analytic-workspace-client 1.2.0__tar.gz → 1.29.0rc1__tar.gz

analytic_workspace_client-1.29.0rc1/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: analytic_workspace_client
+Version: 1.29.0rc1
+Summary: Библиотека для подключения к Analytic Workspace
+Home-page: https://analyticworkspace.ru/
+Author: Analytic Workspace
+Author-email: aw_help@analyticworkspace.ru
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8,<4
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Provides-Extra: ml
+
+# Библиотека для Analytic Workspace
+
+## Получение токена
+
+Перейдите по ссылке https://aw.example.ru/data-master/get-token (вместо https://aw.example.ru/ укажите адрес вашего сервера Analytic Workspace).
+
+Значение токена лучше всего сохранить в отдельном файл или поместить в переменную окружения `AW_DATA_TOKEN`.
+
+## Пример использования
+
+```python
+from aw_client import Session
+
+
+with open('aw_token', 'rt') as f:
+    aw_token = f.read()
+
+session = Session(token=aw_token, aw_url='https://aw.example.ru')
+
+# Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
+# без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
+
+df = session.load()  # df: pandas.DataFrame
+
+display(df)
+```

{analytic_workspace_client-1.2.0 → analytic_workspace_client-1.29.0rc1}/setup.py

@@ -10,7 +10,7 @@ long_description = (here / "README.md").read_text(encoding="utf-8")
 
 setup(
     name='analytic_workspace_client',
-    version='1.2.0',
+    version='1.29.0rc1',
 
     description='Библиотека для подключения к Analytic Workspace',
     long_description=long_description,
@@ -41,8 +41,8 @@ setup(
     ],
 
     extras_require={
-        'dev': ['pyspark==3.4.1'],
-        'ml': ['mlflow>=2.7,<2.8']
+        'dev': ['pyspark==3.5.0', 'pytest>=8.2,<8.3'],
+        'ml': ['mlflow>=2.14,<2.15']
     },
     
     setup_requires=['wheel'],  

analytic_workspace_client-1.29.0rc1/src/analytic_workspace_client.egg-info/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: analytic-workspace-client
+Version: 1.29.0rc1
+Summary: Библиотека для подключения к Analytic Workspace
+Home-page: https://analyticworkspace.ru/
+Author: Analytic Workspace
+Author-email: aw_help@analyticworkspace.ru
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8,<4
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Provides-Extra: ml
+
+# Библиотека для Analytic Workspace
+
+## Получение токена
+
+Перейдите по ссылке https://aw.example.ru/data-master/get-token (вместо https://aw.example.ru/ укажите адрес вашего сервера Analytic Workspace).
+
+Значение токена лучше всего сохранить в отдельном файл или поместить в переменную окружения `AW_DATA_TOKEN`.
+
+## Пример использования
+
+```python
+from aw_client import Session
+
+
+with open('aw_token', 'rt') as f:
+    aw_token = f.read()
+
+session = Session(token=aw_token, aw_url='https://aw.example.ru')
+
+# Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
+# без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
+
+df = session.load()  # df: pandas.DataFrame
+
+display(df)
+```

{analytic_workspace_client-1.2.0 → analytic_workspace_client-1.29.0rc1}/src/analytic_workspace_client.egg-info/SOURCES.txt

@@ -12,15 +12,31 @@ src/aw_client/domain.py
 src/aw_client/exceptions.py
 src/aw_client/session.py
 src/aw_client/tools.py
+src/aw_client/core/__init__.py
+src/aw_client/core/bundle.py
+src/aw_client/core/compiler.py
+src/aw_client/core/model_vault.py
+src/aw_client/core/spark.py
 src/aw_client/data_master/__init__.py
 src/aw_client/data_master/base.py
 src/aw_client/data_master/v0.py
 src/aw_client/data_master/v1.py
 src/aw_client/data_master/v2.py
+src/aw_client/etl_blocks/__init__.py
+src/aw_client/etl_blocks/application.py
+src/aw_client/etl_blocks/dto.py
+src/aw_client/etl_blocks/runtime.py
+src/aw_client/etl_blocks/services.py
+src/aw_client/etl_blocks/test_data.py
+src/aw_client/etl_blocks/tools.py
 src/aw_client/model_dev/__init__.py
 src/aw_client/model_dev/application.py
 src/aw_client/model_dev/cache.py
 src/aw_client/model_dev/runner.py
 src/aw_client/model_dev/virtual_objects.py
 src/aw_client/models/__init__.py
-src/aw_client/models/model_schema.py
+src/aw_client/models/model_schema.py
+src/aw_etl/__init__.py
+src/aw_etl/compiler.py
+src/aw_etl/etl_blocks.py
+src/aw_etl/models.py

{analytic_workspace_client-1.2.0 → analytic_workspace_client-1.29.0rc1}/src/analytic_workspace_client.egg-info/requires.txt

@@ -1,11 +1,12 @@
-colorama<0.5,>=0.4
+python-dotenv<1.1,>=1.0
 httpx<1.0,>=0.25
 pandas
 pydantic<2.0,>=1.10
-python-dotenv<1.1,>=1.0
+colorama<0.5,>=0.4
 
 [dev]
-pyspark==3.4.1
+pyspark==3.5.0
+pytest<8.3,>=8.2
 
 [ml]
-mlflow<2.8,>=2.7
+mlflow<2.15,>=2.14

{analytic_workspace_client-1.2.0 → analytic_workspace_client-1.29.0rc1}/src/analytic_workspace_client.egg-info/top_level.txt

@@ -1 +1,2 @@
 aw_client
+aw_etl

analytic_workspace_client-1.29.0rc1/src/aw_client/core/bundle.py

@@ -0,0 +1,39 @@
+from typing import List, Dict, Union, Optional, Any
+
+
+class NamedObjectsBundle:
+    """ """
+    def __init__(self, objs: Dict[str, Any]):
+        self._obj_list: List[Any] = [df for _, df in objs.items()]
+        self._obj_named: Dict[str, Any] = objs
+
+    def first(self) -> Optional[Any]:
+        """ """
+        return self._obj_list[0] if self._obj_list else None
+    
+    def as_list(self) -> List[Any]:
+        return self._obj_list
+    
+    def as_named(self) -> Dict[str, Any]:
+        return self._obj_named
+    
+    def __getitem__(self, item):
+        if isinstance(item, int):
+            # если item указано целым числом, то возвращаем по индексу
+            return self._obj_list[item]
+        
+        if not item in self._obj_named:
+            raise Exception(f'Данные объекта с именем "{item}" не найдена')
+        return self._obj_named[item]
+    
+    def __iter__(self):
+        return iter(self._obj_named.values())
+
+    def __bool__(self):
+        return len(self._obj_list) > 0
+    
+    def __len__(self):
+        return len(self._obj_list)
+    
+    def items(self):
+        return self._obj_named.items()

analytic_workspace_client-1.29.0rc1/src/aw_client/core/compiler.py

@@ -0,0 +1,165 @@
+from typing import List, Tuple, Optional
+
+import builtins
+
+
+class CompiledModule:
+    """ """
+    def __init__(self, compiled_globals: dict):
+        self.compiled_globals = compiled_globals
+
+    def __contains__(self, item):
+        return item in self.compiled_globals
+
+    def __getattr__(self, item):
+        return self.compiled_globals[item]
+
+    def __getitem__(self, item):
+        return self.compiled_globals[item]
+
+    @property
+    def is_empty(self) -> bool:
+        """ Возвращает True, если в модуле нет кода (только одни комментарии, например) """
+        return len(set(self.compiled_globals) - {'__builtins__'}) == 0
+
+
+class ScriptCompiler:
+    """ """
+
+    class CompilerException(Exception):
+        """ """
+
+    # Список исключений
+    class Misconfigured(CompilerException):
+        """ """
+
+    class ForbiddenImport(CompilerException):
+        """ Исключение, которое возникает при наличии некорректных импортов в коде """
+
+    class CannotCompile(CompilerException):
+        """ Исключение, которое вызывается при невозможности компиляции кода """
+
+    # Перечисление режимов работы компилятора
+    MODE_ETL = 1
+    MODE_VIRTUAL_OBJECT_SCHEMA = 2
+    MODE_ETL_BLOCK = 3
+
+    def __init__(self, mode: Optional[int] = None):
+        self.mode = mode
+
+    def compile(self, source_code: str, mode: int = None) -> CompiledModule:
+        """ """
+        try:
+            byte_code = compile(source_code, filename='<string>', mode='exec')
+        except SyntaxError as e:
+            raise ScriptCompiler.CannotCompile(f'Ошибка компиляции: {e.msg} (line {e.lineno}): {e.text}')
+
+        try:
+            compiled_globals = self._get_globals(mode=mode if mode is not None else self.mode)
+            exec(byte_code, compiled_globals)
+        except Exception as e:
+            raise ScriptCompiler.CompilerException(f'Ошибка компиляции: {e}')
+
+        return CompiledModule(compiled_globals=compiled_globals)
+
+    def _get_globals(self, mode: int) -> dict:
+        """ """
+        if mode == ScriptCompiler.MODE_ETL:
+            safe_names, safe_modules = self._safe_globals_for_model_etl_script()
+        elif mode == ScriptCompiler.MODE_ETL_BLOCK:
+            safe_names, safe_modules = self._safe_globals_for_etl_block()
+        elif mode == ScriptCompiler.MODE_VIRTUAL_OBJECT_SCHEMA:
+            safe_names, safe_modules = self._safe_globals_for_virtual_object_schema_script()
+        else:
+            raise ScriptCompiler.Misconfigured(f'Указан некорректный режим компиляции {self.mode}')
+
+        def safe_import(name, globals=None, locals=None, fromlist=(), level=0):
+            """ """
+            modules = name.split('.')
+            if modules[0] not in safe_modules:
+                raise ScriptCompiler.ForbiddenImport(f'Импорт модуля {modules[0]} запрещен')
+
+            return __import__(name, globals, locals, fromlist, level)
+
+        safe_builtins = {}
+        for name in safe_names:
+            safe_builtins[name] = getattr(builtins, name)
+
+        safe_builtins['__import__'] = safe_import
+
+        return {'__builtins__': safe_builtins}
+
+    @staticmethod
+    def _safe_globals_for_virtual_object_schema_script() -> Tuple[List[str], List[str]]:
+        """ """
+        safe_names = [
+            'None', 'False', 'True', 'bool', 'bytes', 'chr', 'complex', 'float',
+            'hex', 'id', 'int', 'str', 'getattr', 'setattr', 'delattr',
+        ]
+
+        safe_modules = ['pyspark', 'aw_etl']
+
+        return safe_names, safe_modules
+
+    @staticmethod
+    def _safe_globals_full() -> Tuple[List[str], List[str]]:
+        """ Возвращает полный набор разрешенных модулей """
+        safe_names = [
+            'None', 'False', 'True', 
+            'abs', 'all', 'any', 'ascii', 
+            'bin', 'bool', 'bytes', 'bytearray', 
+            'callable', 'chr', 'classmethod', 'complex', 
+            'delattr', 'dict', 'divmod', 
+            'enumerate',
+            'float', 'filter', 'format', 'frozenset',
+            'getattr',
+            'hasattr', 'hash', 'hex', 
+            'id', 'int', 'isinstance', 'issubclass', 'iter',
+            'len', 'list',
+            'map', 'max', 'min',
+            'next', 
+            'object', 'oct', 'ord',
+            'pow', 'print', 'property', 
+            'range', 'repr', 'reversed', 'round', 
+            'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
+            'type', 'tuple', 
+            'vars',
+            'zip', 
+            '__build_class__', '__name__',
+            
+
+            'ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'BufferError', 'BytesWarning',
+            'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'FutureWarning',
+            'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
+            'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 'NotImplementedError', 'OSError',
+            'OverflowError', 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError', 'RuntimeWarning',
+            'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TypeError',
+            'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
+            'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'ZeroDivisionError',
+        ]
+
+        safe_modules = [
+            'pyspark', 'requests', 'pandas', 'numpy', 'aw_etl', 'pyparsing', 'pydantic',
+
+            'mlflow', 'prophet', 'statmodels', 'torch', 'sklearn', 'numpy', 'catboost', 
+
+            'array', 'calendar', 'codecs', 'collections', 'copy', 'csv', 'dataclasses', 'datetime', '_strptime',
+            'decimal', 'enum', 'functools', 'hashlib', 'itertools', 'json', 'math', 'queue', 'random', 're',
+            'statistics', 'string', 'time', 'urllib', 'xml',
+            'zoneinfo', 'typing', 'uuid', 'logging',
+        ]
+
+        return safe_names, safe_modules
+
+    @staticmethod
+    def _safe_globals_for_model_etl_script() -> Tuple[List[str], List[str]]:
+        """ """
+        return ScriptCompiler._safe_globals_full()
+
+    @staticmethod
+    def  _safe_globals_for_etl_block() -> Tuple[List[str], List[str]]:
+        """ """
+        safe_names, safe_modules = ScriptCompiler._safe_globals_full()
+        safe_modules.extend(['sqlglot', 'inspect'])
+
+        return safe_names, safe_modules

analytic_workspace_client-1.29.0rc1/src/aw_client/core/model_vault.py

@@ -0,0 +1,14 @@
+from typing import Optional, Dict, Any
+
+
+class Vault:
+    """ """
+    def __init__(self, values: Optional[dict] = None):
+        self._values = values or {}
+
+    @property
+    def values(self) -> dict:
+        return self._values
+
+    def get(self, name: str) -> Optional[Any]:
+        return self._values.get(name, None)

analytic_workspace_client-1.29.0rc1/src/aw_client/core/spark.py

@@ -0,0 +1,15 @@
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+def build_spark_session():
+    """ """
+    return SparkSession.builder \
+        .master('local[*]') \
+        .config('spark.driver.host', '127.0.0.1') \
+        .config('spark.ui.enabled', 'false') \
+        .getOrCreate()
+

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/__init__.py

@@ -0,0 +1,4 @@
+from .runtime import get_etl_block_schema, get_etl_block_data, get_etl_block_meta
+from .test_data import ModelObjectTestData
+
+

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/application.py

@@ -0,0 +1,49 @@
+from typing import Optional, Callable
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+from aw_client.core.model_vault import Vault
+from aw_client.core.compiler import CompiledModule
+
+
+class ETLBlockApplication:
+    """ """
+    def __init__(self, 
+                 spark_builder: Callable, 
+                 run_mode: str, 
+                 vault: Vault,
+                 model_module: Optional[CompiledModule] = None):
+        self._spark_builder = spark_builder
+        self._spark = None
+        self._run_model = run_mode
+        self._model_module = model_module
+        self._vault = vault
+    
+    @property
+    def spark(self) -> SparkSession:
+        if self._spark is None:
+            self._spark = self._spark_builder()
+        return self._spark
+    
+    @property
+    def is_spark_initialized(self) -> bool:
+        return self._spark is not None
+    
+    @property
+    def model_module(self) -> Optional[CompiledModule]:
+        """ """
+        return self._model_module
+
+    @property
+    def vault(self) -> Vault:
+        """ """
+        return self._vault
+
+    @property
+    def run_mode(self) -> str:
+        """ """
+        return self.run_mode
+    

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/dto.py

@@ -0,0 +1,85 @@
+from typing import Optional, Any, List, Union
+
+import datetime
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class ETLBlockParamType(str, Enum):
+    """ """
+    STRING = 'string'
+    TEXT = 'text'
+    PASSWORD = 'password'
+    SQL_TEXT = 'sql_text'
+    NUMBER = 'number'
+    FLOAT = 'float'
+    BOOL = 'bool'
+    DATE = 'date'
+    DATETIME = 'datetime'
+    SELECT = 'select'
+    ACTION = 'action'
+
+
+class ETLBlockParamGroupType(str, Enum):
+    """ """
+    GROUP = 'group'
+
+
+class ETLBlockParamActionType(str, Enum):
+    """ """
+    ACTION = 'action'
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Метаданные ETL блока
+# ----------------------------------------------------------------------------------------------------------------------
+class ETLBlockParam(BaseModel):
+    """ Параметр ETL блока """
+    code: str
+    name: str
+    type: ETLBlockParamType
+    description: Optional[str] = None
+
+    required: bool
+    mult: bool
+    domain: Optional[Any] = None
+    extra: Optional[Any] = None
+
+
+class ETLBlockParamGroup(BaseModel):
+    """ Группа параметров ETL блока """
+    code: str
+    name: str
+
+    type: ETLBlockParamGroupType
+    description: Optional[str] = None
+    view_options: Optional[dict] = None
+    mult: bool
+    params: List[Union['ETLBlockParam', 'ETLBlockParamAction', 'ETLBlockParamAction']] = []
+    extra: Optional[Any] = None
+
+
+class ETLBlockParamAction(BaseModel):
+    code: str
+    name: str
+    type: ETLBlockParamActionType
+    description: Optional[str] = None
+    action: str
+    extra: Optional[Any] = None
+
+
+class ETLBlockMeta(BaseModel):
+    """ Метаданные ETL-блока """
+    uid: str  # уникальный идентификатор блока
+    name: str  # название блока
+    version: str  # версия блока
+    description: str  # описание блока
+    author: str  # автор блока
+    updated_at: datetime.datetime  # дата и время последнего обновления
+    params: List[Union[ETLBlockParam, ETLBlockParamAction, ETLBlockParamGroup]]
+    engine_requires: Optional[List[str]] = []
+
+    @property
+    def verbose_name(self):
+        return f'{self.name} v{self.version}'

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/runtime.py

@@ -0,0 +1,202 @@
+from typing import Dict, Optional, Any, Union, List, Literal
+from pathlib import Path
+from collections import OrderedDict
+import datetime
+import inspect
+
+from aw_client.core.compiler import ScriptCompiler
+from aw_client.core.model_vault import Vault
+from aw_client.core.spark import build_spark_session
+from aw_client.core.bundle import NamedObjectsBundle
+from aw_client.models.model_schema import ModelObject, ModelObjectField
+from .application import ETLBlockApplication
+from .test_data import ModelObjectTestData
+from .tools import build_dataframe, build_model_object, build_spark_schema
+from .dto import ETLBlockMeta
+
+
+try:
+    from pyspark.sql import SparkSession, DataFrame
+    from pyspark.sql.types import DataType, StringType, DoubleType, TimestampType, LongType, BooleanType, \
+        ByteType, ShortType, IntegerType, DecimalType, FloatType, DateType, StructType, StructField
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+
+def get_etl_block_meta(block_path: Path) -> ETLBlockMeta:
+    """ """
+    block_meta_path = block_path / 'block_meta.json' if block_path.is_dir() else block_path
+
+    if not block_meta_path.exists():
+        raise Exception(f'Файл с метаданным блока не найден: {block_meta_path}')
+
+    with open(block_meta_path, 'rt') as f:
+        return ETLBlockMeta.parse_raw(f.read())
+
+
+def get_etl_block_schema(block_path: Path, 
+                         test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+                         params: Optional[Dict] = None, 
+                         run_mode: Optional[Literal['']] = None, 
+                         vault: Optional[Vault] = None,
+                         model_script_code: Optional[str] = None) -> StructType:
+    """ 
+    Args:
+
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
+    
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    # Дочерние схемы
+    schemas = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        schema = build_spark_schema(td)
+        if not schemas:
+            schemas['child'] = schema
+        schemas[td.model_name] = schema
+
+    upstream_schemas = NamedObjectsBundle(schemas)
+    
+    block_schema_parameters = inspect.signature(block_module['block_schema']).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    # Определение параметров для передачи
+    block_schema_kwargs = {}
+    if 'params' in block_schema_parameters:
+        block_schema_kwargs['params'] = params
+    if 'app' in block_schema_parameters:
+        block_schema_kwargs['app'] = app
+    if 'model_object' in block_schema_parameters:
+        block_schema_kwargs['model_object'] = build_model_object(test_data)
+    if 'schema' in block_schema_parameters:
+        block_schema_kwargs['schema'] = upstream_schemas.first()
+    if 'schemas' in block_schema_parameters:
+        block_schema_kwargs['schemas'] = upstream_schemas
+    if 'upstream_schema' in block_schema_parameters:
+        block_schema_kwargs['upstream_schema'] = upstream_schemas.first()
+    if 'upstream_schemas' in block_schema_parameters:
+        block_schema_kwargs['upstream_schemas'] = upstream_schemas
+    if 'df' in block_schema_parameters:
+        block_schema_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_schema_parameters:
+        block_schema_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_schema_parameters:
+        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_schema_parameters:
+        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+
+    return block_module['block_schema'](**block_schema_kwargs)
+    
+
+def get_etl_block_data(block_path: Path, 
+                       test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+                       params: Optional[Dict] = None, 
+                       run_mode: Optional[Literal['']] = None, 
+                       vault: Optional[Vault] = None,
+                       model_script_code: Optional[str] = None) -> StructType:
+    """ 
+    Args:
+
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
+    
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+        dataframes[td.model_name] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    block_data_parameters = inspect.signature(block_module['block_data']).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    # Определение параметров для передачи
+    block_schema_kwargs = {}
+    if 'params' in block_data_parameters:
+        block_schema_kwargs['params'] = params
+    if 'app' in block_data_parameters:
+        block_schema_kwargs['app'] = app
+    if 'model_object' in block_data_parameters:
+        block_schema_kwargs['model_object'] = build_model_object(test_data)
+    if 'df' in block_data_parameters:
+        block_schema_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_data_parameters:
+        block_schema_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_data_parameters:
+        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_data_parameters:
+        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module['block_data'](**block_schema_kwargs)
+    

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/test_data.py

@@ -0,0 +1,17 @@
+from typing import List, TypedDict, Optional
+
+from dataclasses import dataclass, field
+
+
+class ModelObjectSchemaField(TypedDict):
+    """ """
+    model_name: str
+    simple_type: str
+
+
+@dataclass
+class ModelObjectTestData:
+    """ """
+    model_name: str
+    rows: List[dict] = field(default_factory=list)
+    schema: Optional[List[ModelObjectSchemaField]] = None

analytic_workspace_client-1.29.0rc1/src/aw_client/etl_blocks/tools.py

@@ -0,0 +1,99 @@
+from typing import Any
+import datetime
+
+
+from aw_client.models.model_schema import ModelObject, ModelObjectField
+from .test_data import ModelObjectTestData
+
+
+
+try:
+    from pyspark.sql import SparkSession, DataFrame
+    from pyspark.sql.types import DataType, StringType, DoubleType, TimestampType, LongType, BooleanType, \
+        ByteType, ShortType, IntegerType, DecimalType, FloatType, DateType, StructType, StructField
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+def build_spark_schema(test_data: ModelObjectTestData) -> StructType:
+    """ """
+    if test_data.schema is not None:
+        return StructType(fields=[
+            StructField(sf['model_name'], spark_type_for_simple_type(sf['simple_type']), True) for sf in test_data.schema
+        ])
+    return StructType(fields=[
+        StructField(n, spark_type_for_python_value(v), True) for n, v in test_data.rows[0].items()
+    ])
+
+
+def build_dataframe(spark: SparkSession, test_data: ModelObjectTestData) -> DataFrame:
+    """ """
+    return spark.createDataFrame(test_data.rows, schema=build_spark_schema(test_data))
+
+
+def build_model_object(test_data: ModelObjectTestData):
+    """ """
+    if test_data.schema is not None:
+        fields = [
+            ModelObjectField(name=f['model_name'], model_name=f['model_name'], simple_type=f['simple_type']) for f in test_data.schema
+        ]
+    else:
+        fields = [
+            ModelObjectField(name=n, model_name=n, simple_type=simple_type_for_python_value(v)) for n, v in test_data.rows[0].items()
+        ]
+    return ModelObject(
+        name=test_data.model_name,
+        model_name=test_data.model_name,
+        type='table',
+        sql_text=None,
+        fields=fields
+    )
+
+
+
+
+def spark_type_for_simple_type(simple_type: str) -> DataType:
+    """ 
+    """
+    if simple_type == 'number':
+        return LongType()
+    elif simple_type == 'float':
+        return DoubleType()
+    elif simple_type == 'date':
+        return TimestampType()
+    elif simple_type == 'bool':
+        return BooleanType()
+    else:
+        return StringType()
+        
+
+def spark_type_for_python_value(value: Any) -> DataType:
+    """ 
+    Returns Spark type
+    """
+    if isinstance(value, datetime.date):
+        return DateType()
+    if isinstance(value, datetime.datetime):
+        return TimestampType()
+    if isinstance(value, int):
+        return LongType()
+    if isinstance(value, float):
+        return DoubleType()
+    if isinstance(value, bool):
+        return BooleanType()
+    return StringType()
+
+
+def simple_type_for_python_value(value: Any) -> str:
+    """ """
+    if isinstance(value, datetime.date):
+        return 'date'
+    if isinstance(value, datetime.datetime):
+        return 'date'
+    if isinstance(value, int):
+        return 'number'
+    if isinstance(value, float):
+        return 'float'
+    if isinstance(value, bool):
+        return 'bool'
+    return 'string'

{analytic_workspace_client-1.2.0 → analytic_workspace_client-1.29.0rc1}/src/aw_client/session.py

@@ -136,7 +136,7 @@ class Session:
             import mlflow
         except ImportError:
             raise AwClientMisconfigured(
-                'Для использованиея MLFlow установите с зависимостью ml: pip install aw_client[ml]')
+                'Для использованиея MLFlow установите библиотеку с опцией ml: pip install analytic-workspace-client[ml]')
 
         if not self.token:
             data_master_url = urljoin(self.aw_url, 'data-master/get-token')

analytic_workspace_client-1.29.0rc1/src/aw_etl/compiler.py

@@ -0,0 +1,17 @@
+from typing import Protocol
+
+
+class CompiledModule(Protocol):
+    """ """
+    def __contains__(self, item):
+        """ """
+
+    def __getattr__(self, item):
+        """ """
+
+    def __getitem__(self, item):
+        """ """
+
+    @property
+    def is_empty(self) -> bool:
+        """ """

analytic_workspace_client-1.29.0rc1/src/aw_etl/etl_blocks.py

@@ -0,0 +1,39 @@
+from typing import Protocol, Optional
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+from aw_etl.models import Vault
+from aw_etl.compiler import CompiledModule
+
+
+class InvalidEtlBlock(Exception):
+    """ """
+
+
+class ETLBlockApplication(Protocol):
+    """ """
+    @property
+    def spark(self) -> SparkSession:
+        """ """
+
+    @property
+    def is_spark_initialized(self) -> bool:
+        """ """
+
+    @property
+    def model_module(self) -> Optional[CompiledModule]:
+        """ """
+
+    @property
+    def vault(self) -> Vault:
+        """ """
+
+    @property
+    def run_mode(self) -> str:
+        """ """
+
+    

analytic_workspace_client-1.29.0rc1/src/aw_etl/models.py

@@ -0,0 +1,22 @@
+from typing import Protocol, List, Any
+
+
+class ModelObjectField(Protocol):
+    """ """
+    @property
+    def simple_type(self) -> str:
+        """ """
+
+
+class ModelObject(Protocol):
+    """ """
+    @property
+    def fields(self) -> List[ModelObjectField]:
+        """ """
+
+
+class Vault(Protocol):
+    """ """
+    def get(self, name: str) -> Any:
+        """ """
+

analytic_workspace_client-1.2.0/PKG-INFO

@@ -1,42 +0,0 @@
-Metadata-Version: 2.1
-Name: analytic_workspace_client
-Version: 1.2.0
-Summary: Библиотека для подключения к Analytic Workspace
-Home-page: https://analyticworkspace.ru/
-Author: Analytic Workspace
-Author-email: aw_help@analyticworkspace.ru
-License: UNKNOWN
-Description: # Библиотека для Analytic Workspace
-        
-        ## Получение токена
-        
-        Перейдите по ссылке https://aw.example.ru/data-master/get-token (вместо https://aw.example.ru/ укажите адрес вашего сервера Analytic Workspace).
-        
-        Значение токена лучше всего сохранить в отдельном файл или поместить в переменную окружения `AW_DATA_TOKEN`.
-        
-        ## Пример использования
-        
-        ```python
-        from aw_client import Session
-        
-        
-        with open('aw_token', 'rt') as f:
-            aw_token = f.read()
-        
-        session = Session(token=aw_token, aw_url='https://aw.example.ru')
-        
-        # Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
-        # без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
-        
-        df = session.load()  # df: pandas.DataFrame
-        
-        display(df)
-        ```
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<4
-Description-Content-Type: text/markdown
-Provides-Extra: dev
-Provides-Extra: ml

analytic_workspace_client-1.2.0/src/analytic_workspace_client.egg-info/PKG-INFO

@@ -1,42 +0,0 @@
-Metadata-Version: 2.1
-Name: analytic-workspace-client
-Version: 1.2.0
-Summary: Библиотека для подключения к Analytic Workspace
-Home-page: https://analyticworkspace.ru/
-Author: Analytic Workspace
-Author-email: aw_help@analyticworkspace.ru
-License: UNKNOWN
-Description: # Библиотека для Analytic Workspace
-        
-        ## Получение токена
-        
-        Перейдите по ссылке https://aw.example.ru/data-master/get-token (вместо https://aw.example.ru/ укажите адрес вашего сервера Analytic Workspace).
-        
-        Значение токена лучше всего сохранить в отдельном файл или поместить в переменную окружения `AW_DATA_TOKEN`.
-        
-        ## Пример использования
-        
-        ```python
-        from aw_client import Session
-        
-        
-        with open('aw_token', 'rt') as f:
-            aw_token = f.read()
-        
-        session = Session(token=aw_token, aw_url='https://aw.example.ru')
-        
-        # Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
-        # без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
-        
-        df = session.load()  # df: pandas.DataFrame
-        
-        display(df)
-        ```
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<4
-Description-Content-Type: text/markdown
-Provides-Extra: dev
-Provides-Extra: ml