datablender 0.0.1__py3-none-any.whl

datablender/__init__.py

@@ -0,0 +1,68 @@
+"""
+
+"""
+
+from datablender.base import (
+    File,
+    Connection,
+    readFile,
+    getNextID,
+    normalize_user_name,
+    Directory,
+    DirectoryElement,
+    ZipFile_,
+    Text,
+    formatText,
+    Request,
+    Bot,
+    BotAction,
+    WebElement,
+    QueryBuilder,
+    DataConfiguration,
+    DataLogging,
+    DataEventsTable,
+    DataElement,
+    Data,
+    AsyncConnection,
+    AsyncDataConfiguration,
+    AsyncRequest,
+    DataSets
+)
+
+from datablender.database import (
+    DatabaseElement,
+    SchemaElement,
+    Database,
+    Extension,
+    Role,
+    Schema,
+    Table,
+    View,
+    Function,
+    AsyncView,
+    AsyncTable,
+    AsyncDatabase
+)
+
+from datablender.data import (
+    DataVersion,
+    DataVersionColumn,
+    DataVersionTable,
+    DataVersionValue,
+    DataSource,
+    DataFetcher,
+    RawDataFile,
+    DirectoryElementController,
+    DataProcess,
+    FilesTable,
+    DataFile,
+    DataDirectory,
+    DataDirectoryElement,
+    DataZipFile,
+    DataSourceCore,
+    DataServer,
+    importData,
+    AsyncDataServer,
+    AsyncDataSource,
+    AsyncDataProcess
+)

datablender/base/__init__.py

@@ -0,0 +1,283 @@
+"""
+
+"""
+from typing import Callable,Tuple
+import types
+
+import os
+import sys
+import numpy
+import pandas
+
+import importlib
+import importlib.util
+
+def getDirectoryElementName(
+    path:str=None,
+    directory_name:str=None,
+    element_name:str=None
+) -> Tuple[str,str]:
+    """Get directory name and data directory element name from path.
+
+    Args:
+        path (str, optional): Path. Defaults to None.
+        directory_name (str, optional): Directory name. Defaults to None.
+        element_name (str, optional): Directory element name. Defaults to None.
+
+    Returns:
+        Tuple[str,str]: Directory name and directory element name.
+    """
+    if path:
+        return os.path.dirname(
+            os.path.normpath(path)
+        ),os.path.basename(
+            os.path.normpath(path)
+        )
+    elif directory_name and element_name:
+        return directory_name,element_name
+
+from datablender.base.file import File,ZipFile_
+from datablender.base.directory import Directory,DirectoryElement
+from datablender.base.text import Text,formatText
+from datablender.base.request import Request, AsyncRequest
+
+def normalize_user_name(user_name:str):
+    return user_name.split(' ')[0]
+
+def getNextID(elements:list,id_attribute_name:str='id') -> int:
+    """Get next id in a list of dictionnary.
+
+    Args:
+        elements (list): List of dictionnary.
+        id_attribute_name (str, optional): Id attribute name. Defaults to 'id'.
+
+    Returns:
+        int: Next id.
+    """
+    ids = numpy.append(
+        numpy.array([element[id_attribute_name] for element in elements]),
+        numpy.nan
+    )
+    df = pandas.DataFrame({
+        'column':numpy.sort(ids),
+        'ids':numpy.arange(1,ids.size+1,1)
+    })
+    return df.loc[df['column']!=df['ids'],'ids'].min()
+   
+def readFile(
+    path:str=None,
+    directory_name:str=None,
+    file_name:str=None,
+    **kwargs
+) -> pandas.DataFrame:
+    return File(
+        directory_name,
+        file_name,
+        path
+    ).read(
+        **kwargs
+    ).content
+
+def getFunction(
+    function_name:str,
+    directory_name:str,
+    module:str='__init__',
+    code:str = None,
+    schema_name:str = None,
+    schema_type:str = None
+) -> Callable:
+
+    if module is None:
+        module = '__init__'
+
+    if directory_name is None:
+        directory_name = os.path.join(
+            os.getenv(
+                'code_directory',
+                os.getcwd()
+            ),
+            schema_type,
+            schema_name
+        )
+        os.makedirs(
+            directory_name,
+            exist_ok=True
+        )
+    
+    if code is not None:
+        File(
+            directory_name,
+            os.path.join(directory_name,'{}.py'.format(module))
+        ).write(code)
+        
+    spec = importlib.util.spec_from_file_location(
+        module,
+        os.path.join(directory_name,'{}.py'.format(module))
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return getattr(module, function_name)
+
+def getModule(
+    directory_name:str,
+    module:str='__init__',
+    package_name:str = None,
+    schema_name:str = None,
+    schema_type:str = None
+) -> types.ModuleType:
+    
+    original_path = os.getcwd()
+
+    if package_name:
+        module = package_name
+    else:
+        if module is None:
+            module = '__init__'
+
+        if directory_name is None:
+            directory_name = os.path.join(
+                os.getenv(
+                    'CODE_DIRECTORY',
+                    os.getcwd()
+                ),
+                schema_type,
+                schema_name
+            )
+            os.makedirs(
+                directory_name,
+                exist_ok=True
+            )
+
+    # os.chdir(directory_name)
+    sys.path.append(directory_name)
+
+    module = importlib.import_module(module)
+
+    # spec = importlib.util.spec_from_file_location(
+    #     module,
+    #     os.path.join(directory_name,'{}.py'.format(module))
+    # )
+    # module = importlib.util.module_from_spec(spec)
+    # spec.loader.exec_module(module)
+
+
+    sys.path.append(original_path)
+
+    return module
+
+def manageQuery(
+    name:str=None,
+    query:str=None,
+    file_name:str=None,
+    directory_name:str=None,
+    schema_name:str=None,
+    schema_type:str= None,
+    data_config_is_active:bool=None
+) -> Tuple[str,str,str]:
+    
+    if query is not None:
+        if data_config_is_active:
+            if file_name is None and directory_name is None and name is not None:
+                file_name = name+'.sql'
+                directory_name = os.path.join(
+                    os.getenv(
+                        'CODE_DIRECTORY',
+                        os.getcwd()
+                    ),
+                    schema_type,
+                    schema_name,
+                    'queries'
+                )
+                os.makedirs(
+                    directory_name,
+                    exist_ok=True
+                )
+                
+                File(
+                    directory_name,
+                    file_name
+                ).write(query)
+
+
+            # if os.path.isfile(os.path.join(os.path.abspath(directory_name),file_name)):
+            #     query = File(
+            #         directory_name,
+            #         file_name
+            #     ).read().content            
+
+            # if directory_name is None:
+
+            #     directory_name = os.getenv(
+            #         'CODE_DIRECTORY',
+            #         os.getcwd()
+            #     )
+
+            #     if schema_name:
+            #         new_directory_name = os.path.join(
+            #             directory_name,
+            #             schema_type,
+            #             schema_name,
+            #             'queries'
+            #         )
+            #         if os.path.isdir(new_directory_name):
+            #             directory_name = new_directory_name
+
+            # elif not os.path.isdir(directory_name):
+            #     new_directory_name = os.path.join(
+            #         os.getenv('CODE_DIRECTORY',os.getcwd()),
+            #         directory_name
+            #     )
+            #     if os.path.isdir(new_directory_name):
+            #         directory_name = new_directory_name
+
+            # if file_name is None:
+            #     pass
+
+            # query = File(
+            #     directory_name,
+            #     file_name+'.sql' if '.' not in file_name else file_name
+            # ).read().content            
+
+    elif file_name is not None:   
+        pass
+
+    elif name is not None:
+        if directory_name is None:
+            directory_name=os.getenv(
+                'CODE_DIRECTORY',
+                os.getcwd()
+            )
+            if schema_type:
+                directory_name = os.path.join(
+                    directory_name,
+                    schema_type
+                )
+            if schema_name:
+                directory_name = os.path.join(
+                    directory_name,
+                    schema_name
+                )
+
+        query = File(
+            os.path.join(
+                directory_name,
+                'queries'
+            ),
+            name+'.sql'
+        ).read(encoding = 'utf-8').content            
+
+    return query,file_name,directory_name
+
+from datablender.base.web import Bot,BotAction,WebElement
+from datablender.base.connection import Connection, AsyncConnection
+from datablender.base.queryBuilder import QueryBuilder
+from datablender.base.dataLogging import (
+    DataEventsTable,
+    DataLogging,
+    AsyncDataEventsTable,
+    AsyncDataLogging
+)
+from datablender.base.dataConfiguration import DataConfiguration,AsyncDataConfiguration
+from datablender.base.dataElement import DataElement, AsyncDataElement
+from datablender.base.data import Data
+from datablender.base.dataSets import DataSets

datablender/base/connection.py

@@ -0,0 +1,370 @@
+"""
+
+"""
+from __future__ import annotations
+
+import os
+import psycopg2
+import postgis
+import sqlalchemy
+import aiohttp
+import asyncio
+import asyncpg
+from sqlalchemy.ext.asyncio import (
+    create_async_engine,
+    async_sessionmaker,
+    AsyncSession
+)
+
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT,ISOLATION_LEVEL_DEFAULT
+
+from datablender.base import normalize_user_name
+
+class Connection:
+    """Connection to a database.
+
+    Attributes:
+    ----------
+        host (str): Server's host.
+        port (str): Server's port.
+        database_name (str): Database name.
+        user_name (str): User name.
+        password (str): Password.
+        dialect (str): Dialect.
+        connection_string (str) : Connection string.
+        engine (object) : SQLAlchemy engine.
+        connection (object) : Connection.
+        cursor (object) : Connection's cursor.
+
+    Methods:
+    ----------
+        getConnection (self) -> None: Get the connection.
+        getCursor (self) -> None: Get the connection's cursor.
+        getEngine (self) -> None: Get SQLAlchemy engine.
+        connectActions (self) -> None: Get the connection, cursor and engine.
+        connect (self) -> None: Connect to the database.
+        close (self) -> None: a .
+        registerToPostgis (self) -> None: .
+        setIsolationLevel (self) -> None: .
+
+    """
+
+    def __init__(
+        self,
+        host:str = 'localhost',
+        port:str = '5432',
+        database_name:str = 'postgres',
+        user_name:str = 'postgres',
+        password:str = None,
+        default_database:int = 'postgres',
+        **kwargs
+    ):
+        """Initiate the connection.
+
+        Args:
+            host (str, optional): Server's host. Defaults to 'localhost'.
+            port (str, optional): Server's port. Defaults to '5432'.
+            database_name (str, optional): Database name. Defaults to 'postgres'.
+            user_name (str, optional): User name. Defaults to 'postgres'.
+            password (str, optional): Password. Defaults to None.
+            dialect (str, optional): Dialect. Defaults to 'PostgreSQL'.
+        """
+        self.database_name = database_name
+        self.default_database = default_database
+
+
+        self.host = os.getenv('host',host)
+        self.port = os.getenv('port',port)
+        self.password = os.getenv('PASSWORD',password)
+        
+        self.user_name = user_name if user_name != 'postgres' else normalize_user_name(
+            os.getenv(
+                'username',
+                user_name
+            )
+        )
+        
+        self.connect(**kwargs)
+    
+    @property
+    def connection_string(self) -> str:
+        """Get connection string.
+
+        Returns:
+            str: Connection string.
+        """
+        return  "postgresql+psycopg2://{}:{}@{}:{}/{}".format(
+            self.user_name,
+            self.password,
+            self.host,
+            self.port,
+            self.database_name
+        )
+
+    def getConnection(self) -> None:
+        """Get connection.
+        """
+        self.connection = psycopg2.connect(
+            database = self.database_name,
+            user = self.user_name,
+            host = self.host,
+            password = self.password,
+            port = self.port
+        )
+
+    def getCursor(self,**kwargs) -> None:
+        """Get connection's cursor.
+        """
+        self.cursor = self.connection.cursor(**kwargs)
+
+    def getEngine(self) -> None:
+        """Get SQLAlchemy engine.
+        """
+        self.engine = sqlalchemy.create_engine(
+            self.connection_string
+        )
+
+    def connect(self,**kwargs) -> None:
+        """Connect to the database.
+
+        Raises:
+            TypeError: _description_
+            TypeError: _description_
+
+        Returns:
+            object: self
+        """
+        self.getConnection()
+        self.getCursor(**kwargs)
+        self.getEngine()
+
+    def close(self) -> None:
+        """Close the connected object.
+        """
+        if self.cursor:
+            self.cursor.close()
+        if self.connection:
+            self.connection.close()
+        if self.engine:
+            self.engine.dispose()
+    
+    def registerToPostgis(self) -> None:
+        """Register to Postgis.
+        """
+        postgis.register(self.cursor)
+
+    def setIsolationLevel(
+        self,
+        level:str = 'default'
+    ) -> None:
+        """Set the isolation level.
+
+        Args:
+            level (str, optional): Level to set. Defaults to auto.
+        """
+        self.connection.set_isolation_level(
+            {
+                'auto':ISOLATION_LEVEL_AUTOCOMMIT,
+                'default':ISOLATION_LEVEL_DEFAULT
+            }[level]
+        )
+
+    def setSchema(
+        self,
+        schema:str='public'
+    ) -> None:
+        """Set default schema.
+
+        Args:
+            schema (str, optional): Schema to set to default. Defaults to 'public'.
+        """
+        self.cursor.execute(
+            "SET search_path TO "+schema
+        )
+        self.connection.commit()
+    
+    def setDatabase(
+        self,
+        database_name:str
+    ) -> None:
+        """Change the connection's database.
+
+        Args:
+            database_name (str): Database name.
+        """
+        self.close()
+        self.database_name = database_name
+        self.connect()
+
+    def getDefaultConnection(
+        self,
+        default_database_name:str=None
+    ) -> None:
+        """Change the database connection to the default one
+
+        Args:
+            default_database_name (str, optional): Name of the default database. Defaults to None.
+        """
+        if default_database_name:
+            self.default_database = default_database_name
+            
+        self.setDatabase(
+            self.default_database
+        )
+
+class AsyncConnection:
+    """Connection to a database.
+
+    Attributes:
+    ----------
+        host (str): Server's host.
+        port (str): Server's port.
+        database_name (str): Database name.
+        user_name (str): User name.
+        password (str): Password.
+        dialect (str): Dialect.
+        connection_string (str) : Connection string.
+        engine (object) : SQLAlchemy engine.
+        connection (object) : Connection.
+        cursor (object) : Connection's cursor.
+
+    Methods:
+    ----------
+        getConnection (self) -> None: Get the connection.
+        getCursor (self) -> None: Get the connection's cursor.
+        getEngine (self) -> None: Get SQLAlchemy engine.
+        connectActions (self) -> None: Get the connection, cursor and engine.
+        connect (self) -> None: Connect to the database.
+        close (self) -> None: a .
+        registerToPostgis (self) -> None: .
+        setIsolationLevel (self) -> None: .
+
+    """
+
+    def __init__(
+        self,
+        host:str = 'localhost',
+        port:str = '5432',
+        database_name:str = 'postgres',
+        user_name:str = 'postgres',
+        password:str = None,
+        default_database:int = 'postgres',
+        **kwargs
+    ):
+        """Initiate the connection.
+
+        Args:
+            host (str, optional): Server's host. Defaults to 'localhost'.
+            port (str, optional): Server's port. Defaults to '5432'.
+            database_name (str, optional): Database name. Defaults to 'postgres'.
+            user_name (str, optional): User name. Defaults to 'postgres'.
+            password (str, optional): Password. Defaults to None.
+            dialect (str, optional): Dialect. Defaults to 'PostgreSQL'.
+        """
+        self.database_name = database_name
+        self.default_database = default_database
+
+
+        self.host = os.getenv('host',host)
+        self.port = os.getenv('port',port)
+        self.password = os.getenv('PASSWORD',password)
+        
+        self.user_name = user_name if user_name != 'postgres' else normalize_user_name(
+            os.getenv(
+                'username',
+                user_name
+            )
+        )
+        
+    @property
+    def connection_string(self) -> str:
+        """Get connection string.
+
+        Returns:
+            str: Connection string.
+        """
+        return  "postgresql+asyncpg://{}:{}@{}:{}/{}".format(
+            self.user_name,
+            self.password,
+            self.host,
+            self.port,
+            self.database_name
+        )
+
+    async def connect(self,**kwargs) -> None:
+        """Connect to the database.
+
+        Raises:
+            TypeError: _description_
+            TypeError: _description_
+
+        Returns:
+            object: self
+        """
+        await self.getConnection()
+        self.getEngine()
+
+    async def getConnection(self) -> None:
+        """Get connection.
+        """
+        self.connection:asyncpg.connection.Connection = await asyncpg.connect(
+            database = self.database_name,
+            user = self.user_name,
+            host = self.host,
+            password = self.password,
+            port = self.port
+        )
+
+    def getEngine(self) -> None:
+        """Get SQLAlchemy engine.
+        """
+        self.engine = create_async_engine(
+            self.connection_string
+        )
+        self.async_session = async_sessionmaker(self.engine, expire_on_commit=False)
+
+    async def close(self) -> None:
+        await self.connection.close()
+        await self.engine.dispose()
+
+    async def setSchema(
+        self,
+        schema:str='public'
+    ) -> None:
+        """Set default schema.
+
+        Args:
+            schema (str, optional): Schema to set to default. Defaults to 'public'.
+        """
+        await self.connection.execute(
+            "SET search_path TO "+schema
+        )
+    
+    async def setDatabase(
+        self,
+        database_name:str
+    ) -> None:
+        """Change the connection's database.
+
+        Args:
+            database_name (str): Database name.
+        """
+        await self.close()
+        self.database_name = database_name
+        await self.connect()
+
+    async def getDefaultConnection(
+        self,
+        default_database_name:str=None
+    ) -> None:
+        """Change the database connection to the default one
+
+        Args:
+            default_database_name (str, optional): Name of the default database. Defaults to None.
+        """
+        if default_database_name:
+            self.default_database = default_database_name
+            
+        await self.setDatabase(
+            self.default_database
+        )