connector-for-bb 0.0.1__tar.gz

connector_for_bb-0.0.1/MANIFEST.in

@@ -0,0 +1,10 @@
+global-exclude *.py[co]
+global-exclude __pycache__
+graft tests
+include README.md
+include VERSION
+recursive-include src/dump *.py
+recursive-include src/dump/click_house *.py
+recursive-include src/dump/files *.py
+recursive-include src/dump/postgres *.py
+recursive-include src/dump/s3 *.py

connector_for_bb-0.0.1/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: connector_for_bb
+Version: 0.0.1
+Summary: connector
+Home-page: https://github.com/your_username/your_repository
+Author: Ivan
+Author-email: example@example.com
+License: MIT
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: develop
+
+Dump project
+
+

connector_for_bb-0.0.1/README.md

@@ -0,0 +1 @@
+Dump project

connector_for_bb-0.0.1/VERSION

@@ -0,0 +1 @@
+0.0.1

connector_for_bb-0.0.1/setup.cfg

@@ -0,0 +1,128 @@
+[metadata]
+name = connector_for_bb
+version = 0.0.1
+description = connector
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = Ivan
+author_email = example@example.com
+license = MIT
+url = https://github.com/your_username/your_repository
+classifiers = 
+	Programming Language :: Python :: 3
+	License :: OSI Approved :: MIT License
+	Operating System :: OS Independent
+
+[aliases]
+test = pytest
+
+[coverage:report]
+exclude_lines = 
+	@abc.abstractmethod
+	@abc.abstractproperty
+	CancelledError
+	NotImplementedError
+	pragma: no cover
+	__repr__
+	__str__
+fail_under = 81
+precision = 2
+show_missing = True
+
+[coverage:run]
+branch = True
+source = 
+	src
+	tests
+
+[flake8]
+ignore = E203,W503
+max-line-length = 95
+
+[isort]
+atomic = true
+default_section = THIRDPARTY
+force_grid_wrap = 0
+include_trailing_comma = true
+indent = '    '
+known_first_party = dsp_dags
+known_third_party = 
+line_length = 95
+lines_after_imports = 2
+multi_line_output = 3
+not_skip = __init__.py
+order_by_type = true
+sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
+use_parentheses = True
+
+[mypy]
+check_untyped_defs = True
+
+[mypy-lightfm.*]
+ignore_missing_imports = True
+
+[mypy-numpy.*]
+ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-scipy.*]
+ignore_missing_imports = True
+
+[mypy-sklearn.*]
+ignore_missing_imports = True
+
+[mypy-xgboost.*]
+ignore_missing_imports = True
+
+[mypy-psycopg2.*]
+ignore_missing_imports = True
+
+[mypy-clickhouse_connect.*]
+ignore_missing_imports = True
+
+[options]
+include_package_data = True
+install_requires = 
+	tqdm==4.66.5
+	catboost==1.2.7
+	numpy==1.26.0
+	pandas==2.2.2
+	psycopg2-binary==2.9.9
+	boto3==1.35.30
+	scikit-learn==1.5.2
+	clickhouse-connect==0.8.6
+package_dir = 
+	= src
+packages = find:
+python_requires = >=3.10
+
+[options.extras_require]
+develop = 
+	black==24.8.0
+	coverage==5.5
+	flake8==6.0.0
+	hypothesis==6.9.2
+	isort==5.8.0
+	mypy==0.812
+	pylint==2.17.4
+	pytest==7.3.1
+	responses==0.13.2
+
+[options.packages.find]
+where = src
+
+[tool:pytest]
+addopts = 
+	--junitxml=junit.xml
+	--showlocals
+	--verbose
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

connector_for_bb-0.0.1/setup.py

@@ -0,0 +1,70 @@
+import os
+import subprocess
+from setuptools import setup, find_packages
+from setuptools.command.sdist import sdist as sdist_orig
+
+ROOT = os.path.dirname(__file__)
+VERSION = os.path.join(ROOT, "VERSION")
+
+
+def format_version(raw_version):
+    """
+    Преобразование версии в формат PEP 440.
+    Если версия содержит некорректные символы, добавляем их как метаданные.
+    """
+    if raw_version.startswith("v"):  # Удаляем префикс v, если он есть
+        raw_version = raw_version[1:]
+
+
+    parts = raw_version.split("-")
+    if len(parts) == 1:
+        return raw_version  # Версия уже корректна
+    elif len(parts) == 3:
+        base, distance, commit_hash = parts
+        return f"{base}.dev{distance}+{commit_hash}"
+    else:
+        # Если версия не в ожидаемом формате, добавляем её как метаданные
+        return f"0.1.0+{raw_version}"
+
+
+def project_version():
+    """Получение версии проекта из git или файла VERSION."""
+    version = None
+
+    # Попытка получить версию из Git
+    try:
+        output = subprocess.check_output(
+            ["git", "describe", "--tags", "--always"],
+            stderr=open(os.devnull, "wb"),
+        ).strip().decode()
+        version = format_version(output)
+    except (FileNotFoundError, subprocess.CalledProcessError) as e:
+        print(f"Warning: Unable to retrieve version from git: {e}")
+
+    # Если версия недоступна, читаем из файла VERSION
+    if not version and os.path.exists(VERSION):
+        with open(VERSION) as verfile:
+            raw_version = verfile.read().strip()
+            version = format_version(raw_version)
+
+    # Если версия всё ещё не найдена, вызываем ошибку
+    if not version:
+        raise RuntimeError("Cannot detect project version")
+
+    return version
+
+
+class sdist(sdist_orig):
+
+    def run(self):
+        version = project_version()
+        with open(VERSION, "w") as verfile:
+            verfile.write(version)
+        sdist_orig.run(self)
+
+
+if __name__ == "__main__":
+    setup(
+        version=project_version(),  # Версия из git или файла VERSION
+        cmdclass={"sdist": sdist},  # Использование кастомного класса sdist
+    )

connector_for_bb-0.0.1/src/connector_for_bb.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: connector-for-bb
+Version: 0.0.1
+Summary: connector
+Home-page: https://github.com/your_username/your_repository
+Author: Ivan
+Author-email: example@example.com
+License: MIT
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: develop
+
+Dump project
+
+

connector_for_bb-0.0.1/src/connector_for_bb.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+MANIFEST.in
+README.md
+VERSION
+setup.cfg
+setup.py
+src/connector_for_bb.egg-info/PKG-INFO
+src/connector_for_bb.egg-info/SOURCES.txt
+src/connector_for_bb.egg-info/dependency_links.txt
+src/connector_for_bb.egg-info/requires.txt
+src/connector_for_bb.egg-info/top_level.txt
+src/dump/__init__.py
+src/dump/config_utils.py
+src/dump/click_house/__init__.py
+src/dump/click_house/connectors.py
+src/dump/files/__init__.py
+src/dump/files/tmp_util.py
+src/dump/postgres/__init__.py
+src/dump/postgres/connectors.py
+src/dump/postgres/manipulators.py
+src/dump/postgres/table.py
+src/dump/postgres/types_util.py
+src/dump/s3/__init__.py
+src/dump/s3/connectors.py

connector_for_bb-0.0.1/src/connector_for_bb.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

connector_for_bb-0.0.1/src/connector_for_bb.egg-info/requires.txt

@@ -0,0 +1,19 @@
+boto3==1.35.30
+catboost==1.2.7
+clickhouse-connect==0.8.6
+numpy==1.26.0
+pandas==2.2.2
+psycopg2-binary==2.9.9
+scikit-learn==1.5.2
+tqdm==4.66.5
+
+[develop]
+black==24.8.0
+coverage==5.5
+flake8==6.0.0
+hypothesis==6.9.2
+isort==5.8.0
+mypy==0.812
+pylint==2.17.4
+pytest==7.3.1
+responses==0.13.2

connector_for_bb-0.0.1/src/connector_for_bb.egg-info/top_level.txt

@@ -0,0 +1 @@
+dump

connector_for_bb-0.0.1/src/dump/click_house/connectors.py

@@ -0,0 +1,32 @@
+import clickhouse_connect
+from config_util import load_config
+from pandas import DataFrame as PandasDataFrame
+
+
+class ConnectorCH:
+    def __init__(
+        self,
+        db_config_name: str = "click_house",
+    ) -> None:
+        self.db_config_name = db_config_name
+        self.__config = load_config(section=self.db_config_name)
+
+        self._client = None
+
+    @property
+    def client(self):
+        if self._client is None:
+            try:
+                self._client = clickhouse_connect.get_client(**self.__config)
+                # print("Connected to the ClickHouse server.")
+            except Exception as error:
+                print(error)
+        return self._client
+
+
+class TableCH(ConnectorCH):
+    def __init__(self, db_config_name: str = "click_house") -> None:
+        super().__init__(db_config_name)
+
+    def get_df(self, query: str) -> PandasDataFrame:
+        return self.client.query_df(query)

connector_for_bb-0.0.1/src/dump/config_utils.py

@@ -0,0 +1,19 @@
+from configparser import ConfigParser
+
+
+def load_config(filename="database.ini", section="postgresql"):
+    parser = ConfigParser()
+    parser.read(filename)
+
+    # get section, default to postgresql
+    config = {}
+    if parser.has_section(section):
+        params = parser.items(section)
+        for param in params:
+            config[param[0]] = param[1]
+    else:
+        raise Exception(
+            "Section {0} not found in the {1} file".format(section, filename)
+        )
+
+    return config

connector_for_bb-0.0.1/src/dump/files/tmp_util.py

@@ -0,0 +1,44 @@
+import os
+from tempfile import TemporaryDirectory
+
+
+class TemporaryFileSystem:
+    """
+    Temporary directory utils class
+    used for data manipulators
+    """
+
+    def __init__(self) -> None:
+        self._tmp_dir = TemporaryDirectory()
+        self._last_path: str
+
+    @property
+    def tmp_dir(self) -> str:
+        return self._tmp_dir.name
+
+    @property
+    def files(self) -> list:
+        return os.listdir(str(self.tmp_dir))
+
+    @property
+    def next_filename(self) -> str:
+        number = 0
+        avaliable = [int(x.split(".")[0].split("_")[-1]) for x in self.files]
+        if len(avaliable) != 0:
+            number = max(avaliable) + 1
+        return f"data_{number}.csv"
+
+    def save_path(self) -> str:
+        self._last_path = os.path.join(str(self.tmp_dir), self.next_filename)
+        return self._last_path
+
+    @property
+    def dir_info(self) -> None:
+        """
+        Shows temporary directory info
+        file - file size in mb
+        """
+        for file in self.files:
+            check_file = os.path.join(self.tmp_dir, file)
+            size = os.path.getsize(check_file) / 1024**2
+            print(f"{check_file}: {size} mb")

connector_for_bb-0.0.1/src/dump/postgres/connectors.py

@@ -0,0 +1,186 @@
+import psycopg2
+from dump.config_utils import load_config
+from psycopg2 import connect
+
+
+TYPE_MAPPER = {
+    "bigint": "Int64",
+    "character varying": "str",
+    "integer": "Int64",
+    "double precision": "float",
+    "timestamp without time zone": "timestamp",
+    # "timestamp with time zone": "timestamp",
+    "USER-DEFINED": "str",
+    "date": "timestamp",
+    "character": "str",
+    "boolean": "bool",
+    "real": "Int64",
+    "text": "str",
+}
+
+
+class ConnectorPG:
+    def __init__(
+        self,
+        db_config_name: str = "postgres",
+    ) -> None:
+        self.db_config_name = db_config_name
+        self.__config = load_config(section=self.db_config_name)
+
+        self._conn: connect = None
+        self._cursor = None
+
+    @property
+    def conn(self):
+        if self._conn is None:
+            try:
+                # connecting to the PostgreSQL server
+                with psycopg2.connect(**self.__config) as pgconn:
+                    # print("Connected to the PostgreSQL server.")
+                    self._conn = pgconn
+            except (psycopg2.DatabaseError, Exception) as error:
+                print(error)
+        return self._conn
+
+    @property
+    def cursor(self):
+        if self._cursor is None:
+            self._cursor = self.conn.cursor()
+        return self._cursor
+
+    @property
+    def close(self):
+        self._conn.close()
+
+        self._conn = None
+        self._cursor = None
+
+    @property
+    def commit(self):
+        self.conn.commit()
+
+    def execute(self, query: str):
+        self.cursor.execute(query)
+        self.conn.commit()
+
+    def fetchall(self, queru: str) -> list:
+        self.execute(queru)
+        values = self.cursor.fetchall()
+        self.conn.commit()
+
+        return values
+
+
+class DBUtilsPG(ConnectorPG):
+    @staticmethod
+    def _validate_output(inp: list):
+        return [x[0] for x in inp]
+
+    @property
+    def _schemas(self) -> list:
+        query = """
+            SELECT schema_name
+            FROM information_schema.schemata;
+        """
+        return self._validate_output(self.fetchall(query))
+
+    def get_tables_in_schema(self, schema: str) -> list:
+        query = f"""
+            SELECT table_name
+            FROM information_schema.tables
+            WHERE table_schema = '{schema}'
+        """
+        tables = self._validate_output(self.fetchall(query))
+        return tables
+
+    def get_table_schema(self, table_name: str, db_schema: str) -> dict:
+        query = f"""
+            SELECT column_name, data_type, is_nullable, ordinal_position
+            FROM information_schema.columns
+            WHERE table_name = '{table_name}' and
+            table_schema = '{db_schema}';
+        """
+
+        schema = {}
+        for column_name, data_type, is_nullable, position in self.fetchall(query):
+            schema[column_name] = {
+                "data_type": TYPE_MAPPER.get(data_type, "str"),
+                "is_nullable": is_nullable,
+                "position": position,
+            }
+        return schema
+
+    def get_schema_by_table(self, table_name: str) -> str:
+        query = f"""
+            SELECT table_schema
+            FROM information_schema.tables
+            WHERE table_name = '{table_name}';
+        """
+        schema = self._validate_output(self.fetchall(query))
+        return schema[0]
+
+
+class SequenceUtils:
+    def __init__(
+        self,
+        sequence_name: str = "seq_general_job_id",
+        sequence_schema: str = "public",
+        db_config_name: str = "postgres",
+    ) -> None:
+        self.sequence_name = sequence_name
+        self.sequence_schema = sequence_schema
+
+        self.connector = ConnectorPG(db_config_name)
+
+    @property
+    def next_sequence(self):
+        query = f"""
+            SELECT nextval('{self.sequence_schema}.{self.sequence_name}');
+        """
+        sequence = self.connector.fetchall(query)[0][0]
+        return sequence
+
+    @property
+    def current_sequence(self):
+        query = f"""
+            SELECT last_value FROM {self.sequence_schema}.{self.sequence_name}
+        """
+        sequence = self.connector.fetchall(query)[0][0]
+        return sequence
+
+
+# class TemporaryTablePG(ConnectorPG):
+#     def __init__(self, name: str, db_config_name: str = "postgres") -> None:
+#         super().__init__(db_config_name)
+#         self.name = name
+#         self.data_manipulator = DataManupulationPG(db_config_name=self.db_config_name)
+
+#         # mapping dtypes to PG types
+#         self._KIND_MAPPER = {
+#             "M": "timestamp",
+#             "f": "real",
+#             "i": "bigint",
+#         }
+
+#     def _pd_types_to_pg_types(self, df: PandasDataFrame) -> dict:
+#         df_types = df.dtypes.to_dict()
+#         some = {
+#             col: self._KIND_MAPPER.get(col_type.kind, "varchar")
+#             for col, col_type in df_types.items()
+#         }
+#         return some
+
+#     def _types_to_str_statments(self, types: dict) -> str:
+#         types_statments = ", \n ".join([" ".join(x) for x in types.items()])
+#         return types_statments
+
+#     def create_table(self, df: PandasDataFrame):
+#         types = self._pd_types_to_pg_types(df)
+#         types_statments = self._types_to_str_statments(types)
+#         query = f"""
+#         CREATE TEMP TABLE {self.name}(
+#             {types_statments}
+#         );
+#         """
+#         self.execute(query)
+#         self.commit

connector_for_bb-0.0.1/src/dump/postgres/manipulators.py

@@ -0,0 +1,138 @@
+from typing import Dict, Optional
+
+import pandas as pd
+from dump.files.tmp_util import TemporaryFileSystem
+from pandas import DataFrame as PandasDataFrame
+
+from .connectors import ConnectorPG, DBUtilsPG
+
+
+class GetDataManipulatorPG(ConnectorPG):
+    def __init__(
+        self,
+        encoding="utf-16",
+        size: int = 262144,
+        db_config_name: str = "postgres",
+    ) -> None:
+        super().__init__(db_config_name=db_config_name)
+        self.encoding = encoding
+        self.size = size
+
+        # requires for not execute same query more than one time
+        self.__query_history: Dict[str, str] = {}
+        self.file_system = TemporaryFileSystem()
+
+    def get_data(self, query: str, path: Optional[str] = None) -> str:
+        """
+        fetching data from pg using query
+        and saving into csv locating in temporary directory
+        """
+        to_execute = f"""
+            copy ( {query} )
+            to stdout
+            WITH
+            CSV HEADER
+        """
+        if path is None:
+            path = self.file_system.save_path()
+
+        if query not in self.__query_history:
+            with open(path, "w", encoding=self.encoding) as fp:
+                self.cursor.copy_expert(to_execute, fp, size=self.size)
+                self.close
+                self.__query_history[query] = self.file_system._last_path
+
+        path = self.__query_history[query]
+        return path
+
+    def get_df(self, query: str) -> PandasDataFrame:
+        path = self.get_data(query)
+        return pd.read_csv(path, encoding=self.encoding, low_memory=False)
+
+
+class SaveDataManipulatorPG(ConnectorPG):
+    def __init__(
+        self,
+        table_name: str,
+        db_schema: Optional[str] = None,
+        encoding="utf-16",
+        size: int = 262144,
+        db_config_name: str = "postgres",
+    ) -> None:
+        super().__init__(db_config_name=db_config_name)
+        self.table_name = table_name
+        self.db_schema = db_schema
+        self.encoding = encoding
+        self.size = size
+
+        self.file_system = TemporaryFileSystem()
+
+        self.db_utils = DBUtilsPG(db_config_name)
+        if self.db_schema is None:
+            self.db_schema = self.db_utils.get_schema_by_table(self.table_name)
+
+    @staticmethod
+    def __get_columns_right_order(table_schema: Dict[str, str]) -> list:
+        """
+        Return list of columns sorted by order in destination table
+        """
+        column_positions = {
+            column: meta_info["position"] for column, meta_info in table_schema.items()
+        }
+        columns_sorted_by_position = sorted(column_positions, key=column_positions.get)
+        return columns_sorted_by_position
+
+    def _validate_df_to_save(self, df) -> PandasDataFrame:
+        """
+        Check that DataFrame have all required columns
+        """
+        table_schemа = self.db_utils.get_table_schema(self.table_name, self.db_schema)
+        columns_intersection = set(table_schemа).intersection(df.columns)
+        columns_not_found = set(table_schemа) - columns_intersection
+
+        if len(columns_not_found) != 0:
+            raise Exception(f"Columns not found: {columns_not_found}")
+
+        column_to_write = self.__get_columns_right_order(table_schemа)
+        return df[column_to_write]
+
+    def init_df(self, df: PandasDataFrame) -> str:
+        """
+        Validate df with pg table and save df as csv into temporary directory
+        """
+        df = self._validate_df_to_save(df)
+
+        path = self.file_system.save_path()
+        df.to_csv(path, encoding=self.encoding, index=False)
+        return path
+
+    def save_data(
+        self,
+        path: str,
+    ):
+        """
+        reading file from path
+        and saving file to self.table_save
+
+        self.table_save: str - PG table where to save data
+        path: str - path to file which should be copied into PG
+
+        Using with combination self.init_data
+        or set path manualy
+        """
+
+        to_execute = f"""
+            copy {self.db_schema}.{self.table_name}
+            from STDIN
+            WITH
+            CSV header
+        """
+
+        with open(path, "r", encoding=self.encoding) as fp:
+            self.cursor.copy_expert(to_execute, fp, size=self.size)
+            self.commit
+            self.close
+
+    def save_df(self, df: PandasDataFrame):
+        path = self.init_df(df)
+        self.save_data(path)

connector_for_bb-0.0.1/src/dump/postgres/table.py

@@ -0,0 +1,167 @@
+from typing import Optional
+
+import pandas as pd
+from pandas import DataFrame as PandasDataFrame
+
+from .connectors import ConnectorPG, DBUtilsPG, SequenceUtils
+from .manipulators import GetDataManipulatorPG, SaveDataManipulatorPG
+from .types_util import cast_df_by_schema
+
+
+TECH_COLUMNS = {"_inserted_dttm", "_job_id"}
+
+
+class TablePG(ConnectorPG):
+    def __init__(
+        self,
+        table_name: str,
+        db_schema: Optional[str] = None,
+        limit: int = None,
+        auto_cast: bool = True,
+        clear_tech_cols: bool = True,
+        db_config_name: str = "postgres",
+    ) -> None:
+        super().__init__(db_config_name=db_config_name)
+        self.table_name = table_name
+        self.db_schema = db_schema
+        self._limit = limit
+        self.auto_cast = auto_cast
+        self.clear_tech_cols = clear_tech_cols
+
+        self.data_manipulator = GetDataManipulatorPG(db_config_name=self.db_config_name)
+        self.db_utils = DBUtilsPG(db_config_name=db_config_name)
+
+        if self.db_schema is None:
+            self.db_schema = self.db_utils.get_schema_by_table(self.table_name)
+
+        if self.auto_cast:
+            self.table_schema = self.db_utils.get_table_schema(
+                self.table_name,
+                self.db_schema,
+            )
+
+        self._filters_args: set = set()
+        self._select_args: set = set()
+
+    def filter(self, *args, operation: str = "and"):
+        self._filters_args: set = set()
+        self._filters_args.update([f" {operation} {arg}" for arg in args])
+        return self
+
+    @property
+    def _filters(self) -> str:
+        filters = "WHERE 1=1"
+        filters += " ".join(self._filters_args)
+        return filters
+
+    def select(self, *args):
+        self._select_args: set = set()
+        self._select_args.update(args)
+        return self
+
+    @property
+    def _select(self) -> str:
+        select = "select "
+        if len(self._select_args) == 0:
+            select += "*"
+        select += ", ".join(self._select_args)
+        return select
+
+    @property
+    def query(self) -> str:
+        limit = ""
+        if self._limit is not None:
+            limit = f"LIMIT {self._limit}"
+
+        query = f"""
+            {self._select}
+            FROM {self.db_schema}.{self.table_name}
+            {self._filters}
+            {limit}
+        """
+        return query
+
+    @property
+    def _clear_statments(self):
+        self._filters_args: set = set()
+        self._select_args: set = set()
+        return self
+
+    def get_df(self, query: Optional[str] = None) -> PandasDataFrame:
+        if query is None:
+            query = self.query
+
+        df = self.data_manipulator.get_df(query)
+
+        if self.auto_cast:
+            df = cast_df_by_schema(df, self.table_schema)
+
+        if self.clear_tech_cols:
+            to_drop = TECH_COLUMNS.intersection(df.columns)
+            df = df.drop(columns=to_drop)
+        return df
+
+    def count(self) -> PandasDataFrame:
+        query = f"""select count(*) as cnt from ({self.query}) s"""
+        return self.get_df(query)["cnt"]
+
+
+class JobIDSequence(SequenceUtils):
+    def __init__(
+        self,
+        db_config_name: str = "postgres",
+    ):
+        super().__init__(
+            sequence_name="seq_general_job_id",
+            sequence_schema="meta",
+            db_config_name=db_config_name,
+        )
+
+
+class SaveTablePG:
+    def __init__(
+        self,
+        table_name: str,
+        db_schema: Optional[str] = None,
+        insert_dttm_col_name: str = "_inserted_dttm",
+        job_id_col_name: str = "_job_id",
+        auto_cast: bool = True,
+        size: int = 131072,
+        db_config_name: str = "postgres",
+        sequence_util: SequenceUtils = JobIDSequence,
+    ) -> None:
+        self.table_name = table_name
+        self.db_schema = db_schema
+        self.insert_dttm_col_name = insert_dttm_col_name
+        self.job_id_col_name = job_id_col_name
+        self.auto_cast = auto_cast
+        self.db_config_name = db_config_name
+
+        self.data_manipulator = SaveDataManipulatorPG(
+            size=size,
+            table_name=table_name,
+            db_schema=db_schema,
+            db_config_name=self.db_config_name,
+        )
+        self.sequence_utils = sequence_util(db_config_name=self.db_config_name)
+        self._job_id = None
+
+    def _create_tech_cols(self, df: PandasDataFrame) -> PandasDataFrame:
+        table_schemа = self.data_manipulator.db_utils.get_table_schema(
+            self.table_name, db_schema=self.db_schema
+        )
+        df[self.insert_dttm_col_name] = pd.Timestamp.now()
+
+        if self.job_id_col_name in table_schemа.keys():
+            self._job_id = self.sequence_utils.next_sequence
+            df[self.job_id_col_name] = self._job_id
+            print("current sequence in that job", self._job_id)
+
+        if self.auto_cast:
+            df = cast_df_by_schema(df, table_schemа, cast_timestamp=False)
+        return df
+
+    def save(self, df: PandasDataFrame) -> int:
+        df = self._create_tech_cols(df)
+        self.data_manipulator.save_df(df)
+        return self._job_id

connector_for_bb-0.0.1/src/dump/postgres/types_util.py

@@ -0,0 +1,32 @@
+from typing import Dict
+
+import numpy as np
+import pandas as pd
+from pandas import DataFrame as PandasDataFrame
+
+
+BOOL_MAPPER = {"f": False, "t": True}
+
+
+def cast_df_by_schema(
+    df: PandasDataFrame, schema: Dict[str, str], cast_timestamp: bool = True
+) -> PandasDataFrame:
+    for column in df.columns:
+        if column in schema:
+            cast_to = schema[column]["data_type"]
+            try:
+                if cast_to == "timestamp":
+                    if cast_timestamp:
+                        df[column] = pd.to_datetime(df[column], errors="coerce")
+                    continue
+
+                if cast_to == "bool":
+                    df[column] = df[column].apply(lambda x: BOOL_MAPPER.get(x, np.nan))
+
+                df = df.astype({column: cast_to})
+            except Exception as e:
+                print(f"unable to cast column: {column} to {cast_to} error: {e}")
+                pass
+    df = df.replace("nan", np.nan)
+    df = df.replace("None", np.nan)
+    return df

connector_for_bb-0.0.1/src/dump/s3/connectors.py

@@ -0,0 +1,116 @@
+import os
+from typing import Dict, Optional
+
+import boto3
+import pandas as pd
+from botocore.config import Config
+from dump.config_utils import load_config
+from dump.files.tmp_util import TemporaryFileSystem
+from pandas import DataFrame as PandasDataFrame
+
+
+class S3Client:
+    def __init__(
+        self, creds_section: str, config_filiname: str = "s3_config.ini"
+    ) -> None:
+        self.creds_section = creds_section
+
+        self.config = load_config(filename=config_filiname, section=creds_section)
+        self.conn_config = Config(
+            s3={
+                "addressing_style": "virtual",
+            },
+            retries={"max_attempts": 10, "mode": "standard"},
+            region_name="us-east-1",
+        )
+
+        self.client = boto3.client(
+            "s3",
+            **self.config,
+            config=self.conn_config,
+        )
+
+    def get_files_info(self, bucket_name: str, object_folder: Optional[str] = ""):
+        files_dict = {}
+        response = self.client.list_objects_v2(Bucket=bucket_name, Prefix=object_folder)
+        if "Contents" in response:
+            for obj in response["Contents"]:
+                size = obj["Size"]
+                files_dict[obj["Key"]] = {
+                    "size_bytes": size,
+                    "size_gb": size / 1024**3,
+                }
+        return files_dict
+
+
+class S3Upload(S3Client):
+    def __init__(
+        self,
+        bucket_name: str,
+        object_folder: str = "",
+        creds_section: str = "test",
+    ) -> None:
+        super().__init__(creds_section)
+        self.bucket_name = bucket_name
+        self.object_folder = object_folder
+
+    def _get_path(self, file_name) -> str:
+        """
+        creating path where to save file in s3
+        """
+        path = os.path.join(self.object_folder, file_name)
+        return path
+
+    def upload_to_s3(self, tmp_path_file, file_name: str):
+        """
+        upload local file into s3 bucket
+        """
+        path = self._get_path(file_name)
+        self.client.upload_file(tmp_path_file, self.bucket_name, path)
+
+
+class S3Download(S3Client):
+    def __init__(
+        self,
+        bucket_name: str,
+        object_folder: str = "",
+        creds_section: str = "test",
+    ) -> None:
+        super().__init__(creds_section)
+        self.bucket_name = bucket_name
+        self.object_folder = object_folder
+
+        self.file_system = TemporaryFileSystem()
+
+        # requires for not download same file more than one time
+        self.__downloads_history: Dict[str, str] = {}
+
+    def _check_path_exist(self, path: str) -> bool:
+        exsist_files = self.get_files_info(self.bucket_name, self.object_folder)
+        return path in exsist_files
+
+    def _get_object_path(self, file_name) -> str:
+        path = os.path.join(self.object_folder, file_name)
+        return path
+
+    def download_from_s3(self, download_file_name: str) -> str:
+        """
+        downloading file from s3 and put the file into temporary directory
+        returns path to file in temporary directory
+        """
+        object_path = self._get_object_path(download_file_name)
+        if self._check_path_exist(object_path):
+            if object_path not in self.__downloads_history:
+                save_path = self.file_system.save_path()
+                self.client.download_file(self.bucket_name, object_path, save_path)
+                self.__downloads_history[object_path] = save_path
+
+            return self.__downloads_history[object_path]
+        else:
+            raise ValueError(f"The file {object_path} was not found")
+
+    def download_from_s3_df(
+        self, download_file_name: str, encoding="utf-8"
+    ) -> PandasDataFrame:
+        df_path = self.download_from_s3(download_file_name)
+        return pd.read_csv(df_path, encoding=encoding, low_memory=False)