supertable 0.1.0__tar.gz

supertable-0.1.0/LICENSE

@@ -0,0 +1,84 @@
+SUPER TABLE PUBLIC USE LICENSE (STPUL) v1.0
+
+1. Definitions
+   1.1. “Software” refers to the source code, binary code, documentation,
+        and related materials for the project known as “SuperTable.”
+   1.2. “Licensor” refers to kladna Soft kft.
+   1.3. “You” (or “Your”) means the individual or entity exercising the
+        rights granted under this License.
+
+2. Grant of License
+   2.1. Personal, Non-Commercial, and Evaluation Use
+       - You are granted a worldwide, royalty-free, non-exclusive,
+         non-transferable license to download, install, and use the
+         Software solely for personal, non-commercial, or evaluation
+         purposes at no cost.
+       - Evaluation use is limited to internal testing and shall not
+         exceed 30 days unless otherwise authorized in writing by the
+         Licensor.
+
+   2.2. Commercial or Enterprise Use
+       - If You wish to use the Software for commercial purposes,
+         within an enterprise environment, or as part of any product
+         or service provided to a third party, You must obtain a
+         separate commercial license from the Licensor. Fees may apply.
+
+3. No Modification
+   3.1. You may not modify, reverse engineer, decompile, disassemble,
+        create derivative works from, or otherwise alter the Software
+        in whole or in part without the prior written consent of the
+        Licensor.
+
+4. No Redistribution
+   4.1. You may not publish, distribute, sublicense, or otherwise make
+        the Software available to any third party, unless You have
+        obtained explicit written permission from the Licensor.
+
+5. Ownership and Reservation of Rights
+   5.1. All rights, title, and interest in the Software remain with the
+        Licensor. This License does not transfer any ownership rights.
+   5.2. Access to the source code is provided for transparency and
+        evaluation only. It does not constitute a grant of rights
+        beyond those explicitly stated in this License.
+   5.3. The Licensor reserves all rights not expressly granted to You
+        under this License.
+
+6. Disclaimer of Warranties
+   6.1. THE SOFTWARE IS PROVIDED “AS IS,” WITHOUT WARRANTY OF ANY KIND,
+        EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+        OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+        NON-INFRINGEMENT. THE ENTIRE RISK AS TO THE QUALITY AND
+        PERFORMANCE OF THE SOFTWARE IS WITH YOU.
+
+7. Limitation of Liability
+   7.1. IN NO EVENT SHALL THE LICENSOR BE LIABLE FOR ANY INDIRECT,
+        INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+        (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+        GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+        INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+        WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+        NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+        THE SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+8. Termination
+   8.1. This License and the rights granted hereunder will terminate
+        automatically if You fail to comply with any term or condition
+        of this License. Upon termination, You must cease all use of
+        the Software and destroy any copies of the Software in Your
+        possession or control.
+   8.2. The Licensor reserves the right to revoke this License in the
+        event of a violation or at its sole discretion, subject to
+        applicable law.
+
+9. Governing Law
+   9.1. This License shall be governed by and construed in accordance
+        with the laws of Hungary, without regard to its conflict of
+        law principles.
+
+10. Contact Information
+   10.1. For inquiries about commercial licensing or any other questions
+         regarding this License, please contact:
+         lkupas@kladnasoft.com
+
+BY USING THE SOFTWARE, YOU ACKNOWLEDGE THAT YOU HAVE READ THIS LICENSE,
+UNDERSTAND IT, AND AGREE TO BE BOUND BY ITS TERMS AND CONDITIONS.

supertable-0.1.0/MANIFEST.in

@@ -0,0 +1,17 @@
+# MANIFEST.in
+
+# Essential top‑level docs
+include README.md
+include LICENSE.txt
+include pyproject.toml
+#include setup.py
+
+# Remove bulky or dev‑only content
+prune examples
+prune tests
+prune tmp
+prune generator
+prune */__pycache__
+
+# Exclude compiled artifacts and large sample data
+global-exclude *.py[cod] *.parquet *.csv *.json

supertable-0.1.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: supertable
+Version: 0.1.0
+Summary: SuperTable revolutionizes data management by integrating multiple basic tables into a single, cohesive framework.
+Home-page: https://github.com/kladnasoft/supertable
+Author: Levente Kupas
+Author-email: Levente Kupas <lkupas@kladnasoft.com>
+License: Super Table Public Use License (STPUL) v1.0
+Project-URL: Homepage, https://github.com/kladnasoft/supertable
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# SuperTable
+
+![Python](https://img.shields.io/badge/python-3.10+-blue)
+![License: STPUL](https://img.shields.io/badge/license-STPUL-blue)
+
+**SuperTable — The simplest data warehouse & cataloging system.**  
+A high‑performance, lightweight transaction catalog that integrates multiple
+basic tables into a single, cohesive framework designed for ultimate
+efficiency.
+
+Whether you are processing massive datasets or continuously evolving data
+structures, SuperTable lets you query your information effortlessly. It
+automatically creates and manages tables so you can start running SQL queries
+immediately—no complicated schemas or manual joins required.
+
+By holding all your data in one place, SuperTable removes complexity, boosts
+efficiency, and provides a unified view of your information. From startups to
+enterprises, SuperTable helps you make better decisions, faster.
+
+---
+
+## Key Features
+
+- **Automatic table creation**  
+  Load your data and SuperTable instantly builds the required tables and
+  columns—no predefined schema or extra setup.
+
+- **Self‑referencing architecture**  
+  Combine and analyze data across tables without writing manual joins; tables
+  can dynamically reference each other for richer insights.
+
+- **Staging module with history**  
+  Upload files to a staging area and reload any version at any time, keeping a
+  complete audit trail for tracking and compliance.
+
+- **Columnar storage for speed**  
+  Fully denormalized, column‑partitioned tables deliver lightning‑fast queries,
+  even with thousands of columns.
+
+- **Built‑in RBAC security**  
+  Define users and roles to control row‑ and column‑level access—no external
+  security tools required.
+
+- **Platform independent**  
+  Deploy on any major cloud provider or on‑premise. SuperTable is a pure Python
+  library with no hidden costs.
+
+---
+
+## Benefits
+
+- **Quick start**  
+  Go from raw data to query‑ready in minutes—faster than spreadsheets or
+  traditional databases.
+
+- **Higher efficiency**  
+  Eliminate manual steps and rework so you spend more time analyzing and less
+  time wrangling.
+
+- **Holistic insights**  
+  Analyze datasets individually or together to uncover trends, outliers, and
+  cross‑dependencies.
+
+- **Cost savings**  
+  Consolidate licenses, simplify support, and reinvest the savings in deeper
+  analytics.
+
+---
+
+SuperTable provides a flexible, high‑performance solution that grows with your
+business. Cut complexity, save time, and gain deeper insights—all in a single,
+streamlined platform.
+

supertable-0.1.0/README.md

@@ -0,0 +1,73 @@
+# SuperTable
+
+![Python](https://img.shields.io/badge/python-3.10+-blue)
+![License: STPUL](https://img.shields.io/badge/license-STPUL-blue)
+
+**SuperTable — The simplest data warehouse & cataloging system.**  
+A high‑performance, lightweight transaction catalog that integrates multiple
+basic tables into a single, cohesive framework designed for ultimate
+efficiency.
+
+Whether you are processing massive datasets or continuously evolving data
+structures, SuperTable lets you query your information effortlessly. It
+automatically creates and manages tables so you can start running SQL queries
+immediately—no complicated schemas or manual joins required.
+
+By holding all your data in one place, SuperTable removes complexity, boosts
+efficiency, and provides a unified view of your information. From startups to
+enterprises, SuperTable helps you make better decisions, faster.
+
+---
+
+## Key Features
+
+- **Automatic table creation**  
+  Load your data and SuperTable instantly builds the required tables and
+  columns—no predefined schema or extra setup.
+
+- **Self‑referencing architecture**  
+  Combine and analyze data across tables without writing manual joins; tables
+  can dynamically reference each other for richer insights.
+
+- **Staging module with history**  
+  Upload files to a staging area and reload any version at any time, keeping a
+  complete audit trail for tracking and compliance.
+
+- **Columnar storage for speed**  
+  Fully denormalized, column‑partitioned tables deliver lightning‑fast queries,
+  even with thousands of columns.
+
+- **Built‑in RBAC security**  
+  Define users and roles to control row‑ and column‑level access—no external
+  security tools required.
+
+- **Platform independent**  
+  Deploy on any major cloud provider or on‑premise. SuperTable is a pure Python
+  library with no hidden costs.
+
+---
+
+## Benefits
+
+- **Quick start**  
+  Go from raw data to query‑ready in minutes—faster than spreadsheets or
+  traditional databases.
+
+- **Higher efficiency**  
+  Eliminate manual steps and rework so you spend more time analyzing and less
+  time wrangling.
+
+- **Holistic insights**  
+  Analyze datasets individually or together to uncover trends, outliers, and
+  cross‑dependencies.
+
+- **Cost savings**  
+  Consolidate licenses, simplify support, and reinvest the savings in deeper
+  analytics.
+
+---
+
+SuperTable provides a flexible, high‑performance solution that grows with your
+business. Cut complexity, save time, and gain deeper insights—all in a single,
+streamlined platform.
+

supertable-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "supertable"
+version = "0.1.0"
+description = "SuperTable revolutionizes data management by integrating multiple basic tables into a single, cohesive framework."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Super Table Public Use License (STPUL) v1.0" }
+authors = [
+    { name = "Levente Kupas", email = "lkupas@kladnasoft.com" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: Other/Proprietary License",
+    "Operating System :: OS Independent"
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[project.urls]
+Homepage = "https://github.com/kladnasoft/supertable"

supertable-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

supertable-0.1.0/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="supertable",
+    version="0.1.0",
+    packages=find_packages(include=["supertable", "supertable.*"]),
+    include_package_data=True,
+    author="Levente Kupas",
+    author_email="lkupas@kladnasoft.com",
+    description="A high-performance, lightweight transaction cataloging system designed for ultimate efficiency.",
+    license="Super Table Public Use License (STPUL) v1.0",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    url="https://github.com/kladnasoft/supertable",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: Other/Proprietary License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.10",
+)

supertable-0.1.0/supertable/config/defaults.py

@@ -0,0 +1,81 @@
+import os
+import logging
+import sys
+
+from dotenv import load_dotenv
+from dataclasses import dataclass
+
+import colorlog
+
+# Configure colors for all levels
+handler = colorlog.StreamHandler()
+handler.setFormatter(colorlog.ColoredFormatter(
+    '%(log_color)s%(asctime)s - %(levelname)-8s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    log_colors={
+        'DEBUG': 'cyan',
+        'INFO': 'green',
+        'WARNING': 'yellow',
+        'ERROR': 'red',
+        'CRITICAL': 'red,bg_white',  # White background with red text
+    },
+    secondary_log_colors={},
+    style='%'
+))
+
+logging.basicConfig(level=logging.INFO, handlers=[handler])
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Default:
+    MAX_MEMORY_CHUNK_SIZE: int = 16 * 1024 * 1024
+    DEFAULT_TIMEOUT_SEC: int = 10
+    DEFAULT_LOCK_DURATION_SEC: int = 60
+    LOG_LEVEL: str = "INFO"  # Replaced IS_DEBUG with LOG_LEVEL
+    IS_SHOW_TIMING: bool = True
+    STORAGE_TYPE: str = "LOCAL"
+
+    def update_default(self, **kwargs):
+        """
+        Updates fields of this Default instance in-place
+        with any matching keys in kwargs.
+        """
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+                if key == "LOG_LEVEL":
+                    self._update_log_level()
+
+    def _update_log_level(self):
+        """Update the logging level based on the current setting"""
+        logging.getLogger().setLevel(self.LOG_LEVEL)
+        logger.info(f"Log level changed to {self.LOG_LEVEL}")
+
+
+def load_defaults_from_env(env_file: str = ".env") -> Default:
+    load_dotenv(env_file)
+
+    # Get log level from env, default to INFO
+    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+
+    # Validate log level
+    valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    if log_level not in valid_levels:
+        log_level = "INFO"
+        logger.warning(f"Invalid LOG_LEVEL in .env. Using default INFO. Valid levels are: {valid_levels}")
+
+    # Set the log level immediately
+    logging.getLogger().setLevel(log_level)
+
+    return Default(
+        MAX_MEMORY_CHUNK_SIZE=int(os.getenv("MAX_MEMORY_CHUNK_SIZE", 16 * 1024 * 1024)),
+        DEFAULT_TIMEOUT_SEC=int(os.getenv("DEFAULT_TIMEOUT_SEC", 10)),
+        DEFAULT_LOCK_DURATION_SEC=int(os.getenv("DEFAULT_LOCK_DURATION_SEC", 60)),
+        LOG_LEVEL=log_level,
+        IS_SHOW_TIMING=(os.getenv("IS_SHOW_TIMING", "True").lower() == "true"),
+        STORAGE_TYPE=os.getenv("STORAGE_TYPE", "LOCAL").upper(),
+    )
+
+
+default = load_defaults_from_env()

supertable-0.1.0/supertable/config/homedir.py

@@ -0,0 +1,34 @@
+import os
+import sys
+
+from supertable.config.defaults import default, logger
+
+# If this file is located in a subdirectory, adjust the path logic as needed.
+# Currently appending ".." from __file__ to add the project root directory
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+app_home = os.getenv("SUPERTABLE_HOME", "~/supertable")
+
+def change_to_app_home(home_dir: str) -> None:
+    """
+    Attempts to change the current working directory to `home_dir`.
+    Prints the outcome. Logs (or prints) any error encountered.
+    """
+    expanded_dir = os.path.expanduser(home_dir)
+    try:
+        os.chdir(expanded_dir)
+        logger.debug(f"Changed working directory to {expanded_dir}")
+    except Exception as e:
+        logger.error(f"Failed to change working directory to {expanded_dir}: {e}")
+
+if app_home:
+    change_to_app_home(app_home)
+else:
+    logger.error("SUPERTABLE_HOME environment variable is not set")
+
+
+logger.info(f"Current working directory: {os.getcwd()}")
+
+
+def get_app_home():
+    return app_home

supertable-0.1.0/supertable/data_reader.py

@@ -0,0 +1,213 @@
+from enum import Enum
+
+import duckdb
+import pandas as pd
+
+from supertable.config.defaults import logger
+from supertable.utils.timer import Timer
+from supertable.super_table import SuperTable
+from supertable.query_plan_manager import QueryPlanManager
+from supertable.utils.sql_parser import SQLParser
+from supertable.utils.helper import dict_keys_to_lowercase
+from supertable.plan_extender import PlanStats, extend_execution_plan
+from supertable.rbac.access_control import restrict_read_access
+
+class Status(Enum):
+    OK = "ok"
+    ERROR = "error"
+
+
+class DataReader:
+    def __init__(self, super_name, organization, query):
+        self.super_table = SuperTable(super_name=super_name, organization=organization)
+        self.parser = SQLParser(query)
+        self.parser.parse_sql()
+        self.timer = None
+        self.plan_stats = None
+        self.query_plan_manager = None
+
+    def filter_snapshots(self, super_table_data, super_table_meta):
+        snapshots = super_table_data.get("snapshots")
+        file_count = super_table_meta.get("file_count", 0)
+        total_rows = super_table_meta.get("total_rows", 0)
+        total_file_size = super_table_meta.get("total_file_size", 0)
+        self.plan_stats.add_stat({"TABLE_FILES": file_count})
+        self.plan_stats.add_stat({"TABLE_SIZE": total_file_size})
+        self.plan_stats.add_stat({"TABLE_ROWS": total_rows})
+
+        if self.super_table.super_name.lower() == self.parser.original_table.lower():
+            filtered_snapshots = [
+                s for s in snapshots
+                if not (s["table_name"].startswith("__") and s["table_name"].endswith("__"))
+            ]
+            return filtered_snapshots
+        else:
+            filtered_snapshots = [
+                entry
+                for entry in snapshots
+                if entry["table_name"].lower() == self.parser.original_table.lower()
+            ]
+
+            return filtered_snapshots
+
+    timer = Timer()
+    @timer
+    def execute(self, user_hash: str, with_scan: bool=False):
+        status = Status.ERROR
+        message = None
+        self.timer = Timer()
+        self.plan_stats = PlanStats()
+
+        try:
+            super_table_data, super_table_path, super_table_meta = self.super_table.get_super_table_and_path_with_shared_lock()
+
+            self.timer.capture_and_reset_timing(event="META")
+
+            self.query_plan_manager = QueryPlanManager(super_name=self.super_table.super_name,
+                                                       organization=self.super_table.organization,
+                                                       current_meta_path=super_table_path,
+                                                       parser=self.parser)
+
+            snapshots = self.filter_snapshots(super_table_data=super_table_data,
+                                              super_table_meta=super_table_meta)
+            logger.debug(f"Filtered snapshots: {snapshots}")
+
+            parquet_files, schema = self.process_snapshots(snapshots=snapshots,
+                                                           with_scan=with_scan)
+            logger.debug(f"Parquet Files: {parquet_files}")
+
+            missing_columns = (
+                set([column.lower() for column in self.parser.columns_list])
+                - set("*")
+                - schema
+            )
+            logger.debug(f"Mising Columns: {missing_columns}")
+
+            if len(snapshots) == 0 or missing_columns or not parquet_files:
+                message = (
+                    f"Missing column(s): {', '.join ( missing_columns )}"
+                    if missing_columns
+                    else "No parquet files found"
+                )
+                logger.warning(f"Filter Result: {message}")
+                return pd.DataFrame(), status, message
+
+            restrict_read_access(super_name=self.super_table.super_name,
+                                 organization=self.super_table.organization,
+                                 user_hash=user_hash,
+                                 table_name=self.parser.reflection_table,
+                                 table_schema=schema,
+                                 parsed_columns=self.parser.columns_list,
+                                 parser=self.parser)
+
+            self.timer.capture_and_reset_timing(event="FILTERING")
+
+            result = self.execute_with_duckdb(parquet_files=parquet_files,
+                                              query_manager=self.query_plan_manager)
+
+            status = Status.OK
+        except Exception as e:
+            message = str(e)
+            logger.error(f"Exception: {e}")
+            result = pd.DataFrame()
+        self.timer.capture_and_reset_timing(event="EXECUTING_QUERY")
+
+        try:
+            extend_execution_plan(super_table=self.super_table,
+                                  user_hash=user_hash,
+                                  query_plan_manager=self.query_plan_manager,
+                                  timing=self.timer.timings,
+                                  status=status.value,
+                                  message=message,
+                                  result_shape=result.shape,
+                                  plan_stats=self.plan_stats)
+        except Exception as e:
+            logger.error(f"Exception: {e}")
+
+        self.timer.capture_and_reset_timing(event="EXTENDING_PLAN")
+        self.timer.capture_duration(event="TOTAL_EXECUTE")
+        return result, status, message
+
+    def process_snapshots(self, snapshots, with_scan):
+        parquet_files = []
+        reflection_file_size = 0
+        reflection_rows = 0
+
+        schema = set()
+        for snapshot in snapshots:
+            current_snapshot_path = snapshot["path"]
+            current_snapshot_data = self.super_table.read_simple_table_snapshot(
+                current_snapshot_path
+            )
+
+            current_schema = current_snapshot_data.get("schema", {})
+            resources = current_snapshot_data.get("resources", {})
+            schema.update(dict_keys_to_lowercase(current_schema).keys())
+
+            for resource in resources:
+                file_size = resource.get("file_size", 0)
+                file_rows = resource.get("rows", 0)
+
+                if (
+                    with_scan
+                    or self.parser.columns_csv == "*"
+                    or any(
+                        col in dict_keys_to_lowercase(current_schema).keys()
+                        for col in [
+                            column.lower() for column in self.parser.columns_list
+                        ]
+                    )
+                ):
+                    parquet_files.append(resource["file"])
+                    reflection_file_size += file_size
+                    reflection_rows += file_rows
+
+            logger.debug(f"snapshots: {len ( snapshots )}")
+            logger.debug(f"parquet_files: {len ( parquet_files )}")
+            logger.debug(f"schema: {schema}")
+
+        self.plan_stats.add_stat({"REFLECTIONS": len(parquet_files)})
+        self.plan_stats.add_stat({"REFLECTION_SIZE": reflection_file_size})
+        self.plan_stats.add_stat({"REFLECTION_ROWS": reflection_rows})
+
+        return parquet_files, schema
+
+    def execute_with_duckdb(self, parquet_files, query_manager: QueryPlanManager):
+        # Use DuckDB to read and query the parquet files directly
+        con = duckdb.connect()
+
+        con.execute("PRAGMA memory_limit='2GB';")
+        con.execute(f"PRAGMA temp_directory='{query_manager.temp_dir}';")
+        con.execute("PRAGMA enable_profiling='json';")
+        #con.execute("SET profiling_mode = 'standard';")
+        con.execute(f"PRAGMA profile_output = '{query_manager.query_plan_path}';")
+
+        # Read and register parquet files directly with DuckDB
+        parquet_files_str = ", ".join(f"'{file}'" for file in parquet_files)
+        logger.debug(f"parquet files: {len(parquet_files)}")
+
+        self.timer.capture_and_reset_timing("CONNECTING")
+
+        create_table = f"""
+CREATE TABLE {self.parser.reflection_table} 
+AS 
+SELECT {self.parser.columns_csv}
+FROM parquet_scan([{parquet_files_str}], union_by_name=True, HIVE_PARTITIONING=TRUE);
+          """
+
+        logger.debug(f"create_table: {create_table}")
+        con.execute(create_table)
+
+        create_view = f"""
+CREATE VIEW {self.parser.rbac_view}
+AS
+{self.parser.view_definition}
+"""
+        logger.debug(f"create_view: {create_view}")
+        con.execute(create_view)
+
+        self.timer.capture_and_reset_timing("CREATING_REFLECTION")
+        logger.debug(f"Executing Query: {self.parser.executing_query}")
+        result = con.execute(query=self.parser.executing_query).fetchdf()
+        logger.debug(f"result.shape: {result.shape}")
+        return result