neo-data-svc 0.1.0__tar.gz

neo_data_svc-0.1.0/.gitignore

@@ -0,0 +1,86 @@
+__pycache__/
+*.py[cod]
+*$py.class
+
+*.so
+
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+*.manifest
+*.spec
+
+pip-log.txt
+pip-delete-this-directory.txt
+
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+*.mo
+*.pot
+
+*.log
+local_settings.py
+
+instance/
+.webassets-cache
+
+.scrapy
+
+docs/_build/
+
+target/
+
+.ipynb_checkpoints
+
+dask-worker-space/
+
+.python-version
+
+celerybeat-schedule
+
+*.sage.py
+
+.env
+
+.venv
+venv/
+ENV/
+
+.spyderproject
+.spyproject
+
+.ropeproject
+
+/site
+
+.mypy_cache/
+
+.vscode/
+.idea/
+
+src/emdm/files/
+uv.lock

neo_data_svc-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: neo-data-svc
+Version: 0.1.0
+Summary: A data base svc
+Author-email: Eric <email@example.com>
+License: Proprietary
+Requires-Python: >=3.12.3
+Requires-Dist: delta-spark==2.4.0
+Requires-Dist: pyspark==3.4.4
+Description-Content-Type: text/markdown
+
+# neo-data-svc

neo_data_svc-0.1.0/README.md

@@ -0,0 +1 @@
+# neo-data-svc

neo_data_svc-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "neo-data-svc"
+version = "0.1.0"
+description = "A data base svc"
+authors = [
+    {name = "Eric", email = "email@example.com"},
+]
+readme = "README.md"
+requires-python = ">=3.12.3"
+license = {text = "Proprietary"}
+
+dependencies = [
+    "pyspark==3.4.4",
+    "delta-spark==2.4.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pyright]
+reportMissingImports = false
+reportPrivateImportUsage = false
+reportUndefinedVariable = false
+enableTypeIgnoreComments = true
+reportPrivateUsage = false
+reportGeneralTypeIssues = false
+reportWildcardImportFromLibrary = false

neo_data_svc-0.1.0/src/neo_data_svc/__init__.py

@@ -0,0 +1,8 @@
+__version__ = "0.1.0"
+
+from .core import get_session, install
+
+install()
+
+S, Q = get_session()
+__all__ = ["S", "Q"]

neo_data_svc-0.1.0/src/neo_data_svc/core.py

@@ -0,0 +1,59 @@
+import logging
+
+from delta.tables import DeltaTable
+from pyspark.sql import DataFrame
+
+from .session import M
+
+logger = logging.getLogger(__name__)
+
+
+def neo_write(self, path, mode="overwrite", keys=None, **options):
+    writer = self.write.format("delta")
+
+    for key, value in options.items():
+        writer = writer.option(key, value)
+
+    try:
+        if mode == "overwrite":
+            writer.mode("overwrite").save(path)
+
+        elif mode == "append":
+            writer.mode("append").save(path)
+
+        elif mode == "merge":
+            if not keys:
+                raise ValueError("keys required for merge mode")
+
+            delta_table = DeltaTable.forPath(self.sparkSession, path)
+            condition = " AND ".join(
+                [f"target.{k} = source.{k}" for k in keys])
+
+            (delta_table.alias("target")
+                .merge(self.alias("source"), condition)
+                .whenMatchedUpdateAll()
+                .whenNotMatchedInsertAll()
+                .execute())
+
+        else:
+            raise ValueError(f"Unsupported mode: {mode}")
+
+        logger.info(f"Successfully wrote to {path} using {mode}")
+
+    except Exception as e:
+        if "Path does not exist" in str(e) and mode == "merge":
+            writer.mode("overwrite").save(path)
+            logger.info(f"Created new Delta table at {path}")
+        else:
+            raise
+
+
+def install():
+    if not hasattr(DataFrame, 'neo_write'):
+        setattr(DataFrame, "neo_write", neo_write)
+
+    logger.info("Neo Data Service installed")
+
+
+def get_session():
+    return M.get_session()

neo_data_svc-0.1.0/src/neo_data_svc/session.py

@@ -0,0 +1,95 @@
+import importlib
+import logging
+import sys
+
+from pyspark.sql import SparkSession
+
+logger = logging.getLogger(__name__)
+
+
+class M:
+    _instance = None
+    _config = None
+
+    @classmethod
+    def get_session(cls):
+        if cls._instance is None:
+            cls._init_session()
+        
+        if cls._instance:
+            return (cls._instance, cls._instance.sql)
+
+    @classmethod
+    def _discover_config(cls):
+        if cls._config is not None:
+            return cls._config
+
+        stack = []
+        frame = sys._getframe()
+        for _ in range(6):
+            if frame is None:
+                return stack
+            frame = frame.f_back
+
+        while frame:
+            mod = frame.f_globals.get('__name__', '')
+            if mod and not mod.startswith('neo_data_svc'):
+                stack.append(mod)
+            frame = frame.f_back
+
+        logger.info(f"Import stack: {stack}")
+
+        config_paths = cls._generate_config_paths(stack)
+        config = {}
+
+        for path in config_paths:
+            try:
+                module = importlib.import_module(path)
+                cls._config = cls._parse_config(module, config)
+                logger.info(f"config from: {path}")
+                return cls._config
+            except ImportError as e:
+                logger.debug(f"Failed to import {path}: {e}")
+                continue
+
+        cls._config = config
+        return cls._config
+
+    @classmethod
+    def _generate_config_paths(cls, stack):
+        paths = []
+
+        for module_name in stack:
+            if '.' in module_name:
+                root_package = module_name.split('.')[0]
+                paths.extend([f"{root_package}.config.Cfg"])
+
+        unique_paths = list(dict.fromkeys(paths))
+        return unique_paths
+
+    @classmethod
+    def _parse_config(cls, module, base_config):
+        config = base_config.copy()
+
+        for attr_name in dir(module):
+            value = getattr(module, attr_name)
+            config[attr_name] = value
+        return config
+
+    @classmethod
+    def _init_session(cls):
+        config = cls._discover_config()
+        cls._instance = SparkSession.builder.appName(config['APP']).getOrCreate() # type: ignore
+
+    @classmethod
+    def get_config(cls):
+        if cls._config is None:
+            cls._discover_config()
+        return cls._config
+
+    @classmethod
+    def stop_session(cls):
+        if cls._instance:
+            cls._instance.stop()
+            cls._instance = None
+            cls._config = None