pydpm_xl 0.1.39rc32__py3-none-any.whl → 0.2.0__py3-none-any.whl

py_dpm/{models.py → dpm/models.py}

@@ -22,10 +22,34 @@ from sqlalchemy import func
 import pandas as pd
 import warnings
 
-# Suppress pandas UserWarning about SQLAlchemy connection types
-warnings.filterwarnings("ignore", message=".*pandas only supports SQLAlchemy.*")
+from sqlalchemy.inspection import inspect
 
-Base = declarative_base()
+
+class SerializationMixin:
+    """Mixin to add serialization capabilities to SQLAlchemy models."""
+
+    def to_dict(self):
+        """Convert the model instance to a dictionary."""
+        return {c.key: getattr(self, c.key) for c in inspect(self).mapper.column_attrs}
+
+
+Base = declarative_base(cls=SerializationMixin)
+
+
+def _read_sql_with_connection(sql, session):
+    """
+    Execute pd.read_sql with proper connection handling to avoid pandas warnings.
+
+    Uses the raw DBAPI connection which works reliably with compiled SQL strings,
+    while suppressing the pandas warning about DBAPI2 connections.
+    """
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=".*pandas only supports SQLAlchemy.*",
+            category=UserWarning,
+        )
+        return pd.read_sql(sql, session.connection().connection)
 
 
 def _compile_query_for_pandas(query_statement, session):
@@ -670,6 +694,12 @@ class ModuleVersion(Base):
     module_version_compositions = relationship(
         "ModuleVersionComposition", back_populates="module_version"
     )
+
+    table_versions = relationship(
+        "TableVersion",
+        secondary="ModuleVersionComposition",
+        viewonly=True,
+    )
     operation_scope_compositions = relationship(
         "OperationScopeComposition", back_populates="module_version"
     )
@@ -768,7 +798,7 @@ class ModuleVersion(Base):
                 ]
             )
 
-        from py_dpm.models import TableVersion
+        from py_dpm.dpm.models import TableVersion
 
         query = (
             session.query(
@@ -1371,8 +1401,6 @@ class Release(Base):
     iscurrent = Column("IsCurrent", Boolean)
     rowguid = Column("RowGUID", String(36), ForeignKey("Concept.ConceptGUID"))
     latestvariablegentime = Column("LatestVariableGenTime", DateTime)
-    name = Column("Name", String(50))
-
     # Relationships
     concept = relationship("Concept", foreign_keys=[rowguid])
     changelogs = relationship("Changelog", back_populates="release")
@@ -2224,9 +2252,9 @@ class ViewDatapoints(Base):
                 release_id,
             )
 
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2239,9 +2267,7 @@ class ViewDatapoints(Base):
     def get_datapoints_sample(cls, session, limit=1000):
         """Get a sample of datapoints"""
         query = cls.create_view_query(session)
-        return pd.read_sql_query(
-            query.limit(limit).statement, session.connection().connection
-        )
+        return pd.read_sql_query(query.limit(limit).statement, session.get_bind())
 
     @classmethod
     def export_datapoints_query(cls, session):
@@ -2249,7 +2275,7 @@ class ViewDatapoints(Base):
         query = cls.create_view_query(session)
         return str(
             query.statement.compile(
-                dialect=session.connection().connection.dialect,
+                dialect=session.get_bind().dialect,
                 compile_kwargs={"literal_binds": True},
             )
         )
@@ -2257,9 +2283,9 @@ class ViewDatapoints(Base):
     @classmethod
     def get_all_datapoints(cls, session):
         query = cls.create_view_query(session)
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2364,9 +2390,9 @@ class ViewDatapoints(Base):
                 release_id,
             )
 
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
         # BUGFIX: Remove duplicates based on cell_code
@@ -2405,9 +2431,9 @@ class ViewDatapoints(Base):
             query, ModuleVersion.startreleaseid, ModuleVersion.endreleaseid, release_id
         )
 
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2420,9 +2446,9 @@ class ViewDatapoints(Base):
             ModuleVersion.endreleaseid,
             release_id=None,
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2432,9 +2458,9 @@ class ViewDatapoints(Base):
         query = filter_by_release(
             query, ModuleVersion.startreleaseid, ModuleVersion.endreleaseid, release_id
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2460,9 +2486,9 @@ class ViewDatapoints(Base):
         ).distinct()
 
         query = query.filter(TableVersion.tablevid == table_version_id)
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2494,9 +2520,9 @@ class ViewDatapoints(Base):
         query = filter_by_release(
             query, ModuleVersion.startreleaseid, ModuleVersion.endreleaseid, release_id
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -2522,9 +2548,9 @@ class ViewDatapoints(Base):
         query = filter_by_release(
             query, ModuleVersion.startreleaseid, ModuleVersion.endreleaseid, release_id
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
 
@@ -2614,9 +2640,9 @@ class ViewKeyComponents(Base):
         query = query.distinct()
 
         # Execute and return as DataFrame
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2644,9 +2670,9 @@ class ViewKeyComponents(Base):
         # Add DISTINCT to eliminate duplicate rows from joins with composite PK tables
         query = query.distinct()
 
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2665,9 +2691,9 @@ class ViewKeyComponents(Base):
         # Add DISTINCT to eliminate duplicate rows from joins with composite PK tables
         query = query.distinct()
 
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
 
@@ -2737,9 +2763,9 @@ class ViewOpenKeys(Base):
         # ItemCategory has composite PK (itemid, startreleaseid), so join on itemid creates duplicates
         query = query.distinct()
 
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2758,9 +2784,9 @@ class ViewOpenKeys(Base):
         # Add DISTINCT to eliminate duplicate rows from joins with composite PK tables
         query = query.distinct()
 
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2788,9 +2814,9 @@ class ViewDataTypes(Base):
             )
             query = query.filter(cls.datapoint.in_(datapoints_batch))
             results.append(
-                pd.read_sql(
+                _read_sql_with_connection(
                     _compile_query_for_pandas(query.statement, session),
-                    session.connection().connection,
+                    session,
                 )
             )
             batch_start += batch_size
@@ -2826,9 +2852,9 @@ class ViewSubcategoryItemInfo(Base):
             query, cls.start_release_id, cls.end_release_id, release_id
         )
         query = query.order_by(cls.ordering)
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2857,9 +2883,9 @@ class ViewHierarchyVariables(Base):
         query = filter_by_release(
             query, cls.start_release_id, cls.end_release_id, release_id
         )
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return data
 
@@ -2888,9 +2914,9 @@ class ViewHierarchyVariablesContext(Base):
                 query, cls.start_release_id, cls.end_release_id, release_id
             )
             results.append(
-                pd.read_sql(
+                _read_sql_with_connection(
                     _compile_query_for_pandas(query.statement, session),
-                    session.connection().connection,
+                    session,
                 )
             )
         data = pd.concat(results, axis=0)
@@ -2909,9 +2935,9 @@ class ViewHierarchyPreconditions(Base):
     @classmethod
     def get_preconditions(cls, session):
         query = session.query(cls)
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
 
@@ -2928,9 +2954,9 @@ class ViewOperations(Base):
     @classmethod
     def get_operations(cls, session):
         query = session.query(cls).distinct()
-        operations = pd.read_sql(
+        operations = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         return operations.to_dict(orient="records")
 
@@ -2963,9 +2989,9 @@ class ViewOperations(Base):
         query = session.query(cls)
 
         query = query.filter(cls.operation_version_id.in_(preconditions_ids)).distinct()
-        preconditions = pd.read_sql(
+        preconditions = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
         return preconditions.to_dict(orient="records")
@@ -2982,9 +3008,9 @@ class ViewModules(Base):
     @classmethod
     def get_all_modules(cls, session):
         query = session.query(cls.module_code, cls.table_code).distinct()
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
     @classmethod
@@ -3025,9 +3051,9 @@ class ViewOperationFromModule(Base):
         )
         query = query.filter(cls.module_code == module_code)
         query = filter_by_date(query, cls.from_date, cls.to_date, ref_date)
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3038,9 +3064,9 @@ class ViewOperationFromModule(Base):
         query = query.filter(
             cls.operation_version_id == operation_version_id
         ).distinct()
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3059,9 +3085,9 @@ class ViewOperationFromModule(Base):
             cls.severity,
         )
         query = query.filter(cls.module_version_id == module_version_id).distinct()
-        reference = pd.read_sql(
+        reference = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         not_errors = []
         preconditions_to_remove = []
@@ -3069,9 +3095,7 @@ class ViewOperationFromModule(Base):
             not_errors = session.query(
                 OperationNode.nodeid.label("operation_version_id")
             ).distinct()
-            not_errors = pd.read_sql_query(
-                not_errors.statement, session.connection().connection
-            )
+            not_errors = pd.read_sql_query(not_errors.statement, session.get_bind())
             not_errors = list(not_errors["operation_version_id"])
             reference = reference[reference["operation_version_id"].isin(not_errors)]
         if not with_preconditions:
@@ -3079,7 +3103,7 @@ class ViewOperationFromModule(Base):
                 ViewPreconditionInfo.operation_version_id
             ).distinct()
             preconditions = pd.read_sql_query(
-                preconditions.statement, session.connection().connection
+                preconditions.statement, session.get_bind()
             )
             preconditions_to_remove = list(preconditions["operation_version_id"])
             reference = reference[
@@ -3117,9 +3141,9 @@ class ViewOperationInfo(Base):
         query = session.query(cls).filter(
             cls.operation_version_id == operation_version_id
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3145,9 +3169,9 @@ class ViewOperationInfo(Base):
         query = session.query(cls).filter(
             cls.operation_version_id.in_(operation_version_ids)
         )
-        df = pd.read_sql(
+        df = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         df = df.rename(columns=rename_dict)
         return df
@@ -3171,9 +3195,9 @@ class ViewTableInfo(Base):
             .filter(cls.module_code == module_code)
             .distinct()
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3183,9 +3207,9 @@ class ViewTableInfo(Base):
             .filter(cls.module_version_id == module_version_id)
             .distinct()
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3193,9 +3217,9 @@ class ViewTableInfo(Base):
         query = session.query(cls.variable_id, cls.variable_version_id).filter(
             cls.table_code == table_code
         )
-        data = pd.read_sql(
+        data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         if to_dict:
             return data.to_dict(orient="records")
@@ -3206,17 +3230,17 @@ class ViewTableInfo(Base):
         query = session.query(cls.variable_id, cls.variable_version_id).filter(
             cls.table_version_id == table_version_id
         )
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
     def get_intra_module_variables(cls, session):
         query = session.query(cls.variable_version_id, cls.module_code).distinct()
-        module_data = pd.read_sql(
+        module_data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         intra_module_data = module_data.drop_duplicates(
             subset=["variable_version_id"], keep=False, ignore_index=True
@@ -3235,9 +3259,9 @@ class ViewTableInfo(Base):
             .distinct()
             .filter(cls.table_code.in_(table_codes))
         )
-        module_data = pd.read_sql(
+        module_data = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
 
         all_combinations = (
@@ -3277,9 +3301,9 @@ class ViewPreconditionInfo(Base):
         query = session.query(
             cls.operation_version_id, cls.operation_code, cls.variable_code
         ).distinct()
-        return pd.read_sql(
+        return _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
 
     @classmethod
@@ -3307,9 +3331,9 @@ class ViewHierarchyOperandReferenceInfo(Base):
             .filter(cls.cell_id == cell_id)
             .distinct()
         )
-        operations = pd.read_sql(
+        operations = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
         return operations
 
@@ -3318,9 +3342,9 @@ class ViewHierarchyOperandReferenceInfo(Base):
         query = session.query(cls).filter(cls.variable_id.in_(var_id_list))
         possible_op_codes = []
 
-        df = pd.read_sql(
+        df = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         )
         grouped_code = df.groupby("operation_code")
         for elto_k, elto_v in grouped_code.groups.items():
@@ -3348,8 +3372,8 @@ class ViewReportTypeOperandReferenceInfo(Base):
             .filter(cls.cell_id == cell_id)
             .distinct()
         )
-        operations = pd.read_sql(
+        operations = _read_sql_with_connection(
             _compile_query_for_pandas(query.statement, session),
-            session.connection().connection,
+            session,
         ).to_dict(orient="records")
         return operations

py_dpm/dpm/queries/base.py

@@ -0,0 +1,100 @@
+import warnings
+import pandas as pd
+from typing import List, Dict, Any, Optional, Union
+from sqlalchemy.orm import Query, Session
+from sqlalchemy.sql import Select
+
+
+class BaseQuery:
+    """
+    Base wrapper for SQLAlchemy queries to provide unified output formatting.
+    """
+
+    def __init__(self, session: Session, query: Union[Query, Select]):
+        self.session = session
+        self._query = query
+
+    def filter(self, *criteria):
+        """Apply SQLAlchemy filters."""
+        # Check if it's a legacy ORM Query or composite Select
+        if hasattr(self._query, "filter"):
+            self._query = self._query.filter(*criteria)
+        else:
+            # For 1.4/2.0 style select() objects, use where if filter not avail, but distinct() usually returns Select
+            # SQLAlchemy Select objects usually have .where() or .filter() (which is an alias in 1.4+)
+            self._query = self._query.filter(*criteria)
+        return self
+
+    def apply(self, func, *args, **kwargs):
+        """
+        Apply a function that modifies the query.
+        The function must accept the BaseQuery (or its internal query) as first argument
+        and return a modified query object.
+        """
+        # We pass the internal query to the function, and expect a query back
+        # This allows filters to work on the raw SQLAlchemy object
+        self._query = func(self._query, *args, **kwargs)
+        return self
+
+    @property
+    def statement(self):
+        """Return the underlying SQL statement."""
+        if hasattr(self._query, "statement"):
+            return self._query.statement
+        return self._query
+
+    def _compile_for_pandas(self):
+        """Compile query for safe pandas execution."""
+        stmt = self.statement
+        # Compile with literal binds for pandas compatibility
+        return str(
+            stmt.compile(
+                dialect=self.session.get_bind().dialect,
+                compile_kwargs={"literal_binds": True},
+            )
+        )
+
+    def to_df(self) -> pd.DataFrame:
+        """Execute query and return as Pandas DataFrame."""
+        sql = self._compile_for_pandas()
+
+        # Suppress pandas/SQLAlchemy connection warnings
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message=".*pandas only supports SQLAlchemy.*",
+                category=UserWarning,
+            )
+            return pd.read_sql(sql, self.session.connection().connection)
+
+    def to_dict(self) -> List[Dict[str, Any]]:
+        """
+        Execute query and return as list of dictionaries.
+
+        Handles both ORM objects (using to_dict if available) and KeyedTuples.
+        """
+        # Execute the query
+        if isinstance(self._query, Query):
+            results = self._query.all()
+        else:
+            results = self.session.execute(self._query).all()
+
+        if not results:
+            return []
+
+        # If results are ORM objects with to_dict method
+        first = results[0]
+        if hasattr(first, "to_dict"):
+            return [r.to_dict() for r in results]
+
+        # If results are SQLAlchemy Rows/KeyedTuples
+        if hasattr(first, "_mapping"):
+            return [dict(r._mapping) for r in results]
+
+        # Fallback for older SQLAlchemy or simple tuples (try to map from query column descriptions if possible, but Row/KeyedTuple is standard)
+        # In modern SA, .all() returns Rows which behave like tuples but have _mapping
+        try:
+            return [dict(row) for row in results]
+        except (ValueError, TypeError):
+            # Scalar results?
+            return results

py_dpm/dpm/queries/basic_objects.py

@@ -0,0 +1,33 @@
+from sqlalchemy.orm import Session
+from py_dpm.dpm.models import Release
+from py_dpm.dpm.queries.base import BaseQuery
+
+
+class ReleaseQuery:
+    """
+    Queries related to releases.
+    """
+
+    @staticmethod
+    def get_all_releases(session: Session) -> BaseQuery:
+        """
+        Fetch list of available releases.
+        """
+        q = session.query(Release).order_by(Release.date.desc())
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_release_by_id(session: Session, release_id: int) -> BaseQuery:
+        """
+        Fetch release by id.
+        """
+        q = session.query(Release).filter(Release.releaseid == release_id)
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_release_by_code(session: Session, release_code: str) -> BaseQuery:
+        """
+        Fetch release by code.
+        """
+        q = session.query(Release).filter(Release.code == release_code)
+        return BaseQuery(session, q)