dbgraph 0.2.9__tar.gz → 0.2.11__tar.gz

{dbgraph-0.2.9 → dbgraph-0.2.11}/.idea/workspace.xml

@@ -4,13 +4,10 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="263a96da-9a82-40fd-89b2-ba71dd2c194d" name="Changes" comment="update sql builder and tests">
+    <list default="true" id="263a96da-9a82-40fd-89b2-ba71dd2c194d" name="Changes" comment="update for sqlite">
+      <change afterPath="$PROJECT_DIR$/tests/integrates/test_idma_builder.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/dbgraph.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/dbgraph.iml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/dbgraph/__init__.py" beforeDir="false" afterPath="$PROJECT_DIR$/dbgraph/__init__.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/dbgraph/entity/aspect.py" beforeDir="false" afterPath="$PROJECT_DIR$/dbgraph/entity/aspect.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/dbgraph/persistent/orm_graph_persistent.py" beforeDir="false" afterPath="$PROJECT_DIR$/dbgraph/persistent/orm_graph_persistent.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pyproject.toml" beforeDir="false" afterPath="$PROJECT_DIR$/pyproject.toml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/tests/units/test_orm_graph_persistent.py" beforeDir="false" afterPath="$PROJECT_DIR$/tests/units/test_orm_graph_persistent.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/dbgraph/builder/sql_graph_builder.py" beforeDir="false" afterPath="$PROJECT_DIR$/dbgraph/builder/sql_graph_builder.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/uv.lock" beforeDir="false" afterPath="$PROJECT_DIR$/uv.lock" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
@@ -75,6 +72,7 @@
   &quot;keyToString&quot;: {
     &quot;ASKED_SHARE_PROJECT_CONFIGURATION_FILES&quot;: &quot;true&quot;,
     &quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
+    &quot;Python tests.Python tests for tests.integrates.test_idma_builder.TestIDMABuilder.test_builder.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests for tests.integrates.test_json_graph_loader.TestJSONGraphLoader.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests for tests.integrates.test_sql_graph_builder.TestSQLGraphBuilder.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests for tests.integrates.test_sql_graph_builder.TestSQLGraphBuilder.test_build_graph.executor&quot;: &quot;Run&quot;,
@@ -113,6 +111,8 @@
     &quot;Python tests.Python tests for tests.test_sql_graph_persistent.TestSQLGraphPersistent.test_link.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests for tests.test_sql_graph_persistent.TestSQLGraphPersistent.test_load_graph.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests for tests.test_sql_graph_persistent.TestSQLGraphPersistent.test_save_graph.executor&quot;: &quot;Run&quot;,
+    &quot;Python tests.Python tests for tests.units.test_dbgraph.TestDBGraph.executor&quot;: &quot;Run&quot;,
+    &quot;Python tests.Python tests for tests.units.test_orm_graph_persistent.TestORMGraphPersistent.test_save_load_graph.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests in test_load_graph_sql.py.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests in test_trino_builder.py.executor&quot;: &quot;Run&quot;,
     &quot;Python tests.Python tests in tests.executor&quot;: &quot;Run&quot;,
@@ -146,8 +146,8 @@
       <recent name="C:\Users\minhden\Projects\dbgraph\tests\units" />
     </key>
   </component>
-  <component name="RunManager" selected="Python tests.Python tests in units">
-    <configuration name="Python tests for tests.integrates.test_sql_graph_builder.TestSQLGraphBuilder.test_build_graph" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
+  <component name="RunManager" selected="Python tests.Python tests for tests.integrates.test_idma_builder.TestIDMABuilder.test_builder">
+    <configuration name="Python tests for tests.integrates.test_idma_builder.TestIDMABuilder.test_builder" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
       <module name="dbgraph" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -160,7 +160,7 @@
       <option name="DEBUG_JUST_MY_CODE" value="false" />
       <option name="RUN_TOOL" value="" />
       <option name="_new_additionalArguments" value="&quot;&quot;" />
-      <option name="_new_target" value="&quot;tests.integrates.test_sql_graph_builder.TestSQLGraphBuilder.test_build_graph&quot;" />
+      <option name="_new_target" value="&quot;tests.integrates.test_idma_builder.TestIDMABuilder.test_builder&quot;" />
       <option name="_new_targetType" value="&quot;PYTHON&quot;" />
       <method v="2" />
     </configuration>
@@ -181,7 +181,7 @@
       <option name="_new_targetType" value="&quot;PYTHON&quot;" />
       <method v="2" />
     </configuration>
-    <configuration name="Python tests for tests.integrates.test_trino_builder.TestTrinoBuilder.test__get_table_row_count" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
+    <configuration name="Python tests for tests.units.test_dbgraph.TestDBGraph" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
       <module name="dbgraph" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -194,11 +194,11 @@
       <option name="DEBUG_JUST_MY_CODE" value="false" />
       <option name="RUN_TOOL" value="" />
       <option name="_new_additionalArguments" value="&quot;&quot;" />
-      <option name="_new_target" value="&quot;tests.integrates.test_trino_builder.TestTrinoBuilder.test__get_table_row_count&quot;" />
+      <option name="_new_target" value="&quot;tests.units.test_dbgraph.TestDBGraph&quot;" />
       <option name="_new_targetType" value="&quot;PYTHON&quot;" />
       <method v="2" />
     </configuration>
-    <configuration name="Python tests for tests.integrates.test_trino_builder.TestTrinoBuilder.test_build_graph" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
+    <configuration name="Python tests for tests.units.test_orm_graph_persistent.TestORMGraphPersistent.test_save_load_graph" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
       <module name="dbgraph" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -209,9 +209,9 @@
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <option name="DEBUG_JUST_MY_CODE" value="false" />
-      <option name="RUN_TOOL" value="true" />
+      <option name="RUN_TOOL" value="" />
       <option name="_new_additionalArguments" value="&quot;&quot;" />
-      <option name="_new_target" value="&quot;tests.integrates.test_trino_builder.TestTrinoBuilder.test_build_graph&quot;" />
+      <option name="_new_target" value="&quot;tests.units.test_orm_graph_persistent.TestORMGraphPersistent.test_save_load_graph&quot;" />
       <option name="_new_targetType" value="&quot;PYTHON&quot;" />
       <method v="2" />
     </configuration>
@@ -234,11 +234,11 @@
     </configuration>
     <recent_temporary>
       <list>
+        <item itemvalue="Python tests.Python tests for tests.integrates.test_idma_builder.TestIDMABuilder.test_builder" />
+        <item itemvalue="Python tests.Python tests for tests.units.test_dbgraph.TestDBGraph" />
+        <item itemvalue="Python tests.Python tests for tests.units.test_orm_graph_persistent.TestORMGraphPersistent.test_save_load_graph" />
         <item itemvalue="Python tests.Python tests in units" />
         <item itemvalue="Python tests.Python tests for tests.integrates.test_trino_builder.TestTrinoBuilder.test__get_table_orm" />
-        <item itemvalue="Python tests.Python tests for tests.integrates.test_sql_graph_builder.TestSQLGraphBuilder.test_build_graph" />
-        <item itemvalue="Python tests.Python tests for tests.integrates.test_trino_builder.TestTrinoBuilder.test__get_table_row_count" />
-        <item itemvalue="Python tests.Python tests for tests.integrates.test_trino_builder.TestTrinoBuilder.test_build_graph" />
       </list>
     </recent_temporary>
   </component>
@@ -345,7 +345,23 @@
       <option name="project" value="LOCAL" />
       <updated>1790214726709</updated>
     </task>
-    <option name="localTasksCounter" value="12" />
+    <task id="LOCAL-00012" summary="update sql builder and tests">
+      <option name="closed" value="true" />
+      <created>1790219300115</created>
+      <option name="number" value="00012" />
+      <option name="presentableId" value="LOCAL-00012" />
+      <option name="project" value="LOCAL" />
+      <updated>1790219300115</updated>
+    </task>
+    <task id="LOCAL-00013" summary="update for sqlite">
+      <option name="closed" value="true" />
+      <created>1790220216196</created>
+      <option name="number" value="00013" />
+      <option name="presentableId" value="LOCAL-00013" />
+      <option name="project" value="LOCAL" />
+      <updated>1790220216196</updated>
+    </task>
+    <option name="localTasksCounter" value="14" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -369,6 +385,7 @@
     <MESSAGE value="remove events" />
     <MESSAGE value="update renderer" />
     <MESSAGE value="update sql builder and tests" />
-    <option name="LAST_COMMIT_MESSAGE" value="update sql builder and tests" />
+    <MESSAGE value="update for sqlite" />
+    <option name="LAST_COMMIT_MESSAGE" value="update for sqlite" />
   </component>
 </project>

{dbgraph-0.2.9 → dbgraph-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dbgraph
-Version: 0.2.9
+Version: 0.2.11
 Summary: dbgraph is a database profiling tool which model the database as a graph
 Requires-Python: >=3.11
 Requires-Dist: bm25s==0.3.11

{dbgraph-0.2.9 → dbgraph-0.2.11}/dbgraph/builder/sql_graph_builder.py

@@ -1,9 +1,6 @@
 import logging
-from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
-from unittest import result
 from uuid import uuid4
-from tqdm import tqdm
 
 from sqlalchemy import (
     Column,
@@ -30,7 +27,7 @@ from sqlalchemy import (
     DATE,
     DATETIME,
     distinct,
-    TIME, TIMESTAMP,
+    TIME, TIMESTAMP, VARBINARY,
 )
 from sqlalchemy.orm import Session
 
@@ -70,6 +67,7 @@ class SQLGraphBuilder(GraphBuilder):
     detector_sample_size: int = 200
     textual_length_threshold: int = 30
     textual_distinct_ratio_threshold: float = 0.8
+    run_full_statistics: bool = True
 
     def __post_init__(self):
         logger.info("Creating SQLAlchemy engine...")
@@ -202,23 +200,26 @@ class SQLGraphBuilder(GraphBuilder):
         if isinstance(
                 col.type, (Numeric, Integer, SmallInteger, BigInteger, Float, DECIMAL),
         ):
-            num_aspect = self._get_numerical_stats_aspect(col)
+            if self.run_full_statistics:
+                num_aspect = self._get_numerical_stats_aspect(col)
         elif isinstance(col.type, (String, Text, Unicode, UnicodeText, CHAR, NCHAR)):
             is_textual = self._is_column_textual(col)
             if is_textual:
                 logger.debug("`%s` is textual type", col)
                 cat_aspect = None
             else:
-                cat_aspect = self._get_cat_stats_aspect(col)
+                if self.run_full_statistics:
+                    cat_aspect = self._get_cat_stats_aspect(col)
         elif isinstance(col.type, (DATE, DATETIME, TIME, TIMESTAMP)):
-            temp_aspect = self._get_temp_aspect(col)
-        elif isinstance(col.type, (BLOB,)):
+            if self.run_full_statistics:
+                temp_aspect = self._get_temp_aspect(col)
+        elif isinstance(col.type, (BLOB, VARBINARY)):
             pass
         else:
-            raise NotImplementedError(
-                f"Only support Numeric, String, Date, and BLOB. Got {col.type} for "
-                f"{col}",
-            )
+            logger.warning(f"Only support Numeric, String, Date, and BLOB. Got {col.type} for "
+                f"{col}")
+            pass
+
         return RColumnStatisticsAspect(
             name=f"{col.name}_column_stats",
             numerical_stats=num_aspect,
@@ -231,21 +232,24 @@ class SQLGraphBuilder(GraphBuilder):
 
     def _make_column_asset(self, col: Column, pks: PrimaryKeyConstraint) -> Asset:
         logger.info("`%s` - building column asset...", col)
+        aspects = {}
         schema_aspect = RColumnSchemaAspect(
             name=f"{col.name}_column_schema",
             dtype=str(col.type),
             is_nullable=col.nullable or False,
             is_pk=col.name in [c.name for c in pks.columns],
         )
-        stats_aspect = self._get_stats_aspect(col)
+        aspects["schema_properties"] = schema_aspect
+
+        if self.run_full_statistics:
+            stats_aspect = self._get_stats_aspect(col)
+            aspects["statistical_properties"] = stats_aspect
+
         return Asset(
             asset_id=uuid4(),
             name=col.name,
             type=AssetType.RCOLUMN,
-            aspects={
-                "schema_properties"     : schema_aspect,
-                "statistical_properties": stats_aspect,
-            },
+            aspects=aspects,
         )
 
     def _get_columns_assets(self, table_name: str) -> list[Asset]:
@@ -255,25 +259,6 @@ class SQLGraphBuilder(GraphBuilder):
         pks = table.primary_key
         for column in table.columns:
             assets.append(self._make_column_asset(column, pks))
-        # with ThreadPoolExecutor(max_workers=self.max_worker) as executor:
-        #     if self.use_tqdm:
-        #         assets = list(
-        #             tqdm(
-        #                 executor.map(
-        #                     self._make_column_asset, table.columns,
-        #                     [pks] * len(table.columns),
-        #                 ),
-        #                 total=len(table.columns),
-        #                 desc="Processing columns",
-        #             ),
-        #         )
-        #     else:
-        #         assets = list(
-        #             executor.map(
-        #                 self._make_column_asset, table.columns,
-        #                 [pks] * len(table.columns),
-        #             ),
-        #         )
         return assets
 
     def _get_table_stats_aspect(self, table_name: str) -> RTableStatisticsAspect:
@@ -314,16 +299,17 @@ class SQLGraphBuilder(GraphBuilder):
 
     def _make_table_asset(self, table_name: str) -> Asset:
         logger.info("`%s` - creating table asset...", table_name)
+        aspects = {}
+
         stats_aspect = self._get_table_stats_aspect(table_name)
         schema_aspect = self._get_table_schema_aspect(table_name)
+        aspects["schema_properties"] = schema_aspect
+        aspects["statistical_properties"] = stats_aspect
         table_asset = Asset(
             asset_id=uuid4(),
             name=table_name,
             type=AssetType.RTABLE,
-            aspects={
-                "schema_properties"     : schema_aspect,
-                "statistical_properties": stats_aspect,
-            },
+            aspects=aspects,
         )
         self.columns_assets[table_asset.name] = self._get_columns_assets(table_name)
         self.tables_assets[table_asset.name] = table_asset

{dbgraph-0.2.9 → dbgraph-0.2.11}/dbgraph/persistent/orm_graph_persistent.py

@@ -2,7 +2,7 @@ import json
 from dataclasses import dataclass, asdict
 from uuid import UUID
 
-from sqlalchemy import create_engine, select, delete, and_
+from sqlalchemy import create_engine, select, delete, and_, event
 from sqlalchemy.orm import Session
 
 from dbgraph import Asset, Link, DatabaseGraph, LinkType, AssetType
@@ -35,6 +35,13 @@ class ORMGraphPersistent(GraphPersistent):
 
     def __post_init__(self):
         self.engine = create_engine(self.persistent_uri)
+
+        @event.listens_for(self.engine, "connect")
+        def set_sqlite_pragma(dbapi_connection, connection_record):
+            cursor = dbapi_connection.cursor()
+            cursor.execute("PRAGMA foreign_keys=ON")
+            cursor.close()
+
         Base.metadata.create_all(self.engine)
 
     def insert_graph(self, graph_id: UUID, graph_name: str):

{dbgraph-0.2.9 → dbgraph-0.2.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dbgraph"
-version = "0.2.9"
+version = "0.2.11"
 description = "dbgraph is a database profiling tool which model the database as a graph"
 readme = "README.md"
 requires-python = ">=3.11"

dbgraph-0.2.11/tests/integrates/test_idma_builder.py

@@ -0,0 +1,38 @@
+import logging
+import unittest
+from uuid import uuid4
+
+from dbgraph import SQLGraphBuilder, ORMGraphPersistent
+
+
+class TestIDMABuilder(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger("sqlalchemy").setLevel(
+            logging.WARNING,
+        )  # ví dụ framework SQLAlchemy
+        logging.getLogger("urllib3").setLevel(logging.WARNING)  # ví dụ requests/HTTP
+        logger = logging.getLogger("SQLGraphBuilder")
+        logger.propagate = False
+
+    def setUp(self) -> None:
+        self.uri = "trino://5e313dcb-78c7-4fe0-a399-83d6c55ed169:f4asulq4wluhis651wfi@67cc9102-3edf-44d7-98e8-59e824d365b4.idma.cloud:443/vnptiq_dev/genai_bronze"
+        self.builder = SQLGraphBuilder(
+            self.uri, run_full_statistics=False
+        )
+        persistent_uri = "sqlite:///tests/idma.db"
+        self.persistent = ORMGraphPersistent(persistent_uri)
+
+    def test_builder(self):
+        graph = self.builder.build_graph()
+        self.persistent.save_graph(
+            uuid4(),
+            "idma",
+            graph
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

{dbgraph-0.2.9 → dbgraph-0.2.11}/tests/units/test_orm_graph_persistent.py

@@ -365,6 +365,9 @@ class TestORMGraphPersistent(unittest.TestCase):
         self.persistent.save_graph(target_id, "test", self.graph)
         graph = self.persistent.load_graph(target_id)
         self.assertEqual(self.graph, graph)
+        self.persistent.delete_graph(target_id)
+        assets = self.persistent.get_assets(target_id)
+        self.assertEqual(0, len(assets))
 
 
 if __name__ == "__main__":