dbgraph 0.1.0__tar.gz

dbgraph-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env
+
+site/

dbgraph-0.1.0/.idea/.gitignore

@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/

dbgraph-0.1.0/.idea/dbgraph.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.system.id="pyproject.toml" type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/dbgraph" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="~\Projects\dbgraph\.venv" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+  </component>
+</module>

dbgraph-0.1.0/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

dbgraph-0.1.0/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="~\Projects\dbgraph\.venv" project-jdk-type="Python SDK" />
+</project>

dbgraph-0.1.0/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/dbgraph.iml" filepath="$PROJECT_DIR$/.idea/dbgraph.iml" />
+    </modules>
+  </component>
+</project>

dbgraph-0.1.0/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

dbgraph-0.1.0/.idea/workspace.xml

@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="263a96da-9a82-40fd-89b2-ba71dd2c194d" name="Changes" comment="update packaging" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_BRANCH_BY_REPOSITORY">
+      <map>
+        <entry key="$PROJECT_DIR$" value="packaging" />
+      </map>
+    </option>
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="GitHubPullRequestSearchHistory">{
+  &quot;lastFilter&quot;: {
+    &quot;state&quot;: &quot;OPEN&quot;,
+    &quot;assignee&quot;: &quot;minhdenthedev&quot;
+  }
+}</component>
+  <component name="GithubPullRequestsUISettings"><![CDATA[{
+  "selectedUrlAndAccountId": {
+    "url": "https://github.com/minhdenthedev/dbgraph.git",
+    "accountId": "6c0b7cdc-aa0d-4999-86e3-e6f0a1d74a10"
+  },
+  "recentNewPullRequestHead": {
+    "server": {
+      "useHttp": false,
+      "host": "github.com",
+      "port": null,
+      "suffix": null
+    },
+    "owner": "minhdenthedev",
+    "repository": "dbgraph"
+  }
+}]]></component>
+  <component name="ProjectColorInfo">{
+  &quot;associatedIndex&quot;: 0,
+  &quot;fromUser&quot;: false
+}</component>
+  <component name="ProjectId" id="3IdnkFbc3mGdbjiFJYWIoYOgRXM" />
+  <component name="ProjectLevelVcsManager">
+    <ConfirmationsSetting value="2" id="Add" />
+  </component>
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent">{
+  &quot;keyToString&quot;: {
+    &quot;ASKED_SHARE_PROJECT_CONFIGURATION_FILES&quot;: &quot;true&quot;,
+    &quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
+    &quot;Python tests.Python tests for tests.test_dbgraph.TestDBGraph.executor&quot;: &quot;Run&quot;,
+    &quot;Python tests.Python tests in tests.executor&quot;: &quot;Run&quot;,
+    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    &quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;,
+    &quot;RunOnceActivity.typescript.service.memoryLimit.init&quot;: &quot;true&quot;,
+    &quot;codeWithMe.voiceChat.enabledByDefault&quot;: &quot;false&quot;,
+    &quot;git-widget-placeholder&quot;: &quot;main&quot;,
+    &quot;ignore.virus.scanning.warn.message&quot;: &quot;true&quot;,
+    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;
+  }
+}</component>
+  <component name="RunManager" selected="Python tests.Python tests for tests.test_dbgraph.TestDBGraph">
+    <configuration name="Python tests for tests.test_dbgraph.TestDBGraph" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
+      <module name="dbgraph" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="DEBUG_JUST_MY_CODE" value="false" />
+      <option name="RUN_TOOL" value="" />
+      <option name="_new_additionalArguments" value="&quot;&quot;" />
+      <option name="_new_target" value="&quot;tests.test_dbgraph.TestDBGraph&quot;" />
+      <option name="_new_targetType" value="&quot;PYTHON&quot;" />
+      <method v="2" />
+    </configuration>
+    <configuration name="Python tests in tests" type="tests" factoryName="Autodetect" temporary="true" nameIsGenerated="true">
+      <module name="dbgraph" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/tests" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="DEBUG_JUST_MY_CODE" value="false" />
+      <option name="RUN_TOOL" value="" />
+      <option name="_new_additionalArguments" value="&quot;&quot;" />
+      <option name="_new_target" value="&quot;$PROJECT_DIR$/tests&quot;" />
+      <option name="_new_targetType" value="&quot;PATH&quot;" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python tests.Python tests in tests" />
+        <item itemvalue="Python tests.Python tests for tests.test_dbgraph.TestDBGraph" />
+        <item itemvalue="Python tests.Python tests for tests.test_dbgraph.TestDBGraph" />
+        <item itemvalue="Python tests.Python tests in tests" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SharedIndexes">
+    <attachedChunks>
+      <set>
+        <option value="bundled-python-sdk-821f704e66d9-5bd6d13a2a75-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-262.9437.214" />
+      </set>
+    </attachedChunks>
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="263a96da-9a82-40fd-89b2-ba71dd2c194d" name="Changes" comment="" />
+      <created>1788103453676</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1788103453676</updated>
+    </task>
+    <task id="LOCAL-00001" summary="update packaging">
+      <option name="closed" value="true" />
+      <created>1788163967330</created>
+      <option name="number" value="00001" />
+      <option name="presentableId" value="LOCAL-00001" />
+      <option name="project" value="LOCAL" />
+      <updated>1788163967330</updated>
+    </task>
+    <option name="localTasksCounter" value="2" />
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+  <component name="VcsManagerConfiguration">
+    <MESSAGE value="update packaging" />
+    <option name="LAST_COMMIT_MESSAGE" value="update packaging" />
+  </component>
+</project>

dbgraph-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

dbgraph-0.1.0/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.5
+Name: dbgraph
+Version: 0.1.0
+Summary: dbgraph is a database profiling tool which model the database as a graph
+Requires-Python: >=3.11
+Requires-Dist: bm25s[all]>=0.3.9
+Requires-Dist: build>=1.6.0
+Requires-Dist: grip>=4.6.2
+Requires-Dist: langchain-openai>=1.6.0
+Requires-Dist: langchain>=1.3.16
+Requires-Dist: matplotlib>=3.11.0
+Requires-Dist: mkdocs-gen-files>=0.6.1
+Requires-Dist: mkdocs-material>=9.7.7
+Requires-Dist: mkdocs>=1.6.1
+Requires-Dist: mkdocstrings[python]>=1.0.6
+Requires-Dist: openai>=2.45.0
+Requires-Dist: psutil>=7.2.2
+Requires-Dist: pydantic>=2.13.4
+Requires-Dist: pydot>=4.0.1
+Requires-Dist: pympler>=1.1
+Requires-Dist: pystemmer>=3.1.0
+Requires-Dist: pytest>=9.1.1
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: rustworkx>=0.18.0
+Requires-Dist: sqlalchemy>=2.0.52
+Requires-Dist: tqdm>=4.68.4
+Requires-Dist: trino>=0.339.0
+Description-Content-Type: text/markdown
+
+# DBGraph - Building schema graph with LLM Assistance
+
+**DBGraph** aims to help data scientist with exploring and finding relevant data assets in a huge and complex database.
+
+## Quick Start
+
+Here is a quick example of how to build and query a schema graph with DBGraph:
+
+```python
+# imports
+from dbgraph.builder.sqlite.sqlite_graph_builder import SQLiteGraphBuilder
+from dbgraph.io.json_graph_writer import JSONGraphWriter
+from dbgraph.io.json_graph_loader import JSONGraphLoader
+from dbgraph.search.bm25_search_engine import BM25SearchEngine
+
+# initiate GraphBuilder on an SQLite database
+graph_builder = SQLiteGraphBuilder(Path("data/northwind.db"))
+
+# build the schema graph (include profiling databases)
+graph = graph_builder.build_graph()
+
+# generate descriptions for assets
+graph_descriptor = GraphDescriptorV1(
+    llm=OAICompatibleLLM(
+        model=...,
+        base_url=...,
+        api_key=...,
+    ),
+    system_prompt=...,
+    formating_prompt=...,
+    target_prompt=...
+)
+graph = graph_descriptor.rfill_semantic_aspects(graph)
+
+# save the graph
+graph_writer = JSONGraphWriter(
+    json_path=Path("data/northwind-graph.json"), indent=2
+)
+graph_writer.write()
+
+# load the graph
+graph_loader = JSONGraphLoader(json_path=Path("data/northwind-graph.json"))
+graph = self.graph_loader.load()
+
+# index the graph using BM25
+search_engine = BM25SearchEngine(Path("data/northwind-index"))
+semantic_aspects = {
+    a.asset_id: cast(SemanticAspect, a.aspects["semantic_properties"])
+    for a in graph.assets
+}
+search_engine.index(semantic_aspects)
+
+# retrieve assets using BM25
+assets_ids = search_engine.search(
+    "Give me the total count of orders in each categories"
+)
+```
+
+For visualization purpose, this is the graph saved in JSON:
+
+```json
+{
+  "assets": [
+    {
+      "asset_id": "8ab5a624-0596-497e-a0ee-3996d95dbe63",
+      "name": "Categories",
+      "type": "table",
+      "aspects": {
+        "schema_properties": {
+          "name": "Categories_table_schema",
+          "pks": ["CategoryID"],
+          "indices": {}
+        },
+        "statistical_properties": {
+          "name": "Categories_table_stats",
+          "num_columns": 4,
+          "num_rows": 8
+        },
+        "semantic_properties": {
+          "name": "Categories_semantic",
+          "description": "Stores product category definitions and metadata, serving as a lookup table for classifying products in the inventory system.",
+          "keywords": [
+            "categories",
+            "product classification",
+            "category definitions",
+            "inventory groups",
+            "product types"
+          ]
+        }
+      }
+    },
+    ...
+  ],
+  "links": [
+    {
+      "link_id": "db6bea93-a02c-4426-a2db-449e4a7bba8f",
+      "name": "Categories_CategoryID",
+      "type": "contain",
+      "source_id": "8ab5a624-0596-497e-a0ee-3996d95dbe63",
+      "destination_id": "04c20046-2808-4021-bbf1-99876e0eea6e",
+      "aspects": {}
+    },
+    ...
+  ]
+}
+```
+
+## Usecases
+
+![Usecases of DBGraph](diagrams/usecase.png)
+There are 4 main groups of usecases where DBGraph is applicable:
+
+- **Manipulating database schema**: Build the schema graph, store it, and use it to traverse around the database, find JOIN path, get references tables, ...
+- **Profiling database**: Use the concept of `Aspect` to represent different types of properties attached to a single data asset.
+  Each data asset can have a statistics aspects, semantical aspects, ...
+- **Render graph**: Output schema graph to Markdown or text as context for LLM
+- **LLM Assistance**: Leverage LLM to generate data assets' descriptions and tags. Furthermore, LLM could also be used in the process of SQL generation.
+- **Search for data assets**: Search for wanted data assets based on their descriptions. The descriptions are indexed and retrieved with BM25 algorithms.
+
+## Architecture
+
+![Class diagram of DBGraph](diagrams/entity.png)
+
+To encourage open-ness and extension, DBGraph is designed in a way that is very easy to extend.
+
+1. **The core classes** (_entities_) define the shared business logic of database graphs (traversal, neighborhoods, ...) and core operations
+   within the application (building graphs, profiling databases, ...). The prefix _"R..."_ stands for _"Relational"_, as the class is dedicated
+   Relational databases only. The same stands for _"D.."_ (Document), _"V..."_ (Vector), _"G..."_ (Graph). However, at this point, the only supported
+   paradigm is Relational database.
+2. **The interfaces** (_extensions_) define a part of the system that should be **pluggable**. For example:
+
+   - `RProfiler` and `RGraphBuilder` should works on multiple types of RDBMS, not just SQLite. Hence the abstraction of `RDataGateway`.
+   - We want to support multiple LLM providers. Hence the abstraction of `LLM`.
+   - There are many ways to render a graph into Markdown or text. Hence the abstraction of `RGraphRenderer`.
+   - To store and load the graph's data from some storages, we have multiple options. Hence the abstraction of `GraphGateway`.
+   - To index and retrieve graph's data from some search engines, we also have multiple options. Hence the abstraction of `SearchEngine`

dbgraph-0.1.0/README.md

@@ -0,0 +1,136 @@
+# DBGraph - Building schema graph with LLM Assistance
+
+**DBGraph** aims to help data scientist with exploring and finding relevant data assets in a huge and complex database.
+
+## Quick Start
+
+Here is a quick example of how to build and query a schema graph with DBGraph:
+
+```python
+# imports
+from dbgraph.builder.sqlite.sqlite_graph_builder import SQLiteGraphBuilder
+from dbgraph.io.json_graph_writer import JSONGraphWriter
+from dbgraph.io.json_graph_loader import JSONGraphLoader
+from dbgraph.search.bm25_search_engine import BM25SearchEngine
+
+# initiate GraphBuilder on an SQLite database
+graph_builder = SQLiteGraphBuilder(Path("data/northwind.db"))
+
+# build the schema graph (include profiling databases)
+graph = graph_builder.build_graph()
+
+# generate descriptions for assets
+graph_descriptor = GraphDescriptorV1(
+    llm=OAICompatibleLLM(
+        model=...,
+        base_url=...,
+        api_key=...,
+    ),
+    system_prompt=...,
+    formating_prompt=...,
+    target_prompt=...
+)
+graph = graph_descriptor.rfill_semantic_aspects(graph)
+
+# save the graph
+graph_writer = JSONGraphWriter(
+    json_path=Path("data/northwind-graph.json"), indent=2
+)
+graph_writer.write()
+
+# load the graph
+graph_loader = JSONGraphLoader(json_path=Path("data/northwind-graph.json"))
+graph = self.graph_loader.load()
+
+# index the graph using BM25
+search_engine = BM25SearchEngine(Path("data/northwind-index"))
+semantic_aspects = {
+    a.asset_id: cast(SemanticAspect, a.aspects["semantic_properties"])
+    for a in graph.assets
+}
+search_engine.index(semantic_aspects)
+
+# retrieve assets using BM25
+assets_ids = search_engine.search(
+    "Give me the total count of orders in each categories"
+)
+```
+
+For visualization purpose, this is the graph saved in JSON:
+
+```json
+{
+  "assets": [
+    {
+      "asset_id": "8ab5a624-0596-497e-a0ee-3996d95dbe63",
+      "name": "Categories",
+      "type": "table",
+      "aspects": {
+        "schema_properties": {
+          "name": "Categories_table_schema",
+          "pks": ["CategoryID"],
+          "indices": {}
+        },
+        "statistical_properties": {
+          "name": "Categories_table_stats",
+          "num_columns": 4,
+          "num_rows": 8
+        },
+        "semantic_properties": {
+          "name": "Categories_semantic",
+          "description": "Stores product category definitions and metadata, serving as a lookup table for classifying products in the inventory system.",
+          "keywords": [
+            "categories",
+            "product classification",
+            "category definitions",
+            "inventory groups",
+            "product types"
+          ]
+        }
+      }
+    },
+    ...
+  ],
+  "links": [
+    {
+      "link_id": "db6bea93-a02c-4426-a2db-449e4a7bba8f",
+      "name": "Categories_CategoryID",
+      "type": "contain",
+      "source_id": "8ab5a624-0596-497e-a0ee-3996d95dbe63",
+      "destination_id": "04c20046-2808-4021-bbf1-99876e0eea6e",
+      "aspects": {}
+    },
+    ...
+  ]
+}
+```
+
+## Usecases
+
+![Usecases of DBGraph](diagrams/usecase.png)
+There are 4 main groups of usecases where DBGraph is applicable:
+
+- **Manipulating database schema**: Build the schema graph, store it, and use it to traverse around the database, find JOIN path, get references tables, ...
+- **Profiling database**: Use the concept of `Aspect` to represent different types of properties attached to a single data asset.
+  Each data asset can have a statistics aspects, semantical aspects, ...
+- **Render graph**: Output schema graph to Markdown or text as context for LLM
+- **LLM Assistance**: Leverage LLM to generate data assets' descriptions and tags. Furthermore, LLM could also be used in the process of SQL generation.
+- **Search for data assets**: Search for wanted data assets based on their descriptions. The descriptions are indexed and retrieved with BM25 algorithms.
+
+## Architecture
+
+![Class diagram of DBGraph](diagrams/entity.png)
+
+To encourage open-ness and extension, DBGraph is designed in a way that is very easy to extend.
+
+1. **The core classes** (_entities_) define the shared business logic of database graphs (traversal, neighborhoods, ...) and core operations
+   within the application (building graphs, profiling databases, ...). The prefix _"R..."_ stands for _"Relational"_, as the class is dedicated
+   Relational databases only. The same stands for _"D.."_ (Document), _"V..."_ (Vector), _"G..."_ (Graph). However, at this point, the only supported
+   paradigm is Relational database.
+2. **The interfaces** (_extensions_) define a part of the system that should be **pluggable**. For example:
+
+   - `RProfiler` and `RGraphBuilder` should works on multiple types of RDBMS, not just SQLite. Hence the abstraction of `RDataGateway`.
+   - We want to support multiple LLM providers. Hence the abstraction of `LLM`.
+   - There are many ways to render a graph into Markdown or text. Hence the abstraction of `RGraphRenderer`.
+   - To store and load the graph's data from some storages, we have multiple options. Hence the abstraction of `GraphGateway`.
+   - To index and retrieve graph's data from some search engines, we also have multiple options. Hence the abstraction of `SearchEngine`