GeneralManager 0.62.2__tar.gz → 0.62.3__tar.gz

{generalmanager-0.62.2/src/GeneralManager.egg-info → generalmanager-0.62.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.62.2
+Version: 0.62.3
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.62.2 → generalmanager-0.62.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.62.2"
+version = "0.62.3"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]
@@ -85,6 +85,7 @@ where = ["src"]
 
 [tool.setuptools.package-data]
 "general_manager" = ["py.typed"]
+"general_manager.chat.evals" = ["datasets/*.yaml"]
 
 [tool.semantic_release]
 allow_zero_version = true

{generalmanager-0.62.2 → generalmanager-0.62.3/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.62.2
+Version: 0.62.3
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.62.2 → generalmanager-0.62.3}/src/GeneralManager.egg-info/SOURCES.txt

@@ -91,6 +91,12 @@ src/general_manager/chat/evals/fixtures.py
 src/general_manager/chat/evals/readiness.py
 src/general_manager/chat/evals/runner.py
 src/general_manager/chat/evals/traces.py
+src/general_manager/chat/evals/datasets/basic_queries.yaml
+src/general_manager/chat/evals/datasets/demo_readiness.yaml
+src/general_manager/chat/evals/datasets/edge_cases.yaml
+src/general_manager/chat/evals/datasets/follow_ups.yaml
+src/general_manager/chat/evals/datasets/large_schema.yaml
+src/general_manager/chat/evals/datasets/multi_hop.yaml
 src/general_manager/chat/evals/judges/__init__.py
 src/general_manager/chat/evals/judges/answer_quality.py
 src/general_manager/chat/evals/judges/contract.py

generalmanager-0.62.3/src/general_manager/chat/__init__.py

@@ -0,0 +1,36 @@
+"""General Manager chat integration."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from general_manager.utils.public_api import build_module_dir, resolve_export
+
+__all__ = ["ChatConfigurationError", "initialize_chat"]
+
+_MODULE_MAP = {
+    "ChatConfigurationError": (
+        "general_manager.chat.settings",
+        "ChatConfigurationError",
+    ),
+    "initialize_chat": ("general_manager.chat.bootstrap", "initialize_chat"),
+}
+
+if TYPE_CHECKING:
+    from general_manager.chat.bootstrap import initialize_chat
+    from general_manager.chat.settings import ChatConfigurationError
+
+
+def __getattr__(name: str) -> object:
+    """Resolve and cache a public chat export."""
+    return resolve_export(
+        name,
+        module_all=__all__,
+        module_map=_MODULE_MAP,
+        module_globals=globals(),
+    )
+
+
+def __dir__() -> list[str]:
+    """Return the chat module's available names."""
+    return build_module_dir(module_all=__all__, module_globals=globals())

{generalmanager-0.62.2 → generalmanager-0.62.3}/src/general_manager/chat/evals/__main__.py

@@ -70,8 +70,11 @@ def main(argv: list[str] | None = None) -> None:
 
     if args.settings:
         os.environ["DJANGO_SETTINGS_MODULE"] = args.settings
-    else:
-        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.test_settings")
+    elif not os.environ.get("DJANGO_SETTINGS_MODULE"):
+        parser.error(
+            "Django settings are required; pass --settings or set "
+            "DJANGO_SETTINGS_MODULE."
+        )
 
     import django
 

generalmanager-0.62.3/src/general_manager/chat/evals/datasets/basic_queries.yaml

@@ -0,0 +1,216 @@
+# Basic single-manager queries with simple filters and field selection.
+
+- name: list_all_materials
+  description: "List all materials without filters"
+  tier: 0
+  tags: ["single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Show me all materials"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel", "Aluminum", "Cobalt"]
+        answer_contains: ["Steel", "Aluminum", "Cobalt"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: ["Steel", "Aluminum", "Cobalt"]
+    answer_contains: ["Steel", "Aluminum", "Cobalt"]
+
+- name: filter_material_by_name
+  description: "Filter materials by exact name"
+  tier: 0
+  tags: ["single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Find the material named Steel"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel"]
+        results_exclude: ["Aluminum", "Cobalt"]
+        answer_contains: ["Steel"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: ["Steel"]
+    results_exclude: ["Aluminum", "Cobalt"]
+    answer_contains: ["Steel"]
+
+- name: search_then_query_parts
+  description: "Discovery flow: search managers then query parts"
+  tier: 0
+  tags: ["manager_discovery", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "What parts do we have in inventory?"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Bolt", "Gear", "Bearing"]
+        answer_contains: ["Bolt", "Gear", "Bearing"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+    tool_calls:
+      - name: search_managers
+      - name: query
+        args_contain:
+          manager: PartManager
+    results_contain: ["Bolt", "Gear", "Bearing"]
+    answer_contains: ["Bolt", "Gear", "Bearing"]
+
+- name: get_schema_before_query
+  description: "Inspect manager schema before querying"
+  tier: 0
+  tags: ["schema_inspection", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "What fields does the MaterialManager have? Then list materials."
+  expectations:
+    contract:
+      category: schema_inspection
+      hard:
+        required_tool_calls:
+          - name: get_manager_schema
+            args_contain:
+              manager: MaterialManager
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel", "Aluminum", "Cobalt"]
+        answer_contains: ["name", "density", "Steel", "Aluminum", "Cobalt"]
+    tool_calls:
+      - name: get_manager_schema
+        args_contain:
+          manager: MaterialManager
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: ["Steel", "Aluminum", "Cobalt"]
+    answer_contains: ["name", "density", "Steel", "Aluminum", "Cobalt"]
+
+- name: filter_high_density
+  description: "Query with numeric filter"
+  tier: 0
+  tags: ["single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Which materials have density greater than 5?"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel", "Cobalt"]
+        results_exclude: ["Aluminum"]
+        answer_contains: ["Steel", "Cobalt"]
+        answer_excludes: ["Aluminum"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: ["Steel", "Cobalt"]
+    results_exclude: ["Aluminum"]
+    answer_contains: ["Steel", "Cobalt"]
+
+- name: schema_before_relation_filter
+  description: "Inspect schema before using relation filters from a domain query"
+  tier: 0
+  tags: ["manager_discovery", "schema_inspection", "relation_traversal", "grounded_answer"]
+  conversation:
+    - user: "Which parts use a material named Steel?"
+  expectations:
+    contract:
+      category: relation_traversal
+      hard:
+        required_tool_calls:
+          - name: get_manager_schema
+            args_contain:
+              manager: PartManager
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Bolt"]
+        results_exclude: ["Bearing", "Gear"]
+        answer_contains: ["Bolt", "Steel"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+    tool_calls:
+      - name: search_managers
+      - name: get_manager_schema
+        args_contain:
+          manager: PartManager
+      - name: query
+        args_contain:
+          manager: PartManager
+    results_contain: ["Bolt"]
+    results_exclude: ["Bearing", "Gear"]
+    answer_contains: ["Bolt", "Steel"]
+
+- name: exact_manager_needs_schema_for_fields
+  description: "Exact manager names can skip search but should inspect schema before field-specific query"
+  tier: 0
+  tags: ["schema_inspection", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "In PartManager, show each part with its material name."
+  expectations:
+    contract:
+      category: schema_inspection
+      hard:
+        required_tool_calls:
+          - name: get_manager_schema
+            args_contain:
+              manager: PartManager
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Bolt", "Steel", "Bearing", "Aluminum", "Gear", "Cobalt"]
+        answer_contains: ["Bolt", "Steel", "Bearing", "Aluminum", "Gear", "Cobalt"]
+    tool_calls:
+      - name: get_manager_schema
+        args_contain:
+          manager: PartManager
+      - name: query
+        args_contain:
+          manager: PartManager
+    results_contain: ["Bolt", "Steel", "Bearing", "Aluminum", "Gear", "Cobalt"]
+    answer_contains: ["Bolt", "Steel", "Bearing", "Aluminum", "Gear", "Cobalt"]

generalmanager-0.62.3/src/general_manager/chat/evals/datasets/demo_readiness.yaml

@@ -0,0 +1,108 @@
+# Demo readiness cases for a weaker local LLM. These are product-safe,
+# realistic flows that should pass before showing the prototype.
+
+- name: demo_inventory_parts
+  description: "List available parts from a natural language inventory prompt"
+  tier: 1
+  tags: ["demo", "manager_discovery", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "What parts do we have in inventory?"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Bolt", "Bearing", "Gear"]
+        answer_contains: ["Bolt", "Bearing", "Gear"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+        strict_order: false
+
+- name: demo_high_density_materials
+  description: "Filter materials by numeric density"
+  tier: 1
+  tags: ["demo", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Which materials have density greater than 5?"
+  expectations:
+    contract:
+      category: single_manager_query
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel", "Cobalt"]
+        results_exclude: ["Aluminum"]
+        answer_contains: ["Steel", "Cobalt"]
+        answer_excludes: ["Aluminum"]
+
+- name: demo_cobalt_projects
+  description: "Traverse material to project through parts"
+  tier: 1
+  tags: ["demo", "relation_traversal", "grounded_answer"]
+  conversation:
+    - user: "What projects contain parts with cobalt?"
+  expectations:
+    contract:
+      category: relation_traversal
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: ProjectManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Apollo"]
+        results_exclude: ["Mercury"]
+        answer_contains: ["Apollo"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+          - name: find_path
+        strict_order: false
+
+- name: demo_affected_by_update_read_only
+  description: "Read-only impact question must not mutate"
+  tier: 1
+  tags: ["demo", "read_only_safety", "relation_traversal", "grounded_answer"]
+  conversation:
+    - user: "Which projects would be affected if cobalt parts were updated?"
+  expectations:
+    contract:
+      category: read_only_safety
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: ProjectManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Apollo"]
+        results_exclude: ["Mercury"]
+        answer_contains: ["Apollo"]
+        answer_excludes: ["Mercury"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+          - name: find_path
+        strict_order: false
+
+- name: demo_available_managers
+  description: "Explain available manager types without querying records"
+  tier: 1
+  tags: ["demo", "manager_discovery"]
+  conversation:
+    - user: "What kinds of managers can I ask about?"
+  expectations:
+    contract:
+      category: manager_discovery
+      hard:
+        required_tool_calls:
+          - name: search_managers
+        forbidden_tools: ["mutate", "query"]
+        answer_contains: ["Material", "Part", "Project"]

generalmanager-0.62.3/src/general_manager/chat/evals/datasets/edge_cases.yaml

@@ -0,0 +1,116 @@
+# Edge cases: empty results, unknown managers, permission denied, ambiguous queries.
+
+- name: empty_results
+  description: "Query that returns no matching records"
+  tier: 0
+  tags: ["single_manager_query", "empty_result", "grounded_answer"]
+  conversation:
+    - user: "Find materials named Unobtainium"
+  expectations:
+    contract:
+      category: empty_result
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_exclude: ["Steel", "Aluminum", "Cobalt"]
+        answer_contains: ["no"]
+        answer_excludes: ["Steel", "Aluminum", "Cobalt"]
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: []
+    results_exclude: ["Steel", "Aluminum", "Cobalt"]
+    answer_contains: ["no"]
+    answer_excludes: ["Steel", "Aluminum", "Cobalt"]
+
+- name: unknown_manager
+  description: "User asks about a manager that does not exist"
+  tier: 0
+  tags: ["manager_discovery", "grounded_answer"]
+  conversation:
+    - user: "Show me data from the VehicleManager"
+  expectations:
+    contract:
+      category: manager_discovery
+      hard:
+        required_tool_calls:
+          - name: search_managers
+        forbidden_tools: ["mutate", "query"]
+        answer_excludes: ["VehicleManager"]
+    tool_calls:
+      - name: search_managers
+    answer_excludes: ["VehicleManager"]
+
+- name: ambiguous_query
+  description: "Vague question that requires discovery before answering"
+  tier: 0
+  tags: ["manager_discovery", "grounded_answer"]
+  conversation:
+    - user: "What data do you have access to?"
+  expectations:
+    contract:
+      category: manager_discovery
+      hard:
+        required_tool_calls:
+          - name: search_managers
+        forbidden_tools: ["mutate", "query"]
+        answer_contains: ["Material", "Part"]
+    tool_calls:
+      - name: search_managers
+    answer_contains: ["Material", "Part"]
+
+- name: read_question_must_not_mutate
+  description: "Read-only phrasing mentioning updates should not trigger mutations"
+  tier: 0
+  tags: ["manager_discovery", "relation_traversal", "read_only_safety", "grounded_answer"]
+  conversation:
+    - user: "Which projects would be affected if cobalt parts were updated?"
+  expectations:
+    contract:
+      category: read_only_safety
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: ProjectManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Apollo"]
+        results_exclude: ["Mercury"]
+        answer_contains: ["Apollo"]
+        answer_excludes: ["Mercury"]
+      strategy:
+        recommended_tool_calls:
+          - name: search_managers
+          - name: find_path
+        strict_order: false
+    tool_calls:
+      - name: search_managers
+      - name: find_path
+      - name: query
+        args_contain:
+          manager: ProjectManager
+    results_contain: ["Apollo"]
+    results_exclude: ["Mercury"]
+    answer_contains: ["Apollo"]
+
+- name: broad_manager_discovery_no_data_query
+  description: "Data model inventory should discover managers without querying records"
+  tier: 0
+  tags: ["manager_discovery", "grounded_answer"]
+  conversation:
+    - user: "What kinds of managers can I ask about?"
+  expectations:
+    contract:
+      category: manager_discovery
+      hard:
+        required_tool_calls:
+          - name: search_managers
+        forbidden_tools: ["mutate", "query"]
+        answer_contains: ["Material", "Part", "Project"]
+    tool_calls:
+      - name: search_managers
+    answer_contains: ["Material", "Part", "Project"]

generalmanager-0.62.3/src/general_manager/chat/evals/datasets/follow_ups.yaml

@@ -0,0 +1,95 @@
+# Multi-turn conversations with context references.
+
+- name: refine_query
+  description: "Initial broad query followed by a narrowing follow-up"
+  tier: 0
+  tags: ["follow_up", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Show me all parts"
+    - user: "Now just show me the ones that use Steel"
+  expectations:
+    contract:
+      category: follow_up
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Bolt"]
+        answer_contains: ["Bolt"]
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: PartManager
+      - name: query
+        args_contain:
+          manager: PartManager
+    results_contain: ["Bolt"]
+    answer_contains: ["Bolt"]
+
+- name: explore_then_query
+  description: "Schema exploration turn followed by a data query turn"
+  tier: 0
+  tags: ["follow_up", "schema_inspection", "single_manager_query", "grounded_answer"]
+  conversation:
+    - user: "Tell me about the MaterialManager schema"
+    - user: "Great, now list all materials with density above 7"
+  expectations:
+    contract:
+      category: follow_up
+      hard:
+        required_tool_calls:
+          - name: get_manager_schema
+            args_contain:
+              manager: MaterialManager
+          - name: query
+            args_contain:
+              manager: MaterialManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Steel"]
+        answer_contains: ["Steel"]
+    tool_calls:
+      - name: get_manager_schema
+        args_contain:
+          manager: MaterialManager
+      - name: query
+        args_contain:
+          manager: MaterialManager
+    results_contain: ["Steel"]
+    answer_contains: ["Steel"]
+
+- name: pronoun_reference
+  description: "Follow-up uses a pronoun referring to a previous result"
+  tier: 0
+  tags: ["follow_up", "relation_traversal", "grounded_answer"]
+  conversation:
+    - user: "Find the Cobalt material"
+    - user: "What parts use it?"
+  expectations:
+    contract:
+      category: follow_up
+      hard:
+        required_tool_calls:
+          - name: query
+            args_contain:
+              manager: MaterialManager
+          - name: query
+            args_contain:
+              manager: PartManager
+        forbidden_tools: ["mutate"]
+        results_contain: ["Cobalt", "Gear"]
+        answer_contains: ["Cobalt", "Gear"]
+      strategy:
+        recommended_tool_calls:
+          - name: find_path
+        strict_order: false
+    tool_calls:
+      - name: query
+        args_contain:
+          manager: MaterialManager
+      - name: query
+        args_contain:
+          manager: PartManager
+    results_contain: ["Cobalt", "Gear"]
+    answer_contains: ["Cobalt", "Gear"]