davidkhala.ai 0.1.6__tar.gz → 0.1.7__tar.gz

{davidkhala_ai-0.1.6 → davidkhala_ai-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: davidkhala.ai
-Version: 0.1.6
+Version: 0.1.7
 Summary: misc AI modules
 Requires-Python: >=3.13
 Provides-Extra: ali
@@ -23,10 +23,9 @@ Requires-Dist: onnx; extra == 'hf'
 Requires-Dist: onnxruntime; extra == 'hf'
 Provides-Extra: langchain
 Requires-Dist: langchain; extra == 'langchain'
-Requires-Dist: langchain-openai; extra == 'langchain'
+Requires-Dist: langchain-openai; (python_version < '3.14') and extra == 'langchain'
 Requires-Dist: langgraph; extra == 'langchain'
 Provides-Extra: openrouter
-Requires-Dist: davidkhala-utils[http-request]; extra == 'openrouter'
 Requires-Dist: openrouter; extra == 'openrouter'
 Provides-Extra: ragflow
 Requires-Dist: ragflow-sdk; extra == 'ragflow'

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/api/app.py

@@ -0,0 +1,98 @@
+import json
+from typing import TypedDict
+
+import requests
+from davidkhala.utils.http_request.stream import Request as StreamRequest, as_sse
+from requests import Response, Session
+
+from davidkhala.ai.agent.dify.api import API
+
+
+class Feedbacks(API):
+    def paginate_feedbacks(self, page=1, size=20):
+        """
+        when 'rating'='like', content=None
+        when 'rating'='dislike', content can be filled by end user
+        NOTE: for security reason, api cannot access conversation context associated with the feedback. End user should copy the conversation to comment by themselves.
+        # waiting for https://github.com/langgenius/dify/issues/28067
+        """
+        response = requests.get(f"{self.base_url}/app/feedbacks", params={"page": page, "limit": size}, **self.options)
+        if not response.ok:
+            response.raise_for_status()
+        else:
+            return json.loads(response.text)
+
+    def list_feedbacks(self):
+        return self.paginate_feedbacks()['data']
+
+
+class Conversation(API):
+    """
+    Note: The Service API does not share conversations created by the WebApp. Conversations created through the API are isolated from those created in the WebApp interface.
+    It means you cannot get user conversation content from API, API call has only access to conversation created by API
+    """
+
+    def __init__(self, api_key: str, user: str):
+        super().__init__(api_key)  # base_url need to be configured afterward if not default
+        self.user = user  # user_id, from_end_user_id
+
+    def paginate_messages(self, conversation_id):
+        return self.request(f"{self.base_url}/messages", "GET", params={
+            'conversation_id': conversation_id,
+            'user': self.user,
+        })
+
+    def _chat_request_from(self, template: str, stream, **kwargs):
+        """
+        :param template:
+        :param stream: Note: "Agent Chat App does not support blocking mode"
+        :param kwargs:
+        :return:
+        """
+        return {
+            'url': f"{self.base_url}/chat-messages",
+            'method': "POST",
+            'json': {
+                'query': template,
+                'inputs': kwargs.pop('values', {}),  # to substitute query/template
+                'response_mode': 'streaming' if stream else 'blocking',
+                'conversation_id': kwargs.pop('conversation_id', None),
+                'user': self.user,
+                'files': kwargs.pop('files', [])
+            },
+            **kwargs
+        }
+
+    def async_chat(self, template: str, **kwargs) -> tuple[Response, Session]:
+        s = StreamRequest(self)
+        s.session = Session()
+        return s.request(**self._chat_request_from(template, True, **kwargs)), s.session
+
+    class ChatResult(TypedDict, total=False):
+        thought: list[str]
+        metadata: dict
+
+    @staticmethod
+    def reduce_chat_stream(response: Response) -> ChatResult:
+        r: Conversation.ChatResult = {
+            'thought': [],
+        }
+        for data in as_sse(response):
+            match data['event']:
+                case 'agent_thought':
+                    r['thought'].append(data['thought'])
+                case 'message_end':
+                    r['metadata'] = data['metadata']
+        return r
+
+    def agent_chat(self, template: str, **kwargs) -> ChatResult:
+        r, session = self.async_chat(template, **kwargs)
+        reduced = Conversation.reduce_chat_stream(r)
+        session.close()
+        return reduced
+
+    def bot_chat(self, template: str, **kwargs):
+        r = self.request(**self._chat_request_from(template, False, **kwargs))
+        assert r.pop('event') == 'message'
+        assert r.pop('mode') == 'chat'
+        return r

{davidkhala_ai-0.1.6 → davidkhala_ai-0.1.7}/davidkhala/ai/agent/dify/api/knowledge.py

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
 import requests
 
 from davidkhala.ai.agent.dify.api import API, Iterator
-
+from davidkhala.ai.agent.dify.common import Document as DocumentBase
 
 class DatasetDict(TypedDict):
     id: str
@@ -37,21 +37,14 @@ class DatasetDict(TypedDict):
     external_knowledge_info: dict
 
 
-class DocumentDict(TypedDict):
-    id: str
-    position: int
-    data_source_type: str
+class Document(DocumentBase):
     data_source_info: dict[str, str]
     data_source_detail_dict: dict[str, dict]
     dataset_process_rule_id: str
-    name: str
     created_from: str
     created_by: str
     created_at: int
     tokens: int
-    indexing_status: str
-    error: str
-    enabled: bool
     archived: bool
     display_status: str
     word_count: int
@@ -91,9 +84,8 @@ class Dataset(API):
 
         def upload(self, filename, *, path=None, url=None, document_id=None):
             """
-            don't work for html
-            work for markdown
-            TODO how to simulate console
+            don't work for .html
+            work for .md
             """
             files = {}
             if path:
@@ -124,10 +116,10 @@ class Dataset(API):
                 'limit': size
             })
 
-        def list_documents(self) -> Iterable[DocumentDict]:
+        def list_documents(self) -> Iterable[Document]:
             for document_batch in Iterator(self.paginate_documents, None):
                 for document in document_batch:
-                    yield document
+                    yield Document(**document)
 
         def has_document(self, name) -> bool:
             return any(name == item['name'] for row in self.list_documents() for item in row)
@@ -189,3 +181,10 @@ class Document(API):
     def delete(self):
         if self.exist():
             self.request(self.base_url, "DELETE")
+class Chunk(API):
+    def __init__(self, d: Document, segment_id: str):
+        super().__init__(d.api_key, f"{d.base_url}/segments/{segment_id}")
+    def get(self):
+        r=  self.request(self.base_url, "GET")
+        assert r['doc_form'] # optional value text_model
+        return r['data']

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/common.py

@@ -0,0 +1,36 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins.firecrawl import DataSourceInfo
+
+
+class IndexingStatus(str, Enum):
+    WAITING = "waiting"
+    PARSING = "parsing"
+    SPLITTING = 'splitting'
+    INDEXING = "indexing"
+    COMPLETED = "completed"
+    FAILED = "error"
+
+
+class Document(BaseModel):
+    id: str
+    position: int
+    data_source_type: str
+    data_source_info: dict[str, str]
+    name: str
+    indexing_status: IndexingStatus
+    error: str | None
+    enabled: bool
+
+
+class Dataset(BaseModel):
+    id: str
+    name: str
+    description: str
+
+
+class IndexingError(Exception):
+    """Raised when document indexing fails (indexing_status = 'error')"""
+    pass

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/console/__init__.py

@@ -0,0 +1,9 @@
+from davidkhala.utils.http_request import Request
+
+
+class API(Request):
+    def __init__(self, base_url='http://localhost'):
+        super().__init__()
+        self.base_url = f"{base_url}/console/api"
+        self.__enter__()
+

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/console/knowledge.py

@@ -0,0 +1,156 @@
+from time import sleep
+
+from davidkhala.utils.http_request.stream import as_sse, Request as StreamRequest
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.common import IndexingStatus, IndexingError, Dataset, Document
+from davidkhala.ai.agent.dify.ops.console import API
+from davidkhala.ai.agent.dify.ops.console.session import ConsoleUser
+from davidkhala.ai.agent.dify.ops.db.orm import Node
+
+
+class ConsoleKnowledge(API):
+    def __init__(self, context: ConsoleUser):
+        super().__init__()
+        self.base_url = context.base_url
+        self.session.cookies = context.session.cookies
+        self.options = context.options
+
+
+class Datasource(ConsoleKnowledge):
+    """step 1: Choose a Data Source"""
+
+    class FirecrawlOutput(BaseModel):
+        source_url: str
+        description: str
+        title: str
+        credential_id: str
+        content: str
+
+    def run_firecrawl(self, pipeline: str, node: Node,
+                      *,
+                      inputs: dict,
+                      credential_id: str
+                      ):
+
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/datasource/nodes/{node.id}/run"
+
+        stream_request = StreamRequest(self)
+        response = stream_request.request(url, 'POST', json={
+            'inputs': inputs,
+            'datasource_type': node.datasource_type,
+            'credential_id': credential_id,
+            "response_mode": "streaming"
+        })
+
+        for data in as_sse(response):
+            event = data['event']
+            if event == 'datasource_completed':
+                return data['data']
+            else:
+                assert event == 'datasource_processing'
+                print(data)
+        return None
+
+    def upload(self):
+        "http://localhost/console/api/files/upload?source=datasets"
+        # TODO
+        "form data"
+        {
+            "file": "body"
+        }
+        r = {
+            "id": "3898db5b-eb72-4f11-b507-628ad5d28887",
+            "name": "Professional Diploma Meister Power Electrical Engineering - Technological and Higher Education Institute of Hong Kong.html",
+            "size": 254362,
+            "extension": "html",
+            "mime_type": "text\/html",
+            "created_by": "dbd0b38b-5ef1-4123-8c3f-0c82eb1feacd",
+            "created_at": 1764943811,
+            "source_url": "\/files\/3898db5b-eb72-4f11-b507-628ad5d28887\/file-preview?timestamp=1764943811&nonce=43b0ff5a13372415be79de4cc7ef398c&sign=7OJ2wiVYc4tygl7yvM1sPn7s0WXDlhHxgX76bsGTD94%3D"
+        }
+
+
+class Operation(ConsoleKnowledge):
+    def website_sync(self, dataset: str, document: str, *, wait_until=True):
+        """
+        cannot be used towards a pipeline dataset. Otherwise, you will see error "no website import info found"
+        """
+        doc_url = f"{self.base_url}/datasets/{dataset}/documents/{document}"
+
+        r = self.request(f"{doc_url}/website-sync", "GET")
+        assert r == {"result": "success"}
+        if wait_until:
+            return self.wait_until(dataset, document)
+        return None
+
+    def retry(self, dataset: str, *documents: str, wait_until=True):
+        """
+        It cannot trigger rerun on success documents
+        """
+        url = f"{self.base_url}/datasets/{dataset}/retry"
+        self.request(url, "POST", json={
+            'document_ids': documents,
+        })
+        # response status code will be 204
+        if wait_until:
+            return [self.wait_until(dataset, document) for document in documents]
+        return None
+
+    def rerun(self, dataset: str, *documents: str):
+        for document in documents:
+            try:
+                self.website_sync(dataset, document)
+                assert False, "expect IndexingError"
+            except IndexingError:
+                pass
+        return self.retry(dataset, *documents)
+
+    def wait_until(self, dataset: str, document: str, *,
+                   expect_status=None,
+                   from_status=None,
+                   interval=1
+                   ):
+        if not expect_status:
+            expect_status = [IndexingStatus.FAILED, IndexingStatus.COMPLETED]
+        url = f"{self.base_url}/datasets/{dataset}/documents/{document}/indexing-status"
+        if from_status is None:
+            from_status = [IndexingStatus.WAITING, IndexingStatus.PARSING]
+        r = self.request(url, "GET")
+        status = r['indexing_status']
+        assert status in from_status, f"current status: {status}, expect: {from_status}"
+        while status not in expect_status:
+            sleep(interval)
+            r = self.request(url, "GET")
+            status = r['indexing_status']
+        if status == IndexingStatus.FAILED: raise IndexingError(r['error'])
+        return r
+
+
+class DatasetResult(Dataset):
+    chunk_structure: str
+
+class RunResult(BaseModel):
+    batch: str
+    dataset: DatasetResult
+    documents: list[Document]
+
+class Load(ConsoleKnowledge):
+    """
+    Processing Documents
+    """
+
+    def async_run(self, pipeline: str, node: Node, inputs: dict, datasource_info_list: list[dict])->RunResult:
+        """Ingest new document"""
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/run"
+        r = self.request(url, "POST", json={
+            'inputs': inputs,
+            'start_node_id': node.id,
+            'is_preview': False,
+            'response_mode': "blocking",
+            "datasource_info_list": datasource_info_list,
+            'datasource_type': node.datasource_type
+        })
+        return RunResult(**r)
+
+

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/console/session.py

@@ -0,0 +1,30 @@
+from davidkhala.ai.agent.dify.ops.console import API
+
+
+class ConsoleUser(API):
+    def login(self, email, password,
+              *,
+              remember_me=True,
+              language="en-US"
+              ):
+        url = f"{self.base_url}/login"
+
+        r = self.request(url, "POST", json={
+            'email': email,
+            'password': password,
+            'remember_me': remember_me,
+            'language': language,
+        })
+        assert r == {"result": "success"}
+        self.options['headers']['x-csrf-token'] = self.session.cookies.get("csrf_token")
+        return self.session.cookies
+
+    @property
+    def me(self) -> dict:
+        url = f"{self.base_url}/account/profile"
+        return self.request(url, "GET")
+
+    @property
+    def workspace(self) -> dict:
+        url = f"{self.base_url}/features"
+        return self.request(url, "GET")

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/db/__init__.py

@@ -0,0 +1,17 @@
+from typing import Any
+
+from davidkhala.data.base.pg import Postgres
+
+
+class DB(Postgres):
+
+    def __init__(self, connection_string: str):
+        super().__init__(connection_string)
+        self.connect()
+
+    def get_dict(self,
+                 template: str,
+                 values: dict[str, Any] | None = None,
+                 request_options: dict[str, Any] | None = None
+                 ) -> list[dict]:
+        return Postgres.rows_to_dicts(self.query(template, values, request_options))

davidkhala_ai-0.1.6/davidkhala/ai/agent/dify/ops/db/__init__.py → davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/db/app.py

@@ -1,22 +1,19 @@
-from typing import Any, Optional
-
-from davidkhala.data.base.pg import Postgres
-from sqlalchemy import desc
-from sqlalchemy.orm import Session
-
+from davidkhala.ai.agent.dify.ops.db import DB
 from davidkhala.ai.agent.dify.ops.db.orm import AppModelConfig
+from sqlalchemy.orm import Session
+from sqlalchemy import desc
 
-
-class DB(Postgres):
-
-    def __init__(self, connection_string: str):
-        super().__init__(connection_string)
-        self.connect()
-
-    def get_dict(self, sql): return self.query(sql).mappings().all()
-
+class Studio(DB):
     @property
-    def accounts(self): return self.get_dict("select name, email from accounts where status = 'active'")
+    def user_feedbacks(self):
+        sql = """SELECT mf.conversation_id,
+                        mf.content,
+                        m.query,
+                        m.answer
+                 FROM message_feedbacks mf
+                          LEFT JOIN messages m ON mf.message_id = m.id
+                 WHERE mf.from_source = 'user'"""
+        return self.get_dict(sql)
 
     @property
     def apps(self): return self.get_dict("select id, name, mode from apps where status = 'normal'")
@@ -30,11 +27,11 @@ class DB(Postgres):
                 .first()
             )
 
-    def update_app_config(self, record: AppModelConfig, refresh:bool=False) -> AppModelConfig | None:
+    def update_app_config(self, record: AppModelConfig, refresh: bool = False) -> AppModelConfig | None:
         with Session(self.client) as session:
             session.add(record)
             session.commit()
             if refresh:
-                session.refresh(record)  # 刷新对象，确保拿到数据库生成的字段（如 id）
+                session.refresh(record)
                 return record
-            return None
+            return None

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/db/knowledge.py

@@ -0,0 +1,52 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+from davidkhala.ai.agent.dify.ops.db.orm import Graph
+
+
+class Dataset(DB):
+
+    def dataset_queries(self, dataset_id, limit=20) -> list[str]:
+        template = "select content from dataset_queries where source = 'app' and created_by_role = 'end_user' and dataset_id = :dataset_id limit :limit"
+        return self.query(template, {'dataset_id': dataset_id, 'limit': limit}).scalars().all()
+
+    @property
+    def datasets(self):
+        template = "select id, name, description, indexing_technique, index_struct, embedding_model, embedding_model_provider, collection_binding_id, retrieval_model, icon_info, runtime_mode, pipeline_id, chunk_structure from datasets"
+        return self.get_dict(template)
+
+    def is_pipeline(self, id: str):
+        template = "select runtime_mode = 'rag_pipeline' from datasets where id = :id"
+        return self.query(template, {'id': id}).scalar()
+
+    @property
+    def data_source_credentials(self):
+        template = "select id, name, plugin_id, auth_type from datasource_providers"
+        return self.get_dict(template)
+
+    def credential_id_by(self, name, provider) -> list[str]:
+        template = "select id from datasource_providers where name = :name and provider = :provider"
+        return self.query(template, {'name': name, 'provider': provider}).scalars().all()
+
+
+class Document(DB):
+    def hit_documents(self, top_k: int = 3):
+        template = "SELECT dataset_id, document_id, content FROM document_segments ORDER BY hit_count DESC LIMIT :top_k"
+        return self.get_dict(template, {'top_k': top_k})
+
+    def id_by(self, name) -> list[str]:
+        """multiple ids can be found"""
+        template = "select id from documents where name = :name"
+        return [str(uuid) for uuid in self.query(template, {'name': name}).scalars().all()]
+
+
+class Pipeline(DB):
+    @property
+    def pipelines(self):
+        """unique syntax for pgsql"""
+        template = "SELECT DISTINCT ON (app_id) app_id, graph, rag_pipeline_variables FROM workflows where type = 'rag-pipeline' ORDER BY app_id, created_at DESC"
+        return Graph.convert(*self.get_dict(template))
+
+    def pipeline(self, app_id):
+        template = "select id, graph, rag_pipeline_variables from workflows where type = 'rag-pipeline' and app_id = :app_id"
+        dict_result = self.get_dict(template, {'app_id': app_id})
+        assert len(dict_result) < 2
+        return Graph.convert(*dict_result)

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/db/orm.py

@@ -0,0 +1,151 @@
+import json
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel
+from sqlalchemy import Column, String, Text, JSON, TIMESTAMP, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class DifyBase(Base):
+    __abstract__ = True  # keyword for SQLAlchemy
+    id = Column(UUID(as_uuid=True), primary_key=True, server_default=func.uuid_generate_v4())
+
+
+class AppModelConfig(DifyBase):
+    __tablename__ = "app_model_configs"
+    __table_args__ = {"schema": "public"}
+
+    app_id = Column(UUID(as_uuid=True), nullable=False)
+
+    provider = Column(String(255))
+    model_id = Column(String(255))
+    configs = Column(JSON)
+
+    created_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
+    updated_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
+
+    opening_statement = Column(Text)
+    suggested_questions = Column(Text)
+    suggested_questions_after_answer = Column(Text)
+    more_like_this = Column(Text)
+    model = Column(Text)
+    user_input_form = Column(Text)
+    pre_prompt = Column(Text)
+    agent_mode = Column(Text)
+    speech_to_text = Column(Text)
+    sensitive_word_avoidance = Column(Text)
+    retriever_resource = Column(Text)
+
+    dataset_query_variable = Column(String(255))
+    prompt_type = Column(String(255), nullable=False, server_default="simple")
+
+    chat_prompt_config = Column(Text)
+    completion_prompt_config = Column(Text)
+    dataset_configs = Column(Text)
+    external_data_tools = Column(Text)
+    file_upload = Column(Text)
+    text_to_speech = Column(Text)
+
+    created_by = Column(UUID(as_uuid=True))
+    updated_by = Column(UUID(as_uuid=True))
+
+    def __repr__(self):
+        return f"<AppModelConfig(id={self.id}, app_id={self.app_id}, provider={self.provider}, model_id={self.model_id})>"
+
+
+class Position(BaseModel):
+    x: float
+    y: float
+
+
+class NodeData(BaseModel):
+    class Type(str, Enum):
+        SOURCE = 'datasource'
+        CHUNKER = 'knowledge-index'
+        TOOL = 'tool'
+
+    type: Type | str  # not limit to built-in types
+    title: str | None = None
+    selected: bool
+
+    # datasource
+    datasource_parameters: dict[str, Any] | None = None
+    datasource_configurations: dict[str, Any] | None = None
+    plugin_id: str | None = None
+    provider_type: str | None = None
+    provider_name: str | None = None
+    datasource_name: str | None = None
+    datasource_label: str | None = None
+    plugin_unique_identifier: str | None = None
+
+    # tool
+    tool_parameters: dict[str, Any] | None = None
+    tool_configurations: dict[str, Any] | None = None
+    tool_node_version: str | None = None
+    provider_id: str | None = None
+    provider_icon: str | None = None
+    tool_name: str | None = None
+    tool_label: str | None = None
+    tool_description: str | None = None
+    is_team_authorization: bool | None = None
+    paramSchemas: list[Any] | None = None
+    params: dict[str, Any] | None = None
+
+    # knowledge index
+    index_chunk_variable_selector: list[str] | None = None
+    keyword_number: int | None = None
+    retrieval_model: dict[str, Any] | None = None
+    chunk_structure: str | None = None
+    indexing_technique: str | None = None
+    embedding_model: str | None = None
+    embedding_model_provider: str | None = None
+
+
+class Node(BaseModel):
+    @property
+    def datasource_type(self): return self.data.provider_type
+    id: str
+    type: Literal['custom']
+    data: NodeData
+    position: Position
+    targetPosition: str | None = None
+    sourcePosition: str | None = None
+    positionAbsolute: Position | None = None
+    width: float | None = None
+    height: float | None = None
+    selected: bool
+
+
+class Edge(BaseModel):
+    id: str
+    type: str
+    source: str
+    target: str
+    sourceHandle: str | None = None
+    targetHandle: str | None = None
+    data: dict[str, Any] | None = None
+    zIndex: int | None = None
+
+
+class Viewport(BaseModel):
+    x: float
+    y: float
+    zoom: float
+
+
+class Graph(BaseModel):
+    nodes: list[Node]
+    edges: list[Edge]
+    viewport: Viewport
+
+    @property
+    def datasources(self):
+        return [node for node in self.nodes if node.data.type == NodeData.Type.SOURCE]
+
+    @staticmethod
+    def convert(*records: list[dict]):
+        return [{**record, "graph": Graph(**json.loads(record["graph"]))} for record in records]

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/ops/db/sys.py

@@ -0,0 +1,6 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+
+
+class Info(DB):
+    @property
+    def accounts(self): return self.get_dict("select name, email from accounts where status = 'active'")

davidkhala_ai-0.1.6/davidkhala/ai/agent/dify/plugin.py → davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/plugins/__init__.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from pydantic import BaseModel
 
 class JsonEntry(BaseModel):
@@ -8,3 +10,5 @@ class Output(BaseModel):
     text: str
     files: list
     json: list[JsonEntry]
+class DataSourceTypeAware(BaseModel):
+    datasource_type: Literal["local_file", "online_document", "website_crawl"]

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/plugins/file.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class FileModel(BaseModel):
+    name: str
+    size: int
+    type: str
+    extension: str
+    mime_type: str
+    transfer_method: str
+    url: str
+    related_id: str
+
+
+class DataSourceOutput(DataSourceTypeAware):
+    datasource_type:str = "local_file"
+    file: FileModel

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/plugins/firecrawl.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class DataSourceInfo(BaseModel):
+    source_url: str
+    content: str
+    title: str
+    description: str
+
+
+class DataSourceOutput(DataSourceTypeAware, DataSourceInfo):
+    datasource_type: str = "website_crawl"
+
+
+class CredentialAware(BaseModel):
+    credential_id: str | None
+
+
+class Console(DataSourceOutput, CredentialAware):
+    pass

davidkhala_ai-0.1.7/davidkhala/ai/agent/dify/plugins/jina.py

@@ -0,0 +1,4 @@
+from davidkhala.ai.agent.dify.plugins.firecrawl import DataSourceOutput as FirecrawlDataSourceOutput
+
+class DataSourceOutput(FirecrawlDataSourceOutput):
+    """so far they are the same"""

{davidkhala_ai-0.1.6 → davidkhala_ai-0.1.7}/davidkhala/ai/openrouter/__init__.py

@@ -7,7 +7,7 @@ from openrouter import OpenRouter
 class Client(AbstractClient):
     def __init__(self, api_key: str):
         self.api_key = api_key
-        self.client = OpenRouter(api_key=api_key)
+        self.client = OpenRouter(api_key)
 
     def chat(self, *user_prompt, **kwargs):
         r = self.client.chat.send(
@@ -20,8 +20,16 @@ class Client(AbstractClient):
         return [_.message.content for _ in r.choices]
     def connect(self):
         try:
-            self.client.api_keys.list()
+            self.client.models.list()
             return True
         except UnauthorizedResponseError:
             return False
 
+
+class Admin:
+    def __init__(self, provisioning_key: str):
+        self.provisioning_key = provisioning_key
+        self.client = OpenRouter(provisioning_key)
+    @property
+    def keys(self):
+        return self.client.api_keys.list().data

{davidkhala_ai-0.1.6 → davidkhala_ai-0.1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "davidkhala.ai"
-version = "0.1.6"
+version = "0.1.7"
 description = "misc AI modules"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -8,18 +8,16 @@ requires-python = ">=3.13"
 [project.optional-dependencies]
 langchain = [
     "langgraph", "langchain",
-    "langchain-openai" # required by openRouter
+    "langchain-openai ; python_version < '3.14'" # required by openRouter
 ]
 ragflow = ["ragflow-sdk"]
 google = ["google-genai", "google-adk"]
 api = ["davidkhala.utils[http_request]"]
 hf = [
     "huggingface_hub", 'hf_xet',
-    'onnxruntime','onnx' # for test only
-]
-openrouter = [
-    "openrouter", "davidkhala.utils[http_request]"
+    'onnxruntime', 'onnx' # for test only
 ]
+openrouter = ["openrouter"]
 ali = ["dashscope"]
 azure = ["openai"]
 telemetry = [

davidkhala_ai-0.1.6/davidkhala/ai/agent/dify/api/app.py

@@ -1,38 +0,0 @@
-import json
-
-import requests
-
-from davidkhala.ai.agent.dify.api import API
-
-
-class Feedbacks(API):
-    def paginate_feedbacks(self, page=1, size=20):
-        """
-        when 'rating'='like', content=None
-        when 'rating'='dislike', content can be filled by end user
-        NOTE: for security reason, api cannot access conversation context associated with the feedback. End user should copy the conversation to comment by themselves.
-        """
-        response = requests.get(f"{self.base_url}/app/feedbacks", params={"page": page, "limit": size}, **self.options)
-        if not response.ok:
-            response.raise_for_status()
-        else:
-            return json.loads(response.text)
-
-    def list_feedbacks(self):
-        # TODO https://github.com/langgenius/dify/issues/28067
-        return self.paginate_feedbacks()['data']
-
-class Conversation(API):
-    """
-    Note: The Service API does not share conversations created by the WebApp. Conversations created through the API are isolated from those created in the WebApp interface.
-    It means you cannot get user conversation content from API, API call has only access to conversation created by API
-    """
-    def __init__(self, api_key: str, user: str):
-        super().__init__(api_key) # base_url need to be configured afterward if not default
-        self.user = user  # user_id, from_end_user_id
-
-    def paginate_messages(self, conversation_id):
-        return self.request(f"{self.base_url}/messages", "GET", params={
-            'conversation_id': conversation_id,
-            'user': self.user,
-        })

davidkhala_ai-0.1.6/davidkhala/ai/agent/dify/ops/db/orm.py

@@ -1,50 +0,0 @@
-from sqlalchemy import (
-    Column, String, Text, JSON, TIMESTAMP,
-    func
-)
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.orm import declarative_base
-
-Base = declarative_base()
-
-class AppModelConfig(Base):
-    __tablename__ = "app_model_configs"
-    __table_args__ = {"schema": "public"}
-
-    id = Column(UUID(as_uuid=True), primary_key=True, server_default=func.uuid_generate_v4())
-    app_id = Column(UUID(as_uuid=True), nullable=False)
-
-    provider = Column(String(255))
-    model_id = Column(String(255))
-    configs = Column(JSON)
-
-    created_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
-    updated_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
-
-    opening_statement = Column(Text)
-    suggested_questions = Column(Text)
-    suggested_questions_after_answer = Column(Text)
-    more_like_this = Column(Text)
-    model = Column(Text)
-    user_input_form = Column(Text)
-    pre_prompt = Column(Text)
-    agent_mode = Column(Text)
-    speech_to_text = Column(Text)
-    sensitive_word_avoidance = Column(Text)
-    retriever_resource = Column(Text)
-
-    dataset_query_variable = Column(String(255))
-    prompt_type = Column(String(255), nullable=False, server_default="simple")
-
-    chat_prompt_config = Column(Text)
-    completion_prompt_config = Column(Text)
-    dataset_configs = Column(Text)
-    external_data_tools = Column(Text)
-    file_upload = Column(Text)
-    text_to_speech = Column(Text)
-
-    created_by = Column(UUID(as_uuid=True))
-    updated_by = Column(UUID(as_uuid=True))
-
-    def __repr__(self):
-        return f"<AppModelConfig(id={self.id}, app_id={self.app_id}, provider={self.provider}, model_id={self.model_id})>"