davidkhala.ai 0.1.6__py3-none-any.whl → 0.1.9__py3-none-any.whl

davidkhala/ai/agent/dify/api/app.py

@@ -1,6 +1,9 @@
 import json
+from typing import TypedDict
 
 import requests
+from davidkhala.utils.http_request.stream import Request as StreamRequest, as_sse
+from requests import Response, Session
 
 from davidkhala.ai.agent.dify.api import API
 
@@ -11,6 +14,7 @@ class Feedbacks(API):
         when 'rating'='like', content=None
         when 'rating'='dislike', content can be filled by end user
         NOTE: for security reason, api cannot access conversation context associated with the feedback. End user should copy the conversation to comment by themselves.
+        # waiting for https://github.com/langgenius/dify/issues/28067
         """
         response = requests.get(f"{self.base_url}/app/feedbacks", params={"page": page, "limit": size}, **self.options)
         if not response.ok:
@@ -19,16 +23,17 @@ class Feedbacks(API):
             return json.loads(response.text)
 
     def list_feedbacks(self):
-        # TODO https://github.com/langgenius/dify/issues/28067
         return self.paginate_feedbacks()['data']
 
+
 class Conversation(API):
     """
     Note: The Service API does not share conversations created by the WebApp. Conversations created through the API are isolated from those created in the WebApp interface.
     It means you cannot get user conversation content from API, API call has only access to conversation created by API
     """
+
     def __init__(self, api_key: str, user: str):
-        super().__init__(api_key) # base_url need to be configured afterward if not default
+        super().__init__(api_key)  # base_url need to be configured afterward if not default
         self.user = user  # user_id, from_end_user_id
 
     def paginate_messages(self, conversation_id):
@@ -36,3 +41,58 @@ class Conversation(API):
             'conversation_id': conversation_id,
             'user': self.user,
         })
+
+    def _chat_request_from(self, template: str, stream, **kwargs):
+        """
+        :param template:
+        :param stream: Note: "Agent Chat App does not support blocking mode"
+        :param kwargs:
+        :return:
+        """
+        return {
+            'url': f"{self.base_url}/chat-messages",
+            'method': "POST",
+            'json': {
+                'query': template,
+                'inputs': kwargs.pop('values', {}),  # to substitute query/template
+                'response_mode': 'streaming' if stream else 'blocking',
+                'conversation_id': kwargs.pop('conversation_id', None),
+                'user': self.user,
+                'files': kwargs.pop('files', [])
+            },
+            **kwargs
+        }
+
+    def async_chat(self, template: str, **kwargs) -> tuple[Response, Session]:
+        s = StreamRequest(self)
+        s.session = Session()
+        return s.request(**self._chat_request_from(template, True, **kwargs)), s.session
+
+    class ChatResult(TypedDict, total=False):
+        thought: list[str]
+        metadata: dict
+
+    @staticmethod
+    def reduce_chat_stream(response: Response) -> ChatResult:
+        r: Conversation.ChatResult = {
+            'thought': [],
+        }
+        for data in as_sse(response):
+            match data['event']:
+                case 'agent_thought':
+                    r['thought'].append(data['thought'])
+                case 'message_end':
+                    r['metadata'] = data['metadata']
+        return r
+
+    def agent_chat(self, template: str, **kwargs) -> ChatResult:
+        r, session = self.async_chat(template, **kwargs)
+        reduced = Conversation.reduce_chat_stream(r)
+        session.close()
+        return reduced
+
+    def bot_chat(self, template: str, **kwargs):
+        r = self.request(**self._chat_request_from(template, False, **kwargs))
+        assert r.pop('event') == 'message'
+        assert r.pop('mode') == 'chat'
+        return r

davidkhala/ai/agent/dify/api/knowledge.py

@@ -8,6 +8,7 @@ from urllib.parse import urlparse
 import requests
 
 from davidkhala.ai.agent.dify.api import API, Iterator
+from davidkhala.ai.agent.dify.model import Document as DocumentBase
 
 
 class DatasetDict(TypedDict):
@@ -37,21 +38,14 @@ class DatasetDict(TypedDict):
     external_knowledge_info: dict
 
 
-class DocumentDict(TypedDict):
-    id: str
-    position: int
-    data_source_type: str
+class Document(DocumentBase):
     data_source_info: dict[str, str]
     data_source_detail_dict: dict[str, dict]
     dataset_process_rule_id: str
-    name: str
     created_from: str
     created_by: str
     created_at: int
     tokens: int
-    indexing_status: str
-    error: str
-    enabled: bool
     archived: bool
     display_status: str
     word_count: int
@@ -91,9 +85,8 @@ class Dataset(API):
 
         def upload(self, filename, *, path=None, url=None, document_id=None):
             """
-            don't work for html
-            work for markdown
-            TODO how to simulate console
+            don't work for .html
+            work for .md
             """
             files = {}
             if path:
@@ -124,10 +117,10 @@ class Dataset(API):
                 'limit': size
             })
 
-        def list_documents(self) -> Iterable[DocumentDict]:
+        def list_documents(self) -> Iterable[Document]:
             for document_batch in Iterator(self.paginate_documents, None):
                 for document in document_batch:
-                    yield document
+                    yield Document(**document)
 
         def has_document(self, name) -> bool:
             return any(name == item['name'] for row in self.list_documents() for item in row)
@@ -189,3 +182,10 @@ class Document(API):
     def delete(self):
         if self.exist():
             self.request(self.base_url, "DELETE")
+class Chunk(API):
+    def __init__(self, d: Document, segment_id: str):
+        super().__init__(d.api_key, f"{d.base_url}/segments/{segment_id}")
+    def get(self):
+        r=  self.request(self.base_url, "GET")
+        assert r['doc_form'] # optional value text_model
+        return r['data']

davidkhala/ai/agent/dify/const.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class IndexingStatus(str, Enum):
+    WAITING = "waiting"
+    PARSING = "parsing"
+    SPLITTING = 'splitting'
+    INDEXING = "indexing"
+    COMPLETED = "completed"
+    FAILED = "error"

davidkhala/ai/agent/dify/interface.py

@@ -0,0 +1,3 @@
+class IndexingError(Exception):
+    """Raised when document indexing fails (indexing_status = 'error')"""
+    pass

davidkhala/ai/agent/dify/model.py

@@ -0,0 +1,31 @@
+from pydantic import BaseModel, Field
+
+from davidkhala.ai.agent.dify.const import IndexingStatus
+
+
+class Document(BaseModel):
+    id: str
+    position: int
+    data_source_type: str
+    data_source_info: dict[str, str]
+    name: str
+    indexing_status: IndexingStatus
+    error: str | None
+    enabled: bool
+
+
+class Dataset(BaseModel):
+    id: str
+    name: str
+    description: str
+
+
+class JsonData(BaseModel):
+    data: list
+
+
+class NodeOutput(BaseModel):
+    """Schema for Output of a Dify node"""
+    text: str
+    files: list
+    json_: list[JsonData] = Field(alias="json") # avoid conflict with .json()

davidkhala/ai/agent/dify/ops/console/__init__.py

@@ -0,0 +1,9 @@
+from davidkhala.utils.http_request import Request
+
+
+class API(Request):
+    def __init__(self, base_url='http://localhost'):
+        super().__init__()
+        self.base_url = f"{base_url}/console/api"
+        self.__enter__()
+

davidkhala/ai/agent/dify/ops/console/knowledge.py

@@ -0,0 +1,158 @@
+from time import sleep
+
+from davidkhala.utils.http_request.stream import as_sse, Request as StreamRequest
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.interface import IndexingError
+from davidkhala.ai.agent.dify.model import Document, Dataset
+from davidkhala.ai.agent.dify.const import IndexingStatus
+from davidkhala.ai.agent.dify.ops.console import API
+from davidkhala.ai.agent.dify.ops.console.session import ConsoleUser
+from davidkhala.ai.agent.dify.ops.db.orm import Node
+
+
+class ConsoleKnowledge(API):
+    def __init__(self, context: ConsoleUser):
+        super().__init__()
+        self.base_url = context.base_url
+        self.session.cookies = context.session.cookies
+        self.options = context.options
+
+
+class Datasource(ConsoleKnowledge):
+    """step 1: Choose a Data Source"""
+
+    class FirecrawlOutput(BaseModel):
+        source_url: str
+        description: str
+        title: str
+        credential_id: str
+        content: str
+
+    def run_firecrawl(self, pipeline: str, node: Node,
+                      *,
+                      inputs: dict,
+                      credential_id: str
+                      ):
+
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/datasource/nodes/{node.id}/run"
+
+        stream_request = StreamRequest(self)
+        response = stream_request.request(url, 'POST', json={
+            'inputs': inputs,
+            'datasource_type': node.datasource_type,
+            'credential_id': credential_id,
+            "response_mode": "streaming"
+        })
+
+        for data in as_sse(response):
+            event = data['event']
+            if event == 'datasource_completed':
+                return data['data']
+            else:
+                assert event == 'datasource_processing'
+                print(data)
+        return None
+
+    def upload(self):
+        "http://localhost/console/api/files/upload?source=datasets"
+        # TODO
+        "form data"
+        {
+            "file": "body"
+        }
+        r = {
+            "id": "3898db5b-eb72-4f11-b507-628ad5d28887",
+            "name": "Professional Diploma Meister Power Electrical Engineering - Technological and Higher Education Institute of Hong Kong.html",
+            "size": 254362,
+            "extension": "html",
+            "mime_type": "text\/html",
+            "created_by": "dbd0b38b-5ef1-4123-8c3f-0c82eb1feacd",
+            "created_at": 1764943811,
+            "source_url": "\/files\/3898db5b-eb72-4f11-b507-628ad5d28887\/file-preview?timestamp=1764943811&nonce=43b0ff5a13372415be79de4cc7ef398c&sign=7OJ2wiVYc4tygl7yvM1sPn7s0WXDlhHxgX76bsGTD94%3D"
+        }
+
+
+class Operation(ConsoleKnowledge):
+    def website_sync(self, dataset: str, document: str, *, wait_until=True):
+        """
+        cannot be used towards a pipeline dataset. Otherwise, you will see error "no website import info found"
+        """
+        doc_url = f"{self.base_url}/datasets/{dataset}/documents/{document}"
+
+        r = self.request(f"{doc_url}/website-sync", "GET")
+        assert r == {"result": "success"}
+        if wait_until:
+            return self.wait_until(dataset, document)
+        return None
+
+    def retry(self, dataset: str, *documents: str, wait_until=True):
+        """
+        It cannot trigger rerun on success documents
+        """
+        url = f"{self.base_url}/datasets/{dataset}/retry"
+        self.request(url, "POST", json={
+            'document_ids': documents,
+        })
+        # response status code will be 204
+        if wait_until:
+            return [self.wait_until(dataset, document) for document in documents]
+        return None
+
+    def rerun(self, dataset: str, *documents: str):
+        for document in documents:
+            try:
+                self.website_sync(dataset, document)
+                assert False, "expect IndexingError"
+            except IndexingError:
+                pass
+        return self.retry(dataset, *documents)
+
+    def wait_until(self, dataset: str, document: str, *,
+                   expect_status=None,
+                   from_status=None,
+                   interval=1
+                   ):
+        if not expect_status:
+            expect_status = [IndexingStatus.FAILED, IndexingStatus.COMPLETED]
+        url = f"{self.base_url}/datasets/{dataset}/documents/{document}/indexing-status"
+        if from_status is None:
+            from_status = [IndexingStatus.WAITING, IndexingStatus.PARSING]
+        r = self.request(url, "GET")
+        status = r['indexing_status']
+        assert status in from_status, f"current status: {status}, expect: {from_status}"
+        while status not in expect_status:
+            sleep(interval)
+            r = self.request(url, "GET")
+            status = r['indexing_status']
+        if status == IndexingStatus.FAILED: raise IndexingError(r['error'])
+        return r
+
+
+class DatasetResult(Dataset):
+    chunk_structure: str
+
+class RunResult(BaseModel):
+    batch: str
+    dataset: DatasetResult
+    documents: list[Document]
+
+class Load(ConsoleKnowledge):
+    """
+    Processing Documents
+    """
+
+    def async_run(self, pipeline: str, node: Node, inputs: dict, datasource_info_list: list[dict])->RunResult:
+        """Ingest new document"""
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/run"
+        r = self.request(url, "POST", json={
+            'inputs': inputs,
+            'start_node_id': node.id,
+            'is_preview': False,
+            'response_mode': "blocking",
+            "datasource_info_list": datasource_info_list,
+            'datasource_type': node.datasource_type
+        })
+        return RunResult(**r)
+
+

davidkhala/ai/agent/dify/ops/console/session.py

@@ -0,0 +1,30 @@
+from davidkhala.ai.agent.dify.ops.console import API
+
+
+class ConsoleUser(API):
+    def login(self, email, password,
+              *,
+              remember_me=True,
+              language="en-US"
+              ):
+        url = f"{self.base_url}/login"
+
+        r = self.request(url, "POST", json={
+            'email': email,
+            'password': password,
+            'remember_me': remember_me,
+            'language': language,
+        })
+        assert r == {"result": "success"}
+        self.options['headers']['x-csrf-token'] = self.session.cookies.get("csrf_token")
+        return self.session.cookies
+
+    @property
+    def me(self) -> dict:
+        url = f"{self.base_url}/account/profile"
+        return self.request(url, "GET")
+
+    @property
+    def workspace(self) -> dict:
+        url = f"{self.base_url}/features"
+        return self.request(url, "GET")

davidkhala/ai/agent/dify/ops/db/__init__.py

@@ -1,10 +1,6 @@
-from typing import Any, Optional
+from typing import Any
 
 from davidkhala.data.base.pg import Postgres
-from sqlalchemy import desc
-from sqlalchemy.orm import Session
-
-from davidkhala.ai.agent.dify.ops.db.orm import AppModelConfig
 
 
 class DB(Postgres):
@@ -13,28 +9,9 @@ class DB(Postgres):
         super().__init__(connection_string)
         self.connect()
 
-    def get_dict(self, sql): return self.query(sql).mappings().all()
-
-    @property
-    def accounts(self): return self.get_dict("select name, email from accounts where status = 'active'")
-
-    @property
-    def apps(self): return self.get_dict("select id, name, mode from apps where status = 'normal'")
-
-    def app_config(self, app_id) -> AppModelConfig | None:
-        with Session(self.client) as session:
-            return (
-                session.query(AppModelConfig)
-                .filter(AppModelConfig.app_id == app_id)
-                .order_by(desc(AppModelConfig.created_at))
-                .first()
-            )
-
-    def update_app_config(self, record: AppModelConfig, refresh:bool=False) -> AppModelConfig | None:
-        with Session(self.client) as session:
-            session.add(record)
-            session.commit()
-            if refresh:
-                session.refresh(record)  # 刷新对象，确保拿到数据库生成的字段（如 id）
-                return record
-            return None
+    def get_dict(self,
+                 template: str,
+                 values: dict[str, Any] | None = None,
+                 request_options: dict[str, Any] | None = None
+                 ) -> list[dict]:
+        return Postgres.rows_to_dicts(self.query(template, values, request_options))

davidkhala/ai/agent/dify/ops/db/app.py

@@ -0,0 +1,34 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+from davidkhala.ai.agent.dify.ops.db.orm import AppModelConfig
+from sqlalchemy.orm import Session
+from sqlalchemy import desc
+
+class Studio(DB):
+    user_feedbacks_sql = """SELECT mf.conversation_id,
+                        mf.content,
+                        m.query,
+                        m.answer
+                 FROM message_feedbacks mf
+                          LEFT JOIN messages m ON mf.message_id = m.id
+                 WHERE mf.from_source = 'user'"""
+
+    @property
+    def apps(self): return self.get_dict("select id, name, mode from apps where status = 'normal'")
+
+    def app_config(self, app_id) -> AppModelConfig | None:
+        with Session(self.client) as session:
+            return (
+                session.query(AppModelConfig)
+                .filter(AppModelConfig.app_id == app_id)
+                .order_by(desc(AppModelConfig.created_at))
+                .first()
+            )
+
+    def update_app_config(self, record: AppModelConfig, refresh: bool = False) -> AppModelConfig | None:
+        with Session(self.client) as session:
+            session.add(record)
+            session.commit()
+            if refresh:
+                session.refresh(record)
+                return record
+            return None

davidkhala/ai/agent/dify/ops/db/knowledge.py

@@ -0,0 +1,58 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+from davidkhala.ai.agent.dify.ops.db.orm import Graph
+
+
+class Dataset(DB):
+
+    def dataset_queries(self, dataset_id, limit=20) -> list[str]:
+        template = "select content from dataset_queries where source = 'app' and created_by_role = 'end_user' and dataset_id = :dataset_id limit :limit"
+        return self.query(template, {'dataset_id': dataset_id, 'limit': limit}).scalars().all()
+
+    @property
+    def datasets(self):
+        template = "select id, name, description, indexing_technique, index_struct, embedding_model, embedding_model_provider, collection_binding_id, retrieval_model, icon_info, runtime_mode, pipeline_id, chunk_structure from datasets"
+        return self.get_dict(template)
+
+    def is_pipeline(self, id: str):
+        template = "select runtime_mode = 'rag_pipeline' from datasets where id = :id"
+        return self.query(template, {'id': id}).scalar()
+
+    @property
+    def data_source_credentials(self):
+        template = "select id, name, plugin_id, auth_type from datasource_providers"
+        return self.get_dict(template)
+
+    def credential_id_by(self, name, provider) -> list[str]:
+        template = "select id from datasource_providers where name = :name and provider = :provider"
+        return self.query(template, {'name': name, 'provider': provider}).scalars().all()
+
+    def documents(self, dataset_id: str):
+        template = "select id, name,created_from, created_at from documents where dataset_id = :dataset_id"
+        return self.query(template, {'dataset_id': dataset_id})
+
+
+class Document(DB):
+    def hit_documents(self, top_k: int = 3):
+        template = "SELECT dataset_id, document_id, content FROM document_segments ORDER BY hit_count DESC LIMIT :top_k"
+        return self.get_dict(template, {'top_k': top_k})
+
+    def id_by(self, name: str, dataset_id: str = None) -> list[str]:
+        """multiple ids can be found"""
+        template = "select id from documents where name = :name"
+        if dataset_id:
+            template = "select id from documents where name = :name and dataset_id = :dataset_id"
+        return [str(uuid) for uuid in self.query(template, {'name': name, 'dataset_id': dataset_id}).scalars().all()]
+
+
+class Pipeline(DB):
+    @property
+    def pipelines(self):
+        """unique syntax for pgsql"""
+        template = "SELECT DISTINCT ON (app_id) app_id, graph, rag_pipeline_variables FROM workflows where type = 'rag-pipeline' ORDER BY app_id, created_at DESC"
+        return Graph.convert(*self.get_dict(template))
+
+    def pipeline(self, app_id):
+        template = "select id, graph, rag_pipeline_variables from workflows where type = 'rag-pipeline' and app_id = :app_id"
+        dict_result = self.get_dict(template, {'app_id': app_id})
+        assert len(dict_result) < 2
+        return Graph.convert(*dict_result)

davidkhala/ai/agent/dify/ops/db/orm.py

@@ -1,17 +1,24 @@
-from sqlalchemy import (
-    Column, String, Text, JSON, TIMESTAMP,
-    func
-)
+import json
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel
+from sqlalchemy import Column, String, Text, JSON, TIMESTAMP, func
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.orm import declarative_base
 
 Base = declarative_base()
 
-class AppModelConfig(Base):
+
+class DifyBase(Base):
+    __abstract__ = True  # keyword for SQLAlchemy
+    id = Column(UUID(as_uuid=True), primary_key=True, server_default=func.uuid_generate_v4())
+
+
+class AppModelConfig(DifyBase):
     __tablename__ = "app_model_configs"
     __table_args__ = {"schema": "public"}
 
-    id = Column(UUID(as_uuid=True), primary_key=True, server_default=func.uuid_generate_v4())
     app_id = Column(UUID(as_uuid=True), nullable=False)
 
     provider = Column(String(255))
@@ -48,3 +55,97 @@ class AppModelConfig(Base):
 
     def __repr__(self):
         return f"<AppModelConfig(id={self.id}, app_id={self.app_id}, provider={self.provider}, model_id={self.model_id})>"
+
+
+class Position(BaseModel):
+    x: float
+    y: float
+
+
+class NodeData(BaseModel):
+    class Type(str, Enum):
+        SOURCE = 'datasource'
+        CHUNKER = 'knowledge-index'
+        TOOL = 'tool'
+
+    type: Type | str  # not limit to built-in types
+    title: str | None = None
+    selected: bool
+
+    # datasource
+    datasource_parameters: dict[str, Any] | None = None
+    datasource_configurations: dict[str, Any] | None = None
+    plugin_id: str | None = None
+    provider_type: str | None = None
+    provider_name: str | None = None
+    datasource_name: str | None = None
+    datasource_label: str | None = None
+    plugin_unique_identifier: str | None = None
+
+    # tool
+    tool_parameters: dict[str, Any] | None = None
+    tool_configurations: dict[str, Any] | None = None
+    tool_node_version: str | None = None
+    provider_id: str | None = None
+    provider_icon: str | None = None
+    tool_name: str | None = None
+    tool_label: str | None = None
+    tool_description: str | None = None
+    is_team_authorization: bool | None = None
+    paramSchemas: list[Any] | None = None
+    params: dict[str, Any] | None = None
+
+    # knowledge index
+    index_chunk_variable_selector: list[str] | None = None
+    keyword_number: int | None = None
+    retrieval_model: dict[str, Any] | None = None
+    chunk_structure: str | None = None
+    indexing_technique: str | None = None
+    embedding_model: str | None = None
+    embedding_model_provider: str | None = None
+
+
+class Node(BaseModel):
+    @property
+    def datasource_type(self): return self.data.provider_type
+    id: str
+    type: Literal['custom']
+    data: NodeData
+    position: Position
+    targetPosition: str | None = None
+    sourcePosition: str | None = None
+    positionAbsolute: Position | None = None
+    width: float | None = None
+    height: float | None = None
+    selected: bool
+
+
+class Edge(BaseModel):
+    id: str
+    type: str
+    source: str
+    target: str
+    sourceHandle: str | None = None
+    targetHandle: str | None = None
+    data: dict[str, Any] | None = None
+    zIndex: int | None = None
+
+
+class Viewport(BaseModel):
+    x: float
+    y: float
+    zoom: float
+
+
+class Graph(BaseModel):
+    nodes: list[Node]
+    edges: list[Edge]
+    viewport: Viewport
+
+    @property
+    def datasources(self):
+        return [node for node in self.nodes if node.data.type == NodeData.Type.SOURCE]
+
+    @staticmethod
+    def convert(*records: list[dict]):
+        return [{**record, "graph": Graph(**json.loads(record["graph"]))} for record in records]

davidkhala/ai/agent/dify/ops/db/sys.py

@@ -0,0 +1,6 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+
+
+class Info(DB):
+    @property
+    def accounts(self): return self.get_dict("select name, email from accounts where status = 'active'")

davidkhala/ai/agent/dify/plugins/__init__.py

@@ -0,0 +1,7 @@
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class DataSourceTypeAware(BaseModel):
+    datasource_type: Literal["local_file", "online_document", "website_crawl"]

davidkhala/ai/agent/dify/plugins/file.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class FileModel(BaseModel):
+    name: str
+    size: int
+    type: str
+    extension: str
+    mime_type: str
+    transfer_method: str
+    url: str
+    related_id: str
+
+
+class DataSourceOutput(DataSourceTypeAware):
+    datasource_type:str = "local_file"
+    file: FileModel

davidkhala/ai/agent/dify/plugins/firecrawl.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class DataSourceInfo(BaseModel):
+    source_url: str
+    content: str
+    title: str
+    description: str
+
+
+class DataSourceOutput(DataSourceTypeAware, DataSourceInfo):
+    datasource_type: str = "website_crawl"
+
+
+class CredentialAware(BaseModel):
+    credential_id: str | None
+
+
+class Console(DataSourceOutput, CredentialAware):
+    pass

davidkhala/ai/agent/dify/plugins/jina.py

@@ -0,0 +1,4 @@
+from davidkhala.ai.agent.dify.plugins.firecrawl import DataSourceOutput as FirecrawlDataSourceOutput
+
+class DataSourceOutput(FirecrawlDataSourceOutput):
+    """so far they are the same"""

davidkhala/ai/ali/agentbay.py

@@ -0,0 +1,39 @@
+from agentbay import AgentBay, Session, Config, AgentBayLogger, BrowserOption
+from davidkhala.utils.syntax.interface import ContextAware
+
+AgentBayLogger.setup(level='WARNING')  # Default to INFO
+
+
+class Client(ContextAware):
+    def __init__(self, api_key, *, timeout_ms=10000):
+        self.agent = AgentBay(
+            api_key=api_key,
+            cfg=Config(endpoint="wuyingai.ap-southeast-1.aliyuncs.com", timeout_ms=timeout_ms)
+        )
+        self.session: Session | None = None
+
+    def open(self):
+        r = self.agent.create()
+        if not r.success:
+            return False
+        self.session = r.session
+        return True
+
+    def close(self):
+        self.agent.delete(self.session)
+        del self.session
+
+
+class Browser(ContextAware):
+    def __init__(self, session: Session):
+        self.session = session
+        self.option = BrowserOption()
+        self.endpoint_url: str | None = None
+
+    def open(self) -> bool:
+        success = self.session.browser.initialize(self.option)
+        self.endpoint_url = self.session.browser.get_endpoint_url()
+        return success
+
+    def close(self):
+        self.session.browser.destroy()

davidkhala/ai/openai/azure.py

@@ -1,5 +1,4 @@
-import warnings
-
+from davidkhala.utils.syntax.compat import deprecated
 from openai import AzureOpenAI, OpenAI
 
 from davidkhala.ai.openai import Client
@@ -20,7 +19,7 @@ class ModelDeploymentClient(AzureHosted):
         )
 
 
-@warnings.deprecated("Azure Open AI is deprecated. Please migrate to Azure AI Foundry")
+@deprecated("Azure Open AI is deprecated. Please migrate to Microsoft Foundry")
 class OpenAIClient(AzureHosted):
 
     def __init__(self, api_key, project):

davidkhala/ai/openrouter/__init__.py

@@ -7,7 +7,7 @@ from openrouter import OpenRouter
 class Client(AbstractClient):
     def __init__(self, api_key: str):
         self.api_key = api_key
-        self.client = OpenRouter(api_key=api_key)
+        self.client = OpenRouter(api_key)
 
     def chat(self, *user_prompt, **kwargs):
         r = self.client.chat.send(
@@ -20,8 +20,16 @@ class Client(AbstractClient):
         return [_.message.content for _ in r.choices]
     def connect(self):
         try:
-            self.client.api_keys.list()
+            self.client.models.list()
             return True
         except UnauthorizedResponseError:
             return False
 
+
+class Admin:
+    def __init__(self, provisioning_key: str):
+        self.provisioning_key = provisioning_key
+        self.client = OpenRouter(provisioning_key)
+    @property
+    def keys(self):
+        return self.client.api_keys.list().data

{davidkhala_ai-0.1.6.dist-info → davidkhala_ai-0.1.9.dist-info}/METADATA

@@ -1,13 +1,16 @@
 Metadata-Version: 2.4
 Name: davidkhala.ai
-Version: 0.1.6
+Version: 0.1.9
 Summary: misc AI modules
-Requires-Python: >=3.13
+Requires-Python: >=3.12
 Provides-Extra: ali
 Requires-Dist: dashscope; extra == 'ali'
+Requires-Dist: davidkhala-utils; extra == 'ali'
+Requires-Dist: wuying-agentbay-sdk; extra == 'ali'
 Provides-Extra: api
 Requires-Dist: davidkhala-utils[http-request]; extra == 'api'
 Provides-Extra: azure
+Requires-Dist: davidkhala-utils; extra == 'azure'
 Requires-Dist: openai; extra == 'azure'
 Provides-Extra: dify
 Requires-Dist: davidkhala-databases[pg]; extra == 'dify'
@@ -23,10 +26,9 @@ Requires-Dist: onnx; extra == 'hf'
 Requires-Dist: onnxruntime; extra == 'hf'
 Provides-Extra: langchain
 Requires-Dist: langchain; extra == 'langchain'
-Requires-Dist: langchain-openai; extra == 'langchain'
+Requires-Dist: langchain-openai; (python_version < '3.14') and extra == 'langchain'
 Requires-Dist: langgraph; extra == 'langchain'
 Provides-Extra: openrouter
-Requires-Dist: davidkhala-utils[http-request]; extra == 'openrouter'
 Requires-Dist: openrouter; extra == 'openrouter'
 Provides-Extra: ragflow
 Requires-Dist: ragflow-sdk; extra == 'ragflow'

davidkhala_ai-0.1.9.dist-info/RECORD

@@ -0,0 +1,46 @@
+davidkhala/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/model.py,sha256=1wcXC8X8oqerMatlcPbZmuxZ-nJWdJKmaDSDgiGlUGw,647
+davidkhala/ai/opik.py,sha256=YU1XuweMUAzUkhpjxhltt-SBBDBkR3z-PCNo0DqzBRs,39
+davidkhala/ai/agent/README.md,sha256=kIPsx3gOjrpOw7w2qhNEALuCEQkuh4nYp6uBnijdvHE,178
+davidkhala/ai/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/agent/langgraph.py,sha256=jrc_Yvgo7eJjd3y5UJn0t1FzpnObDGYscwgsuVl2O_I,1052
+davidkhala/ai/agent/ragflow.py,sha256=UaK31us6V0NhAPCthGo07rQsm72vlR-McmihC_NDe1g,273
+davidkhala/ai/agent/dify/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/agent/dify/const.py,sha256=gU4lPBe4U2taakN2jhdPMRWXkqlyCg-YRE8JJmtsblo,218
+davidkhala/ai/agent/dify/interface.py,sha256=bTOI38ZjtkgoSw-ysgFwBZ1QkKVAa92gjOnERDoagQA,118
+davidkhala/ai/agent/dify/model.py,sha256=1LEwKWWkFNmhbBWABEu7I45DRZ_BFGDP5uTHDrvldoo,641
+davidkhala/ai/agent/dify/api/__init__.py,sha256=9-8OesuXF_wPmPrh_gEZpEZP51dcZxb0i6ixOBYKcwQ,876
+davidkhala/ai/agent/dify/api/app.py,sha256=y1mILC-fvQpeH50ASbFBluD9tFAwXu_IWwtwucMV5jM,3801
+davidkhala/ai/agent/dify/api/knowledge.py,sha256=5ePqvzjBHNtQ64Dzt39wBWedYVeQJc23syNe9LFnGw8,5960
+davidkhala/ai/agent/dify/ops/__init__.py,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
+davidkhala/ai/agent/dify/ops/console/__init__.py,sha256=-a81jgCJ3s2B3i1GQ7ge1aZRfbvlALwGDHVu_GEET-A,237
+davidkhala/ai/agent/dify/ops/console/knowledge.py,sha256=I1v0iE_b4VPc2Zsyt4ci_oX080Qbgn3oXObP4uVEphg,5788
+davidkhala/ai/agent/dify/ops/console/session.py,sha256=Kt8vzZJUlyqD8G8_OsrOD-WQwyDor8tqNwV0jJ738wE,902
+davidkhala/ai/agent/dify/ops/db/__init__.py,sha256=HYfJEnoFAoJJck2xvTDYx8zpw9Qao7sHXOGvW0diPqw,517
+davidkhala/ai/agent/dify/ops/db/app.py,sha256=IRiSiR0v387p4p3J7M9xEkJ7pfQyO5DL6chpx7Z2IzA,1319
+davidkhala/ai/agent/dify/ops/db/knowledge.py,sha256=GVaK5QmU_VxB8fDxV60uiYiIeR3JEn3IXJTlJHLiT5U,2917
+davidkhala/ai/agent/dify/ops/db/orm.py,sha256=CnZj8mV2RZhw_7hF1YICTUjROQ66hR5_8OCMQvtujnY,4575
+davidkhala/ai/agent/dify/ops/db/sys.py,sha256=U_qqopUMlgsilhHaG_ids6gtd-pNiR_Jm0kAr9hIL7M,188
+davidkhala/ai/agent/dify/plugins/__init__.py,sha256=iTWvutlkN9bXgptesi05M447nTeF5hKFAIfn4EviFj0,183
+davidkhala/ai/agent/dify/plugins/file.py,sha256=o-HjHSFwRTNIYs8IxqZUSnBbh-xr8f-xMUM3iU9wCCQ,390
+davidkhala/ai/agent/dify/plugins/firecrawl.py,sha256=lB_f8W_bdg-7PeBKmF0-HdwYyakV_0D3nET5iT-Z1KM,460
+davidkhala/ai/agent/dify/plugins/jina.py,sha256=dQ5iJxDLWtChXb1IjCtsHctgUtgjOiDfWOuR2u0aUIM,190
+davidkhala/ai/ali/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/ali/agentbay.py,sha256=O5t71GGwtDgBE1zUXJDYe5djMVwSaNOwn5k8zg1xa18,1200
+davidkhala/ai/ali/dashscope.py,sha256=SZIzRhVHlLx3s5I2RNUh2-u8OoSdrbvoN5e1k8Mh8N0,1943
+davidkhala/ai/api/__init__.py,sha256=q2Ro5nhW5kJx2CYR1MRVamjTT5tTexPZwhrS2hwAvFM,1319
+davidkhala/ai/api/openrouter.py,sha256=khccJr5cBnudFy6Jc2O3A1TNCuHH_5W6Q2tXrkwlUYE,2308
+davidkhala/ai/api/siliconflow.py,sha256=JbnOSv8LJLtwYSNNB8_SMBMQzOgHDtQYZKA9A2BC4sY,2139
+davidkhala/ai/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/google/adk.py,sha256=QwxYoOzT2Hol03V4NM0PF_HAzUGb4fB18VUAYacYbAY,657
+davidkhala/ai/google/gemini.py,sha256=Xf4HDOOcK4-jEBERzuLnQNFsU61P2fFx4K0z-ijvNHE,214
+davidkhala/ai/huggingface/BAAI.py,sha256=LZ9kp5Gfql4UzuTn4osyekI6VV1H3RIfED2IolXFj5c,341
+davidkhala/ai/huggingface/__init__.py,sha256=FJyU8eOfWQWKAvkIa5qwubF9ghsSQ8C0e6p6DKyomgs,521
+davidkhala/ai/huggingface/inference.py,sha256=bYN0PtLF2CaIHzdTP4LaTALJhcawvuLnLR7rhMVqwDE,333
+davidkhala/ai/openai/__init__.py,sha256=GXzWaw2ER3YFGHG6TPD9SmAHV6Tpsnqxj6tXlaWsrko,1897
+davidkhala/ai/openai/azure.py,sha256=WmWSz9pKlUrQLSH25m1jE1l-mNWw9QQARj8uliOv8VU,1138
+davidkhala/ai/openai/native.py,sha256=MB0nDnzCOj_M42RMhdK3HTMVnxGnwpLT2GeLwSrepwI,704
+davidkhala/ai/openrouter/__init__.py,sha256=P8UvolZihN_CVBQ7BT1Fb6mSMFEQLyLY9G5bBDZhC0o,1037
+davidkhala_ai-0.1.9.dist-info/METADATA,sha256=d1eUCeXWEewssHevligqENx8Thz7rd_2wVhcdvAmjKQ,1607
+davidkhala_ai-0.1.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+davidkhala_ai-0.1.9.dist-info/RECORD,,

{davidkhala_ai-0.1.6.dist-info → davidkhala_ai-0.1.9.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

davidkhala/ai/agent/dify/plugin.py

@@ -1,10 +0,0 @@
-from pydantic import BaseModel
-
-class JsonEntry(BaseModel):
-    data: list
-
-class Output(BaseModel):
-    """Class for result of a Dify node"""
-    text: str
-    files: list
-    json: list[JsonEntry]

davidkhala_ai-0.1.6.dist-info/RECORD

@@ -1,33 +0,0 @@
-davidkhala/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/model.py,sha256=1wcXC8X8oqerMatlcPbZmuxZ-nJWdJKmaDSDgiGlUGw,647
-davidkhala/ai/opik.py,sha256=YU1XuweMUAzUkhpjxhltt-SBBDBkR3z-PCNo0DqzBRs,39
-davidkhala/ai/agent/README.md,sha256=kIPsx3gOjrpOw7w2qhNEALuCEQkuh4nYp6uBnijdvHE,178
-davidkhala/ai/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/langgraph.py,sha256=jrc_Yvgo7eJjd3y5UJn0t1FzpnObDGYscwgsuVl2O_I,1052
-davidkhala/ai/agent/ragflow.py,sha256=UaK31us6V0NhAPCthGo07rQsm72vlR-McmihC_NDe1g,273
-davidkhala/ai/agent/dify/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/dify/plugin.py,sha256=wrX47gev8GBbWkF1g7h_9bx4UpgdC3OhhjRRAXw60zs,209
-davidkhala/ai/agent/dify/api/__init__.py,sha256=9-8OesuXF_wPmPrh_gEZpEZP51dcZxb0i6ixOBYKcwQ,876
-davidkhala/ai/agent/dify/api/app.py,sha256=CJT6fdUfLyuQkvtrFEbtfEcKWIBzhcQDYV4J3nKx-DQ,1624
-davidkhala/ai/agent/dify/api/knowledge.py,sha256=cQPTS2S8DRfUKSECrLqFLC-PtObpYTGv2rHEvhkXW-k,5765
-davidkhala/ai/agent/dify/ops/__init__.py,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
-davidkhala/ai/agent/dify/ops/db/__init__.py,sha256=OXEUHs7unxRfw8ozwK_lUhV-SaOgCuEYM27q71F1nXU,1412
-davidkhala/ai/agent/dify/ops/db/orm.py,sha256=NrmVn7oDcWiWw7mCzyJ_QPTTju8ayX3Ar21JICREGpg,1780
-davidkhala/ai/ali/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/ali/dashscope.py,sha256=SZIzRhVHlLx3s5I2RNUh2-u8OoSdrbvoN5e1k8Mh8N0,1943
-davidkhala/ai/api/__init__.py,sha256=q2Ro5nhW5kJx2CYR1MRVamjTT5tTexPZwhrS2hwAvFM,1319
-davidkhala/ai/api/openrouter.py,sha256=khccJr5cBnudFy6Jc2O3A1TNCuHH_5W6Q2tXrkwlUYE,2308
-davidkhala/ai/api/siliconflow.py,sha256=JbnOSv8LJLtwYSNNB8_SMBMQzOgHDtQYZKA9A2BC4sY,2139
-davidkhala/ai/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/google/adk.py,sha256=QwxYoOzT2Hol03V4NM0PF_HAzUGb4fB18VUAYacYbAY,657
-davidkhala/ai/google/gemini.py,sha256=Xf4HDOOcK4-jEBERzuLnQNFsU61P2fFx4K0z-ijvNHE,214
-davidkhala/ai/huggingface/BAAI.py,sha256=LZ9kp5Gfql4UzuTn4osyekI6VV1H3RIfED2IolXFj5c,341
-davidkhala/ai/huggingface/__init__.py,sha256=FJyU8eOfWQWKAvkIa5qwubF9ghsSQ8C0e6p6DKyomgs,521
-davidkhala/ai/huggingface/inference.py,sha256=bYN0PtLF2CaIHzdTP4LaTALJhcawvuLnLR7rhMVqwDE,333
-davidkhala/ai/openai/__init__.py,sha256=GXzWaw2ER3YFGHG6TPD9SmAHV6Tpsnqxj6tXlaWsrko,1897
-davidkhala/ai/openai/azure.py,sha256=QR1uZj8qAyhpCjo3Ks5zNV8GfOp3-enyZs6fBvV-MkA,1110
-davidkhala/ai/openai/native.py,sha256=MB0nDnzCOj_M42RMhdK3HTMVnxGnwpLT2GeLwSrepwI,704
-davidkhala/ai/openrouter/__init__.py,sha256=5vciqhkPwQqBcHEwbuTeuwQgESqb6jsnQmb__EC4nWE,798
-davidkhala_ai-0.1.6.dist-info/METADATA,sha256=bgODlj3_Ma0zhfSwxO-6So3k9L7tonkyQkpTz6sa0CU,1497
-davidkhala_ai-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-davidkhala_ai-0.1.6.dist-info/RECORD,,