davidkhala.ai 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

davidkhala/ai/agent/dify/api/__init__.py

@@ -0,0 +1,27 @@
+from typing import Iterable, Callable, Any, Optional
+
+from davidkhala.utils.http_request import Request
+
+
+class API(Request):
+    def __init__(self, api_key: str, base_url="https://api.dify.ai/v1"):
+        super().__init__({'bearer': api_key})
+        self.base_url = base_url
+        self.api_key = api_key
+
+
+class Iterator(Iterable):
+    def __iter__(self):
+        return self
+
+    def __init__(self, get_fn: Callable[[int, int], Any], r: Optional[dict]):
+        self.response = r
+        self.fn = get_fn
+
+    def __next__(self):
+        if self.response and not self.response['has_more']:
+            raise StopIteration
+        page = 1 if not self.response else self.response['page'] + 1
+        limit = None if not self.response else self.response['limit']
+        self.response = self.fn(page, limit)
+        return self.response['data']

davidkhala/ai/agent/dify/api/app.py

@@ -0,0 +1,98 @@
+import json
+from typing import TypedDict
+
+import requests
+from davidkhala.utils.http_request.stream import Request as StreamRequest, as_sse
+from requests import Response, Session
+
+from davidkhala.ai.agent.dify.api import API
+
+
+class Feedbacks(API):
+    def paginate_feedbacks(self, page=1, size=20):
+        """
+        when 'rating'='like', content=None
+        when 'rating'='dislike', content can be filled by end user
+        NOTE: for security reason, api cannot access conversation context associated with the feedback. End user should copy the conversation to comment by themselves.
+        # waiting for https://github.com/langgenius/dify/issues/28067
+        """
+        response = requests.get(f"{self.base_url}/app/feedbacks", params={"page": page, "limit": size}, **self.options)
+        if not response.ok:
+            response.raise_for_status()
+        else:
+            return json.loads(response.text)
+
+    def list_feedbacks(self):
+        return self.paginate_feedbacks()['data']
+
+
+class Conversation(API):
+    """
+    Note: The Service API does not share conversations created by the WebApp. Conversations created through the API are isolated from those created in the WebApp interface.
+    It means you cannot get user conversation content from API, API call has only access to conversation created by API
+    """
+
+    def __init__(self, api_key: str, user: str):
+        super().__init__(api_key)  # base_url need to be configured afterward if not default
+        self.user = user  # user_id, from_end_user_id
+
+    def paginate_messages(self, conversation_id):
+        return self.request(f"{self.base_url}/messages", "GET", params={
+            'conversation_id': conversation_id,
+            'user': self.user,
+        })
+
+    def _chat_request_from(self, template: str, stream, **kwargs):
+        """
+        :param template:
+        :param stream: Note: "Agent Chat App does not support blocking mode"
+        :param kwargs:
+        :return:
+        """
+        return {
+            'url': f"{self.base_url}/chat-messages",
+            'method': "POST",
+            'json': {
+                'query': template,
+                'inputs': kwargs.pop('values', {}),  # to substitute query/template
+                'response_mode': 'streaming' if stream else 'blocking',
+                'conversation_id': kwargs.pop('conversation_id', None),
+                'user': self.user,
+                'files': kwargs.pop('files', [])
+            },
+            **kwargs
+        }
+
+    def async_chat(self, template: str, **kwargs) -> tuple[Response, Session]:
+        s = StreamRequest(self)
+        s.session = Session()
+        return s.request(**self._chat_request_from(template, True, **kwargs)), s.session
+
+    class ChatResult(TypedDict, total=False):
+        thought: list[str]
+        metadata: dict
+
+    @staticmethod
+    def reduce_chat_stream(response: Response) -> ChatResult:
+        r: Conversation.ChatResult = {
+            'thought': [],
+        }
+        for data in as_sse(response):
+            match data['event']:
+                case 'agent_thought':
+                    r['thought'].append(data['thought'])
+                case 'message_end':
+                    r['metadata'] = data['metadata']
+        return r
+
+    def agent_chat(self, template: str, **kwargs) -> ChatResult:
+        r, session = self.async_chat(template, **kwargs)
+        reduced = Conversation.reduce_chat_stream(r)
+        session.close()
+        return reduced
+
+    def bot_chat(self, template: str, **kwargs):
+        r = self.request(**self._chat_request_from(template, False, **kwargs))
+        assert r.pop('event') == 'message'
+        assert r.pop('mode') == 'chat'
+        return r

davidkhala/ai/agent/dify/{knowledge.py → api/knowledge.py}

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-from typing import Iterable, TypedDict, Callable, Any, Optional
+from typing import Iterable, TypedDict, Optional
 from urllib.parse import urlparse
 
 import requests
 
-from davidkhala.ai.agent.dify.base import API
-
+from davidkhala.ai.agent.dify.api import API, Iterator
+from davidkhala.ai.agent.dify.common import Document as DocumentBase
 
 class DatasetDict(TypedDict):
     id: str
@@ -37,38 +37,14 @@ class DatasetDict(TypedDict):
     external_knowledge_info: dict
 
 
-class Iterator(Iterable):
-    def __iter__(self):
-        return self
-
-    def __init__(self, get_fn: Callable[[int, int], Any], r: Optional[dict]):
-        self.response = r
-        self.fn = get_fn
-
-    def __next__(self):
-        if self.response and not self.response['has_more']:
-            raise StopIteration
-        page = 1 if not self.response else self.response['page'] + 1
-        limit = None if not self.response else self.response['limit']
-        self.response = self.fn(page, limit)
-        return self.response['data']
-
-
-class DocumentDict(TypedDict):
-    id: str
-    position: int
-    data_source_type: str
+class Document(DocumentBase):
     data_source_info: dict[str, str]
     data_source_detail_dict: dict[str, dict]
     dataset_process_rule_id: str
-    name: str
     created_from: str
     created_by: str
     created_at: int
     tokens: int
-    indexing_status: str
-    error: str
-    enabled: bool
     archived: bool
     display_status: str
     word_count: int
@@ -108,9 +84,8 @@ class Dataset(API):
 
         def upload(self, filename, *, path=None, url=None, document_id=None):
             """
-            don't work for html
-            work for markdown
-            TODO how to simulate console
+            don't work for .html
+            work for .md
             """
             files = {}
             if path:
@@ -141,10 +116,10 @@ class Dataset(API):
                 'limit': size
             })
 
-        def list_documents(self) -> Iterable[DocumentDict]:
+        def list_documents(self) -> Iterable[Document]:
             for document_batch in Iterator(self.paginate_documents, None):
                 for document in document_batch:
-                    yield document
+                    yield Document(**document)
 
         def has_document(self, name) -> bool:
             return any(name == item['name'] for row in self.list_documents() for item in row)
@@ -206,3 +181,10 @@ class Document(API):
     def delete(self):
         if self.exist():
             self.request(self.base_url, "DELETE")
+class Chunk(API):
+    def __init__(self, d: Document, segment_id: str):
+        super().__init__(d.api_key, f"{d.base_url}/segments/{segment_id}")
+    def get(self):
+        r=  self.request(self.base_url, "GET")
+        assert r['doc_form'] # optional value text_model
+        return r['data']

davidkhala/ai/agent/dify/common.py

@@ -0,0 +1,36 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins.firecrawl import DataSourceInfo
+
+
+class IndexingStatus(str, Enum):
+    WAITING = "waiting"
+    PARSING = "parsing"
+    SPLITTING = 'splitting'
+    INDEXING = "indexing"
+    COMPLETED = "completed"
+    FAILED = "error"
+
+
+class Document(BaseModel):
+    id: str
+    position: int
+    data_source_type: str
+    data_source_info: dict[str, str]
+    name: str
+    indexing_status: IndexingStatus
+    error: str | None
+    enabled: bool
+
+
+class Dataset(BaseModel):
+    id: str
+    name: str
+    description: str
+
+
+class IndexingError(Exception):
+    """Raised when document indexing fails (indexing_status = 'error')"""
+    pass

davidkhala/ai/agent/dify/ops/__init__.py

@@ -0,0 +1 @@
+

davidkhala/ai/agent/dify/ops/console/__init__.py

@@ -0,0 +1,9 @@
+from davidkhala.utils.http_request import Request
+
+
+class API(Request):
+    def __init__(self, base_url='http://localhost'):
+        super().__init__()
+        self.base_url = f"{base_url}/console/api"
+        self.__enter__()
+

davidkhala/ai/agent/dify/ops/console/knowledge.py

@@ -0,0 +1,156 @@
+from time import sleep
+
+from davidkhala.utils.http_request.stream import as_sse, Request as StreamRequest
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.common import IndexingStatus, IndexingError, Dataset, Document
+from davidkhala.ai.agent.dify.ops.console import API
+from davidkhala.ai.agent.dify.ops.console.session import ConsoleUser
+from davidkhala.ai.agent.dify.ops.db.orm import Node
+
+
+class ConsoleKnowledge(API):
+    def __init__(self, context: ConsoleUser):
+        super().__init__()
+        self.base_url = context.base_url
+        self.session.cookies = context.session.cookies
+        self.options = context.options
+
+
+class Datasource(ConsoleKnowledge):
+    """step 1: Choose a Data Source"""
+
+    class FirecrawlOutput(BaseModel):
+        source_url: str
+        description: str
+        title: str
+        credential_id: str
+        content: str
+
+    def run_firecrawl(self, pipeline: str, node: Node,
+                      *,
+                      inputs: dict,
+                      credential_id: str
+                      ):
+
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/datasource/nodes/{node.id}/run"
+
+        stream_request = StreamRequest(self)
+        response = stream_request.request(url, 'POST', json={
+            'inputs': inputs,
+            'datasource_type': node.datasource_type,
+            'credential_id': credential_id,
+            "response_mode": "streaming"
+        })
+
+        for data in as_sse(response):
+            event = data['event']
+            if event == 'datasource_completed':
+                return data['data']
+            else:
+                assert event == 'datasource_processing'
+                print(data)
+        return None
+
+    def upload(self):
+        "http://localhost/console/api/files/upload?source=datasets"
+        # TODO
+        "form data"
+        {
+            "file": "body"
+        }
+        r = {
+            "id": "3898db5b-eb72-4f11-b507-628ad5d28887",
+            "name": "Professional Diploma Meister Power Electrical Engineering - Technological and Higher Education Institute of Hong Kong.html",
+            "size": 254362,
+            "extension": "html",
+            "mime_type": "text\/html",
+            "created_by": "dbd0b38b-5ef1-4123-8c3f-0c82eb1feacd",
+            "created_at": 1764943811,
+            "source_url": "\/files\/3898db5b-eb72-4f11-b507-628ad5d28887\/file-preview?timestamp=1764943811&nonce=43b0ff5a13372415be79de4cc7ef398c&sign=7OJ2wiVYc4tygl7yvM1sPn7s0WXDlhHxgX76bsGTD94%3D"
+        }
+
+
+class Operation(ConsoleKnowledge):
+    def website_sync(self, dataset: str, document: str, *, wait_until=True):
+        """
+        cannot be used towards a pipeline dataset. Otherwise, you will see error "no website import info found"
+        """
+        doc_url = f"{self.base_url}/datasets/{dataset}/documents/{document}"
+
+        r = self.request(f"{doc_url}/website-sync", "GET")
+        assert r == {"result": "success"}
+        if wait_until:
+            return self.wait_until(dataset, document)
+        return None
+
+    def retry(self, dataset: str, *documents: str, wait_until=True):
+        """
+        It cannot trigger rerun on success documents
+        """
+        url = f"{self.base_url}/datasets/{dataset}/retry"
+        self.request(url, "POST", json={
+            'document_ids': documents,
+        })
+        # response status code will be 204
+        if wait_until:
+            return [self.wait_until(dataset, document) for document in documents]
+        return None
+
+    def rerun(self, dataset: str, *documents: str):
+        for document in documents:
+            try:
+                self.website_sync(dataset, document)
+                assert False, "expect IndexingError"
+            except IndexingError:
+                pass
+        return self.retry(dataset, *documents)
+
+    def wait_until(self, dataset: str, document: str, *,
+                   expect_status=None,
+                   from_status=None,
+                   interval=1
+                   ):
+        if not expect_status:
+            expect_status = [IndexingStatus.FAILED, IndexingStatus.COMPLETED]
+        url = f"{self.base_url}/datasets/{dataset}/documents/{document}/indexing-status"
+        if from_status is None:
+            from_status = [IndexingStatus.WAITING, IndexingStatus.PARSING]
+        r = self.request(url, "GET")
+        status = r['indexing_status']
+        assert status in from_status, f"current status: {status}, expect: {from_status}"
+        while status not in expect_status:
+            sleep(interval)
+            r = self.request(url, "GET")
+            status = r['indexing_status']
+        if status == IndexingStatus.FAILED: raise IndexingError(r['error'])
+        return r
+
+
+class DatasetResult(Dataset):
+    chunk_structure: str
+
+class RunResult(BaseModel):
+    batch: str
+    dataset: DatasetResult
+    documents: list[Document]
+
+class Load(ConsoleKnowledge):
+    """
+    Processing Documents
+    """
+
+    def async_run(self, pipeline: str, node: Node, inputs: dict, datasource_info_list: list[dict])->RunResult:
+        """Ingest new document"""
+        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/run"
+        r = self.request(url, "POST", json={
+            'inputs': inputs,
+            'start_node_id': node.id,
+            'is_preview': False,
+            'response_mode': "blocking",
+            "datasource_info_list": datasource_info_list,
+            'datasource_type': node.datasource_type
+        })
+        return RunResult(**r)
+
+

davidkhala/ai/agent/dify/ops/console/session.py

@@ -0,0 +1,30 @@
+from davidkhala.ai.agent.dify.ops.console import API
+
+
+class ConsoleUser(API):
+    def login(self, email, password,
+              *,
+              remember_me=True,
+              language="en-US"
+              ):
+        url = f"{self.base_url}/login"
+
+        r = self.request(url, "POST", json={
+            'email': email,
+            'password': password,
+            'remember_me': remember_me,
+            'language': language,
+        })
+        assert r == {"result": "success"}
+        self.options['headers']['x-csrf-token'] = self.session.cookies.get("csrf_token")
+        return self.session.cookies
+
+    @property
+    def me(self) -> dict:
+        url = f"{self.base_url}/account/profile"
+        return self.request(url, "GET")
+
+    @property
+    def workspace(self) -> dict:
+        url = f"{self.base_url}/features"
+        return self.request(url, "GET")

davidkhala/ai/agent/dify/ops/db/__init__.py

@@ -0,0 +1,17 @@
+from typing import Any
+
+from davidkhala.data.base.pg import Postgres
+
+
+class DB(Postgres):
+
+    def __init__(self, connection_string: str):
+        super().__init__(connection_string)
+        self.connect()
+
+    def get_dict(self,
+                 template: str,
+                 values: dict[str, Any] | None = None,
+                 request_options: dict[str, Any] | None = None
+                 ) -> list[dict]:
+        return Postgres.rows_to_dicts(self.query(template, values, request_options))

davidkhala/ai/agent/dify/ops/db/app.py

@@ -0,0 +1,37 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+from davidkhala.ai.agent.dify.ops.db.orm import AppModelConfig
+from sqlalchemy.orm import Session
+from sqlalchemy import desc
+
+class Studio(DB):
+    @property
+    def user_feedbacks(self):
+        sql = """SELECT mf.conversation_id,
+                        mf.content,
+                        m.query,
+                        m.answer
+                 FROM message_feedbacks mf
+                          LEFT JOIN messages m ON mf.message_id = m.id
+                 WHERE mf.from_source = 'user'"""
+        return self.get_dict(sql)
+
+    @property
+    def apps(self): return self.get_dict("select id, name, mode from apps where status = 'normal'")
+
+    def app_config(self, app_id) -> AppModelConfig | None:
+        with Session(self.client) as session:
+            return (
+                session.query(AppModelConfig)
+                .filter(AppModelConfig.app_id == app_id)
+                .order_by(desc(AppModelConfig.created_at))
+                .first()
+            )
+
+    def update_app_config(self, record: AppModelConfig, refresh: bool = False) -> AppModelConfig | None:
+        with Session(self.client) as session:
+            session.add(record)
+            session.commit()
+            if refresh:
+                session.refresh(record)
+                return record
+            return None

davidkhala/ai/agent/dify/ops/db/knowledge.py

@@ -0,0 +1,52 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+from davidkhala.ai.agent.dify.ops.db.orm import Graph
+
+
+class Dataset(DB):
+
+    def dataset_queries(self, dataset_id, limit=20) -> list[str]:
+        template = "select content from dataset_queries where source = 'app' and created_by_role = 'end_user' and dataset_id = :dataset_id limit :limit"
+        return self.query(template, {'dataset_id': dataset_id, 'limit': limit}).scalars().all()
+
+    @property
+    def datasets(self):
+        template = "select id, name, description, indexing_technique, index_struct, embedding_model, embedding_model_provider, collection_binding_id, retrieval_model, icon_info, runtime_mode, pipeline_id, chunk_structure from datasets"
+        return self.get_dict(template)
+
+    def is_pipeline(self, id: str):
+        template = "select runtime_mode = 'rag_pipeline' from datasets where id = :id"
+        return self.query(template, {'id': id}).scalar()
+
+    @property
+    def data_source_credentials(self):
+        template = "select id, name, plugin_id, auth_type from datasource_providers"
+        return self.get_dict(template)
+
+    def credential_id_by(self, name, provider) -> list[str]:
+        template = "select id from datasource_providers where name = :name and provider = :provider"
+        return self.query(template, {'name': name, 'provider': provider}).scalars().all()
+
+
+class Document(DB):
+    def hit_documents(self, top_k: int = 3):
+        template = "SELECT dataset_id, document_id, content FROM document_segments ORDER BY hit_count DESC LIMIT :top_k"
+        return self.get_dict(template, {'top_k': top_k})
+
+    def id_by(self, name) -> list[str]:
+        """multiple ids can be found"""
+        template = "select id from documents where name = :name"
+        return [str(uuid) for uuid in self.query(template, {'name': name}).scalars().all()]
+
+
+class Pipeline(DB):
+    @property
+    def pipelines(self):
+        """unique syntax for pgsql"""
+        template = "SELECT DISTINCT ON (app_id) app_id, graph, rag_pipeline_variables FROM workflows where type = 'rag-pipeline' ORDER BY app_id, created_at DESC"
+        return Graph.convert(*self.get_dict(template))
+
+    def pipeline(self, app_id):
+        template = "select id, graph, rag_pipeline_variables from workflows where type = 'rag-pipeline' and app_id = :app_id"
+        dict_result = self.get_dict(template, {'app_id': app_id})
+        assert len(dict_result) < 2
+        return Graph.convert(*dict_result)

davidkhala/ai/agent/dify/ops/db/orm.py

@@ -0,0 +1,151 @@
+import json
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel
+from sqlalchemy import Column, String, Text, JSON, TIMESTAMP, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class DifyBase(Base):
+    __abstract__ = True  # keyword for SQLAlchemy
+    id = Column(UUID(as_uuid=True), primary_key=True, server_default=func.uuid_generate_v4())
+
+
+class AppModelConfig(DifyBase):
+    __tablename__ = "app_model_configs"
+    __table_args__ = {"schema": "public"}
+
+    app_id = Column(UUID(as_uuid=True), nullable=False)
+
+    provider = Column(String(255))
+    model_id = Column(String(255))
+    configs = Column(JSON)
+
+    created_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
+    updated_at = Column(TIMESTAMP, nullable=False, server_default=func.current_timestamp())
+
+    opening_statement = Column(Text)
+    suggested_questions = Column(Text)
+    suggested_questions_after_answer = Column(Text)
+    more_like_this = Column(Text)
+    model = Column(Text)
+    user_input_form = Column(Text)
+    pre_prompt = Column(Text)
+    agent_mode = Column(Text)
+    speech_to_text = Column(Text)
+    sensitive_word_avoidance = Column(Text)
+    retriever_resource = Column(Text)
+
+    dataset_query_variable = Column(String(255))
+    prompt_type = Column(String(255), nullable=False, server_default="simple")
+
+    chat_prompt_config = Column(Text)
+    completion_prompt_config = Column(Text)
+    dataset_configs = Column(Text)
+    external_data_tools = Column(Text)
+    file_upload = Column(Text)
+    text_to_speech = Column(Text)
+
+    created_by = Column(UUID(as_uuid=True))
+    updated_by = Column(UUID(as_uuid=True))
+
+    def __repr__(self):
+        return f"<AppModelConfig(id={self.id}, app_id={self.app_id}, provider={self.provider}, model_id={self.model_id})>"
+
+
+class Position(BaseModel):
+    x: float
+    y: float
+
+
+class NodeData(BaseModel):
+    class Type(str, Enum):
+        SOURCE = 'datasource'
+        CHUNKER = 'knowledge-index'
+        TOOL = 'tool'
+
+    type: Type | str  # not limit to built-in types
+    title: str | None = None
+    selected: bool
+
+    # datasource
+    datasource_parameters: dict[str, Any] | None = None
+    datasource_configurations: dict[str, Any] | None = None
+    plugin_id: str | None = None
+    provider_type: str | None = None
+    provider_name: str | None = None
+    datasource_name: str | None = None
+    datasource_label: str | None = None
+    plugin_unique_identifier: str | None = None
+
+    # tool
+    tool_parameters: dict[str, Any] | None = None
+    tool_configurations: dict[str, Any] | None = None
+    tool_node_version: str | None = None
+    provider_id: str | None = None
+    provider_icon: str | None = None
+    tool_name: str | None = None
+    tool_label: str | None = None
+    tool_description: str | None = None
+    is_team_authorization: bool | None = None
+    paramSchemas: list[Any] | None = None
+    params: dict[str, Any] | None = None
+
+    # knowledge index
+    index_chunk_variable_selector: list[str] | None = None
+    keyword_number: int | None = None
+    retrieval_model: dict[str, Any] | None = None
+    chunk_structure: str | None = None
+    indexing_technique: str | None = None
+    embedding_model: str | None = None
+    embedding_model_provider: str | None = None
+
+
+class Node(BaseModel):
+    @property
+    def datasource_type(self): return self.data.provider_type
+    id: str
+    type: Literal['custom']
+    data: NodeData
+    position: Position
+    targetPosition: str | None = None
+    sourcePosition: str | None = None
+    positionAbsolute: Position | None = None
+    width: float | None = None
+    height: float | None = None
+    selected: bool
+
+
+class Edge(BaseModel):
+    id: str
+    type: str
+    source: str
+    target: str
+    sourceHandle: str | None = None
+    targetHandle: str | None = None
+    data: dict[str, Any] | None = None
+    zIndex: int | None = None
+
+
+class Viewport(BaseModel):
+    x: float
+    y: float
+    zoom: float
+
+
+class Graph(BaseModel):
+    nodes: list[Node]
+    edges: list[Edge]
+    viewport: Viewport
+
+    @property
+    def datasources(self):
+        return [node for node in self.nodes if node.data.type == NodeData.Type.SOURCE]
+
+    @staticmethod
+    def convert(*records: list[dict]):
+        return [{**record, "graph": Graph(**json.loads(record["graph"]))} for record in records]

davidkhala/ai/agent/dify/ops/db/sys.py

@@ -0,0 +1,6 @@
+from davidkhala.ai.agent.dify.ops.db import DB
+
+
+class Info(DB):
+    @property
+    def accounts(self): return self.get_dict("select name, email from accounts where status = 'active'")

davidkhala/ai/agent/dify/plugins/__init__.py

@@ -0,0 +1,14 @@
+from typing import Literal
+
+from pydantic import BaseModel
+
+class JsonEntry(BaseModel):
+    data: list
+
+class Output(BaseModel):
+    """Class for result of a Dify node"""
+    text: str
+    files: list
+    json: list[JsonEntry]
+class DataSourceTypeAware(BaseModel):
+    datasource_type: Literal["local_file", "online_document", "website_crawl"]

davidkhala/ai/agent/dify/plugins/file.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class FileModel(BaseModel):
+    name: str
+    size: int
+    type: str
+    extension: str
+    mime_type: str
+    transfer_method: str
+    url: str
+    related_id: str
+
+
+class DataSourceOutput(DataSourceTypeAware):
+    datasource_type:str = "local_file"
+    file: FileModel

davidkhala/ai/agent/dify/plugins/firecrawl.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from davidkhala.ai.agent.dify.plugins import DataSourceTypeAware
+
+
+class DataSourceInfo(BaseModel):
+    source_url: str
+    content: str
+    title: str
+    description: str
+
+
+class DataSourceOutput(DataSourceTypeAware, DataSourceInfo):
+    datasource_type: str = "website_crawl"
+
+
+class CredentialAware(BaseModel):
+    credential_id: str | None
+
+
+class Console(DataSourceOutput, CredentialAware):
+    pass

davidkhala/ai/agent/dify/plugins/jina.py

@@ -0,0 +1,4 @@
+from davidkhala.ai.agent.dify.plugins.firecrawl import DataSourceOutput as FirecrawlDataSourceOutput
+
+class DataSourceOutput(FirecrawlDataSourceOutput):
+    """so far they are the same"""

davidkhala/ai/openrouter/__init__.py

@@ -0,0 +1,35 @@
+from openrouter.errors import UnauthorizedResponseError
+
+from davidkhala.ai.model import AbstractClient
+from openrouter import OpenRouter
+
+
+class Client(AbstractClient):
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.client = OpenRouter(api_key)
+
+    def chat(self, *user_prompt, **kwargs):
+        r = self.client.chat.send(
+            model=self.model,
+            messages=[
+                *self.messages,
+                *[{'role': 'user', 'content': _} for _ in user_prompt]
+            ]
+        )
+        return [_.message.content for _ in r.choices]
+    def connect(self):
+        try:
+            self.client.models.list()
+            return True
+        except UnauthorizedResponseError:
+            return False
+
+
+class Admin:
+    def __init__(self, provisioning_key: str):
+        self.provisioning_key = provisioning_key
+        self.client = OpenRouter(provisioning_key)
+    @property
+    def keys(self):
+        return self.client.api_keys.list().data

{davidkhala_ai-0.1.5.dist-info → davidkhala_ai-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: davidkhala.ai
-Version: 0.1.5
+Version: 0.1.7
 Summary: misc AI modules
 Requires-Python: >=3.13
 Provides-Extra: ali
@@ -9,6 +9,10 @@ Provides-Extra: api
 Requires-Dist: davidkhala-utils[http-request]; extra == 'api'
 Provides-Extra: azure
 Requires-Dist: openai; extra == 'azure'
+Provides-Extra: dify
+Requires-Dist: davidkhala-databases[pg]; extra == 'dify'
+Requires-Dist: davidkhala-utils[http-request]; extra == 'dify'
+Requires-Dist: dify-plugin; extra == 'dify'
 Provides-Extra: google
 Requires-Dist: google-adk; extra == 'google'
 Requires-Dist: google-genai; extra == 'google'
@@ -19,8 +23,10 @@ Requires-Dist: onnx; extra == 'hf'
 Requires-Dist: onnxruntime; extra == 'hf'
 Provides-Extra: langchain
 Requires-Dist: langchain; extra == 'langchain'
-Requires-Dist: langchain-openai; extra == 'langchain'
+Requires-Dist: langchain-openai; (python_version < '3.14') and extra == 'langchain'
 Requires-Dist: langgraph; extra == 'langchain'
+Provides-Extra: openrouter
+Requires-Dist: openrouter; extra == 'openrouter'
 Provides-Extra: ragflow
 Requires-Dist: ragflow-sdk; extra == 'ragflow'
 Provides-Extra: telemetry
@@ -29,4 +35,5 @@ Description-Content-Type: text/markdown
 
 # davidkhala.ai
 
-For usage of `azure.ai.agents`, goto https://github.com/davidkhala/azure-utils/tree/main/py 
+- For usage of `azure.ai.agents`, goto https://github.com/davidkhala/azure-utils/tree/main/py
+- [openrouter python sdk](https://openrouter.ai/docs/sdks/python) 

davidkhala_ai-0.1.7.dist-info/RECORD

@@ -0,0 +1,43 @@
+davidkhala/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/model.py,sha256=1wcXC8X8oqerMatlcPbZmuxZ-nJWdJKmaDSDgiGlUGw,647
+davidkhala/ai/opik.py,sha256=YU1XuweMUAzUkhpjxhltt-SBBDBkR3z-PCNo0DqzBRs,39
+davidkhala/ai/agent/README.md,sha256=kIPsx3gOjrpOw7w2qhNEALuCEQkuh4nYp6uBnijdvHE,178
+davidkhala/ai/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/agent/langgraph.py,sha256=jrc_Yvgo7eJjd3y5UJn0t1FzpnObDGYscwgsuVl2O_I,1052
+davidkhala/ai/agent/ragflow.py,sha256=UaK31us6V0NhAPCthGo07rQsm72vlR-McmihC_NDe1g,273
+davidkhala/ai/agent/dify/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/agent/dify/common.py,sha256=N7pq4MSrP9TebsarsodY4SyqS5iv20uX_pieplTpHaE,751
+davidkhala/ai/agent/dify/api/__init__.py,sha256=9-8OesuXF_wPmPrh_gEZpEZP51dcZxb0i6ixOBYKcwQ,876
+davidkhala/ai/agent/dify/api/app.py,sha256=y1mILC-fvQpeH50ASbFBluD9tFAwXu_IWwtwucMV5jM,3801
+davidkhala/ai/agent/dify/api/knowledge.py,sha256=1_NIPZlYI8040abrRKiX83hAHTz7vZaKQMbD3BnpO2c,5959
+davidkhala/ai/agent/dify/ops/__init__.py,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
+davidkhala/ai/agent/dify/ops/console/__init__.py,sha256=-a81jgCJ3s2B3i1GQ7ge1aZRfbvlALwGDHVu_GEET-A,237
+davidkhala/ai/agent/dify/ops/console/knowledge.py,sha256=lbOX28slJXAoD4r6ClV_sdNuO0tlsAa3jmgl4pVDeug,5699
+davidkhala/ai/agent/dify/ops/console/session.py,sha256=Kt8vzZJUlyqD8G8_OsrOD-WQwyDor8tqNwV0jJ738wE,902
+davidkhala/ai/agent/dify/ops/db/__init__.py,sha256=HYfJEnoFAoJJck2xvTDYx8zpw9Qao7sHXOGvW0diPqw,517
+davidkhala/ai/agent/dify/ops/db/app.py,sha256=OfpxPJlHKnhzxBJ1sJOhtMMoqWCKxw8yo4GCHp_0nh0,1389
+davidkhala/ai/agent/dify/ops/db/knowledge.py,sha256=pdYlW8dWR_RM56mwNDOjsCJ_lAstnNdS6OuMCuS4gW0,2520
+davidkhala/ai/agent/dify/ops/db/orm.py,sha256=CnZj8mV2RZhw_7hF1YICTUjROQ66hR5_8OCMQvtujnY,4575
+davidkhala/ai/agent/dify/ops/db/sys.py,sha256=U_qqopUMlgsilhHaG_ids6gtd-pNiR_Jm0kAr9hIL7M,188
+davidkhala/ai/agent/dify/plugins/__init__.py,sha256=XsLkLj0rqrNqS0nspAzC3FudVWBSD-iBOPPw8g-Qysc,356
+davidkhala/ai/agent/dify/plugins/file.py,sha256=o-HjHSFwRTNIYs8IxqZUSnBbh-xr8f-xMUM3iU9wCCQ,390
+davidkhala/ai/agent/dify/plugins/firecrawl.py,sha256=lB_f8W_bdg-7PeBKmF0-HdwYyakV_0D3nET5iT-Z1KM,460
+davidkhala/ai/agent/dify/plugins/jina.py,sha256=dQ5iJxDLWtChXb1IjCtsHctgUtgjOiDfWOuR2u0aUIM,190
+davidkhala/ai/ali/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/ali/dashscope.py,sha256=SZIzRhVHlLx3s5I2RNUh2-u8OoSdrbvoN5e1k8Mh8N0,1943
+davidkhala/ai/api/__init__.py,sha256=q2Ro5nhW5kJx2CYR1MRVamjTT5tTexPZwhrS2hwAvFM,1319
+davidkhala/ai/api/openrouter.py,sha256=khccJr5cBnudFy6Jc2O3A1TNCuHH_5W6Q2tXrkwlUYE,2308
+davidkhala/ai/api/siliconflow.py,sha256=JbnOSv8LJLtwYSNNB8_SMBMQzOgHDtQYZKA9A2BC4sY,2139
+davidkhala/ai/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+davidkhala/ai/google/adk.py,sha256=QwxYoOzT2Hol03V4NM0PF_HAzUGb4fB18VUAYacYbAY,657
+davidkhala/ai/google/gemini.py,sha256=Xf4HDOOcK4-jEBERzuLnQNFsU61P2fFx4K0z-ijvNHE,214
+davidkhala/ai/huggingface/BAAI.py,sha256=LZ9kp5Gfql4UzuTn4osyekI6VV1H3RIfED2IolXFj5c,341
+davidkhala/ai/huggingface/__init__.py,sha256=FJyU8eOfWQWKAvkIa5qwubF9ghsSQ8C0e6p6DKyomgs,521
+davidkhala/ai/huggingface/inference.py,sha256=bYN0PtLF2CaIHzdTP4LaTALJhcawvuLnLR7rhMVqwDE,333
+davidkhala/ai/openai/__init__.py,sha256=GXzWaw2ER3YFGHG6TPD9SmAHV6Tpsnqxj6tXlaWsrko,1897
+davidkhala/ai/openai/azure.py,sha256=QR1uZj8qAyhpCjo3Ks5zNV8GfOp3-enyZs6fBvV-MkA,1110
+davidkhala/ai/openai/native.py,sha256=MB0nDnzCOj_M42RMhdK3HTMVnxGnwpLT2GeLwSrepwI,704
+davidkhala/ai/openrouter/__init__.py,sha256=P8UvolZihN_CVBQ7BT1Fb6mSMFEQLyLY9G5bBDZhC0o,1037
+davidkhala_ai-0.1.7.dist-info/METADATA,sha256=uSa6yBpeofpMY7jovYnv71Gyvq9B0Btva6u5PdPKnXE,1458
+davidkhala_ai-0.1.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+davidkhala_ai-0.1.7.dist-info/RECORD,,

{davidkhala_ai-0.1.5.dist-info → davidkhala_ai-0.1.7.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

davidkhala/ai/agent/dify/base.py

@@ -1,7 +0,0 @@
-from davidkhala.utils.http_request import Request
-
-class API(Request):
-    def __init__(self, api_key: str, base_url="https://api.dify.ai/v1"):
-        super().__init__({'bearer': api_key})
-        self.base_url = base_url
-        self.api_key = api_key

davidkhala_ai-0.1.5.dist-info/RECORD

@@ -1,27 +0,0 @@
-davidkhala/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/model.py,sha256=1wcXC8X8oqerMatlcPbZmuxZ-nJWdJKmaDSDgiGlUGw,647
-davidkhala/ai/opik.py,sha256=YU1XuweMUAzUkhpjxhltt-SBBDBkR3z-PCNo0DqzBRs,39
-davidkhala/ai/agent/README.md,sha256=kIPsx3gOjrpOw7w2qhNEALuCEQkuh4nYp6uBnijdvHE,178
-davidkhala/ai/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/langgraph.py,sha256=jrc_Yvgo7eJjd3y5UJn0t1FzpnObDGYscwgsuVl2O_I,1052
-davidkhala/ai/agent/ragflow.py,sha256=UaK31us6V0NhAPCthGo07rQsm72vlR-McmihC_NDe1g,273
-davidkhala/ai/agent/dify/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/dify/base.py,sha256=V4BIhG9oTg9ayPZnYVxxA1Yobl2C4ukSWEOu2U_l0Tw,259
-davidkhala/ai/agent/dify/knowledge.py,sha256=jWx-DmYLvK8ZJ7dMuayKAKnOMW2uehpFJu6jnj-grf8,6328
-davidkhala/ai/ali/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/ali/dashscope.py,sha256=SZIzRhVHlLx3s5I2RNUh2-u8OoSdrbvoN5e1k8Mh8N0,1943
-davidkhala/ai/api/__init__.py,sha256=q2Ro5nhW5kJx2CYR1MRVamjTT5tTexPZwhrS2hwAvFM,1319
-davidkhala/ai/api/openrouter.py,sha256=khccJr5cBnudFy6Jc2O3A1TNCuHH_5W6Q2tXrkwlUYE,2308
-davidkhala/ai/api/siliconflow.py,sha256=JbnOSv8LJLtwYSNNB8_SMBMQzOgHDtQYZKA9A2BC4sY,2139
-davidkhala/ai/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/google/adk.py,sha256=QwxYoOzT2Hol03V4NM0PF_HAzUGb4fB18VUAYacYbAY,657
-davidkhala/ai/google/gemini.py,sha256=Xf4HDOOcK4-jEBERzuLnQNFsU61P2fFx4K0z-ijvNHE,214
-davidkhala/ai/huggingface/BAAI.py,sha256=LZ9kp5Gfql4UzuTn4osyekI6VV1H3RIfED2IolXFj5c,341
-davidkhala/ai/huggingface/__init__.py,sha256=FJyU8eOfWQWKAvkIa5qwubF9ghsSQ8C0e6p6DKyomgs,521
-davidkhala/ai/huggingface/inference.py,sha256=bYN0PtLF2CaIHzdTP4LaTALJhcawvuLnLR7rhMVqwDE,333
-davidkhala/ai/openai/__init__.py,sha256=GXzWaw2ER3YFGHG6TPD9SmAHV6Tpsnqxj6tXlaWsrko,1897
-davidkhala/ai/openai/azure.py,sha256=QR1uZj8qAyhpCjo3Ks5zNV8GfOp3-enyZs6fBvV-MkA,1110
-davidkhala/ai/openai/native.py,sha256=MB0nDnzCOj_M42RMhdK3HTMVnxGnwpLT2GeLwSrepwI,704
-davidkhala_ai-0.1.5.dist-info/METADATA,sha256=O1e0C_WHO5hAtEgn2o_-lVoVItJk_AYP5vfWe_M3DWs,1098
-davidkhala_ai-0.1.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-davidkhala_ai-0.1.5.dist-info/RECORD,,