logdetective 0.4.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

logdetective/server/database/models.py

@@ -1,88 +0,0 @@
-import enum
-import datetime
-
-from typing import Optional
-from sqlalchemy import Column, Integer, Float, DateTime, String, Enum
-
-from logdetective.server.database.base import Base, transaction
-
-
-class EndpointType(enum.Enum):
-    """Different analyze endpoints"""
-
-    ANALYZE = "analyze_log"
-    ANALYZE_STAGED = "analyze_log_staged"
-    ANALYZE_STREAM = "analyze_log_stream"
-
-
-class AnalyzeRequestMetrics(Base):
-    """Store data related to received requests and given responses"""
-
-    __tablename__ = "analyze_request_metrics"
-
-    id = Column(Integer, primary_key=True)
-    endpoint = Column(
-        Enum(EndpointType),
-        nullable=False,
-        index=True,
-        comment="The service endpoint that was called",
-    )
-    request_received_at = Column(
-        DateTime,
-        nullable=False,
-        index=True,
-        default=datetime.datetime.now(datetime.timezone.utc),
-        comment="Timestamp when the request was received",
-    )
-    log_url = Column(
-        String,
-        nullable=False,
-        index=False,
-        comment="Log url for which analysis was requested",
-    )
-    response_sent_at = Column(
-        DateTime, nullable=True, comment="Timestamp when the response was sent back"
-    )
-    response_length = Column(
-        Integer, nullable=True, comment="Length of the response in chars"
-    )
-    response_certainty = Column(
-        Float, nullable=True, comment="Certainty for generated response"
-    )
-
-    @classmethod
-    def create(
-        cls,
-        endpoint: EndpointType,
-        log_url: str,
-        request_received_at: Optional[datetime.datetime] = None,
-    ) -> int:
-        """Create AnalyzeRequestMetrics new line
-        with data related to a received request"""
-        with transaction(commit=True) as session:
-            metrics = AnalyzeRequestMetrics()
-            metrics.endpoint = endpoint
-            metrics.request_received_at = request_received_at or datetime.datetime.now(
-                datetime.timezone.utc
-            )
-            metrics.log_url = log_url
-            session.add(metrics)
-            session.flush()
-            return metrics.id
-
-    @classmethod
-    def update(
-        cls,
-        id_: int,
-        response_sent_at: datetime,
-        response_length: int,
-        response_certainty: float,
-    ) -> None:
-        """Update an AnalyzeRequestMetrics line
-        with data related to the given response"""
-        with transaction(commit=True) as session:
-            metrics = session.query(AnalyzeRequestMetrics).filter_by(id=id_).first()
-            metrics.response_sent_at = response_sent_at
-            metrics.response_length = response_length
-            metrics.response_certainty = response_certainty
-            session.add(metrics)

logdetective-0.4.0.dist-info/METADATA

@@ -1,333 +0,0 @@
-Metadata-Version: 2.3
-Name: logdetective
-Version: 0.4.0
-Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
-License: Apache-2.0
-Author: Jiri Podivin
-Author-email: jpodivin@gmail.com
-Requires-Python: >=3.11,<4.0
-Classifier: Development Status :: 4 - Beta
-Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Topic :: Internet :: Log Analysis
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Software Development :: Debuggers
-Provides-Extra: server
-Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server"
-Requires-Dist: drain3 (>=0.9.11,<0.10.0)
-Requires-Dist: fastapi (>=0.111.1) ; extra == "server"
-Requires-Dist: huggingface-hub (>0.23.2)
-Requires-Dist: llama-cpp-python (>0.2.56,!=0.2.86)
-Requires-Dist: numpy (>=1.26.0)
-Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "server"
-Requires-Dist: pydantic (>=2.8.2,<3.0.0) ; extra == "server"
-Requires-Dist: python-gitlab (>=4.4.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0) ; extra == "server"
-Requires-Dist: requests (>0.2.31)
-Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server"
-Project-URL: homepage, https://github.com/fedora-copr/logdetective
-Project-URL: issues, https://github.com/fedora-copr/logdetective/issues
-Description-Content-Type: text/markdown
-
-Log Detective
-=============
-
-[![PyPI - Version](https://img.shields.io/pypi/v/logdetective?color=blue)][PyPI Releases]
-
-[PyPI Releases]: https://pypi.org/project/logdetective/#history
-
-A Python tool to analyze logs using a Language Model (LLM) and Drain template miner.
-
-Installation
-------------
-
-**Fedora 40+**
-
-    dnf install logdetective
-
-**From Pypi repository**
-
-The logdetective project is published on the the the the the [Pypi repository](https://pypi.org/project/logdetective/). The `pip` tool can be used for installation.
-
-First, ensure that the necessary dependencies for the `llama-cpp-python` project are installed. For Fedora, install `gcc-c++`:
-
-    # for Fedora it will be:
-    dnf install gcc-c++
-
-Then, install the `logdetective` project using pip:
-
-    # then install logdetective project
-    pip install logdetective
-
-**Local repository install**
-
-    pip install .
-
-Usage
------
-
-To analyze a log file, run the script with the following command line arguments:
-- `url` (required): The URL of the log file to be analyzed.
-- `--model` (optional, default: "Mistral-7B-Instruct-v0.2-GGUF"): The path or URL of the language model for analysis. As we are using LLama.cpp we want this to be in the `gguf` format. You can include the download link to the model here. If the model is already on your machine it will skip the download.
-- `--summarizer` (optional, default: "drain"): Choose between LLM and Drain template miner as the log summarizer. You can also provide the path to an existing language model file instead of using a URL.
-- `--n_lines` (optional, default: 8): The number of lines per chunk for LLM analysis. This only makes sense when you are summarizing with LLM.
-- `--n_clusters` (optional, default 8): Number of clusters for Drain to organize log chunks into. This only makes sense when you are summarizing with Drain
-
-Example usage:
-
-    logdetective https://example.com/logs.txt
-
-Or if the log file is stored locally:
-
-    logdetective ./data/logs.txt
-
-Example you want to use a different model:
-
-    logdetective https://example.com/logs.txt --model https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_S.gguf?download=true
-    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF
-
-Note that streaming with some models (notably Meta-Llama-3 is broken) is broken and can be workarounded by `no-stream` option:
-
-    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --no-stream
-
-
-Real Example
-------------
-Let's have a look at a real world example. Log Detective can work with any logs though we optimize it for build logs.
-
-We're going to analyze a failed build of a python-based library that happened in Fedora Koji buildsystem:
-```
-$ logdetective https://kojipkgs.fedoraproject.org//work/tasks/8157/117788157/build.log
-Explanation:
-[Child return code was: 0] : The rpm build process executed successfully without any errors until the 'check' phase.
-
-[wamp/test/test_wamp_component_aio.py::test_asyncio_component] : Pytest found
-two tests marked with '@pytest.mark.asyncio' but they are not async functions.
-This warning can be ignored unless the tests are intended to be run
-asynchronously.
-
-[wamp/test/test_wamp_component_aio.py::test_asyncio_component_404] : Another
-Pytest warning for the same issue as test_asyncio_component.
-
-[-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html] :
-This line is not related to the error, but it is a reminder to refer to Pytest
-documentation for handling warnings.
-
-[=========================== short test summary info
-============================] : This section shows the summary of tests that
-were executed. It shows the number of tests passed, failed, skipped,
-deselected, and warnings.
-
-[FAILED wamp/test/test_wamp_cryptosign.py::TestSigVectors::test_vectors] : A
-failed test is reported with the name of the test file, the name of the test
-method, and the name of the test case that failed. In this case,
-TestSigVectors::test_vectors failed.
-
-[FAILED
-websocket/test/test_websocket_protocol.py::WebSocketClientProtocolTests::test_auto_ping]
-: Another failed test is reported with the same format as the previous test. In
-this case, it is WebSocketClientProtocolTests::test_auto_ping that failed.
-
-[FAILED websocket/test/test_websocket_protocol.py::WebSocketServerProtocolTests::test_interpolate_server_status_template]
-: A third failed test is reported with the same format as the previous tests.
-In this case, it is
-WebSocketServerProtocolTests::test_interpolate_server_status_template that
-failed.
-
-[FAILED websocket/test/test_websocket_protocol.py::WebSocketServerProtocolTests::test_sendClose_reason_with_no_code]
-: Another failed test is reported. This time it is
-WebSocketServerProtocolTests::test_sendClose_reason_with_no_code.
-
-[FAILED websocket/test/test_websocket_protocol.py::WebSocketServerProtocolTests::test_sendClose_str_reason]
-: Another failed test is reported with the same test file and test method name,
-but a different test case name: test_sendClose_str_reason.
-
-[==== 13 failed, 195 passed, 64 skipped, 13 deselected, 2 warnings in 6.55s
-=====] : This is the summary of all tests that were executed, including the
-number of tests that passed, failed, were skipped, deselected, or produced
-warnings. In this case, there were 13 failed tests among a total of 211 tests.
-
-[error: Bad exit status from /var/tmp/rpm-tmp.8C0L25 (%check)] : An error
-message is reported indicating that the 'check' phase of the rpm build process
-failed with a bad exit status.
-```
-
-It looks like a wall of text. Similar to any log. The main difference is that here we have the most significant lines of a logfile wrapped in `[ ] : ` and followed by textual explanation of the log text done by mistral 7b.
-
-
-Contributing
-------------
-
-Contributions are welcome! Please submit a pull request if you have any improvements or new features to add. Make sure your changes pass all existing tests before submitting.
-
-To develop logdetective, you should fork this repository, clone your fork, and install dependencies using pip:
-
-    git clone https://github.com/yourusername/logdetective.git
-    cd logdetective
-    pip install .
-
-Make changes to the code as needed and run pre-commit.
-
-Tests
------
-
-The [tox](https://github.com/tox-dev/tox) is used to manage tests. Please install `tox` package into your distribution and run:
-
-    tox
-
-This will create a virtual environment with dependencies and run all the tests. For more information follow the tox help.
-
-To run only a specific test execute this:
-
-    tox run -e style # to run flake8
-
-or
-
-    tox run -e lint # to run pylint
-
-Visual Studio Code testing with podman/docker-compose
------------------------------------------------------
-
-- In `Containerfile`, add `debugpy` as a dependency
-
-```diff
--RUN pip3 install llama_cpp_python==0.2.85 sse-starlette starlette-context \
-+RUN pip3 install llama_cpp_python==0.2.85 sse-starlette starlette-context debugpy\
-```
-
-- Rebuild server image with new dependencies
-
-```
-make rebuild-server
-```
-
-- Forward debugging port in `docker-compose.yaml` for `server` service.
-
-```diff
-     ports:
-       - "${LOGDETECTIVE_SERVER_PORT:-8080}:${LOGDETECTIVE_SERVER_PORT:-8080}"
-+      - "${VSCODE_DEBUG_PORT:-5678}:${VSCODE_DEBUG_PORT:-5678}"
-```
-
-- Add `debugpy` code in a logdetective file where you want to stop at first.
-
-```diff
-+import debugpy
-+debugpy.listen(("0.0.0.0", 5678))
-+debugpy.wait_for_client()
-```
-
-- Prepare `.vscode/lunch.json` configuration for Visual Studio Code (at least the following configuration is needed)
-
-```json
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Python Debugger: Remote Attach",
-      "type": "debugpy",
-      "request": "attach",
-      "connect": {
-        "host": "localhost",
-        "port": 5678
-      },
-      "pathMappings": [
-        {
-          "localRoot": "${workspaceFolder}",
-          "remoteRoot": "/src"
-        }
-      ]
-    }
-  ]
-}
-```
-
-- Run the server
-
-```
-podman-compose up server
-```
-
-- Run Visual Stdio Code debug configuration named *Python Debug: Remote Attach*
-
-Server
-------
-
-FastApi based server is implemented in `logdetective/server.py`. In order to run it in a development mode,
-simply start llama-cpp-python server with your chosen model as described in llama-cpp-python [docs](https://llama-cpp-python.readthedocs.io/en/latest/server/#running-the-server).
-
-Afterwards, start the logdetective server with `fastapi dev logdetective/server.py --port 8080`.
-Requests can then be made with post requests, for example:
-
-    curl --header "Content-Type: application/json" --request POST --data '{"url":"<YOUR_URL_HERE>"}' http://localhost:8080/analyze
-
-For more accurate responses, you can use `/analyze/staged` endpoint. This will submit snippets to model for individual analysis first.
-Afterwards the model outputs are used to construct final prompt. This will take substantially longer, compared to plain `/analyze`
-
-    curl --header "Content-Type: application/json" --request POST --data '{"url":"<YOUR_URL_HERE>"}' http://localhost:8080/analyze/staged
-
-We also have a Containerfile and composefile to run the logdetective server and llama server in containers.
-
-Before doing `podman-compose up`, make sure to set `MODELS_PATH` environment variable and point to a directory with your local model files:
-```
-$ export MODELS_PATH=/path/to/models/
-$ ll $MODELS_PATH
--rw-r--r--. 1 tt tt 3.9G apr 10 17:18  mistral-7b-instruct-v0.2.Q4_K_S.gguf
-```
-
-If the variable is not set, `./models` is mounted inside by default.
-
-Model can be downloaded from [our Hugging Space](https://huggingface.co/fedora-copr) by:
-```
-$ curl -L -o models/mistral-7b-instruct-v0.2.Q4_K_S.gguf https://huggingface.co/fedora-copr/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/ggml-model-Q4_K_S.gguf
-```
-
-Generate a new database revision with alembic
----------------------------------------------
-
-Modify the database models (`logdetective/server/database/model.py).
-
-Generate a new database revision with the command:
-
-**Warning**: this command will start up a new server
-and shut it down when the operation completes.
-
-```
-CHANGE="A change comment" make alembic-generate-revision
-```
-
-Our production instance
------------------------
-
-Our FastAPI server and model inference server run through `podman-compose` on an
-Amazon AWS intance. The VM is provisioned by an
-[ansible playbook](https://pagure.io/fedora-infra/ansible/blob/main/f/roles/logdetective/tasks/main.yml).
-
-You can control the server through:
-
-```
-cd /root/logdetective
-podman-compose -f docker-compose-prod.yaml ...
-```
-
-The `/root` directory contains valuable data. If moving to a new instance,
-please backup the whole directory and transfer it to the new instance.
-
-Fore some reason, we need to manually run this command after every reboot:
-
-```
-nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
-```
-
-
-License
--------
-
-This project is licensed under the Apache-2.0 License - see the LICENSE file for details.
-

logdetective-0.4.0.dist-info/RECORD

@@ -1,19 +0,0 @@
-logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
-logdetective/constants.py,sha256=SPSs1Bq6zPms3RsFTmsADwgrnFTn4fefNHzrB-M3RAE,1383
-logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
-logdetective/extractors.py,sha256=cjxndfJaQur54GXksIQXL7YTxkOng8I8UnQZMN2t5_w,3388
-logdetective/logdetective.py,sha256=KN0KASW63VAnrjVeXK5AO0ob-vSexutTyeg1fd4uj70,4884
-logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/base.py,sha256=oMJUvbWeapIUP-8Cf_DR9ptFg8CsYeaBAIjOVEzx8SM,1668
-logdetective/server/database/models.py,sha256=8jW4k03Kny_3ld35214hcjYoJqlBvQIr4LH9mfQukXw,2750
-logdetective/server/metric.py,sha256=VYMifrfIhcqgyu6YYN0c1nt8fC1iJ2_LCB7Bh2AheoE,2679
-logdetective/server/models.py,sha256=cTmNJ-w_WT3meD_x0A92wCZ5f6UiSeYpH1f01PNCOy4,5533
-logdetective/server/server.py,sha256=fKGN6ddlLW25V5X0O-NBGHTBlDszRU6R8FpJs0xEYe8,21179
-logdetective/server/utils.py,sha256=osW5-VXxJAxRt7Wd3t1wF7PyW89FE9g4gSZLZCShlLc,1216
-logdetective/utils.py,sha256=eudens1_T6iTtYhyzoYCpwuWgFHUMDSt6eWnrAB-mAI,6188
-logdetective-0.4.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-0.4.0.dist-info/METADATA,sha256=92vGsbePzgQKz1PHOVmdysHvSkgV0Yj0Owtjdzv0G2w,12714
-logdetective-0.4.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-logdetective-0.4.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-0.4.0.dist-info/RECORD,,