eyecore 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eyecore-1.0.0/.github/workflows/ci.yml +87 -0
- eyecore-1.0.0/.gitignore +12 -0
- eyecore-1.0.0/Cargo.lock +133 -0
- eyecore-1.0.0/Cargo.toml +12 -0
- eyecore-1.0.0/LICENSE +21 -0
- eyecore-1.0.0/PKG-INFO +111 -0
- eyecore-1.0.0/README.md +67 -0
- eyecore-1.0.0/pyproject.toml +50 -0
- eyecore-1.0.0/src/eyecore/__init__.py +105 -0
- eyecore-1.0.0/src/eyecore/_compress.py +44 -0
- eyecore-1.0.0/src/eyecore/_core.pyi +3 -0
- eyecore-1.0.0/src/eyecore/_corpus.py +312 -0
- eyecore-1.0.0/src/eyecore/_db.py +64 -0
- eyecore-1.0.0/src/eyecore/_entity_db.py +291 -0
- eyecore-1.0.0/src/eyecore/_feed_report.py +217 -0
- eyecore-1.0.0/src/eyecore/_feed_scraper.py +195 -0
- eyecore-1.0.0/src/eyecore/_feed_store.py +185 -0
- eyecore-1.0.0/src/eyecore/_graph.py +195 -0
- eyecore-1.0.0/src/eyecore/_llm.py +221 -0
- eyecore-1.0.0/src/eyecore/corpus_registry.py +65 -0
- eyecore-1.0.0/src/lib.rs +50 -0
- eyecore-1.0.0/tests/__init__.py +0 -0
- eyecore-1.0.0/tests/conftest.py +19 -0
- eyecore-1.0.0/tests/test_compress.py +159 -0
- eyecore-1.0.0/tests/test_corpus.py +464 -0
- eyecore-1.0.0/tests/test_db.py +168 -0
- eyecore-1.0.0/tests/test_graph.py +348 -0
- eyecore-1.0.0/tests/test_llm.py +303 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
tags: ['v*']
|
|
7
|
+
pull_request:
|
|
8
|
+
branches: [main]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: '3.11'
|
|
18
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
19
|
+
- name: Build wheel
|
|
20
|
+
run: |
|
|
21
|
+
pip install maturin
|
|
22
|
+
maturin build --out dist
|
|
23
|
+
pip install --no-index --find-links dist eyecore
|
|
24
|
+
- name: Run tests
|
|
25
|
+
run: pip install pytest && pytest tests/ -q
|
|
26
|
+
|
|
27
|
+
build-wheels:
|
|
28
|
+
needs: test
|
|
29
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
30
|
+
strategy:
|
|
31
|
+
matrix:
|
|
32
|
+
include:
|
|
33
|
+
- os: ubuntu-latest
|
|
34
|
+
target: x86_64
|
|
35
|
+
- os: windows-latest
|
|
36
|
+
target: x86_64
|
|
37
|
+
- os: macos-latest
|
|
38
|
+
target: x86_64
|
|
39
|
+
- os: macos-latest
|
|
40
|
+
target: aarch64
|
|
41
|
+
runs-on: ${{ matrix.os }}
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
- uses: PyO3/maturin-action@v1
|
|
45
|
+
with:
|
|
46
|
+
target: ${{ matrix.target }}
|
|
47
|
+
args: --release --out dist --find-interpreter
|
|
48
|
+
sccache: 'true'
|
|
49
|
+
manylinux: auto
|
|
50
|
+
- uses: actions/upload-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: wheels-${{ matrix.os }}-${{ matrix.target }}
|
|
53
|
+
path: dist/
|
|
54
|
+
|
|
55
|
+
sdist:
|
|
56
|
+
needs: test
|
|
57
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
58
|
+
runs-on: ubuntu-latest
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/checkout@v4
|
|
61
|
+
- uses: PyO3/maturin-action@v1
|
|
62
|
+
with:
|
|
63
|
+
command: sdist
|
|
64
|
+
args: --out dist
|
|
65
|
+
- uses: actions/upload-artifact@v4
|
|
66
|
+
with:
|
|
67
|
+
name: sdist
|
|
68
|
+
path: dist/
|
|
69
|
+
|
|
70
|
+
publish:
|
|
71
|
+
needs: [build-wheels, sdist]
|
|
72
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
73
|
+
runs-on: ubuntu-latest
|
|
74
|
+
environment: pypi
|
|
75
|
+
permissions:
|
|
76
|
+
id-token: write
|
|
77
|
+
steps:
|
|
78
|
+
- uses: actions/download-artifact@v4
|
|
79
|
+
with:
|
|
80
|
+
pattern: wheels-*
|
|
81
|
+
merge-multiple: true
|
|
82
|
+
path: dist/
|
|
83
|
+
- uses: actions/download-artifact@v4
|
|
84
|
+
with:
|
|
85
|
+
name: sdist
|
|
86
|
+
path: dist/
|
|
87
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
eyecore-1.0.0/.gitignore
ADDED
eyecore-1.0.0/Cargo.lock
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "eyecore-core"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
dependencies = [
|
|
9
|
+
"pyo3",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
[[package]]
|
|
13
|
+
name = "heck"
|
|
14
|
+
version = "0.5.0"
|
|
15
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
16
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
17
|
+
|
|
18
|
+
[[package]]
|
|
19
|
+
name = "libc"
|
|
20
|
+
version = "0.2.186"
|
|
21
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
22
|
+
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
|
23
|
+
|
|
24
|
+
[[package]]
|
|
25
|
+
name = "once_cell"
|
|
26
|
+
version = "1.21.4"
|
|
27
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
28
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
29
|
+
|
|
30
|
+
[[package]]
|
|
31
|
+
name = "portable-atomic"
|
|
32
|
+
version = "1.13.1"
|
|
33
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
34
|
+
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
|
|
35
|
+
|
|
36
|
+
[[package]]
|
|
37
|
+
name = "proc-macro2"
|
|
38
|
+
version = "1.0.106"
|
|
39
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
40
|
+
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
|
41
|
+
dependencies = [
|
|
42
|
+
"unicode-ident",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[[package]]
|
|
46
|
+
name = "pyo3"
|
|
47
|
+
version = "0.28.3"
|
|
48
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
49
|
+
checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
|
|
50
|
+
dependencies = [
|
|
51
|
+
"libc",
|
|
52
|
+
"once_cell",
|
|
53
|
+
"portable-atomic",
|
|
54
|
+
"pyo3-build-config",
|
|
55
|
+
"pyo3-ffi",
|
|
56
|
+
"pyo3-macros",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
[[package]]
|
|
60
|
+
name = "pyo3-build-config"
|
|
61
|
+
version = "0.28.3"
|
|
62
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
63
|
+
checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
|
|
64
|
+
dependencies = [
|
|
65
|
+
"target-lexicon",
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
[[package]]
|
|
69
|
+
name = "pyo3-ffi"
|
|
70
|
+
version = "0.28.3"
|
|
71
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
72
|
+
checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
|
|
73
|
+
dependencies = [
|
|
74
|
+
"libc",
|
|
75
|
+
"pyo3-build-config",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[[package]]
|
|
79
|
+
name = "pyo3-macros"
|
|
80
|
+
version = "0.28.3"
|
|
81
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
82
|
+
checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
|
|
83
|
+
dependencies = [
|
|
84
|
+
"proc-macro2",
|
|
85
|
+
"pyo3-macros-backend",
|
|
86
|
+
"quote",
|
|
87
|
+
"syn",
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
[[package]]
|
|
91
|
+
name = "pyo3-macros-backend"
|
|
92
|
+
version = "0.28.3"
|
|
93
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
94
|
+
checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
|
|
95
|
+
dependencies = [
|
|
96
|
+
"heck",
|
|
97
|
+
"proc-macro2",
|
|
98
|
+
"pyo3-build-config",
|
|
99
|
+
"quote",
|
|
100
|
+
"syn",
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
[[package]]
|
|
104
|
+
name = "quote"
|
|
105
|
+
version = "1.0.45"
|
|
106
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
107
|
+
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
|
108
|
+
dependencies = [
|
|
109
|
+
"proc-macro2",
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
[[package]]
|
|
113
|
+
name = "syn"
|
|
114
|
+
version = "2.0.117"
|
|
115
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
116
|
+
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
|
117
|
+
dependencies = [
|
|
118
|
+
"proc-macro2",
|
|
119
|
+
"quote",
|
|
120
|
+
"unicode-ident",
|
|
121
|
+
]
|
|
122
|
+
|
|
123
|
+
[[package]]
|
|
124
|
+
name = "target-lexicon"
|
|
125
|
+
version = "0.13.5"
|
|
126
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
127
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
128
|
+
|
|
129
|
+
[[package]]
|
|
130
|
+
name = "unicode-ident"
|
|
131
|
+
version = "1.0.24"
|
|
132
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
133
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
eyecore-1.0.0/Cargo.toml
ADDED
eyecore-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrew Watts
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
eyecore-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eyecore
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Database
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Requires-Dist: requests>=2.28 ; extra == 'corpus'
|
|
16
|
+
Requires-Dist: feedparser>=6.0 ; extra == 'feed'
|
|
17
|
+
Requires-Dist: requests>=2.28 ; extra == 'feed'
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.12 ; extra == 'feed'
|
|
19
|
+
Requires-Dist: ollama>=0.1 ; extra == 'llm'
|
|
20
|
+
Requires-Dist: openai>=1.0 ; extra == 'llm'
|
|
21
|
+
Requires-Dist: llama-cpp-python>=0.2 ; extra == 'llm'
|
|
22
|
+
Requires-Dist: llama-cpp-python>=0.2 ; extra == 'llm-cpp'
|
|
23
|
+
Requires-Dist: ollama>=0.1 ; extra == 'llm-ollama'
|
|
24
|
+
Requires-Dist: openai>=1.0 ; extra == 'llm-openai'
|
|
25
|
+
Requires-Dist: pytest>=7.0 ; extra == 'test'
|
|
26
|
+
Requires-Dist: pytest-cov>=4.0 ; extra == 'test'
|
|
27
|
+
Provides-Extra: corpus
|
|
28
|
+
Provides-Extra: feed
|
|
29
|
+
Provides-Extra: llm
|
|
30
|
+
Provides-Extra: llm-cpp
|
|
31
|
+
Provides-Extra: llm-ollama
|
|
32
|
+
Provides-Extra: llm-openai
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Summary: Core data retrieval and organisation library — SQLite DB, topic graph, corpus management, and LLM integration
|
|
36
|
+
Keywords: database,knowledge-graph,corpus,llm
|
|
37
|
+
Author-email: Andrew Watts <andrewkwatts@gmail.com>
|
|
38
|
+
Requires-Python: >=3.10
|
|
39
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
40
|
+
Project-URL: Homepage, https://github.com/andrewkwatts-maker/EyeCore
|
|
41
|
+
Project-URL: Issues, https://github.com/andrewkwatts-maker/EyeCore/issues
|
|
42
|
+
Project-URL: Repository, https://github.com/andrewkwatts-maker/EyeCore
|
|
43
|
+
|
|
44
|
+
# eyecore
|
|
45
|
+
|
|
46
|
+
Core data retrieval and organisation library — SQLite DB management, topic graph, corpus management, and LLM integration.
|
|
47
|
+
|
|
48
|
+
## Features
|
|
49
|
+
|
|
50
|
+
- **BaseDB** — lazy SQLite connection with transparent `.db.gz` decompression to user cache
|
|
51
|
+
- **TopicGraph** — generalized topic registry with parent/child relationships, BFS traversal, and typed links
|
|
52
|
+
- **CorpusManager** — on-demand corpus checkout (Project Gutenberg, URL, git) with FTS5 indexing
|
|
53
|
+
- **LLMClient** — lazy-loaded LLM wrapper with auto-detected backends (Ollama, llama-cpp, OpenAI-compatible)
|
|
54
|
+
- **compress_db / decompress_to_cache** — platform-aware compression utilities for bake scripts
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install eyecore
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
With optional extras:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install "eyecore[llm-ollama]" # Ollama backend
|
|
66
|
+
pip install "eyecore[llm-cpp]" # llama-cpp-python backend
|
|
67
|
+
pip install "eyecore[llm-openai]" # OpenAI-compatible backend
|
|
68
|
+
pip install "eyecore[corpus]" # corpus download support
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Quick start
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
from eyecore import BaseDB, TopicGraph, CorpusManager, LLMClient
|
|
76
|
+
|
|
77
|
+
# Lazy SQLite — decompresses .db.gz to user cache on first access
|
|
78
|
+
db = BaseDB("myapp", gz_path=Path("data/myapp.db.gz"))
|
|
79
|
+
rows = db.fetchall("SELECT * FROM entities LIMIT 10")
|
|
80
|
+
|
|
81
|
+
# Topic graph
|
|
82
|
+
graph = TopicGraph(db.conn)
|
|
83
|
+
related = graph.get_related("topic-id")
|
|
84
|
+
tree = graph.subtree("root-id")
|
|
85
|
+
|
|
86
|
+
# LLM — auto-detects Ollama / llama-cpp / OpenAI
|
|
87
|
+
llm = LLMClient.get()
|
|
88
|
+
if llm.is_available():
|
|
89
|
+
summary = llm.summarize("Some long text to summarize...")
|
|
90
|
+
topics = llm.extract_topics("Article text about AI and machine learning...")
|
|
91
|
+
report = llm.generate_report(articles, "Technology", "title", "summary")
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## LLM configuration
|
|
95
|
+
|
|
96
|
+
Configure via environment variables:
|
|
97
|
+
|
|
98
|
+
| Variable | Default | Description |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `LLM_BACKEND` | auto | `ollama`, `llama-cpp`, or `openai` |
|
|
101
|
+
| `LLM_MODEL` | `llama3` | Model name (Ollama) or model ID (OpenAI) |
|
|
102
|
+
| `LLM_HOST` | `http://localhost:11434` | Ollama server URL |
|
|
103
|
+
| `LLM_MODEL_PATH` | — | Path to GGUF model file (llama-cpp) |
|
|
104
|
+
| `OPENAI_API_KEY` | — | API key for OpenAI-compatible endpoints |
|
|
105
|
+
| `OPENAI_BASE_URL` | — | Base URL for OpenAI-compatible endpoints |
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
## License
|
|
109
|
+
|
|
110
|
+
MIT — see [LICENSE](LICENSE)
|
|
111
|
+
|
eyecore-1.0.0/README.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# eyecore
|
|
2
|
+
|
|
3
|
+
Core data retrieval and organisation library — SQLite DB management, topic graph, corpus management, and LLM integration.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **BaseDB** — lazy SQLite connection with transparent `.db.gz` decompression to user cache
|
|
8
|
+
- **TopicGraph** — generalized topic registry with parent/child relationships, BFS traversal, and typed links
|
|
9
|
+
- **CorpusManager** — on-demand corpus checkout (Project Gutenberg, URL, git) with FTS5 indexing
|
|
10
|
+
- **LLMClient** — lazy-loaded LLM wrapper with auto-detected backends (Ollama, llama-cpp, OpenAI-compatible)
|
|
11
|
+
- **compress_db / decompress_to_cache** — platform-aware compression utilities for bake scripts
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install eyecore
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
With optional extras:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install "eyecore[llm-ollama]" # Ollama backend
|
|
23
|
+
pip install "eyecore[llm-cpp]" # llama-cpp-python backend
|
|
24
|
+
pip install "eyecore[llm-openai]" # OpenAI-compatible backend
|
|
25
|
+
pip install "eyecore[corpus]" # corpus download support
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick start
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from eyecore import BaseDB, TopicGraph, CorpusManager, LLMClient
|
|
33
|
+
|
|
34
|
+
# Lazy SQLite — decompresses .db.gz to user cache on first access
|
|
35
|
+
db = BaseDB("myapp", gz_path=Path("data/myapp.db.gz"))
|
|
36
|
+
rows = db.fetchall("SELECT * FROM entities LIMIT 10")
|
|
37
|
+
|
|
38
|
+
# Topic graph
|
|
39
|
+
graph = TopicGraph(db.conn)
|
|
40
|
+
related = graph.get_related("topic-id")
|
|
41
|
+
tree = graph.subtree("root-id")
|
|
42
|
+
|
|
43
|
+
# LLM — auto-detects Ollama / llama-cpp / OpenAI
|
|
44
|
+
llm = LLMClient.get()
|
|
45
|
+
if llm.is_available():
|
|
46
|
+
summary = llm.summarize("Some long text to summarize...")
|
|
47
|
+
topics = llm.extract_topics("Article text about AI and machine learning...")
|
|
48
|
+
report = llm.generate_report(articles, "Technology", "title", "summary")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## LLM configuration
|
|
52
|
+
|
|
53
|
+
Configure via environment variables:
|
|
54
|
+
|
|
55
|
+
| Variable | Default | Description |
|
|
56
|
+
|---|---|---|
|
|
57
|
+
| `LLM_BACKEND` | auto | `ollama`, `llama-cpp`, or `openai` |
|
|
58
|
+
| `LLM_MODEL` | `llama3` | Model name (Ollama) or model ID (OpenAI) |
|
|
59
|
+
| `LLM_HOST` | `http://localhost:11434` | Ollama server URL |
|
|
60
|
+
| `LLM_MODEL_PATH` | — | Path to GGUF model file (llama-cpp) |
|
|
61
|
+
| `OPENAI_API_KEY` | — | API key for OpenAI-compatible endpoints |
|
|
62
|
+
| `OPENAI_BASE_URL` | — | Base URL for OpenAI-compatible endpoints |
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
## License
|
|
66
|
+
|
|
67
|
+
MIT — see [LICENSE](LICENSE)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.4,<2.0"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "eyecore"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Core data retrieval and organisation library — SQLite DB, topic graph, corpus management, and LLM integration"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {file = "LICENSE"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Andrew Watts", email = "andrewkwatts@gmail.com"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["database", "knowledge-graph", "corpus", "llm"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 5 - Production/Stable",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"License :: OSI Approved :: MIT License",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Topic :: Database",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
llm-ollama = ["ollama>=0.1"]
|
|
32
|
+
llm-cpp = ["llama-cpp-python>=0.2"]
|
|
33
|
+
llm-openai = ["openai>=1.0"]
|
|
34
|
+
llm = ["ollama>=0.1", "openai>=1.0", "llama-cpp-python>=0.2"]
|
|
35
|
+
corpus = ["requests>=2.28"]
|
|
36
|
+
feed = ["feedparser>=6.0", "requests>=2.28", "beautifulsoup4>=4.12"]
|
|
37
|
+
test = ["pytest>=7.0", "pytest-cov>=4.0"]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/andrewkwatts-maker/EyeCore"
|
|
41
|
+
Repository = "https://github.com/andrewkwatts-maker/EyeCore"
|
|
42
|
+
Issues = "https://github.com/andrewkwatts-maker/EyeCore/issues"
|
|
43
|
+
|
|
44
|
+
[tool.maturin]
|
|
45
|
+
python-source = "src"
|
|
46
|
+
module-name = "eyecore._core"
|
|
47
|
+
features = ["pyo3/extension-module"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""
|
|
2
|
+
eyecore — Core data retrieval and organisation library.
|
|
3
|
+
|
|
4
|
+
Provides:
|
|
5
|
+
BaseDB — lazy SQLite connection with transparent gz decompression
|
|
6
|
+
TopicGraph — generalized topic registry with PK/FK parent/child links
|
|
7
|
+
CorpusManager — on-demand corpus download (Gutenberg / URL / git) + FTS indexing
|
|
8
|
+
LLMClient — lazy-loaded LLM wrapper (ollama / llama-cpp / openai-compatible)
|
|
9
|
+
EntityDB — base class for baked SQLite entity databases
|
|
10
|
+
cache_dir — platform-appropriate user cache directory
|
|
11
|
+
compress_db — compress a .db file to .db.gz
|
|
12
|
+
decompress_to_cache — decompress .db.gz to user cache on first use
|
|
13
|
+
|
|
14
|
+
Corpus registry (shared reference texts, usable by any module):
|
|
15
|
+
from eyecore.corpus_registry import ALL_CORPUSES, get_by_topic, get_by_id
|
|
16
|
+
|
|
17
|
+
Feed infrastructure (requires eyecore[feed]):
|
|
18
|
+
feed_data_dir, today_db, open_day, available_days, compress_old_days, insert_articles
|
|
19
|
+
add_source, remove_source, load_sources, scrape_all
|
|
20
|
+
cluster_by_keyword, cluster_with_llm, generate_topic_report, generate_daily_reports
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
from ._core import sha256_hex, score_text, fuzzy_match as _fuzzy_match
|
|
26
|
+
_RUST_CORE = True
|
|
27
|
+
except ImportError:
|
|
28
|
+
_RUST_CORE = False
|
|
29
|
+
|
|
30
|
+
def sha256_hex(data: bytes) -> str:
|
|
31
|
+
h: int = 5381
|
|
32
|
+
for b in data:
|
|
33
|
+
h = ((h * 33) + b) & 0xFFFFFFFFFFFFFFFF
|
|
34
|
+
return format(h, "016x")
|
|
35
|
+
|
|
36
|
+
def score_text(haystack: str, query: str) -> float:
|
|
37
|
+
h, q = haystack.lower(), query.lower()
|
|
38
|
+
if not q:
|
|
39
|
+
return 0.0
|
|
40
|
+
if h.startswith(q):
|
|
41
|
+
return 1000.0
|
|
42
|
+
if q in h:
|
|
43
|
+
return 500.0
|
|
44
|
+
return 0.0
|
|
45
|
+
|
|
46
|
+
def _fuzzy_match(text: str, pattern: str) -> bool:
|
|
47
|
+
qi = iter(pattern)
|
|
48
|
+
pc = next(qi, None)
|
|
49
|
+
for tc in text:
|
|
50
|
+
if tc == pc:
|
|
51
|
+
pc = next(qi, None)
|
|
52
|
+
if pc is None:
|
|
53
|
+
return True
|
|
54
|
+
return pc is None
|
|
55
|
+
|
|
56
|
+
from ._compress import cache_dir, compress_db, decompress_to_cache
|
|
57
|
+
from ._db import BaseDB
|
|
58
|
+
from ._graph import TopicGraph, GRAPH_SCHEMA
|
|
59
|
+
from ._corpus import CorpusManager, CORPUS_REGISTRY_SCHEMA
|
|
60
|
+
from ._llm import LLMClient
|
|
61
|
+
from ._entity_db import EntityDB
|
|
62
|
+
from ._feed_store import (
|
|
63
|
+
data_dir as feed_data_dir,
|
|
64
|
+
today_db,
|
|
65
|
+
open_day,
|
|
66
|
+
available_days,
|
|
67
|
+
compress_old_days,
|
|
68
|
+
insert_articles,
|
|
69
|
+
)
|
|
70
|
+
from ._feed_scraper import add_source, remove_source, load_sources, scrape_all
|
|
71
|
+
from ._feed_report import cluster_by_keyword, cluster_with_llm, generate_topic_report, generate_daily_reports
|
|
72
|
+
|
|
73
|
+
__version__ = "1.0.0"
|
|
74
|
+
|
|
75
|
+
__all__ = [
|
|
76
|
+
"BaseDB",
|
|
77
|
+
"TopicGraph",
|
|
78
|
+
"CorpusManager",
|
|
79
|
+
"LLMClient",
|
|
80
|
+
"EntityDB",
|
|
81
|
+
"cache_dir",
|
|
82
|
+
"compress_db",
|
|
83
|
+
"decompress_to_cache",
|
|
84
|
+
"GRAPH_SCHEMA",
|
|
85
|
+
"CORPUS_REGISTRY_SCHEMA",
|
|
86
|
+
# Feed store
|
|
87
|
+
"feed_data_dir",
|
|
88
|
+
"today_db",
|
|
89
|
+
"open_day",
|
|
90
|
+
"available_days",
|
|
91
|
+
"compress_old_days",
|
|
92
|
+
"insert_articles",
|
|
93
|
+
# Feed scraper
|
|
94
|
+
"add_source",
|
|
95
|
+
"remove_source",
|
|
96
|
+
"load_sources",
|
|
97
|
+
"scrape_all",
|
|
98
|
+
# Feed report
|
|
99
|
+
"cluster_by_keyword",
|
|
100
|
+
"cluster_with_llm",
|
|
101
|
+
"generate_topic_report",
|
|
102
|
+
"generate_daily_reports",
|
|
103
|
+
"__version__",
|
|
104
|
+
"_RUST_CORE",
|
|
105
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Shared compression / cache utilities — used by all libs in the suite."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import gzip
|
|
5
|
+
import os
|
|
6
|
+
import platform
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def cache_dir(app_name: str) -> Path:
|
|
12
|
+
"""Platform-appropriate user cache directory for *app_name*."""
|
|
13
|
+
system = platform.system()
|
|
14
|
+
if system == "Windows":
|
|
15
|
+
base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
|
|
16
|
+
elif system == "Darwin":
|
|
17
|
+
base = Path.home() / "Library" / "Caches"
|
|
18
|
+
else:
|
|
19
|
+
base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
|
|
20
|
+
d = base / app_name
|
|
21
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
return d
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def compress_db(db_path: Path, compresslevel: int = 9) -> Path:
|
|
26
|
+
"""Compress *db_path* to *.db.gz* and delete the original. Returns gz path."""
|
|
27
|
+
gz = db_path.with_suffix(".db.gz")
|
|
28
|
+
with open(db_path, "rb") as src, gzip.open(gz, "wb", compresslevel=compresslevel) as dst:
|
|
29
|
+
shutil.copyfileobj(src, dst)
|
|
30
|
+
db_path.unlink()
|
|
31
|
+
return gz
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def decompress_to_cache(gz_path: Path, app_name: str) -> Path:
|
|
35
|
+
"""Decompress *.db.gz* to the user cache dir on first use.
|
|
36
|
+
|
|
37
|
+
Cache key = gz file size — stable across pip installs, invalidates on re-bake.
|
|
38
|
+
Returns the path to the uncompressed *.db* file.
|
|
39
|
+
"""
|
|
40
|
+
dest = cache_dir(app_name) / f"{app_name}-{gz_path.stat().st_size}.db"
|
|
41
|
+
if not dest.exists():
|
|
42
|
+
with gzip.open(gz_path, "rb") as src, open(dest, "wb") as dst:
|
|
43
|
+
shutil.copyfileobj(src, dst)
|
|
44
|
+
return dest
|