foo-py 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agents.py +10690 -0
- app.py +16234 -0
- boogr/__init__.py +563 -0
- boogr/default_icon.ico +0 -0
- boogr/enums.py +381 -0
- boogr/minion.py +175 -0
- boogr/resources/ico/BooIcon.ico +0 -0
- boogr/resources/ico/Booger.ico +0 -0
- boogr/resources/ico/Save.ico +0 -0
- boogr/resources/ico/adobe.ico +0 -0
- boogr/resources/ico/atk.ico +0 -0
- boogr/resources/ico/b.ico +0 -0
- boogr/resources/ico/batch.ico +0 -0
- boogr/resources/ico/black_sigma.ico +0 -0
- boogr/resources/ico/boo.ico +0 -0
- boogr/resources/ico/boogr.ico +0 -0
- boogr/resources/ico/browse.ico +0 -0
- boogr/resources/ico/chart.ico +0 -0
- boogr/resources/ico/copy.ico +0 -0
- boogr/resources/ico/csv.ico +0 -0
- boogr/resources/ico/dataedit.ico +0 -0
- boogr/resources/ico/doc.ico +0 -0
- boogr/resources/ico/e_logo.ico +0 -0
- boogr/resources/ico/error.ico +0 -0
- boogr/resources/ico/euler_circle.ico +0 -0
- boogr/resources/ico/excel.ico +0 -0
- boogr/resources/ico/file_browse.ico +0 -0
- boogr/resources/ico/filter.ico +0 -0
- boogr/resources/ico/folder_browse.ico +0 -0
- boogr/resources/ico/info.ico +0 -0
- boogr/resources/ico/input.ico +0 -0
- boogr/resources/ico/koolaid.ico +0 -0
- boogr/resources/ico/machinelearning.ico +0 -0
- boogr/resources/ico/message.ico +0 -0
- boogr/resources/ico/pdf.ico +0 -0
- boogr/resources/ico/pi.ico +0 -0
- boogr/resources/ico/setting.ico +0 -0
- boogr/resources/ico/sword_ninja.ico +0 -0
- boogr/resources/ico/textfile.ico +0 -0
- boogr/resources/ico/webcam.ico +0 -0
- boogr/resources/img/atk.png +0 -0
- boogr/resources/img/boogr.png +0 -0
- boogr/resources/img/button/Authority.png +0 -0
- boogr/resources/img/button/BOC.png +0 -0
- boogr/resources/img/button/DERA.png +0 -0
- boogr/resources/img/button/DWH.png +0 -0
- boogr/resources/img/button/EMD.png +0 -0
- boogr/resources/img/button/OAR.png +0 -0
- boogr/resources/img/button/OECA.png +0 -0
- boogr/resources/img/button/OGC.png +0 -0
- boogr/resources/img/button/OMS.png +0 -0
- boogr/resources/img/button/ORD.png +0 -0
- boogr/resources/img/button/OW.png +0 -0
- boogr/resources/img/button/RCRA.png +0 -0
- boogr/resources/img/button/TSCA.png +0 -0
- boogr/resources/img/button/WIFIA.png +0 -0
- boogr/resources/img/button/access.png +0 -0
- boogr/resources/img/button/add.png +0 -0
- boogr/resources/img/button/adobe.png +0 -0
- boogr/resources/img/button/airline.png +0 -0
- boogr/resources/img/button/analytics.png +0 -0
- boogr/resources/img/button/appropriation.png +0 -0
- boogr/resources/img/button/atk.ico +0 -0
- boogr/resources/img/button/atk.png +0 -0
- boogr/resources/img/button/attachment.png +0 -0
- boogr/resources/img/button/bfy.png +0 -0
- boogr/resources/img/button/bluetooth.png +0 -0
- boogr/resources/img/button/browse.png +0 -0
- boogr/resources/img/button/budget.png +0 -0
- boogr/resources/img/button/calculator.png +0 -0
- boogr/resources/img/button/calendar.png +0 -0
- boogr/resources/img/button/cancel.png +0 -0
- boogr/resources/img/button/categoricalgrants.png +0 -0
- boogr/resources/img/button/chart.png +0 -0
- boogr/resources/img/button/chrome.png +0 -0
- boogr/resources/img/button/close.png +0 -0
- boogr/resources/img/button/columndelete.png +0 -0
- boogr/resources/img/button/columnedit.png +0 -0
- boogr/resources/img/button/columninsert.png +0 -0
- boogr/resources/img/button/commandline.png +0 -0
- boogr/resources/img/button/commute.png +0 -0
- boogr/resources/img/button/compass.png +0 -0
- boogr/resources/img/button/contracts.png +0 -0
- boogr/resources/img/button/controlpanel.png +0 -0
- boogr/resources/img/button/csv.png +0 -0
- boogr/resources/img/button/database.png +0 -0
- boogr/resources/img/button/databaseadd.png +0 -0
- boogr/resources/img/button/databasedelete.png +0 -0
- boogr/resources/img/button/databaserefresh.png +0 -0
- boogr/resources/img/button/databasesql.png +0 -0
- boogr/resources/img/button/databaseverify.png +0 -0
- boogr/resources/img/button/datagrid.png +0 -0
- boogr/resources/img/button/delete.png +0 -0
- boogr/resources/img/button/division.png +0 -0
- boogr/resources/img/button/document.png +0 -0
- boogr/resources/img/button/documentadd.png +0 -0
- boogr/resources/img/button/documentation.png +0 -0
- boogr/resources/img/button/documentdelete.png +0 -0
- boogr/resources/img/button/documentedit.png +0 -0
- boogr/resources/img/button/documenterror.png +0 -0
- boogr/resources/img/button/documentsearch.png +0 -0
- boogr/resources/img/button/edge.png +0 -0
- boogr/resources/img/button/edit.png +0 -0
- boogr/resources/img/button/efy.png +0 -0
- boogr/resources/img/button/environment.png +0 -0
- boogr/resources/img/button/ev.png +0 -0
- boogr/resources/img/button/excel.png +0 -0
- boogr/resources/img/button/expenses.png +0 -0
- boogr/resources/img/button/export.png +0 -0
- boogr/resources/img/button/file.png +0 -0
- boogr/resources/img/button/file_word.png +0 -0
- boogr/resources/img/button/fileadd.png +0 -0
- boogr/resources/img/button/filebrowse.png +0 -0
- boogr/resources/img/button/filecopy.png +0 -0
- boogr/resources/img/button/filedelete.png +0 -0
- boogr/resources/img/button/fileedit.png +0 -0
- boogr/resources/img/button/filereader.png +0 -0
- boogr/resources/img/button/filesearch.png +0 -0
- boogr/resources/img/button/filetransfer.png +0 -0
- boogr/resources/img/button/fileverify.png +0 -0
- boogr/resources/img/button/filewriter.png +0 -0
- boogr/resources/img/button/filter.png +0 -0
- boogr/resources/img/button/first.png +0 -0
- boogr/resources/img/button/folder.png +0 -0
- boogr/resources/img/button/folderbrowse.png +0 -0
- boogr/resources/img/button/foldercompress.png +0 -0
- boogr/resources/img/button/foldercopy.png +0 -0
- boogr/resources/img/button/folderdownload.png +0 -0
- boogr/resources/img/button/folderopen.png +0 -0
- boogr/resources/img/button/fte.png +0 -0
- boogr/resources/img/button/function.png +0 -0
- boogr/resources/img/button/gmail.png +0 -0
- boogr/resources/img/button/go.png +0 -0
- boogr/resources/img/button/google.png +0 -0
- boogr/resources/img/button/grants.png +0 -0
- boogr/resources/img/button/guidance.png +0 -0
- boogr/resources/img/button/home.png +0 -0
- boogr/resources/img/button/id.png +0 -0
- boogr/resources/img/button/image.png +0 -0
- boogr/resources/img/button/import.png +0 -0
- boogr/resources/img/button/information.png +0 -0
- boogr/resources/img/button/internet.png +0 -0
- boogr/resources/img/button/justice.png +0 -0
- boogr/resources/img/button/last.png +0 -0
- boogr/resources/img/button/ledger.png +0 -0
- boogr/resources/img/button/left.png +0 -0
- boogr/resources/img/button/levels.png +0 -0
- boogr/resources/img/button/logout.png +0 -0
- boogr/resources/img/button/lust.png +0 -0
- boogr/resources/img/button/menu.png +0 -0
- boogr/resources/img/button/metrics.png +0 -0
- boogr/resources/img/button/mpg.png +0 -0
- boogr/resources/img/button/next.png +0 -0
- boogr/resources/img/button/no.png +0 -0
- boogr/resources/img/button/oil.png +0 -0
- boogr/resources/img/button/ok.png +0 -0
- boogr/resources/img/button/omb.png +0 -0
- boogr/resources/img/button/onenote.png +0 -0
- boogr/resources/img/button/oust.png +0 -0
- boogr/resources/img/button/outlay.png +0 -0
- boogr/resources/img/button/outlook.png +0 -0
- boogr/resources/img/button/pause.png +0 -0
- boogr/resources/img/button/payroll.png +0 -0
- boogr/resources/img/button/pdf.png +0 -0
- boogr/resources/img/button/percentage.png +0 -0
- boogr/resources/img/button/play.png +0 -0
- boogr/resources/img/button/plusminus.png +0 -0
- boogr/resources/img/button/previous.png +0 -0
- boogr/resources/img/button/print.png +0 -0
- boogr/resources/img/button/recertification.png +0 -0
- boogr/resources/img/button/recycle.png +0 -0
- boogr/resources/img/button/redo.png +0 -0
- boogr/resources/img/button/refresh.png +0 -0
- boogr/resources/img/button/remove.png +0 -0
- boogr/resources/img/button/reserve.png +0 -0
- boogr/resources/img/button/right.png +0 -0
- boogr/resources/img/button/row.png +0 -0
- boogr/resources/img/button/rowcopy.png +0 -0
- boogr/resources/img/button/rowdelete.png +0 -0
- boogr/resources/img/button/rowedit.png +0 -0
- boogr/resources/img/button/rowinsert.png +0 -0
- boogr/resources/img/button/save.png +0 -0
- boogr/resources/img/button/scan.png +0 -0
- boogr/resources/img/button/sharepoint.png +0 -0
- boogr/resources/img/button/sigma.png +0 -0
- boogr/resources/img/button/site.png +0 -0
- boogr/resources/img/button/sitetravel.png +0 -0
- boogr/resources/img/button/sort.png +0 -0
- boogr/resources/img/button/spreadsheet.png +0 -0
- boogr/resources/img/button/statistics.png +0 -0
- boogr/resources/img/button/table.png +0 -0
- boogr/resources/img/button/tableadd.png +0 -0
- boogr/resources/img/button/tabledelete.png +0 -0
- boogr/resources/img/button/tablesettings.png +0 -0
- boogr/resources/img/button/text.png +0 -0
- boogr/resources/img/button/traffic.png +0 -0
- boogr/resources/img/button/travel.png +0 -0
- boogr/resources/img/button/undelete.png +0 -0
- boogr/resources/img/button/undo.png +0 -0
- boogr/resources/img/button/wcf.png +0 -0
- boogr/resources/img/button/windows.png +0 -0
- boogr/resources/img/button/word.png +0 -0
- boogr/resources/img/button/xml.png +0 -0
- boogr/resources/img/button/yes.png +0 -0
- boogr/resources/img/button/zipfile.png +0 -0
- boogr/resources/img/gooey.png +0 -0
- boogr/resources/img/web/google.png +0 -0
- config.py +1015 -0
- core.py +170 -0
- data.py +832 -0
- desktop.py +173 -0
- embedders.py +269 -0
- fetchers.py +25142 -0
- foo_assets/__init__.py +1 -0
- foo_assets/resources/images/favicon.ico +0 -0
- foo_assets/resources/images/foo-apikeys.png +0 -0
- foo_assets/resources/images/foo-architecture.png +0 -0
- foo_assets/resources/images/foo-workflows.png +0 -0
- foo_assets/resources/images/foo.ico +0 -0
- foo_assets/resources/images/foo.svg +60 -0
- foo_assets/resources/images/foo_logo.ico +0 -0
- foo_assets/resources/images/foo_logo.png +0 -0
- foo_assets/resources/images/foo_logo.svg +63 -0
- foo_assets/resources/images/foo_portfolio.png +0 -0
- foo_assets/resources/images/foo_project.png +0 -0
- foo_assets/resources/images/system_diagram.svg +46 -0
- foo_assets/resources/images/uml_class_diagram.svg +65 -0
- foo_assets/streamlit_config.toml +85 -0
- foo_cli.py +52 -0
- foo_py-0.1.1.dist-info/METADATA +596 -0
- foo_py-0.1.1.dist-info/RECORD +242 -0
- foo_py-0.1.1.dist-info/WHEEL +5 -0
- foo_py-0.1.1.dist-info/entry_points.txt +2 -0
- foo_py-0.1.1.dist-info/licenses/LICENSE.txt +21 -0
- foo_py-0.1.1.dist-info/top_level.txt +18 -0
- generators.py +3323 -0
- loaders.py +4570 -0
- models.py +344 -0
- processors.py +3216 -0
- scrapers.py +706 -0
- stores/vector.py +365 -0
- writers.py +268 -0
desktop.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""
|
|
2
|
+
******************************************************************************************
|
|
3
|
+
Assembly: Foo
|
|
4
|
+
Filename: desktop.py
|
|
5
|
+
Purpose: Launch Foo in a local Windows desktop webview.
|
|
6
|
+
******************************************************************************************
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import shutil
|
|
13
|
+
import socket
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
from urllib.error import URLError
|
|
18
|
+
from urllib.request import urlopen
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def application_root( ) -> Path:
|
|
22
|
+
"""Return the installed application assets directory.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Path: Bundled resource directory for frozen or source execution.
|
|
26
|
+
"""
|
|
27
|
+
return Path( getattr( sys, '_MEIPASS', Path( __file__ ).resolve( ).parent ) )
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def user_data_root( ) -> Path:
|
|
31
|
+
"""Resolve the writable per-user Foo directory.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
Path: Directory outside the protected installation location.
|
|
35
|
+
"""
|
|
36
|
+
base = os.environ.get( 'LOCALAPPDATA' )
|
|
37
|
+
if base is None:
|
|
38
|
+
base = str( Path.home( ) / 'AppData' / 'Local' )
|
|
39
|
+
location = Path( base ) / 'Foo'
|
|
40
|
+
location.mkdir( parents=True, exist_ok=True )
|
|
41
|
+
return location
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def prepare_user_data( ) -> Path:
|
|
45
|
+
"""Provision writable resources and storage without overwriting user data.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Path: Writable working directory for the Streamlit server.
|
|
49
|
+
"""
|
|
50
|
+
destination = user_data_root( )
|
|
51
|
+
source = application_root( )
|
|
52
|
+
browsers = source / 'playwright-browsers'
|
|
53
|
+
if browsers.exists( ):
|
|
54
|
+
os.environ[ 'PLAYWRIGHT_BROWSERS_PATH' ] = str( browsers )
|
|
55
|
+
for name in ( 'resources', '.streamlit', 'stores' ):
|
|
56
|
+
directory = source / name
|
|
57
|
+
if directory.exists( ):
|
|
58
|
+
shutil.copytree( directory, destination / name, dirs_exist_ok=True,
|
|
59
|
+
copy_function=copy_if_missing )
|
|
60
|
+
(destination / 'stores' / 'sqlite').mkdir( parents=True, exist_ok=True )
|
|
61
|
+
return destination
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def copy_if_missing( source: str, destination: str ) -> str:
|
|
65
|
+
"""Copy bundled defaults only when the destination does not already exist.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
source (str): Original bundled file.
|
|
69
|
+
destination (str): Writable target.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
str: Destination filename.
|
|
73
|
+
"""
|
|
74
|
+
if not Path( destination ).exists( ):
|
|
75
|
+
shutil.copy2( source, destination )
|
|
76
|
+
return destination
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def free_port( ) -> int:
|
|
80
|
+
"""Find a currently unused TCP port on loopback.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
int: Port reserved during inspection; released before server startup.
|
|
84
|
+
"""
|
|
85
|
+
with socket.socket( socket.AF_INET, socket.SOCK_STREAM ) as sock:
|
|
86
|
+
sock.bind( ('127.0.0.1', 0) )
|
|
87
|
+
return int( sock.getsockname( )[ 1 ] )
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def serve( port: int, directory: Path ) -> None:
|
|
91
|
+
"""Run Foo's Streamlit app inside the child process.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
port (int): Loopback TCP port.
|
|
95
|
+
directory (Path): Writable application data location.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
None: Blocks until Streamlit exits.
|
|
99
|
+
"""
|
|
100
|
+
os.chdir( directory )
|
|
101
|
+
from streamlit.web import bootstrap
|
|
102
|
+
app = application_root( ) / 'app.py'
|
|
103
|
+
bootstrap.run( str( app ), False, [ ], {
|
|
104
|
+
'server.address': '127.0.0.1',
|
|
105
|
+
'server.port': port,
|
|
106
|
+
'server.headless': True,
|
|
107
|
+
'browser.gatherUsageStats': False,
|
|
108
|
+
'server.fileWatcherType': 'none',
|
|
109
|
+
'server.enableCORS': True,
|
|
110
|
+
'server.enableXsrfProtection': True,
|
|
111
|
+
} )
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def wait_for_server( process: subprocess.Popen, port: int ) -> None:
|
|
115
|
+
"""Wait for the child server health endpoint or fail on early exit.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
process (subprocess.Popen): Streamlit child process.
|
|
119
|
+
port (int): Child TCP port.
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
None: Server was reachable before timeout.
|
|
123
|
+
|
|
124
|
+
Raises:
|
|
125
|
+
RuntimeError: Server exited or failed to become healthy.
|
|
126
|
+
"""
|
|
127
|
+
endpoint = f'http://127.0.0.1:{port}/_stcore/health'
|
|
128
|
+
for attempt in range( 120 ):
|
|
129
|
+
if process.poll( ) is not None:
|
|
130
|
+
raise RuntimeError( 'Foo Streamlit server exited before startup.' )
|
|
131
|
+
try:
|
|
132
|
+
with urlopen( endpoint, timeout=1 ) as response:
|
|
133
|
+
if response.status == 200:
|
|
134
|
+
return
|
|
135
|
+
except (OSError, URLError):
|
|
136
|
+
time.sleep( 0.5 )
|
|
137
|
+
raise RuntimeError( 'Foo Streamlit server did not become ready within 60 seconds.' )
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def main( ) -> None:
|
|
141
|
+
"""Launch the dedicated webview, terminating its server on window close.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
None: The process exits after the webview is closed.
|
|
145
|
+
"""
|
|
146
|
+
if len( sys.argv ) == 4 and sys.argv[ 1 ] == '--foo-server':
|
|
147
|
+
serve( int( sys.argv[ 2 ] ), Path( sys.argv[ 3 ] ) )
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
import webview
|
|
151
|
+
working_directory = prepare_user_data( )
|
|
152
|
+
port = free_port( )
|
|
153
|
+
command = [ sys.executable, '--foo-server', str( port ), str( working_directory ) ]
|
|
154
|
+
if not getattr( sys, 'frozen', False ):
|
|
155
|
+
command.insert( 1, str( Path( __file__ ).resolve( ) ) )
|
|
156
|
+
server = subprocess.Popen( command, cwd=working_directory )
|
|
157
|
+
try:
|
|
158
|
+
wait_for_server( server, port )
|
|
159
|
+
webview.create_window( 'Foo', f'http://127.0.0.1:{port}',
|
|
160
|
+
width=1400, height=900, min_size=(900, 600) )
|
|
161
|
+
webview.start( gui='edgechromium', debug=False )
|
|
162
|
+
finally:
|
|
163
|
+
if server.poll( ) is None:
|
|
164
|
+
server.terminate( )
|
|
165
|
+
try:
|
|
166
|
+
server.wait( timeout=10 )
|
|
167
|
+
except subprocess.TimeoutExpired:
|
|
168
|
+
server.kill( )
|
|
169
|
+
server.wait( )
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
if __name__ == '__main__':
|
|
173
|
+
main( )
|
embedders.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
'''
|
|
2
|
+
******************************************************************************************
|
|
3
|
+
Assembly: Foo
|
|
4
|
+
Filename: embedders.py
|
|
5
|
+
Author: Terry D. Eppler
|
|
6
|
+
Created: 09-08-2026
|
|
7
|
+
Last Modified By: Terry D. Eppler
|
|
8
|
+
Last Modified On: 09-08-2026
|
|
9
|
+
******************************************************************************************
|
|
10
|
+
<copyright file="embedders.py" company="Terry D. Eppler">
|
|
11
|
+
|
|
12
|
+
Foo is a python framework for web scraping information into ML pipelines.
|
|
13
|
+
|
|
14
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
15
|
+
of this software and associated documentation files (the “Software”),
|
|
16
|
+
to deal in the Software without restriction,
|
|
17
|
+
including without limitation the rights to use,
|
|
18
|
+
copy, modify, merge, publish, distribute, sublicense,
|
|
19
|
+
and/or sell copies of the Software,
|
|
20
|
+
and to permit persons to whom the Software is furnished to do so,
|
|
21
|
+
subject to the following conditions:
|
|
22
|
+
|
|
23
|
+
The above copyright notice and this permission notice shall be included in all
|
|
24
|
+
copies or substantial portions of the Software.
|
|
25
|
+
|
|
26
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
27
|
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
28
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
|
|
29
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
30
|
+
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
31
|
+
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
32
|
+
DEALINGS IN THE SOFTWARE.
|
|
33
|
+
|
|
34
|
+
</copyright>
|
|
35
|
+
<summary>
|
|
36
|
+
LangChain embedding implementations used by Foo document workflows.
|
|
37
|
+
</summary>
|
|
38
|
+
******************************************************************************************
|
|
39
|
+
'''
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Any, List
|
|
43
|
+
from functools import wraps
|
|
44
|
+
|
|
45
|
+
from boogr import Error, Logger
|
|
46
|
+
from langchain_core.embeddings import Embeddings
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def throw_if( name: str, value: object ) -> None:
|
|
50
|
+
"""Validate a required runtime value.
|
|
51
|
+
|
|
52
|
+
Purpose:
|
|
53
|
+
Ensures required embedding configuration and input values are present before provider or
|
|
54
|
+
local-model work begins.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
name (str): Argument name included in validation errors.
|
|
58
|
+
value (object): Runtime value to validate.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
None: This function validates input and does not return a value.
|
|
62
|
+
"""
|
|
63
|
+
if value is None:
|
|
64
|
+
raise ValueError( f'Argument "{name}" cannot be empty!' )
|
|
65
|
+
|
|
66
|
+
if isinstance( value, str ) and not value.strip( ):
|
|
67
|
+
raise ValueError( f'Argument "{name}" cannot be empty!' )
|
|
68
|
+
|
|
69
|
+
if isinstance( value, ( list, tuple, dict, set ) ) and len( value ) == 0:
|
|
70
|
+
raise ValueError( f'Argument "{name}" cannot be empty!' )
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def log_embedding_errors( cause: str, method: str ) -> Any:
|
|
74
|
+
"""Decorate an embedding boundary with Foo's single-log error convention."""
|
|
75
|
+
def decorate( function: Any ) -> Any:
|
|
76
|
+
@wraps( function )
|
|
77
|
+
def execute( *args: Any, **kwargs: Any ) -> Any:
|
|
78
|
+
try:
|
|
79
|
+
return function( *args, **kwargs )
|
|
80
|
+
except Error:
|
|
81
|
+
raise
|
|
82
|
+
except Exception as e:
|
|
83
|
+
exception = Error( e )
|
|
84
|
+
exception.module = 'embedders'
|
|
85
|
+
exception.cause = cause
|
|
86
|
+
exception.method = method
|
|
87
|
+
Logger( ).write( exception )
|
|
88
|
+
raise exception
|
|
89
|
+
return execute
|
|
90
|
+
return decorate
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class LocalGGUFEmbeddings( Embeddings ):
|
|
94
|
+
"""LangChain-compatible local GGUF embedding implementation.
|
|
95
|
+
|
|
96
|
+
Purpose:
|
|
97
|
+
Adapts Chonky's llama.cpp embedding pattern to LangChain's ``Embeddings`` contract so a
|
|
98
|
+
local GGUF model can be used interchangeably with hosted embedding providers and passed
|
|
99
|
+
directly to Chroma or Pinecone vector stores.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
model_path: str
|
|
103
|
+
client: Any
|
|
104
|
+
response: object | None
|
|
105
|
+
|
|
106
|
+
@log_embedding_errors( 'LocalGGUFEmbeddings', '__init__( self, model_path ) -> None' )
|
|
107
|
+
def __init__( self, model_path: str ) -> None:
|
|
108
|
+
"""Initialize the local GGUF embedding implementation.
|
|
109
|
+
|
|
110
|
+
Purpose:
|
|
111
|
+
Stores the configured model path and initializes lazy llama.cpp client state.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
model_path (str): Filesystem path to a local GGUF embedding model.
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
None: This method initializes instance state.
|
|
118
|
+
"""
|
|
119
|
+
throw_if( 'model_path', model_path )
|
|
120
|
+
self.model_path = model_path
|
|
121
|
+
self.client = None
|
|
122
|
+
self.response = None
|
|
123
|
+
|
|
124
|
+
@log_embedding_errors( 'LocalGGUFEmbeddings', 'load( self ) -> None' )
|
|
125
|
+
def load( self ) -> None:
|
|
126
|
+
"""Load the configured local GGUF model once.
|
|
127
|
+
|
|
128
|
+
Purpose:
|
|
129
|
+
Verifies the model path and lazily creates a llama.cpp client configured for
|
|
130
|
+
embedding generation. Repeated calls reuse the loaded client.
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
None: This method initializes the local model client when needed.
|
|
134
|
+
"""
|
|
135
|
+
if self.client is not None:
|
|
136
|
+
return
|
|
137
|
+
|
|
138
|
+
path = Path( self.model_path )
|
|
139
|
+
if not path.is_file( ):
|
|
140
|
+
raise FileNotFoundError( f'Local GGUF model not found: {self.model_path}' )
|
|
141
|
+
|
|
142
|
+
from llama_cpp import Llama
|
|
143
|
+
|
|
144
|
+
self.client = Llama(
|
|
145
|
+
model_path=self.model_path,
|
|
146
|
+
embedding=True,
|
|
147
|
+
verbose=False,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
@log_embedding_errors( 'LocalGGUFEmbeddings', 'embed_documents( self, texts )' )
|
|
151
|
+
def embed_documents( self, texts: List[ str ] ) -> List[ List[ float ] ]:
|
|
152
|
+
"""Create embeddings for document text.
|
|
153
|
+
|
|
154
|
+
Purpose:
|
|
155
|
+
Generates one local embedding vector for every supplied text value while preserving
|
|
156
|
+
positional correspondence between source chunks and returned vectors.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
texts (List[str]): Chunk text values to embed.
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
List[List[float]]: One numeric vector for each supplied text value.
|
|
163
|
+
"""
|
|
164
|
+
throw_if( 'texts', texts )
|
|
165
|
+
values: List[ str ] = [ ]
|
|
166
|
+
|
|
167
|
+
for text in texts:
|
|
168
|
+
throw_if( 'text', text )
|
|
169
|
+
if not isinstance( text, str ):
|
|
170
|
+
raise TypeError( 'Argument "texts" must contain only strings.' )
|
|
171
|
+
values.append( text.strip( ) )
|
|
172
|
+
|
|
173
|
+
self.load( )
|
|
174
|
+
self.response = self.client.create_embedding( values )
|
|
175
|
+
|
|
176
|
+
if isinstance( self.response, dict ) and 'data' in self.response:
|
|
177
|
+
vectors = [ item[ 'embedding' ] for item in self.response[ 'data' ] ]
|
|
178
|
+
else:
|
|
179
|
+
vectors = self.response
|
|
180
|
+
|
|
181
|
+
if not isinstance( vectors, list ) or len( vectors ) != len( values ):
|
|
182
|
+
raise RuntimeError( 'Local GGUF embedding count does not match the input count.' )
|
|
183
|
+
|
|
184
|
+
return vectors
|
|
185
|
+
|
|
186
|
+
@log_embedding_errors( 'LocalGGUFEmbeddings', 'embed_query( self, text ) -> List[ float ]' )
|
|
187
|
+
def embed_query( self, text: str ) -> List[ float ]:
|
|
188
|
+
"""Create an embedding for query text.
|
|
189
|
+
|
|
190
|
+
Purpose:
|
|
191
|
+
Implements the LangChain query-embedding contract using the same local GGUF model
|
|
192
|
+
used for document embeddings.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
text (str): Query text to embed.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
List[float]: Numeric embedding vector for the supplied query.
|
|
199
|
+
"""
|
|
200
|
+
throw_if( 'text', text )
|
|
201
|
+
if not isinstance( text, str ):
|
|
202
|
+
raise TypeError( 'Argument "text" must be a string.' )
|
|
203
|
+
|
|
204
|
+
return self.embed_documents( [ text ] )[ 0 ]
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class EmbeddingFactory( ):
|
|
208
|
+
"""Factory for Foo's supported LangChain embedding implementations."""
|
|
209
|
+
|
|
210
|
+
provider: str
|
|
211
|
+
model: str
|
|
212
|
+
model_path: str
|
|
213
|
+
|
|
214
|
+
def __init__( self ) -> None:
|
|
215
|
+
"""Initialize embedding provider state.
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
None: This method initializes instance state.
|
|
219
|
+
"""
|
|
220
|
+
self.provider = ''
|
|
221
|
+
self.model = ''
|
|
222
|
+
self.model_path = ''
|
|
223
|
+
|
|
224
|
+
@log_embedding_errors( 'EmbeddingFactory', 'create( self, **kwargs ) -> Embeddings' )
|
|
225
|
+
def create( self, provider: str, model: str, model_path: str = '' ) -> Embeddings:
|
|
226
|
+
"""Create the selected LangChain embedding implementation.
|
|
227
|
+
|
|
228
|
+
Purpose:
|
|
229
|
+
Resolves hosted OpenAI, Google Generative AI, Mistral AI, Hugging Face, or local
|
|
230
|
+
GGUF embeddings behind one common LangChain ``Embeddings`` interface.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
provider (str): Embedding provider selected by the user.
|
|
234
|
+
model (str): Hosted or Hugging Face embedding model name.
|
|
235
|
+
model_path (str): Local GGUF model path used only by the Local GGUF provider.
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
Embeddings: Configured LangChain embedding implementation.
|
|
239
|
+
"""
|
|
240
|
+
throw_if( 'provider', provider )
|
|
241
|
+
self.provider = provider
|
|
242
|
+
self.model = model
|
|
243
|
+
self.model_path = model_path
|
|
244
|
+
|
|
245
|
+
if self.provider == 'OpenAI':
|
|
246
|
+
throw_if( 'model', self.model )
|
|
247
|
+
from langchain_openai import OpenAIEmbeddings
|
|
248
|
+
return OpenAIEmbeddings( model=self.model )
|
|
249
|
+
|
|
250
|
+
if self.provider == 'Google Generative AI':
|
|
251
|
+
throw_if( 'model', self.model )
|
|
252
|
+
from langchain_google_genai import GoogleGenerativeAIEmbeddings
|
|
253
|
+
return GoogleGenerativeAIEmbeddings( model=self.model )
|
|
254
|
+
|
|
255
|
+
if self.provider == 'Mistral AI':
|
|
256
|
+
throw_if( 'model', self.model )
|
|
257
|
+
from langchain_mistralai import MistralAIEmbeddings
|
|
258
|
+
return MistralAIEmbeddings( model=self.model )
|
|
259
|
+
|
|
260
|
+
if self.provider == 'Hugging Face':
|
|
261
|
+
throw_if( 'model', self.model )
|
|
262
|
+
from langchain_huggingface import HuggingFaceEmbeddings
|
|
263
|
+
return HuggingFaceEmbeddings( model_name=self.model )
|
|
264
|
+
|
|
265
|
+
if self.provider == 'Local GGUF':
|
|
266
|
+
throw_if( 'model_path', self.model_path )
|
|
267
|
+
return LocalGGUFEmbeddings( model_path=self.model_path )
|
|
268
|
+
|
|
269
|
+
raise ValueError( f'Unsupported embedding provider: {self.provider}' )
|