foo-py 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. agents.py +10690 -0
  2. app.py +16234 -0
  3. boogr/__init__.py +563 -0
  4. boogr/default_icon.ico +0 -0
  5. boogr/enums.py +381 -0
  6. boogr/minion.py +175 -0
  7. boogr/resources/ico/BooIcon.ico +0 -0
  8. boogr/resources/ico/Booger.ico +0 -0
  9. boogr/resources/ico/Save.ico +0 -0
  10. boogr/resources/ico/adobe.ico +0 -0
  11. boogr/resources/ico/atk.ico +0 -0
  12. boogr/resources/ico/b.ico +0 -0
  13. boogr/resources/ico/batch.ico +0 -0
  14. boogr/resources/ico/black_sigma.ico +0 -0
  15. boogr/resources/ico/boo.ico +0 -0
  16. boogr/resources/ico/boogr.ico +0 -0
  17. boogr/resources/ico/browse.ico +0 -0
  18. boogr/resources/ico/chart.ico +0 -0
  19. boogr/resources/ico/copy.ico +0 -0
  20. boogr/resources/ico/csv.ico +0 -0
  21. boogr/resources/ico/dataedit.ico +0 -0
  22. boogr/resources/ico/doc.ico +0 -0
  23. boogr/resources/ico/e_logo.ico +0 -0
  24. boogr/resources/ico/error.ico +0 -0
  25. boogr/resources/ico/euler_circle.ico +0 -0
  26. boogr/resources/ico/excel.ico +0 -0
  27. boogr/resources/ico/file_browse.ico +0 -0
  28. boogr/resources/ico/filter.ico +0 -0
  29. boogr/resources/ico/folder_browse.ico +0 -0
  30. boogr/resources/ico/info.ico +0 -0
  31. boogr/resources/ico/input.ico +0 -0
  32. boogr/resources/ico/koolaid.ico +0 -0
  33. boogr/resources/ico/machinelearning.ico +0 -0
  34. boogr/resources/ico/message.ico +0 -0
  35. boogr/resources/ico/pdf.ico +0 -0
  36. boogr/resources/ico/pi.ico +0 -0
  37. boogr/resources/ico/setting.ico +0 -0
  38. boogr/resources/ico/sword_ninja.ico +0 -0
  39. boogr/resources/ico/textfile.ico +0 -0
  40. boogr/resources/ico/webcam.ico +0 -0
  41. boogr/resources/img/atk.png +0 -0
  42. boogr/resources/img/boogr.png +0 -0
  43. boogr/resources/img/button/Authority.png +0 -0
  44. boogr/resources/img/button/BOC.png +0 -0
  45. boogr/resources/img/button/DERA.png +0 -0
  46. boogr/resources/img/button/DWH.png +0 -0
  47. boogr/resources/img/button/EMD.png +0 -0
  48. boogr/resources/img/button/OAR.png +0 -0
  49. boogr/resources/img/button/OECA.png +0 -0
  50. boogr/resources/img/button/OGC.png +0 -0
  51. boogr/resources/img/button/OMS.png +0 -0
  52. boogr/resources/img/button/ORD.png +0 -0
  53. boogr/resources/img/button/OW.png +0 -0
  54. boogr/resources/img/button/RCRA.png +0 -0
  55. boogr/resources/img/button/TSCA.png +0 -0
  56. boogr/resources/img/button/WIFIA.png +0 -0
  57. boogr/resources/img/button/access.png +0 -0
  58. boogr/resources/img/button/add.png +0 -0
  59. boogr/resources/img/button/adobe.png +0 -0
  60. boogr/resources/img/button/airline.png +0 -0
  61. boogr/resources/img/button/analytics.png +0 -0
  62. boogr/resources/img/button/appropriation.png +0 -0
  63. boogr/resources/img/button/atk.ico +0 -0
  64. boogr/resources/img/button/atk.png +0 -0
  65. boogr/resources/img/button/attachment.png +0 -0
  66. boogr/resources/img/button/bfy.png +0 -0
  67. boogr/resources/img/button/bluetooth.png +0 -0
  68. boogr/resources/img/button/browse.png +0 -0
  69. boogr/resources/img/button/budget.png +0 -0
  70. boogr/resources/img/button/calculator.png +0 -0
  71. boogr/resources/img/button/calendar.png +0 -0
  72. boogr/resources/img/button/cancel.png +0 -0
  73. boogr/resources/img/button/categoricalgrants.png +0 -0
  74. boogr/resources/img/button/chart.png +0 -0
  75. boogr/resources/img/button/chrome.png +0 -0
  76. boogr/resources/img/button/close.png +0 -0
  77. boogr/resources/img/button/columndelete.png +0 -0
  78. boogr/resources/img/button/columnedit.png +0 -0
  79. boogr/resources/img/button/columninsert.png +0 -0
  80. boogr/resources/img/button/commandline.png +0 -0
  81. boogr/resources/img/button/commute.png +0 -0
  82. boogr/resources/img/button/compass.png +0 -0
  83. boogr/resources/img/button/contracts.png +0 -0
  84. boogr/resources/img/button/controlpanel.png +0 -0
  85. boogr/resources/img/button/csv.png +0 -0
  86. boogr/resources/img/button/database.png +0 -0
  87. boogr/resources/img/button/databaseadd.png +0 -0
  88. boogr/resources/img/button/databasedelete.png +0 -0
  89. boogr/resources/img/button/databaserefresh.png +0 -0
  90. boogr/resources/img/button/databasesql.png +0 -0
  91. boogr/resources/img/button/databaseverify.png +0 -0
  92. boogr/resources/img/button/datagrid.png +0 -0
  93. boogr/resources/img/button/delete.png +0 -0
  94. boogr/resources/img/button/division.png +0 -0
  95. boogr/resources/img/button/document.png +0 -0
  96. boogr/resources/img/button/documentadd.png +0 -0
  97. boogr/resources/img/button/documentation.png +0 -0
  98. boogr/resources/img/button/documentdelete.png +0 -0
  99. boogr/resources/img/button/documentedit.png +0 -0
  100. boogr/resources/img/button/documenterror.png +0 -0
  101. boogr/resources/img/button/documentsearch.png +0 -0
  102. boogr/resources/img/button/edge.png +0 -0
  103. boogr/resources/img/button/edit.png +0 -0
  104. boogr/resources/img/button/efy.png +0 -0
  105. boogr/resources/img/button/environment.png +0 -0
  106. boogr/resources/img/button/ev.png +0 -0
  107. boogr/resources/img/button/excel.png +0 -0
  108. boogr/resources/img/button/expenses.png +0 -0
  109. boogr/resources/img/button/export.png +0 -0
  110. boogr/resources/img/button/file.png +0 -0
  111. boogr/resources/img/button/file_word.png +0 -0
  112. boogr/resources/img/button/fileadd.png +0 -0
  113. boogr/resources/img/button/filebrowse.png +0 -0
  114. boogr/resources/img/button/filecopy.png +0 -0
  115. boogr/resources/img/button/filedelete.png +0 -0
  116. boogr/resources/img/button/fileedit.png +0 -0
  117. boogr/resources/img/button/filereader.png +0 -0
  118. boogr/resources/img/button/filesearch.png +0 -0
  119. boogr/resources/img/button/filetransfer.png +0 -0
  120. boogr/resources/img/button/fileverify.png +0 -0
  121. boogr/resources/img/button/filewriter.png +0 -0
  122. boogr/resources/img/button/filter.png +0 -0
  123. boogr/resources/img/button/first.png +0 -0
  124. boogr/resources/img/button/folder.png +0 -0
  125. boogr/resources/img/button/folderbrowse.png +0 -0
  126. boogr/resources/img/button/foldercompress.png +0 -0
  127. boogr/resources/img/button/foldercopy.png +0 -0
  128. boogr/resources/img/button/folderdownload.png +0 -0
  129. boogr/resources/img/button/folderopen.png +0 -0
  130. boogr/resources/img/button/fte.png +0 -0
  131. boogr/resources/img/button/function.png +0 -0
  132. boogr/resources/img/button/gmail.png +0 -0
  133. boogr/resources/img/button/go.png +0 -0
  134. boogr/resources/img/button/google.png +0 -0
  135. boogr/resources/img/button/grants.png +0 -0
  136. boogr/resources/img/button/guidance.png +0 -0
  137. boogr/resources/img/button/home.png +0 -0
  138. boogr/resources/img/button/id.png +0 -0
  139. boogr/resources/img/button/image.png +0 -0
  140. boogr/resources/img/button/import.png +0 -0
  141. boogr/resources/img/button/information.png +0 -0
  142. boogr/resources/img/button/internet.png +0 -0
  143. boogr/resources/img/button/justice.png +0 -0
  144. boogr/resources/img/button/last.png +0 -0
  145. boogr/resources/img/button/ledger.png +0 -0
  146. boogr/resources/img/button/left.png +0 -0
  147. boogr/resources/img/button/levels.png +0 -0
  148. boogr/resources/img/button/logout.png +0 -0
  149. boogr/resources/img/button/lust.png +0 -0
  150. boogr/resources/img/button/menu.png +0 -0
  151. boogr/resources/img/button/metrics.png +0 -0
  152. boogr/resources/img/button/mpg.png +0 -0
  153. boogr/resources/img/button/next.png +0 -0
  154. boogr/resources/img/button/no.png +0 -0
  155. boogr/resources/img/button/oil.png +0 -0
  156. boogr/resources/img/button/ok.png +0 -0
  157. boogr/resources/img/button/omb.png +0 -0
  158. boogr/resources/img/button/onenote.png +0 -0
  159. boogr/resources/img/button/oust.png +0 -0
  160. boogr/resources/img/button/outlay.png +0 -0
  161. boogr/resources/img/button/outlook.png +0 -0
  162. boogr/resources/img/button/pause.png +0 -0
  163. boogr/resources/img/button/payroll.png +0 -0
  164. boogr/resources/img/button/pdf.png +0 -0
  165. boogr/resources/img/button/percentage.png +0 -0
  166. boogr/resources/img/button/play.png +0 -0
  167. boogr/resources/img/button/plusminus.png +0 -0
  168. boogr/resources/img/button/previous.png +0 -0
  169. boogr/resources/img/button/print.png +0 -0
  170. boogr/resources/img/button/recertification.png +0 -0
  171. boogr/resources/img/button/recycle.png +0 -0
  172. boogr/resources/img/button/redo.png +0 -0
  173. boogr/resources/img/button/refresh.png +0 -0
  174. boogr/resources/img/button/remove.png +0 -0
  175. boogr/resources/img/button/reserve.png +0 -0
  176. boogr/resources/img/button/right.png +0 -0
  177. boogr/resources/img/button/row.png +0 -0
  178. boogr/resources/img/button/rowcopy.png +0 -0
  179. boogr/resources/img/button/rowdelete.png +0 -0
  180. boogr/resources/img/button/rowedit.png +0 -0
  181. boogr/resources/img/button/rowinsert.png +0 -0
  182. boogr/resources/img/button/save.png +0 -0
  183. boogr/resources/img/button/scan.png +0 -0
  184. boogr/resources/img/button/sharepoint.png +0 -0
  185. boogr/resources/img/button/sigma.png +0 -0
  186. boogr/resources/img/button/site.png +0 -0
  187. boogr/resources/img/button/sitetravel.png +0 -0
  188. boogr/resources/img/button/sort.png +0 -0
  189. boogr/resources/img/button/spreadsheet.png +0 -0
  190. boogr/resources/img/button/statistics.png +0 -0
  191. boogr/resources/img/button/table.png +0 -0
  192. boogr/resources/img/button/tableadd.png +0 -0
  193. boogr/resources/img/button/tabledelete.png +0 -0
  194. boogr/resources/img/button/tablesettings.png +0 -0
  195. boogr/resources/img/button/text.png +0 -0
  196. boogr/resources/img/button/traffic.png +0 -0
  197. boogr/resources/img/button/travel.png +0 -0
  198. boogr/resources/img/button/undelete.png +0 -0
  199. boogr/resources/img/button/undo.png +0 -0
  200. boogr/resources/img/button/wcf.png +0 -0
  201. boogr/resources/img/button/windows.png +0 -0
  202. boogr/resources/img/button/word.png +0 -0
  203. boogr/resources/img/button/xml.png +0 -0
  204. boogr/resources/img/button/yes.png +0 -0
  205. boogr/resources/img/button/zipfile.png +0 -0
  206. boogr/resources/img/gooey.png +0 -0
  207. boogr/resources/img/web/google.png +0 -0
  208. config.py +1015 -0
  209. core.py +170 -0
  210. data.py +832 -0
  211. desktop.py +173 -0
  212. embedders.py +269 -0
  213. fetchers.py +25142 -0
  214. foo_assets/__init__.py +1 -0
  215. foo_assets/resources/images/favicon.ico +0 -0
  216. foo_assets/resources/images/foo-apikeys.png +0 -0
  217. foo_assets/resources/images/foo-architecture.png +0 -0
  218. foo_assets/resources/images/foo-workflows.png +0 -0
  219. foo_assets/resources/images/foo.ico +0 -0
  220. foo_assets/resources/images/foo.svg +60 -0
  221. foo_assets/resources/images/foo_logo.ico +0 -0
  222. foo_assets/resources/images/foo_logo.png +0 -0
  223. foo_assets/resources/images/foo_logo.svg +63 -0
  224. foo_assets/resources/images/foo_portfolio.png +0 -0
  225. foo_assets/resources/images/foo_project.png +0 -0
  226. foo_assets/resources/images/system_diagram.svg +46 -0
  227. foo_assets/resources/images/uml_class_diagram.svg +65 -0
  228. foo_assets/streamlit_config.toml +85 -0
  229. foo_cli.py +52 -0
  230. foo_py-0.1.1.dist-info/METADATA +596 -0
  231. foo_py-0.1.1.dist-info/RECORD +242 -0
  232. foo_py-0.1.1.dist-info/WHEEL +5 -0
  233. foo_py-0.1.1.dist-info/entry_points.txt +2 -0
  234. foo_py-0.1.1.dist-info/licenses/LICENSE.txt +21 -0
  235. foo_py-0.1.1.dist-info/top_level.txt +18 -0
  236. generators.py +3323 -0
  237. loaders.py +4570 -0
  238. models.py +344 -0
  239. processors.py +3216 -0
  240. scrapers.py +706 -0
  241. stores/vector.py +365 -0
  242. writers.py +268 -0
data.py ADDED
@@ -0,0 +1,832 @@
1
+ '''
2
+ ******************************************************************************************
3
+ Assembly: Foo
4
+ Filename: data.py
5
+ Author: Terry D. Eppler
6
+ Created: 05-31-2022
7
+
8
+ Last Modified By: Terry D. Eppler
9
+ Last Modified On: 05-01-2025
10
+ ******************************************************************************************
11
+ <copyright file="data.py" company="Terry D. Eppler">
12
+
13
+ data.py
14
+ Copyright © 2025 Terry Eppler
15
+
16
+ Permission is hereby granted, free of charge, to any person obtaining a copy
17
+ of this software and associated documentation files (the “Software”),
18
+ to deal in the Software without restriction,
19
+ including without limitation the rights to use,
20
+ copy, modify, merge, publish, distribute, sublicense,
21
+ and/or sell copies of the Software,
22
+ and to permit persons to whom the Software is furnished to do so,
23
+ subject to the following conditions:
24
+
25
+ The above copyright notice and this permission notice shall be included in all
26
+ copies or substantial portions of the Software.
27
+
28
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
29
+ INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30
+ FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
31
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
32
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
33
+ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
34
+ DEALINGS IN THE SOFTWARE.
35
+
36
+ You can contact me at: terryeppler@gmail.com or eppler.terry@epa.gov
37
+
38
+ </copyright>
39
+ <summary>
40
+ data.py
41
+
42
+ Purpose:
43
+ Provides database and vector-store support for the Foo application. The module
44
+ contains shared validation, relational database helpers for SQLite-style data
45
+ operations, provider path utilities, and a ChromaDB wrapper for persistent vector
46
+ storage and semantic retrieval workflows.
47
+ </summary>
48
+ ******************************************************************************************
49
+ '''
50
+ import json
51
+ import numpy as np
52
+ import pandas as pd
53
+ import os
54
+ import sqlite3
55
+ from sqlite3 import Connection, Cursor
56
+ from typing import Optional, Any, List, Tuple
57
+ from boogr import Error, Logger
58
+ from boogr.enums import Source, Provider
59
+ import chromadb
60
+ from chromadb.config import Settings
61
+ import config as cfg
62
+
63
+
64
+ def throw_if( name: str, value: object ) -> None:
65
+ """Throw if.
66
+
67
+ Purpose:
68
+ Provides a input guard used by the Gipity Streamlit application. The function
69
+ supports UI state management, provider coordination, data normalization, or display
70
+ behavior required by the surrounding workflow.
71
+
72
+ Args:
73
+ name (str): Value supplied to the helper.
74
+ value (object): Value supplied to the helper.
75
+
76
+ Raises:
77
+ Error: Re-raised after the exception is wrapped and written to the application logger.
78
+ """
79
+ if value is None:
80
+ raise ValueError( f'Argument "{name}" cannot be empty!' )
81
+ if isinstance( value, str ) and (not value.strip( )):
82
+ raise ValueError( f'Argument "{name}" cannot be empty!' )
83
+ if isinstance( value, (list, tuple, dict, set) ) and len( value ) == 0:
84
+ raise ValueError( f'Argument "{name}" cannot be empty!' )
85
+
86
+
87
+ class DB( ):
88
+ """Provide database provider configuration helpers.
89
+
90
+ Purpose:
91
+ Stores common database provider metadata and resolves provider-specific driver,
92
+ data-path, and connection-string values used by Foo database workflows. The class
93
+ acts as a lightweight base for concrete database implementations.
94
+
95
+ Attributes:
96
+ provider (Optional[Provider]): Selected database provider enum value.
97
+ source (Optional[Source]): Data source enum value associated with the database.
98
+ table_name (Optional[str]): Active table name used by downstream operations.
99
+ column_names (Optional[List[str]]): Active column names associated with a table.
100
+ path (Optional[str]): Resolved provider data path.
101
+ driver (Optional[str]): Provider driver string.
102
+ """
103
+ provider: Optional[ Provider ]
104
+ source: Optional[ Source ]
105
+ table_name: Optional[ str ]
106
+ column_names: Optional[ List[ str ] ]
107
+ path: Optional[ str ]
108
+ driver: Optional[ str ]
109
+
110
+ def __init__( self ):
111
+ """Initialize database configuration state.
112
+
113
+ Purpose:
114
+ Initializes provider, source, table, path, and driver members to empty runtime
115
+ state. Concrete database subclasses and property accessors populate these fields
116
+ later when provider-specific database work is requested.
117
+ """
118
+ self.provider = None
119
+ self.source = None
120
+ self.table_name = None
121
+ self.path = None
122
+ self.driver = None
123
+
124
+ def __dir__( self ) -> list[ str ]:
125
+ """Return visible database configuration members.
126
+
127
+ Purpose:
128
+ Provides a stable ordering of attributes and helper properties for interactive
129
+ inspection, documentation surfaces, and UI components that expose database
130
+ configuration details.
131
+
132
+ Returns:
133
+ Ordered member names exposed by the database base object.
134
+ """
135
+ return [ 'source',
136
+ 'provider',
137
+ 'table_name',
138
+ 'get_driver_info',
139
+ 'path',
140
+ 'adriver',
141
+ 'access_path',
142
+ 'get_data_path',
143
+ 'get_connection_string' ]
144
+
145
+ @property
146
+ def driver_info( self ) -> str:
147
+ """Get provider driver information.
148
+
149
+ Purpose:
150
+ Returns the configured driver string for Access or SQL Server providers and falls
151
+ back to the configured base directory for other provider values. The property also
152
+ stores the selected driver on the instance for later connection-string assembly.
153
+
154
+ Returns:
155
+ Resolved provider driver string or configured base directory fallback.
156
+
157
+ Raises:
158
+ Error: Re-raised after the exception is wrapped and written to the application logger.
159
+ """
160
+ try:
161
+ if self.provider.name == 'Access':
162
+ self.driver = cfg.ACCESS_DRIVER
163
+ return self.driver
164
+ elif self.provider.name == 'SqlServer':
165
+ self.driver = cfg.SQLSERVER_DRIVER
166
+ return self.driver
167
+ elif self.provider.name == 'SqlServer':
168
+ return self.sqlserver_driver
169
+ else:
170
+ return cfg.BASEDIR
171
+ except Exception as e:
172
+ _exc = Error( e )
173
+ _exc.module = 'data'
174
+ _exc.cause = 'DB'
175
+ _exc.method = 'getdriver_info( self )'
176
+ Logger( ).write( _exc )
177
+ raise _exc
178
+
179
+ @property
180
+ def data_path( self ) -> str:
181
+ """Get provider data path.
182
+
183
+ Purpose:
184
+ Builds the provider-specific storage path used by Foo database operations. The
185
+ property maps SQLite, Access, and SQL Server provider values to their configured
186
+ store directories and preserves a SQLite fallback for unsupported providers.
187
+
188
+ Returns:
189
+ Resolved provider data path.
190
+
191
+ Raises:
192
+ Error: Re-raised after the exception is wrapped and written to the application logger.
193
+ """
194
+ try:
195
+ if self.provider.name == 'SQLite':
196
+ self.path = cfg.BASEDIR + r'\stores\sqlite'
197
+ return self.path
198
+ elif self.provider.name == 'Access':
199
+ self.path = cfg.BASEDIR + r'\stores\access'
200
+ return self.path
201
+ elif self.provider.name == 'SqlServer':
202
+ self.path = cfg.BASEDIR + r'\stores\sqlserver'
203
+ return self.path
204
+ else:
205
+ self.path = cfg.BASEDIR + r'\stores\sqlite'
206
+ return self.path
207
+ except Exception as e:
208
+ _exc = Error( e )
209
+ _exc.module = 'data'
210
+ _exc.cause = 'DB'
211
+ _exc.method = 'get_data_path( self )'
212
+ Logger( ).write( _exc )
213
+ raise _exc
214
+
215
+ @property
216
+ def connection_string( self ) -> str:
217
+ """Get provider connection string.
218
+
219
+ Purpose:
220
+ Constructs the connection string associated with the selected database provider.
221
+ Access providers combine the configured Access driver with the resolved path,
222
+ SQL Server providers build an attach-database connection string, and other
223
+ providers return the resolved path fallback.
224
+
225
+ Returns:
226
+ Provider connection string or path fallback.
227
+
228
+ Raises:
229
+ Error: Re-raised after the exception is wrapped and written to the application logger.
230
+ """
231
+ try:
232
+ _path = self.data_path
233
+ if self.provider.name == Provider.Access.name:
234
+ return self.driver_info + _path
235
+ elif self.provider.name == Provider.SqlServer.name:
236
+ return r'DRIVER={ ODBC Driver 17 for SQL Server };Server=.\SQLExpress;' \
237
+ + f'AttachDBFileName={_path}' \
238
+ + f'DATABASE={_path}Trusted_Connection=yes;'
239
+ else:
240
+ return f'{_path} '
241
+ except Exception as e:
242
+ _exc = Error( e )
243
+ _exc.module = 'data'
244
+ _exc.cause = 'DB'
245
+ _exc.method = 'get_connection_string( self )'
246
+ Logger( ).write( _exc )
247
+ raise _exc
248
+
249
+ class SQLite( DB ):
250
+ """Provide SQLite database operations.
251
+
252
+ Purpose:
253
+ Wraps a local SQLite connection and cursor with common table creation, insert,
254
+ batch insert, query, update, delete, Excel import, and close operations. The class
255
+ keeps the existing Foo database behavior while exposing a consistent object surface
256
+ for data-management workflows.
257
+
258
+ Attributes:
259
+ db_path (Optional[str]): Path to the SQLite database file.
260
+ connection (Optional[Connection]): Active SQLite connection.
261
+ cursor (Optional[Cursor]): Active SQLite cursor.
262
+ file_path (Optional[str]): Current file path used by import workflows.
263
+ where (Optional[str]): Current SQL WHERE clause fragment.
264
+ file_name (Optional[str]): Current source file name.
265
+ table_name (Optional[str]): Current table name.
266
+ placeholders (Optional[List[str]]): Placeholder state used by insert operations.
267
+ column_names (Optional[List[str]]): Active column names.
268
+ params (Optional[Tuple]): SQL parameter tuple.
269
+ tables (Optional[List]): Cached table list.
270
+ """
271
+ db_path: Optional[ str ]
272
+ connection: Optional[ Connection ]
273
+ cursor: Optional[ Cursor ]
274
+ file_path: Optional[ str ]
275
+ where: Optional[ str ]
276
+ file_name: Optional[ str ]
277
+ table_name: Optional[ str ]
278
+ placeholders: Optional[ List[ str ] ]
279
+ column_names: Optional[ List[ str ] ]
280
+ params: Optional[ Tuple ]
281
+ tables: Optional[ List ]
282
+
283
+ def __init__( self ):
284
+ """Initialize SQLite connection state.
285
+
286
+ Purpose:
287
+ Initializes the SQLite wrapper with the default Foo data database path, opens the
288
+ connection and cursor, and prepares runtime members used by subsequent CRUD,
289
+ batch-insert, and import operations.
290
+ """
291
+ self.db_path = r'stores\sqlite\datamodels\Data.db'
292
+ self.connection = sqlite3.connect( self.db_path )
293
+ self.cursor = self.connection.cursor( )
294
+ self.file_path = None
295
+ self.where = None
296
+ self.pairs = None
297
+ self.sql = None
298
+ self.file_name = None
299
+ self.table_name = None
300
+ self.placeholders = [ str ]
301
+ self.params = ( )
302
+ self.column_names = [ str ]
303
+ self.tables = [ ]
304
+
305
+ def __dir__( self ):
306
+ """Return visible SQLite members.
307
+
308
+ Purpose:
309
+ Provides a stable ordering of SQLite wrapper attributes and operations for
310
+ interactive inspection, documentation, and UI display surfaces.
311
+
312
+ Returns:
313
+ Ordered member names exposed by the SQLite wrapper.
314
+ """
315
+ return [ 'db_path', 'connection', 'cursor', 'path', 'where', 'pairs', 'sql', 'file_name',
316
+ 'table_name', 'placeholders', 'columns', 'params', 'column_names', 'tables', 'close',
317
+ 'import_excel', 'delete', 'update', 'insert', 'create_table', 'fetch_one',
318
+ 'fetch_all' ]
319
+
320
+ def create( self ) -> None:
321
+ """Create the embeddings table.
322
+
323
+ Purpose:
324
+ Creates the default ``embeddings`` table when it does not already exist. The
325
+ table stores source-file names, chunk indexes, chunk text, serialized embedding
326
+ vectors, and creation timestamps for downstream retrieval workflows.
327
+
328
+ Raises:
329
+ Error: Re-raised after the exception is wrapped and written to the application logger.
330
+ """
331
+ try:
332
+ self.cursor.execute( """
333
+ CREATE TABLE IF NOT EXISTS embeddings
334
+ (
335
+ id
336
+ INTEGER
337
+ PRIMARY
338
+ KEY
339
+ AUTOINCREMENT,
340
+ source_file
341
+ TEXT
342
+ NOT
343
+ NULL,
344
+ chunk_index
345
+ INTEGER
346
+ NOT
347
+ NULL,
348
+ chunk_text
349
+ TEXT
350
+ NOT
351
+ NULL,
352
+ embedding
353
+ TEXT
354
+ NOT
355
+ NULL,
356
+ created_at
357
+ TEXT
358
+ DEFAULT
359
+ CURRENT_TIMESTAMP
360
+ )""" )
361
+
362
+ self.connection.commit( )
363
+ except Exception as e:
364
+ exception = Error( e )
365
+ exception.module = 'data'
366
+ exception.cause = 'SQLite'
367
+ exception.method = 'create( self ) -> None'
368
+ Logger( ).write( exception )
369
+ raise exception
370
+
371
+ def create_table( self, sql: str ) -> None:
372
+ """Create a table from SQL.
373
+
374
+ Purpose:
375
+ Executes a caller-provided SQL table-creation statement against the active SQLite
376
+ connection and commits the result. The method retains the SQL statement on the
377
+ instance for inspection and preserves the existing direct-SQL behavior.
378
+
379
+ Args:
380
+ sql (str): SQL statement to execute.
381
+
382
+ Raises:
383
+ Error: Re-raised after the exception is wrapped and written to the application logger.
384
+ """
385
+ try:
386
+ throw_if( 'sql', sql )
387
+ self.sql = sql
388
+ self.cursor.execute( self.sql )
389
+ self.connection.commit( )
390
+ except Exception as e:
391
+ exception = Error( e )
392
+ exception.module = 'data'
393
+ exception.cause = 'SQLite'
394
+ exception.method = 'create_table( self, sql: str ) -> None'
395
+ Logger( ).write( exception )
396
+ raise exception
397
+
398
+ def insert( self, table: str, columns: List[ str ], values: Tuple[ Any, ... ] ) -> None:
399
+ """Insert one record.
400
+
401
+ Purpose:
402
+ Builds and executes an INSERT statement for the specified table, column list, and
403
+ value tuple. The method uses positional SQLite placeholders for values while
404
+ preserving the caller-provided table and column naming behavior.
405
+
406
+ Args:
407
+ table (str): Target table name.
408
+ columns (List[str]): Column names receiving values.
409
+ values (Tuple[Any, ...]): Values to insert into the target columns.
410
+
411
+ Raises:
412
+ Error: Re-raised after the exception is wrapped and written to the application logger.
413
+ """
414
+ try:
415
+ throw_if( 'table', table )
416
+ throw_if( 'columns', columns )
417
+ throw_if( 'values', values )
418
+ self.placeholders = ', '.join( '?' for _ in values )
419
+ col_names = ', '.join( columns )
420
+ self.sql = f'INSERT INTO {table} ({col_names}) VALUES ({self.placeholders})'
421
+ self.cursor.execute( self.sql, values )
422
+ self.connection.commit( )
423
+ except Exception as e:
424
+ exception = Error( e )
425
+ exception.module = 'data'
426
+ exception.cause = 'SQLite'
427
+ exception.method = ('insert( self, **kwargs ) -> None')
428
+ Logger( ).write( exception )
429
+ raise exception
430
+
431
+ def insert_many( self, source_file: str, chunks: List[ str ], vectors: np.ndarray ) -> None:
432
+ """Insert multiple embedding records.
433
+
434
+ Purpose:
435
+ Serializes a batch of embedding vectors and inserts them with their source file,
436
+ chunk index, and chunk text into the active table. This supports bulk persistence
437
+ of text chunks and embeddings generated by ingestion workflows.
438
+
439
+ Args:
440
+ source_file (str): Name or path of the source document.
441
+ chunks (List[str]): Cleaned text chunks corresponding to the embeddings.
442
+ vectors (np.ndarray): Matrix of embedding vectors to serialize and store.
443
+
444
+ Raises:
445
+ Error: Re-raised after the exception is wrapped and written to the application logger.
446
+ """
447
+ try:
448
+ throw_if( 'source_file', source_file )
449
+ throw_if( 'chuncks', chunks )
450
+ throw_if( 'vectors', vectors )
451
+ records = [ (source_file, i, chunks[ i ], json.dumps( vectors[ i ].tolist( ) ))
452
+ for i in range( len( chunks ) ) ]
453
+
454
+ self.sql = f''' INSERT INTO {self.table_name} ({self.file_name}, chunk_index,
455
+ chunk_text, embedding) VALUES (?, ?, ?, ?) '''
456
+ self.cursor.executemany( self.sql, records )
457
+ self.connection.commit( )
458
+ except Exception as e:
459
+ exception = Error( e )
460
+ exception.module = 'data'
461
+ exception.cause = 'SQLite'
462
+ exception.method = 'insert_many'
463
+ Logger( ).write( exception )
464
+ raise exception
465
+
466
+ def fetch_all( self, table: str ) -> List[ Tuple ] | None:
467
+ """Fetch all rows from a table.
468
+
469
+ Purpose:
470
+ Executes a SELECT statement against the specified table and returns all rows from
471
+ the active cursor. The method keeps the generated SQL on the instance for later
472
+ inspection or debugging.
473
+
474
+ Args:
475
+ table (str): Table name to query.
476
+
477
+ Returns:
478
+ Rows returned by the SQLite cursor, or ``None`` if the wrapped error path is raised.
479
+
480
+ Raises:
481
+ Error: Re-raised after the exception is wrapped and written to the application logger.
482
+ """
483
+ try:
484
+ throw_if( 'table', table )
485
+ self.sql = f'SELECT * FROM {table}'
486
+ self.cursor.execute( self.sql )
487
+ return self.cursor.fetchall( )
488
+ except Exception as e:
489
+ exception = Error( e )
490
+ exception.module = 'data'
491
+ exception.cause = 'SQLite'
492
+ exception.method = 'fetch_all( self, df: str ) -> List[ Tuple ]'
493
+ Logger( ).write( exception )
494
+ raise exception
495
+
496
+ def fetch_one( self, table: str, where: str, params: Tuple[ Any, ... ] ) -> Tuple | None:
497
+ """Fetch one matching row.
498
+
499
+ Purpose:
500
+ Builds a SELECT statement with a caller-provided WHERE clause, executes it with
501
+ the stored parameter tuple, and returns the first matching row from the active
502
+ cursor.
503
+
504
+ Args:
505
+ table (str): Table name to query.
506
+ where (str): WHERE clause fragment without the ``WHERE`` keyword.
507
+ params (Tuple[Any, ...]): Parameters intended for the WHERE clause.
508
+
509
+ Returns:
510
+ First matching row from the cursor, or ``None`` when no row is found.
511
+
512
+ Raises:
513
+ Error: Re-raised after the exception is wrapped and written to the application logger.
514
+ """
515
+ try:
516
+ throw_if( 'params', params )
517
+ throw_if( 'where', where )
518
+ throw_if( 'table', table )
519
+ self.table_name = table
520
+ self.where = where
521
+ self.sql = f'SELECT * FROM {self.table_name} WHERE {self.where} LIMIT 1'
522
+ self.cursor.execute( self.sql, self.params )
523
+ return self.cursor.fetchone( )
524
+ except Exception as e:
525
+ exception = Error( e )
526
+ exception.module = 'data'
527
+ exception.cause = 'SQLite'
528
+ exception.method = 'fetch_one( self, **kwargs )'
529
+ Logger( ).write( exception )
530
+ raise exception
531
+
532
+ def update( self, table: str, pairs: str, where: str, params: Tuple[ Any, ... ] ) -> None:
533
+ """Update matching rows.
534
+
535
+ Purpose:
536
+ Executes an UPDATE statement using the provided table name, SET clause fragment,
537
+ WHERE clause fragment, and SQLite parameter tuple. The method commits the active
538
+ connection after the update is executed.
539
+
540
+ Args:
541
+ table (str): Table name to update.
542
+ pairs (str): SET clause fragment containing target assignments.
543
+ where (str): WHERE clause fragment without the ``WHERE`` keyword.
544
+ params (Tuple[Any, ...]): Parameters for the SET and WHERE clauses.
545
+
546
+ Raises:
547
+ Error: Re-raised after the exception is wrapped and written to the application logger.
548
+ """
549
+ try:
550
+ throw_if( 'pairs', pairs )
551
+ throw_if( 'params', params )
552
+ throw_if( 'where', where )
553
+ throw_if( 'table', table )
554
+ self.table_name = table
555
+ self.where = where
556
+ self.params = params
557
+ self.sql = f'UPDATE {self.table_name} SET {pairs} WHERE {self.where}'
558
+ self.cursor.execute( self.sql, params )
559
+ self.connection.commit( )
560
+ except Exception as e:
561
+ exception = Error( e )
562
+ exception.module = 'data'
563
+ exception.cause = 'SQLite'
564
+ exception.method = 'update( self, **kwargs )'
565
+ Logger( ).write( exception )
566
+ raise exception
567
+
568
+ def delete( self, table: str, where: str, params: Tuple[ Any, ... ] ) -> None:
569
+ """Delete matching rows.
570
+
571
+ Purpose:
572
+ Executes a DELETE statement against the specified table using the provided WHERE
573
+ clause and parameter tuple. The method commits the active SQLite connection after
574
+ the delete operation completes.
575
+
576
+ Args:
577
+ table (str): Table name to delete from.
578
+ where (str): WHERE clause fragment without the ``WHERE`` keyword.
579
+ params (Tuple[Any, ...]): Parameters for the WHERE clause.
580
+
581
+ Raises:
582
+ Error: Re-raised after the exception is wrapped and written to the application logger.
583
+ """
584
+ try:
585
+ throw_if( 'where', where )
586
+ throw_if( 'table', table )
587
+ throw_if( 'params', params )
588
+ self.table_name = table
589
+ self.where = where
590
+ self.params = params
591
+ self.sql = f"DELETE FROM {self.table_name} WHERE {self.where}"
592
+ self.cursor.execute( self.sql, self.params )
593
+ self.connection.commit( )
594
+ except Exception as e:
595
+ exception = Error( e )
596
+ exception.module = 'data'
597
+ exception.cause = 'SQLite'
598
+ exception.method = 'delete( self, **kwargs )'
599
+ Logger( ).write( exception )
600
+ raise exception
601
+
602
+ def import_excel( self, path: str ) -> None:
603
+ """Import worksheets from an Excel workbook.
604
+
605
+ Purpose:
606
+ Reads every worksheet from an Excel workbook into pandas DataFrames and writes
607
+ each worksheet into the active SQLite database using the sheet name as the table
608
+ name. Existing tables with the same sheet names are replaced.
609
+
610
+ Args:
611
+ path (str): Path to the Excel workbook.
612
+
613
+ Raises:
614
+ Error: Re-raised after the exception is wrapped and written to the application logger.
615
+ """
616
+ try:
617
+ throw_if( 'path', path )
618
+ self.file_path = path
619
+ self.file_name = os.path.basename( self.file_path )
620
+ _excel = pd.ExcelFile( self.file_path )
621
+ for _sheet in _excel.sheet_names:
622
+ _df = _excel.parse( _sheet )
623
+ _df.to_sql( _sheet, self.connection, if_exists='replace', index=False )
624
+ except Exception as e:
625
+ exception = Error( e )
626
+ exception.module = 'data'
627
+ exception.cause = 'SQLite'
628
+ exception.method = 'import_excel( self, path: str ) -> None'
629
+ Logger( ).write( exception )
630
+ raise exception
631
+
632
+ def close( self ) -> None:
633
+ """Close the active database connection.
634
+
635
+ Purpose:
636
+ Closes the SQLite connection when it exists. The method preserves the current
637
+ defensive behavior by doing nothing when the connection member is already empty.
638
+
639
+ Raises:
640
+ Error: Re-raised after the exception is wrapped and written to the application logger.
641
+ """
642
+ try:
643
+ if self.connection is not None:
644
+ self.connection.close( )
645
+ except Exception as e:
646
+ exception = Error( e )
647
+ exception.module = 'data'
648
+ exception.cause = 'SQLite'
649
+ exception.method = 'close( self ) -> None'
650
+ Logger( ).write( exception )
651
+ raise exception
652
+
653
+ class Chroma:
654
+ """Provide persistent ChromaDB vector storage.
655
+
656
+ Purpose:
657
+ Wraps a ChromaDB client and collection for persistent embedding storage, similarity
658
+ querying, deletion, record counting, collection clearing, and persistence. The class
659
+ provides Foo vector workflows with a compact interface over the underlying ChromaDB
660
+ collection operations.
661
+
662
+ Attributes:
663
+ client (Optional[chromadb.Client]): Instantiated ChromaDB client.
664
+ collection (Optional[chromadb.Collection]): Vector collection used for add, query,
665
+ delete, count, clear, and persistence operations.
666
+ """
667
+ client: Optional[ chromadb.Client ]
668
+ collection: Optional[ chromadb.Collection ]
669
+
670
+ def __init__( self, path: str = './chroma', collection: str = 'embeddings' ) -> None:
671
+ """Initialize Chroma vector storage.
672
+
673
+ Purpose:
674
+ Initializes a ChromaDB client using the supplied persistence path and retrieves or
675
+ creates the named collection. This prepares the vector-store wrapper for document,
676
+ embedding, metadata, and semantic query operations.
677
+
678
+ Args:
679
+ path (str): Directory used by ChromaDB for persistent storage.
680
+ collection (str): Name of the ChromaDB collection to retrieve or create.
681
+ """
682
+ self.client = chromadb.Client(
683
+ Settings( persist_directory=path, anonymized_telemetry=False ) )
684
+ self.collection = self.client.get_or_create_collection( name=collection )
685
+
686
+ def add( self, ids: List[ str ], texts: List[ str ], embeddings: List[ List[ float ] ],
687
+ metadatas: Optional[ List[ dict ] ] = None ) -> None:
688
+ """Add records to the vector collection.
689
+
690
+ Purpose:
691
+ Adds document text, vector embeddings, unique identifiers, and optional metadata to
692
+ the configured ChromaDB collection. This method is the primary persistence path for
693
+ new vector-search records.
694
+
695
+ Args:
696
+ ids (List[str]): Unique identifiers for each vector-store record.
697
+ texts (List[str]): Document strings associated with the embeddings.
698
+ embeddings (List[List[float]]): Vector representations for the supplied documents.
699
+ metadatas (Optional[List[dict]]): Optional metadata dictionaries for filtering or tagging.
700
+
701
+ Raises:
702
+ Error: Re-raised after the exception is wrapped and written to the application logger.
703
+ """
704
+ try:
705
+ self.collection.add( documents=texts, embeddings=embeddings, ids=ids,
706
+ metadatas=metadatas )
707
+ except Exception as e:
708
+ exception = Error( e )
709
+ exception.module = 'Foo'
710
+ exception.cause = 'Chroma'
711
+ exception.method = 'add( self, **kwargs )'
712
+ Logger( ).write( exception )
713
+ raise exception
714
+
715
+ def query( self, text: List[ str ], num: int = 5, where: Optional[ dict ] = None ) -> List[
716
+ str ] | None:
717
+ """Query similar vector-store records.
718
+
719
+ Purpose:
720
+ Performs a ChromaDB similarity search using one or more query texts, a requested
721
+ result count, and an optional metadata filter. The method returns the first group
722
+ of matching documents from the ChromaDB query result.
723
+
724
+ Args:
725
+ text (List[str]): Query text values submitted to ChromaDB.
726
+ num (int): Number of top matches to request.
727
+ where (Optional[dict]): Optional metadata filter applied to the query.
728
+
729
+ Returns:
730
+ Most relevant document strings returned by the vector search.
731
+
732
+ Raises:
733
+ Error: Re-raised after the exception is wrapped and written to the application logger.
734
+ """
735
+ try:
736
+ result = self.collection.query( query_texts=text, n_results=num, where=where or { } )
737
+ return result.get( 'documents', [ ] )[ 0 ]
738
+ except Exception as e:
739
+ exception = Error( e )
740
+ exception.module = 'Foo'
741
+ exception.cause = 'Chroma'
742
+ exception.method = 'query( )'
743
+ Logger( ).write( exception )
744
+ raise exception
745
+
746
+ def delete( self, ids: List[ str ] ) -> None:
747
+ """Delete vector-store records.
748
+
749
+ Purpose:
750
+ Deletes one or more records from the configured ChromaDB collection by document ID.
751
+ This supports targeted cleanup of obsolete or incorrectly indexed vector records.
752
+
753
+ Args:
754
+ ids (List[str]): Unique document IDs to delete from the collection.
755
+
756
+ Raises:
757
+ Error: Re-raised after the exception is wrapped and written to the application logger.
758
+ """
759
+ try:
760
+ self.collection.delete( ids=ids )
761
+ except Exception as e:
762
+ exception = Error( e )
763
+ exception.module = 'Foo'
764
+ exception.cause = 'Chroma'
765
+ exception.method = 'delete( )'
766
+ Logger( ).write( exception )
767
+ raise exception
768
+
769
+ def count( self ) -> int | None:
770
+ """Count records in the vector collection.
771
+
772
+ Purpose:
773
+ Returns the total number of records stored in the configured ChromaDB collection.
774
+ The value can be used by diagnostics, UI summaries, and validation checks.
775
+
776
+ Returns:
777
+ Number of records in the ChromaDB collection.
778
+
779
+ Raises:
780
+ Error: Re-raised after the exception is wrapped and written to the application logger.
781
+ """
782
+ try:
783
+ return self.collection.count( )
784
+ except Exception as e:
785
+ exception = Error( e )
786
+ exception.module = 'Foo'
787
+ exception.cause = 'Chroma'
788
+ exception.method = 'count( )'
789
+ Logger( ).write( exception )
790
+ raise exception
791
+
792
+ def clear( self ) -> None:
793
+ """Clear the vector collection.
794
+
795
+ Purpose:
796
+ Deletes all documents from the configured ChromaDB collection using the existing
797
+ collection-wide delete behavior. This supports resetting local vector-store state
798
+ for rebuild or testing workflows.
799
+
800
+ Raises:
801
+ Error: Re-raised after the exception is wrapped and written to the application logger.
802
+ """
803
+ try:
804
+ self.collection.delete( where={ } )
805
+ except Exception as e:
806
+ exception = Error( e )
807
+ exception.module = 'Foo'
808
+ exception.cause = 'Chroma'
809
+ exception.method = 'clear( )'
810
+ Logger( ).write( exception )
811
+ raise exception
812
+
813
+ def persist( self ) -> None:
814
+ """Persist vector-store state.
815
+
816
+ Purpose:
817
+ Calls the ChromaDB client persistence operation to save the current collection
818
+ state to disk. This preserves vector-store changes for later application runs.
819
+
820
+ Raises:
821
+ Error: Re-raised after the exception is wrapped and written to the application logger.
822
+ """
823
+ try:
824
+ self.client.persist( )
825
+ except Exception as e:
826
+ exception = Error( e )
827
+ exception.module = 'Foo'
828
+ exception.cause = 'Chroma'
829
+ exception.method = 'persist'
830
+ Logger( ).write( exception )
831
+ raise exception
832
+