tellaro-query-language 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tellaro_query_language-0.1.0.dist-info/LICENSE +21 -0
- tellaro_query_language-0.1.0.dist-info/METADATA +401 -0
- tellaro_query_language-0.1.0.dist-info/RECORD +56 -0
- tellaro_query_language-0.1.0.dist-info/WHEEL +4 -0
- tellaro_query_language-0.1.0.dist-info/entry_points.txt +7 -0
- tql/__init__.py +47 -0
- tql/analyzer.py +385 -0
- tql/cache/__init__.py +7 -0
- tql/cache/base.py +25 -0
- tql/cache/memory.py +63 -0
- tql/cache/redis.py +68 -0
- tql/core.py +929 -0
- tql/core_components/README.md +92 -0
- tql/core_components/__init__.py +20 -0
- tql/core_components/file_operations.py +113 -0
- tql/core_components/opensearch_operations.py +869 -0
- tql/core_components/stats_operations.py +200 -0
- tql/core_components/validation_operations.py +599 -0
- tql/evaluator.py +379 -0
- tql/evaluator_components/README.md +131 -0
- tql/evaluator_components/__init__.py +17 -0
- tql/evaluator_components/field_access.py +176 -0
- tql/evaluator_components/special_expressions.py +296 -0
- tql/evaluator_components/value_comparison.py +315 -0
- tql/exceptions.py +160 -0
- tql/geoip_normalizer.py +233 -0
- tql/mutator_analyzer.py +830 -0
- tql/mutators/__init__.py +222 -0
- tql/mutators/base.py +78 -0
- tql/mutators/dns.py +316 -0
- tql/mutators/encoding.py +218 -0
- tql/mutators/geo.py +363 -0
- tql/mutators/list.py +212 -0
- tql/mutators/network.py +163 -0
- tql/mutators/security.py +225 -0
- tql/mutators/string.py +165 -0
- tql/opensearch.py +78 -0
- tql/opensearch_components/README.md +130 -0
- tql/opensearch_components/__init__.py +17 -0
- tql/opensearch_components/field_mapping.py +399 -0
- tql/opensearch_components/lucene_converter.py +305 -0
- tql/opensearch_components/query_converter.py +775 -0
- tql/opensearch_mappings.py +309 -0
- tql/opensearch_stats.py +451 -0
- tql/parser.py +1363 -0
- tql/parser_components/README.md +72 -0
- tql/parser_components/__init__.py +20 -0
- tql/parser_components/ast_builder.py +162 -0
- tql/parser_components/error_analyzer.py +101 -0
- tql/parser_components/field_extractor.py +112 -0
- tql/parser_components/grammar.py +473 -0
- tql/post_processor.py +737 -0
- tql/scripts.py +124 -0
- tql/stats_evaluator.py +444 -0
- tql/stats_transformer.py +184 -0
- tql/validators.py +110 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Tellaro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: tellaro-query-language
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A flexible, human-friendly query language for searching and filtering structured data
|
|
5
|
+
Home-page: https://github.com/tellaro/tellaro-query-language
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: query,language,opensearch,elasticsearch,search,filter,tql
|
|
8
|
+
Author: Justin Henderson
|
|
9
|
+
Author-email: justin@tellaro.io
|
|
10
|
+
Requires-Python: >=3.11,<3.14
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
20
|
+
Provides-Extra: opensearch
|
|
21
|
+
Requires-Dist: dnspython (>=2.7.0,<3.0.0)
|
|
22
|
+
Requires-Dist: maxminddb (>=2.7.0,<3.0.0)
|
|
23
|
+
Requires-Dist: opensearch-dsl (>=2.1.0,<3.0.0) ; extra == "opensearch"
|
|
24
|
+
Requires-Dist: opensearch-py (>=2.4.2,<3.0.0) ; extra == "opensearch"
|
|
25
|
+
Requires-Dist: pyparsing (>=3.2.1,<4.0.0)
|
|
26
|
+
Requires-Dist: python-dotenv (>=1.0.1,<2.0.0) ; extra == "opensearch"
|
|
27
|
+
Requires-Dist: setuptools (>=80.0.0,<81.0.0)
|
|
28
|
+
Project-URL: Documentation, https://github.com/tellaro/tellaro-query-language/tree/main/docs
|
|
29
|
+
Project-URL: Repository, https://github.com/tellaro/tellaro-query-language
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# Tellaro Query Language
|
|
33
|
+
|
|
34
|
+
[](https://badge.fury.io/py/tellaro-query-language)
|
|
35
|
+
[](./reports/pytest/junit.xml) [](./reports/coverage/index.html) [](./reports/flake8/index.html)
|
|
36
|
+
[](https://www.python.org/)
|
|
37
|
+
[](https://opensource.org/licenses/MIT)
|
|
38
|
+
|
|
39
|
+
## What is TQL?
|
|
40
|
+
|
|
41
|
+
Tellaro Query Language (TQL) is a flexible, human-friendly query language for searching and filtering structured data. TQL is designed to provide a unified, readable syntax for expressing complex queries, supporting both simple and advanced search scenarios. It is especially useful for environments where data may come from different backends (such as OpenSearch or JSON files) and where users want to write queries that are portable and easy to understand.
|
|
42
|
+
|
|
43
|
+
TQL supports:
|
|
44
|
+
- **Field selection** (including nested fields)
|
|
45
|
+
- **Comparison and logical operators**
|
|
46
|
+
- **String, number, and list values**
|
|
47
|
+
- **Collection operators** (ANY, ALL) for working with list fields
|
|
48
|
+
- **Mutators** for post-processing or transforming field values
|
|
49
|
+
- **Operator precedence and parenthetical grouping** (AND, OR, NOT, etc.)
|
|
50
|
+
- **Field extraction** for analyzing query dependencies
|
|
51
|
+
- **Multiple backends** (in-memory evaluation, OpenSearch, file operations)
|
|
52
|
+
- **Statistical aggregations** for data analysis
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## TQL Syntax Overview
|
|
57
|
+
|
|
58
|
+
### Basic Query Structure
|
|
59
|
+
|
|
60
|
+
TQL queries are generally structured as:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
field [| mutator1 | mutator2 ...] operator value
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
- **field**: The field to query (e.g., `computer.name`, `os.ver`).
|
|
67
|
+
- **mutator**: (Optional) One or more transformations to apply to the field before comparison (e.g., `| lowercase`).
|
|
68
|
+
- **operator**: The comparison operator (e.g., `eq`, `contains`, `in`, `>`, `regexp`).
|
|
69
|
+
- **value**: The value to compare against (string, number, identifier, or list).
|
|
70
|
+
|
|
71
|
+
#### Example
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
computer.name | lowercase eq 'ha-jhend'
|
|
75
|
+
os.ver > 10
|
|
76
|
+
os.dataset in ['windows_server', 'enterprise desktop']
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Mutators
|
|
80
|
+
|
|
81
|
+
Mutators allow you to transform field values before comparison. For example, `| lowercase` will convert the field value to lowercase before evaluating the condition.
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
user.email | lowercase eq 'admin@example.com'
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Operators
|
|
88
|
+
|
|
89
|
+
TQL supports a variety of comparison operators, including:
|
|
90
|
+
|
|
91
|
+
- `eq`, `=`, `ne`, `!=` (equals, not equals)
|
|
92
|
+
- `>`, `>=`, `<`, `<=` (greater/less than)
|
|
93
|
+
- `contains`, `in`, `regexp`, `startswith`, `endswith`
|
|
94
|
+
- `is`, `exists`, `range`, `between`, `cidr`
|
|
95
|
+
|
|
96
|
+
### Values
|
|
97
|
+
|
|
98
|
+
Values can be:
|
|
99
|
+
- **Strings**: `'value'` or `"value"`
|
|
100
|
+
- **Numbers**: `123`, `42`, `1.01`
|
|
101
|
+
- **Identifiers**: `computer01`, `admin`
|
|
102
|
+
- **Lists**: `["val1", "val2"]`
|
|
103
|
+
|
|
104
|
+
### Logical Expressions
|
|
105
|
+
|
|
106
|
+
TQL supports logical operators and grouping:
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
field1 eq 'foo' AND (field2 > 10 OR field3 in ['a', 'b'])
|
|
110
|
+
NOT field4 contains 'bar'
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Operators supported: `AND`, `OR`, `NOT`, `ANY`, `ALL` (case-insensitive)
|
|
114
|
+
|
|
115
|
+
### Example Query
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
computer.name | lowercase eq 'ha-jhend' AND (os.ver > 10 OR os.dataset in ['windows_server', 'enterprise desktop'])
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## Why TQL Matters
|
|
124
|
+
|
|
125
|
+
TQL provides a consistent, readable way to express queries across different data sources. It abstracts away backend-specific quirks (like OpenSearch's text vs. keyword fields) and lets users focus on what they want to find, not how to write backend-specific queries.
|
|
126
|
+
|
|
127
|
+
**Key benefits:**
|
|
128
|
+
- **Unified syntax**: Write one query, run it on many backends.
|
|
129
|
+
- **Mutators**: Easily transform data inline (e.g., lowercase, trim).
|
|
130
|
+
- **Readability**: Queries are easy to read and write, even for complex logic.
|
|
131
|
+
- **Extensible**: New operators and mutators can be added as needed.
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Example: TQL in Action
|
|
136
|
+
|
|
137
|
+
Suppose you want to find computers named "HA-JHEND" (case-insensitive), running Windows Server or Enterprise Desktop, and with an OS version greater than 10:
|
|
138
|
+
|
|
139
|
+
```
|
|
140
|
+
computer.name | lowercase eq 'ha-jhend' AND (os.ver > 10 OR os.dataset in ['windows_server', 'enterprise desktop'])
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
This query will:
|
|
144
|
+
- Convert `computer.name` to lowercase and compare to `'ha-jhend'`
|
|
145
|
+
- Check if `os.ver` is greater than 10
|
|
146
|
+
- Check if `os.dataset` is in the provided list
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Implementation Notes
|
|
151
|
+
|
|
152
|
+
TQL is implemented using [pyparsing](https://pyparsing-docs.readthedocs.io/en/latest/) to define the grammar and parse queries. The parser supports mutators, operator precedence, and both standard and reversed operator forms (e.g., `'value' in field`).
|
|
153
|
+
|
|
154
|
+
See `src/tql/` for the implementation, including the parser grammar and evaluation logic.
|
|
155
|
+
|
|
156
|
+
## Documentation
|
|
157
|
+
|
|
158
|
+
For comprehensive documentation, see the [`docs/`](./docs/) folder:
|
|
159
|
+
|
|
160
|
+
- **[Getting Started](./docs/getting-started.md)** - Learn TQL basics with development examples
|
|
161
|
+
- **[Development Guide](./docs/development-guide.md)** - File operations, testing, and common patterns
|
|
162
|
+
- **[OpenSearch Integration](./docs/opensearch-integration.md)** - Convert TQL to OpenSearch DSL and Lucene queries
|
|
163
|
+
- **[Syntax Reference](./docs/syntax-reference.md)** - Complete grammar and syntax specification
|
|
164
|
+
- **[Operators](./docs/operators.md)** - All comparison and logical operators
|
|
165
|
+
- **[Mutators](./docs/mutators.md)** - Field transformation functions (25+ mutators available)
|
|
166
|
+
- **[Stats & Aggregations](./docs/stats.md)** - Statistical analysis and data aggregation functions
|
|
167
|
+
- **[Examples](./docs/examples.md)** - Real-world query examples for security, DevOps, and business use cases
|
|
168
|
+
- **[Best Practices](./docs/best-practices.md)** - Performance optimization and maintainability tips
|
|
169
|
+
|
|
170
|
+
## Quick Start
|
|
171
|
+
|
|
172
|
+
### Installation
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
# Install from PyPI
|
|
176
|
+
pip install tellaro-query-language
|
|
177
|
+
|
|
178
|
+
# Or install with OpenSearch support
|
|
179
|
+
pip install tellaro-query-language[opensearch]
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Basic Usage
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from tql import TQL
|
|
186
|
+
|
|
187
|
+
# Initialize TQL
|
|
188
|
+
tql = TQL()
|
|
189
|
+
|
|
190
|
+
# Query data
|
|
191
|
+
data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
|
|
192
|
+
results = tql.query(data, 'age > 27')
|
|
193
|
+
print(f'Found {len(results)} people over 27: {results}')
|
|
194
|
+
# Output: Found 1 people over 27: [{'name': 'Alice', 'age': 30}]
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Development Setup
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
# Clone the repository
|
|
201
|
+
git clone https://github.com/tellaro/tellaro-query-language.git
|
|
202
|
+
cd tellaro-query-language
|
|
203
|
+
|
|
204
|
+
# Install with poetry
|
|
205
|
+
poetry install
|
|
206
|
+
|
|
207
|
+
# Run tests
|
|
208
|
+
poetry run tests
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
### File Operations
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
from src.tql import TQL
|
|
215
|
+
|
|
216
|
+
# Query JSON files directly
|
|
217
|
+
tql = TQL()
|
|
218
|
+
results = tql.query("data.json", "user.role eq 'admin' AND status eq 'active'")
|
|
219
|
+
|
|
220
|
+
# Query with field mappings for OpenSearch
|
|
221
|
+
mappings = {"hostname": "agent.name.keyword"}
|
|
222
|
+
tql_mapped = TQL(mappings)
|
|
223
|
+
opensearch_dsl = tql_mapped.to_opensearch("hostname eq 'server01'")
|
|
224
|
+
|
|
225
|
+
# Extract fields from a complex query
|
|
226
|
+
query = "process.name eq 'explorer.exe' AND (user.id eq 'admin' OR user.groups contains 'administrators')"
|
|
227
|
+
fields = tql.extract_fields(query)
|
|
228
|
+
print(fields) # ['process.name', 'user.groups', 'user.id']
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
### Query Analysis and Health Evaluation
|
|
232
|
+
|
|
233
|
+
TQL provides context-aware query analysis to help you understand performance implications before execution:
|
|
234
|
+
|
|
235
|
+
```python
|
|
236
|
+
from src.tql import TQL
|
|
237
|
+
|
|
238
|
+
tql = TQL()
|
|
239
|
+
|
|
240
|
+
# Analyze for in-memory execution (default)
|
|
241
|
+
query = "field | lowercase | trim eq 'test'"
|
|
242
|
+
analysis = tql.analyze_query(query) # or explicitly: analyze_query(query, context="in_memory")
|
|
243
|
+
|
|
244
|
+
print(f"Health: {analysis['health']['status']}") # 'good' - fast mutators don't impact in-memory
|
|
245
|
+
print(f"Score: {analysis['health']['score']}") # 100
|
|
246
|
+
print(f"Has mutators: {analysis['stats']['has_mutators']}") # True
|
|
247
|
+
|
|
248
|
+
# Analyze the same query for OpenSearch execution
|
|
249
|
+
analysis = tql.analyze_query(query, context="opensearch")
|
|
250
|
+
print(f"Health: {analysis['health']['status']}") # 'fair' - post-processing required
|
|
251
|
+
print(f"Score: {analysis['health']['score']}") # 85
|
|
252
|
+
|
|
253
|
+
# Check mutator-specific health
|
|
254
|
+
if 'mutator_health' in analysis:
|
|
255
|
+
print(f"Mutator health: {analysis['mutator_health']['health_status']}")
|
|
256
|
+
for reason in analysis['mutator_health']['health_reasons']:
|
|
257
|
+
print(f" - {reason['reason']}")
|
|
258
|
+
|
|
259
|
+
# Slow mutators impact both contexts
|
|
260
|
+
slow_query = "hostname | nslookup contains 'example.com'"
|
|
261
|
+
analysis = tql.analyze_query(slow_query)
|
|
262
|
+
print(f"In-memory health: {analysis['health']['status']}") # 'fair' or 'poor' - network I/O
|
|
263
|
+
|
|
264
|
+
# Query complexity analysis
|
|
265
|
+
complex_query = "(a > 1 OR b < 2) AND (c = 3 OR (d = 4 AND e = 5))"
|
|
266
|
+
analysis = tql.analyze_query(complex_query)
|
|
267
|
+
print(f"Depth: {analysis['complexity']['depth']}")
|
|
268
|
+
print(f"Fields: {analysis['stats']['fields']}")
|
|
269
|
+
print(f"Operators: {analysis['stats']['operators']}")
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
### Post-Processing with OpenSearch
|
|
273
|
+
|
|
274
|
+
TQL intelligently handles mutators based on field mappings. When OpenSearch can't perform certain operations (like case-insensitive searches on keyword fields), TQL applies post-processing:
|
|
275
|
+
|
|
276
|
+
```python
|
|
277
|
+
# Field mappings with only keyword fields
|
|
278
|
+
mappings = {"username": {"type": "keyword"}, "department": {"type": "keyword"}}
|
|
279
|
+
tql = TQL(mappings)
|
|
280
|
+
|
|
281
|
+
# This query requires post-processing since keyword fields can't do case-insensitive contains
|
|
282
|
+
query = "username | lowercase contains 'admin' AND department eq 'Engineering'"
|
|
283
|
+
|
|
284
|
+
# Analyze the query (analyze_opensearch_query is deprecated, use analyze_query instead)
|
|
285
|
+
analysis = tql.analyze_query(query, context="opensearch")
|
|
286
|
+
print(f"Health: {analysis['health']['status']}") # 'fair' (post-processing required)
|
|
287
|
+
|
|
288
|
+
# Execute with automatic post-processing
|
|
289
|
+
result = tql.execute_opensearch(
|
|
290
|
+
opensearch_client=client,
|
|
291
|
+
index="users",
|
|
292
|
+
query=query
|
|
293
|
+
)
|
|
294
|
+
# OpenSearch returns all Engineering users, TQL filters to only those with 'admin' in username
|
|
295
|
+
|
|
296
|
+
# Run the demo to see this in action
|
|
297
|
+
# poetry run python post_processing_demo.py
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
### Development Examples
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
# Run comprehensive demos
|
|
304
|
+
poetry run python demo.py # Basic functionality
|
|
305
|
+
poetry run python intelligent_mapping_demo.py # Field mapping features
|
|
306
|
+
poetry run python test_requested_functionality.py # Core functionality tests
|
|
307
|
+
poetry run python field_extraction_demo.py # Field extraction
|
|
308
|
+
poetry run python post_processing_demo.py # Post-processing filtering
|
|
309
|
+
|
|
310
|
+
# Run tests
|
|
311
|
+
poetry run pytest tests/ -v
|
|
312
|
+
|
|
313
|
+
# Run integration tests with OpenSearch (requires OpenSearch)
|
|
314
|
+
# 1. Copy .env.example to .env and configure connection settings
|
|
315
|
+
# 2. Set OPENSEARCH_INTEGRATION_TEST=true in .env
|
|
316
|
+
poetry run pytest tests/test_opensearch_integration.py -v
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
## Contributing
|
|
320
|
+
|
|
321
|
+
TQL supports 25+ mutators including string manipulation, encoding/decoding, DNS operations, and network analysis. See the [Mutators documentation](./docs/mutators.md) for the complete list.
|
|
322
|
+
|
|
323
|
+
To add new mutators or operators, see the implementation in `src/tql/mutators.py` and `src/tql/parser.py`.
|
|
324
|
+
|
|
325
|
+
### Statistical Aggregations
|
|
326
|
+
|
|
327
|
+
TQL supports powerful data analysis with stats expressions:
|
|
328
|
+
|
|
329
|
+
```tql
|
|
330
|
+
# Simple aggregation
|
|
331
|
+
| stats sum(revenue)
|
|
332
|
+
|
|
333
|
+
# Grouped analysis
|
|
334
|
+
| stats count(requests), average(response_time) by server_name
|
|
335
|
+
|
|
336
|
+
# Top N analysis
|
|
337
|
+
| stats sum(sales, top 10) by product_category
|
|
338
|
+
|
|
339
|
+
# Complex analytics
|
|
340
|
+
status eq 'success'
|
|
341
|
+
| stats count(requests), sum(bytes), average(response_time), max(cpu_usage) by endpoint
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
Stats functions include: `sum`, `min`, `max`, `count`, `unique_count`, `average`, `median`, `percentile_rank`, `zscore`, `std`
|
|
345
|
+
|
|
346
|
+
## Documentation
|
|
347
|
+
|
|
348
|
+
Comprehensive documentation is available in the [docs](./docs/) directory:
|
|
349
|
+
|
|
350
|
+
- [**Getting Started**](./docs/getting-started.md) - Quick introduction to TQL
|
|
351
|
+
- [**Syntax Reference**](./docs/syntax-reference.md) - Complete syntax guide
|
|
352
|
+
- [**Operators**](./docs/operators.md) - All comparison and logical operators
|
|
353
|
+
- [**Mutators**](./docs/mutators.md) - Field transformation functions
|
|
354
|
+
- [**Mutator Caching & Security**](./docs/mutator-caching.md) - Performance optimization and security controls
|
|
355
|
+
- [**OpenSearch Integration**](./docs/opensearch-integration.md) - Using TQL with OpenSearch
|
|
356
|
+
- [**Examples**](./docs/examples.md) - Real-world query examples
|
|
357
|
+
- [**Architecture**](./docs/architecture.md) - Modular architecture and design
|
|
358
|
+
- [**Migration Guide**](./docs/migration-guide.md) - Upgrading from older versions
|
|
359
|
+
|
|
360
|
+
## Development
|
|
361
|
+
|
|
362
|
+
### Installation
|
|
363
|
+
|
|
364
|
+
```bash
|
|
365
|
+
# Clone the repository
|
|
366
|
+
git clone https://github.com/tellaro/tellaro-query-language.git
|
|
367
|
+
cd tellaro-query-language
|
|
368
|
+
|
|
369
|
+
# Install with poetry
|
|
370
|
+
poetry install
|
|
371
|
+
|
|
372
|
+
# Or install with pip
|
|
373
|
+
pip install -e .
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
### Testing
|
|
377
|
+
|
|
378
|
+
This project supports Python 3.11, 3.12, 3.13, and 3.14. We use `nox` for automated testing across all versions.
|
|
379
|
+
|
|
380
|
+
```bash
|
|
381
|
+
# Install test dependencies
|
|
382
|
+
poetry install --with dev
|
|
383
|
+
|
|
384
|
+
# Run tests on all Python versions
|
|
385
|
+
poetry run nox -s tests
|
|
386
|
+
|
|
387
|
+
# Run tests on a specific version
|
|
388
|
+
poetry run nox -s tests-3.12
|
|
389
|
+
|
|
390
|
+
# Quick test run (fail fast, no coverage)
|
|
391
|
+
poetry run nox -s test_quick
|
|
392
|
+
|
|
393
|
+
# Run linting and formatting
|
|
394
|
+
poetry run nox -s lint
|
|
395
|
+
poetry run nox -s format
|
|
396
|
+
|
|
397
|
+
# Run all checks
|
|
398
|
+
poetry run nox -s all
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
For more detailed testing instructions, see [TESTING.md](TESTING.md).
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
tql/__init__.py,sha256=mRrrtun-1Xx9k8g0aaiqxYhhNAwWEyRC4zrVMO49Kkg,1260
|
|
2
|
+
tql/analyzer.py,sha256=Sfzj6f7YzqylT8HIL9hDbXdhl0lf8q8DNoafrxkD-F8,15456
|
|
3
|
+
tql/cache/__init__.py,sha256=GIzIEMZUZEYJj72sAhuVLEG-OJEKUG2srUWNM3Ix-T8,213
|
|
4
|
+
tql/cache/base.py,sha256=0b-8uyh3JltayGmXQI45snTqsM5sQu9u0KcNvZIRa-I,687
|
|
5
|
+
tql/cache/memory.py,sha256=ibcmQSAxNvqCy6DksbU7gLu6UArYp1u3fW-oLubxtV0,2056
|
|
6
|
+
tql/cache/redis.py,sha256=ZU_IsVDvpSYpNvPfnZ4iulJDODpEGx3c4dkXLzPzPVc,2309
|
|
7
|
+
tql/core.py,sha256=HXz_95f9-yuBiI5iYGAPrpeqzokgSC5bsitOBLheuDU,37278
|
|
8
|
+
tql/core_components/README.md,sha256=Rm7w4UHdQ0vPBEFybE5b62IOvSA5Nzq2GRvtBHOapmc,3068
|
|
9
|
+
tql/core_components/__init__.py,sha256=v8BBybPlqV7dkVY9mw1mblvqyAFJZ7Pf_bEc-jAL7FI,643
|
|
10
|
+
tql/core_components/file_operations.py,sha256=Jr0kkxz_OP2KHOAsIr7KMtYe_lbu8LuBUySt2LQbjJw,3925
|
|
11
|
+
tql/core_components/opensearch_operations.py,sha256=x0GPdQfMX-NYCzzAlJCbtzYFa6HmaKxGEVl2JdAhKJY,35447
|
|
12
|
+
tql/core_components/stats_operations.py,sha256=jNk7cQZLwwsIoN__hdFHME8e17AjDHYLvZ-5kwPxNUY,6853
|
|
13
|
+
tql/core_components/validation_operations.py,sha256=_VPXh0HABBjsXF99jFT7B6-5QAPsADOCy6poinGrxeE,22454
|
|
14
|
+
tql/evaluator.py,sha256=YdgS1vuxUEPAHhUsZey-Y4NydeS8CTYOy_O8R5_K8cE,15421
|
|
15
|
+
tql/evaluator_components/README.md,sha256=c59yf2au34yPhrru7JWgGop_ORteB6w5vfMhsac8j3k,3882
|
|
16
|
+
tql/evaluator_components/__init__.py,sha256=DourRUSYXWPnCghBFj7W0YfMeymT3X8YTDCwnLIyP1c,535
|
|
17
|
+
tql/evaluator_components/field_access.py,sha256=BuXvL9jlv4H77neT70Vh7_qokmzs-d4EbSDA2FB1IT0,6435
|
|
18
|
+
tql/evaluator_components/special_expressions.py,sha256=rC1klZRA6cXLeS14xBi5LYYMi0V1qU3yg6_B4etRRBs,12139
|
|
19
|
+
tql/evaluator_components/value_comparison.py,sha256=A-RJ_52FLvsMO0ciEhJEdfhfFsoE4c3viNk7qYMFILI,15636
|
|
20
|
+
tql/exceptions.py,sha256=hatIixXci6p57J9RrkfdvmKM_2i-JKb8ViL2kU4z7a8,5550
|
|
21
|
+
tql/geoip_normalizer.py,sha256=tvie-5xevJEeLp2KmjoXDjYdND8AvyVE7lCO8qgUzGY,10486
|
|
22
|
+
tql/mutator_analyzer.py,sha256=nCP64sVmEVV_zhpACfuTVTcuRV2NmypGbkWgS2taWOs,38039
|
|
23
|
+
tql/mutators/__init__.py,sha256=zuzv6OMu2kVveDI85mu-6FA_-CZHsJEnIBIgtsjfXgY,6798
|
|
24
|
+
tql/mutators/base.py,sha256=4Ze_x1sTO11OILXfcF2XN7ttyHcZ4gwn96UXFMMaC6M,2523
|
|
25
|
+
tql/mutators/dns.py,sha256=Rmn6Kv-BmhfhMSYJ4GrkNmYms5rmpcxuG6leNxJspOQ,13387
|
|
26
|
+
tql/mutators/encoding.py,sha256=yt12BJrHAIJfBesP8VOSfVlvJqB1yOmEeT_8QDPvNN8,7985
|
|
27
|
+
tql/mutators/geo.py,sha256=fFQSg_Li3KjFKS3TI26yDzrDpWsmC3MfmgcsxYoQMgM,14507
|
|
28
|
+
tql/mutators/list.py,sha256=949ZrKKhL4INkH2Od8bq7Ey80kFX_23PEfRKueG82cU,7084
|
|
29
|
+
tql/mutators/network.py,sha256=1lZpmKt1GoTfNxiXUmSXkTwJIzPQZnQEgU7ojpBSm3A,5458
|
|
30
|
+
tql/mutators/security.py,sha256=BE5x1IFCjq85UD22Q2sgSue_xXwhCk0umvf91a1XqJY,8202
|
|
31
|
+
tql/mutators/string.py,sha256=8CzqZ2G5vpDeaEQ_jSA3NRidfUqfOrkUT9o2cmUUAEA,6288
|
|
32
|
+
tql/opensearch.py,sha256=J7LhfVJfaXEWtyZqVDqNZaeIbIcUYJr2cQtfKzdyIhM,3362
|
|
33
|
+
tql/opensearch_components/README.md,sha256=gt-qLmmach8Kh7-QwLZmoAxxIL79XIG1EDqJum8PMZE,3756
|
|
34
|
+
tql/opensearch_components/__init__.py,sha256=_zIZY8Fns7mkEcY6w2p9FNRBXtEmmPFFJEcFRfrVyXA,514
|
|
35
|
+
tql/opensearch_components/field_mapping.py,sha256=N4r7VkzNeXjIhNDt2cfnd1LbkvaS_9O298PIDAcR_Hw,17569
|
|
36
|
+
tql/opensearch_components/lucene_converter.py,sha256=ZbupWZ2smGhWfE9cSIrNo7MZVY35l2t86HYM-bd7nKw,12436
|
|
37
|
+
tql/opensearch_components/query_converter.py,sha256=9Nkhehb8X7jIQcyRaHCa5WCKWtovdRGKuSdDsBulrG0,36722
|
|
38
|
+
tql/opensearch_mappings.py,sha256=zJCCdMrxK7mswrkxd5LiOhunQ9GIJNZdhktVoGXgVgk,11529
|
|
39
|
+
tql/opensearch_stats.py,sha256=svcE23lD1gDe1iYGlC388WE8X3x9C8tjfBylUVy2Uqo,17216
|
|
40
|
+
tql/parser.py,sha256=6WbCpMuYOJymSYHEgTFS7yTgCOPXsCmGsi9m_u5oQzc,73954
|
|
41
|
+
tql/parser_components/README.md,sha256=lvQX72ckq2zyotGs8QIHHCIFqaA7bOHwkP44wU8Zoiw,2322
|
|
42
|
+
tql/parser_components/__init__.py,sha256=zBwHBMPJyHSBbaOojf6qTrJYjJg5A6tPUE8nHFdRiQs,521
|
|
43
|
+
tql/parser_components/ast_builder.py,sha256=-pbcYhZNoRm0AnjmJRAAlXLCAwHfauchTpX_6KO0plE,6793
|
|
44
|
+
tql/parser_components/error_analyzer.py,sha256=qlCD9vKyW73aeKQYI33P1OjIWSJ3LPd08wuN9cis2fU,4012
|
|
45
|
+
tql/parser_components/field_extractor.py,sha256=TumeuUo2c5gPYVbTPsmU43C3TJFC8chAAWERu5v_Q3c,4182
|
|
46
|
+
tql/parser_components/grammar.py,sha256=bfThefrhdbCMPLWb9ZXwZDXx8ZW8_PXihjwCL-P3Gi8,18669
|
|
47
|
+
tql/post_processor.py,sha256=-vA2wgbuLij2FVnj5I9HDHtw5bKj9Cu3EE9mtoeSWk8,28859
|
|
48
|
+
tql/scripts.py,sha256=VOr5vCjIvKlW36kwvJx7JGFIRM16IJZlbJcWlBexBtk,3728
|
|
49
|
+
tql/stats_evaluator.py,sha256=7ymCebPOepY3QKHGz0QEhkTmYYHjck648rrI8cWhhDI,15158
|
|
50
|
+
tql/stats_transformer.py,sha256=MT-4rDWZSySgn4Fuq9H0c-mvwFYLM6FqWpPv2rHX-rE,7588
|
|
51
|
+
tql/validators.py,sha256=e9MlX-zQ_O3M8YP8vXyMjKU8iiJMTh6mMK0iv0_4gTY,3771
|
|
52
|
+
tellaro_query_language-0.1.0.dist-info/LICENSE,sha256=zRhQ85LnW55fWgAjQctckwQ67DX5Jmt64lq343ThZFU,1063
|
|
53
|
+
tellaro_query_language-0.1.0.dist-info/METADATA,sha256=8C1D6P1nsAfTpWWEuRP43wCVLnV33jnUAR5LDV7MbRY,14567
|
|
54
|
+
tellaro_query_language-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
55
|
+
tellaro_query_language-0.1.0.dist-info/entry_points.txt,sha256=H43APfGBMsZkKsUCnFTaqprQPW-Kce2yz2qsBL3dZrw,164
|
|
56
|
+
tellaro_query_language-0.1.0.dist-info/RECORD,,
|
tql/__init__.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Tellaro Query Language package.
|
|
2
|
+
|
|
3
|
+
A flexible, human-friendly query language for searching and filtering structured data.
|
|
4
|
+
TQL provides a unified syntax for expressing complex queries with support for:
|
|
5
|
+
- Field selection (including nested fields)
|
|
6
|
+
- Comparison and logical operators
|
|
7
|
+
- String, number, and list values
|
|
8
|
+
- Mutators for field transformation
|
|
9
|
+
- Direct file operations and OpenSearch integration
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from .core import TQL
|
|
13
|
+
from .exceptions import (
|
|
14
|
+
TQLExecutionError,
|
|
15
|
+
TQLFieldError,
|
|
16
|
+
TQLOperatorError,
|
|
17
|
+
TQLParseError,
|
|
18
|
+
TQLSyntaxError,
|
|
19
|
+
TQLTypeError,
|
|
20
|
+
TQLUnsupportedOperationError,
|
|
21
|
+
TQLValidationError,
|
|
22
|
+
TQLValueError,
|
|
23
|
+
)
|
|
24
|
+
from .opensearch import OpenSearchBackend
|
|
25
|
+
from .opensearch_mappings import (
|
|
26
|
+
discover_field_mappings_for_query,
|
|
27
|
+
extract_field_mappings_from_opensearch,
|
|
28
|
+
get_sample_data_from_index,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
__version__ = "0.1.0"
|
|
32
|
+
__all__ = [
|
|
33
|
+
"TQL",
|
|
34
|
+
"TQLParseError",
|
|
35
|
+
"TQLExecutionError",
|
|
36
|
+
"TQLValidationError",
|
|
37
|
+
"TQLSyntaxError",
|
|
38
|
+
"TQLTypeError",
|
|
39
|
+
"TQLFieldError",
|
|
40
|
+
"TQLOperatorError",
|
|
41
|
+
"TQLValueError",
|
|
42
|
+
"TQLUnsupportedOperationError",
|
|
43
|
+
"OpenSearchBackend",
|
|
44
|
+
"extract_field_mappings_from_opensearch",
|
|
45
|
+
"discover_field_mappings_for_query",
|
|
46
|
+
"get_sample_data_from_index",
|
|
47
|
+
]
|