filter-pipe 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- filter_pipe-1.0.0/.devcontainer/Dockerfile +4 -0
- filter_pipe-1.0.0/.devcontainer/devcontainer.json +30 -0
- filter_pipe-1.0.0/.devcontainer/setup_devcontainer.sh +6 -0
- filter_pipe-1.0.0/.github/workflows/build.yaml +27 -0
- filter_pipe-1.0.0/.github/workflows/ci.yaml +32 -0
- filter_pipe-1.0.0/.gitignore +95 -0
- filter_pipe-1.0.0/.pre-commit-config.yaml +16 -0
- filter_pipe-1.0.0/PKG-INFO +191 -0
- filter_pipe-1.0.0/README.md +182 -0
- filter_pipe-1.0.0/pyproject.toml +21 -0
- filter_pipe-1.0.0/requirements.txt +4 -0
- filter_pipe-1.0.0/setup.cfg +4 -0
- filter_pipe-1.0.0/src/__init__.py +0 -0
- filter_pipe-1.0.0/src/app.py +13 -0
- filter_pipe-1.0.0/src/filter_pipe.egg-info/PKG-INFO +191 -0
- filter_pipe-1.0.0/src/filter_pipe.egg-info/SOURCES.txt +25 -0
- filter_pipe-1.0.0/src/filter_pipe.egg-info/dependency_links.txt +1 -0
- filter_pipe-1.0.0/src/filter_pipe.egg-info/top_level.txt +3 -0
- filter_pipe-1.0.0/src/pipeline/__init__.py +0 -0
- filter_pipe-1.0.0/src/pipeline/filters.py +233 -0
- filter_pipe-1.0.0/src/pipeline/math_ops.py +99 -0
- filter_pipe-1.0.0/src/pipeline/pipe.py +47 -0
- filter_pipe-1.0.0/src/pipeline/pipeline.py +62 -0
- filter_pipe-1.0.0/tests/test_filters.py +181 -0
- filter_pipe-1.0.0/tests/test_math.py +57 -0
- filter_pipe-1.0.0/tests/test_pipeline.py +70 -0
- filter_pipe-1.0.0/tools/index.html +351 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Devcontainer",
|
|
3
|
+
// "image": "",
|
|
4
|
+
"build": {
|
|
5
|
+
"context": "..",
|
|
6
|
+
"dockerfile": "Dockerfile"
|
|
7
|
+
},
|
|
8
|
+
"customizations": {
|
|
9
|
+
"vscode": {
|
|
10
|
+
"extensions": [
|
|
11
|
+
"ms-python.debugpy",
|
|
12
|
+
"ms-python.python",
|
|
13
|
+
"ms-python.vscode-pylance",
|
|
14
|
+
"ms-python.isort",
|
|
15
|
+
"ritwickdey.LiveServer"
|
|
16
|
+
]
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"mounts":[
|
|
20
|
+
"source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind",
|
|
21
|
+
"source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,readonly",
|
|
22
|
+
"source=${localEnv:HOME}/.gitconfig,target=/home/vscode//.gitconfig,type=bind,readonly",
|
|
23
|
+
],
|
|
24
|
+
"containerEnv": {
|
|
25
|
+
},
|
|
26
|
+
"runArgs": [
|
|
27
|
+
],
|
|
28
|
+
"postCreateCommand": ".devcontainer/setup_devcontainer.sh",
|
|
29
|
+
"remoteUser": "vscode"
|
|
30
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Build and Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "*"
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
deploy:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write
|
|
13
|
+
contents: read
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
with:
|
|
17
|
+
fetch-depth: 0
|
|
18
|
+
- name: Setup Python environment
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: '3.x'
|
|
22
|
+
- run: python3 -m pip install -r requirements.txt
|
|
23
|
+
- name: Verify setuptools-scm detected version
|
|
24
|
+
run: python3 -m setuptools_scm
|
|
25
|
+
- run: python3 -m build
|
|
26
|
+
- name: Publish package to PyPi
|
|
27
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
push:
|
|
6
|
+
branches:
|
|
7
|
+
- master
|
|
8
|
+
|
|
9
|
+
defaults:
|
|
10
|
+
run:
|
|
11
|
+
shell: bash
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
test:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
# - run: |
|
|
19
|
+
# sudo apt update
|
|
20
|
+
# sudo apt upgrade -y
|
|
21
|
+
# sudo apt install -y python3 python3-pip
|
|
22
|
+
- name: setup env
|
|
23
|
+
run: |
|
|
24
|
+
/bin/bash .devcontainer/setup_devcontainer.sh
|
|
25
|
+
env:
|
|
26
|
+
GITHUB_WORKSPACE: ${{ github.workspace }}
|
|
27
|
+
- name: run pre-commit
|
|
28
|
+
run: |
|
|
29
|
+
pre-commit run --all-files
|
|
30
|
+
- name: run tests
|
|
31
|
+
run:
|
|
32
|
+
python3 -m pytest
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
*.egg-info/
|
|
24
|
+
.installed.cfg
|
|
25
|
+
*.egg
|
|
26
|
+
|
|
27
|
+
# PyInstaller
|
|
28
|
+
# Usually these files are written by a python script from a template
|
|
29
|
+
*.manifest
|
|
30
|
+
*.spec
|
|
31
|
+
|
|
32
|
+
# Installer logs
|
|
33
|
+
pip-log.txt
|
|
34
|
+
pip-delete-this-directory.txt
|
|
35
|
+
pip-wheel-metadata/
|
|
36
|
+
|
|
37
|
+
# Unit test / coverage reports
|
|
38
|
+
htmlcov/
|
|
39
|
+
.tox/
|
|
40
|
+
.nox/
|
|
41
|
+
.coverage
|
|
42
|
+
.coverage.*
|
|
43
|
+
.cache
|
|
44
|
+
nosetests.xml
|
|
45
|
+
coverage.xml
|
|
46
|
+
*.cover
|
|
47
|
+
*.py,cover
|
|
48
|
+
.hypothesis/
|
|
49
|
+
|
|
50
|
+
# Jupyter Notebook
|
|
51
|
+
.ipynb_checkpoints
|
|
52
|
+
|
|
53
|
+
# pyenv
|
|
54
|
+
.python-version
|
|
55
|
+
|
|
56
|
+
# pipenv
|
|
57
|
+
Pipfile.lock
|
|
58
|
+
|
|
59
|
+
# poetry
|
|
60
|
+
poetry.lock
|
|
61
|
+
.venv/
|
|
62
|
+
venv/
|
|
63
|
+
ENV/
|
|
64
|
+
env/
|
|
65
|
+
env.bak/
|
|
66
|
+
venv.bak/
|
|
67
|
+
|
|
68
|
+
# venv
|
|
69
|
+
*/bin/activate
|
|
70
|
+
*/Lib/site-packages/
|
|
71
|
+
*/pyvenv.cfg
|
|
72
|
+
|
|
73
|
+
# Spyder project settings
|
|
74
|
+
.spyderproject
|
|
75
|
+
.spyproject
|
|
76
|
+
|
|
77
|
+
# Rope project settings
|
|
78
|
+
.ropeproject
|
|
79
|
+
|
|
80
|
+
# mkdocs documentation
|
|
81
|
+
/site
|
|
82
|
+
|
|
83
|
+
# mypy
|
|
84
|
+
.mypy_cache/
|
|
85
|
+
.dmypy.json
|
|
86
|
+
dmypy.json
|
|
87
|
+
|
|
88
|
+
# pyre
|
|
89
|
+
.pyre/
|
|
90
|
+
|
|
91
|
+
# IDEs
|
|
92
|
+
.vscode/
|
|
93
|
+
.idea/
|
|
94
|
+
*.sublime-project
|
|
95
|
+
*.sublime-workspace
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
exclude: src/jair_turtlesim/test/|src/jair_turtlesim/setup.py
|
|
2
|
+
repos:
|
|
3
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
4
|
+
rev: v4.5.0
|
|
5
|
+
hooks:
|
|
6
|
+
- id: trailing-whitespace
|
|
7
|
+
- id: end-of-file-fixer
|
|
8
|
+
- id: mixed-line-ending
|
|
9
|
+
args: [--fix=lf]
|
|
10
|
+
- id: check-yaml
|
|
11
|
+
args: [--unsafe]
|
|
12
|
+
- id: debug-statements
|
|
13
|
+
- id: double-quote-string-fixer
|
|
14
|
+
- id: requirements-txt-fixer
|
|
15
|
+
- id: check-added-large-files
|
|
16
|
+
args: ["--maxkb=512"]
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: filter_pipe
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Build and expose flexible calculation pipelines that give the end user total control over dynamic variables and data logic.
|
|
5
|
+
Author: Jair
|
|
6
|
+
Project-URL: Repository, https://github.com/Jair-F/filter_pipe
|
|
7
|
+
Requires-Python: >=3.8
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# ๐ filter_pipe
|
|
11
|
+
|
|
12
|
+
A powerful and flexible Python library for building data processing pipelines with filters and mathematical operations. Chain multiple filters together to process numerical data in a clean, intuitive way.
|
|
13
|
+
|
|
14
|
+
[](https://pypi.org/project/filter_pipe/) [](https://pypi.org/project/filter_pipe/)
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## โจ Features
|
|
19
|
+
|
|
20
|
+
- โ๏ธ **Chainable Pipelines** - Combine filters seamlessly using pipe syntax
|
|
21
|
+
- ๐ **Signal Processing Filters** - Moving average, low-pass, high-pass, band-pass, notch filters
|
|
22
|
+
- ๐งฎ **Mathematical Operations** - Add, subtract, multiply, divide operations
|
|
23
|
+
- ๐ฏ **Flexible Configuration** - Easy-to-read pipeline syntax with customizable parameters
|
|
24
|
+
- ๐ **Regex-based Validation** - Robust pattern matching for pipeline chunks
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## ๐ Quick Start
|
|
29
|
+
|
|
30
|
+
### Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install -r requirements.txt
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### Basic Usage
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from src.pipeline.pipeline import Pipeline
|
|
40
|
+
|
|
41
|
+
# Create a pipeline with multiple filters
|
|
42
|
+
pipeline = Pipeline('mavg(n=10) | lpass(alpha=0.2) | str(ndigits=2)')
|
|
43
|
+
|
|
44
|
+
# Process a value through the pipeline
|
|
45
|
+
result = pipeline.calc(2134.1231231)
|
|
46
|
+
print(result) # Output: '2064.17'
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Mathematical Operations
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from src.pipeline.math_ops import Divide, Multiply, Add
|
|
53
|
+
|
|
54
|
+
# Use mathematical operations
|
|
55
|
+
divider = Divide(' / 2')
|
|
56
|
+
result = divider.calc(10)
|
|
57
|
+
print(result) # Output: 5.0
|
|
58
|
+
|
|
59
|
+
# Combine in a pipeline
|
|
60
|
+
pipeline = Pipeline('*2 | +5 | /3')
|
|
61
|
+
result = pipeline.calc(10) # (10 * 2 + 5) / 3 = 25/3 โ 8.33
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Advanced Pipeline Example
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
# Signal smoothing pipeline: moving average โ low-pass filter โ round to 2 decimals
|
|
68
|
+
pipeline = Pipeline('mavg(n=5) | lpass(alpha=0.3) | str(ndigits=2)')
|
|
69
|
+
values = [100.2, 101.5, 99.8, 102.1, 100.9]
|
|
70
|
+
|
|
71
|
+
for value in values:
|
|
72
|
+
smoothed = pipeline.calc(value)
|
|
73
|
+
print(f"Input: {value:6.1f} โ Output: {smoothed}")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## ๐ฆ Available Pipeline Chunks
|
|
79
|
+
|
|
80
|
+
### ๐ฏ Filters
|
|
81
|
+
|
|
82
|
+
| Chunk | Syntax | Description | Parameters |
|
|
83
|
+
|-------|--------|-------------|------------|
|
|
84
|
+
| **Moving Average** | `mavg(n=N)` | Calculates the average of the last N values | `n` - window size (integer) |
|
|
85
|
+
| **Low-Pass Filter** | `lpass(alpha=A)` | Smooths data by attenuating high frequencies | `alpha` - smoothing factor (0-1) |
|
|
86
|
+
| **High-Pass Filter** | `hpass(alpha=A)` | Attenuates low frequencies, preserves high frequencies | `alpha` - smoothing factor (0-1) |
|
|
87
|
+
| **Band-Pass Filter** | `bpass(low_alpha=A,high_alpha=B)` | Passes frequencies within a band | `low_alpha`, `high_alpha` - bounds (0-1) |
|
|
88
|
+
| **Notch Filter** | `notch(low_alpha=A,high_alpha=B)` | Removes frequencies within a band | `low_alpha`, `high_alpha` - bounds (0-1) |
|
|
89
|
+
| **High Cut** | `hcut(cut=C)` | Caps maximum value | `cut` - maximum threshold |
|
|
90
|
+
| **Low Cut** | `lcut(cut=C)` | Caps minimum value | `cut` - minimum threshold |
|
|
91
|
+
| **To String** | `str(ndigits=N)` | Converts to formatted string | `ndigits` - decimal places (optional, default=0) |
|
|
92
|
+
|
|
93
|
+
### ๐งฎ Math Operations
|
|
94
|
+
|
|
95
|
+
| Chunk | Syntax | Description | Parameters |
|
|
96
|
+
|-------|--------|-------------|------------|
|
|
97
|
+
| **Divide** | `/ D` | Divides value by D | `D` - divisor (number) |
|
|
98
|
+
| **Multiply** | `* M` | Multiplies value by M | `M` - multiplier (number) |
|
|
99
|
+
| **Add** | `+ V` | Adds V to value | `V` - value to add (number) |
|
|
100
|
+
| **Subtract** | `- S` | Subtracts S from value | `S` - value to subtract (number) |
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## ๐ก Examples
|
|
105
|
+
|
|
106
|
+
### Example 1: Temperature Smoothing
|
|
107
|
+
Smooth noisy temperature sensor readings:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from src.pipeline.pipeline import Pipeline
|
|
111
|
+
|
|
112
|
+
# Combine moving average with low-pass filter
|
|
113
|
+
pipeline = Pipeline('mavg(n=3) | lpass(alpha=0.5) | str(ndigits=1)')
|
|
114
|
+
|
|
115
|
+
temperatures = [20.1, 25.3, 19.8, 21.2, 20.9]
|
|
116
|
+
for temp in temperatures:
|
|
117
|
+
smoothed = pipeline.calc(temp)
|
|
118
|
+
print(f"Temperature: {temp}ยฐC โ Smoothed: {smoothed}ยฐC")
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Example 2: Financial Data Processing
|
|
122
|
+
Process stock price data:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
# 10-period moving average โ normalize โ round to 2 decimals
|
|
126
|
+
pipeline = Pipeline('mavg(n=10) | str(ndigits=2)')
|
|
127
|
+
|
|
128
|
+
stock_prices = [150.25, 149.80, 151.10, 150.95, 149.50]
|
|
129
|
+
for price in stock_prices:
|
|
130
|
+
avg = pipeline.calc(price)
|
|
131
|
+
print(f"${price} โ MA: ${avg}")
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Example 3: Signal Processing
|
|
135
|
+
Apply advanced filtering:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
# Remove noise with band-pass โ apply notch โ convert to string
|
|
139
|
+
pipeline = Pipeline('bpass(low_alpha=0.2,high_alpha=0.8) | notch(low_alpha=0.3,high_alpha=0.7) | str(ndigits=3)')
|
|
140
|
+
|
|
141
|
+
signal_data = [0.5, 0.6, 0.55, 0.58, 0.52]
|
|
142
|
+
for signal in signal_data:
|
|
143
|
+
filtered = pipeline.calc(signal)
|
|
144
|
+
print(f"Signal: {signal} โ Filtered: {filtered}")
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Example 4: Value Constraints
|
|
148
|
+
Bound values within a range:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
# Scale value, then apply min/max constraints
|
|
152
|
+
pipeline = Pipeline('*10 | lcut(cut=0) | hcut(cut=100)')
|
|
153
|
+
|
|
154
|
+
raw_values = [-1.5, 5.3, 12.8, -0.2]
|
|
155
|
+
for val in raw_values:
|
|
156
|
+
bounded = pipeline.calc(val)
|
|
157
|
+
print(f"{val} โ {bounded} (0-100 range)")
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## ๐งช Running Tests
|
|
163
|
+
|
|
164
|
+
Execute the test suite to verify functionality:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
python3 -m pytest
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Tests are located in the `tests/` directory:
|
|
171
|
+
- `test_filters.py` - Filter functionality tests
|
|
172
|
+
- `test_math.py` - Math operations tests
|
|
173
|
+
- `test_pipeline.py` - Pipeline chaining tests
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## ๐ Resources
|
|
178
|
+
|
|
179
|
+
- **๐ฆ PyPI Package**: Find this project on [PyPI](https://pypi.org/project/filter_pipe/)
|
|
180
|
+
- **๏ฟฝ๐ผ Regex Patterns**: Learn and test regex patterns used in this project at [regex101.com](https://regex101.com/)
|
|
181
|
+
- **๐ Tools**: See `tools/index.html` for playing around with the filters and learn how to calibrate the gains.
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## ๐ License
|
|
186
|
+
|
|
187
|
+
This project is open-source and available for educational and commercial use.
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
**Built with โค๏ธ for data processing pipelines**
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# ๐ filter_pipe
|
|
2
|
+
|
|
3
|
+
A powerful and flexible Python library for building data processing pipelines with filters and mathematical operations. Chain multiple filters together to process numerical data in a clean, intuitive way.
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/filter_pipe/) [](https://pypi.org/project/filter_pipe/)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## โจ Features
|
|
10
|
+
|
|
11
|
+
- โ๏ธ **Chainable Pipelines** - Combine filters seamlessly using pipe syntax
|
|
12
|
+
- ๐ **Signal Processing Filters** - Moving average, low-pass, high-pass, band-pass, notch filters
|
|
13
|
+
- ๐งฎ **Mathematical Operations** - Add, subtract, multiply, divide operations
|
|
14
|
+
- ๐ฏ **Flexible Configuration** - Easy-to-read pipeline syntax with customizable parameters
|
|
15
|
+
- ๐ **Regex-based Validation** - Robust pattern matching for pipeline chunks
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## ๐ Quick Start
|
|
20
|
+
|
|
21
|
+
### Installation
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install -r requirements.txt
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### Basic Usage
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from src.pipeline.pipeline import Pipeline
|
|
31
|
+
|
|
32
|
+
# Create a pipeline with multiple filters
|
|
33
|
+
pipeline = Pipeline('mavg(n=10) | lpass(alpha=0.2) | str(ndigits=2)')
|
|
34
|
+
|
|
35
|
+
# Process a value through the pipeline
|
|
36
|
+
result = pipeline.calc(2134.1231231)
|
|
37
|
+
print(result) # Output: '2064.17'
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Mathematical Operations
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from src.pipeline.math_ops import Divide, Multiply, Add
|
|
44
|
+
|
|
45
|
+
# Use mathematical operations
|
|
46
|
+
divider = Divide(' / 2')
|
|
47
|
+
result = divider.calc(10)
|
|
48
|
+
print(result) # Output: 5.0
|
|
49
|
+
|
|
50
|
+
# Combine in a pipeline
|
|
51
|
+
pipeline = Pipeline('*2 | +5 | /3')
|
|
52
|
+
result = pipeline.calc(10) # (10 * 2 + 5) / 3 = 25/3 โ 8.33
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Advanced Pipeline Example
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
# Signal smoothing pipeline: moving average โ low-pass filter โ round to 2 decimals
|
|
59
|
+
pipeline = Pipeline('mavg(n=5) | lpass(alpha=0.3) | str(ndigits=2)')
|
|
60
|
+
values = [100.2, 101.5, 99.8, 102.1, 100.9]
|
|
61
|
+
|
|
62
|
+
for value in values:
|
|
63
|
+
smoothed = pipeline.calc(value)
|
|
64
|
+
print(f"Input: {value:6.1f} โ Output: {smoothed}")
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## ๐ฆ Available Pipeline Chunks
|
|
70
|
+
|
|
71
|
+
### ๐ฏ Filters
|
|
72
|
+
|
|
73
|
+
| Chunk | Syntax | Description | Parameters |
|
|
74
|
+
|-------|--------|-------------|------------|
|
|
75
|
+
| **Moving Average** | `mavg(n=N)` | Calculates the average of the last N values | `n` - window size (integer) |
|
|
76
|
+
| **Low-Pass Filter** | `lpass(alpha=A)` | Smooths data by attenuating high frequencies | `alpha` - smoothing factor (0-1) |
|
|
77
|
+
| **High-Pass Filter** | `hpass(alpha=A)` | Attenuates low frequencies, preserves high frequencies | `alpha` - smoothing factor (0-1) |
|
|
78
|
+
| **Band-Pass Filter** | `bpass(low_alpha=A,high_alpha=B)` | Passes frequencies within a band | `low_alpha`, `high_alpha` - bounds (0-1) |
|
|
79
|
+
| **Notch Filter** | `notch(low_alpha=A,high_alpha=B)` | Removes frequencies within a band | `low_alpha`, `high_alpha` - bounds (0-1) |
|
|
80
|
+
| **High Cut** | `hcut(cut=C)` | Caps maximum value | `cut` - maximum threshold |
|
|
81
|
+
| **Low Cut** | `lcut(cut=C)` | Caps minimum value | `cut` - minimum threshold |
|
|
82
|
+
| **To String** | `str(ndigits=N)` | Converts to formatted string | `ndigits` - decimal places (optional, default=0) |
|
|
83
|
+
|
|
84
|
+
### ๐งฎ Math Operations
|
|
85
|
+
|
|
86
|
+
| Chunk | Syntax | Description | Parameters |
|
|
87
|
+
|-------|--------|-------------|------------|
|
|
88
|
+
| **Divide** | `/ D` | Divides value by D | `D` - divisor (number) |
|
|
89
|
+
| **Multiply** | `* M` | Multiplies value by M | `M` - multiplier (number) |
|
|
90
|
+
| **Add** | `+ V` | Adds V to value | `V` - value to add (number) |
|
|
91
|
+
| **Subtract** | `- S` | Subtracts S from value | `S` - value to subtract (number) |
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## ๐ก Examples
|
|
96
|
+
|
|
97
|
+
### Example 1: Temperature Smoothing
|
|
98
|
+
Smooth noisy temperature sensor readings:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from src.pipeline.pipeline import Pipeline
|
|
102
|
+
|
|
103
|
+
# Combine moving average with low-pass filter
|
|
104
|
+
pipeline = Pipeline('mavg(n=3) | lpass(alpha=0.5) | str(ndigits=1)')
|
|
105
|
+
|
|
106
|
+
temperatures = [20.1, 25.3, 19.8, 21.2, 20.9]
|
|
107
|
+
for temp in temperatures:
|
|
108
|
+
smoothed = pipeline.calc(temp)
|
|
109
|
+
print(f"Temperature: {temp}ยฐC โ Smoothed: {smoothed}ยฐC")
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Example 2: Financial Data Processing
|
|
113
|
+
Process stock price data:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
# 10-period moving average โ normalize โ round to 2 decimals
|
|
117
|
+
pipeline = Pipeline('mavg(n=10) | str(ndigits=2)')
|
|
118
|
+
|
|
119
|
+
stock_prices = [150.25, 149.80, 151.10, 150.95, 149.50]
|
|
120
|
+
for price in stock_prices:
|
|
121
|
+
avg = pipeline.calc(price)
|
|
122
|
+
print(f"${price} โ MA: ${avg}")
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### Example 3: Signal Processing
|
|
126
|
+
Apply advanced filtering:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
# Remove noise with band-pass โ apply notch โ convert to string
|
|
130
|
+
pipeline = Pipeline('bpass(low_alpha=0.2,high_alpha=0.8) | notch(low_alpha=0.3,high_alpha=0.7) | str(ndigits=3)')
|
|
131
|
+
|
|
132
|
+
signal_data = [0.5, 0.6, 0.55, 0.58, 0.52]
|
|
133
|
+
for signal in signal_data:
|
|
134
|
+
filtered = pipeline.calc(signal)
|
|
135
|
+
print(f"Signal: {signal} โ Filtered: {filtered}")
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Example 4: Value Constraints
|
|
139
|
+
Bound values within a range:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
# Scale value, then apply min/max constraints
|
|
143
|
+
pipeline = Pipeline('*10 | lcut(cut=0) | hcut(cut=100)')
|
|
144
|
+
|
|
145
|
+
raw_values = [-1.5, 5.3, 12.8, -0.2]
|
|
146
|
+
for val in raw_values:
|
|
147
|
+
bounded = pipeline.calc(val)
|
|
148
|
+
print(f"{val} โ {bounded} (0-100 range)")
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## ๐งช Running Tests
|
|
154
|
+
|
|
155
|
+
Execute the test suite to verify functionality:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
python3 -m pytest
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Tests are located in the `tests/` directory:
|
|
162
|
+
- `test_filters.py` - Filter functionality tests
|
|
163
|
+
- `test_math.py` - Math operations tests
|
|
164
|
+
- `test_pipeline.py` - Pipeline chaining tests
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## ๐ Resources
|
|
169
|
+
|
|
170
|
+
- **๐ฆ PyPI Package**: Find this project on [PyPI](https://pypi.org/project/filter_pipe/)
|
|
171
|
+
- **๏ฟฝ๐ผ Regex Patterns**: Learn and test regex patterns used in this project at [regex101.com](https://regex101.com/)
|
|
172
|
+
- **๐ Tools**: See `tools/index.html` for playing around with the filters and learn how to calibrate the gains.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## ๐ License
|
|
177
|
+
|
|
178
|
+
This project is open-source and available for educational and commercial use.
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
**Built with โค๏ธ for data processing pipelines**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "setuptools-scm>=7.0", "twine", "build"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "filter_pipe"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
authors = [
|
|
10
|
+
{name = "Jair"},
|
|
11
|
+
]
|
|
12
|
+
description = "Build and expose flexible calculation pipelines that give the end user total control over dynamic variables and data logic."
|
|
13
|
+
requires-python = ">=3.8"
|
|
14
|
+
dependencies = []
|
|
15
|
+
|
|
16
|
+
[project.urls]
|
|
17
|
+
Repository = "https://github.com/Jair-F/filter_pipe"
|
|
18
|
+
|
|
19
|
+
[tool.setuptools_scm]
|
|
20
|
+
# You can leave this table empty; setuptools-scm uses smart defaults.
|
|
21
|
+
# It automatically detects the version from your Git tags.
|
|
File without changes
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from src.pipeline.math_ops import Divide
|
|
2
|
+
from src.pipeline.pipeline import Pipeline
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
if __name__ == '__main__':
|
|
6
|
+
pipeline = Pipeline(' mavg(n=10) | lpass(alpha=0.2) | str(ndigits=2)')
|
|
7
|
+
print(F"pipeline: {pipeline.calc(2134.1231231)}")
|
|
8
|
+
|
|
9
|
+
divider = Divide(' / 2')
|
|
10
|
+
print(F"10/div: {divider.calc(10)}")
|
|
11
|
+
|
|
12
|
+
print(F"value: {str(float(round(123.123, ndigits=2)))}")
|
|
13
|
+
print(F"value: {str(123)}")
|