pipeline-eds 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipeline_eds-0.2.4/LICENSE +14 -0
- pipeline_eds-0.2.4/PKG-INFO +238 -0
- pipeline_eds-0.2.4/README.md +206 -0
- pipeline_eds-0.2.4/pyproject.toml +60 -0
- pipeline_eds-0.2.4/src/pipeline/__init__.py +4 -0
- pipeline_eds-0.2.4/src/pipeline/__main__.py +1 -0
- pipeline_eds-0.2.4/src/pipeline/api/__init__.py +0 -0
- pipeline_eds-0.2.4/src/pipeline/api/eds.py +980 -0
- pipeline_eds-0.2.4/src/pipeline/api/rjn.py +157 -0
- pipeline_eds-0.2.4/src/pipeline/api/status_api.py +9 -0
- pipeline_eds-0.2.4/src/pipeline/calls.py +108 -0
- pipeline_eds-0.2.4/src/pipeline/cli.py +282 -0
- pipeline_eds-0.2.4/src/pipeline/configrationmanager.py +22 -0
- pipeline_eds-0.2.4/src/pipeline/decorators.py +13 -0
- pipeline_eds-0.2.4/src/pipeline/env.py +61 -0
- pipeline_eds-0.2.4/src/pipeline/environment.py +59 -0
- pipeline_eds-0.2.4/src/pipeline/gui_fastapi_plotly_live.py +78 -0
- pipeline_eds-0.2.4/src/pipeline/gui_mpl_live.py +113 -0
- pipeline_eds-0.2.4/src/pipeline/helpers.py +125 -0
- pipeline_eds-0.2.4/src/pipeline/logging_setup.py +45 -0
- pipeline_eds-0.2.4/src/pipeline/pastehelpers.py +10 -0
- pipeline_eds-0.2.4/src/pipeline/philosophy.py +62 -0
- pipeline_eds-0.2.4/src/pipeline/plotbuffer.py +21 -0
- pipeline_eds-0.2.4/src/pipeline/points_loader.py +19 -0
- pipeline_eds-0.2.4/src/pipeline/queriesmanager.py +122 -0
- pipeline_eds-0.2.4/src/pipeline/time_manager.py +211 -0
- pipeline_eds-0.2.4/src/pipeline/workspace_manager.py +253 -0
- pipeline_eds-0.2.4/workspaces/default-workspace.toml +3 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/__init__.py +0 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/__init__.py +0 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/aggregator.py +84 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/collector.py +60 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/sanitizer.py +40 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/storage.py +16 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/configurations/config_time.toml +11 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/configurations/configuration.toml +2 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/README.md +7 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/README.md +7 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data - Copy.csv +355 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data_EFF.csv +17521 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data_INF.csv +17521 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/export_eds_points_neo.txt +11015 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.csv +8759 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.xlsx +0 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_effluent.csv +8759 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_influent.csv +8759 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_wetwell.csv +8761 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/history/time_sample.txt +0 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsD321E_sid11003.toml +14 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsFI8001_sid8528.toml +14 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsM100FI_sid2308.toml +14 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsM310LI_sid2382.toml +14 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/default-queries.toml +4 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/points-maxson.csv +4 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/points-stiles.csv +4 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/timestamps_success.json +20 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/scripts/__init__.py +0 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/scripts/daemon_runner.py +212 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/secrets/README.md +24 -0
- pipeline_eds-0.2.4/workspaces/eds_to_rjn/secrets/secrets-example.yaml +15 -0
- pipeline_eds-0.2.4/workspaces/eds_to_termux/..txt +0 -0
@@ -0,0 +1,14 @@
|
|
1
|
+
BSD-3 License
|
2
|
+
|
3
|
+
Copyright (c) 2025 George Clayton Bennett
|
4
|
+
|
5
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
6
|
+
|
7
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
8
|
+
|
9
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
10
|
+
|
11
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
12
|
+
|
13
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
14
|
+
|
@@ -0,0 +1,238 @@
|
|
1
|
+
Metadata-Version: 2.3
|
2
|
+
Name: pipeline-eds
|
3
|
+
Version: 0.2.4
|
4
|
+
Summary: The official API pipeline library for mulch-based projects. Key target: Emerson Ovation EDS REST API.
|
5
|
+
License: BSD-3
|
6
|
+
Author: George Clayton Bennett
|
7
|
+
Author-email: george.bennett@memphistn.gov
|
8
|
+
Requires-Python: >=3.11,<4.0.0
|
9
|
+
Classifier: License :: Other/Proprietary License
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
14
|
+
Requires-Dist: certifi (>=2025.1.31,<2026.0.0)
|
15
|
+
Requires-Dist: fastapi (>=0.115.12,<0.116.0)
|
16
|
+
Requires-Dist: mulch (>=0.2.8,<0.3.0)
|
17
|
+
Requires-Dist: mysql-connector-python (>=9.3.0,<10.0.0)
|
18
|
+
Requires-Dist: pendulum (>=3.1.0,<4.0.0)
|
19
|
+
Requires-Dist: plotly (>=6.2.0,<7.0.0)
|
20
|
+
Requires-Dist: pyodbc (>=5.2.0,<6.0.0)
|
21
|
+
Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
|
22
|
+
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
23
|
+
Requires-Dist: schedule (>=1.2.2,<2.0.0)
|
24
|
+
Requires-Dist: toml (>=0.10.2,<0.11.0)
|
25
|
+
Requires-Dist: tzdata (>=2025.2,<2026.0)
|
26
|
+
Requires-Dist: urllib3 (>=2.4.0,<3.0.0)
|
27
|
+
Requires-Dist: uvicorn (>=0.34.3,<0.35.0)
|
28
|
+
Project-URL: Homepage, https://github.com/city-of-memphis-wastewater/pipeline
|
29
|
+
Project-URL: Repository, https://github.com/city-of-memphis-wastewater/pipeline
|
30
|
+
Description-Content-Type: text/markdown
|
31
|
+
|
32
|
+
# pipeline
|
33
|
+
The primary purpose of this project is to ease API access to Enterprise Data Server (EDS) machines set up by Emerson to compliment an Emerson Ovation local system.
|
34
|
+
Use-cases include data exchange with third-party contractors and also data access for in-house employees on work and personal devices.
|
35
|
+
|
36
|
+
*Scroll down to ***Rollout, setup, etc.*** to see information about dependencies, Poetry, and pyenv.*
|
37
|
+
|
38
|
+
## How to run pipeline
|
39
|
+
|
40
|
+
Check that the secrets.yaml file in the default-workspace is loading:
|
41
|
+
```
|
42
|
+
poetry run python -m src.pipeline.env
|
43
|
+
```
|
44
|
+
Recieve an export file of all the points known to your EDS:
|
45
|
+
```
|
46
|
+
poetry run python -m src.pipeline.api.eds demo-point-export
|
47
|
+
```
|
48
|
+
Run the existing eds_to_rjn workspace:
|
49
|
+
```
|
50
|
+
poetry run python -m workspaces.eds_to_rjn.scripts.daemon_runner test
|
51
|
+
```
|
52
|
+
Check connectivity:
|
53
|
+
```
|
54
|
+
poetry run python -m src.pipeline.api.eds ping
|
55
|
+
poetry run python -m src.pipeline.api.rjn ping
|
56
|
+
```
|
57
|
+
Other commands:
|
58
|
+
```
|
59
|
+
.\main.bat # put the daemon into service: ill advised.
|
60
|
+
.\main.ps1
|
61
|
+
```
|
62
|
+
|
63
|
+
## Implementation
|
64
|
+
The current ideal implementation of `pipeline` involves `Windows Task Scheduler`.
|
65
|
+
This is installed on an `EDS` server, to pull data and send it to a third party.
|
66
|
+
The Task is set up to call `main_eds_to_rjn_quiet.ps1` as the entry point.
|
67
|
+
The iterative hourly timing handled is by `Windows Task Scheduler` rather than pythonically with the (no unused) `setup_schedules()` function.
|
68
|
+
The `main` function from the `daemon_runner` file is run; this used to call the `run_hourly_tabular_trend_eds_to_rjn()` function.
|
69
|
+
Environment managemenet is handled with `venv` rather than `pyenv` and `poetry`, because `Task Scheduler` directs the process to a different user.
|
70
|
+
|
71
|
+
## secrets.yaml
|
72
|
+
Access will not work without a secrets.yaml file in /pipeline/workspaces/your-workspace-name/config/secrets.yaml
|
73
|
+
|
74
|
+
*API keys are specific to each the workspace, and can be referenced in the workspace scripts.*
|
75
|
+
You would edit the secrets.yaml file to specify your own EDS server and credentials.
|
76
|
+
Important: You need to VPN onto the same network as your server, EDS or otherwise, if it is not available to the outside world.
|
77
|
+
### Example secrets.yaml:
|
78
|
+
```
|
79
|
+
eds_apis:
|
80
|
+
MyServer1:
|
81
|
+
url: "http://127.0.0.1:43084/api/v1/"
|
82
|
+
username: "admin"
|
83
|
+
password: ""
|
84
|
+
MyServer2:
|
85
|
+
url: "http://some-ip-address:port/api/v1/"
|
86
|
+
username: "admin"
|
87
|
+
password: ""
|
88
|
+
|
89
|
+
contractor_apis:
|
90
|
+
MySpecialContractor:
|
91
|
+
url: "https://contractor-api.com/v1/special/"
|
92
|
+
client_id: "special-user"
|
93
|
+
password: "2685steam"
|
94
|
+
```
|
95
|
+
The ***workspaces*** folder is designed to accomodate any custom workspaces or projects that you might add. You can alter the default projet file by altering the directory name indicated in the /pipeline/workspaces/default-workspace.toml file.
|
96
|
+
|
97
|
+
# Future goals:
|
98
|
+
- Submit a pipeline library to PyPi.
|
99
|
+
- Generate a cookiecutter template for building new project directories.
|
100
|
+
- Use mulchcli (another ongoing repo) to guide users through setting up secrets.yaml files step-by-step.
|
101
|
+
|
102
|
+
# Maintenance Notes:
|
103
|
+
- Implement session = requests.Session(), like an adult.
|
104
|
+
- daemon_runner.py should use both Maxson and Stiles access.
|
105
|
+
|
106
|
+
# Rollout, setup, etc.
|
107
|
+
It is recommended that you use **pyenv** for setting the Python version and generating a virtual environment, though this is optional.
|
108
|
+
|
109
|
+
To benefit from the pyproject.toml rollout for this project, **Poetry** is entirely necessary for installing requirements.
|
110
|
+
|
111
|
+
### Why pyenv?
|
112
|
+
venv and virtualenv are confusing, and pyenv is not confusing. With pyenv, it is easy to run many projects at once, with different requirements for each.
|
113
|
+
You only need to install pyenv once on you system, and then you use it to access different versions of Python.
|
114
|
+
|
115
|
+
#### Note to anyone who doesn't yet understand the glory of virtual environments in Python:
|
116
|
+
Do it. You do not want to install special requirements to your system version of Python.
|
117
|
+
|
118
|
+
### Why Poetry?
|
119
|
+
**Poetry** is my favorite dependency management tool. It is very easy. You only need to install it once. **Poetry** also has the benefit of generating a directory-specific virtual environment.
|
120
|
+
|
121
|
+
## Use pyenv to set your Python version
|
122
|
+
(3.11.9 or other 3.11.xx).
|
123
|
+
### Install pyenv (skip if pyenv is already installed on your system)
|
124
|
+
How to install pyenv-win (https://github.com/pyenv-win/pyenv-win)
|
125
|
+
```
|
126
|
+
Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy RemoteSigned
|
127
|
+
Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1"; &"./install-pyenv-win.ps1"
|
128
|
+
```
|
129
|
+
It is worth it to make the pyenv command persistent in PowerShell, by editing the $profile file:
|
130
|
+
```
|
131
|
+
notepad $profile
|
132
|
+
```
|
133
|
+
to include something like:
|
134
|
+
```
|
135
|
+
$env:PYENV = "$HOME\.pyenv\pyenv-win"
|
136
|
+
$env:PYENV_ROOT = "$HOME\.pyenv\pyenv-win"
|
137
|
+
$env:PYENV_HOME = "$HOME\.pyenv\pyenv-win"
|
138
|
+
$env:Path += ";$env:PYENV\bin;$env:PYENV\shims"
|
139
|
+
|
140
|
+
# Initialize pyenv
|
141
|
+
#$pyenvInit = & $env:PYENV_HOME\bin\pyenv init --path
|
142
|
+
#Invoke-Expression $pyenvInit
|
143
|
+
|
144
|
+
# Manually set up the pyenv environment
|
145
|
+
function Invoke-PyenvInit {
|
146
|
+
# Update PATH to include pyenv directories
|
147
|
+
$pyenvShims = "$env:PYENV\shims"
|
148
|
+
$pyenvBin = "$env:PYENV\bin"
|
149
|
+
$env:PATH = "$pyenvBin;$pyenvShims;$env:PATH"
|
150
|
+
}
|
151
|
+
|
152
|
+
# Initialize pyenv
|
153
|
+
Invoke-PyenvInit
|
154
|
+
```
|
155
|
+
How to install pyenv on linux (https://github.com/pyenv/pyenv)
|
156
|
+
```
|
157
|
+
curl -fsSL https://pyenv.run | bash
|
158
|
+
```
|
159
|
+
To make the pyenv command persistent in the Bourne Again SHell, edit ~/.bashrc
|
160
|
+
```
|
161
|
+
nano ~/.bashrc
|
162
|
+
```
|
163
|
+
to include something like:
|
164
|
+
```
|
165
|
+
export Path="$HOME/.local/bin:$PATH"
|
166
|
+
export PYENV_ROOT="$HOME/.pyenv"
|
167
|
+
[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
|
168
|
+
eval "$(pyenv init - bash)"
|
169
|
+
```
|
170
|
+
### Post-install, leverage the benefits of pyenv
|
171
|
+
Install Python 3.11.9 using pyenv:
|
172
|
+
```
|
173
|
+
# pyenv install --list # See all Python versions able with pyenv.
|
174
|
+
pyenv install 3.11.9
|
175
|
+
# pyenv local 3.11.9 # to set your current directory version
|
176
|
+
# pyenv global 3.11.9 # to set the assumed version in any directory
|
177
|
+
# pyenv global system # revert to your system installation as the assumed version
|
178
|
+
```
|
179
|
+
## Use Poetry to run deploy the requirements for this project
|
180
|
+
How to install poetry (https://github.com/python-poetry/poetry)
|
181
|
+
```
|
182
|
+
Remove-Item Alias:curl # Solution to a common PowerShell issue
|
183
|
+
curl -sSL https://install.python-poetry.org | python3
|
184
|
+
# Alternatively:
|
185
|
+
// pip install poetry
|
186
|
+
# Or, even:
|
187
|
+
// pyenv exec pip install poetry
|
188
|
+
```
|
189
|
+
## Git clone pipeline, open source
|
190
|
+
```
|
191
|
+
git clone https://github.com/City-of-Memphis-Wastewater/pipeline.git
|
192
|
+
cd pipline
|
193
|
+
pyenv local 3.11.9 # to set your current directory version
|
194
|
+
```
|
195
|
+
Explicitly set poetry to use the local pyenv version.
|
196
|
+
```
|
197
|
+
poetry python list
|
198
|
+
# You'll see something like this:
|
199
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.13.exe
|
200
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python313.exe
|
201
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.exe
|
202
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python.exe
|
203
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.11.exe
|
204
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python311.exe
|
205
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.exe
|
206
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python.exe
|
207
|
+
# Copy and paste ~any~ of the comparable paths to pyenv 3.11.9 ...
|
208
|
+
poetry use C:\Users\<user>\.pyenv\pyenv-win\versions\3.11.9\python.exe
|
209
|
+
```
|
210
|
+
Pull the requirements from the pyproject.toml file for packagage installation.
|
211
|
+
```
|
212
|
+
poetry install
|
213
|
+
# This is where the magic happens.
|
214
|
+
# When in doubt, run this again.
|
215
|
+
# Especially if you get a ModuleNotFound warning.
|
216
|
+
# Sometimes it doesn't take until you use "poetry run python".
|
217
|
+
poetry run python
|
218
|
+
poetry run python -m src.pipeline.env
|
219
|
+
poetry run python -m src.pipeline.api.eds ping
|
220
|
+
poetry run python -m workspaces.eds_to_rjn.scripts.main
|
221
|
+
```
|
222
|
+
|
223
|
+
# References
|
224
|
+
PEP621.toml
|
225
|
+
|
226
|
+
# Installation on Termux on Android
|
227
|
+
|
228
|
+
Due to `maturin` (Rust build dependency) required by newer versions of `pydantic`, use older versions of FastAPI and Pydantic that don't require Rust, to successfully install on Termux.
|
229
|
+
|
230
|
+
Also, Termux does not allow `poetry` or `pyenv`, so install packages directly to your generic environement with pip.
|
231
|
+
|
232
|
+
```
|
233
|
+
pip install -r requirements-termux.txt
|
234
|
+
```
|
235
|
+
|
236
|
+
Note that Termux does not allow `numpy` or `pandas`. Nor does it allow any plotting through a pop up window, like with tkinter, freesimplegui, matplotlib, etc.
|
237
|
+
|
238
|
+
|
@@ -0,0 +1,206 @@
|
|
1
|
+
# pipeline
|
2
|
+
The primary purpose of this project is to ease API access to Enterprise Data Server (EDS) machines set up by Emerson to compliment an Emerson Ovation local system.
|
3
|
+
Use-cases include data exchange with third-party contractors and also data access for in-house employees on work and personal devices.
|
4
|
+
|
5
|
+
*Scroll down to ***Rollout, setup, etc.*** to see information about dependencies, Poetry, and pyenv.*
|
6
|
+
|
7
|
+
## How to run pipeline
|
8
|
+
|
9
|
+
Check that the secrets.yaml file in the default-workspace is loading:
|
10
|
+
```
|
11
|
+
poetry run python -m src.pipeline.env
|
12
|
+
```
|
13
|
+
Recieve an export file of all the points known to your EDS:
|
14
|
+
```
|
15
|
+
poetry run python -m src.pipeline.api.eds demo-point-export
|
16
|
+
```
|
17
|
+
Run the existing eds_to_rjn workspace:
|
18
|
+
```
|
19
|
+
poetry run python -m workspaces.eds_to_rjn.scripts.daemon_runner test
|
20
|
+
```
|
21
|
+
Check connectivity:
|
22
|
+
```
|
23
|
+
poetry run python -m src.pipeline.api.eds ping
|
24
|
+
poetry run python -m src.pipeline.api.rjn ping
|
25
|
+
```
|
26
|
+
Other commands:
|
27
|
+
```
|
28
|
+
.\main.bat # put the daemon into service: ill advised.
|
29
|
+
.\main.ps1
|
30
|
+
```
|
31
|
+
|
32
|
+
## Implementation
|
33
|
+
The current ideal implementation of `pipeline` involves `Windows Task Scheduler`.
|
34
|
+
This is installed on an `EDS` server, to pull data and send it to a third party.
|
35
|
+
The Task is set up to call `main_eds_to_rjn_quiet.ps1` as the entry point.
|
36
|
+
The iterative hourly timing handled is by `Windows Task Scheduler` rather than pythonically with the (no unused) `setup_schedules()` function.
|
37
|
+
The `main` function from the `daemon_runner` file is run; this used to call the `run_hourly_tabular_trend_eds_to_rjn()` function.
|
38
|
+
Environment managemenet is handled with `venv` rather than `pyenv` and `poetry`, because `Task Scheduler` directs the process to a different user.
|
39
|
+
|
40
|
+
## secrets.yaml
|
41
|
+
Access will not work without a secrets.yaml file in /pipeline/workspaces/your-workspace-name/config/secrets.yaml
|
42
|
+
|
43
|
+
*API keys are specific to each the workspace, and can be referenced in the workspace scripts.*
|
44
|
+
You would edit the secrets.yaml file to specify your own EDS server and credentials.
|
45
|
+
Important: You need to VPN onto the same network as your server, EDS or otherwise, if it is not available to the outside world.
|
46
|
+
### Example secrets.yaml:
|
47
|
+
```
|
48
|
+
eds_apis:
|
49
|
+
MyServer1:
|
50
|
+
url: "http://127.0.0.1:43084/api/v1/"
|
51
|
+
username: "admin"
|
52
|
+
password: ""
|
53
|
+
MyServer2:
|
54
|
+
url: "http://some-ip-address:port/api/v1/"
|
55
|
+
username: "admin"
|
56
|
+
password: ""
|
57
|
+
|
58
|
+
contractor_apis:
|
59
|
+
MySpecialContractor:
|
60
|
+
url: "https://contractor-api.com/v1/special/"
|
61
|
+
client_id: "special-user"
|
62
|
+
password: "2685steam"
|
63
|
+
```
|
64
|
+
The ***workspaces*** folder is designed to accomodate any custom workspaces or projects that you might add. You can alter the default projet file by altering the directory name indicated in the /pipeline/workspaces/default-workspace.toml file.
|
65
|
+
|
66
|
+
# Future goals:
|
67
|
+
- Submit a pipeline library to PyPi.
|
68
|
+
- Generate a cookiecutter template for building new project directories.
|
69
|
+
- Use mulchcli (another ongoing repo) to guide users through setting up secrets.yaml files step-by-step.
|
70
|
+
|
71
|
+
# Maintenance Notes:
|
72
|
+
- Implement session = requests.Session(), like an adult.
|
73
|
+
- daemon_runner.py should use both Maxson and Stiles access.
|
74
|
+
|
75
|
+
# Rollout, setup, etc.
|
76
|
+
It is recommended that you use **pyenv** for setting the Python version and generating a virtual environment, though this is optional.
|
77
|
+
|
78
|
+
To benefit from the pyproject.toml rollout for this project, **Poetry** is entirely necessary for installing requirements.
|
79
|
+
|
80
|
+
### Why pyenv?
|
81
|
+
venv and virtualenv are confusing, and pyenv is not confusing. With pyenv, it is easy to run many projects at once, with different requirements for each.
|
82
|
+
You only need to install pyenv once on you system, and then you use it to access different versions of Python.
|
83
|
+
|
84
|
+
#### Note to anyone who doesn't yet understand the glory of virtual environments in Python:
|
85
|
+
Do it. You do not want to install special requirements to your system version of Python.
|
86
|
+
|
87
|
+
### Why Poetry?
|
88
|
+
**Poetry** is my favorite dependency management tool. It is very easy. You only need to install it once. **Poetry** also has the benefit of generating a directory-specific virtual environment.
|
89
|
+
|
90
|
+
## Use pyenv to set your Python version
|
91
|
+
(3.11.9 or other 3.11.xx).
|
92
|
+
### Install pyenv (skip if pyenv is already installed on your system)
|
93
|
+
How to install pyenv-win (https://github.com/pyenv-win/pyenv-win)
|
94
|
+
```
|
95
|
+
Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy RemoteSigned
|
96
|
+
Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1"; &"./install-pyenv-win.ps1"
|
97
|
+
```
|
98
|
+
It is worth it to make the pyenv command persistent in PowerShell, by editing the $profile file:
|
99
|
+
```
|
100
|
+
notepad $profile
|
101
|
+
```
|
102
|
+
to include something like:
|
103
|
+
```
|
104
|
+
$env:PYENV = "$HOME\.pyenv\pyenv-win"
|
105
|
+
$env:PYENV_ROOT = "$HOME\.pyenv\pyenv-win"
|
106
|
+
$env:PYENV_HOME = "$HOME\.pyenv\pyenv-win"
|
107
|
+
$env:Path += ";$env:PYENV\bin;$env:PYENV\shims"
|
108
|
+
|
109
|
+
# Initialize pyenv
|
110
|
+
#$pyenvInit = & $env:PYENV_HOME\bin\pyenv init --path
|
111
|
+
#Invoke-Expression $pyenvInit
|
112
|
+
|
113
|
+
# Manually set up the pyenv environment
|
114
|
+
function Invoke-PyenvInit {
|
115
|
+
# Update PATH to include pyenv directories
|
116
|
+
$pyenvShims = "$env:PYENV\shims"
|
117
|
+
$pyenvBin = "$env:PYENV\bin"
|
118
|
+
$env:PATH = "$pyenvBin;$pyenvShims;$env:PATH"
|
119
|
+
}
|
120
|
+
|
121
|
+
# Initialize pyenv
|
122
|
+
Invoke-PyenvInit
|
123
|
+
```
|
124
|
+
How to install pyenv on linux (https://github.com/pyenv/pyenv)
|
125
|
+
```
|
126
|
+
curl -fsSL https://pyenv.run | bash
|
127
|
+
```
|
128
|
+
To make the pyenv command persistent in the Bourne Again SHell, edit ~/.bashrc
|
129
|
+
```
|
130
|
+
nano ~/.bashrc
|
131
|
+
```
|
132
|
+
to include something like:
|
133
|
+
```
|
134
|
+
export Path="$HOME/.local/bin:$PATH"
|
135
|
+
export PYENV_ROOT="$HOME/.pyenv"
|
136
|
+
[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
|
137
|
+
eval "$(pyenv init - bash)"
|
138
|
+
```
|
139
|
+
### Post-install, leverage the benefits of pyenv
|
140
|
+
Install Python 3.11.9 using pyenv:
|
141
|
+
```
|
142
|
+
# pyenv install --list # See all Python versions able with pyenv.
|
143
|
+
pyenv install 3.11.9
|
144
|
+
# pyenv local 3.11.9 # to set your current directory version
|
145
|
+
# pyenv global 3.11.9 # to set the assumed version in any directory
|
146
|
+
# pyenv global system # revert to your system installation as the assumed version
|
147
|
+
```
|
148
|
+
## Use Poetry to run deploy the requirements for this project
|
149
|
+
How to install poetry (https://github.com/python-poetry/poetry)
|
150
|
+
```
|
151
|
+
Remove-Item Alias:curl # Solution to a common PowerShell issue
|
152
|
+
curl -sSL https://install.python-poetry.org | python3
|
153
|
+
# Alternatively:
|
154
|
+
// pip install poetry
|
155
|
+
# Or, even:
|
156
|
+
// pyenv exec pip install poetry
|
157
|
+
```
|
158
|
+
## Git clone pipeline, open source
|
159
|
+
```
|
160
|
+
git clone https://github.com/City-of-Memphis-Wastewater/pipeline.git
|
161
|
+
cd pipline
|
162
|
+
pyenv local 3.11.9 # to set your current directory version
|
163
|
+
```
|
164
|
+
Explicitly set poetry to use the local pyenv version.
|
165
|
+
```
|
166
|
+
poetry python list
|
167
|
+
# You'll see something like this:
|
168
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.13.exe
|
169
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python313.exe
|
170
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.exe
|
171
|
+
>> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python.exe
|
172
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.11.exe
|
173
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python311.exe
|
174
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.exe
|
175
|
+
>> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python.exe
|
176
|
+
# Copy and paste ~any~ of the comparable paths to pyenv 3.11.9 ...
|
177
|
+
poetry use C:\Users\<user>\.pyenv\pyenv-win\versions\3.11.9\python.exe
|
178
|
+
```
|
179
|
+
Pull the requirements from the pyproject.toml file for packagage installation.
|
180
|
+
```
|
181
|
+
poetry install
|
182
|
+
# This is where the magic happens.
|
183
|
+
# When in doubt, run this again.
|
184
|
+
# Especially if you get a ModuleNotFound warning.
|
185
|
+
# Sometimes it doesn't take until you use "poetry run python".
|
186
|
+
poetry run python
|
187
|
+
poetry run python -m src.pipeline.env
|
188
|
+
poetry run python -m src.pipeline.api.eds ping
|
189
|
+
poetry run python -m workspaces.eds_to_rjn.scripts.main
|
190
|
+
```
|
191
|
+
|
192
|
+
# References
|
193
|
+
PEP621.toml
|
194
|
+
|
195
|
+
# Installation on Termux on Android
|
196
|
+
|
197
|
+
Due to `maturin` (Rust build dependency) required by newer versions of `pydantic`, use older versions of FastAPI and Pydantic that don't require Rust, to successfully install on Termux.
|
198
|
+
|
199
|
+
Also, Termux does not allow `poetry` or `pyenv`, so install packages directly to your generic environement with pip.
|
200
|
+
|
201
|
+
```
|
202
|
+
pip install -r requirements-termux.txt
|
203
|
+
```
|
204
|
+
|
205
|
+
Note that Termux does not allow `numpy` or `pandas`. Nor does it allow any plotting through a pop up window, like with tkinter, freesimplegui, matplotlib, etc.
|
206
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
[tool.poetry]
|
2
|
+
name = "pipeline-eds"
|
3
|
+
version = "0.2.4"
|
4
|
+
description = "The official API pipeline library for mulch-based projects. Key target: Emerson Ovation EDS REST API."
|
5
|
+
authors = ["George Clayton Bennett <george.bennett@memphistn.gov>"]
|
6
|
+
license = "BSD-3"
|
7
|
+
readme = "README.md"
|
8
|
+
packages = [
|
9
|
+
{ include = "pipeline", from = "src" },
|
10
|
+
{ include = "workspaces", from = "." }
|
11
|
+
] # This makes `pipeline` available as an importable top-level module
|
12
|
+
homepage = "https://github.com/city-of-memphis-wastewater/pipeline"
|
13
|
+
repository = "https://github.com/city-of-memphis-wastewater/pipeline"
|
14
|
+
|
15
|
+
[tool.poetry.scripts]
|
16
|
+
#time-manager = "pipeline.time_manager:main"
|
17
|
+
#workspace-manager = "pipeline.workspace_manager:main" # included in init
|
18
|
+
#collector = "pipeline.collector:main"
|
19
|
+
#eds-client = "pipeline.api.eds:main"
|
20
|
+
#rjn-client = "pipeline.api.rjn:main"
|
21
|
+
pipeline = "pipeline.cli:app" # for installation of pipeline as a cli. Ex: pipeline trend M100FI -s June3 -f June17
|
22
|
+
|
23
|
+
|
24
|
+
[tool.pytest.ini_options]
|
25
|
+
markers = [
|
26
|
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
27
|
+
]
|
28
|
+
#addopts = "--cov=pipeline --cov-report=html" # visually inspect coverage, e.g. before a release or PR.
|
29
|
+
addopts = "-ra --cov=pipeline --cov-report=term-missing" # for development — fast and informative in terminal.
|
30
|
+
testpaths = ["tests"]
|
31
|
+
pythonpath = ["src", "."]
|
32
|
+
|
33
|
+
[tool.poetry.dependencies]
|
34
|
+
python = ">=3.11,<4.0.0"
|
35
|
+
requests = ">=2.32.3,<3.0.0"
|
36
|
+
toml = ">=0.10.2,<0.11.0"
|
37
|
+
pyyaml = ">=6.0.2,<7.0.0"
|
38
|
+
certifi = ">=2025.1.31,<2026.0.0"
|
39
|
+
schedule = ">=1.2.2,<2.0.0"
|
40
|
+
fastapi = ">=0.115.12,<0.116.0"
|
41
|
+
urllib3 = "^2.4.0"
|
42
|
+
plotly = "^6.2.0"
|
43
|
+
uvicorn = "^0.34.3"
|
44
|
+
mulch = "^0.2.8"
|
45
|
+
mysql-connector-python = "^9.3.0"
|
46
|
+
pyodbc = "^5.2.0"
|
47
|
+
tzdata = "^2025.2"
|
48
|
+
pendulum = "^3.1.0"
|
49
|
+
|
50
|
+
[tool.poetry.group.dev.dependencies]
|
51
|
+
pytest = "^8.4.1"
|
52
|
+
pytest-cov = "^6.2.1"
|
53
|
+
pygal = "^3.0.5"
|
54
|
+
|
55
|
+
[build-system]
|
56
|
+
requires = ["poetry-core>=2.0.0,<3.0.0"]
|
57
|
+
build-backend = "poetry.core.masonry.api"
|
58
|
+
|
59
|
+
[tool.poetry.plugins."scripts"]
|
60
|
+
controller = "pipeline.daemon.controller:main_cli"
|
@@ -0,0 +1 @@
|
|
1
|
+
# src/__main__.py
|
File without changes
|