pipeline-eds 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. pipeline_eds-0.2.4/LICENSE +14 -0
  2. pipeline_eds-0.2.4/PKG-INFO +238 -0
  3. pipeline_eds-0.2.4/README.md +206 -0
  4. pipeline_eds-0.2.4/pyproject.toml +60 -0
  5. pipeline_eds-0.2.4/src/pipeline/__init__.py +4 -0
  6. pipeline_eds-0.2.4/src/pipeline/__main__.py +1 -0
  7. pipeline_eds-0.2.4/src/pipeline/api/__init__.py +0 -0
  8. pipeline_eds-0.2.4/src/pipeline/api/eds.py +980 -0
  9. pipeline_eds-0.2.4/src/pipeline/api/rjn.py +157 -0
  10. pipeline_eds-0.2.4/src/pipeline/api/status_api.py +9 -0
  11. pipeline_eds-0.2.4/src/pipeline/calls.py +108 -0
  12. pipeline_eds-0.2.4/src/pipeline/cli.py +282 -0
  13. pipeline_eds-0.2.4/src/pipeline/configrationmanager.py +22 -0
  14. pipeline_eds-0.2.4/src/pipeline/decorators.py +13 -0
  15. pipeline_eds-0.2.4/src/pipeline/env.py +61 -0
  16. pipeline_eds-0.2.4/src/pipeline/environment.py +59 -0
  17. pipeline_eds-0.2.4/src/pipeline/gui_fastapi_plotly_live.py +78 -0
  18. pipeline_eds-0.2.4/src/pipeline/gui_mpl_live.py +113 -0
  19. pipeline_eds-0.2.4/src/pipeline/helpers.py +125 -0
  20. pipeline_eds-0.2.4/src/pipeline/logging_setup.py +45 -0
  21. pipeline_eds-0.2.4/src/pipeline/pastehelpers.py +10 -0
  22. pipeline_eds-0.2.4/src/pipeline/philosophy.py +62 -0
  23. pipeline_eds-0.2.4/src/pipeline/plotbuffer.py +21 -0
  24. pipeline_eds-0.2.4/src/pipeline/points_loader.py +19 -0
  25. pipeline_eds-0.2.4/src/pipeline/queriesmanager.py +122 -0
  26. pipeline_eds-0.2.4/src/pipeline/time_manager.py +211 -0
  27. pipeline_eds-0.2.4/src/pipeline/workspace_manager.py +253 -0
  28. pipeline_eds-0.2.4/workspaces/default-workspace.toml +3 -0
  29. pipeline_eds-0.2.4/workspaces/eds_to_rjn/__init__.py +0 -0
  30. pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/__init__.py +0 -0
  31. pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/aggregator.py +84 -0
  32. pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/collector.py +60 -0
  33. pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/sanitizer.py +40 -0
  34. pipeline_eds-0.2.4/workspaces/eds_to_rjn/code/storage.py +16 -0
  35. pipeline_eds-0.2.4/workspaces/eds_to_rjn/configurations/config_time.toml +11 -0
  36. pipeline_eds-0.2.4/workspaces/eds_to_rjn/configurations/configuration.toml +2 -0
  37. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/README.md +7 -0
  38. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/README.md +7 -0
  39. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data - Copy.csv +355 -0
  40. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data_EFF.csv +17521 -0
  41. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/aggregate/live_data_INF.csv +17521 -0
  42. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/export_eds_points_neo.txt +11015 -0
  43. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.csv +8759 -0
  44. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.xlsx +0 -0
  45. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_effluent.csv +8759 -0
  46. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_influent.csv +8759 -0
  47. pipeline_eds-0.2.4/workspaces/eds_to_rjn/exports/manual_wetwell.csv +8761 -0
  48. pipeline_eds-0.2.4/workspaces/eds_to_rjn/history/time_sample.txt +0 -0
  49. pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsD321E_sid11003.toml +14 -0
  50. pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsFI8001_sid8528.toml +14 -0
  51. pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsM100FI_sid2308.toml +14 -0
  52. pipeline_eds-0.2.4/workspaces/eds_to_rjn/imports/zdMaxson_idcsM310LI_sid2382.toml +14 -0
  53. pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/default-queries.toml +4 -0
  54. pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/points-maxson.csv +4 -0
  55. pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/points-stiles.csv +4 -0
  56. pipeline_eds-0.2.4/workspaces/eds_to_rjn/queries/timestamps_success.json +20 -0
  57. pipeline_eds-0.2.4/workspaces/eds_to_rjn/scripts/__init__.py +0 -0
  58. pipeline_eds-0.2.4/workspaces/eds_to_rjn/scripts/daemon_runner.py +212 -0
  59. pipeline_eds-0.2.4/workspaces/eds_to_rjn/secrets/README.md +24 -0
  60. pipeline_eds-0.2.4/workspaces/eds_to_rjn/secrets/secrets-example.yaml +15 -0
  61. pipeline_eds-0.2.4/workspaces/eds_to_termux/..txt +0 -0
@@ -0,0 +1,14 @@
1
+ BSD-3 License
2
+
3
+ Copyright (c) 2025 George Clayton Bennett
4
+
5
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6
+
7
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8
+
9
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
10
+
11
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
12
+
13
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
14
+
@@ -0,0 +1,238 @@
1
+ Metadata-Version: 2.3
2
+ Name: pipeline-eds
3
+ Version: 0.2.4
4
+ Summary: The official API pipeline library for mulch-based projects. Key target: Emerson Ovation EDS REST API.
5
+ License: BSD-3
6
+ Author: George Clayton Bennett
7
+ Author-email: george.bennett@memphistn.gov
8
+ Requires-Python: >=3.11,<4.0.0
9
+ Classifier: License :: Other/Proprietary License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Requires-Dist: certifi (>=2025.1.31,<2026.0.0)
15
+ Requires-Dist: fastapi (>=0.115.12,<0.116.0)
16
+ Requires-Dist: mulch (>=0.2.8,<0.3.0)
17
+ Requires-Dist: mysql-connector-python (>=9.3.0,<10.0.0)
18
+ Requires-Dist: pendulum (>=3.1.0,<4.0.0)
19
+ Requires-Dist: plotly (>=6.2.0,<7.0.0)
20
+ Requires-Dist: pyodbc (>=5.2.0,<6.0.0)
21
+ Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
22
+ Requires-Dist: requests (>=2.32.3,<3.0.0)
23
+ Requires-Dist: schedule (>=1.2.2,<2.0.0)
24
+ Requires-Dist: toml (>=0.10.2,<0.11.0)
25
+ Requires-Dist: tzdata (>=2025.2,<2026.0)
26
+ Requires-Dist: urllib3 (>=2.4.0,<3.0.0)
27
+ Requires-Dist: uvicorn (>=0.34.3,<0.35.0)
28
+ Project-URL: Homepage, https://github.com/city-of-memphis-wastewater/pipeline
29
+ Project-URL: Repository, https://github.com/city-of-memphis-wastewater/pipeline
30
+ Description-Content-Type: text/markdown
31
+
32
+ # pipeline
33
+ The primary purpose of this project is to ease API access to Enterprise Data Server (EDS) machines set up by Emerson to compliment an Emerson Ovation local system.
34
+ Use-cases include data exchange with third-party contractors and also data access for in-house employees on work and personal devices.
35
+
36
+ *Scroll down to ***Rollout, setup, etc.*** to see information about dependencies, Poetry, and pyenv.*
37
+
38
+ ## How to run pipeline
39
+
40
+ Check that the secrets.yaml file in the default-workspace is loading:
41
+ ```
42
+ poetry run python -m src.pipeline.env
43
+ ```
44
+ Recieve an export file of all the points known to your EDS:
45
+ ```
46
+ poetry run python -m src.pipeline.api.eds demo-point-export
47
+ ```
48
+ Run the existing eds_to_rjn workspace:
49
+ ```
50
+ poetry run python -m workspaces.eds_to_rjn.scripts.daemon_runner test
51
+ ```
52
+ Check connectivity:
53
+ ```
54
+ poetry run python -m src.pipeline.api.eds ping
55
+ poetry run python -m src.pipeline.api.rjn ping
56
+ ```
57
+ Other commands:
58
+ ```
59
+ .\main.bat # put the daemon into service: ill advised.
60
+ .\main.ps1
61
+ ```
62
+
63
+ ## Implementation
64
+ The current ideal implementation of `pipeline` involves `Windows Task Scheduler`.
65
+ This is installed on an `EDS` server, to pull data and send it to a third party.
66
+ The Task is set up to call `main_eds_to_rjn_quiet.ps1` as the entry point.
67
+ The iterative hourly timing handled is by `Windows Task Scheduler` rather than pythonically with the (no unused) `setup_schedules()` function.
68
+ The `main` function from the `daemon_runner` file is run; this used to call the `run_hourly_tabular_trend_eds_to_rjn()` function.
69
+ Environment managemenet is handled with `venv` rather than `pyenv` and `poetry`, because `Task Scheduler` directs the process to a different user.
70
+
71
+ ## secrets.yaml
72
+ Access will not work without a secrets.yaml file in /pipeline/workspaces/your-workspace-name/config/secrets.yaml
73
+
74
+ *API keys are specific to each the workspace, and can be referenced in the workspace scripts.*
75
+ You would edit the secrets.yaml file to specify your own EDS server and credentials.
76
+ Important: You need to VPN onto the same network as your server, EDS or otherwise, if it is not available to the outside world.
77
+ ### Example secrets.yaml:
78
+ ```
79
+ eds_apis:
80
+ MyServer1:
81
+ url: "http://127.0.0.1:43084/api/v1/"
82
+ username: "admin"
83
+ password: ""
84
+ MyServer2:
85
+ url: "http://some-ip-address:port/api/v1/"
86
+ username: "admin"
87
+ password: ""
88
+
89
+ contractor_apis:
90
+ MySpecialContractor:
91
+ url: "https://contractor-api.com/v1/special/"
92
+ client_id: "special-user"
93
+ password: "2685steam"
94
+ ```
95
+ The ***workspaces*** folder is designed to accomodate any custom workspaces or projects that you might add. You can alter the default projet file by altering the directory name indicated in the /pipeline/workspaces/default-workspace.toml file.
96
+
97
+ # Future goals:
98
+ - Submit a pipeline library to PyPi.
99
+ - Generate a cookiecutter template for building new project directories.
100
+ - Use mulchcli (another ongoing repo) to guide users through setting up secrets.yaml files step-by-step.
101
+
102
+ # Maintenance Notes:
103
+ - Implement session = requests.Session(), like an adult.
104
+ - daemon_runner.py should use both Maxson and Stiles access.
105
+
106
+ # Rollout, setup, etc.
107
+ It is recommended that you use **pyenv** for setting the Python version and generating a virtual environment, though this is optional.
108
+
109
+ To benefit from the pyproject.toml rollout for this project, **Poetry** is entirely necessary for installing requirements.
110
+
111
+ ### Why pyenv?
112
+ venv and virtualenv are confusing, and pyenv is not confusing. With pyenv, it is easy to run many projects at once, with different requirements for each.
113
+ You only need to install pyenv once on you system, and then you use it to access different versions of Python.
114
+
115
+ #### Note to anyone who doesn't yet understand the glory of virtual environments in Python:
116
+ Do it. You do not want to install special requirements to your system version of Python.
117
+
118
+ ### Why Poetry?
119
+ **Poetry** is my favorite dependency management tool. It is very easy. You only need to install it once. **Poetry** also has the benefit of generating a directory-specific virtual environment.
120
+
121
+ ## Use pyenv to set your Python version
122
+ (3.11.9 or other 3.11.xx).
123
+ ### Install pyenv (skip if pyenv is already installed on your system)
124
+ How to install pyenv-win (https://github.com/pyenv-win/pyenv-win)
125
+ ```
126
+ Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy RemoteSigned
127
+ Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1"; &"./install-pyenv-win.ps1"
128
+ ```
129
+ It is worth it to make the pyenv command persistent in PowerShell, by editing the $profile file:
130
+ ```
131
+ notepad $profile
132
+ ```
133
+ to include something like:
134
+ ```
135
+ $env:PYENV = "$HOME\.pyenv\pyenv-win"
136
+ $env:PYENV_ROOT = "$HOME\.pyenv\pyenv-win"
137
+ $env:PYENV_HOME = "$HOME\.pyenv\pyenv-win"
138
+ $env:Path += ";$env:PYENV\bin;$env:PYENV\shims"
139
+
140
+ # Initialize pyenv
141
+ #$pyenvInit = & $env:PYENV_HOME\bin\pyenv init --path
142
+ #Invoke-Expression $pyenvInit
143
+
144
+ # Manually set up the pyenv environment
145
+ function Invoke-PyenvInit {
146
+ # Update PATH to include pyenv directories
147
+ $pyenvShims = "$env:PYENV\shims"
148
+ $pyenvBin = "$env:PYENV\bin"
149
+ $env:PATH = "$pyenvBin;$pyenvShims;$env:PATH"
150
+ }
151
+
152
+ # Initialize pyenv
153
+ Invoke-PyenvInit
154
+ ```
155
+ How to install pyenv on linux (https://github.com/pyenv/pyenv)
156
+ ```
157
+ curl -fsSL https://pyenv.run | bash
158
+ ```
159
+ To make the pyenv command persistent in the Bourne Again SHell, edit ~/.bashrc
160
+ ```
161
+ nano ~/.bashrc
162
+ ```
163
+ to include something like:
164
+ ```
165
+ export Path="$HOME/.local/bin:$PATH"
166
+ export PYENV_ROOT="$HOME/.pyenv"
167
+ [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
168
+ eval "$(pyenv init - bash)"
169
+ ```
170
+ ### Post-install, leverage the benefits of pyenv
171
+ Install Python 3.11.9 using pyenv:
172
+ ```
173
+ # pyenv install --list # See all Python versions able with pyenv.
174
+ pyenv install 3.11.9
175
+ # pyenv local 3.11.9 # to set your current directory version
176
+ # pyenv global 3.11.9 # to set the assumed version in any directory
177
+ # pyenv global system # revert to your system installation as the assumed version
178
+ ```
179
+ ## Use Poetry to run deploy the requirements for this project
180
+ How to install poetry (https://github.com/python-poetry/poetry)
181
+ ```
182
+ Remove-Item Alias:curl # Solution to a common PowerShell issue
183
+ curl -sSL https://install.python-poetry.org | python3
184
+ # Alternatively:
185
+ // pip install poetry
186
+ # Or, even:
187
+ // pyenv exec pip install poetry
188
+ ```
189
+ ## Git clone pipeline, open source
190
+ ```
191
+ git clone https://github.com/City-of-Memphis-Wastewater/pipeline.git
192
+ cd pipline
193
+ pyenv local 3.11.9 # to set your current directory version
194
+ ```
195
+ Explicitly set poetry to use the local pyenv version.
196
+ ```
197
+ poetry python list
198
+ # You'll see something like this:
199
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.13.exe
200
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python313.exe
201
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.exe
202
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python.exe
203
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.11.exe
204
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python311.exe
205
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.exe
206
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python.exe
207
+ # Copy and paste ~any~ of the comparable paths to pyenv 3.11.9 ...
208
+ poetry use C:\Users\<user>\.pyenv\pyenv-win\versions\3.11.9\python.exe
209
+ ```
210
+ Pull the requirements from the pyproject.toml file for packagage installation.
211
+ ```
212
+ poetry install
213
+ # This is where the magic happens.
214
+ # When in doubt, run this again.
215
+ # Especially if you get a ModuleNotFound warning.
216
+ # Sometimes it doesn't take until you use "poetry run python".
217
+ poetry run python
218
+ poetry run python -m src.pipeline.env
219
+ poetry run python -m src.pipeline.api.eds ping
220
+ poetry run python -m workspaces.eds_to_rjn.scripts.main
221
+ ```
222
+
223
+ # References
224
+ PEP621.toml
225
+
226
+ # Installation on Termux on Android
227
+
228
+ Due to `maturin` (Rust build dependency) required by newer versions of `pydantic`, use older versions of FastAPI and Pydantic that don't require Rust, to successfully install on Termux.
229
+
230
+ Also, Termux does not allow `poetry` or `pyenv`, so install packages directly to your generic environement with pip.
231
+
232
+ ```
233
+ pip install -r requirements-termux.txt
234
+ ```
235
+
236
+ Note that Termux does not allow `numpy` or `pandas`. Nor does it allow any plotting through a pop up window, like with tkinter, freesimplegui, matplotlib, etc.
237
+
238
+
@@ -0,0 +1,206 @@
1
+ # pipeline
2
+ The primary purpose of this project is to ease API access to Enterprise Data Server (EDS) machines set up by Emerson to compliment an Emerson Ovation local system.
3
+ Use-cases include data exchange with third-party contractors and also data access for in-house employees on work and personal devices.
4
+
5
+ *Scroll down to ***Rollout, setup, etc.*** to see information about dependencies, Poetry, and pyenv.*
6
+
7
+ ## How to run pipeline
8
+
9
+ Check that the secrets.yaml file in the default-workspace is loading:
10
+ ```
11
+ poetry run python -m src.pipeline.env
12
+ ```
13
+ Recieve an export file of all the points known to your EDS:
14
+ ```
15
+ poetry run python -m src.pipeline.api.eds demo-point-export
16
+ ```
17
+ Run the existing eds_to_rjn workspace:
18
+ ```
19
+ poetry run python -m workspaces.eds_to_rjn.scripts.daemon_runner test
20
+ ```
21
+ Check connectivity:
22
+ ```
23
+ poetry run python -m src.pipeline.api.eds ping
24
+ poetry run python -m src.pipeline.api.rjn ping
25
+ ```
26
+ Other commands:
27
+ ```
28
+ .\main.bat # put the daemon into service: ill advised.
29
+ .\main.ps1
30
+ ```
31
+
32
+ ## Implementation
33
+ The current ideal implementation of `pipeline` involves `Windows Task Scheduler`.
34
+ This is installed on an `EDS` server, to pull data and send it to a third party.
35
+ The Task is set up to call `main_eds_to_rjn_quiet.ps1` as the entry point.
36
+ The iterative hourly timing handled is by `Windows Task Scheduler` rather than pythonically with the (no unused) `setup_schedules()` function.
37
+ The `main` function from the `daemon_runner` file is run; this used to call the `run_hourly_tabular_trend_eds_to_rjn()` function.
38
+ Environment managemenet is handled with `venv` rather than `pyenv` and `poetry`, because `Task Scheduler` directs the process to a different user.
39
+
40
+ ## secrets.yaml
41
+ Access will not work without a secrets.yaml file in /pipeline/workspaces/your-workspace-name/config/secrets.yaml
42
+
43
+ *API keys are specific to each the workspace, and can be referenced in the workspace scripts.*
44
+ You would edit the secrets.yaml file to specify your own EDS server and credentials.
45
+ Important: You need to VPN onto the same network as your server, EDS or otherwise, if it is not available to the outside world.
46
+ ### Example secrets.yaml:
47
+ ```
48
+ eds_apis:
49
+ MyServer1:
50
+ url: "http://127.0.0.1:43084/api/v1/"
51
+ username: "admin"
52
+ password: ""
53
+ MyServer2:
54
+ url: "http://some-ip-address:port/api/v1/"
55
+ username: "admin"
56
+ password: ""
57
+
58
+ contractor_apis:
59
+ MySpecialContractor:
60
+ url: "https://contractor-api.com/v1/special/"
61
+ client_id: "special-user"
62
+ password: "2685steam"
63
+ ```
64
+ The ***workspaces*** folder is designed to accomodate any custom workspaces or projects that you might add. You can alter the default projet file by altering the directory name indicated in the /pipeline/workspaces/default-workspace.toml file.
65
+
66
+ # Future goals:
67
+ - Submit a pipeline library to PyPi.
68
+ - Generate a cookiecutter template for building new project directories.
69
+ - Use mulchcli (another ongoing repo) to guide users through setting up secrets.yaml files step-by-step.
70
+
71
+ # Maintenance Notes:
72
+ - Implement session = requests.Session(), like an adult.
73
+ - daemon_runner.py should use both Maxson and Stiles access.
74
+
75
+ # Rollout, setup, etc.
76
+ It is recommended that you use **pyenv** for setting the Python version and generating a virtual environment, though this is optional.
77
+
78
+ To benefit from the pyproject.toml rollout for this project, **Poetry** is entirely necessary for installing requirements.
79
+
80
+ ### Why pyenv?
81
+ venv and virtualenv are confusing, and pyenv is not confusing. With pyenv, it is easy to run many projects at once, with different requirements for each.
82
+ You only need to install pyenv once on you system, and then you use it to access different versions of Python.
83
+
84
+ #### Note to anyone who doesn't yet understand the glory of virtual environments in Python:
85
+ Do it. You do not want to install special requirements to your system version of Python.
86
+
87
+ ### Why Poetry?
88
+ **Poetry** is my favorite dependency management tool. It is very easy. You only need to install it once. **Poetry** also has the benefit of generating a directory-specific virtual environment.
89
+
90
+ ## Use pyenv to set your Python version
91
+ (3.11.9 or other 3.11.xx).
92
+ ### Install pyenv (skip if pyenv is already installed on your system)
93
+ How to install pyenv-win (https://github.com/pyenv-win/pyenv-win)
94
+ ```
95
+ Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy RemoteSigned
96
+ Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1"; &"./install-pyenv-win.ps1"
97
+ ```
98
+ It is worth it to make the pyenv command persistent in PowerShell, by editing the $profile file:
99
+ ```
100
+ notepad $profile
101
+ ```
102
+ to include something like:
103
+ ```
104
+ $env:PYENV = "$HOME\.pyenv\pyenv-win"
105
+ $env:PYENV_ROOT = "$HOME\.pyenv\pyenv-win"
106
+ $env:PYENV_HOME = "$HOME\.pyenv\pyenv-win"
107
+ $env:Path += ";$env:PYENV\bin;$env:PYENV\shims"
108
+
109
+ # Initialize pyenv
110
+ #$pyenvInit = & $env:PYENV_HOME\bin\pyenv init --path
111
+ #Invoke-Expression $pyenvInit
112
+
113
+ # Manually set up the pyenv environment
114
+ function Invoke-PyenvInit {
115
+ # Update PATH to include pyenv directories
116
+ $pyenvShims = "$env:PYENV\shims"
117
+ $pyenvBin = "$env:PYENV\bin"
118
+ $env:PATH = "$pyenvBin;$pyenvShims;$env:PATH"
119
+ }
120
+
121
+ # Initialize pyenv
122
+ Invoke-PyenvInit
123
+ ```
124
+ How to install pyenv on linux (https://github.com/pyenv/pyenv)
125
+ ```
126
+ curl -fsSL https://pyenv.run | bash
127
+ ```
128
+ To make the pyenv command persistent in the Bourne Again SHell, edit ~/.bashrc
129
+ ```
130
+ nano ~/.bashrc
131
+ ```
132
+ to include something like:
133
+ ```
134
+ export Path="$HOME/.local/bin:$PATH"
135
+ export PYENV_ROOT="$HOME/.pyenv"
136
+ [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
137
+ eval "$(pyenv init - bash)"
138
+ ```
139
+ ### Post-install, leverage the benefits of pyenv
140
+ Install Python 3.11.9 using pyenv:
141
+ ```
142
+ # pyenv install --list # See all Python versions able with pyenv.
143
+ pyenv install 3.11.9
144
+ # pyenv local 3.11.9 # to set your current directory version
145
+ # pyenv global 3.11.9 # to set the assumed version in any directory
146
+ # pyenv global system # revert to your system installation as the assumed version
147
+ ```
148
+ ## Use Poetry to run deploy the requirements for this project
149
+ How to install poetry (https://github.com/python-poetry/poetry)
150
+ ```
151
+ Remove-Item Alias:curl # Solution to a common PowerShell issue
152
+ curl -sSL https://install.python-poetry.org | python3
153
+ # Alternatively:
154
+ // pip install poetry
155
+ # Or, even:
156
+ // pyenv exec pip install poetry
157
+ ```
158
+ ## Git clone pipeline, open source
159
+ ```
160
+ git clone https://github.com/City-of-Memphis-Wastewater/pipeline.git
161
+ cd pipline
162
+ pyenv local 3.11.9 # to set your current directory version
163
+ ```
164
+ Explicitly set poetry to use the local pyenv version.
165
+ ```
166
+ poetry python list
167
+ # You'll see something like this:
168
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.13.exe
169
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python313.exe
170
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python3.exe
171
+ >> 3.13.2 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.13.2/python.exe
172
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.11.exe
173
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python311.exe
174
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python3.exe
175
+ >> 3.11.9 CPython System C:/Users/<user>/.pyenv/pyenv-win/versions/3.11.9/python.exe
176
+ # Copy and paste ~any~ of the comparable paths to pyenv 3.11.9 ...
177
+ poetry use C:\Users\<user>\.pyenv\pyenv-win\versions\3.11.9\python.exe
178
+ ```
179
+ Pull the requirements from the pyproject.toml file for packagage installation.
180
+ ```
181
+ poetry install
182
+ # This is where the magic happens.
183
+ # When in doubt, run this again.
184
+ # Especially if you get a ModuleNotFound warning.
185
+ # Sometimes it doesn't take until you use "poetry run python".
186
+ poetry run python
187
+ poetry run python -m src.pipeline.env
188
+ poetry run python -m src.pipeline.api.eds ping
189
+ poetry run python -m workspaces.eds_to_rjn.scripts.main
190
+ ```
191
+
192
+ # References
193
+ PEP621.toml
194
+
195
+ # Installation on Termux on Android
196
+
197
+ Due to `maturin` (Rust build dependency) required by newer versions of `pydantic`, use older versions of FastAPI and Pydantic that don't require Rust, to successfully install on Termux.
198
+
199
+ Also, Termux does not allow `poetry` or `pyenv`, so install packages directly to your generic environement with pip.
200
+
201
+ ```
202
+ pip install -r requirements-termux.txt
203
+ ```
204
+
205
+ Note that Termux does not allow `numpy` or `pandas`. Nor does it allow any plotting through a pop up window, like with tkinter, freesimplegui, matplotlib, etc.
206
+
@@ -0,0 +1,60 @@
1
+ [tool.poetry]
2
+ name = "pipeline-eds"
3
+ version = "0.2.4"
4
+ description = "The official API pipeline library for mulch-based projects. Key target: Emerson Ovation EDS REST API."
5
+ authors = ["George Clayton Bennett <george.bennett@memphistn.gov>"]
6
+ license = "BSD-3"
7
+ readme = "README.md"
8
+ packages = [
9
+ { include = "pipeline", from = "src" },
10
+ { include = "workspaces", from = "." }
11
+ ] # This makes `pipeline` available as an importable top-level module
12
+ homepage = "https://github.com/city-of-memphis-wastewater/pipeline"
13
+ repository = "https://github.com/city-of-memphis-wastewater/pipeline"
14
+
15
+ [tool.poetry.scripts]
16
+ #time-manager = "pipeline.time_manager:main"
17
+ #workspace-manager = "pipeline.workspace_manager:main" # included in init
18
+ #collector = "pipeline.collector:main"
19
+ #eds-client = "pipeline.api.eds:main"
20
+ #rjn-client = "pipeline.api.rjn:main"
21
+ pipeline = "pipeline.cli:app" # for installation of pipeline as a cli. Ex: pipeline trend M100FI -s June3 -f June17
22
+
23
+
24
+ [tool.pytest.ini_options]
25
+ markers = [
26
+ "slow: marks tests as slow (deselect with '-m \"not slow\"')",
27
+ ]
28
+ #addopts = "--cov=pipeline --cov-report=html" # visually inspect coverage, e.g. before a release or PR.
29
+ addopts = "-ra --cov=pipeline --cov-report=term-missing" # for development — fast and informative in terminal.
30
+ testpaths = ["tests"]
31
+ pythonpath = ["src", "."]
32
+
33
+ [tool.poetry.dependencies]
34
+ python = ">=3.11,<4.0.0"
35
+ requests = ">=2.32.3,<3.0.0"
36
+ toml = ">=0.10.2,<0.11.0"
37
+ pyyaml = ">=6.0.2,<7.0.0"
38
+ certifi = ">=2025.1.31,<2026.0.0"
39
+ schedule = ">=1.2.2,<2.0.0"
40
+ fastapi = ">=0.115.12,<0.116.0"
41
+ urllib3 = "^2.4.0"
42
+ plotly = "^6.2.0"
43
+ uvicorn = "^0.34.3"
44
+ mulch = "^0.2.8"
45
+ mysql-connector-python = "^9.3.0"
46
+ pyodbc = "^5.2.0"
47
+ tzdata = "^2025.2"
48
+ pendulum = "^3.1.0"
49
+
50
+ [tool.poetry.group.dev.dependencies]
51
+ pytest = "^8.4.1"
52
+ pytest-cov = "^6.2.1"
53
+ pygal = "^3.0.5"
54
+
55
+ [build-system]
56
+ requires = ["poetry-core>=2.0.0,<3.0.0"]
57
+ build-backend = "poetry.core.masonry.api"
58
+
59
+ [tool.poetry.plugins."scripts"]
60
+ controller = "pipeline.daemon.controller:main_cli"
@@ -0,0 +1,4 @@
1
+ # pipeline/__init__.py
2
+ from pipeline.workspace_manager import WorkspaceManager,establish_default_workspace
3
+
4
+ __all__ = ["WorkspaceManager","establish_default_workspace"]
@@ -0,0 +1 @@
1
+ # src/__main__.py
File without changes