fielddash 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fielddash-0.1.0/LICENSE +21 -0
- fielddash-0.1.0/PKG-INFO +267 -0
- fielddash-0.1.0/README.md +230 -0
- fielddash-0.1.0/fielddash/__init__.py +17 -0
- fielddash-0.1.0/fielddash/access.py +73 -0
- fielddash-0.1.0/fielddash/app.py +24 -0
- fielddash-0.1.0/fielddash/cli.py +97 -0
- fielddash-0.1.0/fielddash/core/__init__.py +0 -0
- fielddash-0.1.0/fielddash/core/config.py +115 -0
- fielddash-0.1.0/fielddash/core/context.py +14 -0
- fielddash-0.1.0/fielddash/core/env.py +20 -0
- fielddash-0.1.0/fielddash/core/loader.py +47 -0
- fielddash-0.1.0/fielddash/core/model.py +84 -0
- fielddash-0.1.0/fielddash/core/normalize.py +104 -0
- fielddash-0.1.0/fielddash/core/registry.py +27 -0
- fielddash-0.1.0/fielddash/core/schema.py +123 -0
- fielddash-0.1.0/fielddash/scaffold.py +149 -0
- fielddash-0.1.0/fielddash/sources/__init__.py +0 -0
- fielddash-0.1.0/fielddash/sources/base.py +62 -0
- fielddash-0.1.0/fielddash/sources/epicollect.py +196 -0
- fielddash-0.1.0/fielddash/sources/json_file.py +35 -0
- fielddash-0.1.0/fielddash/ui/__init__.py +0 -0
- fielddash-0.1.0/fielddash/ui/charts.py +202 -0
- fielddash-0.1.0/fielddash/ui/compat.py +23 -0
- fielddash-0.1.0/fielddash/ui/filters.py +90 -0
- fielddash-0.1.0/fielddash/ui/maps.py +56 -0
- fielddash-0.1.0/fielddash/views/__init__.py +0 -0
- fielddash-0.1.0/fielddash/views/data.py +36 -0
- fielddash-0.1.0/fielddash/views/highlights.py +37 -0
- fielddash-0.1.0/fielddash/views/overview.py +41 -0
- fielddash-0.1.0/fielddash/views/questions.py +62 -0
- fielddash-0.1.0/fielddash/web.py +93 -0
- fielddash-0.1.0/fielddash.egg-info/PKG-INFO +267 -0
- fielddash-0.1.0/fielddash.egg-info/SOURCES.txt +42 -0
- fielddash-0.1.0/fielddash.egg-info/dependency_links.txt +1 -0
- fielddash-0.1.0/fielddash.egg-info/entry_points.txt +2 -0
- fielddash-0.1.0/fielddash.egg-info/requires.txt +14 -0
- fielddash-0.1.0/fielddash.egg-info/top_level.txt +1 -0
- fielddash-0.1.0/pyproject.toml +67 -0
- fielddash-0.1.0/setup.cfg +4 -0
- fielddash-0.1.0/tests/test_app.py +86 -0
- fielddash-0.1.0/tests/test_epicollect.py +70 -0
- fielddash-0.1.0/tests/test_init.py +89 -0
- fielddash-0.1.0/tests/test_schema.py +67 -0
fielddash-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sergio Costa, André Moura, LambdaGeo / UFMA
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
fielddash-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fielddash
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Schema-driven dashboards for field data collection (Epicollect5).
|
|
5
|
+
Author: André Moura Lima
|
|
6
|
+
Author-email: Sergio Souza Costa <sergio.costa@ufma.br>, Ricardo Luvizotto Santos <ricardo.luvizotto@ufma.br>, Marianna Basso Jorge <mb.jorge@ufma.br>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/LambdaGeo/fielddash
|
|
9
|
+
Project-URL: Repository, https://github.com/LambdaGeo/fielddash
|
|
10
|
+
Project-URL: Documentation, https://lambdageo.github.io/fielddash/
|
|
11
|
+
Project-URL: Bug Tracker, https://github.com/LambdaGeo/fielddash/issues
|
|
12
|
+
Keywords: epicollect5,fieldwork,data-collection,streamlit,gis,survey,geoprocessing
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: streamlit>=1.39
|
|
25
|
+
Requires-Dist: pandas>=2.0
|
|
26
|
+
Requires-Dist: plotly>=5.0
|
|
27
|
+
Requires-Dist: folium>=0.15
|
|
28
|
+
Requires-Dist: requests>=2.28
|
|
29
|
+
Requires-Dist: python-dotenv>=1.0
|
|
30
|
+
Requires-Dist: PyYAML>=6.0
|
|
31
|
+
Requires-Dist: openpyxl>=3.1
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: mkdocs-material<10,>=9.5; extra == "docs"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# fielddash
|
|
39
|
+
|
|
40
|
+
**Documentation: <https://lambdageo.github.io/fielddash/>**
|
|
41
|
+
|
|
42
|
+
Schema-driven dashboards for **field data collection** (Epicollect5), built directly from the **form schema**. Every question is identified by Epicollect's stable `ref`, and its input type (`radio`, `checkbox`, `integer`, `location`, ...) automatically dictates the filter, chart, and map representation. Modifying, adding, or reordering questions in the form does not break the dashboard, and setting up a new fieldwork survey requires only a YAML configuration file.
|
|
43
|
+
|
|
44
|
+
Ready-to-use pages:
|
|
45
|
+
- **Overview**: summary indicators, map, submissions over time, collectors.
|
|
46
|
+
- **Highlights**: curated charts configured in your YAML.
|
|
47
|
+
- **Questions**: all questions grouped by section, with search, text-field toggle, and dynamic cross-tabulation.
|
|
48
|
+
- **Data**: searchable table with CSV and Excel export.
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
Install it in an isolated environment rather than globally.
|
|
53
|
+
|
|
54
|
+
**One environment per project (recommended).** Works everywhere and is required for deployment and custom pages:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
mkdir my-survey && cd my-survey
|
|
58
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
59
|
+
pip install fielddash
|
|
60
|
+
fielddash init .
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
**Nothing installed first.** [pipx](https://pipx.pypa.io) or [uv](https://docs.astral.sh/uv/) can run `init` once; then create the environment from the generated `requirements.txt`:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pipx run fielddash init my-survey # or: uvx fielddash init my-survey
|
|
67
|
+
cd my-survey
|
|
68
|
+
python -m venv .venv && source .venv/bin/activate
|
|
69
|
+
pip install -r requirements.txt
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
**Command line only.** If you never edit code in the project, `pipx install fielddash` (or `uv tool install fielddash`) is enough for `init`, `fields` and `run`. The `streamlit_app.py` generated by `--deploy` imports `fielddash`, so it needs the per-project environment above.
|
|
73
|
+
|
|
74
|
+
For development: `git clone https://github.com/LambdaGeo/fielddash && cd fielddash && pip install -e ".[dev]"`.
|
|
75
|
+
|
|
76
|
+
## Quickstart
|
|
77
|
+
|
|
78
|
+
The example project lives in the repository; it is not part of the PyPI package.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
git clone https://github.com/LambdaGeo/fielddash && cd fielddash
|
|
82
|
+
python -m venv .venv && source .venv/bin/activate
|
|
83
|
+
pip install -e .
|
|
84
|
+
fielddash run examples/waste/project.yaml
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The example uses anonymized household survey data on solid waste management (Itaqui-Bacanga, São Luís – MA) and showcases a custom project page (`Recycling`).
|
|
88
|
+
|
|
89
|
+
## Setting Up a New Field Project
|
|
90
|
+
|
|
91
|
+
1. Scaffold the project folder:
|
|
92
|
+
```bash
|
|
93
|
+
fielddash init my-survey # project.yaml, requirements.txt, .env.example, .gitignore
|
|
94
|
+
fielddash init my-survey --deploy # + streamlit_app.py, secrets example
|
|
95
|
+
fielddash init my-survey --source json # offline project reading data/ (git-ignored)
|
|
96
|
+
```
|
|
97
|
+
Then `cp .env.example .env` and fill in the project slug and the Epicollect app credentials (generated under *Apps* in your project's administration area):
|
|
98
|
+
```bash
|
|
99
|
+
PROJECT_MY_SURVEY=project-slug
|
|
100
|
+
MY_SURVEY_CLIENT_ID=...
|
|
101
|
+
MY_SURVEY_CLIENT_SECRET=...
|
|
102
|
+
```
|
|
103
|
+
*Public projects do not require credentials: simply omit `credentials` in the config.*
|
|
104
|
+
|
|
105
|
+
2. The generated `project.yaml` starts minimal (or write it by hand):
|
|
106
|
+
```yaml
|
|
107
|
+
title: "My Field Survey"
|
|
108
|
+
source:
|
|
109
|
+
type: epicollect
|
|
110
|
+
project: ${PROJECT_MY_SURVEY} # slug, or the literal value
|
|
111
|
+
credentials: MY_SURVEY
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
3. Inspect the form fields to select aliases, filters, and highlights:
|
|
115
|
+
```bash
|
|
116
|
+
fielddash fields project.yaml
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
4. Complete the configuration (all keys below are optional) and run `fielddash run project.yaml`:
|
|
120
|
+
```yaml
|
|
121
|
+
subtitle: "Research team, institution..."
|
|
122
|
+
fields: # alias -> ref suffix, column, or question label
|
|
123
|
+
neighborhood: "5401ce"
|
|
124
|
+
waste_dest: "48c35e"
|
|
125
|
+
types: # force type (e.g. treat free text as category)
|
|
126
|
+
neighborhood: category
|
|
127
|
+
ignore: [created_by, "3401cb"] # hide fields (e.g. collector email)
|
|
128
|
+
filters: [created_at, neighborhood]
|
|
129
|
+
highlights:
|
|
130
|
+
- {field: waste_dest, by: neighborhood, title: "Waste destination by neighborhood"}
|
|
131
|
+
sections: # tabs on the Questions page (default: form groups)
|
|
132
|
+
- {title: "Profile", fields: [age, gender]}
|
|
133
|
+
map: {field: location, popup: [neighborhood]}
|
|
134
|
+
extensions: [pages] # .py files or directories with custom pages
|
|
135
|
+
timezone: America/Fortaleza
|
|
136
|
+
cache_minutes: 5
|
|
137
|
+
```
|
|
138
|
+
*(Portuguese keys from early versions — `titulo`, `fonte`, `campos`, ... — still load but are deprecated and emit a warning.)*
|
|
139
|
+
|
|
140
|
+
Running `fielddash run folder/` will scan all `.yaml` files in the directory and present a project selector in the sidebar.
|
|
141
|
+
To work offline or test without internet access, use `source: {type: json, data: ..., schema: ...}` (as shown in `examples/waste/project.yaml`).
|
|
142
|
+
|
|
143
|
+
## Using in a Custom Streamlit Script / Deployment
|
|
144
|
+
|
|
145
|
+
The dashboard can also be invoked as a Python function inside your own Streamlit app:
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
# streamlit_app.py
|
|
149
|
+
import fielddash
|
|
150
|
+
|
|
151
|
+
fielddash.dashboard("projects/") # or "projects/waste.yaml"
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Relative paths are resolved relative to the current directory or the script folder.
|
|
155
|
+
Run with:
|
|
156
|
+
```bash
|
|
157
|
+
streamlit run streamlit_app.py
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Streamlit Community Cloud
|
|
161
|
+
|
|
162
|
+
1. Create the project with the deploy files and try it locally:
|
|
163
|
+
```bash
|
|
164
|
+
mkdir my-survey && cd my-survey
|
|
165
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
166
|
+
pip install fielddash
|
|
167
|
+
fielddash init . --deploy
|
|
168
|
+
cp .env.example .env # fill in, then edit project.yaml
|
|
169
|
+
streamlit run streamlit_app.py
|
|
170
|
+
```
|
|
171
|
+
2. Push the folder to a GitHub repository. `.env` and `.streamlit/secrets.toml` are git-ignored: credentials never go to the repo.
|
|
172
|
+
3. On [share.streamlit.io](https://share.streamlit.io) choose *Create app*, pick the repository and branch, and set `streamlit_app.py` as the main file.
|
|
173
|
+
4. Under *Advanced settings → Secrets* (later: *Settings → Secrets*) paste the content of `.streamlit/secrets.toml.example`, filled in:
|
|
174
|
+
```toml
|
|
175
|
+
FIELD_ACCESS_PIN = "choose-a-code"
|
|
176
|
+
PROJECT_MY_SURVEY = "project-slug"
|
|
177
|
+
MY_SURVEY_CLIENT_ID = "..."
|
|
178
|
+
MY_SURVEY_CLIENT_SECRET = "..."
|
|
179
|
+
```
|
|
180
|
+
5. Deploy, then share the app link, or `https://<your-app>.streamlit.app/?token=<code>` to skip the login screen.
|
|
181
|
+
|
|
182
|
+
Notes:
|
|
183
|
+
- Use `source: {type: epicollect}` for the cloud. A `type: json` project reads `data/`, which is git-ignored on purpose (it may hold personal data), so it only works locally.
|
|
184
|
+
- Without `FIELD_ACCESS_PIN` the app is public: anyone with the link sees the data.
|
|
185
|
+
|
|
186
|
+
`fielddash` checks environment variables and `.env` first, then falls back to `st.secrets` (also resolving `${VAR}` references in YAML). The data cache is shared among server sessions, so Epicollect receives at most one request per project every `cache_minutes`.
|
|
187
|
+
|
|
188
|
+
### Restricting access
|
|
189
|
+
|
|
190
|
+
A deployed Streamlit app is public by default. Add `fielddash.require_access()` before `fielddash.dashboard(...)` (`fielddash init --deploy` already does) and set `FIELD_ACCESS_PIN` in the secrets: visitors then see a login screen, or can use a link ending in `?token=<code>`. Without the variable the call does nothing.
|
|
191
|
+
|
|
192
|
+
### Self-Hosted Server
|
|
193
|
+
```bash
|
|
194
|
+
fielddash run projects/ --server.port 8501 --server.headless true
|
|
195
|
+
```
|
|
196
|
+
(e.g., as a systemd service behind an Nginx reverse proxy with HTTPS).
|
|
197
|
+
|
|
198
|
+
## Custom Pages
|
|
199
|
+
|
|
200
|
+
A project-specific page is a standard `.py` file listed in `extensions`:
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
from fielddash.core.registry import page
|
|
204
|
+
from fielddash.ui.charts import render_field
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@page("Recycling", order=40)
|
|
208
|
+
def render(ctx):
|
|
209
|
+
ds = ctx.dataset
|
|
210
|
+
render_field(ds, ctx.df, ds.field("recycle_habit"), by=ds.field("neighborhood"))
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
- `ctx.df` provides the data with all active sidebar filters applied.
|
|
214
|
+
- `ds.field("alias")` resolves the field (column, type, options) dynamically by its stable schema ref.
|
|
215
|
+
- Additional data sources (e.g. KoboToolbox, CSV) can be registered with `@source("type")`.
|
|
216
|
+
|
|
217
|
+
## Project Structure
|
|
218
|
+
|
|
219
|
+
```
|
|
220
|
+
fielddash/
|
|
221
|
+
cli.py CLI commands `fielddash init`, `run` and `fields`
|
|
222
|
+
scaffold.py templates and writer behind `fielddash init`
|
|
223
|
+
access.py `fielddash.require_access()`: optional PIN/token gate
|
|
224
|
+
web.py `fielddash.dashboard()`: config, cache, filters, navigation
|
|
225
|
+
app.py Streamlit entry point used by `fielddash run`
|
|
226
|
+
core/schema.py Epicollect schema parser -> list[Field], column naming, reconciliation
|
|
227
|
+
core/normalize.py raw entries -> typed series (ordered categories, lists, coords, datetimes)
|
|
228
|
+
core/config.py YAML config parser (+ ${VAR} resolution from .env and secrets)
|
|
229
|
+
sources/ data source plugins (@source): epicollect (API, pagination), json
|
|
230
|
+
ui/ dynamic filters, charts, and maps based on field types
|
|
231
|
+
views/ default pages (@page): Overview, Highlights, Questions, Data
|
|
232
|
+
examples/ example projects with anonymized datasets
|
|
233
|
+
tests/ pytest test suite (schema, index shifting, AppTest integration)
|
|
234
|
+
docs/ MkDocs site (`mkdocs serve`); docs/history/ holds the original development plan
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
## Running Tests
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
pytest
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Building the Documentation
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
pip install -e ".[docs]"
|
|
247
|
+
mkdocs serve # http://127.0.0.1:8000
|
|
248
|
+
mkdocs build --strict # what CI runs
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
## Releasing (maintainers)
|
|
252
|
+
|
|
253
|
+
1. Bump the version in `pyproject.toml`, `fielddash/__init__.py` and `CITATION.cff`.
|
|
254
|
+
2. Publish a GitHub Release: `.github/workflows/publish.yml` builds the package and uploads it to PyPI through Trusted Publishing (no token stored).
|
|
255
|
+
|
|
256
|
+
One-time setup on pypi.org → *Publishing* → *Add a new pending publisher*: project `fielddash`, owner `LambdaGeo`, repository `fielddash`, workflow `publish.yml`. PyPI never lets a published version be overwritten, so check with `python -m build && twine check dist/*` (and optionally TestPyPI) first.
|
|
257
|
+
|
|
258
|
+
## Authors
|
|
259
|
+
|
|
260
|
+
- **Sergio Costa** ([@LambdaGeo](https://github.com/LambdaGeo)) — Universidade Federal do Maranhão (UFMA)
|
|
261
|
+
- **André Moura Lima** ([@AndreMouraL](https://github.com/AndreMouraL)) — Universidade Federal do Maranhão (UFMA)
|
|
262
|
+
- **Ricardo Luvizotto Santos** — Universidade Federal do Maranhão (UFMA)
|
|
263
|
+
- **Marianna Basso Jorge** — Universidade Federal do Maranhão (UFMA)
|
|
264
|
+
|
|
265
|
+
## License
|
|
266
|
+
|
|
267
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
# fielddash
|
|
2
|
+
|
|
3
|
+
**Documentation: <https://lambdageo.github.io/fielddash/>**
|
|
4
|
+
|
|
5
|
+
Schema-driven dashboards for **field data collection** (Epicollect5), built directly from the **form schema**. Every question is identified by Epicollect's stable `ref`, and its input type (`radio`, `checkbox`, `integer`, `location`, ...) automatically dictates the filter, chart, and map representation. Modifying, adding, or reordering questions in the form does not break the dashboard, and setting up a new fieldwork survey requires only a YAML configuration file.
|
|
6
|
+
|
|
7
|
+
Ready-to-use pages:
|
|
8
|
+
- **Overview**: summary indicators, map, submissions over time, collectors.
|
|
9
|
+
- **Highlights**: curated charts configured in your YAML.
|
|
10
|
+
- **Questions**: all questions grouped by section, with search, text-field toggle, and dynamic cross-tabulation.
|
|
11
|
+
- **Data**: searchable table with CSV and Excel export.
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
Install it in an isolated environment rather than globally.
|
|
16
|
+
|
|
17
|
+
**One environment per project (recommended).** Works everywhere and is required for deployment and custom pages:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
mkdir my-survey && cd my-survey
|
|
21
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
22
|
+
pip install fielddash
|
|
23
|
+
fielddash init .
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
**Nothing installed first.** [pipx](https://pipx.pypa.io) or [uv](https://docs.astral.sh/uv/) can run `init` once; then create the environment from the generated `requirements.txt`:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pipx run fielddash init my-survey # or: uvx fielddash init my-survey
|
|
30
|
+
cd my-survey
|
|
31
|
+
python -m venv .venv && source .venv/bin/activate
|
|
32
|
+
pip install -r requirements.txt
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
**Command line only.** If you never edit code in the project, `pipx install fielddash` (or `uv tool install fielddash`) is enough for `init`, `fields` and `run`. The `streamlit_app.py` generated by `--deploy` imports `fielddash`, so it needs the per-project environment above.
|
|
36
|
+
|
|
37
|
+
For development: `git clone https://github.com/LambdaGeo/fielddash && cd fielddash && pip install -e ".[dev]"`.
|
|
38
|
+
|
|
39
|
+
## Quickstart
|
|
40
|
+
|
|
41
|
+
The example project lives in the repository; it is not part of the PyPI package.
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
git clone https://github.com/LambdaGeo/fielddash && cd fielddash
|
|
45
|
+
python -m venv .venv && source .venv/bin/activate
|
|
46
|
+
pip install -e .
|
|
47
|
+
fielddash run examples/waste/project.yaml
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The example uses anonymized household survey data on solid waste management (Itaqui-Bacanga, São Luís – MA) and showcases a custom project page (`Recycling`).
|
|
51
|
+
|
|
52
|
+
## Setting Up a New Field Project
|
|
53
|
+
|
|
54
|
+
1. Scaffold the project folder:
|
|
55
|
+
```bash
|
|
56
|
+
fielddash init my-survey # project.yaml, requirements.txt, .env.example, .gitignore
|
|
57
|
+
fielddash init my-survey --deploy # + streamlit_app.py, secrets example
|
|
58
|
+
fielddash init my-survey --source json # offline project reading data/ (git-ignored)
|
|
59
|
+
```
|
|
60
|
+
Then `cp .env.example .env` and fill in the project slug and the Epicollect app credentials (generated under *Apps* in your project's administration area):
|
|
61
|
+
```bash
|
|
62
|
+
PROJECT_MY_SURVEY=project-slug
|
|
63
|
+
MY_SURVEY_CLIENT_ID=...
|
|
64
|
+
MY_SURVEY_CLIENT_SECRET=...
|
|
65
|
+
```
|
|
66
|
+
*Public projects do not require credentials: simply omit `credentials` in the config.*
|
|
67
|
+
|
|
68
|
+
2. The generated `project.yaml` starts minimal (or write it by hand):
|
|
69
|
+
```yaml
|
|
70
|
+
title: "My Field Survey"
|
|
71
|
+
source:
|
|
72
|
+
type: epicollect
|
|
73
|
+
project: ${PROJECT_MY_SURVEY} # slug, or the literal value
|
|
74
|
+
credentials: MY_SURVEY
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
3. Inspect the form fields to select aliases, filters, and highlights:
|
|
78
|
+
```bash
|
|
79
|
+
fielddash fields project.yaml
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
4. Complete the configuration (all keys below are optional) and run `fielddash run project.yaml`:
|
|
83
|
+
```yaml
|
|
84
|
+
subtitle: "Research team, institution..."
|
|
85
|
+
fields: # alias -> ref suffix, column, or question label
|
|
86
|
+
neighborhood: "5401ce"
|
|
87
|
+
waste_dest: "48c35e"
|
|
88
|
+
types: # force type (e.g. treat free text as category)
|
|
89
|
+
neighborhood: category
|
|
90
|
+
ignore: [created_by, "3401cb"] # hide fields (e.g. collector email)
|
|
91
|
+
filters: [created_at, neighborhood]
|
|
92
|
+
highlights:
|
|
93
|
+
- {field: waste_dest, by: neighborhood, title: "Waste destination by neighborhood"}
|
|
94
|
+
sections: # tabs on the Questions page (default: form groups)
|
|
95
|
+
- {title: "Profile", fields: [age, gender]}
|
|
96
|
+
map: {field: location, popup: [neighborhood]}
|
|
97
|
+
extensions: [pages] # .py files or directories with custom pages
|
|
98
|
+
timezone: America/Fortaleza
|
|
99
|
+
cache_minutes: 5
|
|
100
|
+
```
|
|
101
|
+
*(Portuguese keys from early versions — `titulo`, `fonte`, `campos`, ... — still load but are deprecated and emit a warning.)*
|
|
102
|
+
|
|
103
|
+
Running `fielddash run folder/` will scan all `.yaml` files in the directory and present a project selector in the sidebar.
|
|
104
|
+
To work offline or test without internet access, use `source: {type: json, data: ..., schema: ...}` (as shown in `examples/waste/project.yaml`).
|
|
105
|
+
|
|
106
|
+
## Using in a Custom Streamlit Script / Deployment
|
|
107
|
+
|
|
108
|
+
The dashboard can also be invoked as a Python function inside your own Streamlit app:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
# streamlit_app.py
|
|
112
|
+
import fielddash
|
|
113
|
+
|
|
114
|
+
fielddash.dashboard("projects/") # or "projects/waste.yaml"
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Relative paths are resolved relative to the current directory or the script folder.
|
|
118
|
+
Run with:
|
|
119
|
+
```bash
|
|
120
|
+
streamlit run streamlit_app.py
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Streamlit Community Cloud
|
|
124
|
+
|
|
125
|
+
1. Create the project with the deploy files and try it locally:
|
|
126
|
+
```bash
|
|
127
|
+
mkdir my-survey && cd my-survey
|
|
128
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
129
|
+
pip install fielddash
|
|
130
|
+
fielddash init . --deploy
|
|
131
|
+
cp .env.example .env # fill in, then edit project.yaml
|
|
132
|
+
streamlit run streamlit_app.py
|
|
133
|
+
```
|
|
134
|
+
2. Push the folder to a GitHub repository. `.env` and `.streamlit/secrets.toml` are git-ignored: credentials never go to the repo.
|
|
135
|
+
3. On [share.streamlit.io](https://share.streamlit.io) choose *Create app*, pick the repository and branch, and set `streamlit_app.py` as the main file.
|
|
136
|
+
4. Under *Advanced settings → Secrets* (later: *Settings → Secrets*) paste the content of `.streamlit/secrets.toml.example`, filled in:
|
|
137
|
+
```toml
|
|
138
|
+
FIELD_ACCESS_PIN = "choose-a-code"
|
|
139
|
+
PROJECT_MY_SURVEY = "project-slug"
|
|
140
|
+
MY_SURVEY_CLIENT_ID = "..."
|
|
141
|
+
MY_SURVEY_CLIENT_SECRET = "..."
|
|
142
|
+
```
|
|
143
|
+
5. Deploy, then share the app link, or `https://<your-app>.streamlit.app/?token=<code>` to skip the login screen.
|
|
144
|
+
|
|
145
|
+
Notes:
|
|
146
|
+
- Use `source: {type: epicollect}` for the cloud. A `type: json` project reads `data/`, which is git-ignored on purpose (it may hold personal data), so it only works locally.
|
|
147
|
+
- Without `FIELD_ACCESS_PIN` the app is public: anyone with the link sees the data.
|
|
148
|
+
|
|
149
|
+
`fielddash` checks environment variables and `.env` first, then falls back to `st.secrets` (also resolving `${VAR}` references in YAML). The data cache is shared among server sessions, so Epicollect receives at most one request per project every `cache_minutes`.
|
|
150
|
+
|
|
151
|
+
### Restricting access
|
|
152
|
+
|
|
153
|
+
A deployed Streamlit app is public by default. Add `fielddash.require_access()` before `fielddash.dashboard(...)` (`fielddash init --deploy` already does) and set `FIELD_ACCESS_PIN` in the secrets: visitors then see a login screen, or can use a link ending in `?token=<code>`. Without the variable the call does nothing.
|
|
154
|
+
|
|
155
|
+
### Self-Hosted Server
|
|
156
|
+
```bash
|
|
157
|
+
fielddash run projects/ --server.port 8501 --server.headless true
|
|
158
|
+
```
|
|
159
|
+
(e.g., as a systemd service behind an Nginx reverse proxy with HTTPS).
|
|
160
|
+
|
|
161
|
+
## Custom Pages
|
|
162
|
+
|
|
163
|
+
A project-specific page is a standard `.py` file listed in `extensions`:
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from fielddash.core.registry import page
|
|
167
|
+
from fielddash.ui.charts import render_field
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@page("Recycling", order=40)
|
|
171
|
+
def render(ctx):
|
|
172
|
+
ds = ctx.dataset
|
|
173
|
+
render_field(ds, ctx.df, ds.field("recycle_habit"), by=ds.field("neighborhood"))
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
- `ctx.df` provides the data with all active sidebar filters applied.
|
|
177
|
+
- `ds.field("alias")` resolves the field (column, type, options) dynamically by its stable schema ref.
|
|
178
|
+
- Additional data sources (e.g. KoboToolbox, CSV) can be registered with `@source("type")`.
|
|
179
|
+
|
|
180
|
+
## Project Structure
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
fielddash/
|
|
184
|
+
cli.py CLI commands `fielddash init`, `run` and `fields`
|
|
185
|
+
scaffold.py templates and writer behind `fielddash init`
|
|
186
|
+
access.py `fielddash.require_access()`: optional PIN/token gate
|
|
187
|
+
web.py `fielddash.dashboard()`: config, cache, filters, navigation
|
|
188
|
+
app.py Streamlit entry point used by `fielddash run`
|
|
189
|
+
core/schema.py Epicollect schema parser -> list[Field], column naming, reconciliation
|
|
190
|
+
core/normalize.py raw entries -> typed series (ordered categories, lists, coords, datetimes)
|
|
191
|
+
core/config.py YAML config parser (+ ${VAR} resolution from .env and secrets)
|
|
192
|
+
sources/ data source plugins (@source): epicollect (API, pagination), json
|
|
193
|
+
ui/ dynamic filters, charts, and maps based on field types
|
|
194
|
+
views/ default pages (@page): Overview, Highlights, Questions, Data
|
|
195
|
+
examples/ example projects with anonymized datasets
|
|
196
|
+
tests/ pytest test suite (schema, index shifting, AppTest integration)
|
|
197
|
+
docs/ MkDocs site (`mkdocs serve`); docs/history/ holds the original development plan
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Running Tests
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
pytest
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## Building the Documentation
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
pip install -e ".[docs]"
|
|
210
|
+
mkdocs serve # http://127.0.0.1:8000
|
|
211
|
+
mkdocs build --strict # what CI runs
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## Releasing (maintainers)
|
|
215
|
+
|
|
216
|
+
1. Bump the version in `pyproject.toml`, `fielddash/__init__.py` and `CITATION.cff`.
|
|
217
|
+
2. Publish a GitHub Release: `.github/workflows/publish.yml` builds the package and uploads it to PyPI through Trusted Publishing (no token stored).
|
|
218
|
+
|
|
219
|
+
One-time setup on pypi.org → *Publishing* → *Add a new pending publisher*: project `fielddash`, owner `LambdaGeo`, repository `fielddash`, workflow `publish.yml`. PyPI never lets a published version be overwritten, so check with `python -m build && twine check dist/*` (and optionally TestPyPI) first.
|
|
220
|
+
|
|
221
|
+
## Authors
|
|
222
|
+
|
|
223
|
+
- **Sergio Costa** ([@LambdaGeo](https://github.com/LambdaGeo)) — Universidade Federal do Maranhão (UFMA)
|
|
224
|
+
- **André Moura Lima** ([@AndreMouraL](https://github.com/AndreMouraL)) — Universidade Federal do Maranhão (UFMA)
|
|
225
|
+
- **Ricardo Luvizotto Santos** — Universidade Federal do Maranhão (UFMA)
|
|
226
|
+
- **Marianna Basso Jorge** — Universidade Federal do Maranhão (UFMA)
|
|
227
|
+
|
|
228
|
+
## License
|
|
229
|
+
|
|
230
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""fielddash: schema-driven dashboards for field data collection (Epicollect5)."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def dashboard(target=".", *, configure_page: bool = True):
|
|
7
|
+
"""Renders the dashboard for a project (.yaml) or directory of projects in a Streamlit script."""
|
|
8
|
+
from fielddash.web import dashboard as _dashboard # imports streamlit only when called
|
|
9
|
+
|
|
10
|
+
return _dashboard(target, configure_page=configure_page)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def require_access():
|
|
14
|
+
"""Optional PIN gate: stops the Streamlit script on a login screen unless FIELD_ACCESS_PIN matches."""
|
|
15
|
+
from fielddash.access import require_access as _require_access # imports streamlit only when called
|
|
16
|
+
|
|
17
|
+
return _require_access()
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Optional PIN gate for dashboards shared with a field team (Streamlit).
|
|
2
|
+
|
|
3
|
+
import fielddash
|
|
4
|
+
fielddash.require_access() # no-op unless FIELD_ACCESS_PIN is set (env, .env or Streamlit secrets)
|
|
5
|
+
fielddash.dashboard("project.yaml", configure_page=False)
|
|
6
|
+
|
|
7
|
+
Team members enter the code on a login screen or open a link ending in `?token=<code>`.
|
|
8
|
+
"""
|
|
9
|
+
import hmac
|
|
10
|
+
|
|
11
|
+
import streamlit as st
|
|
12
|
+
|
|
13
|
+
from fielddash.core.env import getenv
|
|
14
|
+
|
|
15
|
+
PIN_VARS = ("FIELD_ACCESS_PIN", "FIELD_ACCESS_TOKEN", "ACCESS_PIN")
|
|
16
|
+
URL_PARAMS = ("token", "pin", "key")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def required_pin():
|
|
20
|
+
"""The configured access code, or None when the dashboard is open to everyone."""
|
|
21
|
+
for name in PIN_VARS:
|
|
22
|
+
value = getenv(name)
|
|
23
|
+
if value:
|
|
24
|
+
return str(value).strip()
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _matches(given, expected) -> bool:
|
|
29
|
+
return hmac.compare_digest(str(given).strip().encode(), str(expected).encode())
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _url_token():
|
|
33
|
+
for name in URL_PARAMS:
|
|
34
|
+
value = st.query_params.get(name)
|
|
35
|
+
if value:
|
|
36
|
+
return value
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _sidebar_logout():
|
|
41
|
+
st.sidebar.caption("🟢 Access granted (field team)")
|
|
42
|
+
if st.sidebar.button("End session", key="logout_btn"):
|
|
43
|
+
st.session_state["authenticated"] = False
|
|
44
|
+
for name in URL_PARAMS: # otherwise the link would log the user straight back in
|
|
45
|
+
if name in st.query_params:
|
|
46
|
+
del st.query_params[name]
|
|
47
|
+
st.rerun()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def require_access() -> None:
|
|
51
|
+
"""Stops the script on a login screen unless the visitor has the access code."""
|
|
52
|
+
pin = required_pin()
|
|
53
|
+
if not pin:
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
token = _url_token()
|
|
57
|
+
if st.session_state.get("authenticated") or (token and _matches(token, pin)):
|
|
58
|
+
st.session_state["authenticated"] = True
|
|
59
|
+
_sidebar_logout()
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
_, center, _ = st.columns([1, 2, 1])
|
|
63
|
+
with center:
|
|
64
|
+
st.markdown("## 🔐 Restricted area — field team")
|
|
65
|
+
st.info("This dashboard contains field-work data. Enter the access code provided by the research coordination.")
|
|
66
|
+
with st.form("login_form"):
|
|
67
|
+
entered = st.text_input("Access code / PIN:", type="password")
|
|
68
|
+
if st.form_submit_button("Enter dashboard"):
|
|
69
|
+
if entered and _matches(entered, pin):
|
|
70
|
+
st.session_state["authenticated"] = True
|
|
71
|
+
st.rerun()
|
|
72
|
+
st.error("Wrong access code. Check with the responsible team.")
|
|
73
|
+
st.stop()
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Streamlit script used by `fielddash run`.
|
|
2
|
+
|
|
3
|
+
fielddash run project.yaml # single project
|
|
4
|
+
fielddash run dir/ # choose between .yaml projects in sidebar
|
|
5
|
+
|
|
6
|
+
To deploy (e.g. Streamlit Community Cloud), use `fielddash.dashboard(...)`
|
|
7
|
+
in your project's script.
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
from fielddash.web import dashboard
|
|
13
|
+
|
|
14
|
+
ENV_CONFIG = "FIELDDASH_CONFIG"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _config_arg() -> str:
|
|
18
|
+
parser = argparse.ArgumentParser()
|
|
19
|
+
parser.add_argument("--config")
|
|
20
|
+
args, _ = parser.parse_known_args()
|
|
21
|
+
return args.config or os.environ.get(ENV_CONFIG) or "."
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
dashboard(_config_arg())
|