fielddash 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. fielddash-0.1.0/LICENSE +21 -0
  2. fielddash-0.1.0/PKG-INFO +267 -0
  3. fielddash-0.1.0/README.md +230 -0
  4. fielddash-0.1.0/fielddash/__init__.py +17 -0
  5. fielddash-0.1.0/fielddash/access.py +73 -0
  6. fielddash-0.1.0/fielddash/app.py +24 -0
  7. fielddash-0.1.0/fielddash/cli.py +97 -0
  8. fielddash-0.1.0/fielddash/core/__init__.py +0 -0
  9. fielddash-0.1.0/fielddash/core/config.py +115 -0
  10. fielddash-0.1.0/fielddash/core/context.py +14 -0
  11. fielddash-0.1.0/fielddash/core/env.py +20 -0
  12. fielddash-0.1.0/fielddash/core/loader.py +47 -0
  13. fielddash-0.1.0/fielddash/core/model.py +84 -0
  14. fielddash-0.1.0/fielddash/core/normalize.py +104 -0
  15. fielddash-0.1.0/fielddash/core/registry.py +27 -0
  16. fielddash-0.1.0/fielddash/core/schema.py +123 -0
  17. fielddash-0.1.0/fielddash/scaffold.py +149 -0
  18. fielddash-0.1.0/fielddash/sources/__init__.py +0 -0
  19. fielddash-0.1.0/fielddash/sources/base.py +62 -0
  20. fielddash-0.1.0/fielddash/sources/epicollect.py +196 -0
  21. fielddash-0.1.0/fielddash/sources/json_file.py +35 -0
  22. fielddash-0.1.0/fielddash/ui/__init__.py +0 -0
  23. fielddash-0.1.0/fielddash/ui/charts.py +202 -0
  24. fielddash-0.1.0/fielddash/ui/compat.py +23 -0
  25. fielddash-0.1.0/fielddash/ui/filters.py +90 -0
  26. fielddash-0.1.0/fielddash/ui/maps.py +56 -0
  27. fielddash-0.1.0/fielddash/views/__init__.py +0 -0
  28. fielddash-0.1.0/fielddash/views/data.py +36 -0
  29. fielddash-0.1.0/fielddash/views/highlights.py +37 -0
  30. fielddash-0.1.0/fielddash/views/overview.py +41 -0
  31. fielddash-0.1.0/fielddash/views/questions.py +62 -0
  32. fielddash-0.1.0/fielddash/web.py +93 -0
  33. fielddash-0.1.0/fielddash.egg-info/PKG-INFO +267 -0
  34. fielddash-0.1.0/fielddash.egg-info/SOURCES.txt +42 -0
  35. fielddash-0.1.0/fielddash.egg-info/dependency_links.txt +1 -0
  36. fielddash-0.1.0/fielddash.egg-info/entry_points.txt +2 -0
  37. fielddash-0.1.0/fielddash.egg-info/requires.txt +14 -0
  38. fielddash-0.1.0/fielddash.egg-info/top_level.txt +1 -0
  39. fielddash-0.1.0/pyproject.toml +67 -0
  40. fielddash-0.1.0/setup.cfg +4 -0
  41. fielddash-0.1.0/tests/test_app.py +86 -0
  42. fielddash-0.1.0/tests/test_epicollect.py +70 -0
  43. fielddash-0.1.0/tests/test_init.py +89 -0
  44. fielddash-0.1.0/tests/test_schema.py +67 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sergio Costa, André Moura, LambdaGeo / UFMA
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,267 @@
1
+ Metadata-Version: 2.4
2
+ Name: fielddash
3
+ Version: 0.1.0
4
+ Summary: Schema-driven dashboards for field data collection (Epicollect5).
5
+ Author: André Moura Lima
6
+ Author-email: Sergio Souza Costa <sergio.costa@ufma.br>, Ricardo Luvizotto Santos <ricardo.luvizotto@ufma.br>, Marianna Basso Jorge <mb.jorge@ufma.br>
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://github.com/LambdaGeo/fielddash
9
+ Project-URL: Repository, https://github.com/LambdaGeo/fielddash
10
+ Project-URL: Documentation, https://lambdageo.github.io/fielddash/
11
+ Project-URL: Bug Tracker, https://github.com/LambdaGeo/fielddash/issues
12
+ Keywords: epicollect5,fieldwork,data-collection,streamlit,gis,survey,geoprocessing
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: GIS
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: streamlit>=1.39
25
+ Requires-Dist: pandas>=2.0
26
+ Requires-Dist: plotly>=5.0
27
+ Requires-Dist: folium>=0.15
28
+ Requires-Dist: requests>=2.28
29
+ Requires-Dist: python-dotenv>=1.0
30
+ Requires-Dist: PyYAML>=6.0
31
+ Requires-Dist: openpyxl>=3.1
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest>=8.0; extra == "dev"
34
+ Provides-Extra: docs
35
+ Requires-Dist: mkdocs-material<10,>=9.5; extra == "docs"
36
+ Dynamic: license-file
37
+
38
+ # fielddash
39
+
40
+ **Documentation: <https://lambdageo.github.io/fielddash/>**
41
+
42
+ Schema-driven dashboards for **field data collection** (Epicollect5), built directly from the **form schema**. Every question is identified by Epicollect's stable `ref`, and its input type (`radio`, `checkbox`, `integer`, `location`, ...) automatically dictates the filter, chart, and map representation. Modifying, adding, or reordering questions in the form does not break the dashboard, and setting up a new fieldwork survey requires only a YAML configuration file.
43
+
44
+ Ready-to-use pages:
45
+ - **Overview**: summary indicators, map, submissions over time, collectors.
46
+ - **Highlights**: curated charts configured in your YAML.
47
+ - **Questions**: all questions grouped by section, with search, text-field toggle, and dynamic cross-tabulation.
48
+ - **Data**: searchable table with CSV and Excel export.
49
+
50
+ ## Installation
51
+
52
+ Install it in an isolated environment rather than globally.
53
+
54
+ **One environment per project (recommended).** Works everywhere and is required for deployment and custom pages:
55
+
56
+ ```bash
57
+ mkdir my-survey && cd my-survey
58
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
59
+ pip install fielddash
60
+ fielddash init .
61
+ ```
62
+
63
+ **Nothing installed first.** [pipx](https://pipx.pypa.io) or [uv](https://docs.astral.sh/uv/) can run `init` once; then create the environment from the generated `requirements.txt`:
64
+
65
+ ```bash
66
+ pipx run fielddash init my-survey # or: uvx fielddash init my-survey
67
+ cd my-survey
68
+ python -m venv .venv && source .venv/bin/activate
69
+ pip install -r requirements.txt
70
+ ```
71
+
72
+ **Command line only.** If you never edit code in the project, `pipx install fielddash` (or `uv tool install fielddash`) is enough for `init`, `fields` and `run`. The `streamlit_app.py` generated by `--deploy` imports `fielddash`, so it needs the per-project environment above.
73
+
74
+ For development: `git clone https://github.com/LambdaGeo/fielddash && cd fielddash && pip install -e ".[dev]"`.
75
+
76
+ ## Quickstart
77
+
78
+ The example project lives in the repository; it is not part of the PyPI package.
79
+
80
+ ```bash
81
+ git clone https://github.com/LambdaGeo/fielddash && cd fielddash
82
+ python -m venv .venv && source .venv/bin/activate
83
+ pip install -e .
84
+ fielddash run examples/waste/project.yaml
85
+ ```
86
+
87
+ The example uses anonymized household survey data on solid waste management (Itaqui-Bacanga, São Luís – MA) and showcases a custom project page (`Recycling`).
88
+
89
+ ## Setting Up a New Field Project
90
+
91
+ 1. Scaffold the project folder:
92
+ ```bash
93
+ fielddash init my-survey # project.yaml, requirements.txt, .env.example, .gitignore
94
+ fielddash init my-survey --deploy # + streamlit_app.py, secrets example
95
+ fielddash init my-survey --source json # offline project reading data/ (git-ignored)
96
+ ```
97
+ Then `cp .env.example .env` and fill in the project slug and the Epicollect app credentials (generated under *Apps* in your project's administration area):
98
+ ```bash
99
+ PROJECT_MY_SURVEY=project-slug
100
+ MY_SURVEY_CLIENT_ID=...
101
+ MY_SURVEY_CLIENT_SECRET=...
102
+ ```
103
+ *Public projects do not require credentials: simply omit `credentials` in the config.*
104
+
105
+ 2. The generated `project.yaml` starts minimal (or write it by hand):
106
+ ```yaml
107
+ title: "My Field Survey"
108
+ source:
109
+ type: epicollect
110
+ project: ${PROJECT_MY_SURVEY} # slug, or the literal value
111
+ credentials: MY_SURVEY
112
+ ```
113
+
114
+ 3. Inspect the form fields to select aliases, filters, and highlights:
115
+ ```bash
116
+ fielddash fields project.yaml
117
+ ```
118
+
119
+ 4. Complete the configuration (all keys below are optional) and run `fielddash run project.yaml`:
120
+ ```yaml
121
+ subtitle: "Research team, institution..."
122
+ fields: # alias -> ref suffix, column, or question label
123
+ neighborhood: "5401ce"
124
+ waste_dest: "48c35e"
125
+ types: # force type (e.g. treat free text as category)
126
+ neighborhood: category
127
+ ignore: [created_by, "3401cb"] # hide fields (e.g. collector email)
128
+ filters: [created_at, neighborhood]
129
+ highlights:
130
+ - {field: waste_dest, by: neighborhood, title: "Waste destination by neighborhood"}
131
+ sections: # tabs on the Questions page (default: form groups)
132
+ - {title: "Profile", fields: [age, gender]}
133
+ map: {field: location, popup: [neighborhood]}
134
+ extensions: [pages] # .py files or directories with custom pages
135
+ timezone: America/Fortaleza
136
+ cache_minutes: 5
137
+ ```
138
+ *(Portuguese keys from early versions — `titulo`, `fonte`, `campos`, ... — still load but are deprecated and emit a warning.)*
139
+
140
+ Running `fielddash run folder/` will scan all `.yaml` files in the directory and present a project selector in the sidebar.
141
+ To work offline or test without internet access, use `source: {type: json, data: ..., schema: ...}` (as shown in `examples/waste/project.yaml`).
142
+
143
+ ## Using in a Custom Streamlit Script / Deployment
144
+
145
+ The dashboard can also be invoked as a Python function inside your own Streamlit app:
146
+
147
+ ```python
148
+ # streamlit_app.py
149
+ import fielddash
150
+
151
+ fielddash.dashboard("projects/") # or "projects/waste.yaml"
152
+ ```
153
+
154
+ Relative paths are resolved relative to the current directory or the script folder.
155
+ Run with:
156
+ ```bash
157
+ streamlit run streamlit_app.py
158
+ ```
159
+
160
+ ### Streamlit Community Cloud
161
+
162
+ 1. Create the project with the deploy files and try it locally:
163
+ ```bash
164
+ mkdir my-survey && cd my-survey
165
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
166
+ pip install fielddash
167
+ fielddash init . --deploy
168
+ cp .env.example .env # fill in, then edit project.yaml
169
+ streamlit run streamlit_app.py
170
+ ```
171
+ 2. Push the folder to a GitHub repository. `.env` and `.streamlit/secrets.toml` are git-ignored: credentials never go to the repo.
172
+ 3. On [share.streamlit.io](https://share.streamlit.io) choose *Create app*, pick the repository and branch, and set `streamlit_app.py` as the main file.
173
+ 4. Under *Advanced settings → Secrets* (later: *Settings → Secrets*) paste the content of `.streamlit/secrets.toml.example`, filled in:
174
+ ```toml
175
+ FIELD_ACCESS_PIN = "choose-a-code"
176
+ PROJECT_MY_SURVEY = "project-slug"
177
+ MY_SURVEY_CLIENT_ID = "..."
178
+ MY_SURVEY_CLIENT_SECRET = "..."
179
+ ```
180
+ 5. Deploy, then share the app link, or `https://<your-app>.streamlit.app/?token=<code>` to skip the login screen.
181
+
182
+ Notes:
183
+ - Use `source: {type: epicollect}` for the cloud. A `type: json` project reads `data/`, which is git-ignored on purpose (it may hold personal data), so it only works locally.
184
+ - Without `FIELD_ACCESS_PIN` the app is public: anyone with the link sees the data.
185
+
186
+ `fielddash` checks environment variables and `.env` first, then falls back to `st.secrets` (also resolving `${VAR}` references in YAML). The data cache is shared among server sessions, so Epicollect receives at most one request per project every `cache_minutes`.
187
+
188
+ ### Restricting access
189
+
190
+ A deployed Streamlit app is public by default. Add `fielddash.require_access()` before `fielddash.dashboard(...)` (`fielddash init --deploy` already does) and set `FIELD_ACCESS_PIN` in the secrets: visitors then see a login screen, or can use a link ending in `?token=<code>`. Without the variable the call does nothing.
191
+
192
+ ### Self-Hosted Server
193
+ ```bash
194
+ fielddash run projects/ --server.port 8501 --server.headless true
195
+ ```
196
+ (e.g., as a systemd service behind an Nginx reverse proxy with HTTPS).
197
+
198
+ ## Custom Pages
199
+
200
+ A project-specific page is a standard `.py` file listed in `extensions`:
201
+
202
+ ```python
203
+ from fielddash.core.registry import page
204
+ from fielddash.ui.charts import render_field
205
+
206
+
207
+ @page("Recycling", order=40)
208
+ def render(ctx):
209
+ ds = ctx.dataset
210
+ render_field(ds, ctx.df, ds.field("recycle_habit"), by=ds.field("neighborhood"))
211
+ ```
212
+
213
+ - `ctx.df` provides the data with all active sidebar filters applied.
214
+ - `ds.field("alias")` resolves the field (column, type, options) dynamically by its stable schema ref.
215
+ - Additional data sources (e.g. KoboToolbox, CSV) can be registered with `@source("type")`.
216
+
217
+ ## Project Structure
218
+
219
+ ```
220
+ fielddash/
221
+ cli.py CLI commands `fielddash init`, `run` and `fields`
222
+ scaffold.py templates and writer behind `fielddash init`
223
+ access.py `fielddash.require_access()`: optional PIN/token gate
224
+ web.py `fielddash.dashboard()`: config, cache, filters, navigation
225
+ app.py Streamlit entry point used by `fielddash run`
226
+ core/schema.py Epicollect schema parser -> list[Field], column naming, reconciliation
227
+ core/normalize.py raw entries -> typed series (ordered categories, lists, coords, datetimes)
228
+ core/config.py YAML config parser (+ ${VAR} resolution from .env and secrets)
229
+ sources/ data source plugins (@source): epicollect (API, pagination), json
230
+ ui/ dynamic filters, charts, and maps based on field types
231
+ views/ default pages (@page): Overview, Highlights, Questions, Data
232
+ examples/ example projects with anonymized datasets
233
+ tests/ pytest test suite (schema, index shifting, AppTest integration)
234
+ docs/ MkDocs site (`mkdocs serve`); docs/history/ holds the original development plan
235
+ ```
236
+
237
+ ## Running Tests
238
+
239
+ ```bash
240
+ pytest
241
+ ```
242
+
243
+ ## Building the Documentation
244
+
245
+ ```bash
246
+ pip install -e ".[docs]"
247
+ mkdocs serve # http://127.0.0.1:8000
248
+ mkdocs build --strict # what CI runs
249
+ ```
250
+
251
+ ## Releasing (maintainers)
252
+
253
+ 1. Bump the version in `pyproject.toml`, `fielddash/__init__.py` and `CITATION.cff`.
254
+ 2. Publish a GitHub Release: `.github/workflows/publish.yml` builds the package and uploads it to PyPI through Trusted Publishing (no token stored).
255
+
256
+ One-time setup on pypi.org → *Publishing* → *Add a new pending publisher*: project `fielddash`, owner `LambdaGeo`, repository `fielddash`, workflow `publish.yml`. PyPI never lets a published version be overwritten, so check with `python -m build && twine check dist/*` (and optionally TestPyPI) first.
257
+
258
+ ## Authors
259
+
260
+ - **Sergio Costa** ([@LambdaGeo](https://github.com/LambdaGeo)) — Universidade Federal do Maranhão (UFMA)
261
+ - **André Moura Lima** ([@AndreMouraL](https://github.com/AndreMouraL)) — Universidade Federal do Maranhão (UFMA)
262
+ - **Ricardo Luvizotto Santos** — Universidade Federal do Maranhão (UFMA)
263
+ - **Marianna Basso Jorge** — Universidade Federal do Maranhão (UFMA)
264
+
265
+ ## License
266
+
267
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,230 @@
1
+ # fielddash
2
+
3
+ **Documentation: <https://lambdageo.github.io/fielddash/>**
4
+
5
+ Schema-driven dashboards for **field data collection** (Epicollect5), built directly from the **form schema**. Every question is identified by Epicollect's stable `ref`, and its input type (`radio`, `checkbox`, `integer`, `location`, ...) automatically dictates the filter, chart, and map representation. Modifying, adding, or reordering questions in the form does not break the dashboard, and setting up a new fieldwork survey requires only a YAML configuration file.
6
+
7
+ Ready-to-use pages:
8
+ - **Overview**: summary indicators, map, submissions over time, collectors.
9
+ - **Highlights**: curated charts configured in your YAML.
10
+ - **Questions**: all questions grouped by section, with search, text-field toggle, and dynamic cross-tabulation.
11
+ - **Data**: searchable table with CSV and Excel export.
12
+
13
+ ## Installation
14
+
15
+ Install it in an isolated environment rather than globally.
16
+
17
+ **One environment per project (recommended).** Works everywhere and is required for deployment and custom pages:
18
+
19
+ ```bash
20
+ mkdir my-survey && cd my-survey
21
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
22
+ pip install fielddash
23
+ fielddash init .
24
+ ```
25
+
26
+ **Nothing installed first.** [pipx](https://pipx.pypa.io) or [uv](https://docs.astral.sh/uv/) can run `init` once; then create the environment from the generated `requirements.txt`:
27
+
28
+ ```bash
29
+ pipx run fielddash init my-survey # or: uvx fielddash init my-survey
30
+ cd my-survey
31
+ python -m venv .venv && source .venv/bin/activate
32
+ pip install -r requirements.txt
33
+ ```
34
+
35
+ **Command line only.** If you never edit code in the project, `pipx install fielddash` (or `uv tool install fielddash`) is enough for `init`, `fields` and `run`. The `streamlit_app.py` generated by `--deploy` imports `fielddash`, so it needs the per-project environment above.
36
+
37
+ For development: `git clone https://github.com/LambdaGeo/fielddash && cd fielddash && pip install -e ".[dev]"`.
38
+
39
+ ## Quickstart
40
+
41
+ The example project lives in the repository; it is not part of the PyPI package.
42
+
43
+ ```bash
44
+ git clone https://github.com/LambdaGeo/fielddash && cd fielddash
45
+ python -m venv .venv && source .venv/bin/activate
46
+ pip install -e .
47
+ fielddash run examples/waste/project.yaml
48
+ ```
49
+
50
+ The example uses anonymized household survey data on solid waste management (Itaqui-Bacanga, São Luís – MA) and showcases a custom project page (`Recycling`).
51
+
52
+ ## Setting Up a New Field Project
53
+
54
+ 1. Scaffold the project folder:
55
+ ```bash
56
+ fielddash init my-survey # project.yaml, requirements.txt, .env.example, .gitignore
57
+ fielddash init my-survey --deploy # + streamlit_app.py, secrets example
58
+ fielddash init my-survey --source json # offline project reading data/ (git-ignored)
59
+ ```
60
+ Then `cp .env.example .env` and fill in the project slug and the Epicollect app credentials (generated under *Apps* in your project's administration area):
61
+ ```bash
62
+ PROJECT_MY_SURVEY=project-slug
63
+ MY_SURVEY_CLIENT_ID=...
64
+ MY_SURVEY_CLIENT_SECRET=...
65
+ ```
66
+ *Public projects do not require credentials: simply omit `credentials` in the config.*
67
+
68
+ 2. The generated `project.yaml` starts minimal (or write it by hand):
69
+ ```yaml
70
+ title: "My Field Survey"
71
+ source:
72
+ type: epicollect
73
+ project: ${PROJECT_MY_SURVEY} # slug, or the literal value
74
+ credentials: MY_SURVEY
75
+ ```
76
+
77
+ 3. Inspect the form fields to select aliases, filters, and highlights:
78
+ ```bash
79
+ fielddash fields project.yaml
80
+ ```
81
+
82
+ 4. Complete the configuration (all keys below are optional) and run `fielddash run project.yaml`:
83
+ ```yaml
84
+ subtitle: "Research team, institution..."
85
+ fields: # alias -> ref suffix, column, or question label
86
+ neighborhood: "5401ce"
87
+ waste_dest: "48c35e"
88
+ types: # force type (e.g. treat free text as category)
89
+ neighborhood: category
90
+ ignore: [created_by, "3401cb"] # hide fields (e.g. collector email)
91
+ filters: [created_at, neighborhood]
92
+ highlights:
93
+ - {field: waste_dest, by: neighborhood, title: "Waste destination by neighborhood"}
94
+ sections: # tabs on the Questions page (default: form groups)
95
+ - {title: "Profile", fields: [age, gender]}
96
+ map: {field: location, popup: [neighborhood]}
97
+ extensions: [pages] # .py files or directories with custom pages
98
+ timezone: America/Fortaleza
99
+ cache_minutes: 5
100
+ ```
101
+ *(Portuguese keys from early versions — `titulo`, `fonte`, `campos`, ... — still load but are deprecated and emit a warning.)*
102
+
103
+ Running `fielddash run folder/` will scan all `.yaml` files in the directory and present a project selector in the sidebar.
104
+ To work offline or test without internet access, use `source: {type: json, data: ..., schema: ...}` (as shown in `examples/waste/project.yaml`).
105
+
106
+ ## Using in a Custom Streamlit Script / Deployment
107
+
108
+ The dashboard can also be invoked as a Python function inside your own Streamlit app:
109
+
110
+ ```python
111
+ # streamlit_app.py
112
+ import fielddash
113
+
114
+ fielddash.dashboard("projects/") # or "projects/waste.yaml"
115
+ ```
116
+
117
+ Relative paths are resolved relative to the current directory or the script folder.
118
+ Run with:
119
+ ```bash
120
+ streamlit run streamlit_app.py
121
+ ```
122
+
123
+ ### Streamlit Community Cloud
124
+
125
+ 1. Create the project with the deploy files and try it locally:
126
+ ```bash
127
+ mkdir my-survey && cd my-survey
128
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
129
+ pip install fielddash
130
+ fielddash init . --deploy
131
+ cp .env.example .env # fill in, then edit project.yaml
132
+ streamlit run streamlit_app.py
133
+ ```
134
+ 2. Push the folder to a GitHub repository. `.env` and `.streamlit/secrets.toml` are git-ignored: credentials never go to the repo.
135
+ 3. On [share.streamlit.io](https://share.streamlit.io) choose *Create app*, pick the repository and branch, and set `streamlit_app.py` as the main file.
136
+ 4. Under *Advanced settings → Secrets* (later: *Settings → Secrets*) paste the content of `.streamlit/secrets.toml.example`, filled in:
137
+ ```toml
138
+ FIELD_ACCESS_PIN = "choose-a-code"
139
+ PROJECT_MY_SURVEY = "project-slug"
140
+ MY_SURVEY_CLIENT_ID = "..."
141
+ MY_SURVEY_CLIENT_SECRET = "..."
142
+ ```
143
+ 5. Deploy, then share the app link, or `https://<your-app>.streamlit.app/?token=<code>` to skip the login screen.
144
+
145
+ Notes:
146
+ - Use `source: {type: epicollect}` for the cloud. A `type: json` project reads `data/`, which is git-ignored on purpose (it may hold personal data), so it only works locally.
147
+ - Without `FIELD_ACCESS_PIN` the app is public: anyone with the link sees the data.
148
+
149
+ `fielddash` checks environment variables and `.env` first, then falls back to `st.secrets` (also resolving `${VAR}` references in YAML). The data cache is shared among server sessions, so Epicollect receives at most one request per project every `cache_minutes`.
150
+
151
+ ### Restricting access
152
+
153
+ A deployed Streamlit app is public by default. Add `fielddash.require_access()` before `fielddash.dashboard(...)` (`fielddash init --deploy` already does) and set `FIELD_ACCESS_PIN` in the secrets: visitors then see a login screen, or can use a link ending in `?token=<code>`. Without the variable the call does nothing.
154
+
155
+ ### Self-Hosted Server
156
+ ```bash
157
+ fielddash run projects/ --server.port 8501 --server.headless true
158
+ ```
159
+ (e.g., as a systemd service behind an Nginx reverse proxy with HTTPS).
160
+
161
+ ## Custom Pages
162
+
163
+ A project-specific page is a standard `.py` file listed in `extensions`:
164
+
165
+ ```python
166
+ from fielddash.core.registry import page
167
+ from fielddash.ui.charts import render_field
168
+
169
+
170
+ @page("Recycling", order=40)
171
+ def render(ctx):
172
+ ds = ctx.dataset
173
+ render_field(ds, ctx.df, ds.field("recycle_habit"), by=ds.field("neighborhood"))
174
+ ```
175
+
176
+ - `ctx.df` provides the data with all active sidebar filters applied.
177
+ - `ds.field("alias")` resolves the field (column, type, options) dynamically by its stable schema ref.
178
+ - Additional data sources (e.g. KoboToolbox, CSV) can be registered with `@source("type")`.
179
+
180
+ ## Project Structure
181
+
182
+ ```
183
+ fielddash/
184
+ cli.py CLI commands `fielddash init`, `run` and `fields`
185
+ scaffold.py templates and writer behind `fielddash init`
186
+ access.py `fielddash.require_access()`: optional PIN/token gate
187
+ web.py `fielddash.dashboard()`: config, cache, filters, navigation
188
+ app.py Streamlit entry point used by `fielddash run`
189
+ core/schema.py Epicollect schema parser -> list[Field], column naming, reconciliation
190
+ core/normalize.py raw entries -> typed series (ordered categories, lists, coords, datetimes)
191
+ core/config.py YAML config parser (+ ${VAR} resolution from .env and secrets)
192
+ sources/ data source plugins (@source): epicollect (API, pagination), json
193
+ ui/ dynamic filters, charts, and maps based on field types
194
+ views/ default pages (@page): Overview, Highlights, Questions, Data
195
+ examples/ example projects with anonymized datasets
196
+ tests/ pytest test suite (schema, index shifting, AppTest integration)
197
+ docs/ MkDocs site (`mkdocs serve`); docs/history/ holds the original development plan
198
+ ```
199
+
200
+ ## Running Tests
201
+
202
+ ```bash
203
+ pytest
204
+ ```
205
+
206
+ ## Building the Documentation
207
+
208
+ ```bash
209
+ pip install -e ".[docs]"
210
+ mkdocs serve # http://127.0.0.1:8000
211
+ mkdocs build --strict # what CI runs
212
+ ```
213
+
214
+ ## Releasing (maintainers)
215
+
216
+ 1. Bump the version in `pyproject.toml`, `fielddash/__init__.py` and `CITATION.cff`.
217
+ 2. Publish a GitHub Release: `.github/workflows/publish.yml` builds the package and uploads it to PyPI through Trusted Publishing (no token stored).
218
+
219
+ One-time setup on pypi.org → *Publishing* → *Add a new pending publisher*: project `fielddash`, owner `LambdaGeo`, repository `fielddash`, workflow `publish.yml`. PyPI never lets a published version be overwritten, so check with `python -m build && twine check dist/*` (and optionally TestPyPI) first.
220
+
221
+ ## Authors
222
+
223
+ - **Sergio Costa** ([@LambdaGeo](https://github.com/LambdaGeo)) — Universidade Federal do Maranhão (UFMA)
224
+ - **André Moura Lima** ([@AndreMouraL](https://github.com/AndreMouraL)) — Universidade Federal do Maranhão (UFMA)
225
+ - **Ricardo Luvizotto Santos** — Universidade Federal do Maranhão (UFMA)
226
+ - **Marianna Basso Jorge** — Universidade Federal do Maranhão (UFMA)
227
+
228
+ ## License
229
+
230
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,17 @@
1
+ """fielddash: schema-driven dashboards for field data collection (Epicollect5)."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+
6
+ def dashboard(target=".", *, configure_page: bool = True):
7
+ """Renders the dashboard for a project (.yaml) or directory of projects in a Streamlit script."""
8
+ from fielddash.web import dashboard as _dashboard # imports streamlit only when called
9
+
10
+ return _dashboard(target, configure_page=configure_page)
11
+
12
+
13
+ def require_access():
14
+ """Optional PIN gate: stops the Streamlit script on a login screen unless FIELD_ACCESS_PIN matches."""
15
+ from fielddash.access import require_access as _require_access # imports streamlit only when called
16
+
17
+ return _require_access()
@@ -0,0 +1,73 @@
1
+ """Optional PIN gate for dashboards shared with a field team (Streamlit).
2
+
3
+ import fielddash
4
+ fielddash.require_access() # no-op unless FIELD_ACCESS_PIN is set (env, .env or Streamlit secrets)
5
+ fielddash.dashboard("project.yaml", configure_page=False)
6
+
7
+ Team members enter the code on a login screen or open a link ending in `?token=<code>`.
8
+ """
9
+ import hmac
10
+
11
+ import streamlit as st
12
+
13
+ from fielddash.core.env import getenv
14
+
15
+ PIN_VARS = ("FIELD_ACCESS_PIN", "FIELD_ACCESS_TOKEN", "ACCESS_PIN")
16
+ URL_PARAMS = ("token", "pin", "key")
17
+
18
+
19
+ def required_pin():
20
+ """The configured access code, or None when the dashboard is open to everyone."""
21
+ for name in PIN_VARS:
22
+ value = getenv(name)
23
+ if value:
24
+ return str(value).strip()
25
+ return None
26
+
27
+
28
+ def _matches(given, expected) -> bool:
29
+ return hmac.compare_digest(str(given).strip().encode(), str(expected).encode())
30
+
31
+
32
+ def _url_token():
33
+ for name in URL_PARAMS:
34
+ value = st.query_params.get(name)
35
+ if value:
36
+ return value
37
+ return None
38
+
39
+
40
+ def _sidebar_logout():
41
+ st.sidebar.caption("🟢 Access granted (field team)")
42
+ if st.sidebar.button("End session", key="logout_btn"):
43
+ st.session_state["authenticated"] = False
44
+ for name in URL_PARAMS: # otherwise the link would log the user straight back in
45
+ if name in st.query_params:
46
+ del st.query_params[name]
47
+ st.rerun()
48
+
49
+
50
+ def require_access() -> None:
51
+ """Stops the script on a login screen unless the visitor has the access code."""
52
+ pin = required_pin()
53
+ if not pin:
54
+ return
55
+
56
+ token = _url_token()
57
+ if st.session_state.get("authenticated") or (token and _matches(token, pin)):
58
+ st.session_state["authenticated"] = True
59
+ _sidebar_logout()
60
+ return
61
+
62
+ _, center, _ = st.columns([1, 2, 1])
63
+ with center:
64
+ st.markdown("## 🔐 Restricted area — field team")
65
+ st.info("This dashboard contains field-work data. Enter the access code provided by the research coordination.")
66
+ with st.form("login_form"):
67
+ entered = st.text_input("Access code / PIN:", type="password")
68
+ if st.form_submit_button("Enter dashboard"):
69
+ if entered and _matches(entered, pin):
70
+ st.session_state["authenticated"] = True
71
+ st.rerun()
72
+ st.error("Wrong access code. Check with the responsible team.")
73
+ st.stop()
@@ -0,0 +1,24 @@
1
+ """Streamlit script used by `fielddash run`.
2
+
3
+ fielddash run project.yaml # single project
4
+ fielddash run dir/ # choose between .yaml projects in sidebar
5
+
6
+ To deploy (e.g. Streamlit Community Cloud), use `fielddash.dashboard(...)`
7
+ in your project's script.
8
+ """
9
+ import argparse
10
+ import os
11
+
12
+ from fielddash.web import dashboard
13
+
14
+ ENV_CONFIG = "FIELDDASH_CONFIG"
15
+
16
+
17
+ def _config_arg() -> str:
18
+ parser = argparse.ArgumentParser()
19
+ parser.add_argument("--config")
20
+ args, _ = parser.parse_known_args()
21
+ return args.config or os.environ.get(ENV_CONFIG) or "."
22
+
23
+
24
+ dashboard(_config_arg())