fielddash 0.1.0__py3-none-any.whl

fielddash/__init__.py

@@ -0,0 +1,17 @@
+"""fielddash: schema-driven dashboards for field data collection (Epicollect5)."""
+
+__version__ = "0.1.0"
+
+
+def dashboard(target=".", *, configure_page: bool = True):
+    """Renders the dashboard for a project (.yaml) or directory of projects in a Streamlit script."""
+    from fielddash.web import dashboard as _dashboard  # imports streamlit only when called
+
+    return _dashboard(target, configure_page=configure_page)
+
+
+def require_access():
+    """Optional PIN gate: stops the Streamlit script on a login screen unless FIELD_ACCESS_PIN matches."""
+    from fielddash.access import require_access as _require_access  # imports streamlit only when called
+
+    return _require_access()

fielddash/access.py

@@ -0,0 +1,73 @@
+"""Optional PIN gate for dashboards shared with a field team (Streamlit).
+
+    import fielddash
+    fielddash.require_access()      # no-op unless FIELD_ACCESS_PIN is set (env, .env or Streamlit secrets)
+    fielddash.dashboard("project.yaml", configure_page=False)
+
+Team members enter the code on a login screen or open a link ending in `?token=<code>`.
+"""
+import hmac
+
+import streamlit as st
+
+from fielddash.core.env import getenv
+
+PIN_VARS = ("FIELD_ACCESS_PIN", "FIELD_ACCESS_TOKEN", "ACCESS_PIN")
+URL_PARAMS = ("token", "pin", "key")
+
+
+def required_pin():
+    """The configured access code, or None when the dashboard is open to everyone."""
+    for name in PIN_VARS:
+        value = getenv(name)
+        if value:
+            return str(value).strip()
+    return None
+
+
+def _matches(given, expected) -> bool:
+    return hmac.compare_digest(str(given).strip().encode(), str(expected).encode())
+
+
+def _url_token():
+    for name in URL_PARAMS:
+        value = st.query_params.get(name)
+        if value:
+            return value
+    return None
+
+
+def _sidebar_logout():
+    st.sidebar.caption("🟢 Access granted (field team)")
+    if st.sidebar.button("End session", key="logout_btn"):
+        st.session_state["authenticated"] = False
+        for name in URL_PARAMS:  # otherwise the link would log the user straight back in
+            if name in st.query_params:
+                del st.query_params[name]
+        st.rerun()
+
+
+def require_access() -> None:
+    """Stops the script on a login screen unless the visitor has the access code."""
+    pin = required_pin()
+    if not pin:
+        return
+
+    token = _url_token()
+    if st.session_state.get("authenticated") or (token and _matches(token, pin)):
+        st.session_state["authenticated"] = True
+        _sidebar_logout()
+        return
+
+    _, center, _ = st.columns([1, 2, 1])
+    with center:
+        st.markdown("## 🔐 Restricted area — field team")
+        st.info("This dashboard contains field-work data. Enter the access code provided by the research coordination.")
+        with st.form("login_form"):
+            entered = st.text_input("Access code / PIN:", type="password")
+            if st.form_submit_button("Enter dashboard"):
+                if entered and _matches(entered, pin):
+                    st.session_state["authenticated"] = True
+                    st.rerun()
+                st.error("Wrong access code. Check with the responsible team.")
+    st.stop()

fielddash/app.py

@@ -0,0 +1,24 @@
+"""Streamlit script used by `fielddash run`.
+
+    fielddash run project.yaml        # single project
+    fielddash run dir/                # choose between .yaml projects in sidebar
+
+To deploy (e.g. Streamlit Community Cloud), use `fielddash.dashboard(...)`
+in your project's script.
+"""
+import argparse
+import os
+
+from fielddash.web import dashboard
+
+ENV_CONFIG = "FIELDDASH_CONFIG"
+
+
+def _config_arg() -> str:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config")
+    args, _ = parser.parse_known_args()
+    return args.config or os.environ.get(ENV_CONFIG) or "."
+
+
+dashboard(_config_arg())

fielddash/cli.py

@@ -0,0 +1,97 @@
+"""Command line interface for fielddash.
+
+    fielddash run project.yaml [streamlit options, e.g. --server.port 8600]
+    fielddash run dir/                   # select project in sidebar
+    fielddash fields project.yaml        # inspect form fields to write config
+    fielddash init my-survey             # scaffold a new project folder
+"""
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+from fielddash.core.config import load_config
+from fielddash.core.loader import bootstrap, load_dataset
+from fielddash.scaffold import init_project
+
+APP = Path(__file__).resolve().parent / "app.py"
+
+
+def run(target: str, streamlit_args: list) -> int:
+    path = Path(target).resolve()
+    if not path.exists():
+        sys.exit(f"Not found: {path}")
+    command = [sys.executable, "-m", "streamlit", "run", str(APP), *streamlit_args, "--", "--config", str(path)]
+    return subprocess.call(command)
+
+
+def fields(target: str) -> int:
+    config = load_config(target)
+    bootstrap(config)
+    dataset = load_dataset(config)
+    print(f"{dataset.title}: {len(dataset.df)} responses\n")
+    print(f"{'ref (suffix)':<14}{'alias':<20}{'type':<12}{'column':<22}question")
+    for f in dataset.fields:
+        ref = f.ref if f.system else f.ref[-6:]
+        print(f"{ref:<14}{f.alias or '':<20}{f.type:<12}{f.column:<22}{f.label[:70]}")
+    for warning in dataset.warnings:
+        print(f"⚠ {warning}")
+    return 0
+
+
+def init(args) -> int:
+    directory = Path(args.directory)
+    created, skipped = init_project(directory, args.name, args.source, args.deploy, args.force)
+    print(f"{directory.resolve()}")
+    for relative in created:
+        print(f"  + {relative}")
+    for relative in skipped:
+        print(f"  = {relative} (already exists, use --force to overwrite)")
+    steps = []
+    if directory != Path("."):
+        steps.append(f"cd {directory}")
+    steps.append("python -m venv .venv && source .venv/bin/activate   # Windows: .venv\\Scripts\\activate (skip if you already use an environment)")
+    steps.append("pip install -r requirements.txt")
+    if args.source == "epicollect":
+        steps.append("cp .env.example .env   # fill in the project slug and credentials")
+    else:
+        steps.append("save the form schema and entries to data/form.json and data/entries.json")
+    steps += ["fielddash fields project.yaml   # list fields, then edit project.yaml", "fielddash run project.yaml"]
+    print("\nNext steps:")
+    for number, step in enumerate(steps, 1):
+        print(f"  {number}. {step}")
+    return 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="fielddash", description="Schema-driven dashboards for field data collection.")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    run_parser = commands.add_parser("run", help="launch dashboard for a project (.yaml) or project directory")
+    run_parser.add_argument("project", nargs="?", default=".")
+
+    fields_parser = commands.add_parser("fields", help="list form fields for a project")
+    fields_parser.add_argument("project")
+
+    init_parser = commands.add_parser("init", help="create a new project folder (project.yaml, .env.example, ...)")
+    init_parser.add_argument("directory", nargs="?", default=".", help="folder to create/use (default: current)")
+    init_parser.add_argument("--name", help="project name, used for title and variable names (default: folder name)")
+    init_parser.add_argument("--source", choices=["epicollect", "json"], default="epicollect")
+    init_parser.add_argument("--deploy", action="store_true", help="also create streamlit_app.py, requirements.txt and secrets example")
+    init_parser.add_argument("--force", action="store_true", help="overwrite existing files")
+
+    args, extra = parser.parse_known_args(argv)
+    if args.command == "run":
+        return run(args.project, extra)
+    if extra:
+        parser.error(f"unrecognized arguments: {' '.join(extra)}")
+    if args.command == "init":
+        return init(args)
+    try:
+        return fields(args.project)
+    except (OSError, KeyError, ValueError, RuntimeError) as error:
+        sys.exit(f"fielddash: {error}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fielddash/core/config.py

@@ -0,0 +1,115 @@
+import re
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+from dotenv import load_dotenv
+
+from fielddash.core.env import getenv
+
+_ENV = re.compile(r"\$\{([A-Za-z0-9_]+)\}")
+
+
+def _expand(value):
+    if isinstance(value, str):
+        def replace(match):
+            name = match.group(1)
+            value = getenv(name)
+            if value is None:
+                raise KeyError(f"Variable not defined in .env or Streamlit secrets: {name}")
+            return value
+        return _ENV.sub(replace, value)
+    if isinstance(value, list):
+        return [_expand(v) for v in value]
+    if isinstance(value, dict):
+        return {k: _expand(v) for k, v in value.items()}
+    return value
+
+
+# Portuguese keys from earlier versions: still accepted (with a FutureWarning), use English keys.
+LEGACY_KEYS = {
+    "titulo": "title",
+    "subtitulo": "subtitle",
+    "fonte": "source",
+    "campos": "fields",
+    "ignorar": "ignore",
+    "tipos": "types",
+    "filtros": "filters",
+    "destaques": "highlights",
+    "secoes": "sections",
+    "mapa": "map",
+    "extensoes": "extensions",
+    "fuso": "timezone",
+    "cache_minutos": "cache_minutes",
+}
+
+LEGACY_SOURCE_KEYS = {
+    "tipo": "type",
+    "projeto": "project",
+    "credenciais": "credentials",
+    "dados": "data",
+}
+
+
+@dataclass
+class Config:
+    path: Path
+    title: str
+    source: dict
+    subtitle: str = ""
+    fields: dict = field(default_factory=dict)       # alias -> ref / column / question
+    ignore: list = field(default_factory=list)
+    types: dict = field(default_factory=dict)        # field -> forced type (e.g. neighborhood: category)
+    filters: list = field(default_factory=list)
+    highlights: list = field(default_factory=list)
+    sections: list = field(default_factory=list)
+    map: dict = field(default_factory=dict)
+    extensions: list = field(default_factory=list)   # .py files or directories with project pages/sources
+    timezone: str = "America/Fortaleza"
+    cache_minutes: int = 5
+
+    @property
+    def base_dir(self) -> Path:
+        return self.path.parent
+
+    def resolve(self, relative: str) -> Path:
+        return (self.base_dir / relative).resolve()
+
+
+def load_config(path) -> Config:
+    path = Path(path).resolve()
+    # .env in current directory and in project directory
+    load_dotenv(Path.cwd() / ".env")
+    load_dotenv(path.parent / ".env")
+    with open(path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f) or {}
+
+    normalized, legacy = {}, []
+    for key, value in raw.items():
+        canonical = LEGACY_KEYS.get(key, key)
+        if canonical != key:
+            legacy.append(f"{key} -> {canonical}")
+        normalized[canonical] = value
+
+    source = {}
+    for key, value in _expand(normalized.get("source") or {}).items():
+        canonical = LEGACY_SOURCE_KEYS.get(key, key)
+        if canonical != key:
+            legacy.append(f"source.{key} -> source.{canonical}")
+        source[canonical] = value
+    normalized["source"] = source
+
+    if legacy:
+        warnings.warn(f"{path.name}: Portuguese config keys are deprecated, rename: {', '.join(legacy)}", FutureWarning, stacklevel=2)
+
+    if "type" not in source:
+        raise ValueError(f"{path.name}: 'source.type' is required")
+
+    known = Config.__dataclass_fields__.keys() - {"path"}
+    unknown = set(normalized) - known
+    if unknown:
+        raise ValueError(f"{path.name}: unknown keys: {', '.join(sorted(unknown))}")
+
+    normalized.setdefault("title", path.stem)
+    return Config(path=path, **{k: v for k, v in normalized.items() if v is not None})

fielddash/core/context.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+
+import pandas as pd
+
+from fielddash.core.config import Config
+from fielddash.core.model import Dataset
+
+
+@dataclass
+class PageContext:
+    config: Config
+    dataset: Dataset      # complete data + schema
+    df: pd.DataFrame      # data after sidebar filters
+    filters: list         # active filter labels

fielddash/core/env.py

@@ -0,0 +1,20 @@
+"""Configuration variables: environment (.env) or Streamlit Secrets."""
+import os
+
+
+def getenv(name: str) -> str | None:
+    """Retrieves value of `name` from environment (including .env) or `st.secrets`.
+
+    `st.secrets` comes from `.streamlit/secrets.toml` or Streamlit Community Cloud
+    dashboard Secrets, where there is no local .env file.
+    """
+    value = os.environ.get(name)
+    if value:
+        return value
+    try:
+        import streamlit as st
+
+        value = st.secrets.get(name)
+    except Exception:  # no secrets configured or outside of Streamlit runtime
+        return None
+    return str(value) if value is not None else None

fielddash/core/loader.py

@@ -0,0 +1,47 @@
+import importlib
+import importlib.util
+import pkgutil
+import sys
+from pathlib import Path
+
+from fielddash.core.config import Config
+from fielddash.core.model import Dataset
+from fielddash.core.registry import SOURCE_REGISTRY
+
+_loaded_extensions = set()
+
+
+def bootstrap(config: Config | None = None):
+    """Registers sources and views from the package, and project extensions if config is given."""
+    for package in ("fielddash.sources", "fielddash.views"):
+        module = importlib.import_module(package)
+        for info in pkgutil.iter_modules(module.__path__):
+            importlib.import_module(f"{package}.{info.name}")
+
+    if config is not None:
+        for entry in config.extensions:
+            path = config.resolve(entry)
+            for file in (sorted(path.glob("*.py")) if path.is_dir() else [path]):
+                _load_extension(file)
+
+
+def _load_extension(file: Path):
+    """Imports a project module (e.g. project-specific @page) once."""
+    file = file.resolve()
+    if file in _loaded_extensions or file.name.startswith("_"):
+        return
+    if not file.exists():
+        raise FileNotFoundError(f"Extension not found: {file}")
+    name = f"fielddash_ext.{file.stem}"
+    spec = importlib.util.spec_from_file_location(name, file)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    _loaded_extensions.add(file)
+
+
+def load_dataset(config: Config) -> Dataset:
+    kind = config.source.get("type")
+    if kind not in SOURCE_REGISTRY:
+        raise ValueError(f"Unknown data source: {kind!r} (available: {', '.join(SOURCE_REGISTRY)})")
+    return SOURCE_REGISTRY[kind](config).load()

fielddash/core/model.py

@@ -0,0 +1,84 @@
+from dataclasses import dataclass, field as dc_field
+
+import pandas as pd
+
+# Grouping of Epicollect5 types by how they are handled in the dashboard
+CATEGORICAL = {"radio", "dropdown", "searchsingle", "category"}
+MULTI = {"checkbox", "searchmultiple"}
+NUMERIC = {"integer", "decimal"}
+DATE = {"date", "datetime"}
+LOCATION = {"location"}
+TEXT = {"text", "textarea", "phone", "barcode", "time"}
+MEDIA = {"photo", "audio", "video"}
+
+
+@dataclass
+class Field:
+    ref: str
+    column: str
+    label: str
+    type: str
+    options: list = dc_field(default_factory=list)
+    group: str | None = None
+    alias: str | None = None
+    system: bool = False  # Epicollect system fields (created_at, created_by...)
+    question: str = ""    # original question text (used to match columns)
+
+    @property
+    def kind(self) -> str:
+        """Category used by the UI: categorical, multi, numeric, date, location, text, media or other."""
+        for kind, types in (
+            ("categorical", CATEGORICAL),
+            ("multi", MULTI),
+            ("numeric", NUMERIC),
+            ("date", DATE),
+            ("location", LOCATION),
+            ("text", TEXT),
+            ("media", MEDIA),
+        ):
+            if self.type in types:
+                return kind
+        return "other"
+
+    @property
+    def lat(self) -> str:
+        return f"{self.column}__lat"
+
+    @property
+    def lon(self) -> str:
+        return f"{self.column}__lon"
+
+
+@dataclass
+class Dataset:
+    df: pd.DataFrame
+    fields: list
+    title: str = ""
+    warnings: list = dc_field(default_factory=list)
+
+    def field(self, key: str) -> Field:
+        """Looks up a field by alias, ref (full or suffix), column, or question text."""
+        found = self.find(key)
+        if found is None:
+            raise KeyError(f"Field not found: {key!r}")
+        return found
+
+    def find(self, key: str) -> Field | None:
+        key = str(key).strip()
+        lowered = key.lower()
+        matchers = (
+            lambda f: f.alias == key,
+            lambda f: f.ref == key or f.column == key,
+            lambda f: f.label.strip().lower() == lowered,
+            lambda f: len(key) >= 6 and f.ref.endswith(key),
+        )
+        for match in matchers:
+            hits = [f for f in self.fields if match(f)]
+            if len(hits) == 1:
+                return hits[0]
+            if len(hits) > 1:
+                raise KeyError(f"Ambiguous field: {key!r} ({', '.join(f.column for f in hits)})")
+        return None
+
+    def of_kind(self, *kinds) -> list:
+        return [f for f in self.fields if f.kind in kinds]

fielddash/core/normalize.py

@@ -0,0 +1,104 @@
+"""Converts raw Epicollect responses into types directly used by the UI."""
+import numpy as np
+import pandas as pd
+
+
+def _empty_to_nan(value):
+    if isinstance(value, str) and value.strip() == "":
+        return np.nan
+    return value
+
+
+def _as_list(value):
+    if isinstance(value, list):
+        return [str(v) for v in value if str(v).strip()]
+    if value is None or (isinstance(value, float) and np.isnan(value)):
+        return []
+    text = str(value).strip()
+    # Epicollect CSV brings multiple-choice answers separated by comma
+    return [v.strip() for v in text.split(",") if v.strip()] if text else []
+
+
+def _first(value):
+    if isinstance(value, list):
+        return value[0] if value else np.nan
+    return value
+
+
+def _coord(value, key):
+    if isinstance(value, dict):
+        return value.get(key)
+    return np.nan
+
+
+def _unify_case(series):
+    """Typed text responses ("foo bar", "Foo  Bar", "FooBar") become the most frequent casing."""
+    def key(value):
+        return "".join(value.lower().split())
+
+    counts = series.dropna().value_counts()
+    canonical = {}
+    for value in counts.index:
+        canonical.setdefault(key(value), value)
+    return series.map(lambda v: v if pd.isna(v) else canonical[key(v)])
+
+
+def _categorical(series, options):
+    observed = [v for v in series.dropna().unique() if v not in options]
+    return pd.Categorical(series, categories=list(options) + sorted(map(str, observed)))
+
+
+def normalize(df: pd.DataFrame, fields: list, timezone: str | None = None) -> pd.DataFrame:
+    df = df.copy()
+    for f in fields:
+        if f.column not in df.columns:
+            continue
+        col = df[f.column]
+        kind = f.kind
+
+        if kind == "location":
+            df[f.lat] = pd.to_numeric(col.map(lambda v: _coord(v, "latitude")), errors="coerce")
+            df[f.lon] = pd.to_numeric(col.map(lambda v: _coord(v, "longitude")), errors="coerce")
+            continue
+
+        if kind == "multi":
+            df[f.column] = col.map(_as_list)
+            continue
+
+        col = col.map(_empty_to_nan)
+
+        if kind == "categorical":
+            col = col.map(_first).map(lambda v: v if pd.isna(v) else " ".join(str(v).split()))
+            if not f.options:
+                col = _unify_case(col)
+            options = f.options or sorted(col.dropna().unique(), key=str.lower)
+            df[f.column] = _categorical(col, options)
+        elif kind == "numeric":
+            df[f.column] = pd.to_numeric(col, errors="coerce")
+        elif kind == "date":
+            parsed = pd.to_datetime(col, errors="coerce", utc=True, format="mixed")
+            if timezone:
+                parsed = parsed.dt.tz_convert(timezone)
+            df[f.column] = parsed.dt.tz_localize(None)
+        else:
+            df[f.column] = col
+    return df
+
+
+def flat_for_export(df: pd.DataFrame, fields: list) -> pd.DataFrame:
+    """Flat table (joined lists, location split into lat/lon) with question labels as column headers."""
+    out = pd.DataFrame(index=df.index)
+    for f in fields:
+        if f.kind == "location":
+            if f.lat in df.columns:
+                out[f"{f.label} (lat)"] = df[f.lat]
+                out[f"{f.label} (lon)"] = df[f.lon]
+        elif f.column in df.columns:
+            col = df[f.column]
+            if f.kind == "multi":
+                col = col.map(lambda v: "; ".join(v) if isinstance(v, list) else v)
+            elif f.kind == "categorical":
+                col = col.astype(object)
+            name = f.label if f.label not in out.columns else f"{f.label} [{f.column}]"
+            out[name] = col
+    return out

fielddash/core/registry.py

@@ -0,0 +1,27 @@
+PAGE_REGISTRY = {}
+SOURCE_REGISTRY = {}
+
+
+def page(name: str, order: int = 100, available=None):
+    """Registers a dashboard page. The decorated function receives a PageContext.
+
+    `available(ctx) -> bool` hides the page when it does not apply to the project.
+    """
+    def decorator(func):
+        func.page_order = order
+        func.page_available = available or (lambda ctx: True)
+        PAGE_REGISTRY[name] = func
+        return func
+    return decorator
+
+
+def source(kind: str):
+    """Registers a data source class by the value of `source.type` in config."""
+    def decorator(cls):
+        SOURCE_REGISTRY[kind] = cls
+        return cls
+    return decorator
+
+
+def ordered_pages() -> dict:
+    return dict(sorted(PAGE_REGISTRY.items(), key=lambda item: item[1].page_order))