suntdataset 0.1.0__tar.gz

suntdataset-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: suntdataset
+Version: 0.1.0
+Summary: A Dataset for the Salvador Unified Network for Transport (SUNT).
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: tqdm
+Provides-Extra: vis
+Requires-Dist: pydeck; extra == "vis"
+Requires-Dist: folium; extra == "vis"
+Requires-Dist: geopandas; extra == "vis"
+Provides-Extra: dashboard
+Requires-Dist: streamlit; extra == "dashboard"
+Provides-Extra: all
+Requires-Dist: pydeck; extra == "all"
+Requires-Dist: folium; extra == "all"
+Requires-Dist: geopandas; extra == "all"
+Requires-Dist: streamlit; extra == "all"
+Requires-Dist: networkx; extra == "all"

suntdataset-0.1.0/README.md

@@ -0,0 +1,154 @@
+# SUNT-Lib: Salvador Unified Network for Transport Library
+
+**SUNT-Lib** is a Python library designed to streamline the access, processing, and visualization of the **SUNT (Salvador Unified Network for Transport)** dataset. It is built for researchers, urban planners, and data scientists working with multimodal public transport data from Salvador, Brazil.
+
+With just a few lines of code, you can load large blocks of historical data (AVL, AFC, GTFS), filter by day types (workdays/weekends), and generate high-fidelity 3D visualizations or academic-ready tables.
+
+## 🚀 Features
+
+* **Smart Data Loading**: Direct integration with the SUNT GitHub repository.
+* **Temporal Batching**: Load data for $X$ days or months without manual URL formatting.
+* **Smart Filtering**: Built-in logic to select only Workdays, Saturdays, or Sundays.
+
+* **Advanced Visualization**:
+* **3D Hexagon Maps**: View passenger density and loading with `PyDeck`.
+* **Interactive Heatmaps**: Traditional spatial hotspots with `Folium`.
+* **Graph Analysis**: Convert OD matrices into `NetworkX` directed graphs.
+
+
+
+---
+
+## 🛠 Installation
+
+You can install the library locally for development:
+
+```bash
+git clone https://github.com/your-username/sunt-lib.git
+cd sunt-lib
+pip install -e .
+
+```
+
+Or via pip (once published):
+
+```bash
+# Basic data loading only
+pip install sunt-lib
+
+# With 3D Mapping support
+pip install "sunt-lib[vis]"
+
+# Full suite (Dashboards, Graphs, Maps)
+pip install "sunt-lib[all]"
+
+```
+
+---
+
+## 📖 Quick Start
+
+### 1. Loading Data by Blocks
+
+You don't need to provide the exact start date if you want to use the dataset default (**2024-03-01**).
+
+```python
+from sunt_lib.loader import SUNTLoader
+
+loader = SUNTLoader()
+
+# Load AVL data for all SUNDAYS in a 4-week period
+df_sundays = loader.load_batch(
+    dataset_type='avl-lines', 
+    periods=4, 
+    freq='W', 
+    day_type='sundays'
+)
+
+# Load Boarding/Alighting flow for 15 WORKDAYS
+df_work = loader.load_batch(
+    dataset_type='boarding-alighting',
+    periods=15,
+    day_type='workdays'
+)
+
+```
+
+### 2. Geographic Visualization
+
+Visualize passenger loading or origin-destination flows in 3D.
+
+```python
+from sunt_lib.visualizer import SUNTVisualizer
+
+viz = SUNTVisualizer(df_work)
+
+# Generate a 3D Hexagon map (PyDeck)
+deck_map = viz.plot_3d_flow(weight_col='n-boarding')
+deck_map.to_html("salvador_density.html")
+
+```
+
+### 3. Academic Export
+
+Generate a summary table for your paper or thesis.
+
+```python
+from sunt_lib.utils import export_academic_table
+
+# Creates a .tex file with mean boarding and speed stats
+export_academic_table(df_work, name="results_table.tex")
+
+```
+
+### 4. OD Graph (NetworkX)
+
+Generate and export a directed OD graph using the SUNT parquet file.
+
+```python
+import pandas as pd
+from suntdataset.visualizer import SUNTVisualizer
+
+url = "https://github.com/LabIA-UFBA/SUNT/raw/main/data/od/od-2024-03-01.parquet"
+df = pd.read_parquet(url)
+
+viz = SUNTVisualizer(df)
+graph = viz.build_od_graph()
+viz.export_od_graph("outputs/od_2024-03-01.graphml", fmt="graphml")
+viz.export_od_graph("outputs/od_2024-03-01.gexf", fmt="gexf")
+```
+
+---
+
+## 📊 Dataset Types Supported
+
+| Type | Description |
+| --- | --- |
+| `avl-full` | Full telemetry including GPS, speed, and occupancy. |
+| `avl-lines` | Information concerning routes and schedules. |
+| `trip-timeseries` | Node activity (loading, boarding) in 5-min intervals. |
+| `boarding-alighting` | Passenger flow at stop level. |
+| `gtfs-stops` | Geospatial data for all transit stops. |
+
+---
+
+## 🏛 Academic Use & Citation
+
+This library is a wrapper for the **SUNT Dataset** (UFBA). If you use this tool in your research, please cite:
+
+> *Nery, A. S., et al. (2024). SUNT: Salvador Unified Network for Transport Dataset.*
+
+---
+
+## 🛠 Development: Running Locally
+
+To test changes before publishing to PyPI:
+
+1. Create a virtual environment: `python -m venv venv`
+2. Activate it: `source venv/bin/activate`
+3. Install dependencies: `pip install -r requirements.txt`
+4. Run the sample dashboard: `streamlit run examples/dashboard_app.py`
+
+---
+
+**Would you like me to help you configure the `__init__.py` file so these classes are available directly at the top level of the library?**

suntdataset-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "suntdataset"
+version = "0.1.0"
+description = "A Dataset for the Salvador Unified Network for Transport (SUNT)."
+dependencies = [
+    "pandas",
+    "pyarrow",
+    "requests",
+    "tqdm"
+]
+
+[tool.setuptools]
+packages = ["suntdataset"]
+
+[project.optional-dependencies]
+vis = ["pydeck", "folium", "geopandas"]
+dashboard = ["streamlit"]
+all = ["pydeck", "folium", "geopandas", "streamlit", "networkx"]
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

suntdataset-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

suntdataset-0.1.0/suntdataset/__init__.py

@@ -0,0 +1,4 @@
+from .loader import SUNTLoader
+from .visualizer import SUNTVisualizer
+
+__version__ = "0.1.0"

suntdataset-0.1.0/suntdataset/loader.py

@@ -0,0 +1,53 @@
+import pandas as pd
+from tqdm import tqdm
+
+
+class SUNTLoader:
+    BASE_URL = "https://raw.githubusercontent.com/Nery-Ufba/SUNT/main/data/"
+    ROUTES_URL = "https://raw.githubusercontent.com/Nery-Ufba/SUNT/main/data/raw/GTFS_INTEGRA_SALVADOR2/"
+    DEFAULT_START = "2024-03-01"
+
+    def _filter_days(self, dates, day_type):
+        df = pd.DataFrame({'date': dates})
+        df['dow'] = df['date'].dt.dayofweek
+        if day_type == 'workdays': return df[df['dow'] < 5]['date']
+        if day_type == 'saturdays': return df[df['dow'] == 5]['date']
+        if day_type == 'sundays': return df[df['dow'] == 6]['date']
+        return df['date']
+
+    def load_batch(self, dataset_type, start_date=None, periods=1, freq='D', day_type='all'):
+        start = start_date if start_date else self.DEFAULT_START
+        dates = pd.date_range(start=start, periods=periods, freq=freq)
+        selected_dates = self._filter_days(dates, day_type)
+        
+        dataframes = []
+        for dt in tqdm(selected_dates, desc=f"Carregando {dataset_type}"):
+            y, m, d = dt.year, f"{dt.month:02d}", f"{dt.day:02d}"
+            
+            # Mapeamento conforme datasets.md do SUNT
+            paths = {
+                'od': f"{self.BASE_URL}od/od-{y}-{m}-{d}.parquet",                
+                'gtfs-stops': f"{self.ROUTES_URL}stops.txt",
+                'gtfs-trips': f"{self.ROUTES_URL}trips.txt",
+                'gtfs-stop-times': f"{self.ROUTES_URL}stop_times.txt",                
+                'gtfs-routes': f"{self.ROUTES_URL}routes.txt",                
+                'gtfs-shapes': f"{self.ROUTES_URL}shapes.txt",  
+                'gtfs-agency': f"{self.ROUTES_URL}agency.txt",  
+            }
+            
+            try:
+                url = paths[dataset_type]
+                if url.endswith('.parquet'):
+                    df = pd.read_parquet(url)
+                elif url.endswith('.csv'):
+                    df = pd.read_csv(url)
+                elif url.endswith('.txt'):
+                    df = pd.read_csv(url)
+                else:
+                    raise ValueError(f"Formato nao suportado para: {url}")
+                df['date_ref'] = dt
+                dataframes.append(df)
+            except Exception as e:
+                continue
+                
+        return pd.concat(dataframes, ignore_index=True) if dataframes else pd.DataFrame()

suntdataset-0.1.0/suntdataset/visualizer.py

@@ -0,0 +1,108 @@
+import importlib.util
+from pathlib import Path
+import pandas as pd
+
+def _has_pkg(name):
+    return importlib.util.find_spec(name) is not None
+
+
+class SUNTVisualizer:
+    def __init__(self, df):
+        self.df = df
+
+    def build_od_graph(
+        self,
+        origin_col="stop_id",
+        trip_col="trip_id",
+        sequence_col="pt_sequence",
+        time_col="stop_time",
+        weight_col="n-boardings",
+    ):
+        if not _has_pkg("networkx"):
+            raise ImportError("NetworkX não encontrado. Instale com: pip install networkx")
+
+        import networkx as nx
+
+        required_cols = {origin_col, trip_col}
+        if sequence_col:
+            required_cols.add(sequence_col)
+        if time_col:
+            required_cols.add(time_col)
+        if weight_col:
+            required_cols.add(weight_col)
+
+        missing = [col for col in required_cols if col not in self.df.columns]
+        if missing:
+            raise ValueError(f"Colunas ausentes para gerar grafo OD: {missing}")
+
+        df = self.df.copy()
+        if time_col and time_col in df.columns:
+            df[time_col] = pd.to_datetime(df[time_col], errors="coerce")
+
+        sort_cols = [trip_col]
+        if sequence_col and sequence_col in df.columns:
+            sort_cols.append(sequence_col)
+        elif time_col and time_col in df.columns:
+            sort_cols.append(time_col)
+        else:
+            raise ValueError("Informe sequence_col ou time_col válido para ordenar os deslocamentos.")
+
+        df = df.sort_values(sort_cols)
+        df["next_stop_id"] = df.groupby(trip_col)[origin_col].shift(-1)
+        edges_df = df.dropna(subset=[origin_col, "next_stop_id"])
+        edges_df = edges_df[edges_df[origin_col] != edges_df["next_stop_id"]]
+
+        if edges_df.empty:
+            return nx.DiGraph()
+
+        if weight_col and weight_col in edges_df.columns:
+            edges_df[weight_col] = pd.to_numeric(edges_df[weight_col], errors="coerce").fillna(0)
+            grouped = (
+                edges_df.groupby([origin_col, "next_stop_id"], as_index=False)
+                .agg(
+                    trips=(trip_col, "nunique"),
+                    passengers=(weight_col, "sum"),
+                )
+            )
+        else:
+            grouped = (
+                edges_df.groupby([origin_col, "next_stop_id"], as_index=False)
+                .agg(trips=(trip_col, "nunique"))
+            )
+            grouped["passengers"] = grouped["trips"]
+
+        graph = nx.DiGraph()
+        for row in grouped.itertuples(index=False):
+            origin = str(getattr(row, origin_col))
+            destination = str(row.next_stop_id)
+            trips = float(row.trips)
+            passengers = float(row.passengers)
+            graph.add_edge(
+                origin,
+                destination,
+                trips=trips,
+                passengers=passengers,
+                weight=passengers,
+            )
+
+        return graph
+
+    def export_od_graph(self, output_path, fmt="graphml", **build_kwargs):
+        if not _has_pkg("networkx"):
+            raise ImportError("NetworkX não encontrado. Instale com: pip install networkx")
+
+        import networkx as nx
+
+        graph = self.build_od_graph(**build_kwargs)
+        output = Path(output_path)
+        output.parent.mkdir(parents=True, exist_ok=True)
+
+        fmt_normalized = fmt.strip().lower()
+        if fmt_normalized == "graphml":
+            nx.write_graphml(graph, output)
+        elif fmt_normalized == "gexf":
+            nx.write_gexf(graph, output)
+        else:
+            raise ValueError("Formato inválido. Use 'graphml' ou 'gexf'.")
+
+        return output

suntdataset-0.1.0/suntdataset.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: suntdataset
+Version: 0.1.0
+Summary: A Dataset for the Salvador Unified Network for Transport (SUNT).
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: tqdm
+Provides-Extra: vis
+Requires-Dist: pydeck; extra == "vis"
+Requires-Dist: folium; extra == "vis"
+Requires-Dist: geopandas; extra == "vis"
+Provides-Extra: dashboard
+Requires-Dist: streamlit; extra == "dashboard"
+Provides-Extra: all
+Requires-Dist: pydeck; extra == "all"
+Requires-Dist: folium; extra == "all"
+Requires-Dist: geopandas; extra == "all"
+Requires-Dist: streamlit; extra == "all"
+Requires-Dist: networkx; extra == "all"

suntdataset-0.1.0/suntdataset.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+suntdataset/__init__.py
+suntdataset/loader.py
+suntdataset/visualizer.py
+suntdataset.egg-info/PKG-INFO
+suntdataset.egg-info/SOURCES.txt
+suntdataset.egg-info/dependency_links.txt
+suntdataset.egg-info/requires.txt
+suntdataset.egg-info/top_level.txt
+tests/test_local.py
+tests/test_visualizer.py

suntdataset-0.1.0/suntdataset.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

suntdataset-0.1.0/suntdataset.egg-info/requires.txt

@@ -0,0 +1,19 @@
+pandas
+pyarrow
+requests
+tqdm
+
+[all]
+pydeck
+folium
+geopandas
+streamlit
+networkx
+
+[dashboard]
+streamlit
+
+[vis]
+pydeck
+folium
+geopandas

suntdataset-0.1.0/suntdataset.egg-info/top_level.txt

@@ -0,0 +1 @@
+suntdataset

suntdataset-0.1.0/tests/test_local.py

@@ -0,0 +1,60 @@
+from suntdataset.loader import SUNTLoader
+from suntdataset.visualizer import SUNTVisualizer
+import pandas as pd
+import pytest
+
+
+def test_load_batch_od_parquet():
+    loader = SUNTLoader()
+    df = loader.load_batch(dataset_type="od", periods=1)
+
+    assert df is not None
+    if not df.empty:
+        assert "date_ref" in df.columns
+
+
+def test_load_batch_gtfs_parquet():
+    loader = SUNTLoader()
+    dataset_types = [
+        "od",
+        "gtfs-stops",
+        "gtfs-trips",
+        "gtfs-stop-times",
+        "gtfs-routes",
+    ]
+
+    for dataset_type in dataset_types:
+        df = loader.load_batch(dataset_type=dataset_type, periods=1)
+        assert df is not None
+        if not df.empty:            
+            print(f"{dataset_type} - len: {len(df)}")
+            assert "date_ref" in df.columns
+
+
+def test_build_od_graph_from_dataframe():
+    pytest.importorskip("networkx")
+
+    df = pd.DataFrame(
+        {
+            "trip_id": ["t1", "t1", "t1", "t2", "t2"],
+            "stop_id": ["A", "B", "C", "A", "C"],
+            "pt_sequence": [1, 2, 3, 1, 2],
+            "stop_time": [
+                "2024-03-01 08:00:00",
+                "2024-03-01 08:10:00",
+                "2024-03-01 08:20:00",
+                "2024-03-01 09:00:00",
+                "2024-03-01 09:12:00",
+            ],
+            "n-boardings": [5, 3, 1, 4, 2],
+        }
+    )
+
+    viz = SUNTVisualizer(df)
+    graph = viz.build_od_graph()
+
+    assert graph.number_of_nodes() == 3
+    assert graph.number_of_edges() == 3
+    assert graph.has_edge("A", "B")
+    assert graph.has_edge("B", "C")
+    assert graph.has_edge("A", "C")

suntdataset-0.1.0/tests/test_visualizer.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import pytest
+
+from suntdataset.visualizer import SUNTVisualizer
+
+
+nx = pytest.importorskip("networkx")
+OD_URL = "https://github.com/LabIA-UFBA/SUNT/raw/main/data/od/od-2024-03-01.parquet"
+
+
+def _sample_od_df():
+    return pd.DataFrame(
+        {
+            "trip_id": ["t1", "t1", "t1", "t2", "t2"],
+            "stop_id": ["A", "B", "C", "A", "B"],
+            "pt_sequence": [1, 2, 3, 1, 2],
+            "stop_time": [
+                "2024-03-01 08:00:00",
+                "2024-03-01 08:10:00",
+                "2024-03-01 08:20:00",
+                "2024-03-01 09:00:00",
+                "2024-03-01 09:10:00",
+            ],
+            "n-boardings": [10, 20, 30, 5, 7],
+        }
+    )
+
+
+@pytest.fixture(scope="module")
+def real_od_df():
+    return pd.read_parquet(OD_URL)
+
+
+def test_build_od_graph_aggregates_edges():
+    viz = SUNTVisualizer(_sample_od_df())
+    graph = viz.build_od_graph()
+
+    assert isinstance(graph, nx.DiGraph)
+    assert graph.number_of_nodes() == 3
+    assert graph.number_of_edges() == 2
+
+    assert graph.has_edge("A", "B")
+    assert graph["A"]["B"]["trips"] == 2.0
+    assert graph["A"]["B"]["passengers"] == 15.0
+    assert graph["A"]["B"]["weight"] == 15.0
+
+    assert graph.has_edge("B", "C")
+    assert graph["B"]["C"]["trips"] == 1.0
+    assert graph["B"]["C"]["passengers"] == 20.0
+
+
+def test_build_od_graph_works_with_time_order_only():
+    df = _sample_od_df().drop(columns=["pt_sequence"])
+    viz = SUNTVisualizer(df)
+    graph = viz.build_od_graph(sequence_col=None, time_col="stop_time")
+
+    assert graph.number_of_edges() == 2
+    assert graph.has_edge("A", "B")
+    assert graph.has_edge("B", "C")
+
+
+def test_build_od_graph_missing_columns_raises():
+    df = pd.DataFrame({"trip_id": ["t1"], "stop_id": ["A"]})
+    viz = SUNTVisualizer(df)
+
+    with pytest.raises(ValueError, match="Colunas ausentes"):
+        viz.build_od_graph()
+
+
+def test_export_od_graph_graphml_and_gexf(tmp_path):
+    viz = SUNTVisualizer(_sample_od_df())
+
+    graphml_path = viz.export_od_graph(tmp_path / "od.graphml", fmt="graphml")
+    gexf_path = viz.export_od_graph(tmp_path / "od.gexf", fmt="gexf")
+
+    assert graphml_path.exists()
+    assert graphml_path.suffix == ".graphml"
+    assert gexf_path.exists()
+    assert gexf_path.suffix == ".gexf"
+
+
+def test_export_od_graph_invalid_format_raises(tmp_path):
+    viz = SUNTVisualizer(_sample_od_df())
+
+    with pytest.raises(ValueError, match="Formato invalido|Formato inválido"):
+        viz.export_od_graph(tmp_path / "od.invalid", fmt="json")
+
+
+def test_build_od_graph_with_real_od_data(real_od_df):
+    viz = SUNTVisualizer(real_od_df)
+    graph = viz.build_od_graph(
+        origin_col="stop_id",
+        trip_col="trip_id",
+        sequence_col="pt_sequence",
+        time_col="stop_time",
+        weight_col="n-boardings",
+    )
+
+    assert isinstance(graph, nx.DiGraph)
+    assert graph.number_of_nodes() > 0
+    assert graph.number_of_edges() > 0
+    assert sum(data["passengers"] for _, _, data in graph.edges(data=True)) > 0