tangra-dataforge 0.1.0a1__tar.gz

tangra_dataforge-0.1.0a1/.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Install build tools
+        run: python -m pip install build twine
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Verify distributions
+        run: python -m twine check dist/*
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/project/tangra-dataforge/
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

tangra_dataforge-0.1.0a1/.github/workflows/tests.yml

@@ -0,0 +1,33 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,scraping,analytics,visualization]"
+
+      - name: Run tests
+        run: python -m pytest -v
+
+      - name: Verify dependencies
+        run: python -m pip check

tangra_dataforge-0.1.0a1/.gitignore

@@ -0,0 +1,9 @@
+
+# Python generated files
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+dist/
+build/
+.venv/

tangra_dataforge-0.1.0a1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Irena Popova
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

tangra_dataforge-0.1.0a1/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.5
+Name: tangra-dataforge
+Version: 0.1.0a1
+Summary: Python toolkit for data collection, cleaning, analysis, and investigative journalism.
+Author: Irena Popova
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: matplotlib>=3.8
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: pandas>=2.0
+Requires-Dist: requests>=2.31
+Provides-Extra: analytics
+Requires-Dist: duckdb>=1.1; extra == 'analytics'
+Requires-Dist: pyarrow>=17.0; extra == 'analytics'
+Requires-Dist: scipy>=1.13; extra == 'analytics'
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: mypy>=1.14; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.9; extra == 'dev'
+Provides-Extra: scraping
+Requires-Dist: beautifulsoup4>=4.12; extra == 'scraping'
+Requires-Dist: httpx>=0.27; extra == 'scraping'
+Requires-Dist: lxml>=5.0; extra == 'scraping'
+Requires-Dist: playwright>=1.50; extra == 'scraping'
+Requires-Dist: scrapy>=2.12; extra == 'scraping'
+Provides-Extra: visualization
+Requires-Dist: geopandas>=1.0; extra == 'visualization'
+Requires-Dist: plotly>=5.24; extra == 'visualization'
+Description-Content-Type: text/markdown
+
+# TANGRA DataForge
+
+**Open-Source Python Toolkit for Data Engineering, Analysis & Investigative Journalism**
+
+TANGRA DataForge is an open-source Python project focused on collecting, processing, analyzing, and visualizing data. It is designed for data journalism, investigative research, and reproducible data analysis.
+
+The project aims to transform raw data from public datasets, websites, and APIs into structured, verifiable, and meaningful information.
+
+## Project Goals
+
+- Collect data from public APIs, websites, CSV, JSON, and Excel files.
+- Extract structured information through web scraping.
+- Clean, validate, and transform datasets using Python and Pandas.
+- Perform statistical analysis and identify patterns, trends, and anomalies.
+- Create informative visualizations and charts.
+- Support reproducible investigations with transparent methodologies and documented sources.
+
+## Technology Stack
+
+- **Python** — Core programming language
+- **Pandas** — Data manipulation and analysis
+- **NumPy** — Numerical computing
+- **Beautiful Soup** — HTML parsing and web scraping
+- **Requests** — HTTP requests and API integration
+- **Matplotlib / Plotly** — Data visualization
+- **pytest** — Automated testing
+- **Git / GitHub** — Version control and collaboration
+
+These technologies represent the planned toolkit. Dependencies will be introduced as features are implemented.
+
+## Planned Architecture
+
+```text
+tangra-dataforge/
+├── src/
+│   └── tangra_dataforge/
+│       ├── __init__.py
+│       ├── collect.py
+│       ├── clean.py
+│       ├── analyze.py
+│       └── visualize.py
+├── investigations/
+├── examples/
+├── tests/
+├── pyproject.toml
+├── README.md
+└── .gitignore
+```
+
+## Data Journalism & Research
+
+TANGRA DataForge is being developed to support data-driven investigations into economic, social, political, and technological developments.
+
+Planned research applications include:
+
+- Economic indicators and regional inequality
+- Housing affordability and income development
+- Labour markets and workforce shortages
+- Education and professional qualifications
+- Technology, digital transformation, and society
+
+Investigations will prioritize publicly accessible data, source attribution, transparent analysis, and reproducible results.
+
+## Development Status
+
+**Early development — project initialization**
+
+The toolkit is currently being established. Its first development phase will focus on Python/Pandas data processing, dataset validation, and reproducible analysis workflows.
+
+The project is not yet available as an installable Python package.
+
+## Roadmap
+
+- [ ] Establish Python package structure
+- [ ] Implement CSV and JSON data loading
+- [ ] Develop Pandas-based cleaning functions
+- [ ] Add data validation and descriptive statistics
+- [ ] Integrate API data collection
+- [ ] Implement web scraping components
+- [ ] Add visualization utilities
+- [ ] Create a reproducible journalism case study
+- [ ] Add automated tests and documentation
+- [ ] Publish an initial release on PyPI
+
+## Responsible Data Collection
+
+Data collection functionality will be developed with attention to applicable laws, data protection, website access policies, rate limits, and source attribution.
+
+## Author
+
+**Irena Popova**
+
+Independent Researcher | Python & Full-Stack Developer | Author & Publisher
+
+[Website](https://www.irena-popova.com) | [GitHub](https://github.com/irenapopova)
+
+## License
+
+An open-source license will be added before the first public software release.

tangra_dataforge-0.1.0a1/README.md

@@ -0,0 +1,99 @@
+# TANGRA DataForge
+
+**Open-Source Python Toolkit for Data Engineering, Analysis & Investigative Journalism**
+
+TANGRA DataForge is an open-source Python project focused on collecting, processing, analyzing, and visualizing data. It is designed for data journalism, investigative research, and reproducible data analysis.
+
+The project aims to transform raw data from public datasets, websites, and APIs into structured, verifiable, and meaningful information.
+
+## Project Goals
+
+- Collect data from public APIs, websites, CSV, JSON, and Excel files.
+- Extract structured information through web scraping.
+- Clean, validate, and transform datasets using Python and Pandas.
+- Perform statistical analysis and identify patterns, trends, and anomalies.
+- Create informative visualizations and charts.
+- Support reproducible investigations with transparent methodologies and documented sources.
+
+## Technology Stack
+
+- **Python** — Core programming language
+- **Pandas** — Data manipulation and analysis
+- **NumPy** — Numerical computing
+- **Beautiful Soup** — HTML parsing and web scraping
+- **Requests** — HTTP requests and API integration
+- **Matplotlib / Plotly** — Data visualization
+- **pytest** — Automated testing
+- **Git / GitHub** — Version control and collaboration
+
+These technologies represent the planned toolkit. Dependencies will be introduced as features are implemented.
+
+## Planned Architecture
+
+```text
+tangra-dataforge/
+├── src/
+│   └── tangra_dataforge/
+│       ├── __init__.py
+│       ├── collect.py
+│       ├── clean.py
+│       ├── analyze.py
+│       └── visualize.py
+├── investigations/
+├── examples/
+├── tests/
+├── pyproject.toml
+├── README.md
+└── .gitignore
+```
+
+## Data Journalism & Research
+
+TANGRA DataForge is being developed to support data-driven investigations into economic, social, political, and technological developments.
+
+Planned research applications include:
+
+- Economic indicators and regional inequality
+- Housing affordability and income development
+- Labour markets and workforce shortages
+- Education and professional qualifications
+- Technology, digital transformation, and society
+
+Investigations will prioritize publicly accessible data, source attribution, transparent analysis, and reproducible results.
+
+## Development Status
+
+**Early development — project initialization**
+
+The toolkit is currently being established. Its first development phase will focus on Python/Pandas data processing, dataset validation, and reproducible analysis workflows.
+
+The project is not yet available as an installable Python package.
+
+## Roadmap
+
+- [ ] Establish Python package structure
+- [ ] Implement CSV and JSON data loading
+- [ ] Develop Pandas-based cleaning functions
+- [ ] Add data validation and descriptive statistics
+- [ ] Integrate API data collection
+- [ ] Implement web scraping components
+- [ ] Add visualization utilities
+- [ ] Create a reproducible journalism case study
+- [ ] Add automated tests and documentation
+- [ ] Publish an initial release on PyPI
+
+## Responsible Data Collection
+
+Data collection functionality will be developed with attention to applicable laws, data protection, website access policies, rate limits, and source attribution.
+
+## Author
+
+**Irena Popova**
+
+Independent Researcher | Python & Full-Stack Developer | Author & Publisher
+
+[Website](https://www.irena-popova.com) | [GitHub](https://github.com/irenapopova)
+
+## License
+
+An open-source license will be added before the first public software release.

tangra_dataforge-0.1.0a1/pyproject.toml

@@ -0,0 +1,56 @@
+
+
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tangra-dataforge"
+version = "0.1.0a1"
+description = "Python toolkit for data collection, cleaning, analysis, and investigative journalism."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+    { name = "Irena Popova" }
+]
+
+dependencies = [
+    "pandas>=2.0",
+    "requests>=2.31",
+    "openpyxl>=3.1",
+    "matplotlib>=3.8"
+]
+
+[project.optional-dependencies]
+scraping = [
+    "scrapy>=2.12",
+    "beautifulsoup4>=4.12",
+    "lxml>=5.0",
+    "httpx>=0.27",
+    "playwright>=1.50"
+]
+
+analytics = [
+    "scipy>=1.13",
+    "duckdb>=1.1",
+    "pyarrow>=17.0"
+]
+
+visualization = [
+    "plotly>=5.24",
+    "geopandas>=1.0"
+]
+
+dev = [
+    "pytest>=8.0",
+    "build>=1.0",
+    "ruff>=0.9",
+    "mypy>=1.14"
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/tangra_dataforge"]

tangra_dataforge-0.1.0a1/src/tangra_dataforge/analyze.py

@@ -0,0 +1,83 @@
+
+"""Statistical analysis utilities for TANGRA DataForge."""
+
+import pandas as pd
+
+
+def describe_data(df: pd.DataFrame) -> pd.DataFrame:
+    """Calculate descriptive statistics for numeric columns."""
+    return df.describe()
+
+
+def calculate_percentage_change(
+    df: pd.DataFrame,
+    column: str,
+    periods: int = 1,
+) -> pd.Series:
+    """Calculate percentage change between observations."""
+    if column not in df.columns:
+        raise KeyError(f"Column not found: {column}")
+
+    if periods < 1:
+        raise ValueError("periods must be at least 1")
+
+    return df[column].pct_change(periods=periods) * 100
+
+
+def calculate_growth_rate(
+    initial_value: float,
+    final_value: float,
+) -> float:
+    """Calculate growth between two values as a percentage."""
+    if initial_value == 0:
+        raise ValueError("Initial value cannot be zero.")
+
+    return (
+        (final_value - initial_value)
+        / abs(initial_value)
+    ) * 100
+
+
+def compare_columns(
+    df: pd.DataFrame,
+    column_a: str,
+    column_b: str,
+) -> pd.DataFrame:
+    """Compare two numeric columns using absolute differences."""
+    if column_a not in df.columns or column_b not in df.columns:
+        raise KeyError("One or both columns were not found.")
+
+    result = df[[column_a, column_b]].copy()
+    result["difference"] = result[column_a] - result[column_b]
+
+    return result
+
+
+def detect_outliers(
+    df: pd.DataFrame,
+    column: str,
+    threshold: float = 1.5,
+) -> pd.DataFrame:
+    """Detect outliers using the interquartile range (IQR)."""
+    if column not in df.columns:
+        raise KeyError(f"Column not found: {column}")
+
+    if threshold <= 0:
+        raise ValueError("threshold must be positive")
+
+    values = df[column].dropna()
+
+    if values.empty:
+        return df.iloc[0:0].copy()
+
+    q1 = values.quantile(0.25)
+    q3 = values.quantile(0.75)
+    iqr = q3 - q1
+
+    lower_bound = q1 - threshold * iqr
+    upper_bound = q3 + threshold * iqr
+
+    return df[
+        (df[column] < lower_bound)
+        | (df[column] > upper_bound)
+    ].copy()

tangra_dataforge-0.1.0a1/src/tangra_dataforge/clean.py

@@ -0,0 +1,13 @@
+"""Data cleaning utilities for TANGRA DataForge."""
+
+import pandas as pd
+
+
+def remove_duplicates(df: pd.DataFrame) -> pd.DataFrame:
+    """Remove duplicate rows and reset the index."""
+    return df.drop_duplicates().reset_index(drop=True)
+
+
+def remove_empty_rows(df: pd.DataFrame) -> pd.DataFrame:
+    """Remove rows where all values are missing."""
+    return df.dropna(how="all").reset_index(drop=True)

tangra_dataforge-0.1.0a1/src/tangra_dataforge/collect.py

@@ -0,0 +1,66 @@
+"""Data collection utilities for TANGRA DataForge."""
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+
+def load_csv(
+    path: str | Path,
+    **kwargs: Any,
+) -> pd.DataFrame:
+    """Load a CSV file into a Pandas DataFrame."""
+    return pd.read_csv(path, **kwargs)
+
+
+def load_json(
+    path: str | Path,
+    **kwargs: Any,
+) -> pd.DataFrame:
+    """Load a JSON file into a Pandas DataFrame."""
+    return pd.read_json(path, **kwargs)
+
+
+def load_excel(
+    path: str | Path,
+    sheet_name: str | int = 0,
+    **kwargs: Any,
+) -> pd.DataFrame:
+    """Load one Excel worksheet into a DataFrame."""
+    return pd.read_excel(
+        path,
+        sheet_name=sheet_name,
+        **kwargs,
+    )
+
+
+def fetch_api(
+    url: str,
+    *,
+    timeout: float = 15.0,
+    params: dict[str, Any] | None = None,
+) -> pd.DataFrame:
+    """Fetch a JSON array of records from a public API.
+
+    The endpoint must return a JSON list of objects.
+    """
+    import requests
+
+    response = requests.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    response.raise_for_status()
+
+    data = response.json()
+
+    if not isinstance(data, list) or not all(
+        isinstance(item, dict) for item in data
+    ):
+        raise ValueError(
+            "API response must be a JSON list of objects."
+        )
+
+    return pd.DataFrame(data)

tangra_dataforge-0.1.0a1/src/tangra_dataforge/database.py

@@ -0,0 +1,48 @@
+
+
+"""DuckDB integration for TANGRA DataForge.
+
+Query Pandas DataFrames, CSV files, and Parquet datasets
+using SQL.
+"""
+
+from pathlib import Path
+
+import duckdb
+import pandas as pd
+
+
+def query_dataframe(
+    df: pd.DataFrame,
+    sql: str,
+) -> pd.DataFrame:
+    """Query a DataFrame registered as 'data'."""
+    with duckdb.connect(database=":memory:") as con:
+        con.register("data", df)
+        return con.execute(sql).fetchdf()
+
+
+def query_csv(
+    path: str | Path,
+    sql: str,
+) -> pd.DataFrame:
+    """Query a CSV file registered as 'data'."""
+    path = Path(path)
+
+    with duckdb.connect(database=":memory:") as con:
+        data = con.read_csv(str(path))
+        con.register("data", data)
+        return con.execute(sql).fetchdf()
+
+
+def query_parquet(
+    path: str | Path,
+    sql: str,
+) -> pd.DataFrame:
+    """Query a Parquet file registered as 'data'."""
+    path = Path(path)
+
+    with duckdb.connect(database=":memory:") as con:
+        data = con.read_parquet(str(path))
+        con.register("data", data)
+        return con.execute(sql).fetchdf()