3phi-framework 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- 3phi_framework-0.2.5/.env +11 -0
- 3phi_framework-0.2.5/.github/workflows/docs.yml +53 -0
- 3phi_framework-0.2.5/.gitignore +10 -0
- 3phi_framework-0.2.5/.gitlab/merge_request_templates/mr-template.md +10 -0
- 3phi_framework-0.2.5/.gitlab-ci.yml +106 -0
- 3phi_framework-0.2.5/.pre-commit-config.yaml +7 -0
- 3phi_framework-0.2.5/PKG-INFO +214 -0
- 3phi_framework-0.2.5/README.md +205 -0
- 3phi_framework-0.2.5/docker/clean_volume.sh +3 -0
- 3phi_framework-0.2.5/docker/db/init/1_debug.sh +3 -0
- 3phi_framework-0.2.5/docker/db/init/2_add_user.sql +1 -0
- 3phi_framework-0.2.5/docker/docker-compose.yml +33 -0
- 3phi_framework-0.2.5/docs/Makefile +20 -0
- 3phi_framework-0.2.5/docs/_static/.gitkeep +0 -0
- 3phi_framework-0.2.5/docs/api/index.rst +29 -0
- 3phi_framework-0.2.5/docs/conf.py +51 -0
- 3phi_framework-0.2.5/docs/index.rst +8 -0
- 3phi_framework-0.2.5/docs/make.bat +35 -0
- 3phi_framework-0.2.5/docs/requirements.txt +9 -0
- 3phi_framework-0.2.5/execute_data_app.sh +21 -0
- 3phi_framework-0.2.5/pyproject.toml +52 -0
- 3phi_framework-0.2.5/requirements-ci.txt +9 -0
- 3phi_framework-0.2.5/requirements.txt +15 -0
- 3phi_framework-0.2.5/setup.cfg +4 -0
- 3phi_framework-0.2.5/src/3phi_framework.egg-info/PKG-INFO +214 -0
- 3phi_framework-0.2.5/src/3phi_framework.egg-info/SOURCES.txt +93 -0
- 3phi_framework-0.2.5/src/3phi_framework.egg-info/dependency_links.txt +1 -0
- 3phi_framework-0.2.5/src/3phi_framework.egg-info/top_level.txt +1 -0
- 3phi_framework-0.2.5/src/threephi_framework/__init__.py +33 -0
- 3phi_framework-0.2.5/src/threephi_framework/aggregator.py +72 -0
- 3phi_framework-0.2.5/src/threephi_framework/controllers/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/controllers/meta.py +480 -0
- 3phi_framework-0.2.5/src/threephi_framework/controllers/time_series.py +35 -0
- 3phi_framework-0.2.5/src/threephi_framework/controllers/topology.py +256 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/base.py +73 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/base_config.py +7 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/sm_classifier.py +691 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/stat_labeler.py +164 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/timeseries_ingestor.py +45 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_apps/topology_ingestor.py +45 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_extractor/data_extractor.py +3198 -0
- 3phi_framework-0.2.5/src/threephi_framework/data_extractor/schemas/phase_measurements/v1.py +36 -0
- 3phi_framework-0.2.5/src/threephi_framework/db/db.py +63 -0
- 3phi_framework-0.2.5/src/threephi_framework/db_connector.py +254 -0
- 3phi_framework-0.2.5/src/threephi_framework/dtu/sm_classifier.py +363 -0
- 3phi_framework-0.2.5/src/threephi_framework/dtu/stat_labeler.py +676 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/base.py +5 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/meta/meta_schema_mixin.py +5 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/meta/meter.py +22 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/meta/run_result.py +66 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/cabinet.py +14 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/delivery_point.py +20 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/feeder.py +20 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/meter.py +25 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/secondary_substation.py +17 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/transformer.py +22 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/cable.py +26 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/edge.py +42 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/edge_cable.py +22 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/node.py +43 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/topology_version.py +21 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/lv_schema_mixin.py +5 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/node_type_enum.py +10 -0
- 3phi_framework-0.2.5/src/threephi_framework/models/topology/utilities.py +26 -0
- 3phi_framework-0.2.5/src/threephi_framework/object_storage/base_connector.py +266 -0
- 3phi_framework-0.2.5/src/threephi_framework/object_storage/s3_connector.py +220 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/base.py +30 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/meta/meter.py +38 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/meta/run_result.py +106 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/sanity.py +22 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/staging.py +54 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/cabinet.py +17 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/delivery_point.py +43 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/feeder.py +27 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/meter.py +308 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/secondary_substation.py +24 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/transformer.py +27 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/__init__.py +0 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/cable.py +69 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/edge.py +313 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/edge_cable.py +46 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/node.py +130 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/topology_version.py +19 -0
- 3phi_framework-0.2.5/src/threephi_framework/resources/topology/topology_export.py +283 -0
- 3phi_framework-0.2.5/src/threephi_framework/schemas/v1/phase_measurements.py +36 -0
- 3phi_framework-0.2.5/src/threephi_framework/schemas/v1/topology.py +96 -0
- 3phi_framework-0.2.5/src/threephi_framework/topology_cleaner.py +432 -0
- 3phi_framework-0.2.5/src/threephi_framework/util/util.py +9 -0
- 3phi_framework-0.2.5/tests/data/lv_topology.csv +25500 -0
- 3phi_framework-0.2.5/tests/data/meter_cabinet_connection.csv +62346 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Build and deploy Sphinx docs
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
pages: write
|
|
11
|
+
id-token: write
|
|
12
|
+
|
|
13
|
+
concurrency:
|
|
14
|
+
group: "pages"
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
build:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
|
|
23
|
+
- uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
|
|
27
|
+
- name: Install deps
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip
|
|
30
|
+
pip install -r docs/requirements.txt
|
|
31
|
+
pip install -e .
|
|
32
|
+
|
|
33
|
+
- name: Build HTML
|
|
34
|
+
run: |
|
|
35
|
+
sphinx-build -b html -W --keep-going -T docs docs/_build/html
|
|
36
|
+
|
|
37
|
+
- name: Configure Pages
|
|
38
|
+
uses: actions/configure-pages@v5
|
|
39
|
+
|
|
40
|
+
- name: Upload Pages artifact
|
|
41
|
+
uses: actions/upload-pages-artifact@v3
|
|
42
|
+
with:
|
|
43
|
+
path: docs/_build/html
|
|
44
|
+
|
|
45
|
+
deploy:
|
|
46
|
+
needs: build
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
environment:
|
|
49
|
+
name: github-pages
|
|
50
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
51
|
+
steps:
|
|
52
|
+
- id: deployment
|
|
53
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
stages: [lint, style, build, publish]
|
|
2
|
+
image: python:3.12
|
|
3
|
+
|
|
4
|
+
variables:
|
|
5
|
+
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
|
6
|
+
PIP_NO_CACHE_DIR: "1"
|
|
7
|
+
# Ensure tags are available to the job
|
|
8
|
+
GIT_STRATEGY: fetch
|
|
9
|
+
GIT_DEPTH: "0"
|
|
10
|
+
|
|
11
|
+
# ---------- Ruff lint and format with MR annotations ----------
|
|
12
|
+
.ruff:
|
|
13
|
+
image: python:3.11-slim
|
|
14
|
+
before_script:
|
|
15
|
+
- python -V
|
|
16
|
+
- pip install --no-cache-dir ruff==0.6.8
|
|
17
|
+
|
|
18
|
+
ruff_lint:
|
|
19
|
+
extends: .ruff
|
|
20
|
+
stage: lint
|
|
21
|
+
rules:
|
|
22
|
+
- when: always
|
|
23
|
+
script:
|
|
24
|
+
- ruff check --output-format=junit --output-file=ruff-junit.xml .
|
|
25
|
+
artifacts:
|
|
26
|
+
when: always
|
|
27
|
+
reports:
|
|
28
|
+
junit: ruff-junit.xml
|
|
29
|
+
|
|
30
|
+
ruff_format_check:
|
|
31
|
+
extends: .ruff
|
|
32
|
+
stage: style
|
|
33
|
+
rules:
|
|
34
|
+
- when: always
|
|
35
|
+
script:
|
|
36
|
+
- ruff format --check .
|
|
37
|
+
|
|
38
|
+
# ---------- Release (tag) ----------
|
|
39
|
+
build_release:
|
|
40
|
+
stage: build
|
|
41
|
+
needs: [ruff_lint, ruff_format_check]
|
|
42
|
+
rules:
|
|
43
|
+
- if: '$CI_COMMIT_TAG'
|
|
44
|
+
script:
|
|
45
|
+
- |
|
|
46
|
+
python -m pip install -U pip build
|
|
47
|
+
python -m build
|
|
48
|
+
artifacts:
|
|
49
|
+
when: always
|
|
50
|
+
paths:
|
|
51
|
+
- dist/
|
|
52
|
+
|
|
53
|
+
publish_release:
|
|
54
|
+
stage: publish
|
|
55
|
+
needs: [build_release]
|
|
56
|
+
rules:
|
|
57
|
+
- if: '$CI_COMMIT_TAG'
|
|
58
|
+
script:
|
|
59
|
+
- |
|
|
60
|
+
python -m pip install -U twine
|
|
61
|
+
TWINE_USERNAME=gitlab-ci-token TWINE_PASSWORD="$CI_JOB_TOKEN" \
|
|
62
|
+
python -m twine upload \
|
|
63
|
+
--repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" \
|
|
64
|
+
dist/*
|
|
65
|
+
|
|
66
|
+
# ---------- Manual PR build (dev) ----------
|
|
67
|
+
build_pr_manual:
|
|
68
|
+
stage: build
|
|
69
|
+
needs: [ruff_lint, ruff_format_check]
|
|
70
|
+
rules:
|
|
71
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
72
|
+
when: always
|
|
73
|
+
script:
|
|
74
|
+
- |
|
|
75
|
+
python -m pip install -U pip build
|
|
76
|
+
# Ensure we have tags (safety if runner ignores GIT_DEPTH)
|
|
77
|
+
git fetch --force --prune --tags
|
|
78
|
+
# Get latest semver-like tag (vX.Y.Z or X.Y.Z); fallback to 0.0.0
|
|
79
|
+
BASE_TAG="$(git tag --list --sort=-v:refname | grep -E '^(v)?[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 || true)"
|
|
80
|
+
if [ -z "$BASE_TAG" ]; then
|
|
81
|
+
BASE_VERSION="0.0.0"
|
|
82
|
+
else
|
|
83
|
+
BASE_VERSION="${BASE_TAG#v}" # strip leading v if present
|
|
84
|
+
fi
|
|
85
|
+
export SETUPTOOLS_SCM_PRETEND_VERSION="${BASE_VERSION}.dev${CI_PIPELINE_IID}+pr.${CI_MERGE_REQUEST_IID}.sha${CI_COMMIT_SHORT_SHA}"
|
|
86
|
+
echo "Building version: $SETUPTOOLS_SCM_PRETEND_VERSION"
|
|
87
|
+
python -m build
|
|
88
|
+
artifacts:
|
|
89
|
+
when: always
|
|
90
|
+
expire_in: 7 days
|
|
91
|
+
paths:
|
|
92
|
+
- dist/
|
|
93
|
+
|
|
94
|
+
publish_pr_manual:
|
|
95
|
+
stage: publish
|
|
96
|
+
needs: [build_pr_manual]
|
|
97
|
+
rules:
|
|
98
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
99
|
+
when: manual
|
|
100
|
+
script:
|
|
101
|
+
- |
|
|
102
|
+
python -m pip install -U twine
|
|
103
|
+
TWINE_USERNAME=gitlab-ci-token TWINE_PASSWORD="$CI_JOB_TOKEN" \
|
|
104
|
+
python -m twine upload \
|
|
105
|
+
--repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" \
|
|
106
|
+
dist/*
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: 3phi-framework
|
|
3
|
+
Version: 0.2.5
|
|
4
|
+
Summary: Framework for 3phi project
|
|
5
|
+
Author-email: Christian Fotteler <chris@inilab.dk>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# 3phi Framework
|
|
11
|
+
|
|
12
|
+
Utility classes for **DB** access, **S3** interactions, and **data processing** via a `DataExtractor`.
|
|
13
|
+
Distributed privately via GitLab’s Package Registry (private PyPI).
|
|
14
|
+
|
|
15
|
+
> **Install name:** `3phi-framework`
|
|
16
|
+
> **Import package:** `threephi_framework`
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Features (MVP)
|
|
21
|
+
|
|
22
|
+
- **DBConnector** – simple connector + query helpers.
|
|
23
|
+
- **S3Connector** – upload/download and safe path helpers.
|
|
24
|
+
- **DataExtractor** – API to interact with the Data in S3.
|
|
25
|
+
|
|
26
|
+
> This is an early version. API may evolve until `v1.0`. See “Versioning” below.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Requirements
|
|
31
|
+
|
|
32
|
+
- Python **3.9+**
|
|
33
|
+
- Access to the GitLab **Package Registry** for this project or group.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Build a new version
|
|
38
|
+
|
|
39
|
+
Write & commit your code and then tag it:
|
|
40
|
+
```
|
|
41
|
+
git tag -a vx.x.x -m "vx.x.x"
|
|
42
|
+
git push --tags
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
## Installation (GitLab private PyPI)
|
|
47
|
+
|
|
48
|
+
Create a **Deploy Token** with scope `read_package_registry` in this project.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install \
|
|
52
|
+
--index-url "https://<DEPLOY_USER>:<DEPLOY_TOKEN>@gitlab.com/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
|
|
53
|
+
3phi-framework
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Other ways to install
|
|
57
|
+
Replace placeholders like <PROJECT_ID>, <USERNAME>, <TOKEN>, and use the gitlab domain (gitlab.3pi-dev.io).
|
|
58
|
+
|
|
59
|
+
1) Quick one-liner (private project, for code completion etc.)
|
|
60
|
+
|
|
61
|
+
Use a Deploy Token (Settings → Repository → Deploy tokens) with read_package_registry, or a PAT with read_api.
|
|
62
|
+
```
|
|
63
|
+
# safer: export creds so they don't land in shell history
|
|
64
|
+
export GL_USER="gitlab+deploy-token-<id>" # or your GitLab username if using a PAT
|
|
65
|
+
export GL_TOKEN="<token>"
|
|
66
|
+
|
|
67
|
+
pip install \
|
|
68
|
+
--index-url "https://$GL_USER:$GL_TOKEN@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
|
|
69
|
+
--extra-index-url "https://pypi.org/simple" \
|
|
70
|
+
3phi-framework==0.1.2
|
|
71
|
+
```
|
|
72
|
+
Package name to install: 3phi-framework
|
|
73
|
+
Module you import: threephi_framework
|
|
74
|
+
--extra-index-url https://pypi.org/simple lets pip resolve any third-party deps from PyPI.
|
|
75
|
+
|
|
76
|
+
2) Put it in requirements.txt (useful for docker image builds)
|
|
77
|
+
|
|
78
|
+
At the top of requirements.txt:
|
|
79
|
+
```
|
|
80
|
+
--index-url https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
|
|
81
|
+
--extra-index-url https://pypi.org/simple
|
|
82
|
+
3phi-framework==0.1.2
|
|
83
|
+
```
|
|
84
|
+
Then install as you normally would.
|
|
85
|
+
|
|
86
|
+
3) Persist credentials in pip.conf (developer machines)
|
|
87
|
+
|
|
88
|
+
Linux/macOS: ~/.pip/pip.conf
|
|
89
|
+
Windows: %APPDATA%\pip\pip.ini
|
|
90
|
+
```
|
|
91
|
+
[global]
|
|
92
|
+
index-url = https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
|
|
93
|
+
extra-index-url = https://pypi.org/simple
|
|
94
|
+
```
|
|
95
|
+
Then you can just:
|
|
96
|
+
```
|
|
97
|
+
pip install 3phi-framework==0.1.2
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Quickstart
|
|
103
|
+
|
|
104
|
+
The framework is set up for local development as well as for being used in a deployment.
|
|
105
|
+
To set up your environment for local development, follow these steps:
|
|
106
|
+
|
|
107
|
+
### Set up virtual environment
|
|
108
|
+
|
|
109
|
+
[execute_data_app.sh](execute_data_app.sh) expects a virtual environment to be set up under [.venv]. See the [python docs](https://docs.python.org/3/library/venv.html) on how to set it up.
|
|
110
|
+
|
|
111
|
+
### Seed data
|
|
112
|
+
|
|
113
|
+
Obtain seed data for the database and the object storage and copy it to:
|
|
114
|
+
- [3_db_seed.sql](docker/db/init/3_db_seed.sql): This should be a sql dump/snapshot of a working 3phi Database, PSQL will automatically seed the DB when it is created using docker compose.
|
|
115
|
+
- [3phi](docker/object_storage/3phi): This should be a copy of a bucket from a working object storage. It will be mounted in the minio object storage as a bucket.
|
|
116
|
+
|
|
117
|
+
### Spin up DB and Object Storage
|
|
118
|
+
|
|
119
|
+
Navigate to [docker](./docker) and run `docker compose up -d`.
|
|
120
|
+
This will bring up a local DB and a MinIO Object Storage seeded with the data you provided.
|
|
121
|
+
|
|
122
|
+
### Run a data app locally
|
|
123
|
+
|
|
124
|
+
Use the utility script [execute_data_app.sh](execute_data_app.sh) and pass the data app name as an argument, e.g.:
|
|
125
|
+
```
|
|
126
|
+
./exectue_data_app.sh sm_classifier
|
|
127
|
+
```
|
|
128
|
+
In case the script is not executable, make it executable:
|
|
129
|
+
```
|
|
130
|
+
chmod +x execute_data_app.sh
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The script will install the dependencies in [requirements.txt](requirements.txt) in your virtual environment, set up environment variables as they are listed in [.env](.env) and execute the data app as a python module.
|
|
134
|
+
|
|
135
|
+
## Contibuting to 3phi Framework
|
|
136
|
+
|
|
137
|
+
The framework is currently not open-source, but the aim is to open-source it eventually.
|
|
138
|
+
|
|
139
|
+
### Commit Message Guidelines
|
|
140
|
+
|
|
141
|
+
Commit Messages should follow the recommendations of [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/), supplemented by the [Angular Convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines). Every commit must contain `type` and `description`, while `scope`, `body` and `footer` are optional:
|
|
142
|
+
|
|
143
|
+
```
|
|
144
|
+
<type>(<scope>): <description>
|
|
145
|
+
<BLANK LINE>
|
|
146
|
+
<body>
|
|
147
|
+
<BLANK LINE>
|
|
148
|
+
<footer>
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
#### Type
|
|
152
|
+
`type` is mandatory, and should be one of the following:
|
|
153
|
+
|
|
154
|
+
- `feat`: A new feature
|
|
155
|
+
- `fix`: A bug fix
|
|
156
|
+
- `build`: Changes that affect the build system or external dependencies
|
|
157
|
+
- `ci`: Changes to our CI configuration files and scripts
|
|
158
|
+
- `docs`: Documentation only changes
|
|
159
|
+
- `perf`: A code change that improves performance
|
|
160
|
+
- `refactor`: A code change that neither fixes a bug nor adds a feature
|
|
161
|
+
- `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
|
|
162
|
+
- `test`: Adding missing tests or correcting existing tests
|
|
163
|
+
|
|
164
|
+
#### Scope
|
|
165
|
+
`scope`is optional, but if included it should be a noun that gives context to the part of the codebase that is affected by the commit. Please do not use issue identifiers as scope.
|
|
166
|
+
|
|
167
|
+
#### Description
|
|
168
|
+
`description` is mandatory, and provides a short and concise description of the change. Be mindful of the following:
|
|
169
|
+
|
|
170
|
+
- use the imperative, present tense: "change" not "changed" nor "changes"
|
|
171
|
+
- don't capitalize the first letter
|
|
172
|
+
- no dot (.) at the end
|
|
173
|
+
|
|
174
|
+
#### Body
|
|
175
|
+
`body` is optional, but if included, it should use the imperative, present tense: "change" not "changed" nor "changes" (just as in `description`). The body should include the motivation for the change and contrast this with previous behavior.
|
|
176
|
+
|
|
177
|
+
#### Footer
|
|
178
|
+
`footer` is optional, but if included, it should use `-`in place of whitespace to distinguish it from a `body`or multiline `body`. It is the place to reference git issues that this commit closes.
|
|
179
|
+
|
|
180
|
+
#### Breaking Change
|
|
181
|
+
A commit that make incompatible API changes, should be denoted `Breaking Change`, either by a `!` right after `scope` or by adding `Breaking Change` or `Breaking-Change` to `footer`.
|
|
182
|
+
|
|
183
|
+
### Linter and formatter
|
|
184
|
+
[Ruff](https://docs.astral.sh/ruff/linter/) is used for linting and formatting in both local development (pre-commit) and CI, with a shared configuration in pyproject.toml.
|
|
185
|
+
|
|
186
|
+
#### Ruff in CI pipeline
|
|
187
|
+
- Runs on pushes/merge requests in GitLab.
|
|
188
|
+
- Checks only (no auto-fix). If any check fails, the pipeline fails and the change cannot be merged.
|
|
189
|
+
|
|
190
|
+
#### Ruff in pre-commit hook
|
|
191
|
+
- Runs locally on commit (and optionally on push).
|
|
192
|
+
- Applies fixes and then fails if it changed files so you can review and commit the updates.
|
|
193
|
+
- First-time setup:
|
|
194
|
+
```
|
|
195
|
+
pre-commit install
|
|
196
|
+
# optional: also run on push
|
|
197
|
+
pre-commit install --hook-type pre-push
|
|
198
|
+
```
|
|
199
|
+
- Manual sweep of the whole repo:
|
|
200
|
+
```
|
|
201
|
+
pre-commit run --all-files
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
## Documentation
|
|
206
|
+
|
|
207
|
+
This Project uses Sphinx for API docs. From the [docs](./docs/) run:
|
|
208
|
+
```
|
|
209
|
+
make html
|
|
210
|
+
```
|
|
211
|
+
to generate the docs. Then you can open the [docs index](./docs/_build/html/index.html) in your browser.
|
|
212
|
+
|
|
213
|
+
We leverage [autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to generate the documentation from docstrings in code.
|
|
214
|
+
To configure which classes are included in the API docs, edit and extend [index.rst](./docs/api/index.rst).
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# 3phi Framework
|
|
2
|
+
|
|
3
|
+
Utility classes for **DB** access, **S3** interactions, and **data processing** via a `DataExtractor`.
|
|
4
|
+
Distributed privately via GitLab’s Package Registry (private PyPI).
|
|
5
|
+
|
|
6
|
+
> **Install name:** `3phi-framework`
|
|
7
|
+
> **Import package:** `threephi_framework`
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Features (MVP)
|
|
12
|
+
|
|
13
|
+
- **DBConnector** – simple connector + query helpers.
|
|
14
|
+
- **S3Connector** – upload/download and safe path helpers.
|
|
15
|
+
- **DataExtractor** – API to interact with the Data in S3.
|
|
16
|
+
|
|
17
|
+
> This is an early version. API may evolve until `v1.0`. See “Versioning” below.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## Requirements
|
|
22
|
+
|
|
23
|
+
- Python **3.9+**
|
|
24
|
+
- Access to the GitLab **Package Registry** for this project or group.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## Build a new version
|
|
29
|
+
|
|
30
|
+
Write & commit your code and then tag it:
|
|
31
|
+
```
|
|
32
|
+
git tag -a vx.x.x -m "vx.x.x"
|
|
33
|
+
git push --tags
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
## Installation (GitLab private PyPI)
|
|
38
|
+
|
|
39
|
+
Create a **Deploy Token** with scope `read_package_registry` in this project.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install \
|
|
43
|
+
--index-url "https://<DEPLOY_USER>:<DEPLOY_TOKEN>@gitlab.com/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
|
|
44
|
+
3phi-framework
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Other ways to install
|
|
48
|
+
Replace placeholders like <PROJECT_ID>, <USERNAME>, <TOKEN>, and use the gitlab domain (gitlab.3pi-dev.io).
|
|
49
|
+
|
|
50
|
+
1) Quick one-liner (private project, for code completion etc.)
|
|
51
|
+
|
|
52
|
+
Use a Deploy Token (Settings → Repository → Deploy tokens) with read_package_registry, or a PAT with read_api.
|
|
53
|
+
```
|
|
54
|
+
# safer: export creds so they don't land in shell history
|
|
55
|
+
export GL_USER="gitlab+deploy-token-<id>" # or your GitLab username if using a PAT
|
|
56
|
+
export GL_TOKEN="<token>"
|
|
57
|
+
|
|
58
|
+
pip install \
|
|
59
|
+
--index-url "https://$GL_USER:$GL_TOKEN@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
|
|
60
|
+
--extra-index-url "https://pypi.org/simple" \
|
|
61
|
+
3phi-framework==0.1.2
|
|
62
|
+
```
|
|
63
|
+
Package name to install: 3phi-framework
|
|
64
|
+
Module you import: threephi_framework
|
|
65
|
+
--extra-index-url https://pypi.org/simple lets pip resolve any third-party deps from PyPI.
|
|
66
|
+
|
|
67
|
+
2) Put it in requirements.txt (useful for docker image builds)
|
|
68
|
+
|
|
69
|
+
At the top of requirements.txt:
|
|
70
|
+
```
|
|
71
|
+
--index-url https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
|
|
72
|
+
--extra-index-url https://pypi.org/simple
|
|
73
|
+
3phi-framework==0.1.2
|
|
74
|
+
```
|
|
75
|
+
Then install as you normally would.
|
|
76
|
+
|
|
77
|
+
3) Persist credentials in pip.conf (developer machines)
|
|
78
|
+
|
|
79
|
+
Linux/macOS: ~/.pip/pip.conf
|
|
80
|
+
Windows: %APPDATA%\pip\pip.ini
|
|
81
|
+
```
|
|
82
|
+
[global]
|
|
83
|
+
index-url = https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
|
|
84
|
+
extra-index-url = https://pypi.org/simple
|
|
85
|
+
```
|
|
86
|
+
Then you can just:
|
|
87
|
+
```
|
|
88
|
+
pip install 3phi-framework==0.1.2
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Quickstart
|
|
94
|
+
|
|
95
|
+
The framework is set up for local development as well as for being used in a deployment.
|
|
96
|
+
To set up your environment for local development, follow these steps:
|
|
97
|
+
|
|
98
|
+
### Set up virtual environment
|
|
99
|
+
|
|
100
|
+
[execute_data_app.sh](execute_data_app.sh) expects a virtual environment to be set up under [.venv]. See the [python docs](https://docs.python.org/3/library/venv.html) on how to set it up.
|
|
101
|
+
|
|
102
|
+
### Seed data
|
|
103
|
+
|
|
104
|
+
Obtain seed data for the database and the object storage and copy it to:
|
|
105
|
+
- [3_db_seed.sql](docker/db/init/3_db_seed.sql): This should be a sql dump/snapshot of a working 3phi Database, PSQL will automatically seed the DB when it is created using docker compose.
|
|
106
|
+
- [3phi](docker/object_storage/3phi): This should be a copy of a bucket from a working object storage. It will be mounted in the minio object storage as a bucket.
|
|
107
|
+
|
|
108
|
+
### Spin up DB and Object Storage
|
|
109
|
+
|
|
110
|
+
Navigate to [docker](./docker) and run `docker compose up -d`.
|
|
111
|
+
This will bring up a local DB and a MinIO Object Storage seeded with the data you provided.
|
|
112
|
+
|
|
113
|
+
### Run a data app locally
|
|
114
|
+
|
|
115
|
+
Use the utility script [execute_data_app.sh](execute_data_app.sh) and pass the data app name as an argument, e.g.:
|
|
116
|
+
```
|
|
117
|
+
./exectue_data_app.sh sm_classifier
|
|
118
|
+
```
|
|
119
|
+
In case the script is not executable, make it executable:
|
|
120
|
+
```
|
|
121
|
+
chmod +x execute_data_app.sh
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The script will install the dependencies in [requirements.txt](requirements.txt) in your virtual environment, set up environment variables as they are listed in [.env](.env) and execute the data app as a python module.
|
|
125
|
+
|
|
126
|
+
## Contibuting to 3phi Framework
|
|
127
|
+
|
|
128
|
+
The framework is currently not open-source, but the aim is to open-source it eventually.
|
|
129
|
+
|
|
130
|
+
### Commit Message Guidelines
|
|
131
|
+
|
|
132
|
+
Commit Messages should follow the recommendations of [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/), supplemented by the [Angular Convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines). Every commit must contain `type` and `description`, while `scope`, `body` and `footer` are optional:
|
|
133
|
+
|
|
134
|
+
```
|
|
135
|
+
<type>(<scope>): <description>
|
|
136
|
+
<BLANK LINE>
|
|
137
|
+
<body>
|
|
138
|
+
<BLANK LINE>
|
|
139
|
+
<footer>
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
#### Type
|
|
143
|
+
`type` is mandatory, and should be one of the following:
|
|
144
|
+
|
|
145
|
+
- `feat`: A new feature
|
|
146
|
+
- `fix`: A bug fix
|
|
147
|
+
- `build`: Changes that affect the build system or external dependencies
|
|
148
|
+
- `ci`: Changes to our CI configuration files and scripts
|
|
149
|
+
- `docs`: Documentation only changes
|
|
150
|
+
- `perf`: A code change that improves performance
|
|
151
|
+
- `refactor`: A code change that neither fixes a bug nor adds a feature
|
|
152
|
+
- `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
|
|
153
|
+
- `test`: Adding missing tests or correcting existing tests
|
|
154
|
+
|
|
155
|
+
#### Scope
|
|
156
|
+
`scope`is optional, but if included it should be a noun that gives context to the part of the codebase that is affected by the commit. Please do not use issue identifiers as scope.
|
|
157
|
+
|
|
158
|
+
#### Description
|
|
159
|
+
`description` is mandatory, and provides a short and concise description of the change. Be mindful of the following:
|
|
160
|
+
|
|
161
|
+
- use the imperative, present tense: "change" not "changed" nor "changes"
|
|
162
|
+
- don't capitalize the first letter
|
|
163
|
+
- no dot (.) at the end
|
|
164
|
+
|
|
165
|
+
#### Body
|
|
166
|
+
`body` is optional, but if included, it should use the imperative, present tense: "change" not "changed" nor "changes" (just as in `description`). The body should include the motivation for the change and contrast this with previous behavior.
|
|
167
|
+
|
|
168
|
+
#### Footer
|
|
169
|
+
`footer` is optional, but if included, it should use `-`in place of whitespace to distinguish it from a `body`or multiline `body`. It is the place to reference git issues that this commit closes.
|
|
170
|
+
|
|
171
|
+
#### Breaking Change
|
|
172
|
+
A commit that make incompatible API changes, should be denoted `Breaking Change`, either by a `!` right after `scope` or by adding `Breaking Change` or `Breaking-Change` to `footer`.
|
|
173
|
+
|
|
174
|
+
### Linter and formatter
|
|
175
|
+
[Ruff](https://docs.astral.sh/ruff/linter/) is used for linting and formatting in both local development (pre-commit) and CI, with a shared configuration in pyproject.toml.
|
|
176
|
+
|
|
177
|
+
#### Ruff in CI pipeline
|
|
178
|
+
- Runs on pushes/merge requests in GitLab.
|
|
179
|
+
- Checks only (no auto-fix). If any check fails, the pipeline fails and the change cannot be merged.
|
|
180
|
+
|
|
181
|
+
#### Ruff in pre-commit hook
|
|
182
|
+
- Runs locally on commit (and optionally on push).
|
|
183
|
+
- Applies fixes and then fails if it changed files so you can review and commit the updates.
|
|
184
|
+
- First-time setup:
|
|
185
|
+
```
|
|
186
|
+
pre-commit install
|
|
187
|
+
# optional: also run on push
|
|
188
|
+
pre-commit install --hook-type pre-push
|
|
189
|
+
```
|
|
190
|
+
- Manual sweep of the whole repo:
|
|
191
|
+
```
|
|
192
|
+
pre-commit run --all-files
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
## Documentation
|
|
197
|
+
|
|
198
|
+
This Project uses Sphinx for API docs. From the [docs](./docs/) run:
|
|
199
|
+
```
|
|
200
|
+
make html
|
|
201
|
+
```
|
|
202
|
+
to generate the docs. Then you can open the [docs index](./docs/_build/html/index.html) in your browser.
|
|
203
|
+
|
|
204
|
+
We leverage [autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to generate the documentation from docstrings in code.
|
|
205
|
+
To configure which classes are included in the API docs, edit and extend [index.rst](./docs/api/index.rst).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CREATE ROLE "threephi_db_user" WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION PASSWORD 'userpass';
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
services:
|
|
2
|
+
|
|
3
|
+
local-db:
|
|
4
|
+
container_name: local-3phi-db
|
|
5
|
+
image: postgres:18-alpine
|
|
6
|
+
environment:
|
|
7
|
+
POSTGRES_USER: postgres
|
|
8
|
+
POSTGRES_PASSWORD: password
|
|
9
|
+
POSTGRES_DB: 3phi-db
|
|
10
|
+
ports:
|
|
11
|
+
- "5432:5432"
|
|
12
|
+
volumes:
|
|
13
|
+
- ./tmp/pgdata:/var/lib/postgresql
|
|
14
|
+
- ./db/init:/docker-entrypoint-initdb.d
|
|
15
|
+
|
|
16
|
+
local-object-storage:
|
|
17
|
+
container_name: local-minio
|
|
18
|
+
image: minio/minio:latest
|
|
19
|
+
command: server /data --console-address ":9001"
|
|
20
|
+
ports:
|
|
21
|
+
- "19000:9000"
|
|
22
|
+
- "19001:9001"
|
|
23
|
+
environment:
|
|
24
|
+
MINIO_ROOT_USER: minioadmin
|
|
25
|
+
MINIO_ROOT_PASSWORD: minioadmin
|
|
26
|
+
volumes:
|
|
27
|
+
- ./object_storage:/data
|
|
28
|
+
healthcheck:
|
|
29
|
+
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
|
|
30
|
+
interval: 5s
|
|
31
|
+
timeout: 3s
|
|
32
|
+
retries: 20
|
|
33
|
+
restart: unless-stopped
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Minimal makefile for Sphinx documentation
|
|
2
|
+
#
|
|
3
|
+
|
|
4
|
+
# You can set these variables from the command line, and also
|
|
5
|
+
# from the environment for the first two.
|
|
6
|
+
SPHINXOPTS ?=
|
|
7
|
+
SPHINXBUILD ?= sphinx-build
|
|
8
|
+
SOURCEDIR = .
|
|
9
|
+
BUILDDIR = _build
|
|
10
|
+
|
|
11
|
+
# Put it first so that "make" without argument is like "make help".
|
|
12
|
+
help:
|
|
13
|
+
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
|
14
|
+
|
|
15
|
+
.PHONY: help Makefile
|
|
16
|
+
|
|
17
|
+
# Catch-all target: route all unknown targets to Sphinx using the new
|
|
18
|
+
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
|
19
|
+
%: Makefile
|
|
20
|
+
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
|
File without changes
|