3phi-framework 0.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. 3phi_framework-0.2.5/.env +11 -0
  2. 3phi_framework-0.2.5/.github/workflows/docs.yml +53 -0
  3. 3phi_framework-0.2.5/.gitignore +10 -0
  4. 3phi_framework-0.2.5/.gitlab/merge_request_templates/mr-template.md +10 -0
  5. 3phi_framework-0.2.5/.gitlab-ci.yml +106 -0
  6. 3phi_framework-0.2.5/.pre-commit-config.yaml +7 -0
  7. 3phi_framework-0.2.5/PKG-INFO +214 -0
  8. 3phi_framework-0.2.5/README.md +205 -0
  9. 3phi_framework-0.2.5/docker/clean_volume.sh +3 -0
  10. 3phi_framework-0.2.5/docker/db/init/1_debug.sh +3 -0
  11. 3phi_framework-0.2.5/docker/db/init/2_add_user.sql +1 -0
  12. 3phi_framework-0.2.5/docker/docker-compose.yml +33 -0
  13. 3phi_framework-0.2.5/docs/Makefile +20 -0
  14. 3phi_framework-0.2.5/docs/_static/.gitkeep +0 -0
  15. 3phi_framework-0.2.5/docs/api/index.rst +29 -0
  16. 3phi_framework-0.2.5/docs/conf.py +51 -0
  17. 3phi_framework-0.2.5/docs/index.rst +8 -0
  18. 3phi_framework-0.2.5/docs/make.bat +35 -0
  19. 3phi_framework-0.2.5/docs/requirements.txt +9 -0
  20. 3phi_framework-0.2.5/execute_data_app.sh +21 -0
  21. 3phi_framework-0.2.5/pyproject.toml +52 -0
  22. 3phi_framework-0.2.5/requirements-ci.txt +9 -0
  23. 3phi_framework-0.2.5/requirements.txt +15 -0
  24. 3phi_framework-0.2.5/setup.cfg +4 -0
  25. 3phi_framework-0.2.5/src/3phi_framework.egg-info/PKG-INFO +214 -0
  26. 3phi_framework-0.2.5/src/3phi_framework.egg-info/SOURCES.txt +93 -0
  27. 3phi_framework-0.2.5/src/3phi_framework.egg-info/dependency_links.txt +1 -0
  28. 3phi_framework-0.2.5/src/3phi_framework.egg-info/top_level.txt +1 -0
  29. 3phi_framework-0.2.5/src/threephi_framework/__init__.py +33 -0
  30. 3phi_framework-0.2.5/src/threephi_framework/aggregator.py +72 -0
  31. 3phi_framework-0.2.5/src/threephi_framework/controllers/__init__.py +0 -0
  32. 3phi_framework-0.2.5/src/threephi_framework/controllers/meta.py +480 -0
  33. 3phi_framework-0.2.5/src/threephi_framework/controllers/time_series.py +35 -0
  34. 3phi_framework-0.2.5/src/threephi_framework/controllers/topology.py +256 -0
  35. 3phi_framework-0.2.5/src/threephi_framework/data_apps/base.py +73 -0
  36. 3phi_framework-0.2.5/src/threephi_framework/data_apps/base_config.py +7 -0
  37. 3phi_framework-0.2.5/src/threephi_framework/data_apps/sm_classifier.py +691 -0
  38. 3phi_framework-0.2.5/src/threephi_framework/data_apps/stat_labeler.py +164 -0
  39. 3phi_framework-0.2.5/src/threephi_framework/data_apps/timeseries_ingestor.py +45 -0
  40. 3phi_framework-0.2.5/src/threephi_framework/data_apps/topology_ingestor.py +45 -0
  41. 3phi_framework-0.2.5/src/threephi_framework/data_extractor/data_extractor.py +3198 -0
  42. 3phi_framework-0.2.5/src/threephi_framework/data_extractor/schemas/phase_measurements/v1.py +36 -0
  43. 3phi_framework-0.2.5/src/threephi_framework/db/db.py +63 -0
  44. 3phi_framework-0.2.5/src/threephi_framework/db_connector.py +254 -0
  45. 3phi_framework-0.2.5/src/threephi_framework/dtu/sm_classifier.py +363 -0
  46. 3phi_framework-0.2.5/src/threephi_framework/dtu/stat_labeler.py +676 -0
  47. 3phi_framework-0.2.5/src/threephi_framework/models/__init__.py +0 -0
  48. 3phi_framework-0.2.5/src/threephi_framework/models/base.py +5 -0
  49. 3phi_framework-0.2.5/src/threephi_framework/models/meta/meta_schema_mixin.py +5 -0
  50. 3phi_framework-0.2.5/src/threephi_framework/models/meta/meter.py +22 -0
  51. 3phi_framework-0.2.5/src/threephi_framework/models/meta/run_result.py +66 -0
  52. 3phi_framework-0.2.5/src/threephi_framework/models/topology/__init__.py +0 -0
  53. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/__init__.py +0 -0
  54. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/cabinet.py +14 -0
  55. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/delivery_point.py +20 -0
  56. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/feeder.py +20 -0
  57. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/meter.py +25 -0
  58. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/secondary_substation.py +17 -0
  59. 3phi_framework-0.2.5/src/threephi_framework/models/topology/assets/transformer.py +22 -0
  60. 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/cable.py +26 -0
  61. 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/edge.py +42 -0
  62. 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/edge_cable.py +22 -0
  63. 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/node.py +43 -0
  64. 3phi_framework-0.2.5/src/threephi_framework/models/topology/graph/topology_version.py +21 -0
  65. 3phi_framework-0.2.5/src/threephi_framework/models/topology/lv_schema_mixin.py +5 -0
  66. 3phi_framework-0.2.5/src/threephi_framework/models/topology/node_type_enum.py +10 -0
  67. 3phi_framework-0.2.5/src/threephi_framework/models/topology/utilities.py +26 -0
  68. 3phi_framework-0.2.5/src/threephi_framework/object_storage/base_connector.py +266 -0
  69. 3phi_framework-0.2.5/src/threephi_framework/object_storage/s3_connector.py +220 -0
  70. 3phi_framework-0.2.5/src/threephi_framework/resources/__init__.py +0 -0
  71. 3phi_framework-0.2.5/src/threephi_framework/resources/base.py +30 -0
  72. 3phi_framework-0.2.5/src/threephi_framework/resources/meta/meter.py +38 -0
  73. 3phi_framework-0.2.5/src/threephi_framework/resources/meta/run_result.py +106 -0
  74. 3phi_framework-0.2.5/src/threephi_framework/resources/sanity.py +22 -0
  75. 3phi_framework-0.2.5/src/threephi_framework/resources/staging.py +54 -0
  76. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/__init__.py +0 -0
  77. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/cabinet.py +17 -0
  78. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/delivery_point.py +43 -0
  79. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/feeder.py +27 -0
  80. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/meter.py +308 -0
  81. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/secondary_substation.py +24 -0
  82. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/assets/transformer.py +27 -0
  83. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/__init__.py +0 -0
  84. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/cable.py +69 -0
  85. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/edge.py +313 -0
  86. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/edge_cable.py +46 -0
  87. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/node.py +130 -0
  88. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/graph/topology_version.py +19 -0
  89. 3phi_framework-0.2.5/src/threephi_framework/resources/topology/topology_export.py +283 -0
  90. 3phi_framework-0.2.5/src/threephi_framework/schemas/v1/phase_measurements.py +36 -0
  91. 3phi_framework-0.2.5/src/threephi_framework/schemas/v1/topology.py +96 -0
  92. 3phi_framework-0.2.5/src/threephi_framework/topology_cleaner.py +432 -0
  93. 3phi_framework-0.2.5/src/threephi_framework/util/util.py +9 -0
  94. 3phi_framework-0.2.5/tests/data/lv_topology.csv +25500 -0
  95. 3phi_framework-0.2.5/tests/data/meter_cabinet_connection.csv +62346 -0
@@ -0,0 +1,11 @@
1
+ # object storage
2
+ S3_ENDPOINT_URL=http://localhost:19000
3
+ S3_ACCESS_KEY=minioadmin
4
+ S3_SECRET_KEY=minioadmin
5
+ # db connection
6
+ DB_TYPE=POSTGRES
7
+ DB_USER=postgres
8
+ DB_PASSWORD=password
9
+ DB_NAME=3phi-db
10
+ DB_HOST=localhost
11
+ DB_PORT=5432
@@ -0,0 +1,53 @@
1
+ name: Build and deploy Sphinx docs
2
+
3
+ on:
4
+ push:
5
+ branches: [ main ]
6
+ workflow_dispatch:
7
+
8
+ permissions:
9
+ contents: read
10
+ pages: write
11
+ id-token: write
12
+
13
+ concurrency:
14
+ group: "pages"
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ build:
19
+ runs-on: ubuntu-latest
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+
23
+ - uses: actions/setup-python@v5
24
+ with:
25
+ python-version: "3.12"
26
+
27
+ - name: Install deps
28
+ run: |
29
+ python -m pip install --upgrade pip
30
+ pip install -r docs/requirements.txt
31
+ pip install -e .
32
+
33
+ - name: Build HTML
34
+ run: |
35
+ sphinx-build -b html -W --keep-going -T docs docs/_build/html
36
+
37
+ - name: Configure Pages
38
+ uses: actions/configure-pages@v5
39
+
40
+ - name: Upload Pages artifact
41
+ uses: actions/upload-pages-artifact@v3
42
+ with:
43
+ path: docs/_build/html
44
+
45
+ deploy:
46
+ needs: build
47
+ runs-on: ubuntu-latest
48
+ environment:
49
+ name: github-pages
50
+ url: ${{ steps.deployment.outputs.page_url }}
51
+ steps:
52
+ - id: deployment
53
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,10 @@
1
+ .idea
2
+ dist
3
+ **/__pycache__/
4
+ src/3phi_framework.egg-info
5
+ .DS_Store
6
+ docs/_build
7
+ docker/tmp
8
+ docker/db/init/3_db_seed.sql
9
+ docker/object_storage/*
10
+ **.bak
@@ -0,0 +1,10 @@
1
+ ## Summary
2
+ Why is this change needed?
3
+
4
+ ## Issue Link
5
+ <put link here or remove if not available>
6
+
7
+ ## Checklist
8
+ - [ ] Linked issue if available
9
+ - [ ] Updated README.md
10
+ - [ ] No sensitive data
@@ -0,0 +1,106 @@
1
+ stages: [lint, style, build, publish]
2
+ image: python:3.12
3
+
4
+ variables:
5
+ PIP_DISABLE_PIP_VERSION_CHECK: "1"
6
+ PIP_NO_CACHE_DIR: "1"
7
+ # Ensure tags are available to the job
8
+ GIT_STRATEGY: fetch
9
+ GIT_DEPTH: "0"
10
+
11
+ # ---------- Ruff lint and format with MR annotations ----------
12
+ .ruff:
13
+ image: python:3.11-slim
14
+ before_script:
15
+ - python -V
16
+ - pip install --no-cache-dir ruff==0.6.8
17
+
18
+ ruff_lint:
19
+ extends: .ruff
20
+ stage: lint
21
+ rules:
22
+ - when: always
23
+ script:
24
+ - ruff check --output-format=junit --output-file=ruff-junit.xml .
25
+ artifacts:
26
+ when: always
27
+ reports:
28
+ junit: ruff-junit.xml
29
+
30
+ ruff_format_check:
31
+ extends: .ruff
32
+ stage: style
33
+ rules:
34
+ - when: always
35
+ script:
36
+ - ruff format --check .
37
+
38
+ # ---------- Release (tag) ----------
39
+ build_release:
40
+ stage: build
41
+ needs: [ruff_lint, ruff_format_check]
42
+ rules:
43
+ - if: '$CI_COMMIT_TAG'
44
+ script:
45
+ - |
46
+ python -m pip install -U pip build
47
+ python -m build
48
+ artifacts:
49
+ when: always
50
+ paths:
51
+ - dist/
52
+
53
+ publish_release:
54
+ stage: publish
55
+ needs: [build_release]
56
+ rules:
57
+ - if: '$CI_COMMIT_TAG'
58
+ script:
59
+ - |
60
+ python -m pip install -U twine
61
+ TWINE_USERNAME=gitlab-ci-token TWINE_PASSWORD="$CI_JOB_TOKEN" \
62
+ python -m twine upload \
63
+ --repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" \
64
+ dist/*
65
+
66
+ # ---------- Manual PR build (dev) ----------
67
+ build_pr_manual:
68
+ stage: build
69
+ needs: [ruff_lint, ruff_format_check]
70
+ rules:
71
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
72
+ when: always
73
+ script:
74
+ - |
75
+ python -m pip install -U pip build
76
+ # Ensure we have tags (safety if runner ignores GIT_DEPTH)
77
+ git fetch --force --prune --tags
78
+ # Get latest semver-like tag (vX.Y.Z or X.Y.Z); fallback to 0.0.0
79
+ BASE_TAG="$(git tag --list --sort=-v:refname | grep -E '^(v)?[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 || true)"
80
+ if [ -z "$BASE_TAG" ]; then
81
+ BASE_VERSION="0.0.0"
82
+ else
83
+ BASE_VERSION="${BASE_TAG#v}" # strip leading v if present
84
+ fi
85
+ export SETUPTOOLS_SCM_PRETEND_VERSION="${BASE_VERSION}.dev${CI_PIPELINE_IID}+pr.${CI_MERGE_REQUEST_IID}.sha${CI_COMMIT_SHORT_SHA}"
86
+ echo "Building version: $SETUPTOOLS_SCM_PRETEND_VERSION"
87
+ python -m build
88
+ artifacts:
89
+ when: always
90
+ expire_in: 7 days
91
+ paths:
92
+ - dist/
93
+
94
+ publish_pr_manual:
95
+ stage: publish
96
+ needs: [build_pr_manual]
97
+ rules:
98
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
99
+ when: manual
100
+ script:
101
+ - |
102
+ python -m pip install -U twine
103
+ TWINE_USERNAME=gitlab-ci-token TWINE_PASSWORD="$CI_JOB_TOKEN" \
104
+ python -m twine upload \
105
+ --repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" \
106
+ dist/*
@@ -0,0 +1,7 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.14.1
4
+ hooks:
5
+ - id: ruff-check
6
+ args: [--fix, --exit-non-zero-on-fix, --diff]
7
+ - id: ruff-format
@@ -0,0 +1,214 @@
1
+ Metadata-Version: 2.4
2
+ Name: 3phi-framework
3
+ Version: 0.2.5
4
+ Summary: Framework for 3phi project
5
+ Author-email: Christian Fotteler <chris@inilab.dk>
6
+ License: MIT
7
+ Requires-Python: >=3.12
8
+ Description-Content-Type: text/markdown
9
+
10
+ # 3phi Framework
11
+
12
+ Utility classes for **DB** access, **S3** interactions, and **data processing** via a `DataExtractor`.
13
+ Distributed privately via GitLab’s Package Registry (private PyPI).
14
+
15
+ > **Install name:** `3phi-framework`
16
+ > **Import package:** `threephi_framework`
17
+
18
+ ---
19
+
20
+ ## Features (MVP)
21
+
22
+ - **DBConnector** – simple connector + query helpers.
23
+ - **S3Connector** – upload/download and safe path helpers.
24
+ - **DataExtractor** – API to interact with the Data in S3.
25
+
26
+ > This is an early version. API may evolve until `v1.0`. See “Versioning” below.
27
+
28
+ ---
29
+
30
+ ## Requirements
31
+
32
+ - Python **3.9+**
33
+ - Access to the GitLab **Package Registry** for this project or group.
34
+
35
+ ---
36
+
37
+ ## Build a new version
38
+
39
+ Write & commit your code and then tag it:
40
+ ```
41
+ git tag -a vx.x.x -m "vx.x.x"
42
+ git push --tags
43
+ ```
44
+
45
+
46
+ ## Installation (GitLab private PyPI)
47
+
48
+ Create a **Deploy Token** with scope `read_package_registry` in this project.
49
+
50
+ ```bash
51
+ pip install \
52
+ --index-url "https://<DEPLOY_USER>:<DEPLOY_TOKEN>@gitlab.com/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
53
+ 3phi-framework
54
+ ```
55
+
56
+ ### Other ways to install
57
+ Replace placeholders like <PROJECT_ID>, <USERNAME>, <TOKEN>, and use the gitlab domain (gitlab.3pi-dev.io).
58
+
59
+ 1) Quick one-liner (private project, for code completion etc.)
60
+
61
+ Use a Deploy Token (Settings → Repository → Deploy tokens) with read_package_registry, or a PAT with read_api.
62
+ ```
63
+ # safer: export creds so they don't land in shell history
64
+ export GL_USER="gitlab+deploy-token-<id>" # or your GitLab username if using a PAT
65
+ export GL_TOKEN="<token>"
66
+
67
+ pip install \
68
+ --index-url "https://$GL_USER:$GL_TOKEN@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
69
+ --extra-index-url "https://pypi.org/simple" \
70
+ 3phi-framework==0.1.2
71
+ ```
72
+ Package name to install: 3phi-framework
73
+ Module you import: threephi_framework
74
+ --extra-index-url https://pypi.org/simple lets pip resolve any third-party deps from PyPI.
75
+
76
+ 2) Put it in requirements.txt (useful for docker image builds)
77
+
78
+ At the top of requirements.txt:
79
+ ```
80
+ --index-url https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
81
+ --extra-index-url https://pypi.org/simple
82
+ 3phi-framework==0.1.2
83
+ ```
84
+ Then install as you normally would.
85
+
86
+ 3) Persist credentials in pip.conf (developer machines)
87
+
88
+ Linux/macOS: ~/.pip/pip.conf
89
+ Windows: %APPDATA%\pip\pip.ini
90
+ ```
91
+ [global]
92
+ index-url = https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
93
+ extra-index-url = https://pypi.org/simple
94
+ ```
95
+ Then you can just:
96
+ ```
97
+ pip install 3phi-framework==0.1.2
98
+ ```
99
+
100
+ ---
101
+
102
+ ## Quickstart
103
+
104
+ The framework is set up for local development as well as for being used in a deployment.
105
+ To set up your environment for local development, follow these steps:
106
+
107
+ ### Set up virtual environment
108
+
109
+ [execute_data_app.sh](execute_data_app.sh) expects a virtual environment to be set up under [.venv]. See the [python docs](https://docs.python.org/3/library/venv.html) on how to set it up.
110
+
111
+ ### Seed data
112
+
113
+ Obtain seed data for the database and the object storage and copy it to:
114
+ - [3_db_seed.sql](docker/db/init/3_db_seed.sql): This should be a sql dump/snapshot of a working 3phi Database, PSQL will automatically seed the DB when it is created using docker compose.
115
+ - [3phi](docker/object_storage/3phi): This should be a copy of a bucket from a working object storage. It will be mounted in the minio object storage as a bucket.
116
+
117
+ ### Spin up DB and Object Storage
118
+
119
+ Navigate to [docker](./docker) and run `docker compose up -d`.
120
+ This will bring up a local DB and a MinIO Object Storage seeded with the data you provided.
121
+
122
+ ### Run a data app locally
123
+
124
+ Use the utility script [execute_data_app.sh](execute_data_app.sh) and pass the data app name as an argument, e.g.:
125
+ ```
126
+ ./exectue_data_app.sh sm_classifier
127
+ ```
128
+ In case the script is not executable, make it executable:
129
+ ```
130
+ chmod +x execute_data_app.sh
131
+ ```
132
+
133
+ The script will install the dependencies in [requirements.txt](requirements.txt) in your virtual environment, set up environment variables as they are listed in [.env](.env) and execute the data app as a python module.
134
+
135
+ ## Contibuting to 3phi Framework
136
+
137
+ The framework is currently not open-source, but the aim is to open-source it eventually.
138
+
139
+ ### Commit Message Guidelines
140
+
141
+ Commit Messages should follow the recommendations of [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/), supplemented by the [Angular Convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines). Every commit must contain `type` and `description`, while `scope`, `body` and `footer` are optional:
142
+
143
+ ```
144
+ <type>(<scope>): <description>
145
+ <BLANK LINE>
146
+ <body>
147
+ <BLANK LINE>
148
+ <footer>
149
+ ```
150
+
151
+ #### Type
152
+ `type` is mandatory, and should be one of the following:
153
+
154
+ - `feat`: A new feature
155
+ - `fix`: A bug fix
156
+ - `build`: Changes that affect the build system or external dependencies
157
+ - `ci`: Changes to our CI configuration files and scripts
158
+ - `docs`: Documentation only changes
159
+ - `perf`: A code change that improves performance
160
+ - `refactor`: A code change that neither fixes a bug nor adds a feature
161
+ - `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
162
+ - `test`: Adding missing tests or correcting existing tests
163
+
164
+ #### Scope
165
+ `scope`is optional, but if included it should be a noun that gives context to the part of the codebase that is affected by the commit. Please do not use issue identifiers as scope.
166
+
167
+ #### Description
168
+ `description` is mandatory, and provides a short and concise description of the change. Be mindful of the following:
169
+
170
+ - use the imperative, present tense: "change" not "changed" nor "changes"
171
+ - don't capitalize the first letter
172
+ - no dot (.) at the end
173
+
174
+ #### Body
175
+ `body` is optional, but if included, it should use the imperative, present tense: "change" not "changed" nor "changes" (just as in `description`). The body should include the motivation for the change and contrast this with previous behavior.
176
+
177
+ #### Footer
178
+ `footer` is optional, but if included, it should use `-`in place of whitespace to distinguish it from a `body`or multiline `body`. It is the place to reference git issues that this commit closes.
179
+
180
+ #### Breaking Change
181
+ A commit that make incompatible API changes, should be denoted `Breaking Change`, either by a `!` right after `scope` or by adding `Breaking Change` or `Breaking-Change` to `footer`.
182
+
183
+ ### Linter and formatter
184
+ [Ruff](https://docs.astral.sh/ruff/linter/) is used for linting and formatting in both local development (pre-commit) and CI, with a shared configuration in pyproject.toml.
185
+
186
+ #### Ruff in CI pipeline
187
+ - Runs on pushes/merge requests in GitLab.
188
+ - Checks only (no auto-fix). If any check fails, the pipeline fails and the change cannot be merged.
189
+
190
+ #### Ruff in pre-commit hook
191
+ - Runs locally on commit (and optionally on push).
192
+ - Applies fixes and then fails if it changed files so you can review and commit the updates.
193
+ - First-time setup:
194
+ ```
195
+ pre-commit install
196
+ # optional: also run on push
197
+ pre-commit install --hook-type pre-push
198
+ ```
199
+ - Manual sweep of the whole repo:
200
+ ```
201
+ pre-commit run --all-files
202
+ ```
203
+
204
+
205
+ ## Documentation
206
+
207
+ This Project uses Sphinx for API docs. From the [docs](./docs/) run:
208
+ ```
209
+ make html
210
+ ```
211
+ to generate the docs. Then you can open the [docs index](./docs/_build/html/index.html) in your browser.
212
+
213
+ We leverage [autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to generate the documentation from docstrings in code.
214
+ To configure which classes are included in the API docs, edit and extend [index.rst](./docs/api/index.rst).
@@ -0,0 +1,205 @@
1
+ # 3phi Framework
2
+
3
+ Utility classes for **DB** access, **S3** interactions, and **data processing** via a `DataExtractor`.
4
+ Distributed privately via GitLab’s Package Registry (private PyPI).
5
+
6
+ > **Install name:** `3phi-framework`
7
+ > **Import package:** `threephi_framework`
8
+
9
+ ---
10
+
11
+ ## Features (MVP)
12
+
13
+ - **DBConnector** – simple connector + query helpers.
14
+ - **S3Connector** – upload/download and safe path helpers.
15
+ - **DataExtractor** – API to interact with the Data in S3.
16
+
17
+ > This is an early version. API may evolve until `v1.0`. See “Versioning” below.
18
+
19
+ ---
20
+
21
+ ## Requirements
22
+
23
+ - Python **3.9+**
24
+ - Access to the GitLab **Package Registry** for this project or group.
25
+
26
+ ---
27
+
28
+ ## Build a new version
29
+
30
+ Write & commit your code and then tag it:
31
+ ```
32
+ git tag -a vx.x.x -m "vx.x.x"
33
+ git push --tags
34
+ ```
35
+
36
+
37
+ ## Installation (GitLab private PyPI)
38
+
39
+ Create a **Deploy Token** with scope `read_package_registry` in this project.
40
+
41
+ ```bash
42
+ pip install \
43
+ --index-url "https://<DEPLOY_USER>:<DEPLOY_TOKEN>@gitlab.com/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
44
+ 3phi-framework
45
+ ```
46
+
47
+ ### Other ways to install
48
+ Replace placeholders like <PROJECT_ID>, <USERNAME>, <TOKEN>, and use the gitlab domain (gitlab.3pi-dev.io).
49
+
50
+ 1) Quick one-liner (private project, for code completion etc.)
51
+
52
+ Use a Deploy Token (Settings → Repository → Deploy tokens) with read_package_registry, or a PAT with read_api.
53
+ ```
54
+ # safer: export creds so they don't land in shell history
55
+ export GL_USER="gitlab+deploy-token-<id>" # or your GitLab username if using a PAT
56
+ export GL_TOKEN="<token>"
57
+
58
+ pip install \
59
+ --index-url "https://$GL_USER:$GL_TOKEN@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple" \
60
+ --extra-index-url "https://pypi.org/simple" \
61
+ 3phi-framework==0.1.2
62
+ ```
63
+ Package name to install: 3phi-framework
64
+ Module you import: threephi_framework
65
+ --extra-index-url https://pypi.org/simple lets pip resolve any third-party deps from PyPI.
66
+
67
+ 2) Put it in requirements.txt (useful for docker image builds)
68
+
69
+ At the top of requirements.txt:
70
+ ```
71
+ --index-url https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
72
+ --extra-index-url https://pypi.org/simple
73
+ 3phi-framework==0.1.2
74
+ ```
75
+ Then install as you normally would.
76
+
77
+ 3) Persist credentials in pip.conf (developer machines)
78
+
79
+ Linux/macOS: ~/.pip/pip.conf
80
+ Windows: %APPDATA%\pip\pip.ini
81
+ ```
82
+ [global]
83
+ index-url = https://<USER>:<TOKEN>@gitlab.3pi-dev.io/api/v4/projects/<PROJECT_ID>/packages/pypi/simple
84
+ extra-index-url = https://pypi.org/simple
85
+ ```
86
+ Then you can just:
87
+ ```
88
+ pip install 3phi-framework==0.1.2
89
+ ```
90
+
91
+ ---
92
+
93
+ ## Quickstart
94
+
95
+ The framework is set up for local development as well as for being used in a deployment.
96
+ To set up your environment for local development, follow these steps:
97
+
98
+ ### Set up virtual environment
99
+
100
+ [execute_data_app.sh](execute_data_app.sh) expects a virtual environment to be set up under [.venv]. See the [python docs](https://docs.python.org/3/library/venv.html) on how to set it up.
101
+
102
+ ### Seed data
103
+
104
+ Obtain seed data for the database and the object storage and copy it to:
105
+ - [3_db_seed.sql](docker/db/init/3_db_seed.sql): This should be a sql dump/snapshot of a working 3phi Database, PSQL will automatically seed the DB when it is created using docker compose.
106
+ - [3phi](docker/object_storage/3phi): This should be a copy of a bucket from a working object storage. It will be mounted in the minio object storage as a bucket.
107
+
108
+ ### Spin up DB and Object Storage
109
+
110
+ Navigate to [docker](./docker) and run `docker compose up -d`.
111
+ This will bring up a local DB and a MinIO Object Storage seeded with the data you provided.
112
+
113
+ ### Run a data app locally
114
+
115
+ Use the utility script [execute_data_app.sh](execute_data_app.sh) and pass the data app name as an argument, e.g.:
116
+ ```
117
+ ./exectue_data_app.sh sm_classifier
118
+ ```
119
+ In case the script is not executable, make it executable:
120
+ ```
121
+ chmod +x execute_data_app.sh
122
+ ```
123
+
124
+ The script will install the dependencies in [requirements.txt](requirements.txt) in your virtual environment, set up environment variables as they are listed in [.env](.env) and execute the data app as a python module.
125
+
126
+ ## Contibuting to 3phi Framework
127
+
128
+ The framework is currently not open-source, but the aim is to open-source it eventually.
129
+
130
+ ### Commit Message Guidelines
131
+
132
+ Commit Messages should follow the recommendations of [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/), supplemented by the [Angular Convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines). Every commit must contain `type` and `description`, while `scope`, `body` and `footer` are optional:
133
+
134
+ ```
135
+ <type>(<scope>): <description>
136
+ <BLANK LINE>
137
+ <body>
138
+ <BLANK LINE>
139
+ <footer>
140
+ ```
141
+
142
+ #### Type
143
+ `type` is mandatory, and should be one of the following:
144
+
145
+ - `feat`: A new feature
146
+ - `fix`: A bug fix
147
+ - `build`: Changes that affect the build system or external dependencies
148
+ - `ci`: Changes to our CI configuration files and scripts
149
+ - `docs`: Documentation only changes
150
+ - `perf`: A code change that improves performance
151
+ - `refactor`: A code change that neither fixes a bug nor adds a feature
152
+ - `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
153
+ - `test`: Adding missing tests or correcting existing tests
154
+
155
+ #### Scope
156
+ `scope`is optional, but if included it should be a noun that gives context to the part of the codebase that is affected by the commit. Please do not use issue identifiers as scope.
157
+
158
+ #### Description
159
+ `description` is mandatory, and provides a short and concise description of the change. Be mindful of the following:
160
+
161
+ - use the imperative, present tense: "change" not "changed" nor "changes"
162
+ - don't capitalize the first letter
163
+ - no dot (.) at the end
164
+
165
+ #### Body
166
+ `body` is optional, but if included, it should use the imperative, present tense: "change" not "changed" nor "changes" (just as in `description`). The body should include the motivation for the change and contrast this with previous behavior.
167
+
168
+ #### Footer
169
+ `footer` is optional, but if included, it should use `-`in place of whitespace to distinguish it from a `body`or multiline `body`. It is the place to reference git issues that this commit closes.
170
+
171
+ #### Breaking Change
172
+ A commit that make incompatible API changes, should be denoted `Breaking Change`, either by a `!` right after `scope` or by adding `Breaking Change` or `Breaking-Change` to `footer`.
173
+
174
+ ### Linter and formatter
175
+ [Ruff](https://docs.astral.sh/ruff/linter/) is used for linting and formatting in both local development (pre-commit) and CI, with a shared configuration in pyproject.toml.
176
+
177
+ #### Ruff in CI pipeline
178
+ - Runs on pushes/merge requests in GitLab.
179
+ - Checks only (no auto-fix). If any check fails, the pipeline fails and the change cannot be merged.
180
+
181
+ #### Ruff in pre-commit hook
182
+ - Runs locally on commit (and optionally on push).
183
+ - Applies fixes and then fails if it changed files so you can review and commit the updates.
184
+ - First-time setup:
185
+ ```
186
+ pre-commit install
187
+ # optional: also run on push
188
+ pre-commit install --hook-type pre-push
189
+ ```
190
+ - Manual sweep of the whole repo:
191
+ ```
192
+ pre-commit run --all-files
193
+ ```
194
+
195
+
196
+ ## Documentation
197
+
198
+ This Project uses Sphinx for API docs. From the [docs](./docs/) run:
199
+ ```
200
+ make html
201
+ ```
202
+ to generate the docs. Then you can open the [docs index](./docs/_build/html/index.html) in your browser.
203
+
204
+ We leverage [autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to generate the documentation from docstrings in code.
205
+ To configure which classes are included in the API docs, edit and extend [index.rst](./docs/api/index.rst).
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+ rm -rf ./tmp/pgdata
3
+ mkdir ./tmp/pgdata
@@ -0,0 +1,3 @@
1
+ # ./db/init/debug.sh
2
+ #!/bin/bash
3
+ echo "✅ DB INIT SCRIPT IS RUNNING"
@@ -0,0 +1 @@
1
+ CREATE ROLE "threephi_db_user" WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION PASSWORD 'userpass';
@@ -0,0 +1,33 @@
1
+ services:
2
+
3
+ local-db:
4
+ container_name: local-3phi-db
5
+ image: postgres:18-alpine
6
+ environment:
7
+ POSTGRES_USER: postgres
8
+ POSTGRES_PASSWORD: password
9
+ POSTGRES_DB: 3phi-db
10
+ ports:
11
+ - "5432:5432"
12
+ volumes:
13
+ - ./tmp/pgdata:/var/lib/postgresql
14
+ - ./db/init:/docker-entrypoint-initdb.d
15
+
16
+ local-object-storage:
17
+ container_name: local-minio
18
+ image: minio/minio:latest
19
+ command: server /data --console-address ":9001"
20
+ ports:
21
+ - "19000:9000"
22
+ - "19001:9001"
23
+ environment:
24
+ MINIO_ROOT_USER: minioadmin
25
+ MINIO_ROOT_PASSWORD: minioadmin
26
+ volumes:
27
+ - ./object_storage:/data
28
+ healthcheck:
29
+ test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
30
+ interval: 5s
31
+ timeout: 3s
32
+ retries: 20
33
+ restart: unless-stopped
@@ -0,0 +1,20 @@
1
+ # Minimal makefile for Sphinx documentation
2
+ #
3
+
4
+ # You can set these variables from the command line, and also
5
+ # from the environment for the first two.
6
+ SPHINXOPTS ?=
7
+ SPHINXBUILD ?= sphinx-build
8
+ SOURCEDIR = .
9
+ BUILDDIR = _build
10
+
11
+ # Put it first so that "make" without argument is like "make help".
12
+ help:
13
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14
+
15
+ .PHONY: help Makefile
16
+
17
+ # Catch-all target: route all unknown targets to Sphinx using the new
18
+ # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19
+ %: Makefile
20
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
File without changes