lamindb_setup 1.18.2__tar.gz → 1.19.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/.github/workflows/build.yml +320 -294
  2. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/.github/workflows/doc-changes.yml +23 -23
  3. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/.gitignore +118 -118
  4. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/.pre-commit-config.yaml +48 -48
  5. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/LICENSE +201 -201
  6. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/PKG-INFO +4 -2
  7. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/README.md +6 -6
  8. lamindb_setup-1.18.2/docs/reference.md → lamindb_setup-1.19.1/docs/api.md +5 -5
  9. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/changelog.md +654 -654
  10. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/01-init-local-instance.ipynb +144 -144
  11. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/02-connect-local-instance.ipynb +184 -184
  12. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/03-add-managed-storage.ipynb +561 -561
  13. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/04-test-bionty.ipynb +159 -159
  14. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/05-init-hosted-instance.ipynb +159 -159
  15. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/06-connect-hosted-instance.ipynb +232 -232
  16. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/07-keep-artifacts-local.ipynb +266 -266
  17. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/08-test-multi-session.ipynb +261 -261
  18. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/09-test-migrate.ipynb +94 -94
  19. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-cloud/test_notebooks.py +8 -8
  20. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-cache-management.ipynb +405 -405
  21. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-cloud-sync.ipynb +863 -863
  22. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-connect-anonymously.ipynb +88 -88
  23. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-empty-init.ipynb +126 -126
  24. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-import-schema.ipynb +132 -132
  25. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-init-load-local-anonymously.ipynb +148 -140
  26. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-invalid-schema.ipynb +54 -54
  27. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test-sqlite-lock.ipynb +283 -283
  28. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/hub-prod/test_notebooks2.py +8 -8
  29. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/index.md +12 -12
  30. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/docs/notebooks.md +25 -25
  31. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/__init__.py +4 -19
  32. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_cache.py +87 -87
  33. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_check.py +7 -7
  34. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_check_setup.py +131 -131
  35. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_connect_instance.py +443 -438
  36. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_delete.py +155 -151
  37. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_disconnect.py +38 -38
  38. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_django.py +39 -39
  39. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_entry_points.py +19 -19
  40. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_init_instance.py +423 -429
  41. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_migrate.py +331 -327
  42. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_register_instance.py +32 -32
  43. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_schema.py +27 -27
  44. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_schema_metadata.py +451 -451
  45. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_set_managed_storage.py +81 -80
  46. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_setup_user.py +198 -198
  47. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/_silence_loggers.py +46 -46
  48. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/__init__.py +25 -34
  49. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_aws_options.py +276 -266
  50. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_aws_storage.py +57 -55
  51. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_clone.py +50 -50
  52. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_deprecated.py +62 -62
  53. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_docs.py +14 -14
  54. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_client.py +288 -294
  55. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_core.py +0 -2
  56. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_crud.py +247 -247
  57. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_utils.py +100 -100
  58. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_private_django_api.py +80 -80
  59. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings.py +440 -434
  60. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_instance.py +32 -7
  61. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_load.py +162 -159
  62. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_save.py +108 -96
  63. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_storage.py +433 -433
  64. lamindb_setup-1.19.1/lamindb_setup/core/_settings_store.py +162 -0
  65. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_user.py +55 -55
  66. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/_setup_bionty_sources.py +44 -44
  67. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/cloud_sqlite_locker.py +240 -240
  68. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/django.py +414 -413
  69. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/exceptions.py +1 -1
  70. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/hashing.py +134 -134
  71. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/types.py +1 -1
  72. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/upath.py +1031 -1028
  73. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/errors.py +72 -70
  74. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/io.py +423 -416
  75. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/types.py +17 -17
  76. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/noxfile.py +142 -142
  77. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/pyproject.toml +165 -164
  78. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/connectivity/conftest.py +33 -33
  79. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/connectivity/test_proxies_certificates.py +58 -58
  80. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_db_import_export.py +388 -379
  81. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_entry_point.py +76 -76
  82. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_hashing.py +64 -64
  83. lamindb_setup-1.19.1/tests/core/test_settings_store_load_save.py +149 -0
  84. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_storage_access.py +57 -57
  85. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_storage_basis.py +27 -27
  86. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_storage_settings.py +46 -46
  87. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_storage_stats.py +92 -92
  88. {lamindb_setup-1.18.2/tests/storage → lamindb_setup-1.19.1/tests/core}/test_to_url.py +32 -32
  89. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/scripts/script-init-pass-user-no-writes.py +36 -36
  90. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py +26 -26
  91. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_connect_instance.py +181 -180
  92. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_delete_instance.py +10 -10
  93. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_edge_request.py +63 -63
  94. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_fail_managed_storage.py +13 -13
  95. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_init_instance.py +225 -225
  96. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_init_pass_user_no_writes.py +47 -47
  97. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_login.py +80 -80
  98. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-cloud/test_set_storage.py +15 -15
  99. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-local/README.md +7 -7
  100. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-local/conftest.py +210 -210
  101. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-local/scripts/script-connect-fine-grained-access.py +78 -71
  102. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-local/test_all.py +358 -358
  103. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-local/test_update_schema_in_hub.py +248 -248
  104. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/conftest.py +24 -24
  105. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_aws_options_manager.py +18 -18
  106. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_django.py +5 -5
  107. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_global_settings.py +80 -80
  108. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_migrate.py +13 -13
  109. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_switch_and_fallback_env.py +50 -50
  110. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/tests/hub-prod/test_upath.py +86 -86
  111. lamindb_setup-1.19.1/tests/profiling/connect_cli.py +3 -0
  112. lamindb_setup-1.19.1/tests/profiling/import_lamindb_setup.py +1 -0
  113. lamindb_setup-1.19.1/tests/profiling/print_lamindb_setup_settings.py +3 -0
  114. lamindb_setup-1.18.2/lamindb_setup/core/_settings_store.py +0 -92
  115. lamindb_setup-1.18.2/tests/storage/conftest.py +0 -10
  116. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/core/lamin.db.gz +0 -0
  117. {lamindb_setup-1.18.2 → lamindb_setup-1.19.1}/lamindb_setup/py.typed +0 -0
@@ -1,294 +1,320 @@
1
- name: build
2
-
3
- on:
4
- push:
5
- branches: [main, staging]
6
- pull_request:
7
- branches: [main]
8
- repository_dispatch:
9
- types: [build]
10
-
11
- jobs:
12
- # tests only on production hub
13
- hub-prod:
14
- runs-on: ubuntu-latest
15
- timeout-minutes: 13
16
- steps:
17
- - uses: actions/checkout@v4
18
- - uses: actions/setup-python@v6
19
- with:
20
- python-version: "3.10"
21
- cache: "pip"
22
- - uses: aws-actions/configure-aws-credentials@v4
23
- with:
24
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
25
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
26
- aws-region: eu-central-1
27
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
28
- - run: nox -s "install(group='hub-prod')"
29
- - run: nox -s "build(lamin_env='prod', group='hub-prod')"
30
- - uses: actions/upload-artifact@v4
31
- with:
32
- name: coverage--hub-prod
33
- path: .coverage
34
- include-hidden-files: true
35
-
36
- # tests both on production and staging hub
37
- hub-cloud:
38
- runs-on: ubuntu-latest
39
- strategy:
40
- fail-fast: false
41
- matrix:
42
- include:
43
- - lamin_env: "prod"
44
- python-version: "3.11"
45
- - lamin_env: "staging"
46
- python-version: "3.10"
47
- timeout-minutes: 13
48
- steps:
49
- - uses: aws-actions/configure-aws-credentials@v4
50
- with:
51
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
52
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53
- aws-region: eu-central-1
54
- - uses: actions/checkout@v4
55
- - uses: actions/setup-python@v6
56
- with:
57
- python-version: ${{ matrix.python-version }}
58
- cache: "pip"
59
- cache-dependency-path: ".github/workflows/build.yml"
60
- - uses: actions/checkout@v4
61
- with:
62
- repository: laminlabs/laminhub
63
- token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
64
- path: laminhub
65
- ref: main
66
- - uses: actions/cache@v4
67
- with:
68
- path: ~/.cache/pre-commit
69
- key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
70
- - id: cache-postgres
71
- uses: actions/cache@v4
72
- with:
73
- path: ~/postgres.tar
74
- key: cache-postgres-0
75
- - if: steps.cache-postgres.outputs.cache-hit != 'true'
76
- run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
77
- - if: steps.cache-postgres.outputs.cache-hit == 'true'
78
- run: docker image load --input ~/postgres.tar
79
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
80
- # account for in a different job
81
- # - uses: "google-github-actions/auth@v0"
82
- # with:
83
- # credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
84
- # - uses: "google-github-actions/setup-gcloud@v0"
85
- - run: nox -s "install(group='hub-cloud')"
86
- - run: nox -s "build(lamin_env='${{ matrix.lamin_env }}', group='hub-cloud')"
87
- - uses: actions/upload-artifact@v4
88
- if: ${{ matrix.lamin_env == 'prod' }}
89
- with:
90
- name: coverage--hub-cloud
91
- path: .coverage
92
- include-hidden-files: true
93
-
94
- # test user access to storage
95
- storage:
96
- runs-on: ubuntu-latest
97
- timeout-minutes: 13
98
- steps:
99
- - uses: actions/checkout@v4
100
- - uses: actions/setup-python@v6
101
- with:
102
- python-version: "3.10"
103
- cache: "pip"
104
- cache-dependency-path: ".github/workflows/build.yml"
105
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
106
- - run: nox -s "install(group='storage')"
107
- - run: nox -s lint
108
- - run: nox -s storage
109
- env:
110
- TMP_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
111
- TMP_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
112
- - uses: actions/upload-artifact@v4
113
- with:
114
- name: coverage--storage
115
- path: .coverage
116
- include-hidden-files: true
117
-
118
- # test low-level hub functionality
119
- hub-local:
120
- runs-on: ubuntu-latest
121
- timeout-minutes: 13
122
- steps:
123
- - uses: aws-actions/configure-aws-credentials@v4
124
- with:
125
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
126
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
127
- aws-region: eu-central-1
128
- - uses: actions/checkout@v4
129
- - uses: actions/checkout@v4
130
- with:
131
- repository: laminlabs/laminhub
132
- token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
133
- path: laminhub
134
- ref: main
135
- - name: Set env file for local test of edge functions
136
- run: |
137
- touch .env.local
138
- echo "AWS_ACCESS_KEY_ID_HOSTED_S3=${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env.local
139
- echo "AWS_SECRET_ACCESS_KEY_HOSTED_S3=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env.local
140
- working-directory: laminhub/backend/services/central/supabase
141
- - uses: actions/setup-python@v6
142
- with:
143
- python-version: "3.11" # we need to run everything for coverage on 3.11
144
- cache: "pip"
145
- cache-dependency-path: ".github/workflows/build.yml"
146
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
147
- - run: nox -s "install(group='hub-local')"
148
- - id: cache-supabase
149
- uses: actions/cache@v4
150
- with:
151
- path: /var/lib/docker
152
- key: cache-supabase
153
- - uses: supabase/setup-cli@v1
154
- with:
155
- version: "2.6.8"
156
- - run: nox -s hub_local
157
- env:
158
- LAMIN_ENV: "local"
159
- - uses: actions/upload-artifact@v4
160
- with:
161
- name: coverage--hub-local
162
- path: .coverage
163
- include-hidden-files: true
164
-
165
- # test custom proxies and certificates
166
- connectivity:
167
- runs-on: ubuntu-latest
168
- timeout-minutes: 13
169
- steps:
170
- - uses: aws-actions/configure-aws-credentials@v4
171
- with:
172
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
173
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
174
- aws-region: eu-central-1
175
- - uses: actions/checkout@v4
176
- - uses: actions/setup-python@v6
177
- with:
178
- python-version: "3.11"
179
- cache: "pip"
180
- cache-dependency-path: ".github/workflows/build.yml"
181
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
182
- - run: nox -s "install(group='connectivity')"
183
- - name: cache mitmproxy
184
- id: cache-mitmproxy
185
- uses: actions/cache@v4
186
- with:
187
- path: ~/mitmproxy.tar
188
- key: cache-mitmproxy-0
189
- - name: cache mitmproxy miss
190
- if: ${{ steps.cache-mitmproxy.outputs.cache-hit != 'true' }}
191
- run: docker pull mitmproxy/mitmproxy:latest && docker image save mitmproxy/mitmproxy:latest --output ~/mitmproxy.tar
192
- - name: cache mitmproxy use
193
- if: ${{ steps.cache-mitmproxy.outputs.cache-hit == 'true' }}
194
- run: docker image load --input ~/mitmproxy.tar
195
- - name: start mitmproxy
196
- run: |
197
- # Start mitmdump (headless mitmproxy) as a forward proxy on 8080
198
- docker run -d \
199
- --name mitmproxy \
200
- -p 8080:8080 \
201
- mitmproxy/mitmproxy:latest \
202
- mitmdump --mode regular --listen-port 8080
203
- # Give it a few seconds to start and generate the CA
204
- sleep 10
205
- echo "mitmproxy state:"
206
- docker ps -a
207
- echo "Container ~/.mitmproxy contents:"
208
- docker exec mitmproxy ls -l /home/mitmproxy/.mitmproxy || true
209
- echo "Container logs (if it still exited):"
210
- docker logs mitmproxy || true
211
- - name: export mitmproxy certificate
212
- run: |
213
- # Copy CA cert from inside the container to the workspace
214
- docker cp mitmproxy:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./mitmproxy-ca.pem
215
- ls -l ./mitmproxy-ca.pem
216
- - name: run tests
217
- run: nox -s connectivity
218
-
219
- coverage:
220
- needs: [hub-prod, hub-cloud, storage, hub-local]
221
- runs-on: ubuntu-latest
222
- steps:
223
- - uses: actions/checkout@v4
224
- - uses: actions/setup-python@v6
225
- with:
226
- python-version: "3.11"
227
- cache: "pip"
228
- - run: |
229
- pip install coverage[toml]
230
- pip install --no-deps .
231
- - uses: actions/download-artifact@v4
232
- - name: run coverage
233
- run: |
234
- coverage combine coverage--*/.coverage*
235
- coverage report --fail-under=0
236
- coverage xml
237
- - uses: codecov/codecov-action@v2
238
- with:
239
- token: ${{ secrets.CODECOV_TOKEN }}
240
-
241
- docs:
242
- needs: hub-cloud
243
- runs-on: ubuntu-latest
244
- steps:
245
- - uses: aws-actions/configure-aws-credentials@v4
246
- with:
247
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
248
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
249
- aws-region: eu-central-1
250
- - uses: actions/checkout@v4
251
- with:
252
- submodules: recursive
253
- fetch-depth: 0
254
- - name: checkout lndocs
255
- uses: actions/checkout@v4
256
- with:
257
- repository: laminlabs/lndocs
258
- ssh-key: ${{ secrets.READ_LNDOCS }}
259
- path: lndocs
260
- ref: main
261
- - uses: actions/setup-python@v6
262
- with:
263
- python-version: "3.10"
264
- cache: "pip"
265
- cache-dependency-path: ".github/workflows/build.yml"
266
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
267
- - run: nox -s "install(group='docs')"
268
- - uses: actions/download-artifact@v4
269
- - run: nox -s docs
270
- - uses: nwtgck/actions-netlify@v1.2
271
- with:
272
- publish-dir: "_build/html"
273
- production-deploy: ${{ github.event_name == 'push' }}
274
- github-token: ${{ secrets.GITHUB_TOKEN }}
275
- enable-commit-comment: false
276
- env:
277
- NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
278
- NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
279
-
280
- notify:
281
- if: always()
282
- # this here is only for notifying developers of laminhub
283
- # hence, it only needs groups hub-local and hub-cloud
284
- needs: [hub-local, hub-cloud]
285
- runs-on: ubuntu-latest
286
- steps:
287
- - uses: voxmedia/github-action-slack-notify-build@v1
288
- if: ${{ ( needs.hub-local.result == 'failure' || needs.hub-cloud.result == 'failure' ) && github.event_name == 'repository_dispatch' }}
289
- env:
290
- SLACK_BOT_TOKEN: ${{ secrets.SLACK_GITHUB_ACTION }}
291
- with:
292
- channel_id: C05S2C02JHM
293
- status: FAILURE
294
- color: danger
1
+ name: build
2
+
3
+ on:
4
+ push:
5
+ branches: [main, staging]
6
+ pull_request:
7
+ branches: [main]
8
+ repository_dispatch:
9
+ types: [build]
10
+
11
+ jobs:
12
+ # tests only on production hub
13
+ hub-prod:
14
+ runs-on: ubuntu-latest
15
+ timeout-minutes: 13
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v6
19
+ with:
20
+ python-version: "3.10"
21
+ cache: "pip"
22
+ - uses: aws-actions/configure-aws-credentials@v4
23
+ with:
24
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
25
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
26
+ aws-region: eu-central-1
27
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
28
+ - run: nox -s "install(group='hub-prod')"
29
+ - run: nox -s "build(lamin_env='prod', group='hub-prod')"
30
+ - uses: actions/upload-artifact@v4
31
+ with:
32
+ name: coverage--hub-prod
33
+ path: .coverage
34
+ include-hidden-files: true
35
+
36
+ # tests both on production and staging hub
37
+ hub-cloud:
38
+ runs-on: ubuntu-latest
39
+ strategy:
40
+ fail-fast: false
41
+ matrix:
42
+ include:
43
+ - lamin_env: "prod"
44
+ python-version: "3.11"
45
+ - lamin_env: "staging"
46
+ python-version: "3.10"
47
+ timeout-minutes: 13
48
+ steps:
49
+ - uses: aws-actions/configure-aws-credentials@v4
50
+ with:
51
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
52
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53
+ aws-region: eu-central-1
54
+ - uses: actions/checkout@v4
55
+ - uses: actions/setup-python@v6
56
+ with:
57
+ python-version: ${{ matrix.python-version }}
58
+ cache: "pip"
59
+ cache-dependency-path: ".github/workflows/build.yml"
60
+ - uses: actions/checkout@v4
61
+ with:
62
+ repository: laminlabs/laminhub
63
+ token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
64
+ path: laminhub
65
+ ref: main
66
+ - uses: actions/cache@v4
67
+ with:
68
+ path: ~/.cache/pre-commit
69
+ key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
70
+ - id: cache-postgres
71
+ uses: actions/cache@v4
72
+ with:
73
+ path: ~/postgres.tar
74
+ key: cache-postgres-0
75
+ - if: steps.cache-postgres.outputs.cache-hit != 'true'
76
+ run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
77
+ - if: steps.cache-postgres.outputs.cache-hit == 'true'
78
+ run: docker image load --input ~/postgres.tar
79
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
80
+ # account for in a different job
81
+ # - uses: "google-github-actions/auth@v0"
82
+ # with:
83
+ # credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
84
+ # - uses: "google-github-actions/setup-gcloud@v0"
85
+ - run: nox -s "install(group='hub-cloud')"
86
+ - run: nox -s "build(lamin_env='${{ matrix.lamin_env }}', group='hub-cloud')"
87
+ - uses: actions/upload-artifact@v4
88
+ if: ${{ matrix.lamin_env == 'prod' }}
89
+ with:
90
+ name: coverage--hub-cloud
91
+ path: .coverage
92
+ include-hidden-files: true
93
+
94
+ core:
95
+ runs-on: ubuntu-latest
96
+ timeout-minutes: 13
97
+ steps:
98
+ - uses: actions/checkout@v4
99
+ - uses: actions/setup-python@v6
100
+ with:
101
+ python-version: "3.10"
102
+ cache: "pip"
103
+ cache-dependency-path: ".github/workflows/build.yml"
104
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
105
+ - run: nox -s "install(group='core')"
106
+ - run: nox -s lint
107
+ - run: nox -s core
108
+ env:
109
+ TMP_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
110
+ TMP_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
111
+ - uses: actions/upload-artifact@v4
112
+ with:
113
+ name: coverage--core
114
+ path: .coverage
115
+ include-hidden-files: true
116
+
117
+ # test low-level hub functionality
118
+ hub-local:
119
+ runs-on: ubuntu-latest
120
+ timeout-minutes: 13
121
+ steps:
122
+ - uses: aws-actions/configure-aws-credentials@v4
123
+ with:
124
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
125
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
126
+ aws-region: eu-central-1
127
+ - uses: actions/checkout@v4
128
+ - uses: actions/checkout@v4
129
+ with:
130
+ repository: laminlabs/laminhub
131
+ token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
132
+ path: laminhub
133
+ ref: main
134
+ - name: Set env file for local test of edge functions
135
+ run: |
136
+ touch .env.local
137
+ echo "AWS_ACCESS_KEY_ID_HOSTED_S3=${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env.local
138
+ echo "AWS_SECRET_ACCESS_KEY_HOSTED_S3=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env.local
139
+ working-directory: laminhub/backend/services/central/supabase
140
+ - uses: actions/setup-python@v6
141
+ with:
142
+ python-version: "3.11" # we need to run everything for coverage on 3.11
143
+ cache: "pip"
144
+ cache-dependency-path: ".github/workflows/build.yml"
145
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
146
+ - run: nox -s "install(group='hub-local')"
147
+ - id: cache-supabase
148
+ uses: actions/cache@v4
149
+ with:
150
+ path: /var/lib/docker
151
+ key: cache-supabase
152
+ - uses: supabase/setup-cli@v1
153
+ with:
154
+ version: "2.72.7"
155
+ - run: nox -s hub_local
156
+ env:
157
+ LAMIN_ENV: "local"
158
+ - uses: actions/upload-artifact@v4
159
+ with:
160
+ name: coverage--hub-local
161
+ path: .coverage
162
+ include-hidden-files: true
163
+
164
+ # test custom proxies and certificates
165
+ connectivity:
166
+ runs-on: ubuntu-latest
167
+ timeout-minutes: 13
168
+ steps:
169
+ - uses: aws-actions/configure-aws-credentials@v4
170
+ with:
171
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
172
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
173
+ aws-region: eu-central-1
174
+ - uses: actions/checkout@v4
175
+ - uses: actions/setup-python@v6
176
+ with:
177
+ python-version: "3.11"
178
+ cache: "pip"
179
+ cache-dependency-path: ".github/workflows/build.yml"
180
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
181
+ - run: nox -s "install(group='connectivity')"
182
+ - name: cache mitmproxy
183
+ id: cache-mitmproxy
184
+ uses: actions/cache@v4
185
+ with:
186
+ path: ~/mitmproxy.tar
187
+ key: cache-mitmproxy-0
188
+ - name: cache mitmproxy miss
189
+ if: ${{ steps.cache-mitmproxy.outputs.cache-hit != 'true' }}
190
+ run: docker pull mitmproxy/mitmproxy:latest && docker image save mitmproxy/mitmproxy:latest --output ~/mitmproxy.tar
191
+ - name: cache mitmproxy use
192
+ if: ${{ steps.cache-mitmproxy.outputs.cache-hit == 'true' }}
193
+ run: docker image load --input ~/mitmproxy.tar
194
+ - name: start mitmproxy
195
+ run: |
196
+ # Start mitmdump (headless mitmproxy) as a forward proxy on 8080
197
+ docker run -d \
198
+ --name mitmproxy \
199
+ -p 8080:8080 \
200
+ mitmproxy/mitmproxy:latest \
201
+ mitmdump --mode regular --listen-port 8080
202
+ # Give it a few seconds to start and generate the CA
203
+ sleep 10
204
+ echo "mitmproxy state:"
205
+ docker ps -a
206
+ echo "Container ~/.mitmproxy contents:"
207
+ docker exec mitmproxy ls -l /home/mitmproxy/.mitmproxy || true
208
+ echo "Container logs (if it still exited):"
209
+ docker logs mitmproxy || true
210
+ - name: export mitmproxy certificate
211
+ run: |
212
+ # Copy CA cert from inside the container to the workspace
213
+ docker cp mitmproxy:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./mitmproxy-ca.pem
214
+ ls -l ./mitmproxy-ca.pem
215
+ - name: run tests
216
+ run: nox -s connectivity
217
+
218
+ profile:
219
+ runs-on: ubuntu-latest
220
+ timeout-minutes: 10
221
+ env:
222
+ LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
223
+ steps:
224
+ - uses: actions/checkout@v4
225
+ with:
226
+ submodules: recursive
227
+ fetch-depth: 0
228
+ - uses: actions/setup-python@v6
229
+ with:
230
+ python-version: |
231
+ ${{ github.ref == 'refs/heads/release' && '3.11' ||
232
+ '3.13'
233
+ }}
234
+ - run: pip install git+https://github.com/laminlabs/laminci
235
+ - run: uv pip install --system git+https://github.com/laminlabs/lamin-cli
236
+ - run: uv pip install --system git+https://github.com/laminlabs/lamindb
237
+ - run: uv pip install --system git+https://github.com/laminlabs/laminprofiler
238
+ - run: uv pip install --system -e .
239
+ - run: lamin login
240
+ - run: lamin connect laminlabs/lamindata
241
+ - run: laminprofiler check tests/profiling/import_lamindb_setup.py --threshold 0.25
242
+ - run: laminprofiler check tests/profiling/print_lamindb_setup_settings.py --threshold 0.4
243
+ - run: laminprofiler check tests/profiling/connect_cli.py --threshold 2.0
244
+
245
+ coverage:
246
+ needs: [hub-prod, hub-cloud, core, hub-local]
247
+ runs-on: ubuntu-latest
248
+ steps:
249
+ - uses: actions/checkout@v4
250
+ - uses: actions/setup-python@v6
251
+ with:
252
+ python-version: "3.11"
253
+ cache: "pip"
254
+ - run: |
255
+ pip install coverage[toml]
256
+ pip install --no-deps .
257
+ - uses: actions/download-artifact@v4
258
+ - name: run coverage
259
+ run: |
260
+ coverage combine coverage--*/.coverage*
261
+ coverage report --fail-under=0
262
+ coverage xml
263
+ - uses: codecov/codecov-action@v2
264
+ with:
265
+ token: ${{ secrets.CODECOV_TOKEN }}
266
+
267
+ docs:
268
+ needs: hub-cloud
269
+ runs-on: ubuntu-latest
270
+ steps:
271
+ - uses: aws-actions/configure-aws-credentials@v4
272
+ with:
273
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
274
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
275
+ aws-region: eu-central-1
276
+ - uses: actions/checkout@v4
277
+ with:
278
+ submodules: recursive
279
+ fetch-depth: 0
280
+ - name: checkout lndocs
281
+ uses: actions/checkout@v4
282
+ with:
283
+ repository: laminlabs/lndocs
284
+ ssh-key: ${{ secrets.READ_LNDOCS }}
285
+ path: lndocs
286
+ ref: main
287
+ - uses: actions/setup-python@v6
288
+ with:
289
+ python-version: "3.10"
290
+ cache: "pip"
291
+ cache-dependency-path: ".github/workflows/build.yml"
292
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
293
+ - run: nox -s "install(group='docs')"
294
+ - uses: actions/download-artifact@v4
295
+ - run: nox -s docs
296
+ - uses: nwtgck/actions-netlify@v1.2
297
+ with:
298
+ publish-dir: "_build/html"
299
+ production-deploy: ${{ github.event_name == 'push' }}
300
+ github-token: ${{ secrets.GITHUB_TOKEN }}
301
+ enable-commit-comment: false
302
+ env:
303
+ NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
304
+ NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
305
+
306
+ notify:
307
+ if: always()
308
+ # this here is only for notifying developers of laminhub
309
+ # hence, it only needs groups hub-local and hub-cloud
310
+ needs: [hub-local, hub-cloud]
311
+ runs-on: ubuntu-latest
312
+ steps:
313
+ - uses: voxmedia/github-action-slack-notify-build@v1
314
+ if: ${{ ( needs.hub-local.result == 'failure' || needs.hub-cloud.result == 'failure' ) && github.event_name == 'repository_dispatch' }}
315
+ env:
316
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_GITHUB_ACTION }}
317
+ with:
318
+ channel_id: C05S2C02JHM
319
+ status: FAILURE
320
+ color: danger