lamindb_setup 1.19.0__tar.gz → 1.19.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/.github/workflows/build.yml +320 -320
  2. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/.github/workflows/doc-changes.yml +23 -23
  3. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/.gitignore +118 -118
  4. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/.pre-commit-config.yaml +48 -48
  5. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/LICENSE +201 -201
  6. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/PKG-INFO +3 -2
  7. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/README.md +6 -6
  8. lamindb_setup-1.19.0/docs/reference.md → lamindb_setup-1.19.1/docs/api.md +5 -5
  9. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/changelog.md +654 -654
  10. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/01-init-local-instance.ipynb +144 -144
  11. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/02-connect-local-instance.ipynb +184 -184
  12. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/03-add-managed-storage.ipynb +561 -561
  13. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/04-test-bionty.ipynb +159 -159
  14. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/05-init-hosted-instance.ipynb +159 -159
  15. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/06-connect-hosted-instance.ipynb +232 -232
  16. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/07-keep-artifacts-local.ipynb +266 -266
  17. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/08-test-multi-session.ipynb +261 -261
  18. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/09-test-migrate.ipynb +94 -94
  19. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-cloud/test_notebooks.py +8 -8
  20. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-cache-management.ipynb +405 -405
  21. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-cloud-sync.ipynb +863 -863
  22. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-connect-anonymously.ipynb +88 -88
  23. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-empty-init.ipynb +126 -126
  24. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-import-schema.ipynb +132 -132
  25. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-init-load-local-anonymously.ipynb +148 -140
  26. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-invalid-schema.ipynb +54 -54
  27. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test-sqlite-lock.ipynb +283 -283
  28. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/hub-prod/test_notebooks2.py +8 -8
  29. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/index.md +12 -12
  30. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/docs/notebooks.md +25 -25
  31. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/__init__.py +1 -1
  32. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_cache.py +87 -87
  33. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_check.py +7 -7
  34. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_check_setup.py +131 -131
  35. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_connect_instance.py +443 -441
  36. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_delete.py +155 -155
  37. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_disconnect.py +38 -38
  38. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_django.py +39 -39
  39. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_entry_points.py +19 -19
  40. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_init_instance.py +423 -423
  41. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_migrate.py +331 -331
  42. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_register_instance.py +32 -32
  43. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_schema.py +27 -27
  44. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_schema_metadata.py +451 -451
  45. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_set_managed_storage.py +81 -81
  46. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_setup_user.py +198 -198
  47. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/_silence_loggers.py +46 -46
  48. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/__init__.py +25 -34
  49. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_aws_options.py +276 -276
  50. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_aws_storage.py +57 -57
  51. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_clone.py +50 -50
  52. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_deprecated.py +62 -62
  53. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_docs.py +14 -14
  54. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_client.py +288 -288
  55. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_crud.py +247 -247
  56. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_utils.py +100 -100
  57. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_private_django_api.py +80 -80
  58. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings.py +440 -434
  59. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_instance.py +22 -1
  60. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_load.py +162 -162
  61. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_save.py +108 -108
  62. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_storage.py +433 -433
  63. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_store.py +162 -162
  64. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_settings_user.py +55 -55
  65. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_setup_bionty_sources.py +44 -44
  66. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/cloud_sqlite_locker.py +240 -240
  67. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/django.py +414 -413
  68. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/exceptions.py +1 -1
  69. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/hashing.py +134 -134
  70. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/types.py +1 -1
  71. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/upath.py +1031 -1028
  72. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/errors.py +72 -72
  73. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/io.py +423 -423
  74. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/types.py +17 -17
  75. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/noxfile.py +142 -142
  76. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/pyproject.toml +165 -165
  77. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/connectivity/conftest.py +33 -33
  78. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/connectivity/test_proxies_certificates.py +58 -58
  79. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_db_import_export.py +388 -388
  80. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_entry_point.py +76 -76
  81. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_hashing.py +64 -64
  82. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_settings_store_load_save.py +149 -149
  83. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_storage_access.py +57 -57
  84. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_storage_basis.py +27 -27
  85. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_storage_settings.py +46 -46
  86. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_storage_stats.py +92 -92
  87. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/core/test_to_url.py +32 -32
  88. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/scripts/script-init-pass-user-no-writes.py +36 -36
  89. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py +26 -26
  90. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_connect_instance.py +181 -180
  91. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_delete_instance.py +10 -10
  92. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_edge_request.py +63 -63
  93. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_fail_managed_storage.py +13 -13
  94. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_init_instance.py +225 -225
  95. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_init_pass_user_no_writes.py +47 -47
  96. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_login.py +80 -80
  97. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-cloud/test_set_storage.py +15 -15
  98. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-local/README.md +7 -7
  99. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-local/conftest.py +210 -210
  100. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-local/scripts/script-connect-fine-grained-access.py +78 -71
  101. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-local/test_all.py +358 -358
  102. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-local/test_update_schema_in_hub.py +248 -248
  103. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/conftest.py +24 -24
  104. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_aws_options_manager.py +18 -18
  105. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_django.py +5 -5
  106. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_global_settings.py +80 -80
  107. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_migrate.py +13 -13
  108. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_switch_and_fallback_env.py +50 -50
  109. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/hub-prod/test_upath.py +86 -86
  110. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/profiling/connect_cli.py +3 -3
  111. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/profiling/import_lamindb_setup.py +1 -1
  112. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/tests/profiling/print_lamindb_setup_settings.py +3 -3
  113. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/_hub_core.py +0 -0
  114. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/core/lamin.db.gz +0 -0
  115. {lamindb_setup-1.19.0 → lamindb_setup-1.19.1}/lamindb_setup/py.typed +0 -0
@@ -1,320 +1,320 @@
1
- name: build
2
-
3
- on:
4
- push:
5
- branches: [main, staging]
6
- pull_request:
7
- branches: [main]
8
- repository_dispatch:
9
- types: [build]
10
-
11
- jobs:
12
- # tests only on production hub
13
- hub-prod:
14
- runs-on: ubuntu-latest
15
- timeout-minutes: 13
16
- steps:
17
- - uses: actions/checkout@v4
18
- - uses: actions/setup-python@v6
19
- with:
20
- python-version: "3.10"
21
- cache: "pip"
22
- - uses: aws-actions/configure-aws-credentials@v4
23
- with:
24
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
25
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
26
- aws-region: eu-central-1
27
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
28
- - run: nox -s "install(group='hub-prod')"
29
- - run: nox -s "build(lamin_env='prod', group='hub-prod')"
30
- - uses: actions/upload-artifact@v4
31
- with:
32
- name: coverage--hub-prod
33
- path: .coverage
34
- include-hidden-files: true
35
-
36
- # tests both on production and staging hub
37
- hub-cloud:
38
- runs-on: ubuntu-latest
39
- strategy:
40
- fail-fast: false
41
- matrix:
42
- include:
43
- - lamin_env: "prod"
44
- python-version: "3.11"
45
- - lamin_env: "staging"
46
- python-version: "3.10"
47
- timeout-minutes: 13
48
- steps:
49
- - uses: aws-actions/configure-aws-credentials@v4
50
- with:
51
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
52
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53
- aws-region: eu-central-1
54
- - uses: actions/checkout@v4
55
- - uses: actions/setup-python@v6
56
- with:
57
- python-version: ${{ matrix.python-version }}
58
- cache: "pip"
59
- cache-dependency-path: ".github/workflows/build.yml"
60
- - uses: actions/checkout@v4
61
- with:
62
- repository: laminlabs/laminhub
63
- token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
64
- path: laminhub
65
- ref: main
66
- - uses: actions/cache@v4
67
- with:
68
- path: ~/.cache/pre-commit
69
- key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
70
- - id: cache-postgres
71
- uses: actions/cache@v4
72
- with:
73
- path: ~/postgres.tar
74
- key: cache-postgres-0
75
- - if: steps.cache-postgres.outputs.cache-hit != 'true'
76
- run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
77
- - if: steps.cache-postgres.outputs.cache-hit == 'true'
78
- run: docker image load --input ~/postgres.tar
79
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
80
- # account for in a different job
81
- # - uses: "google-github-actions/auth@v0"
82
- # with:
83
- # credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
84
- # - uses: "google-github-actions/setup-gcloud@v0"
85
- - run: nox -s "install(group='hub-cloud')"
86
- - run: nox -s "build(lamin_env='${{ matrix.lamin_env }}', group='hub-cloud')"
87
- - uses: actions/upload-artifact@v4
88
- if: ${{ matrix.lamin_env == 'prod' }}
89
- with:
90
- name: coverage--hub-cloud
91
- path: .coverage
92
- include-hidden-files: true
93
-
94
- core:
95
- runs-on: ubuntu-latest
96
- timeout-minutes: 13
97
- steps:
98
- - uses: actions/checkout@v4
99
- - uses: actions/setup-python@v6
100
- with:
101
- python-version: "3.10"
102
- cache: "pip"
103
- cache-dependency-path: ".github/workflows/build.yml"
104
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
105
- - run: nox -s "install(group='core')"
106
- - run: nox -s lint
107
- - run: nox -s core
108
- env:
109
- TMP_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
110
- TMP_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
111
- - uses: actions/upload-artifact@v4
112
- with:
113
- name: coverage--core
114
- path: .coverage
115
- include-hidden-files: true
116
-
117
- # test low-level hub functionality
118
- hub-local:
119
- runs-on: ubuntu-latest
120
- timeout-minutes: 13
121
- steps:
122
- - uses: aws-actions/configure-aws-credentials@v4
123
- with:
124
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
125
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
126
- aws-region: eu-central-1
127
- - uses: actions/checkout@v4
128
- - uses: actions/checkout@v4
129
- with:
130
- repository: laminlabs/laminhub
131
- token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
132
- path: laminhub
133
- ref: main
134
- - name: Set env file for local test of edge functions
135
- run: |
136
- touch .env.local
137
- echo "AWS_ACCESS_KEY_ID_HOSTED_S3=${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env.local
138
- echo "AWS_SECRET_ACCESS_KEY_HOSTED_S3=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env.local
139
- working-directory: laminhub/backend/services/central/supabase
140
- - uses: actions/setup-python@v6
141
- with:
142
- python-version: "3.11" # we need to run everything for coverage on 3.11
143
- cache: "pip"
144
- cache-dependency-path: ".github/workflows/build.yml"
145
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
146
- - run: nox -s "install(group='hub-local')"
147
- - id: cache-supabase
148
- uses: actions/cache@v4
149
- with:
150
- path: /var/lib/docker
151
- key: cache-supabase
152
- - uses: supabase/setup-cli@v1
153
- with:
154
- version: "2.72.7"
155
- - run: nox -s hub_local
156
- env:
157
- LAMIN_ENV: "local"
158
- - uses: actions/upload-artifact@v4
159
- with:
160
- name: coverage--hub-local
161
- path: .coverage
162
- include-hidden-files: true
163
-
164
- # test custom proxies and certificates
165
- connectivity:
166
- runs-on: ubuntu-latest
167
- timeout-minutes: 13
168
- steps:
169
- - uses: aws-actions/configure-aws-credentials@v4
170
- with:
171
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
172
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
173
- aws-region: eu-central-1
174
- - uses: actions/checkout@v4
175
- - uses: actions/setup-python@v6
176
- with:
177
- python-version: "3.11"
178
- cache: "pip"
179
- cache-dependency-path: ".github/workflows/build.yml"
180
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
181
- - run: nox -s "install(group='connectivity')"
182
- - name: cache mitmproxy
183
- id: cache-mitmproxy
184
- uses: actions/cache@v4
185
- with:
186
- path: ~/mitmproxy.tar
187
- key: cache-mitmproxy-0
188
- - name: cache mitmproxy miss
189
- if: ${{ steps.cache-mitmproxy.outputs.cache-hit != 'true' }}
190
- run: docker pull mitmproxy/mitmproxy:latest && docker image save mitmproxy/mitmproxy:latest --output ~/mitmproxy.tar
191
- - name: cache mitmproxy use
192
- if: ${{ steps.cache-mitmproxy.outputs.cache-hit == 'true' }}
193
- run: docker image load --input ~/mitmproxy.tar
194
- - name: start mitmproxy
195
- run: |
196
- # Start mitmdump (headless mitmproxy) as a forward proxy on 8080
197
- docker run -d \
198
- --name mitmproxy \
199
- -p 8080:8080 \
200
- mitmproxy/mitmproxy:latest \
201
- mitmdump --mode regular --listen-port 8080
202
- # Give it a few seconds to start and generate the CA
203
- sleep 10
204
- echo "mitmproxy state:"
205
- docker ps -a
206
- echo "Container ~/.mitmproxy contents:"
207
- docker exec mitmproxy ls -l /home/mitmproxy/.mitmproxy || true
208
- echo "Container logs (if it still exited):"
209
- docker logs mitmproxy || true
210
- - name: export mitmproxy certificate
211
- run: |
212
- # Copy CA cert from inside the container to the workspace
213
- docker cp mitmproxy:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./mitmproxy-ca.pem
214
- ls -l ./mitmproxy-ca.pem
215
- - name: run tests
216
- run: nox -s connectivity
217
-
218
- profile:
219
- runs-on: ubuntu-latest
220
- timeout-minutes: 10
221
- env:
222
- LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
223
- steps:
224
- - uses: actions/checkout@v4
225
- with:
226
- submodules: recursive
227
- fetch-depth: 0
228
- - uses: actions/setup-python@v6
229
- with:
230
- python-version: |
231
- ${{ github.ref == 'refs/heads/release' && '3.11' ||
232
- '3.13'
233
- }}
234
- - run: pip install git+https://github.com/laminlabs/laminci
235
- - run: uv pip install --system git+https://github.com/laminlabs/lamin-cli
236
- - run: uv pip install --system git+https://github.com/laminlabs/lamindb
237
- - run: uv pip install --system git+https://github.com/laminlabs/laminprofiler
238
- - run: uv pip install --system -e .
239
- - run: lamin login
240
- - run: lamin connect laminlabs/lamindata
241
- - run: laminprofiler check tests/profiling/import_lamindb_setup.py --threshold 0.25
242
- - run: laminprofiler check tests/profiling/print_lamindb_setup_settings.py --threshold 0.4
243
- - run: laminprofiler check tests/profiling/connect_cli.py --threshold 2.0
244
-
245
- coverage:
246
- needs: [hub-prod, hub-cloud, core, hub-local]
247
- runs-on: ubuntu-latest
248
- steps:
249
- - uses: actions/checkout@v4
250
- - uses: actions/setup-python@v6
251
- with:
252
- python-version: "3.11"
253
- cache: "pip"
254
- - run: |
255
- pip install coverage[toml]
256
- pip install --no-deps .
257
- - uses: actions/download-artifact@v4
258
- - name: run coverage
259
- run: |
260
- coverage combine coverage--*/.coverage*
261
- coverage report --fail-under=0
262
- coverage xml
263
- - uses: codecov/codecov-action@v2
264
- with:
265
- token: ${{ secrets.CODECOV_TOKEN }}
266
-
267
- docs:
268
- needs: hub-cloud
269
- runs-on: ubuntu-latest
270
- steps:
271
- - uses: aws-actions/configure-aws-credentials@v4
272
- with:
273
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
274
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
275
- aws-region: eu-central-1
276
- - uses: actions/checkout@v4
277
- with:
278
- submodules: recursive
279
- fetch-depth: 0
280
- - name: checkout lndocs
281
- uses: actions/checkout@v4
282
- with:
283
- repository: laminlabs/lndocs
284
- ssh-key: ${{ secrets.READ_LNDOCS }}
285
- path: lndocs
286
- ref: main
287
- - uses: actions/setup-python@v6
288
- with:
289
- python-version: "3.10"
290
- cache: "pip"
291
- cache-dependency-path: ".github/workflows/build.yml"
292
- - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
293
- - run: nox -s "install(group='docs')"
294
- - uses: actions/download-artifact@v4
295
- - run: nox -s docs
296
- - uses: nwtgck/actions-netlify@v1.2
297
- with:
298
- publish-dir: "_build/html"
299
- production-deploy: ${{ github.event_name == 'push' }}
300
- github-token: ${{ secrets.GITHUB_TOKEN }}
301
- enable-commit-comment: false
302
- env:
303
- NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
304
- NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
305
-
306
- notify:
307
- if: always()
308
- # this here is only for notifying developers of laminhub
309
- # hence, it only needs groups hub-local and hub-cloud
310
- needs: [hub-local, hub-cloud]
311
- runs-on: ubuntu-latest
312
- steps:
313
- - uses: voxmedia/github-action-slack-notify-build@v1
314
- if: ${{ ( needs.hub-local.result == 'failure' || needs.hub-cloud.result == 'failure' ) && github.event_name == 'repository_dispatch' }}
315
- env:
316
- SLACK_BOT_TOKEN: ${{ secrets.SLACK_GITHUB_ACTION }}
317
- with:
318
- channel_id: C05S2C02JHM
319
- status: FAILURE
320
- color: danger
1
+ name: build
2
+
3
+ on:
4
+ push:
5
+ branches: [main, staging]
6
+ pull_request:
7
+ branches: [main]
8
+ repository_dispatch:
9
+ types: [build]
10
+
11
+ jobs:
12
+ # tests only on production hub
13
+ hub-prod:
14
+ runs-on: ubuntu-latest
15
+ timeout-minutes: 13
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v6
19
+ with:
20
+ python-version: "3.10"
21
+ cache: "pip"
22
+ - uses: aws-actions/configure-aws-credentials@v4
23
+ with:
24
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
25
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
26
+ aws-region: eu-central-1
27
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
28
+ - run: nox -s "install(group='hub-prod')"
29
+ - run: nox -s "build(lamin_env='prod', group='hub-prod')"
30
+ - uses: actions/upload-artifact@v4
31
+ with:
32
+ name: coverage--hub-prod
33
+ path: .coverage
34
+ include-hidden-files: true
35
+
36
+ # tests both on production and staging hub
37
+ hub-cloud:
38
+ runs-on: ubuntu-latest
39
+ strategy:
40
+ fail-fast: false
41
+ matrix:
42
+ include:
43
+ - lamin_env: "prod"
44
+ python-version: "3.11"
45
+ - lamin_env: "staging"
46
+ python-version: "3.10"
47
+ timeout-minutes: 13
48
+ steps:
49
+ - uses: aws-actions/configure-aws-credentials@v4
50
+ with:
51
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
52
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53
+ aws-region: eu-central-1
54
+ - uses: actions/checkout@v4
55
+ - uses: actions/setup-python@v6
56
+ with:
57
+ python-version: ${{ matrix.python-version }}
58
+ cache: "pip"
59
+ cache-dependency-path: ".github/workflows/build.yml"
60
+ - uses: actions/checkout@v4
61
+ with:
62
+ repository: laminlabs/laminhub
63
+ token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
64
+ path: laminhub
65
+ ref: main
66
+ - uses: actions/cache@v4
67
+ with:
68
+ path: ~/.cache/pre-commit
69
+ key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
70
+ - id: cache-postgres
71
+ uses: actions/cache@v4
72
+ with:
73
+ path: ~/postgres.tar
74
+ key: cache-postgres-0
75
+ - if: steps.cache-postgres.outputs.cache-hit != 'true'
76
+ run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
77
+ - if: steps.cache-postgres.outputs.cache-hit == 'true'
78
+ run: docker image load --input ~/postgres.tar
79
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
80
+ # account for in a different job
81
+ # - uses: "google-github-actions/auth@v0"
82
+ # with:
83
+ # credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
84
+ # - uses: "google-github-actions/setup-gcloud@v0"
85
+ - run: nox -s "install(group='hub-cloud')"
86
+ - run: nox -s "build(lamin_env='${{ matrix.lamin_env }}', group='hub-cloud')"
87
+ - uses: actions/upload-artifact@v4
88
+ if: ${{ matrix.lamin_env == 'prod' }}
89
+ with:
90
+ name: coverage--hub-cloud
91
+ path: .coverage
92
+ include-hidden-files: true
93
+
94
+ core:
95
+ runs-on: ubuntu-latest
96
+ timeout-minutes: 13
97
+ steps:
98
+ - uses: actions/checkout@v4
99
+ - uses: actions/setup-python@v6
100
+ with:
101
+ python-version: "3.10"
102
+ cache: "pip"
103
+ cache-dependency-path: ".github/workflows/build.yml"
104
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
105
+ - run: nox -s "install(group='core')"
106
+ - run: nox -s lint
107
+ - run: nox -s core
108
+ env:
109
+ TMP_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
110
+ TMP_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
111
+ - uses: actions/upload-artifact@v4
112
+ with:
113
+ name: coverage--core
114
+ path: .coverage
115
+ include-hidden-files: true
116
+
117
+ # test low-level hub functionality
118
+ hub-local:
119
+ runs-on: ubuntu-latest
120
+ timeout-minutes: 13
121
+ steps:
122
+ - uses: aws-actions/configure-aws-credentials@v4
123
+ with:
124
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
125
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
126
+ aws-region: eu-central-1
127
+ - uses: actions/checkout@v4
128
+ - uses: actions/checkout@v4
129
+ with:
130
+ repository: laminlabs/laminhub
131
+ token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
132
+ path: laminhub
133
+ ref: main
134
+ - name: Set env file for local test of edge functions
135
+ run: |
136
+ touch .env.local
137
+ echo "AWS_ACCESS_KEY_ID_HOSTED_S3=${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env.local
138
+ echo "AWS_SECRET_ACCESS_KEY_HOSTED_S3=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env.local
139
+ working-directory: laminhub/backend/services/central/supabase
140
+ - uses: actions/setup-python@v6
141
+ with:
142
+ python-version: "3.11" # we need to run everything for coverage on 3.11
143
+ cache: "pip"
144
+ cache-dependency-path: ".github/workflows/build.yml"
145
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
146
+ - run: nox -s "install(group='hub-local')"
147
+ - id: cache-supabase
148
+ uses: actions/cache@v4
149
+ with:
150
+ path: /var/lib/docker
151
+ key: cache-supabase
152
+ - uses: supabase/setup-cli@v1
153
+ with:
154
+ version: "2.72.7"
155
+ - run: nox -s hub_local
156
+ env:
157
+ LAMIN_ENV: "local"
158
+ - uses: actions/upload-artifact@v4
159
+ with:
160
+ name: coverage--hub-local
161
+ path: .coverage
162
+ include-hidden-files: true
163
+
164
+ # test custom proxies and certificates
165
+ connectivity:
166
+ runs-on: ubuntu-latest
167
+ timeout-minutes: 13
168
+ steps:
169
+ - uses: aws-actions/configure-aws-credentials@v4
170
+ with:
171
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
172
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
173
+ aws-region: eu-central-1
174
+ - uses: actions/checkout@v4
175
+ - uses: actions/setup-python@v6
176
+ with:
177
+ python-version: "3.11"
178
+ cache: "pip"
179
+ cache-dependency-path: ".github/workflows/build.yml"
180
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
181
+ - run: nox -s "install(group='connectivity')"
182
+ - name: cache mitmproxy
183
+ id: cache-mitmproxy
184
+ uses: actions/cache@v4
185
+ with:
186
+ path: ~/mitmproxy.tar
187
+ key: cache-mitmproxy-0
188
+ - name: cache mitmproxy miss
189
+ if: ${{ steps.cache-mitmproxy.outputs.cache-hit != 'true' }}
190
+ run: docker pull mitmproxy/mitmproxy:latest && docker image save mitmproxy/mitmproxy:latest --output ~/mitmproxy.tar
191
+ - name: cache mitmproxy use
192
+ if: ${{ steps.cache-mitmproxy.outputs.cache-hit == 'true' }}
193
+ run: docker image load --input ~/mitmproxy.tar
194
+ - name: start mitmproxy
195
+ run: |
196
+ # Start mitmdump (headless mitmproxy) as a forward proxy on 8080
197
+ docker run -d \
198
+ --name mitmproxy \
199
+ -p 8080:8080 \
200
+ mitmproxy/mitmproxy:latest \
201
+ mitmdump --mode regular --listen-port 8080
202
+ # Give it a few seconds to start and generate the CA
203
+ sleep 10
204
+ echo "mitmproxy state:"
205
+ docker ps -a
206
+ echo "Container ~/.mitmproxy contents:"
207
+ docker exec mitmproxy ls -l /home/mitmproxy/.mitmproxy || true
208
+ echo "Container logs (if it still exited):"
209
+ docker logs mitmproxy || true
210
+ - name: export mitmproxy certificate
211
+ run: |
212
+ # Copy CA cert from inside the container to the workspace
213
+ docker cp mitmproxy:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./mitmproxy-ca.pem
214
+ ls -l ./mitmproxy-ca.pem
215
+ - name: run tests
216
+ run: nox -s connectivity
217
+
218
+ profile:
219
+ runs-on: ubuntu-latest
220
+ timeout-minutes: 10
221
+ env:
222
+ LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
223
+ steps:
224
+ - uses: actions/checkout@v4
225
+ with:
226
+ submodules: recursive
227
+ fetch-depth: 0
228
+ - uses: actions/setup-python@v6
229
+ with:
230
+ python-version: |
231
+ ${{ github.ref == 'refs/heads/release' && '3.11' ||
232
+ '3.13'
233
+ }}
234
+ - run: pip install git+https://github.com/laminlabs/laminci
235
+ - run: uv pip install --system git+https://github.com/laminlabs/lamin-cli
236
+ - run: uv pip install --system git+https://github.com/laminlabs/lamindb
237
+ - run: uv pip install --system git+https://github.com/laminlabs/laminprofiler
238
+ - run: uv pip install --system -e .
239
+ - run: lamin login
240
+ - run: lamin connect laminlabs/lamindata
241
+ - run: laminprofiler check tests/profiling/import_lamindb_setup.py --threshold 0.25
242
+ - run: laminprofiler check tests/profiling/print_lamindb_setup_settings.py --threshold 0.4
243
+ - run: laminprofiler check tests/profiling/connect_cli.py --threshold 2.0
244
+
245
+ coverage:
246
+ needs: [hub-prod, hub-cloud, core, hub-local]
247
+ runs-on: ubuntu-latest
248
+ steps:
249
+ - uses: actions/checkout@v4
250
+ - uses: actions/setup-python@v6
251
+ with:
252
+ python-version: "3.11"
253
+ cache: "pip"
254
+ - run: |
255
+ pip install coverage[toml]
256
+ pip install --no-deps .
257
+ - uses: actions/download-artifact@v4
258
+ - name: run coverage
259
+ run: |
260
+ coverage combine coverage--*/.coverage*
261
+ coverage report --fail-under=0
262
+ coverage xml
263
+ - uses: codecov/codecov-action@v2
264
+ with:
265
+ token: ${{ secrets.CODECOV_TOKEN }}
266
+
267
+ docs:
268
+ needs: hub-cloud
269
+ runs-on: ubuntu-latest
270
+ steps:
271
+ - uses: aws-actions/configure-aws-credentials@v4
272
+ with:
273
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
274
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
275
+ aws-region: eu-central-1
276
+ - uses: actions/checkout@v4
277
+ with:
278
+ submodules: recursive
279
+ fetch-depth: 0
280
+ - name: checkout lndocs
281
+ uses: actions/checkout@v4
282
+ with:
283
+ repository: laminlabs/lndocs
284
+ ssh-key: ${{ secrets.READ_LNDOCS }}
285
+ path: lndocs
286
+ ref: main
287
+ - uses: actions/setup-python@v6
288
+ with:
289
+ python-version: "3.10"
290
+ cache: "pip"
291
+ cache-dependency-path: ".github/workflows/build.yml"
292
+ - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
293
+ - run: nox -s "install(group='docs')"
294
+ - uses: actions/download-artifact@v4
295
+ - run: nox -s docs
296
+ - uses: nwtgck/actions-netlify@v1.2
297
+ with:
298
+ publish-dir: "_build/html"
299
+ production-deploy: ${{ github.event_name == 'push' }}
300
+ github-token: ${{ secrets.GITHUB_TOKEN }}
301
+ enable-commit-comment: false
302
+ env:
303
+ NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
304
+ NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
305
+
306
+ notify:
307
+ if: always()
308
+ # this here is only for notifying developers of laminhub
309
+ # hence, it only needs groups hub-local and hub-cloud
310
+ needs: [hub-local, hub-cloud]
311
+ runs-on: ubuntu-latest
312
+ steps:
313
+ - uses: voxmedia/github-action-slack-notify-build@v1
314
+ if: ${{ ( needs.hub-local.result == 'failure' || needs.hub-cloud.result == 'failure' ) && github.event_name == 'repository_dispatch' }}
315
+ env:
316
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_GITHUB_ACTION }}
317
+ with:
318
+ channel_id: C05S2C02JHM
319
+ status: FAILURE
320
+ color: danger