frogml-core 0.0.95__py3-none-any.whl → 0.0.97__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. frogml_core/__init__.py +1 -1
  2. frogml_core/clients/instance_template/client.py +6 -4
  3. frogml_core/clients/model_version_manager/client.py +4 -4
  4. frogml_core/clients/prompt_manager/model_descriptor_mapper.py +4 -4
  5. frogml_core/feature_store/_common/artifact_utils.py +3 -3
  6. frogml_core/feature_store/data_sources/batch/athena.py +3 -3
  7. frogml_core/feature_store/feature_sets/context.py +2 -6
  8. frogml_core/feature_store/feature_sets/streaming.py +3 -3
  9. frogml_core/feature_store/feature_sets/streaming_backfill.py +1 -1
  10. frogml_core/feature_store/online/client.py +6 -6
  11. frogml_core/feature_store/sinks/streaming/factory.py +1 -1
  12. frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/git/git_strategy.py +3 -3
  13. frogml_core/inner/di_configuration/account.py +2 -2
  14. frogml_core/inner/tool/auth.py +2 -3
  15. frogml_core/llmops/provider/openai/provider.py +3 -3
  16. frogml_core/model/tools/adapters/output.py +1 -1
  17. frogml_core/tools/logger/logger.py +1 -1
  18. frogml_core/utils/proto_utils.py +1 -1
  19. frogml_core-0.0.97.dist-info/METADATA +411 -0
  20. {frogml_core-0.0.95.dist-info → frogml_core-0.0.97.dist-info}/RECORD +69 -25
  21. frogml_proto/qwak/model_group/model_group_repository_details_pb2.py +86 -0
  22. frogml_proto/qwak/model_group/model_group_repository_details_pb2.pyi +107 -0
  23. frogml_proto/qwak/model_group/model_group_repository_details_pb2_grpc.py +4 -0
  24. frogml_proto/qwak/models/models_pb2.py +102 -81
  25. frogml_proto/qwak/models/models_pb2.pyi +57 -0
  26. frogml_proto/qwak/models/models_pb2_grpc.py +34 -0
  27. frogml_proto/qwak/models/models_query_pb2.py +77 -0
  28. frogml_proto/qwak/models/models_query_pb2.pyi +157 -0
  29. frogml_proto/qwak/models/models_query_pb2_grpc.py +4 -0
  30. frogml_storage/__init__.py +1 -0
  31. frogml_storage/_environment.py +22 -0
  32. frogml_storage/artifactory/__init__.py +1 -0
  33. frogml_storage/artifactory/_artifactory_api.py +315 -0
  34. frogml_storage/authentication/login/__init__.py +1 -0
  35. frogml_storage/authentication/login/_login_cli.py +229 -0
  36. frogml_storage/authentication/login/_login_command.py +71 -0
  37. frogml_storage/authentication/models/__init__.py +3 -0
  38. frogml_storage/authentication/models/_auth.py +20 -0
  39. frogml_storage/authentication/models/_auth_config.py +70 -0
  40. frogml_storage/authentication/models/_login.py +22 -0
  41. frogml_storage/authentication/utils/__init__.py +17 -0
  42. frogml_storage/authentication/utils/_authentication_utils.py +281 -0
  43. frogml_storage/authentication/utils/_login_checks_utils.py +114 -0
  44. frogml_storage/base_storage.py +140 -0
  45. frogml_storage/constants.py +56 -0
  46. frogml_storage/exceptions/checksum_verification_error.py +3 -0
  47. frogml_storage/exceptions/validation_error.py +4 -0
  48. frogml_storage/frog_ml.py +668 -0
  49. frogml_storage/http/__init__.py +1 -0
  50. frogml_storage/http/http_client.py +83 -0
  51. frogml_storage/logging/__init__.py +1 -0
  52. frogml_storage/logging/_log_config.py +45 -0
  53. frogml_storage/logging/log_utils.py +21 -0
  54. frogml_storage/models/__init__.py +1 -0
  55. frogml_storage/models/_download_context.py +54 -0
  56. frogml_storage/models/dataset_manifest.py +13 -0
  57. frogml_storage/models/entity_manifest.py +93 -0
  58. frogml_storage/models/frogml_dataset_version.py +21 -0
  59. frogml_storage/models/frogml_entity_type_info.py +48 -0
  60. frogml_storage/models/frogml_entity_version.py +34 -0
  61. frogml_storage/models/frogml_model_version.py +21 -0
  62. frogml_storage/models/model_manifest.py +60 -0
  63. frogml_storage/models/serialization_metadata.py +15 -0
  64. frogml_storage/utils/__init__.py +12 -0
  65. frogml_storage/utils/_input_checks_utility.py +104 -0
  66. frogml_storage/utils/_storage_utils.py +15 -0
  67. frogml_storage/utils/_url_utils.py +27 -0
  68. frogml_core-0.0.95.dist-info/METADATA +0 -46
  69. {frogml_core-0.0.95.dist-info → frogml_core-0.0.97.dist-info}/WHEEL +0 -0
  70. {frogml_proto → frogml_storage/authentication}/__init__.py +0 -0
frogml_core/__init__.py CHANGED
@@ -1,7 +1,7 @@
1
1
  """Top-level package for frogml."""
2
2
 
3
3
  __author__ = "jfrog"
4
- __version__ = "0.0.95"
4
+ __version__ = "0.0.97"
5
5
 
6
6
  from frogml_core.inner.di_configuration import wire_dependencies
7
7
  from frogml_core.model.model_version_tracking import ( # noqa: F401,E501
@@ -50,10 +50,12 @@ class InstanceTemplateManagementClient:
50
50
 
51
51
  def list_instance_templates(self) -> List[InstanceTemplateSpec]:
52
52
  try:
53
- result: ListInstanceTemplatesResponse = self._instance_template_service.ListInstanceTemplates(
54
- ListInstanceTemplatesRequest(
55
- optional_instance_filter=InstanceFilter(
56
- instance_type_filter=InstanceTypeFilter.INSTANCE_TYPE_FILTER_ALL
53
+ result: ListInstanceTemplatesResponse = (
54
+ self._instance_template_service.ListInstanceTemplates(
55
+ ListInstanceTemplatesRequest(
56
+ optional_instance_filter=InstanceFilter(
57
+ instance_type_filter=InstanceTypeFilter.INSTANCE_TYPE_FILTER_ALL
58
+ )
57
59
  )
58
60
  )
59
61
  )
@@ -2,7 +2,7 @@ from platform import python_version
2
2
  from typing import List, Optional, Dict, Union, Tuple
3
3
 
4
4
  from dependency_injector.wiring import Provide
5
- from frogml_storage.entity_manifest import Artifact
5
+ from frogml_storage.models.entity_manifest import Artifact
6
6
  from grpc import RpcError
7
7
 
8
8
  from frogml_core.exceptions import FrogmlException
@@ -210,9 +210,9 @@ class ModelVersionManagerClient:
210
210
  Optional[List[ArtifactProto]],
211
211
  List[ArtifactProto],
212
212
  ]:
213
- model_artifact_proto: List[
214
- ArtifactProto
215
- ] = ProtoUtils.convert_artifacts_to_artifacts_proto(model_artifact)
213
+ model_artifact_proto: List[ArtifactProto] = (
214
+ ProtoUtils.convert_artifacts_to_artifacts_proto(model_artifact)
215
+ )
216
216
  dependency_artifacts_proto = (
217
217
  ProtoUtils.convert_artifacts_to_artifacts_proto(dependency_artifacts)
218
218
  if dependency_artifacts
@@ -57,10 +57,10 @@ class ModelDescriptorMapper:
57
57
  model_id: str, openai_chat_params: ProtoOpenAIChatModelParams
58
58
  ) -> OpenAIChat:
59
59
  p = openai_chat_params
60
- _tool_choice: Union[
61
- str, ChatCompletionNamedToolChoiceParam
62
- ] = ModelDescriptorMapper._from_tool_choice(
63
- openai_chat_params=openai_chat_params
60
+ _tool_choice: Union[str, ChatCompletionNamedToolChoiceParam] = (
61
+ ModelDescriptorMapper._from_tool_choice(
62
+ openai_chat_params=openai_chat_params
63
+ )
64
64
  )
65
65
  _tools: List[ChatCompletionToolParam] = []
66
66
 
@@ -45,9 +45,9 @@ class ArtifactsUploader:
45
45
  featureset_name: str,
46
46
  __instance_module_path__: str,
47
47
  ) -> Optional[ArtifactSpec]:
48
- transformation_functions: Optional[
49
- List[Callable[..., Any]]
50
- ] = transformation.get_functions()
48
+ transformation_functions: Optional[List[Callable[..., Any]]] = (
49
+ transformation.get_functions()
50
+ )
51
51
  if transformation_functions is not None and transformation_functions:
52
52
  return ArtifactSpec(
53
53
  artifact_name=featureset_name,
@@ -130,9 +130,9 @@ class AthenaSource(JdbcSource):
130
130
  )
131
131
  )
132
132
 
133
- time_partition_columns: Optional[
134
- TimePartitionColumns
135
- ] = AthenaSource._extract_partition_column(proto_athena_source)
133
+ time_partition_columns: Optional[TimePartitionColumns] = (
134
+ AthenaSource._extract_partition_column(proto_athena_source)
135
+ )
136
136
  workgroup: Optional[str] = (
137
137
  proto_athena_source.workgroup
138
138
  if proto_athena_source.HasField("workgroup")
@@ -5,9 +5,5 @@ from typing import Union
5
5
 
6
6
  @dataclass
7
7
  class Context:
8
- start_time: Union[
9
- str, datetime
10
- ] = "${qwak_ingestion_start_timestamp}" # todo mlops-2312 - rename?
11
- end_time: Union[
12
- str, datetime
13
- ] = "${qwak_ingestion_end_timestamp}" # todo mlops-2312 - rename?
8
+ start_time: Union[str, datetime] = "${qwak_ingestion_start_timestamp}"
9
+ end_time: Union[str, datetime] = "${qwak_ingestion_end_timestamp}"
@@ -131,9 +131,9 @@ def feature_set(
131
131
  offline_scheduling_policy=offline_scheduling_policy,
132
132
  )
133
133
 
134
- streaming_backfill: Optional[
135
- StreamingBackfill
136
- ] = StreamingBackfill.get_streaming_backfill_from_function(function=function)
134
+ streaming_backfill: Optional[StreamingBackfill] = (
135
+ StreamingBackfill.get_streaming_backfill_from_function(function=function)
136
+ )
137
137
 
138
138
  fs_name = name or function.__name__
139
139
  streaming_feature_set = StreamingFeatureSet(
@@ -230,7 +230,7 @@ class StreamingBackfill:
230
230
 
231
231
  @staticmethod
232
232
  def _get_normalized_backfill_sources_spec(
233
- data_sources: Union[List[str], List[DataSourceBackfillSpec]]
233
+ data_sources: Union[List[str], List[DataSourceBackfillSpec]],
234
234
  ) -> List[DataSourceBackfillSpec]:
235
235
  # reformat all data source specs to 'DataSourceBackfillSpec'
236
236
  return [
@@ -197,12 +197,12 @@ class OnlineClient:
197
197
  )
198
198
  ordered_entities = [entity[0] for entity in ordered_entities_tuple]
199
199
 
200
- request_chunks: List[
201
- Tuple[RequestedEntitiesMatrix, pd.DataFrame]
202
- ] = OnlineClient._split_entities(
203
- entity_names=ordered_entities,
204
- population_df=df,
205
- max_entities_per_split=max_keys_per_request,
200
+ request_chunks: List[Tuple[RequestedEntitiesMatrix, pd.DataFrame]] = (
201
+ OnlineClient._split_entities(
202
+ entity_names=ordered_entities,
203
+ population_df=df,
204
+ max_entities_per_split=max_keys_per_request,
205
+ )
206
206
  )
207
207
 
208
208
  results: List[pd.DataFrame] = []
@@ -25,7 +25,7 @@ class StreamingSinkFactory:
25
25
  def get_streaming_sink(proto_streaming_sink: ProtoStreamingSink) -> BaseSink:
26
26
  sink_type = proto_streaming_sink.WhichOneof("sink_type")
27
27
 
28
- auth_conf: BaseAuthentication
28
+ auth_conf: BaseAuthentication # noqa: F842
29
29
  if sink_type == "kafka_sink":
30
30
  proto_kafka_sink: ProtoKafkaSink = proto_streaming_sink.kafka_sink
31
31
  auth_configuration: BaseAuthentication = cast(
@@ -161,9 +161,9 @@ def make_ssh_key_file(git_ssh_key: str) -> str:
161
161
 
162
162
 
163
163
  def add_ssh_file_to_env(ssh_key_file_path: str) -> None:
164
- os.environ[
165
- "GIT_SSH_COMMAND"
166
- ] = f"ssh -i {ssh_key_file_path} -o StrictHostKeyChecking=no"
164
+ os.environ["GIT_SSH_COMMAND"] = (
165
+ f"ssh -i {ssh_key_file_path} -o StrictHostKeyChecking=no"
166
+ )
167
167
  os.environ["GIT_SSH"] = f"ssh -i {ssh_key_file_path} -o StrictHostKeyChecking=no"
168
168
 
169
169
 
@@ -3,8 +3,8 @@ import os
3
3
  from dataclasses import dataclass
4
4
  from typing import Optional
5
5
 
6
- from frogml_storage.authentication._authentication_utils import get_credentials
7
- from frogml_storage.cli._login_cli import login as frogml_login
6
+ from frogml_storage.authentication.utils import get_credentials
7
+ from frogml_storage.authentication.login import frogml_login
8
8
  from frogml_storage.constants import CONFIG_FILE_PATH
9
9
 
10
10
  from frogml_core.exceptions import FrogmlLoginException
@@ -1,9 +1,8 @@
1
1
  from typing import Optional, Union, cast
2
2
 
3
3
  import requests
4
- from frogml_storage._utils import BearerAuth
5
- from frogml_storage.authentication._authentication_utils import get_credentials
6
- from frogml_storage.authentication.models._auth_config import AuthConfig
4
+ from frogml_storage.authentication.models import AuthConfig, BearerAuth
5
+ from frogml_storage.authentication.utils import get_credentials
7
6
  from requests.auth import AuthBase
8
7
  from typing_extensions import Self
9
8
 
@@ -30,9 +30,9 @@ class OpenAIProvider:
30
30
  self.client = OpenAIClient()
31
31
 
32
32
  def _get_random_openai_api_key(self) -> Optional[str]:
33
- openai_api_keys: List[
34
- OpenAIApiKeySystemSecret
35
- ] = IntegrationUtils().get_openai_api_keys()
33
+ openai_api_keys: List[OpenAIApiKeySystemSecret] = (
34
+ IntegrationUtils().get_openai_api_keys()
35
+ )
36
36
  if len(openai_api_keys) == 0:
37
37
  return None
38
38
 
@@ -46,7 +46,7 @@ def get_output_adapter(
46
46
 
47
47
  first_result = (
48
48
  return_result[0]
49
- if type(return_result) == list and len(return_result) > 0
49
+ if isinstance(return_result, list) and len(return_result) > 0
50
50
  else return_result
51
51
  )
52
52
  if issubclass(type(first_result), Message):
@@ -276,7 +276,7 @@ def set_file_handler_log_file(
276
276
  logger: logging.Logger, handler_name: str, log_file: Path
277
277
  ):
278
278
  existing_handler = get_handler_from_logger(logger, handler_name)
279
- if type(existing_handler) != RotatingFileHandler:
279
+ if not isinstance(existing_handler, RotatingFileHandler):
280
280
  raise FrogmlException(
281
281
  f"Error in setting log file. Error message: handler of name {handler_name} is not a file logger handler"
282
282
  )
@@ -1,6 +1,6 @@
1
1
  from typing import List
2
2
 
3
- from frogml_storage.entity_manifest import Artifact
3
+ from frogml_storage.models.entity_manifest import Artifact
4
4
 
5
5
  from frogml_proto.jfml.model_version.v1.artifact_pb2 import (
6
6
  Artifact as ArtifactProto,
@@ -0,0 +1,411 @@
1
+ Metadata-Version: 2.3
2
+ Name: frogml-core
3
+ Version: 0.0.97
4
+ Summary: frogml Core contains the necessary objects and communication tools for using the Jfrog ml Platform
5
+ License: Apache-2.0
6
+ Keywords: mlops,ml,deployment,serving,model
7
+ Author: Jfrog
8
+ Requires-Python: >=3.9,<3.12
9
+ Classifier: License :: OSI Approved :: Apache Software License
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: Implementation :: CPython
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Provides-Extra: feature-store
18
+ Requires-Dist: PyYAML
19
+ Requires-Dist: cachetools
20
+ Requires-Dist: chevron (==0.14.0)
21
+ Requires-Dist: cloudpickle (==2.2.1) ; extra == "feature-store"
22
+ Requires-Dist: dacite (==1.8.1)
23
+ Requires-Dist: dependency-injector (>=4.0)
24
+ Requires-Dist: filelock
25
+ Requires-Dist: grpcio (>=1.57.0)
26
+ Requires-Dist: joblib (>=1.3.2,<2.0.0)
27
+ Requires-Dist: marshmallow-dataclass (>=8.5.8,<9.0.0)
28
+ Requires-Dist: protobuf (>=3.10,<4) ; python_version < "3.10"
29
+ Requires-Dist: protobuf (>=4.21.6) ; python_version >= "3.10"
30
+ Requires-Dist: pyarrow (>=6.0.0) ; extra == "feature-store"
31
+ Requires-Dist: pyathena (>=2.2.0,!=2.18.0) ; extra == "feature-store"
32
+ Requires-Dist: pydantic
33
+ Requires-Dist: pyspark (==3.4.2) ; extra == "feature-store"
34
+ Requires-Dist: python-jose[cryptography] (>=3.4.0)
35
+ Requires-Dist: python-json-logger (>=2.0.2)
36
+ Requires-Dist: requests
37
+ Requires-Dist: retrying (==1.3.4)
38
+ Requires-Dist: tqdm
39
+ Requires-Dist: typeguard (>=2,<3)
40
+ Project-URL: Home page, https://www.jfrog.com/
41
+ Description-Content-Type: text/markdown
42
+
43
+ # Frogml Core
44
+
45
+ Frogml is an end-to-end production ML platform designed to allow data scientists to build, deploy, and monitor their models in production with minimal engineering friction.
46
+ Frogml Core contains all the objects and tools necessary to use the Frogml Platform
47
+
48
+ # Frog ML Storage
49
+
50
+ ## Table of contents:
51
+
52
+ - [Overview](#overview)
53
+ - [Working with Artifactory](#Working-with-Artifactory)
54
+ - [Login by adding authentication details to your python code](#Login-by-adding-authentication-details-to-your-python-code)
55
+ - [Login via environment variables](#Login-via-environment-variables)
56
+ - [Login via cli](#Login-via-cli)
57
+ - [Login by a single command line with options](#Login-by-a-single-command-line-with-options)
58
+ - [Login by interactive flow in Cli](#Login-by-interactive-flow-in-Cli)
59
+ - [Upload ML model to Artifactory](#Upload-ML-model-to-Artifactory)
60
+ - [Download ML model from Artifactory](#Download-ML-model-from-Artifactory)
61
+ - [Upload ML dataset to Artifactory](#Upload-ML-dataset-to-Artifactory)
62
+ - [Download ML dataset from Artifactory](#Download-ML-dataset-from-Artifactory)
63
+ - [Testing](#Testing)
64
+ - [Locally run integration tests using local artifactory](#Locally-run-integration-tests-using-artifactory)
65
+ - [Locally run tests using existing Artifactory](#Locally-run-tests-using-existing-Artifactory)
66
+ - [Packaging](#Packaging)
67
+ - [Linters](#Linters)
68
+
69
+ ## Overview
70
+
71
+ JFrog ML Storage is a smart python client library providing a simple and efficient method of storing and downloading models, model data and datasets from the JFrog platform, utilizing the advanced capabilities of the JFrog platform.
72
+
73
+ ## Working with Artifactory
74
+
75
+ FrogML Storage Library support is available from Artifactory version 7.84.x.
76
+
77
+ To be able to use FrogML Storage with Artifactory, you should authenticate the frogml storage client against Artifactory.
78
+ JFrog implements a credentials provider chain. It sequentially checks each place where you can set the credentials to authenticate with FrogML, and then selects the first one you set.
79
+
80
+ The credentials retrieval order is as follows:
81
+
82
+ 1. [Login by adding authentication details to your python code](#Login-by-adding-authentication-details-to-your-python-code)
83
+ 2. [Login via environment variables](#Login-via-environment-variables)
84
+ 3. [Login via cli](#login-via-cli)
85
+
86
+
87
+ #### Login by adding authentication details to your python code
88
+
89
+ ---
90
+ **NOTE**
91
+
92
+ It is strongly discouraged to use credentials in clear text. Use this method for testing purposes only.
93
+
94
+ ---
95
+
96
+ You can authenticate the FrogML client directly via your Python code, using any of the following credentials (overriding env vars and the credentials in the configuration file):
97
+ - Username and Password
98
+ - Access Token
99
+
100
+ To log in as an anonymous user, log in first via CLI, and then you will be able to log in via Python using only your JFrog Platform domain.
101
+
102
+ Authentication by username and password:
103
+
104
+ ```
105
+ from frogml_storage.frog_ml import FrogMLStorage
106
+ from frogml_storage.authentication.models._auth_config import AuthConfig
107
+
108
+ arti = FrogMLStorage(AuthConfig.by_basic_auth("http://myorg.jfrog.io", <username>, <password>))
109
+ ```
110
+
111
+ Authentication by access token:
112
+
113
+ ```
114
+ from frogml_storage.frog_ml import FrogMLStorage
115
+ from frogml_storage.authentication.models._auth_config import AuthConfig
116
+
117
+ arti = FrogMLStorage(AuthConfig.by_access_token("http://myorg.jfrog.io", <token>))
118
+ ```
119
+
120
+ #### Login via environment variables
121
+
122
+ You can also authenticate the frogml client using the following environment variables:
123
+
124
+ - JF_URL - your JFrog platform domain, for example 'http://myorg.jfrog.io'
125
+ - JF_ACCESS_TOKEN - your artifactory token for this domain. To generate a token, log in to your artifactory, navigate to your FrogML repository and click on "Set Me Up".
126
+
127
+ After setting the environment variables, you can log in to the frogml client without specifying credentials.
128
+
129
+
130
+ ```
131
+ from frogml_storage.frog_ml import FrogMLStorage
132
+ from frogml_storage.authentication.models._auth_config import AuthConfig
133
+
134
+ arti = FrogMLStorage()
135
+ ```
136
+
137
+
138
+ ### Login via cli
139
+
140
+ It is possible to authenticate the frogml client using any of the following methods:
141
+
142
+ 1. Login by a single CLI command
143
+ 2. Interactive flow
144
+
145
+ After each login attempt, the authentication result (success or failure) is printed on the screen.
146
+ If the login attempt succeeded, the authentication details will be saved as frogml configuration file under the path: ~/.frogml/config.json and from that point you can login again without specifying credentials.
147
+
148
+ In both interactive flow and the single command flow, it is possible to authenticate the frogml client by:
149
+
150
+ 1. Username and password
151
+ 2. Access token
152
+ 3. Anonymous authentication
153
+
154
+
155
+
156
+ #### Login by a single command line with options
157
+
158
+ The below examples show the frogml login options using the cli:
159
+
160
+ Login using existing jfrog-cli of frogml configuration files (~/.jfrog/jfrog-cli.conf.v6 or ~/.frogml/config.json, respectively):
161
+
162
+ ```
163
+ frogml login
164
+ ```
165
+ If no configuration file is found, interactive mode will be triggered.
166
+
167
+ Login by username and password:
168
+
169
+ ```
170
+ frogml login --url <artifactory_url> --username <username> --password <password>
171
+ ```
172
+
173
+ Where:
174
+ - artifactory_url is your JFrog platform domain, for example 'http://myorg.jfrog.io'
175
+ - username and password are your artifactory credentials for this domain
176
+
177
+ Login by access token:
178
+
179
+ ```
180
+ frogml login --url <artifactory_url> --token <access_token>
181
+ ```
182
+
183
+ Where:
184
+ - artifactory_url is your JFrog platform domain, for example 'http://myorg.jfrog.io'
185
+ - token - your artifactory token for this domain. To generate a token, log in to your artifactory and navigate to Administration -> User Management -> Access Tokens.
186
+
187
+ Login by anonymous access:
188
+
189
+ ```
190
+ frogml login --url <artifactory_url> --anonymous
191
+ ```
192
+
193
+ #### Login by interactive flow in cli:
194
+
195
+ To start an interactive flow in the cli, run the command:
196
+
197
+ ```
198
+ frogml login --interactive
199
+ ```
200
+
201
+ After executing the command above, the cli prompts two options as follows:
202
+
203
+ ```
204
+ frogml login --interactive
205
+ Please select from the following options:
206
+ 1.Login by jfrog-cli configuration file: ~/.jfrog/jfrog-cli.conf.v6
207
+ 2.Connecting to a new server
208
+ ```
209
+
210
+ On choosing the first option, the cli attempts to retrieve your authentication credentials from your JFrog CLI configuration file and sends them to Artifactory.
211
+
212
+ On choosing the second option, the cli prompts you to input your JFrog platform domain URL. Afterwards, you can select the method you wish to use for authenticating the FrogML library.
213
+
214
+ ```
215
+ Enter artifactory base url: http://myorg.jfrog.io
216
+ Choose your preferred authentication option:
217
+ 0: Username and Password
218
+ 1: Access Token
219
+ 2: Anonymous Access
220
+ ```
221
+
222
+
223
+ ### Upload ML model to Artifactory
224
+
225
+ You can upload a model to a FrogML repository using the upload_model_version() function.
226
+ You can upload a single file or an entire folder.
227
+ This function uses checksum upload, assigning a SHA2 value to each model for retrieval from storage. If the binary content cannot be reused, the smart upload mechanism performs regular upload instead.
228
+ After uploading the model, FrogML generates a file named model-info.json which contains the model name and its related files and dependencies.
229
+
230
+ The version parameter is optional. If not specified, Artifactory will set the version as the timestamp of the time you uploaded the model in your time zone, in UTC format: yyyy-MM-dd-HH-mm-ss.
231
+ Additionally, you can add properties to the model in Artifactory to categorize and label it.
232
+ The function upload_model_version returns an instance of FrogMlModelVersion, which includes the model's name, version, and namespace.
233
+
234
+ The below examples show how to upload a model to Artifactory:
235
+
236
+ ---
237
+ **NOTE**
238
+
239
+ namespace, version, properties, dependencies_files_paths and code_archive_file_path are optional.
240
+ model_path can point to a single file or a directory, in which case the whole directory is uploaded.
241
+ model_type can be written as JSON or as SerializationMetadata object imported from jfrog_ml.serialization_metadata.
242
+ All of SerializationMetadata fields must be populated.
243
+
244
+ ---
245
+
246
+
247
+ Upload an entire folder as model:
248
+
249
+ ```
250
+ from frogml_storage.frog_ml import FrogMLStorage
251
+
252
+ arti = FrogMLStorage()
253
+ arti.upload_model_version(repository=<repository_key>,
254
+ namespace=<namespce>,
255
+ model_name=<model_name>,
256
+ model_path="~/model_to_upload/",
257
+ model_type={"framework": "tensorflow", "framework_version": "2.3.0", "serialization_format": "H5", "runtime": "python", "runtime_version": "3.7"},
258
+ properties={"model_type": "keras", "experiment": "my-exp"},
259
+ dependencies_files_paths=["path/to/req/file1", "path/to/req/file2"],
260
+ code_archive_file_path="path/to/code/archieve/file"
261
+ )
262
+ ```
263
+
264
+ Upload a model with a specified version, and no dependencies and code archive:
265
+
266
+ ```
267
+ from frogml_storage.frog_ml import FrogMLStorage
268
+
269
+ arti = FrogMLStorage()
270
+ arti.upload_model_version(repository=<repository_key>,
271
+ namespace=<namespce>,
272
+ model_name=<model_name>,
273
+ version=<version>,
274
+ model_path="~/model_to_upload/",
275
+ model_type={"framework": "tensorflow", "framework_version": "2.3.0", "serialization_format": "H5", "runtime": "python", "runtime_version": "3.7"}
276
+ )
277
+ ```
278
+
279
+ ---
280
+
281
+ #### Download ML model from Artifactory
282
+
283
+ The below example shows how to download a model from Artifactory:
284
+
285
+ ```
286
+ from frogml_storage.frog_ml import FrogMLStorage
287
+
288
+ arti = FrogMLStorage()
289
+
290
+ arti.download_model_version(repository=<repository_key>,
291
+ namespace=<namespace>,
292
+ model_name=<model_name>,
293
+ target_path="~/models",
294
+ version=<version>)
295
+ ```
296
+
297
+ ---
298
+ **NOTE**
299
+
300
+ The dependencies and code archive cannot be downloaded.
301
+
302
+ ---
303
+
304
+ ### Upload ML dataset to Artifactory
305
+
306
+ Upload an entire folder as dataset:
307
+
308
+
309
+ ```
310
+ from frogml_storage.frog_ml import FrogMLStorage
311
+
312
+ arti = FrogMLStorage()
313
+ arti.upload_dataset_version(repository=<repository_key>,
314
+ namespace=<namespce>,
315
+ dataset_name=<dataset_name>,
316
+ source_path="~/dataset_to_upload/",
317
+ properties={"dataset_type": "kerras", "experiment": "my-exp"})
318
+ ```
319
+
320
+ Upload a dataset with specified version:
321
+
322
+ ```
323
+ from frogml_storage.frog_ml import FrogMLStorage
324
+
325
+ arti = FrogMLStorage()
326
+ arti.upload_dataset_version(repository=<repository_key>,
327
+ namespace=<namespce>,
328
+ dataset_name=<dataset_name>,
329
+ version=<version>,
330
+ source_path="~/dataset_to_upload/")
331
+ ```
332
+
333
+ Upload a single file as a dataset:
334
+
335
+ ```
336
+ from frogml_storage.frog_ml import FrogMLStorage
337
+
338
+ arti = FrogMLStorage()
339
+ arti.upload_dataset_version(repository=<repository_key>,
340
+ namespace=<namespce>,
341
+ dataset_name=<dataset_name>,
342
+ version=<version>,
343
+ source_path="~/dataset_to_upload/config.json")
344
+ ```
345
+
346
+ #### Download ML dataset from Artifactory
347
+
348
+ The below example shows how to download a dataset from Artifactory:
349
+
350
+ ```
351
+ from frogml_storage.frog_ml import FrogMLStorage
352
+
353
+ arti = FrogMLStorage()
354
+
355
+ arti.download_dataset_version(repository=<repository_key>,
356
+ namespace=<namespace>,
357
+ dataset_name=<dataset_name>,
358
+ target_path="~/datasets",
359
+ version=<version>)
360
+ ```
361
+
362
+ ## Testing
363
+
364
+ ### Locally run integration tests using artifactory
365
+
366
+ Prerequisites:
367
+ - A user credentials (username, password)
368
+
369
+ To run the integration tests, use the ```poetry run pytest tests/integrations/test_artifactory_integration.py``` command.
370
+ In addition, you will need to supply your
371
+ ARTIFACTORY_URL in a `http(s)://` format (if not supplied, default will be as defined [here](tests/integrations/conftest.py)), ARTIFACTORY_USERNAME, ARTIFACTORY_PASSWORD.
372
+ the test will create a local repository in RT, will upload and download model and datasets using the provided details,
373
+ and will delete the repository after the test.
374
+
375
+ example:
376
+ ```
377
+ export ARTIFACTORY_URL=<artifactory_url>
378
+ export ARTIFACTORY_USERNAME=<username>
379
+ export ARTIFACTORY_PASSWORD=<password>
380
+
381
+ poetry run pytest tests/integrations/test_artifactory_integration.py
382
+ ```
383
+
384
+ ### Locally run tests using existing Artifactory
385
+
386
+ To run the tests, use the ```pytest``` command pointing it to an existing Artifactory host.
387
+ Prerequisites:
388
+ - A generic local repository in the Artifactory instance
389
+ - A user token or user credentials (username, password)
390
+
391
+ To run the test:
392
+
393
+ ```
394
+ python3 -m pytest tests/integrations/test_artifactory_integration.py --rt_url "<artifactory_url>" --rt_access_token <token> --repo_name <generic-repo-name> -s
395
+ ```
396
+
397
+ ## Packaging
398
+
399
+ ```
400
+ poetry build
401
+ ```
402
+
403
+ ## Linters
404
+ Fix spaces and linebreaks with:
405
+ ```
406
+ make format
407
+ ```
408
+ Run linter check:
409
+ ```
410
+ make format
411
+ ```