trainml 0.5.9__py3-none-any.whl → 0.5.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. tests/integration/projects/conftest.py +3 -1
  2. tests/integration/projects/test_projects_credentials_integration.py +45 -0
  3. tests/integration/projects/test_projects_data_connectors_integration.py +44 -0
  4. tests/integration/projects/test_projects_datastores_integration.py +42 -0
  5. tests/integration/projects/test_projects_secrets_integration.py +1 -1
  6. tests/integration/projects/test_projects_services_integration.py +44 -0
  7. tests/integration/test_checkpoints_integration.py +1 -2
  8. tests/integration/test_models_integration.py +0 -1
  9. tests/unit/cli/projects/__init__.py +0 -0
  10. tests/unit/cli/projects/test_cli_project_credential_unit.py +26 -0
  11. tests/unit/cli/projects/test_cli_project_data_connector_unit.py +28 -0
  12. tests/unit/cli/projects/test_cli_project_datastore_unit.py +26 -0
  13. tests/unit/cli/projects/test_cli_project_key_unit.py +26 -0
  14. tests/unit/cli/projects/test_cli_project_secret_unit.py +26 -0
  15. tests/unit/cli/projects/test_cli_project_service_unit.py +26 -0
  16. tests/unit/cli/projects/test_cli_project_unit.py +19 -0
  17. tests/unit/cloudbender/test_datastores_unit.py +1 -5
  18. tests/unit/conftest.py +79 -6
  19. tests/unit/projects/test_project_credentials_unit.py +100 -0
  20. tests/unit/projects/test_projects_unit.py +1 -1
  21. tests/unit/test_checkpoints_unit.py +15 -23
  22. tests/unit/test_datasets_unit.py +15 -20
  23. tests/unit/test_models_unit.py +13 -16
  24. tests/unit/test_volumes_unit.py +3 -0
  25. trainml/__init__.py +1 -1
  26. trainml/checkpoints.py +14 -3
  27. trainml/cli/cloudbender/datastore.py +2 -7
  28. trainml/cli/job/create.py +16 -16
  29. trainml/cli/project/__init__.py +4 -73
  30. trainml/cli/project/credential.py +128 -0
  31. trainml/cli/project/data_connector.py +61 -0
  32. trainml/cli/project/datastore.py +61 -0
  33. trainml/cli/project/secret.py +12 -3
  34. trainml/cli/project/service.py +61 -0
  35. trainml/cloudbender/data_connectors.py +8 -0
  36. trainml/cloudbender/datastores.py +9 -19
  37. trainml/cloudbender/nodes.py +44 -1
  38. trainml/cloudbender/providers.py +53 -0
  39. trainml/cloudbender/regions.py +48 -0
  40. trainml/datasets.py +14 -3
  41. trainml/exceptions.py +51 -0
  42. trainml/jobs.py +2 -13
  43. trainml/models.py +14 -3
  44. trainml/projects/credentials.py +71 -0
  45. trainml/projects/projects.py +7 -4
  46. trainml/projects/secrets.py +1 -1
  47. trainml/volumes.py +15 -3
  48. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/METADATA +1 -1
  49. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/RECORD +53 -46
  50. tests/integration/test_projects_integration.py +0 -44
  51. tests/unit/cli/cloudbender/test_cli_reservation_unit.py +0 -34
  52. tests/unit/cli/test_cli_project_unit.py +0 -42
  53. tests/unit/cloudbender/test_reservations_unit.py +0 -173
  54. tests/unit/test_auth.py +0 -30
  55. tests/unit/test_projects_unit.py +0 -320
  56. tests/unit/test_trainml.py +0 -54
  57. trainml/cli/cloudbender/reservation.py +0 -159
  58. trainml/cli/project.py +0 -149
  59. trainml/cloudbender/reservations.py +0 -126
  60. trainml/projects.py +0 -228
  61. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/LICENSE +0 -0
  62. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/WHEEL +0 -0
  63. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/entry_points.txt +0 -0
  64. {trainml-0.5.9.dist-info → trainml-0.5.12.dist-info}/top_level.txt +0 -0
@@ -1,159 +0,0 @@
1
- import click
2
- from trainml.cli import cli, pass_config, search_by_id_name
3
- from trainml.cli.cloudbender import cloudbender
4
-
5
-
6
- @cloudbender.group()
7
- @pass_config
8
- def reservation(config):
9
- """trainML CloudBender reservation commands."""
10
- pass
11
-
12
-
13
- @reservation.command()
14
- @click.option(
15
- "--provider",
16
- "-p",
17
- type=click.STRING,
18
- required=True,
19
- help="The provider ID of the region.",
20
- )
21
- @click.option(
22
- "--region",
23
- "-r",
24
- type=click.STRING,
25
- required=True,
26
- help="The region ID to list reservations for.",
27
- )
28
- @pass_config
29
- def list(config, provider, region):
30
- """List reservations."""
31
- data = [
32
- ["ID", "NAME", "TYPE", "RESOURCE", "HOSTNAME"],
33
- [
34
- "-" * 80,
35
- "-" * 80,
36
- "-" * 80,
37
- "-" * 80,
38
- "-" * 80,
39
- ],
40
- ]
41
-
42
- reservations = config.trainml.run(
43
- config.trainml.client.cloudbender.reservations.list(
44
- provider_uuid=provider, region_uuid=region
45
- )
46
- )
47
-
48
- for reservation in reservations:
49
- data.append(
50
- [
51
- reservation.id,
52
- reservation.name,
53
- reservation.type,
54
- reservation.resource,
55
- reservation.hostname,
56
- ]
57
- )
58
-
59
- for row in data:
60
- click.echo(
61
- "{: >37.36} {: >29.28} {: >9.8} {: >9.8} {: >29.28}"
62
- "".format(*row),
63
- file=config.stdout,
64
- )
65
-
66
-
67
- @reservation.command()
68
- @click.option(
69
- "--provider",
70
- "-p",
71
- type=click.STRING,
72
- required=True,
73
- help="The provider ID of the region.",
74
- )
75
- @click.option(
76
- "--region",
77
- "-r",
78
- type=click.STRING,
79
- required=True,
80
- help="The region ID to create the reservation in.",
81
- )
82
- @click.option(
83
- "--type",
84
- "-t",
85
- type=click.Choice(
86
- [
87
- "port",
88
- ],
89
- case_sensitive=False,
90
- ),
91
- required=True,
92
- help="The type of reservation to create.",
93
- )
94
- @click.option(
95
- "--hostname",
96
- "-h",
97
- type=click.STRING,
98
- required=True,
99
- help="The hostname to make the reservation on",
100
- )
101
- @click.option(
102
- "--resource",
103
- "-r",
104
- type=click.STRING,
105
- required=True,
106
- help="The resource to reserve",
107
- )
108
- @click.argument("name", type=click.STRING, required=True)
109
- @pass_config
110
- def create(config, provider, region, type, hostname, resource, name):
111
- """
112
- Creates a reservation.
113
- """
114
- return config.trainml.run(
115
- config.trainml.client.cloudbender.reservations.create(
116
- provider_uuid=provider,
117
- region_uuid=region,
118
- name=name,
119
- hostname=hostname,
120
- resource=resource,
121
- type=type,
122
- )
123
- )
124
-
125
-
126
- @reservation.command()
127
- @click.option(
128
- "--provider",
129
- "-p",
130
- type=click.STRING,
131
- required=True,
132
- help="The provider ID of the region.",
133
- )
134
- @click.option(
135
- "--region",
136
- "-r",
137
- type=click.STRING,
138
- required=True,
139
- help="The region ID to remove the reservation from.",
140
- )
141
- @click.argument("reservation", type=click.STRING)
142
- @pass_config
143
- def remove(config, provider, region, reservation):
144
- """
145
- Remove a reservation.
146
-
147
- RESERVATION may be specified by name or ID, but ID is preferred.
148
- """
149
- reservations = config.trainml.run(
150
- config.trainml.client.cloudbender.reservations.list(
151
- provider_uuid=provider, region_uuid=region
152
- )
153
- )
154
-
155
- found = search_by_id_name(reservation, reservations)
156
- if None is found:
157
- raise click.UsageError("Cannot find specified reservation.")
158
-
159
- return config.trainml.run(found.remove())
trainml/cli/project.py DELETED
@@ -1,149 +0,0 @@
1
- import click
2
- from trainml.cli import cli, pass_config, search_by_id_name
3
-
4
-
5
- @cli.group()
6
- @pass_config
7
- def project(config):
8
- """trainML project commands."""
9
- pass
10
-
11
-
12
- @project.command()
13
- @pass_config
14
- def list(config):
15
- """List projects."""
16
- data = [
17
- ["ID", "NAME", "OWNER", "MINE"],
18
- [
19
- "-" * 80,
20
- "-" * 80,
21
- "-" * 80,
22
- "-" * 80,
23
- ],
24
- ]
25
-
26
- projects = config.trainml.run(config.trainml.client.projects.list())
27
-
28
- for project in projects:
29
- data.append(
30
- [
31
- project.id,
32
- project.name,
33
- project.owner_name,
34
- "X" if project.is_owner else "",
35
- ]
36
- )
37
-
38
- for row in data:
39
- click.echo(
40
- "{: >38.36} {: >30.28} {: >15.13} {: >4.4}" "".format(*row),
41
- file=config.stdout,
42
- )
43
-
44
-
45
- @project.command()
46
- @click.argument("name", type=click.STRING)
47
- @pass_config
48
- def create(config, name):
49
- """
50
- Create a project.
51
-
52
- Project is created with the specified NAME.
53
- """
54
-
55
- return config.trainml.run(
56
- config.trainml.client.projects.create(
57
- name=name,
58
- )
59
- )
60
-
61
-
62
- @project.command()
63
- @click.argument("project", type=click.STRING)
64
- @pass_config
65
- def remove(config, project):
66
- """
67
- Remove a project.
68
-
69
- PROJECT may be specified by name or ID, but ID is preferred.
70
- """
71
- projects = config.trainml.run(config.trainml.client.projects.list())
72
-
73
- found = search_by_id_name(project, projects)
74
- if None is found:
75
- raise click.UsageError("Cannot find specified project.")
76
-
77
- return config.trainml.run(found.remove())
78
-
79
-
80
- @project.command()
81
- @pass_config
82
- def list_datastores(config):
83
- """List project datastores."""
84
- data = [
85
- ["ID", "NAME", "TYPE", "REGION_UUID"],
86
- [
87
- "-" * 80,
88
- "-" * 80,
89
- "-" * 80,
90
- "-" * 80,
91
- ],
92
- ]
93
- project = config.trainml.run(
94
- config.trainml.client.projects.get(config.trainml.client.project)
95
- )
96
-
97
- datastores = config.trainml.run(project.list_datastores())
98
-
99
- for datastore in datastores:
100
- data.append(
101
- [
102
- datastore.id,
103
- datastore.name,
104
- datastore.type,
105
- datastore.region_uuid,
106
- ]
107
- )
108
-
109
- for row in data:
110
- click.echo(
111
- "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
112
- file=config.stdout,
113
- )
114
-
115
-
116
- @project.command()
117
- @pass_config
118
- def list_services(config):
119
- """List project services."""
120
- data = [
121
- ["ID", "NAME", "HOSTNAME", "REGION_UUID"],
122
- [
123
- "-" * 80,
124
- "-" * 80,
125
- "-" * 80,
126
- "-" * 80,
127
- ],
128
- ]
129
- project = config.trainml.run(
130
- config.trainml.client.projects.get(config.trainml.client.project)
131
- )
132
-
133
- services = config.trainml.run(project.list_services())
134
-
135
- for service in services:
136
- data.append(
137
- [
138
- service.id,
139
- service.name,
140
- service.hostname,
141
- service.region_uuid,
142
- ]
143
- )
144
-
145
- for row in data:
146
- click.echo(
147
- "{: >38.36} {: >30.28} {: >30.28} {: >38.36}" "".format(*row),
148
- file=config.stdout,
149
- )
@@ -1,126 +0,0 @@
1
- import json
2
- import logging
3
-
4
-
5
- class Reservations(object):
6
- def __init__(self, trainml):
7
- self.trainml = trainml
8
-
9
- async def get(self, provider_uuid, region_uuid, id, **kwargs):
10
- resp = await self.trainml._query(
11
- f"/provider/{provider_uuid}/region/{region_uuid}/reservation/{id}",
12
- "GET",
13
- kwargs,
14
- )
15
- return Reservation(self.trainml, **resp)
16
-
17
- async def list(self, provider_uuid, region_uuid, **kwargs):
18
- resp = await self.trainml._query(
19
- f"/provider/{provider_uuid}/region/{region_uuid}/reservation",
20
- "GET",
21
- kwargs,
22
- )
23
- reservations = [
24
- Reservation(self.trainml, **reservation) for reservation in resp
25
- ]
26
- return reservations
27
-
28
- async def create(
29
- self,
30
- provider_uuid,
31
- region_uuid,
32
- name,
33
- type,
34
- resource,
35
- hostname,
36
- **kwargs,
37
- ):
38
- logging.info(f"Creating Reservation {name}")
39
- data = dict(
40
- name=name,
41
- type=type,
42
- resource=resource,
43
- hostname=hostname,
44
- **kwargs,
45
- )
46
- payload = {k: v for k, v in data.items() if v is not None}
47
- resp = await self.trainml._query(
48
- f"/provider/{provider_uuid}/region/{region_uuid}/reservation",
49
- "POST",
50
- None,
51
- payload,
52
- )
53
- reservation = Reservation(self.trainml, **resp)
54
- logging.info(f"Created Reservation {name} with id {reservation.id}")
55
- return reservation
56
-
57
- async def remove(self, provider_uuid, region_uuid, id, **kwargs):
58
- await self.trainml._query(
59
- f"/provider/{provider_uuid}/region/{region_uuid}/reservation/{id}",
60
- "DELETE",
61
- kwargs,
62
- )
63
-
64
-
65
- class Reservation:
66
- def __init__(self, trainml, **kwargs):
67
- self.trainml = trainml
68
- self._reservation = kwargs
69
- self._id = self._reservation.get("reservation_id")
70
- self._provider_uuid = self._reservation.get("provider_uuid")
71
- self._region_uuid = self._reservation.get("region_uuid")
72
- self._type = self._reservation.get("type")
73
- self._name = self._reservation.get("name")
74
- self._resource = self._reservation.get("resource")
75
- self._hostname = self._reservation.get("hostname")
76
-
77
- @property
78
- def id(self) -> str:
79
- return self._id
80
-
81
- @property
82
- def provider_uuid(self) -> str:
83
- return self._provider_uuid
84
-
85
- @property
86
- def region_uuid(self) -> str:
87
- return self._region_uuid
88
-
89
- @property
90
- def type(self) -> str:
91
- return self._type
92
-
93
- @property
94
- def name(self) -> str:
95
- return self._name
96
-
97
- @property
98
- def resource(self) -> str:
99
- return self._resource
100
-
101
- @property
102
- def hostname(self) -> str:
103
- return self._hostname
104
-
105
- def __str__(self):
106
- return json.dumps({k: v for k, v in self._reservation.items()})
107
-
108
- def __repr__(self):
109
- return f"Reservation( trainml , **{self._reservation.__repr__()})"
110
-
111
- def __bool__(self):
112
- return bool(self._id)
113
-
114
- async def remove(self):
115
- await self.trainml._query(
116
- f"/provider/{self._provider_uuid}/region/{self._region_uuid}/reservation/{self._id}",
117
- "DELETE",
118
- )
119
-
120
- async def refresh(self):
121
- resp = await self.trainml._query(
122
- f"/provider/{self._provider_uuid}/region/{self._region_uuid}/reservation/{self._id}",
123
- "GET",
124
- )
125
- self.__init__(self.trainml, **resp)
126
- return self
trainml/projects.py DELETED
@@ -1,228 +0,0 @@
1
- import json
2
- import logging
3
-
4
-
5
- class Projects(object):
6
- def __init__(self, trainml):
7
- self.trainml = trainml
8
-
9
- async def get(self, id, **kwargs):
10
- resp = await self.trainml._query(f"/project/{id}", "GET", kwargs)
11
- return Project(self.trainml, **resp)
12
-
13
- async def get_current(self, **kwargs):
14
- resp = await self.trainml._query(
15
- f"/project/{self.trainml.project}", "GET", kwargs
16
- )
17
- return Project(self.trainml, **resp)
18
-
19
- async def list(self, **kwargs):
20
- resp = await self.trainml._query(f"/project", "GET", kwargs)
21
- projects = [Project(self.trainml, **project) for project in resp]
22
- return projects
23
-
24
- async def create(self, name, copy_keys=False, **kwargs):
25
- data = dict(
26
- name=name,
27
- copy_keys=copy_keys,
28
- )
29
- payload = {k: v for k, v in data.items() if v is not None}
30
- logging.info(f"Creating Project {name}")
31
- resp = await self.trainml._query("/project", "POST", None, payload)
32
- project = Project(self.trainml, **resp)
33
- logging.info(f"Created Project {name} with id {project.id}")
34
-
35
- return project
36
-
37
- async def remove(self, id, **kwargs):
38
- await self.trainml._query(f"/project/{id}", "DELETE", kwargs)
39
-
40
-
41
- class ProjectDatastore:
42
- def __init__(self, trainml, **kwargs):
43
- self.trainml = trainml
44
- self._datastore = kwargs
45
- self._id = self._datastore.get("id")
46
- self._project_uuid = self._datastore.get("project_uuid")
47
- self._name = self._datastore.get("name")
48
- self._type = self._datastore.get("type")
49
- self._region_uuid = self._datastore.get("region_uuid")
50
-
51
- @property
52
- def id(self) -> str:
53
- return self._id
54
-
55
- @property
56
- def project_uuid(self) -> str:
57
- return self._project_uuid
58
-
59
- @property
60
- def name(self) -> str:
61
- return self._name
62
-
63
- @property
64
- def type(self) -> str:
65
- return self._type
66
-
67
- @property
68
- def region_uuid(self) -> str:
69
- return self._region_uuid
70
-
71
- def __str__(self):
72
- return json.dumps({k: v for k, v in self._datastore.items()})
73
-
74
- def __repr__(self):
75
- return f"ProjectDatastore( trainml , **{self._datastore.__repr__()})"
76
-
77
- def __bool__(self):
78
- return bool(self._id)
79
-
80
-
81
- class ProjectDataConnector:
82
- def __init__(self, trainml, **kwargs):
83
- self.trainml = trainml
84
- self._data_connector = kwargs
85
- self._id = self._data_connector.get("id")
86
- self._project_uuid = self._data_connector.get("project_uuid")
87
- self._name = self._data_connector.get("name")
88
- self._type = self._data_connector.get("type")
89
- self._region_uuid = self._data_connector.get("region_uuid")
90
-
91
- @property
92
- def id(self) -> str:
93
- return self._id
94
-
95
- @property
96
- def project_uuid(self) -> str:
97
- return self._project_uuid
98
-
99
- @property
100
- def name(self) -> str:
101
- return self._name
102
-
103
- @property
104
- def type(self) -> str:
105
- return self._type
106
-
107
- @property
108
- def region_uuid(self) -> str:
109
- return self._region_uuid
110
-
111
- def __str__(self):
112
- return json.dumps({k: v for k, v in self._data_connector.items()})
113
-
114
- def __repr__(self):
115
- return f"ProjectDataConnector( trainml , **{self._data_connector.__repr__()})"
116
-
117
- def __bool__(self):
118
- return bool(self._id)
119
-
120
-
121
- class ProjectService:
122
- def __init__(self, trainml, **kwargs):
123
- self.trainml = trainml
124
- self._service = kwargs
125
- self._id = self._service.get("id")
126
- self._project_uuid = self._service.get("project_uuid")
127
- self._name = self._service.get("name")
128
- self._hostname = self._service.get("hostname")
129
- self._public = self._service.get("public")
130
- self._region_uuid = self._service.get("region_uuid")
131
-
132
- @property
133
- def id(self) -> str:
134
- return self._id
135
-
136
- @property
137
- def project_uuid(self) -> str:
138
- return self._project_uuid
139
-
140
- @property
141
- def name(self) -> str:
142
- return self._name
143
-
144
- @property
145
- def hostname(self) -> str:
146
- return self._hostname
147
-
148
- @property
149
- def public(self) -> bool:
150
- return self._public
151
-
152
- @property
153
- def region_uuid(self) -> str:
154
- return self._region_uuid
155
-
156
- def __str__(self):
157
- return json.dumps({k: v for k, v in self._service.items()})
158
-
159
- def __repr__(self):
160
- return f"ProjectService( trainml , **{self._service.__repr__()})"
161
-
162
- def __bool__(self):
163
- return bool(self._id)
164
-
165
-
166
- class Project:
167
- def __init__(self, trainml, **kwargs):
168
- self.trainml = trainml
169
- self._project = kwargs
170
- self._id = self._project.get("id")
171
- self._name = self._project.get("name")
172
- self._is_owner = self._project.get("owner")
173
- self._owner_name = self._project.get("owner_name")
174
-
175
- @property
176
- def id(self) -> str:
177
- return self._id
178
-
179
- @property
180
- def name(self) -> str:
181
- return self._name
182
-
183
- @property
184
- def is_owner(self) -> bool:
185
- return self._is_owner
186
-
187
- @property
188
- def owner_name(self) -> str:
189
- return self._owner_name
190
-
191
- def __str__(self):
192
- return json.dumps({k: v for k, v in self._project.items()})
193
-
194
- def __repr__(self):
195
- return f"Project( trainml , **{self._project.__repr__()})"
196
-
197
- def __bool__(self):
198
- return bool(self._id)
199
-
200
- async def remove(self):
201
- await self.trainml._query(f"/project/{self._id}", "DELETE")
202
-
203
- async def list_datastores(self):
204
- resp = await self.trainml._query(f"/project/{self._id}/datastores", "GET")
205
- datastores = [ProjectDatastore(self.trainml, **datastore) for datastore in resp]
206
- return datastores
207
-
208
- async def list_data_connectors(self):
209
- resp = await self.trainml._query(f"/project/{self._id}/data_connectors", "GET")
210
- data_connectors = [
211
- ProjectDataConnector(self.trainml, **data_connector)
212
- for data_connector in resp
213
- ]
214
- return data_connectors
215
-
216
- async def list_services(self):
217
- resp = await self.trainml._query(f"/project/{self._id}/services", "GET")
218
- services = [ProjectService(self.trainml, **service) for service in resp]
219
- return services
220
-
221
- async def refresh_datastores(self):
222
- await self.trainml._query(f"/project/{self._id}/datastores", "PATCH")
223
-
224
- async def refresh_data_connectors(self):
225
- await self.trainml._query(f"/project/{self._id}/data_connectors", "PATCH")
226
-
227
- async def refresh_services(self):
228
- await self.trainml._query(f"/project/{self._id}/services", "PATCH")