splight-cli 5.1.2__tar.gz → 5.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {splight_cli-5.1.2 → splight_cli-5.1.4}/PKG-INFO +49 -223
- {splight_cli-5.1.2 → splight_cli-5.1.4}/README.md +48 -222
- {splight_cli-5.1.2 → splight_cli-5.1.4}/pyproject.toml +1 -1
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/cli.py +23 -3
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/main.py +0 -20
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/context/__init__.py +2 -2
- splight_cli-5.1.4/splight_cli/context/exceptions.py +3 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/context/workspace.py +5 -1
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/component.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/loaders.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/.splightignore +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/Initialization +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/README.md +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/auto_readme.md +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/spec.json +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/templates/tests.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/tests/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/tests/test_component_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/utils.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/config/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/constants.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/context/context.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/alert/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/asset/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/attribute/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/component/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/datalake/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/file/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/tests/test_datalake_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/tests/test_resource_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/secret/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/component_builder.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/hub_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/resources/Dockerfile +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/resources/installer.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/component/tests/test_hub_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/server/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/server/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/server/server_builder.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/server/server_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/solution/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/solution/constants.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/solution/exceptions.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/hub/solution/solution_manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/settings.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/dict.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/parser.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/diff.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/logger.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/manager.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/alert.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/asset.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/attribute.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/base.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/component.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/component_object.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/file.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/function.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/metadata.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/routine.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/secret.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/state.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/tests/test_component/Initialization +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/tests/test_component/README.md +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/tests/test_component/main.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/tests/test_component/spec.json +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/tests/test_component/tests.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/__init__.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/input.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/json.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/loader.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/pprint.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/template.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/utils/yaml.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/version.py +0 -0
- {splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/workspace/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: splight-cli
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.4
|
|
4
4
|
Summary: Splight Command Line Interface
|
|
5
5
|
Author: Splight Dev
|
|
6
6
|
Author-email: dev@splight-ae.com
|
|
@@ -186,36 +186,6 @@ component, here we will only cover the different sub-commands you can use
|
|
|
186
186
|
code in the same directory where the command was executed. If `<path>`
|
|
187
187
|
is specified, then the files will be located in the specified path.
|
|
188
188
|
|
|
189
|
-
- Install component requirements
|
|
190
|
-
|
|
191
|
-
The command
|
|
192
|
-
|
|
193
|
-
```bash
|
|
194
|
-
splight component install-requirements <path>
|
|
195
|
-
```
|
|
196
|
-
|
|
197
|
-
Where the parameter `<path>` is the path to the component that will be installed
|
|
198
|
-
locally.
|
|
199
|
-
|
|
200
|
-
This command is useful for running locally a component for testing and development.
|
|
201
|
-
|
|
202
|
-
- Run locally a component
|
|
203
|
-
|
|
204
|
-
You can run locally a component, this is quite usefull for testing a comoponent
|
|
205
|
-
before push it and for development a new component. The command is the following
|
|
206
|
-
|
|
207
|
-
```bash
|
|
208
|
-
splight component run <path> [-r] [-rs]
|
|
209
|
-
```
|
|
210
|
-
|
|
211
|
-
This command will run a component locally. Optionally you can use the flag
|
|
212
|
-
`-r/--reset-input`, so you will be asked to configure some parameters for the
|
|
213
|
-
component. If it is the first time you run a component, you will see some messages in
|
|
214
|
-
the terminal for input some parameters values that are needed for running correctly
|
|
215
|
-
the component. Also, you can use the flag `-rs/--run-spec` for using a custom
|
|
216
|
-
configuration different to the one defined in the `spec.json` file. In the following
|
|
217
|
-
section we will dive in in the usage of the file `spec.json`.
|
|
218
|
-
|
|
219
189
|
- Create component Readme
|
|
220
190
|
|
|
221
191
|
As a component developer, you can generate a README.md file automatically using the
|
|
@@ -379,63 +349,81 @@ This will create a directory with the following structure:
|
|
|
379
349
|
|
|
380
350
|
```
|
|
381
351
|
<name>-<version>
|
|
382
|
-
│
|
|
352
|
+
│ main.py
|
|
383
353
|
│ Initialization
|
|
384
354
|
│ spec.json
|
|
385
355
|
│ README.md
|
|
386
356
|
```
|
|
387
357
|
|
|
388
|
-
- `
|
|
358
|
+
- `main.py` : The main file to contian the component's code.
|
|
389
359
|
- `Initialization` : Execute instructions for the component to be initialized.
|
|
390
360
|
- `spec.json` : JSON file where the component metadata is set.
|
|
391
361
|
- `README.md` : Text that describes the component and how it works
|
|
392
362
|
|
|
393
363
|
#### Component Core
|
|
394
364
|
|
|
395
|
-
When creating a component, inside `
|
|
365
|
+
When creating a component, inside `main.py` you will find a component template
|
|
366
|
+
already written in python for you, in order to make it easier to write the component code.
|
|
396
367
|
|
|
397
|
-
For example, when you create an algorithm component, in `
|
|
368
|
+
For example, when you create an algorithm component, in `main.py` will have the following:
|
|
398
369
|
|
|
399
370
|
```python
|
|
400
371
|
import random
|
|
372
|
+
from typing import Optional, Type
|
|
401
373
|
|
|
402
|
-
|
|
403
|
-
from splight_lib.component
|
|
374
|
+
import typer
|
|
375
|
+
from splight_lib.component import SplightBaseComponent
|
|
404
376
|
from splight_lib.execution import Task
|
|
377
|
+
from splight_lib.logging import getLogger
|
|
378
|
+
from splight_lib.models import Number
|
|
405
379
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
logger = logging.getLogger()
|
|
380
|
+
app = typer.Typer(pretty_exceptions_enable=False)
|
|
410
381
|
|
|
411
382
|
|
|
412
|
-
class
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
logger.info("It worked!")
|
|
383
|
+
class ExampleComponent(SplightBaseComponent):
|
|
384
|
+
def __init__(self, component_id: str):
|
|
385
|
+
super().__init__(component_id)
|
|
386
|
+
self._logger = getLogger("MyComponent")
|
|
417
387
|
|
|
418
388
|
def start(self):
|
|
419
|
-
|
|
420
|
-
self.execution_client.start(
|
|
389
|
+
self.execution_engine.start(
|
|
421
390
|
Task(
|
|
422
|
-
handler=self.
|
|
423
|
-
args=
|
|
424
|
-
period=self.period
|
|
391
|
+
handler=self._run,
|
|
392
|
+
args=(self.input.min, self.input.max),
|
|
393
|
+
period=self.input.period,
|
|
425
394
|
)
|
|
426
395
|
)
|
|
427
396
|
|
|
428
|
-
def
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
self.datalake_client.save(
|
|
435
|
-
Variable,
|
|
436
|
-
instances=[Variable(args=args)],
|
|
437
|
-
collection=self.collection_name
|
|
397
|
+
def _run(self, min_value: float, max_value: float):
|
|
398
|
+
value = self._give_a_random_value(
|
|
399
|
+
self.input.lower_bound, self.input.upper_bound
|
|
400
|
+
)
|
|
401
|
+
preds = Number(
|
|
402
|
+
value=value,
|
|
438
403
|
)
|
|
404
|
+
preds.save()
|
|
405
|
+
self._logger.info(f"\nValue = {value}\n")
|
|
406
|
+
|
|
407
|
+
def _give_a_random_value(self, min: float, max: float) -> float:
|
|
408
|
+
return (max - min) * random.random() + min
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
@app.command()
|
|
412
|
+
def main(
|
|
413
|
+
component_id: str = typer.Option(...),
|
|
414
|
+
input: Optional[str] = typer.Option(None),
|
|
415
|
+
):
|
|
416
|
+
logger = getLogger("MyComponent")
|
|
417
|
+
component = ExampleComponent(component_id=component_id)
|
|
418
|
+
try:
|
|
419
|
+
component.start()
|
|
420
|
+
except Exception as exc:
|
|
421
|
+
logger.exception(exc, tags="EXCEPTION")
|
|
422
|
+
component.stop()
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
if __name__ == "__main__":
|
|
426
|
+
app()
|
|
439
427
|
```
|
|
440
428
|
|
|
441
429
|
The component class must always be called `Main` and must inherit from one of
|
|
@@ -445,165 +433,3 @@ should be implemented by the developer that is writting the component. Also, we
|
|
|
445
433
|
provide you a lot of useful functions in our package so you can use them to interact
|
|
446
434
|
with the platform and make better components
|
|
447
435
|
|
|
448
|
-
#### Component Initialization
|
|
449
|
-
|
|
450
|
-
In the case you need some previous steps to be run before the component is executed,
|
|
451
|
-
you can use the file `Initialization`. When you create your component with Splight Hub,
|
|
452
|
-
`Initialization` will have the following:
|
|
453
|
-
|
|
454
|
-
```json
|
|
455
|
-
RUN pip install splight-lib==<some lib version>
|
|
456
|
-
```
|
|
457
|
-
|
|
458
|
-
This command will be run before the component is executed. You can add more lines like
|
|
459
|
-
this, or maybe even create a requirements.txt file in the component directory and just
|
|
460
|
-
leave `Initialization` as:
|
|
461
|
-
|
|
462
|
-
```json
|
|
463
|
-
RUN pip install -r requirements.txt
|
|
464
|
-
```
|
|
465
|
-
|
|
466
|
-
#### Component Configuration
|
|
467
|
-
|
|
468
|
-
The file `spec.json` defines all the specification for the component, in this file you
|
|
469
|
-
will find generic information about the component like the name and version but also
|
|
470
|
-
you will find the input and output parameters.
|
|
471
|
-
|
|
472
|
-
The structure of the file `spec.json` is the following
|
|
473
|
-
|
|
474
|
-
```json
|
|
475
|
-
{
|
|
476
|
-
"name": "component_name",
|
|
477
|
-
"version": "component_version",
|
|
478
|
-
"tags": ["tag1", "tag2"],
|
|
479
|
-
"custom_types": [],
|
|
480
|
-
"input": [],
|
|
481
|
-
"output": []
|
|
482
|
-
}
|
|
483
|
-
```
|
|
484
|
-
|
|
485
|
-
A component can have different inputs and outputs that are previously defined, these
|
|
486
|
-
parameters can any or primite Python types like `str`, `int`, `float`, `bool` or `file`,
|
|
487
|
-
can also take the value of any Splight parameter like `Asset`, `Attribute`,
|
|
488
|
-
`Component`, `Mapping`, `Query`, but also can be custom type defined in the
|
|
489
|
-
`"custom_types"` key.
|
|
490
|
-
A parameter have the structure
|
|
491
|
-
|
|
492
|
-
```json
|
|
493
|
-
{
|
|
494
|
-
"name": <name>,
|
|
495
|
-
"type": <type>,
|
|
496
|
-
"required": <required>,
|
|
497
|
-
"value": <value>
|
|
498
|
-
}
|
|
499
|
-
```
|
|
500
|
-
|
|
501
|
-
and also accepts the key `"multiple"` with `true` or `false` value to specify if the
|
|
502
|
-
parameter can take multiple value that is parsed as a list when the component is running.
|
|
503
|
-
|
|
504
|
-
A custom type is defined by the following format
|
|
505
|
-
|
|
506
|
-
```json
|
|
507
|
-
{
|
|
508
|
-
"name": "<custom_type_name",
|
|
509
|
-
"fields": [
|
|
510
|
-
{
|
|
511
|
-
"name": "<name>",
|
|
512
|
-
"type": "<type>",
|
|
513
|
-
"required": "<required>",
|
|
514
|
-
"multiple": "<multiple>",
|
|
515
|
-
"value": "<value>"
|
|
516
|
-
}
|
|
517
|
-
]
|
|
518
|
-
}
|
|
519
|
-
```
|
|
520
|
-
|
|
521
|
-
Then the custom type can be used as input or output. For example, we create
|
|
522
|
-
a custom type:
|
|
523
|
-
|
|
524
|
-
```json
|
|
525
|
-
{
|
|
526
|
-
"name": "AssetAttribute",
|
|
527
|
-
"fields": [
|
|
528
|
-
{
|
|
529
|
-
"name": "asset",
|
|
530
|
-
"type": "Asset",
|
|
531
|
-
"required": true,
|
|
532
|
-
"multiple": false
|
|
533
|
-
},
|
|
534
|
-
{
|
|
535
|
-
"name": "attribute",
|
|
536
|
-
"type": "Attribute",
|
|
537
|
-
"depends_on": "asset",
|
|
538
|
-
"required": true,
|
|
539
|
-
"multiple": false
|
|
540
|
-
}
|
|
541
|
-
]
|
|
542
|
-
}
|
|
543
|
-
```
|
|
544
|
-
|
|
545
|
-
So we can use it as follows
|
|
546
|
-
|
|
547
|
-
```json
|
|
548
|
-
{
|
|
549
|
-
"name": "CustomTypeVar",
|
|
550
|
-
"type": "AssetAttribute",
|
|
551
|
-
"required": true,
|
|
552
|
-
"multiple": false,
|
|
553
|
-
"value": [
|
|
554
|
-
{
|
|
555
|
-
"name": "asset",
|
|
556
|
-
"type": "Asset",
|
|
557
|
-
"required": true,
|
|
558
|
-
"multiple": false,
|
|
559
|
-
"value": <asset_id>
|
|
560
|
-
},
|
|
561
|
-
{
|
|
562
|
-
"name": "attribute",
|
|
563
|
-
"type": "Attribute",
|
|
564
|
-
"required": true,
|
|
565
|
-
"multiple": false,
|
|
566
|
-
"value": <attribute_id>
|
|
567
|
-
}
|
|
568
|
-
]
|
|
569
|
-
}
|
|
570
|
-
```
|
|
571
|
-
|
|
572
|
-
This is just an example but you can create custom types as complex as you want,
|
|
573
|
-
the limit is your imagination.
|
|
574
|
-
|
|
575
|
-
#### Running Locally
|
|
576
|
-
|
|
577
|
-
You can run the component locally before pushing it to the platform with the `--local` option:
|
|
578
|
-
|
|
579
|
-
```bash
|
|
580
|
-
splight component run <component directory> --local
|
|
581
|
-
```
|
|
582
|
-
|
|
583
|
-
This way, the component will run using local clients for database and datalake. This is extremely
|
|
584
|
-
useful for development since you can create instances of the different database objects in the
|
|
585
|
-
local database for running different scenearios or differents tests. The same can be applied for
|
|
586
|
-
datalake data, the local client stores the data in files. In both cases, for database and datalake,
|
|
587
|
-
the files are created in the same directory as the `__init__.py` file of the component, so you
|
|
588
|
-
can modified it based on your needs.
|
|
589
|
-
|
|
590
|
-
You can interact with the local databases using the library, for example
|
|
591
|
-
|
|
592
|
-
```python
|
|
593
|
-
from splight_models import Asset, Attribute, Number
|
|
594
|
-
from splight_lib.client.database import LocalDatabaseClient
|
|
595
|
-
from splight_lib.client.datalake import LocalDatalakeClient
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
component_path = ...
|
|
599
|
-
db_client = LocalDatabaseClient(namespace="default", path=component_path)
|
|
600
|
-
dl_client = LocalDatalakeClient(namespace="default", path=component_path)
|
|
601
|
-
|
|
602
|
-
all_assets = db_client.get(Asset)
|
|
603
|
-
attribute = Attribute(name="SomeAttribute")
|
|
604
|
-
|
|
605
|
-
db_client.save(attribute)
|
|
606
|
-
|
|
607
|
-
df = client.get_dataframe(Number, asset=all_assets[0].id, attribute=attribute.id)
|
|
608
|
-
```
|
|
609
|
-
|
|
@@ -159,36 +159,6 @@ component, here we will only cover the different sub-commands you can use
|
|
|
159
159
|
code in the same directory where the command was executed. If `<path>`
|
|
160
160
|
is specified, then the files will be located in the specified path.
|
|
161
161
|
|
|
162
|
-
- Install component requirements
|
|
163
|
-
|
|
164
|
-
The command
|
|
165
|
-
|
|
166
|
-
```bash
|
|
167
|
-
splight component install-requirements <path>
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
Where the parameter `<path>` is the path to the component that will be installed
|
|
171
|
-
locally.
|
|
172
|
-
|
|
173
|
-
This command is useful for running locally a component for testing and development.
|
|
174
|
-
|
|
175
|
-
- Run locally a component
|
|
176
|
-
|
|
177
|
-
You can run locally a component, this is quite usefull for testing a comoponent
|
|
178
|
-
before push it and for development a new component. The command is the following
|
|
179
|
-
|
|
180
|
-
```bash
|
|
181
|
-
splight component run <path> [-r] [-rs]
|
|
182
|
-
```
|
|
183
|
-
|
|
184
|
-
This command will run a component locally. Optionally you can use the flag
|
|
185
|
-
`-r/--reset-input`, so you will be asked to configure some parameters for the
|
|
186
|
-
component. If it is the first time you run a component, you will see some messages in
|
|
187
|
-
the terminal for input some parameters values that are needed for running correctly
|
|
188
|
-
the component. Also, you can use the flag `-rs/--run-spec` for using a custom
|
|
189
|
-
configuration different to the one defined in the `spec.json` file. In the following
|
|
190
|
-
section we will dive in in the usage of the file `spec.json`.
|
|
191
|
-
|
|
192
162
|
- Create component Readme
|
|
193
163
|
|
|
194
164
|
As a component developer, you can generate a README.md file automatically using the
|
|
@@ -352,63 +322,81 @@ This will create a directory with the following structure:
|
|
|
352
322
|
|
|
353
323
|
```
|
|
354
324
|
<name>-<version>
|
|
355
|
-
│
|
|
325
|
+
│ main.py
|
|
356
326
|
│ Initialization
|
|
357
327
|
│ spec.json
|
|
358
328
|
│ README.md
|
|
359
329
|
```
|
|
360
330
|
|
|
361
|
-
- `
|
|
331
|
+
- `main.py` : The main file to contian the component's code.
|
|
362
332
|
- `Initialization` : Execute instructions for the component to be initialized.
|
|
363
333
|
- `spec.json` : JSON file where the component metadata is set.
|
|
364
334
|
- `README.md` : Text that describes the component and how it works
|
|
365
335
|
|
|
366
336
|
#### Component Core
|
|
367
337
|
|
|
368
|
-
When creating a component, inside `
|
|
338
|
+
When creating a component, inside `main.py` you will find a component template
|
|
339
|
+
already written in python for you, in order to make it easier to write the component code.
|
|
369
340
|
|
|
370
|
-
For example, when you create an algorithm component, in `
|
|
341
|
+
For example, when you create an algorithm component, in `main.py` will have the following:
|
|
371
342
|
|
|
372
343
|
```python
|
|
373
344
|
import random
|
|
345
|
+
from typing import Optional, Type
|
|
374
346
|
|
|
375
|
-
|
|
376
|
-
from splight_lib.component
|
|
347
|
+
import typer
|
|
348
|
+
from splight_lib.component import SplightBaseComponent
|
|
377
349
|
from splight_lib.execution import Task
|
|
350
|
+
from splight_lib.logging import getLogger
|
|
351
|
+
from splight_lib.models import Number
|
|
378
352
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
logger = logging.getLogger()
|
|
353
|
+
app = typer.Typer(pretty_exceptions_enable=False)
|
|
383
354
|
|
|
384
355
|
|
|
385
|
-
class
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
logger.info("It worked!")
|
|
356
|
+
class ExampleComponent(SplightBaseComponent):
|
|
357
|
+
def __init__(self, component_id: str):
|
|
358
|
+
super().__init__(component_id)
|
|
359
|
+
self._logger = getLogger("MyComponent")
|
|
390
360
|
|
|
391
361
|
def start(self):
|
|
392
|
-
|
|
393
|
-
self.execution_client.start(
|
|
362
|
+
self.execution_engine.start(
|
|
394
363
|
Task(
|
|
395
|
-
handler=self.
|
|
396
|
-
args=
|
|
397
|
-
period=self.period
|
|
364
|
+
handler=self._run,
|
|
365
|
+
args=(self.input.min, self.input.max),
|
|
366
|
+
period=self.input.period,
|
|
398
367
|
)
|
|
399
368
|
)
|
|
400
369
|
|
|
401
|
-
def
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
self.datalake_client.save(
|
|
408
|
-
Variable,
|
|
409
|
-
instances=[Variable(args=args)],
|
|
410
|
-
collection=self.collection_name
|
|
370
|
+
def _run(self, min_value: float, max_value: float):
|
|
371
|
+
value = self._give_a_random_value(
|
|
372
|
+
self.input.lower_bound, self.input.upper_bound
|
|
373
|
+
)
|
|
374
|
+
preds = Number(
|
|
375
|
+
value=value,
|
|
411
376
|
)
|
|
377
|
+
preds.save()
|
|
378
|
+
self._logger.info(f"\nValue = {value}\n")
|
|
379
|
+
|
|
380
|
+
def _give_a_random_value(self, min: float, max: float) -> float:
|
|
381
|
+
return (max - min) * random.random() + min
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@app.command()
|
|
385
|
+
def main(
|
|
386
|
+
component_id: str = typer.Option(...),
|
|
387
|
+
input: Optional[str] = typer.Option(None),
|
|
388
|
+
):
|
|
389
|
+
logger = getLogger("MyComponent")
|
|
390
|
+
component = ExampleComponent(component_id=component_id)
|
|
391
|
+
try:
|
|
392
|
+
component.start()
|
|
393
|
+
except Exception as exc:
|
|
394
|
+
logger.exception(exc, tags="EXCEPTION")
|
|
395
|
+
component.stop()
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
if __name__ == "__main__":
|
|
399
|
+
app()
|
|
412
400
|
```
|
|
413
401
|
|
|
414
402
|
The component class must always be called `Main` and must inherit from one of
|
|
@@ -417,165 +405,3 @@ execution of the component starts when the method `start()` is called, so the me
|
|
|
417
405
|
should be implemented by the developer that is writting the component. Also, we
|
|
418
406
|
provide you a lot of useful functions in our package so you can use them to interact
|
|
419
407
|
with the platform and make better components
|
|
420
|
-
|
|
421
|
-
#### Component Initialization
|
|
422
|
-
|
|
423
|
-
In the case you need some previous steps to be run before the component is executed,
|
|
424
|
-
you can use the file `Initialization`. When you create your component with Splight Hub,
|
|
425
|
-
`Initialization` will have the following:
|
|
426
|
-
|
|
427
|
-
```json
|
|
428
|
-
RUN pip install splight-lib==<some lib version>
|
|
429
|
-
```
|
|
430
|
-
|
|
431
|
-
This command will be run before the component is executed. You can add more lines like
|
|
432
|
-
this, or maybe even create a requirements.txt file in the component directory and just
|
|
433
|
-
leave `Initialization` as:
|
|
434
|
-
|
|
435
|
-
```json
|
|
436
|
-
RUN pip install -r requirements.txt
|
|
437
|
-
```
|
|
438
|
-
|
|
439
|
-
#### Component Configuration
|
|
440
|
-
|
|
441
|
-
The file `spec.json` defines all the specification for the component, in this file you
|
|
442
|
-
will find generic information about the component like the name and version but also
|
|
443
|
-
you will find the input and output parameters.
|
|
444
|
-
|
|
445
|
-
The structure of the file `spec.json` is the following
|
|
446
|
-
|
|
447
|
-
```json
|
|
448
|
-
{
|
|
449
|
-
"name": "component_name",
|
|
450
|
-
"version": "component_version",
|
|
451
|
-
"tags": ["tag1", "tag2"],
|
|
452
|
-
"custom_types": [],
|
|
453
|
-
"input": [],
|
|
454
|
-
"output": []
|
|
455
|
-
}
|
|
456
|
-
```
|
|
457
|
-
|
|
458
|
-
A component can have different inputs and outputs that are previously defined, these
|
|
459
|
-
parameters can any or primite Python types like `str`, `int`, `float`, `bool` or `file`,
|
|
460
|
-
can also take the value of any Splight parameter like `Asset`, `Attribute`,
|
|
461
|
-
`Component`, `Mapping`, `Query`, but also can be custom type defined in the
|
|
462
|
-
`"custom_types"` key.
|
|
463
|
-
A parameter have the structure
|
|
464
|
-
|
|
465
|
-
```json
|
|
466
|
-
{
|
|
467
|
-
"name": <name>,
|
|
468
|
-
"type": <type>,
|
|
469
|
-
"required": <required>,
|
|
470
|
-
"value": <value>
|
|
471
|
-
}
|
|
472
|
-
```
|
|
473
|
-
|
|
474
|
-
and also accepts the key `"multiple"` with `true` or `false` value to specify if the
|
|
475
|
-
parameter can take multiple value that is parsed as a list when the component is running.
|
|
476
|
-
|
|
477
|
-
A custom type is defined by the following format
|
|
478
|
-
|
|
479
|
-
```json
|
|
480
|
-
{
|
|
481
|
-
"name": "<custom_type_name",
|
|
482
|
-
"fields": [
|
|
483
|
-
{
|
|
484
|
-
"name": "<name>",
|
|
485
|
-
"type": "<type>",
|
|
486
|
-
"required": "<required>",
|
|
487
|
-
"multiple": "<multiple>",
|
|
488
|
-
"value": "<value>"
|
|
489
|
-
}
|
|
490
|
-
]
|
|
491
|
-
}
|
|
492
|
-
```
|
|
493
|
-
|
|
494
|
-
Then the custom type can be used as input or output. For example, we create
|
|
495
|
-
a custom type:
|
|
496
|
-
|
|
497
|
-
```json
|
|
498
|
-
{
|
|
499
|
-
"name": "AssetAttribute",
|
|
500
|
-
"fields": [
|
|
501
|
-
{
|
|
502
|
-
"name": "asset",
|
|
503
|
-
"type": "Asset",
|
|
504
|
-
"required": true,
|
|
505
|
-
"multiple": false
|
|
506
|
-
},
|
|
507
|
-
{
|
|
508
|
-
"name": "attribute",
|
|
509
|
-
"type": "Attribute",
|
|
510
|
-
"depends_on": "asset",
|
|
511
|
-
"required": true,
|
|
512
|
-
"multiple": false
|
|
513
|
-
}
|
|
514
|
-
]
|
|
515
|
-
}
|
|
516
|
-
```
|
|
517
|
-
|
|
518
|
-
So we can use it as follows
|
|
519
|
-
|
|
520
|
-
```json
|
|
521
|
-
{
|
|
522
|
-
"name": "CustomTypeVar",
|
|
523
|
-
"type": "AssetAttribute",
|
|
524
|
-
"required": true,
|
|
525
|
-
"multiple": false,
|
|
526
|
-
"value": [
|
|
527
|
-
{
|
|
528
|
-
"name": "asset",
|
|
529
|
-
"type": "Asset",
|
|
530
|
-
"required": true,
|
|
531
|
-
"multiple": false,
|
|
532
|
-
"value": <asset_id>
|
|
533
|
-
},
|
|
534
|
-
{
|
|
535
|
-
"name": "attribute",
|
|
536
|
-
"type": "Attribute",
|
|
537
|
-
"required": true,
|
|
538
|
-
"multiple": false,
|
|
539
|
-
"value": <attribute_id>
|
|
540
|
-
}
|
|
541
|
-
]
|
|
542
|
-
}
|
|
543
|
-
```
|
|
544
|
-
|
|
545
|
-
This is just an example but you can create custom types as complex as you want,
|
|
546
|
-
the limit is your imagination.
|
|
547
|
-
|
|
548
|
-
#### Running Locally
|
|
549
|
-
|
|
550
|
-
You can run the component locally before pushing it to the platform with the `--local` option:
|
|
551
|
-
|
|
552
|
-
```bash
|
|
553
|
-
splight component run <component directory> --local
|
|
554
|
-
```
|
|
555
|
-
|
|
556
|
-
This way, the component will run using local clients for database and datalake. This is extremely
|
|
557
|
-
useful for development since you can create instances of the different database objects in the
|
|
558
|
-
local database for running different scenearios or differents tests. The same can be applied for
|
|
559
|
-
datalake data, the local client stores the data in files. In both cases, for database and datalake,
|
|
560
|
-
the files are created in the same directory as the `__init__.py` file of the component, so you
|
|
561
|
-
can modified it based on your needs.
|
|
562
|
-
|
|
563
|
-
You can interact with the local databases using the library, for example
|
|
564
|
-
|
|
565
|
-
```python
|
|
566
|
-
from splight_models import Asset, Attribute, Number
|
|
567
|
-
from splight_lib.client.database import LocalDatabaseClient
|
|
568
|
-
from splight_lib.client.datalake import LocalDatalakeClient
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
component_path = ...
|
|
572
|
-
db_client = LocalDatabaseClient(namespace="default", path=component_path)
|
|
573
|
-
dl_client = LocalDatalakeClient(namespace="default", path=component_path)
|
|
574
|
-
|
|
575
|
-
all_assets = db_client.get(Asset)
|
|
576
|
-
attribute = Attribute(name="SomeAttribute")
|
|
577
|
-
|
|
578
|
-
db_client.save(attribute)
|
|
579
|
-
|
|
580
|
-
df = client.get_dataframe(Number, asset=all_assets[0].id, attribute=attribute.id)
|
|
581
|
-
```
|
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
3
|
import typer
|
|
4
|
+
from rich.console import Console
|
|
4
5
|
|
|
5
6
|
from splight_cli.component import component_app
|
|
6
7
|
from splight_cli.config import config_app
|
|
8
|
+
from splight_cli.constants import error_style
|
|
7
9
|
from splight_cli.context import Context
|
|
10
|
+
from splight_cli.context.exceptions import MissingConfigurationFile
|
|
8
11
|
from splight_cli.engine import engine_app
|
|
9
12
|
from splight_cli.hub import hub_app
|
|
10
13
|
from splight_cli.solution import solution_app
|
|
11
14
|
from splight_cli.version import __version__
|
|
12
15
|
from splight_cli.workspace import workspace_app
|
|
13
16
|
|
|
17
|
+
console = Console()
|
|
14
18
|
app = typer.Typer(
|
|
15
19
|
name="Splight Command Line",
|
|
16
20
|
add_completion=True,
|
|
17
21
|
rich_markup_mode="rich",
|
|
18
|
-
|
|
22
|
+
invoke_without_command=True,
|
|
19
23
|
pretty_exceptions_enable=False,
|
|
20
24
|
)
|
|
21
25
|
|
|
@@ -39,5 +43,21 @@ def main(
|
|
|
39
43
|
version: Optional[bool] = typer.Option(
|
|
40
44
|
None, "--version", "-v", callback=version_callback, is_eager=True
|
|
41
45
|
),
|
|
42
|
-
):
|
|
43
|
-
|
|
46
|
+
) -> None:
|
|
47
|
+
try:
|
|
48
|
+
ctx.obj = Context(
|
|
49
|
+
new_workspace=(
|
|
50
|
+
True if ctx.invoked_subcommand == "configure" else False
|
|
51
|
+
)
|
|
52
|
+
)
|
|
53
|
+
except MissingConfigurationFile as exc:
|
|
54
|
+
console.print(
|
|
55
|
+
(
|
|
56
|
+
f"Error running Splight: {str(exc)}. "
|
|
57
|
+
"Run command 'splight configure' to configure your environment"
|
|
58
|
+
),
|
|
59
|
+
style=error_style,
|
|
60
|
+
)
|
|
61
|
+
raise typer.Exit(code=1)
|
|
62
|
+
if ctx.invoked_subcommand is None:
|
|
63
|
+
ctx.get_help()
|
|
@@ -8,8 +8,6 @@ from splight_lib.logging import getLogger
|
|
|
8
8
|
from splight_lib.models import Number
|
|
9
9
|
|
|
10
10
|
app = typer.Typer(pretty_exceptions_enable=False)
|
|
11
|
-
MyAsset = Type["MyAsset"]
|
|
12
|
-
MyRoutine = Type["MyRoutine"]
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
class {{component_name}}(SplightBaseComponent):
|
|
@@ -39,24 +37,6 @@ class {{component_name}}(SplightBaseComponent):
|
|
|
39
37
|
def _give_a_random_value(self, min: float, max: float) -> float:
|
|
40
38
|
return (max - min) * random.random() + min
|
|
41
39
|
|
|
42
|
-
def command_myasset_print(self, my_asset: MyAsset):
|
|
43
|
-
print("Command for MyAsset")
|
|
44
|
-
|
|
45
|
-
def handle_myasset_create(self, my_asset: MyAsset):
|
|
46
|
-
print("MyAsset create")
|
|
47
|
-
|
|
48
|
-
def handle_myasset_delete(self, my_asset: MyAsset):
|
|
49
|
-
print("MyAsset delete")
|
|
50
|
-
|
|
51
|
-
def handle_MyRoutine_create(self, routine: MyRoutine):
|
|
52
|
-
print("MyRoutine create")
|
|
53
|
-
|
|
54
|
-
def handle_MyRoutine_update(self, routine: MyRoutine):
|
|
55
|
-
print("MyRoutine update")
|
|
56
|
-
|
|
57
|
-
def handle_MyRoutine_delete(self, routine: MyRoutine):
|
|
58
|
-
print("MyRoutine delete")
|
|
59
|
-
|
|
60
40
|
|
|
61
41
|
@app.command()
|
|
62
42
|
def main(
|
|
@@ -8,8 +8,8 @@ console = Console()
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class Context:
|
|
11
|
-
def __init__(self):
|
|
12
|
-
self.__workspace = WorkspaceManager()
|
|
11
|
+
def __init__(self, new_workspace: bool = False):
|
|
12
|
+
self.__workspace = WorkspaceManager(new_workspace=new_workspace)
|
|
13
13
|
|
|
14
14
|
@property
|
|
15
15
|
def workspace(self):
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from typing import List
|
|
3
4
|
|
|
@@ -7,6 +8,7 @@ from splight_cli.constants import (
|
|
|
7
8
|
DEFAULT_WORKSPACE_NAME,
|
|
8
9
|
DEFAULT_WORKSPACES,
|
|
9
10
|
)
|
|
11
|
+
from splight_cli.context.exceptions import MissingConfigurationFile
|
|
10
12
|
from splight_cli.settings import SplightCLIConfig, SplightCLISettings
|
|
11
13
|
from splight_cli.utils.yaml import get_yaml_from_file, save_yaml_to_file
|
|
12
14
|
|
|
@@ -31,7 +33,9 @@ class NotExistingWorkspace(Exception):
|
|
|
31
33
|
|
|
32
34
|
|
|
33
35
|
class WorkspaceManager:
|
|
34
|
-
def __init__(self):
|
|
36
|
+
def __init__(self, new_workspace: bool = False):
|
|
37
|
+
if not os.path.exists(CONFIG_FILE) and not new_workspace:
|
|
38
|
+
raise MissingConfigurationFile("Config file does not exist")
|
|
35
39
|
self.config_file = CONFIG_FILE
|
|
36
40
|
path = Path(self.config_file)
|
|
37
41
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/component/tests/test_component_manager.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/tests/test_datalake_manager.py
RENAMED
|
File without changes
|
{splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/engine/manager/tests/test_resource_manager.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{splight_cli-5.1.2 → splight_cli-5.1.4}/splight_cli/solution/resource/models/component_object.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|