orchestrator-core 3.1.0rc1__py3-none-any.whl → 3.1.2rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
orchestrator/__init__.py CHANGED
@@ -13,7 +13,7 @@
13
13
 
14
14
  """This is the orchestrator workflow engine."""
15
15
 
16
- __version__ = "3.1.0rc1"
16
+ __version__ = "3.1.2rc1"
17
17
 
18
18
  from orchestrator.app import OrchestratorCore
19
19
  from orchestrator.settings import app_settings
orchestrator/db/models.py CHANGED
@@ -92,7 +92,7 @@ class InputStateTable(BaseModel):
92
92
 
93
93
  input_state_id = mapped_column(UUIDType, primary_key=True, server_default=text("uuid_generate_v4()"), index=True)
94
94
  process_id = mapped_column("pid", UUIDType, ForeignKey("processes.pid"), nullable=False)
95
- input_state = mapped_column(pg.JSONB(), nullable=False) # type: ignore
95
+ input_state = mapped_column(pg.JSONB(), nullable=False)
96
96
  input_time = mapped_column(UtcTimestamp, server_default=text("current_timestamp()"), nullable=False)
97
97
  input_type = mapped_column(Enum(InputType), nullable=False)
98
98
 
@@ -137,7 +137,7 @@ class ProcessStepTable(BaseModel):
137
137
  )
138
138
  name = mapped_column(String(), nullable=False)
139
139
  status = mapped_column(String(50), nullable=False)
140
- state = mapped_column(pg.JSONB(), nullable=False) # type: ignore
140
+ state = mapped_column(pg.JSONB(), nullable=False)
141
141
  created_by = mapped_column(String(255), nullable=True)
142
142
  executed_at = mapped_column(UtcTimestamp, server_default=text("statement_timestamp()"), nullable=False)
143
143
  commit_hash = mapped_column(String(40), nullable=True, default=GIT_COMMIT_HASH)
@@ -642,7 +642,7 @@ class SubscriptionMetadataTable(BaseModel):
642
642
  primary_key=True,
643
643
  index=True,
644
644
  )
645
- metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False) # type: ignore
645
+ metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)
646
646
 
647
647
  @staticmethod
648
648
  def find_by_subscription_id(subscription_id: str) -> SubscriptionMetadataTable | None:
@@ -651,6 +651,8 @@ class SubscriptionMetadataTable(BaseModel):
651
651
 
652
652
  class SubscriptionSearchView(BaseModel):
653
653
  __tablename__ = "subscriptions_search"
654
+ __table_args__ = {"info": {"materialized_view": True}}
655
+
654
656
  subscription_id = mapped_column(
655
657
  UUIDType, ForeignKey("subscriptions.subscription_id"), nullable=False, index=True, primary_key=True
656
658
  )
@@ -41,6 +41,14 @@ target_metadata = BaseModel.metadata
41
41
  # ... etc.
42
42
 
43
43
 
44
+ def include_object(object, name, type_, reflected, compare_to): # type: ignore
45
+ """Determines if an object should be included."""
46
+
47
+ if type_ == "table" and object.info.get("materialized_view", False):
48
+ return False
49
+ return True
50
+
51
+
44
52
  def run_migrations_offline() -> None:
45
53
  """Run migrations in 'offline' mode.
46
54
 
@@ -55,7 +63,11 @@ def run_migrations_offline() -> None:
55
63
  """
56
64
  url = config.get_main_option("sqlalchemy.url")
57
65
  context.configure(
58
- url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}
66
+ url=url,
67
+ target_metadata=target_metadata,
68
+ literal_binds=True,
69
+ dialect_opts={"paramstyle": "named"},
70
+ include_object=include_object,
59
71
  )
60
72
 
61
73
  with context.begin_transaction():
@@ -90,6 +102,7 @@ def run_migrations_online() -> None:
90
102
  target_metadata=target_metadata,
91
103
  process_revision_directives=process_revision_directives,
92
104
  compare_type=True,
105
+ include_object=include_object,
93
106
  )
94
107
 
95
108
  try:
@@ -157,7 +157,7 @@ def upgrade() -> None:
157
157
  sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
158
158
  sa.Column("name", sa.String(), nullable=False),
159
159
  sa.Column("status", sa.String(length=50), nullable=False),
160
- sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False), # type: ignore
160
+ sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
161
161
  sa.Column("created_by", sa.String(length=255), nullable=True),
162
162
  sa.Column(
163
163
  "executed_at",
@@ -29,7 +29,7 @@ def upgrade() -> None:
29
29
  nullable=False,
30
30
  index=True,
31
31
  ),
32
- sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False), # type: ignore
32
+ sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
33
33
  sa.ForeignKeyConstraint(["subscription_id"], ["subscriptions.subscription_id"], ondelete="CASCADE"),
34
34
  )
35
35
 
@@ -31,7 +31,7 @@ def upgrade() -> None:
31
31
  nullable=False,
32
32
  ),
33
33
  sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
34
- sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False), # type: ignore
34
+ sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
35
35
  sa.Column(
36
36
  "input_time",
37
37
  db.models.UtcTimestamp(timezone=True),
@@ -150,65 +150,85 @@ def _build_arguments(func: StepFunc | InputStepFunc, state: State) -> list: # n
150
150
 
151
151
  Raises:
152
152
  KeyError: if requested argument is not in the state, or cannot be reconstructed as an initial domain model.
153
+ ValueError: if requested argument cannot be converted to the expected type.
153
154
 
154
155
  """
156
+
155
157
  sig = inspect.signature(func)
158
+ if not sig.parameters:
159
+ return []
160
+
161
+ def _convert_to_uuid(v: Any) -> UUID:
162
+ """Converts the value to a UUID instance if it is not already one."""
163
+ return v if isinstance(v, UUID) else UUID(v)
164
+
156
165
  arguments: list[Any] = []
157
- if sig.parameters:
158
- for name, param in sig.parameters.items():
159
- # Ignore dynamic arguments. Mostly need to deal with `const`
160
- if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
161
- logger.warning("*args and **kwargs are not supported as step params")
162
- continue
163
-
164
- # If we find an argument named "state" we use the whole state as argument to
165
- # This is mainly to be backward compatible with code that needs the whole state...
166
- # TODO: Remove this construction
167
- if name == "state":
168
- arguments.append(state)
169
- continue
170
-
171
- # Workaround for the fact that you can't call issubclass on typing types
166
+ for name, param in sig.parameters.items():
167
+ # Ignore dynamic arguments. Mostly need to deal with `const`
168
+ if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
169
+ logger.warning("*args and **kwargs are not supported as step params")
170
+ continue
171
+
172
+ # If we find an argument named "state" we use the whole state as argument to
173
+ # This is mainly to be backward compatible with code that needs the whole state...
174
+ # TODO: Remove this construction
175
+ if name == "state":
176
+ arguments.append(state)
177
+ continue
178
+
179
+ # Workaround for the fact that you can't call issubclass on typing types
180
+ try:
181
+ is_subscription_model_type = issubclass(param.annotation, SubscriptionModel)
182
+ except Exception:
183
+ is_subscription_model_type = False
184
+
185
+ if is_subscription_model_type:
186
+ subscription_id = _get_sub_id(state.get(name))
187
+ if subscription_id:
188
+ sub_mod = param.annotation.from_subscription(subscription_id)
189
+ arguments.append(sub_mod)
190
+ else:
191
+ logger.error("Could not find key in state.", key=name, state=state)
192
+ raise KeyError(f"Could not find key '{name}' in state.")
193
+ elif is_list_type(param.annotation, SubscriptionModel):
194
+ subscription_ids = [_get_sub_id(item) for item in state.get(name, [])]
195
+ # Actual type is first argument from list type
196
+ if (actual_type := get_args(param.annotation)[0]) == Any:
197
+ raise ValueError(
198
+ f"Step function argument '{param.name}' cannot be serialized from database with type 'Any'"
199
+ )
200
+ subscriptions = [actual_type.from_subscription(subscription_id) for subscription_id in subscription_ids]
201
+ arguments.append(subscriptions)
202
+ elif is_optional_type(param.annotation, SubscriptionModel):
203
+ subscription_id = _get_sub_id(state.get(name))
204
+ if subscription_id:
205
+ # Actual type is first argument from optional type
206
+ sub_mod = get_args(param.annotation)[0].from_subscription(subscription_id)
207
+ arguments.append(sub_mod)
208
+ else:
209
+ arguments.append(None)
210
+ elif param.default is not inspect.Parameter.empty:
211
+ arguments.append(state.get(name, param.default))
212
+ else:
172
213
  try:
173
- is_subscription_model_type = issubclass(param.annotation, SubscriptionModel)
174
- except Exception:
175
- is_subscription_model_type = False
176
-
177
- if is_subscription_model_type:
178
- subscription_id = _get_sub_id(state.get(name))
179
- if subscription_id:
180
- sub_mod = param.annotation.from_subscription(subscription_id)
181
- arguments.append(sub_mod)
182
- else:
183
- logger.error("Could not find key in state.", key=name, state=state)
184
- raise KeyError(f"Could not find key '{name}' in state.")
185
- elif is_list_type(param.annotation, SubscriptionModel):
186
- subscription_ids = map(_get_sub_id, state.get(name, []))
187
- # Actual type is first argument from list type
188
- if (actual_type := get_args(param.annotation)[0]) == Any:
189
- raise ValueError(
190
- f"Step function argument '{param.name}' cannot be serialized from database with type 'Any'"
191
- )
192
- subscriptions = [actual_type.from_subscription(subscription_id) for subscription_id in subscription_ids]
193
- arguments.append(subscriptions)
194
- elif is_optional_type(param.annotation, SubscriptionModel):
195
- subscription_id = _get_sub_id(state.get(name))
196
- if subscription_id:
197
- # Actual type is first argument from optional type
198
- sub_mod = get_args(param.annotation)[0].from_subscription(subscription_id)
199
- arguments.append(sub_mod)
214
+ value = state[name]
215
+ if param.annotation == UUID:
216
+ arguments.append(_convert_to_uuid(value))
217
+ elif is_list_type(param.annotation, UUID):
218
+ arguments.append([_convert_to_uuid(item) for item in value])
219
+ elif is_optional_type(param.annotation, UUID):
220
+ arguments.append(None if value is None else _convert_to_uuid(value))
200
221
  else:
201
- arguments.append(None)
202
- elif param.default is not inspect.Parameter.empty:
203
- arguments.append(state.get(name, param.default))
204
- else:
205
- try:
206
- arguments.append(state[name])
207
- except KeyError as key_error:
208
- logger.error("Could not find key in state.", key=name, state=state)
209
- raise KeyError(
210
- f"Could not find key '{name}' in state. for function {func.__module__}.{func.__qualname__}"
211
- ) from key_error
222
+ arguments.append(value)
223
+ except KeyError as key_error:
224
+ logger.error("Could not find key in state.", key=name, state=state)
225
+ raise KeyError(
226
+ f"Could not find key '{name}' in state. for function {func.__module__}.{func.__qualname__}"
227
+ ) from key_error
228
+ except ValueError as value_error:
229
+ logger.error("Could not convert value to expected type.", key=name, state=state, value=state[name])
230
+ raise ValueError(f"Could not convert value '{state[name]}' to {param.annotation}") from value_error
231
+
212
232
  return arguments
213
233
 
214
234
 
@@ -0,0 +1,321 @@
1
+ Metadata-Version: 2.4
2
+ Name: orchestrator-core
3
+ Version: 3.1.2rc1
4
+ Summary: This is the orchestrator workflow engine.
5
+ Author-email: SURF <automation-beheer@surf.nl>
6
+ Requires-Python: >=3.11,<3.14
7
+ Description-Content-Type: text/markdown
8
+ License-Expression: Apache-2.0
9
+ Classifier: Intended Audience :: Information Technology
10
+ Classifier: Intended Audience :: System Administrators
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Topic :: Internet
15
+ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
16
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
+ Classifier: Topic :: Software Development :: Libraries
18
+ Classifier: Topic :: Software Development
19
+ Classifier: Typing :: Typed
20
+ Classifier: Development Status :: 5 - Production/Stable
21
+ Classifier: Environment :: Web Environment
22
+ Classifier: Framework :: AsyncIO
23
+ Classifier: Framework :: FastAPI
24
+ Classifier: Intended Audience :: Developers
25
+ Classifier: Intended Audience :: Telecommunications Industry
26
+ Classifier: Programming Language :: Python :: 3 :: Only
27
+ Classifier: Programming Language :: Python :: 3.13
28
+ Classifier: Programming Language :: Python :: 3.12
29
+ Classifier: Programming Language :: Python :: 3.11
30
+ Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
31
+ Classifier: Topic :: Internet :: WWW/HTTP
32
+ License-File: LICENSE
33
+ Requires-Dist: alembic==1.15.1
34
+ Requires-Dist: anyio>=3.7.0
35
+ Requires-Dist: click==8.*
36
+ Requires-Dist: deprecated
37
+ Requires-Dist: deepmerge==2.0
38
+ Requires-Dist: fastapi~=0.115.2
39
+ Requires-Dist: fastapi-etag==0.4.0
40
+ Requires-Dist: more-itertools~=10.6.0
41
+ Requires-Dist: itsdangerous
42
+ Requires-Dist: Jinja2==3.1.6
43
+ Requires-Dist: orjson==3.10.15
44
+ Requires-Dist: psycopg[binary]==3.2.6
45
+ Requires-Dist: pydantic[email]~=2.10.6
46
+ Requires-Dist: pydantic-settings~=2.8.0
47
+ Requires-Dist: python-dateutil==2.8.2
48
+ Requires-Dist: python-rapidjson>=1.18,<1.21
49
+ Requires-Dist: pytz==2025.1
50
+ Requires-Dist: redis==5.1.1
51
+ Requires-Dist: schedule==1.1.0
52
+ Requires-Dist: sentry-sdk[fastapi]~=2.22.0
53
+ Requires-Dist: SQLAlchemy==2.0.39
54
+ Requires-Dist: SQLAlchemy-Utils==0.41.2
55
+ Requires-Dist: structlog
56
+ Requires-Dist: typer==0.15.2
57
+ Requires-Dist: uvicorn[standard]~=0.34.0
58
+ Requires-Dist: nwa-stdlib~=1.9.0
59
+ Requires-Dist: oauth2-lib~=2.4.0
60
+ Requires-Dist: tabulate==0.9.0
61
+ Requires-Dist: strawberry-graphql>=0.246.2
62
+ Requires-Dist: pydantic-forms~=1.4.0
63
+ Requires-Dist: celery~=5.4.0 ; extra == "celery"
64
+ Requires-Dist: toml ; extra == "dev"
65
+ Requires-Dist: bumpversion ; extra == "dev"
66
+ Requires-Dist: mypy_extensions ; extra == "dev"
67
+ Requires-Dist: pre-commit ; extra == "dev"
68
+ Requires-Dist: pydocstyle ; extra == "dev"
69
+ Requires-Dist: python-dotenv ; extra == "dev"
70
+ Requires-Dist: watchdog ; extra == "dev"
71
+ Requires-Dist: mkdocs ; extra == "doc"
72
+ Requires-Dist: mkdocs-material[imaging] ; extra == "doc"
73
+ Requires-Dist: mkdocs-render-swagger-plugin ; extra == "doc"
74
+ Requires-Dist: mkdocs-include-markdown-plugin ; extra == "doc"
75
+ Requires-Dist: mkdocstrings[python] ; extra == "doc"
76
+ Requires-Dist: mkdocs-open-in-new-tab ; extra == "doc"
77
+ Requires-Dist: mkdocs-macros-plugin ; extra == "doc"
78
+ Requires-Dist: mkdocs-embed-external-markdown ; extra == "doc"
79
+ Requires-Dist: apache-license-check ; extra == "test"
80
+ Requires-Dist: black ; extra == "test"
81
+ Requires-Dist: blinker ; extra == "test"
82
+ Requires-Dist: deepdiff ; extra == "test"
83
+ Requires-Dist: dirty-equals ; extra == "test"
84
+ Requires-Dist: jsonref ; extra == "test"
85
+ Requires-Dist: mypy==1.9 ; extra == "test"
86
+ Requires-Dist: pyinstrument ; extra == "test"
87
+ Requires-Dist: pytest==8.3.5 ; extra == "test"
88
+ Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
89
+ Requires-Dist: pytest-codspeed ; extra == "test"
90
+ Requires-Dist: pytest-cov ; extra == "test"
91
+ Requires-Dist: pytest-httpx ; extra == "test"
92
+ Requires-Dist: pytest-xdist ; extra == "test"
93
+ Requires-Dist: requests-mock ; extra == "test"
94
+ Requires-Dist: ruff ; extra == "test"
95
+ Requires-Dist: sqlalchemy[mypy] ; extra == "test"
96
+ Requires-Dist: urllib3-mock ; extra == "test"
97
+ Requires-Dist: types-Deprecated ; extra == "test"
98
+ Requires-Dist: types-Jinja2 ; extra == "test"
99
+ Requires-Dist: types-aiofiles ; extra == "test"
100
+ Requires-Dist: types-certifi ; extra == "test"
101
+ Requires-Dist: types-click ; extra == "test"
102
+ Requires-Dist: types-itsdangerous ; extra == "test"
103
+ Requires-Dist: types-orjson ; extra == "test"
104
+ Requires-Dist: types-python-dateutil ; extra == "test"
105
+ Requires-Dist: types-pytz ; extra == "test"
106
+ Requires-Dist: types-redis ; extra == "test"
107
+ Requires-Dist: types-requests ; extra == "test"
108
+ Requires-Dist: types-setuptools ; extra == "test"
109
+ Requires-Dist: types-tabulate ; extra == "test"
110
+ Requires-Dist: types-toml ; extra == "test"
111
+ Requires-Dist: types-ujson ; extra == "test"
112
+ Requires-Dist: types-PyYAML ; extra == "test"
113
+ Project-URL: documentation, https://workfloworchestrator.org/orchestrator-core/
114
+ Project-URL: releasenotes, https://github.com/workfloworchestrator/orchestrator-core/releases
115
+ Project-URL: source, https://github.com/workfloworchestrator/orchestrator-core
116
+ Provides-Extra: celery
117
+ Provides-Extra: dev
118
+ Provides-Extra: doc
119
+ Provides-Extra: test
120
+
121
+ # Orchestrator-Core
122
+ [![Downloads](https://pepy.tech/badge/orchestrator-core/month)](https://pepy.tech/project/orchestrator-core)
123
+ [![codecov](https://codecov.io/gh/workfloworchestrator/orchestrator-core/branch/main/graph/badge.svg?token=5ANQFI2DHS)](https://codecov.io/gh/workfloworchestrator/orchestrator-core)
124
+ [![pypi_version](https://img.shields.io/pypi/v/orchestrator-core?color=%2334D058&label=pypi%20package)](https://pypi.org/project/orchestrator-core)
125
+ [![Supported python versions](https://img.shields.io/pypi/pyversions/orchestrator-core.svg?color=%2334D058)](https://pypi.org/project/orchestrator-core)
126
+ ![Discord](https://img.shields.io/discord/1295834294270558280?style=flat&logo=discord&label=discord&link=https%3A%2F%2Fdiscord.gg%2FKNgF6gE8)
127
+
128
+ <p style="text-align: center"><em>Production ready Orchestration Framework to manage product lifecycle and workflows. Easy to use, Built on top of FastAPI</em></p>
129
+
130
+
131
+ ## Documentation
132
+ Can be found [here](https://workfloworchestrator.org/orchestrator-core/)
133
+
134
+ ## Usage
135
+ This project can be installed as follows:
136
+
137
+ #### Step 1:
138
+ Install the core.
139
+ ```shell
140
+ pip install orchestrator-core
141
+ ```
142
+
143
+ #### Step 2:
144
+ Create a postgres database:
145
+ ```shell
146
+ createuser -sP nwa
147
+ createdb orchestrator-core -O nwa
148
+ ```
149
+
150
+ #### Step 3 (optional):
151
+ When using multiple workers, you will need a redis server for live updates with websockets.
152
+
153
+ By default it will use memory which works with only one worker.
154
+ ```shell
155
+ export WEBSOCKET_BROADCASTER_URL="memory://"
156
+ ```
157
+
158
+ For the redis connection you need to set the env variable with the connection url.
159
+ ```shell
160
+ export WEBSOCKET_BROADCASTER_URL="redis://localhost:6379"
161
+ ```
162
+
163
+
164
+ Websockets can also be turned off with:
165
+ ```shell
166
+ export ENABLE_WEBSOCKETS=False
167
+ ```
168
+
169
+ If you want to use pickle for CACHE serialization you will need to set the `CACHE_HMAC_SECRET`:
170
+ ```shell
171
+ export CACHE_HMAC_SECRET="SOMESECRET"
172
+ ```
173
+ **NOTE**: The key can be any length. However, the recommended size is 1024 bits.
174
+
175
+ #### Step 4:
176
+ Create a `main.py` file.
177
+
178
+ ```python
179
+ from orchestrator import OrchestratorCore
180
+ from orchestrator.cli.main import app as core_cli
181
+ from orchestrator.settings import AppSettings
182
+
183
+ app = OrchestratorCore(base_settings=AppSettings())
184
+
185
+ if __name__ == "__main__":
186
+ core_cli()
187
+ ```
188
+
189
+ #### Step 5 (Optional):
190
+ OrchestratorCore comes with a graphql interface that can to be registered after you create your OrchestratorApp.
191
+ If you add it after registering your `SUBSCRIPTION_MODEL_REGISTRY` it will automatically create graphql types for them.
192
+ More info can be found in `docs/architecture/application/graphql.md`
193
+
194
+ example:
195
+ ```python
196
+ from orchestrator import OrchestratorCore
197
+ from orchestrator.settings import AppSettings
198
+
199
+ app = OrchestratorCore(base_settings=AppSettings())
200
+ # register SUBSCRIPTION_MODEL_REGISTRY
201
+ app.register_graphql()
202
+ ```
203
+
204
+ #### Step 6:
205
+ Initialize the migration environment.
206
+ ```shell
207
+ PYTHONPATH=. python main.py db init
208
+ PYTHONPATH=. python main.py db upgrade heads
209
+ ```
210
+
211
+ ### Step 7:
212
+ Profit :)
213
+
214
+ Authentication and authorization are default enabled, to disable set `OAUTH2_ACTIVE` and `OAUTH2_AUTHORIZATION_ACTIVE` to `False`.
215
+
216
+ ```shell
217
+ uvicorn --reload --host 127.0.0.1 --port 8080 main:app
218
+ ```
219
+
220
+ Visit [http://127.0.0.1:8080/api/redoc](http://127.0.0.1:8080/api/redoc) to view the api documentation.
221
+
222
+
223
+ ## Setting up a development environment
224
+
225
+ To add features to the repository follow the following procedure to setup a working development environment.
226
+
227
+ ### Installation (Development standalone)
228
+ Install the project and its dependencies to develop on the code.
229
+
230
+ #### Step 1 - install flit:
231
+
232
+ ```shell
233
+ python3 -m venv venv
234
+ source venv/bin/activate
235
+ pip install flit
236
+ ```
237
+
238
+ #### Step 2 - install the development code:
239
+
240
+ !!! danger
241
+ Make sure to use the flit binary that is installed in your environment. You can check the correct
242
+ path by running
243
+ ```shell
244
+ which flit
245
+ ```
246
+
247
+ To be sure that the packages will be installed against the correct venv you can also prepend the python interpreter
248
+ that you want to use:
249
+
250
+ ```shell
251
+ flit install --deps develop --symlink --python venv/bin/python
252
+ ```
253
+
254
+
255
+ ### Running tests
256
+ Run the unit-test suite to verify a correct setup.
257
+
258
+ #### Step 1 - Create a database
259
+
260
+ ```shell
261
+ createuser -sP nwa
262
+ createdb orchestrator-core-test -O nwa
263
+ ```
264
+
265
+ #### Step 2 - Run tests
266
+ ```shell
267
+ pytest test/unit_tests
268
+ ```
269
+ or with xdist:
270
+
271
+ ```shell
272
+ pytest -n auto test/unit_tests
273
+ ```
274
+
275
+ If you do not encounter any failures in the test, you should be able to develop features in the orchestrator-core.
276
+
277
+ ### Installation (Development symlinked into orchestrator SURF)
278
+
279
+ If you are working on a project that already uses the `orchestrator-core` and you want to test your new core features
280
+ against it, you can use some `flit` magic to symlink the dev version of the core to your project. It will
281
+ automatically replace the pypi dep with a symlink to the development version
282
+ of the core and update/downgrade all required packages in your own orchestrator project.
283
+
284
+ #### Step 1 - install flit:
285
+
286
+ ```shell
287
+ python - m venv venv
288
+ source venv/bin/activate
289
+ pip install flit
290
+ ```
291
+
292
+ ### Step 2 - symlink the core to your own project
293
+
294
+ ```shell
295
+ flit install --deps develop --symlink --python /path/to/a/orchestrator-project/venv/bin/python
296
+ ```
297
+
298
+ So if you have the core and your own orchestrator project repo in the same folder and the main project folder is
299
+ `orchestrator` and you want to use relative links, this will be last step:
300
+
301
+ ```shell
302
+ flit install --deps develop --symlink --python ../orchestrator/venv/bin/python
303
+ ```
304
+
305
+ # Increasing the version number for a (pre) release.
306
+
307
+ When your PR is accepted you will get a version number.
308
+
309
+ You can do the necessary change with a clean, e.g. every change committed, branch:
310
+
311
+ ```shell
312
+ bumpversion patch --new-version 0.4.1-rc3
313
+ ```
314
+
315
+ ### Changing the Core database schema
316
+ When you would like to change the core database schema, execute the following steps.
317
+
318
+ - Create the new model `orchestrator/database/models.py`
319
+ - `cd orchestrator/migrations`
320
+ - `alembic revision --autogenerate -m "Name of the migratioin"`
321
+
@@ -1,4 +1,4 @@
1
- orchestrator/__init__.py,sha256=44uEQENUfGzHI2rM8eaZMGP3Zt5u-wn0O2SF5kiHMdk,1058
1
+ orchestrator/__init__.py,sha256=ZP1Jysb045z9lRfVO6EzA1vnZiG_nSFXoDDnIg34onM,1058
2
2
  orchestrator/app.py,sha256=8GMzoHjdR0bkgRBCejiG8nIUjeo43f12I3WNNZ89pKE,11659
3
3
  orchestrator/exception_handlers.py,sha256=UsW3dw8q0QQlNLcV359bIotah8DYjMsj2Ts1LfX4ClY,1268
4
4
  orchestrator/log_config.py,sha256=1tPRX5q65e57a6a_zEii_PFK8SzWT0mnA5w2sKg4hh8,1853
@@ -106,7 +106,7 @@ orchestrator/db/database.py,sha256=MU_w_e95ho2dVb2JDnt_KFYholx___XDkiQXbc8wCkI,1
106
106
  orchestrator/db/helpers.py,sha256=L8kEdnSSNGnUpZhdeGx2arCodakWN8vSpKdfjoLuHdY,831
107
107
  orchestrator/db/listeners.py,sha256=UBPYcH0FE3a7aZQu_D0O_JMXpXIRYXC0gjSAvlv5GZo,1142
108
108
  orchestrator/db/loaders.py,sha256=escBOUNf5bHmjIuNH37fGgNSeZLzMiJvQgQFy4r4MYY,6244
109
- orchestrator/db/models.py,sha256=LOdqw2G45TGV2Bkukj2GgXtc4nRYk1rARE0wDk_QZUE,26850
109
+ orchestrator/db/models.py,sha256=x0x4CT9LFm5XrqmY3LO9JWL7p_C7YSpbow4IRB_QLOg,26862
110
110
  orchestrator/db/filters/__init__.py,sha256=RUj6P0XxEBhYj0SN5wH5-Vf_Wt_ilZR_n9DSar5m9oM,371
111
111
  orchestrator/db/filters/filters.py,sha256=55RtpQwM2rhrk4A6CCSeSXoo-BT9GnQoNTryA8CtLEg,5020
112
112
  orchestrator/db/filters/process.py,sha256=xvGhyfo_MZ1xhLvFC6yULjcT4mJk0fKc1glJIYgsWLE,4018
@@ -198,7 +198,7 @@ orchestrator/graphql/utils/override_class.py,sha256=blwPXVHxLyXQga3KjiDzWozmMhHE
198
198
  orchestrator/graphql/utils/to_graphql_result_page.py,sha256=8ObkJP8reVf-TQOQVPKv1mNdfmSEMS1sG7s_-T7-pUU,902
199
199
  orchestrator/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
200
200
  orchestrator/migrations/alembic.ini,sha256=kMoADqhGeubU8xanILNaqm4oixLy9m4ngYtdGpZcc7I,873
201
- orchestrator/migrations/env.py,sha256=AwlgBPYbV2hr5rHNwlOPJ5rs-vRyfmzcWyxae0btpZ4,3382
201
+ orchestrator/migrations/env.py,sha256=0NrhF3vVeKMtU1kGNrc-nSfk8EnK4KbcDdUeGvWGCBM,3732
202
202
  orchestrator/migrations/helpers.py,sha256=U-b64Gp6VBq5sTDN0fqrG8mbXcpncCFVgYObW9y7ffs,43778
203
203
  orchestrator/migrations/script.py.mako,sha256=607Zrgp-Z-m9WGLt4wewN1QDOmHeifxcePUdADkSZyM,510
204
204
  orchestrator/migrations/templates/alembic.ini.j2,sha256=jA-QykVparwWSNt5XDP0Zk7epLOhK7D87Af-i2shJV4,905
@@ -206,13 +206,13 @@ orchestrator/migrations/templates/env.py.j2,sha256=RfLAQItZ56Jlzwi6LJfBo92m1-th_
206
206
  orchestrator/migrations/templates/helpers.py.j2,sha256=3MWNMICGrcQFObyBQefL-FPjoVKUgP0QIlbk4TdMZds,98
207
207
  orchestrator/migrations/versions/schema/2020-10-19_3323bcb934e7_fix_tsv_triggers.py,sha256=ufe6OFUELNpx6N2663bvdwgB4lP-v71fuMuJtx9CmJc,2698
208
208
  orchestrator/migrations/versions/schema/2020-10-19_a76b9185b334_add_generic_workflows_to_core.py,sha256=EHj87IXucruyB8KuxEWcc7JK1NIizZ5Jzmj-bzY0t1Y,1265
209
- orchestrator/migrations/versions/schema/2020-10-19_c112305b07d3_initial_schema_migration.py,sha256=gYg8JBKGu6VH7gryU-AqDPvAzx1oAAQnyGS9HMl0nSY,39307
209
+ orchestrator/migrations/versions/schema/2020-10-19_c112305b07d3_initial_schema_migration.py,sha256=-_dEwEXbl1E2HQpdcigMsSsq6H98eRcmaE8g5NR36iE,39291
210
210
  orchestrator/migrations/versions/schema/2021-04-06_3c8b9185c221_add_validate_products_task.py,sha256=rLuEl8WuQHcwmOBZdADbgCBqzJ5EZ4KWXrTVwXrRyx4,950
211
211
  orchestrator/migrations/versions/schema/2021-07-01_6896a54e9483_add_product_block_relations.py,sha256=xw01x5YTNDDxZiMUCeBPuzp0LKsKeMMR4YWF2aWI9ZI,1214
212
212
  orchestrator/migrations/versions/schema/2021-11-17_19cdd3ab86f6_fix_parse_websearch.py,sha256=FUWxAPpi32SgowU_WdZiC903BbLUA5zktBICOi4ecpQ,1603
213
213
  orchestrator/migrations/versions/schema/2022-02-16_bed6bc0b197a_rename_parent_and_child_block_relations.py,sha256=2hiV8aFwlcgRQ7EFVvGhV13j2j-p7cMLadyUfXezIF8,5106
214
214
  orchestrator/migrations/versions/schema/2023-03-06_e05bb1967eff_add_subscriptions_search_view.py,sha256=EUArauxUk_D_QWO9AKuhGIOD1a4PonH4-rVmdSx0dbs,7964
215
- orchestrator/migrations/versions/schema/2023-05-25_b1970225392d_add_subscription_metadata_workflow.py,sha256=0DhrxNXtB6cKkish5SanfpRkSx86yNL10ReKazlWuv4,1014
215
+ orchestrator/migrations/versions/schema/2023-05-25_b1970225392d_add_subscription_metadata_workflow.py,sha256=6_sH1geyl5Ier2oUmGohobpPW5OMp_jN59885DIox8g,998
216
216
  orchestrator/migrations/versions/schema/2023-06-28_a09ac125ea73_add_throttling_to_refresh_subscriptions.py,sha256=GgWBhv2QboFpES3ltoJpVM23lqp8HI9NvRuKkdF3Qzk,591
217
217
  orchestrator/migrations/versions/schema/2023-06-28_a09ac125ea73_add_throttling_to_refresh_subscriptions.sql,sha256=HYKQmWzSm7A7igjzJoOoKyGp3nQUYtThJ10OWWQo3Xs,755
218
218
  orchestrator/migrations/versions/schema/2023-07-17_165303a20fb1_customer_id_to_varchar.py,sha256=fOjZULujncwoa2QciQ4d2qcxi3iEqc_KdhdRFyfHuPc,967
@@ -223,7 +223,7 @@ orchestrator/migrations/versions/schema/2023-12-06_048219045729_add_workflow_id_
223
223
  orchestrator/migrations/versions/schema/2024-09-27_460ec6748e37_add_uuid_search_workaround.py,sha256=GzHBzOwOc6FaO1kYwoSNIhb8sKstXo8Cfxdqy3Rmeg4,972
224
224
  orchestrator/migrations/versions/schema/2024-09-27_460ec6748e37_add_uuid_search_workaround.sql,sha256=mhPnqjG5H3W8_BD7w5tYzXUQSxFOM7Rahn_MudEPTIE,5383
225
225
  orchestrator/migrations/versions/schema/2025-01-08_4c5859620539_add_version_column_to_subscription.py,sha256=xAhe74U0ZiVRo9Z8Uq7491RBbATMMUnYpTBjbG-BYL0,1690
226
- orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py,sha256=BPtx8blYEotaJv9Gnzq9Pf4ihOyzx4tfk9qwr81i8MU,1769
226
+ orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py,sha256=0vBDGltmOs_gcTTYlWBNOgItzqCXm8qqT02jZfTpL5c,1753
227
227
  orchestrator/migrations/versions/schema/2025-10-19_4fjdn13f83ga_add_validate_product_type_task.py,sha256=O0GfCISIDnyohGf3Ot_2HKedGRbMqLVox6t7Wd3PMvo,894
228
228
  orchestrator/schedules/__init__.py,sha256=JnnaglfK1qYUBKI6Dd9taV-tCZIPlAdAkHtnkJDMXxY,1066
229
229
  orchestrator/schedules/resume_workflows.py,sha256=kSotzTAXjX7p9fpSYiGOpuxuTQfv54eRFAe0YSG0DHc,832
@@ -273,7 +273,7 @@ orchestrator/utils/json.py,sha256=7386sdqkrKYyy4sbn5NscwctH_v1hLyw5172P__rU3g,83
273
273
  orchestrator/utils/redis.py,sha256=E2vrMO3uQHb4nJENgA3WnpB0iw2C615YMuaWT-4gqoI,7027
274
274
  orchestrator/utils/redis_client.py,sha256=9rhsvedjK_CyClAjUicQyge0mVIViATqKFGZyjBY3XA,1384
275
275
  orchestrator/utils/search_query.py,sha256=ji5LHtrzohGz6b1IG41cnPdpWXzLEzz4SGWgHly_yfU,16205
276
- orchestrator/utils/state.py,sha256=DLHBnpEjhHQNeBGYV6H6geqZclToeMuWwqU26TVy220,13185
276
+ orchestrator/utils/state.py,sha256=eEmYCo9gczy0R6pIIahNyEjOnNJWI0oC5aI-1gqy5Ns,14012
277
277
  orchestrator/utils/strings.py,sha256=N0gWjmQaMjE9_99VtRvRaU8IBLTKMgBKSXcTZ9TpWAg,1077
278
278
  orchestrator/utils/validate_data_version.py,sha256=3Eioy2wE2EWKSgkyMKcEKrkCAfUIAq-eb73iRcpgppw,184
279
279
  orchestrator/websocket/__init__.py,sha256=V79jskk1z3uPIYgu0Gt6JLzuqr7NGfNeAZ-hbBqoUv4,5745
@@ -291,7 +291,7 @@ orchestrator/workflows/tasks/resume_workflows.py,sha256=R0I3jxGToiqDr5mF3YjDd6dN
291
291
  orchestrator/workflows/tasks/validate_product_type.py,sha256=5FwhRQyMNgtys5DM846EIIY0uXKvnSYy3Orf7lOg0DA,3176
292
292
  orchestrator/workflows/tasks/validate_products.py,sha256=5uXX7MXMDDP13cXRvfLDNvvCp4nG7zLQBm_IYdf8BSs,8513
293
293
  orchestrator/workflows/translations/en-GB.json,sha256=ST53HxkphFLTMjFHonykDBOZ7-P_KxksktZU3GbxLt0,846
294
- orchestrator_core-3.1.0rc1.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
295
- orchestrator_core-3.1.0rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
296
- orchestrator_core-3.1.0rc1.dist-info/METADATA,sha256=HWbNmbgLR-tuB1ifP702BFVr7x4La_ymTtl678oxtwU,4993
297
- orchestrator_core-3.1.0rc1.dist-info/RECORD,,
294
+ orchestrator_core-3.1.2rc1.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
295
+ orchestrator_core-3.1.2rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
296
+ orchestrator_core-3.1.2rc1.dist-info/METADATA,sha256=y1VUynKBYQow9miuCnXpMg_4VkNQdyODnTBr8X6v-xU,11160
297
+ orchestrator_core-3.1.2rc1.dist-info/RECORD,,
@@ -1,116 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: orchestrator-core
3
- Version: 3.1.0rc1
4
- Summary: This is the orchestrator workflow engine.
5
- Requires-Python: >=3.11,<3.14
6
- Classifier: Intended Audience :: Information Technology
7
- Classifier: Intended Audience :: System Administrators
8
- Classifier: Operating System :: OS Independent
9
- Classifier: Programming Language :: Python :: 3
10
- Classifier: Programming Language :: Python
11
- Classifier: Topic :: Internet
12
- Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
13
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
14
- Classifier: Topic :: Software Development :: Libraries
15
- Classifier: Topic :: Software Development
16
- Classifier: Typing :: Typed
17
- Classifier: Development Status :: 5 - Production/Stable
18
- Classifier: Environment :: Web Environment
19
- Classifier: Framework :: AsyncIO
20
- Classifier: Framework :: FastAPI
21
- Classifier: Intended Audience :: Developers
22
- Classifier: Intended Audience :: Telecommunications Industry
23
- Classifier: License :: OSI Approved :: Apache Software License
24
- Classifier: Programming Language :: Python :: 3 :: Only
25
- Classifier: Programming Language :: Python :: 3.13
26
- Classifier: Programming Language :: Python :: 3.12
27
- Classifier: Programming Language :: Python :: 3.11
28
- Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
29
- Classifier: Topic :: Internet :: WWW/HTTP
30
- License-File: LICENSE
31
- Requires-Dist: alembic==1.14.1
32
- Requires-Dist: anyio>=3.7.0
33
- Requires-Dist: click==8.*
34
- Requires-Dist: deprecated
35
- Requires-Dist: deepmerge==2.0
36
- Requires-Dist: fastapi~=0.115.2
37
- Requires-Dist: fastapi-etag==0.4.0
38
- Requires-Dist: more-itertools~=10.6.0
39
- Requires-Dist: itsdangerous
40
- Requires-Dist: Jinja2==3.1.5
41
- Requires-Dist: orjson==3.10.15
42
- Requires-Dist: psycopg[binary]==3.2.5
43
- Requires-Dist: pydantic[email]~=2.8.2
44
- Requires-Dist: pydantic-settings~=2.8.0
45
- Requires-Dist: python-dateutil==2.8.2
46
- Requires-Dist: python-rapidjson>=1.18,<1.21
47
- Requires-Dist: pytz==2025.1
48
- Requires-Dist: redis==5.0.3
49
- Requires-Dist: schedule==1.1.0
50
- Requires-Dist: sentry-sdk[fastapi]~=2.22.0
51
- Requires-Dist: SQLAlchemy==2.0.38
52
- Requires-Dist: SQLAlchemy-Utils==0.41.2
53
- Requires-Dist: structlog
54
- Requires-Dist: typer==0.15.1
55
- Requires-Dist: uvicorn[standard]~=0.32.0
56
- Requires-Dist: nwa-stdlib~=1.9.0
57
- Requires-Dist: oauth2-lib~=2.4.0
58
- Requires-Dist: tabulate==0.9.0
59
- Requires-Dist: strawberry-graphql>=0.246.2
60
- Requires-Dist: pydantic-forms~=1.3.0
61
- Requires-Dist: celery~=5.4.0 ; extra == "celery"
62
- Requires-Dist: toml ; extra == "dev"
63
- Requires-Dist: bumpversion ; extra == "dev"
64
- Requires-Dist: mypy_extensions ; extra == "dev"
65
- Requires-Dist: pre-commit ; extra == "dev"
66
- Requires-Dist: pydocstyle ; extra == "dev"
67
- Requires-Dist: python-dotenv ; extra == "dev"
68
- Requires-Dist: watchdog ; extra == "dev"
69
- Requires-Dist: mkdocs ; extra == "doc"
70
- Requires-Dist: mkdocs-material[imaging] ; extra == "doc"
71
- Requires-Dist: mkdocs-render-swagger-plugin ; extra == "doc"
72
- Requires-Dist: mkdocs-include-markdown-plugin ; extra == "doc"
73
- Requires-Dist: mkdocstrings[python] ; extra == "doc"
74
- Requires-Dist: mkdocs-open-in-new-tab ; extra == "doc"
75
- Requires-Dist: mkdocs-macros-plugin ; extra == "doc"
76
- Requires-Dist: mkdocs-embed-external-markdown ; extra == "doc"
77
- Requires-Dist: apache-license-check ; extra == "test"
78
- Requires-Dist: black ; extra == "test"
79
- Requires-Dist: blinker ; extra == "test"
80
- Requires-Dist: deepdiff ; extra == "test"
81
- Requires-Dist: dirty-equals ; extra == "test"
82
- Requires-Dist: jsonref ; extra == "test"
83
- Requires-Dist: mypy==1.9 ; extra == "test"
84
- Requires-Dist: pyinstrument ; extra == "test"
85
- Requires-Dist: pytest==8.3.4 ; extra == "test"
86
- Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
87
- Requires-Dist: pytest-codspeed ; extra == "test"
88
- Requires-Dist: pytest-cov ; extra == "test"
89
- Requires-Dist: pytest-httpx ; extra == "test"
90
- Requires-Dist: pytest-xdist ; extra == "test"
91
- Requires-Dist: requests-mock ; extra == "test"
92
- Requires-Dist: ruff ; extra == "test"
93
- Requires-Dist: sqlalchemy[mypy] ; extra == "test"
94
- Requires-Dist: urllib3-mock ; extra == "test"
95
- Requires-Dist: types-Deprecated ; extra == "test"
96
- Requires-Dist: types-Jinja2 ; extra == "test"
97
- Requires-Dist: types-aiofiles ; extra == "test"
98
- Requires-Dist: types-certifi ; extra == "test"
99
- Requires-Dist: types-click ; extra == "test"
100
- Requires-Dist: types-itsdangerous ; extra == "test"
101
- Requires-Dist: types-orjson ; extra == "test"
102
- Requires-Dist: types-python-dateutil ; extra == "test"
103
- Requires-Dist: types-pytz ; extra == "test"
104
- Requires-Dist: types-redis ; extra == "test"
105
- Requires-Dist: types-requests ; extra == "test"
106
- Requires-Dist: types-setuptools ; extra == "test"
107
- Requires-Dist: types-tabulate ; extra == "test"
108
- Requires-Dist: types-toml ; extra == "test"
109
- Requires-Dist: types-ujson ; extra == "test"
110
- Requires-Dist: types-PyYAML ; extra == "test"
111
- Project-URL: Documentation, https://workfloworchestrator.org/orchestrator-core/
112
- Project-URL: Source, https://github.com/workfloworchestrator/orchestrator-core
113
- Provides-Extra: celery
114
- Provides-Extra: dev
115
- Provides-Extra: doc
116
- Provides-Extra: test