orchestrator-core 4.7.1__tar.gz → 4.8.0rc1__tar.gz

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.7.1
+Version: 4.8.0rc1
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
 Requires-Python: >=3.11,<3.15
@@ -31,7 +31,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 License-File: LICENSE
-Requires-Dist: alembic==1.18.1
+Requires-Dist: alembic==1.18.3
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: apscheduler>=3.11.0
 Requires-Dist: click==8.*
@@ -44,7 +44,7 @@ Requires-Dist: jinja2==3.1.6
 Requires-Dist: more-itertools~=10.8.0
 Requires-Dist: nwa-stdlib~=1.11.0
 Requires-Dist: oauth2-lib>=2.5.0
-Requires-Dist: orjson==3.11.5
+Requires-Dist: orjson==3.11.7
 Requires-Dist: pgvector>=0.4.1
 Requires-Dist: prometheus-client==0.24.1
 Requires-Dist: psycopg2-binary==2.9.11
@@ -57,7 +57,7 @@ Requires-Dist: pytz==2025.2
 Requires-Dist: redis==7.1.0
 Requires-Dist: semver==3.0.4
 Requires-Dist: sentry-sdk[fastapi]>=2.29.1
-Requires-Dist: sqlalchemy==2.0.45
+Requires-Dist: sqlalchemy==2.0.46
 Requires-Dist: sqlalchemy-utils==0.42.1
 Requires-Dist: strawberry-graphql>=0.281.0,<0.285.0
 Requires-Dist: structlog>=25.4.0

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.7.1"
+__version__ = "4.8.0rc1"
 
 
 from structlog import get_logger

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/api/api_v1/endpoints/search.py

@@ -16,6 +16,7 @@ from fastapi import APIRouter, HTTPException, Query, status
 
 from orchestrator.db import db
 from orchestrator.schemas.search import (
+    CursorInfoSchema,
     ExportResponse,
     PageInfoSchema,
     PathsResponse,
@@ -81,10 +82,24 @@ async def _perform_search_and_fetch(
 
         next_page_cursor = encode_next_page_cursor(search_response, page_cursor, query)
         has_next_page = next_page_cursor is not None
-        page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
+        page_info = PageInfoSchema(
+            has_next_page=has_next_page,
+            next_page_cursor=next_page_cursor,
+        )
+
+        cursor_info = None
+        if search_response.total_items:
+            cursor_info = CursorInfoSchema(
+                total_items=search_response.total_items,
+                start_cursor=search_response.start_cursor,
+                end_cursor=search_response.end_cursor,
+            )
 
         return SearchResultsSchema(
-            data=search_response.results, page_info=page_info, search_metadata=search_response.metadata
+            data=search_response.results,
+            page_info=page_info,
+            search_metadata=search_response.metadata,
+            cursor=cursor_info,
         )
     except (InvalidCursorError, ValueError) as e:
         raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/db/models.py

@@ -38,7 +38,6 @@ from sqlalchemy import (
     Select,
     String,
     Table,
-    Text,
     TypeDecorator,
     UniqueConstraint,
     select,
@@ -69,6 +68,37 @@ logger = structlog.get_logger(__name__)
 TAG_LENGTH = 20
 STATUS_LENGTH = 255
 
+# Field length limits chosen based on expected usage patterns
+# These values are intended to be reasonable, but give lots of wiggle room
+# If these values are updated, they also need to be updated in a migration, as in migration d69e10434a04
+NOTE_LENGTH = 5000
+DESCRIPTION_LENGTH = 2000
+FAILED_REASON_LENGTH = 10000
+TRACEBACK_LENGTH = 50000
+RESOURCE_VALUE_LENGTH = 10000
+DOMAIN_MODEL_ATTR_LENGTH = 255
+
+
+class StringThatAutoConvertsToNullWhenEmpty(TypeDecorator):
+    """A String type that converts empty strings to NULL on save."""
+
+    impl = String
+    cache_ok = True
+    python_type = str
+
+    def __init__(self, length: int | None = None):
+        super().__init__(length)
+
+    def process_bind_param(self, value: str | None, dialect: Dialect) -> str | None:
+        """Called when saving to DB - convert empty/whitespace to NULL."""
+        if value is not None and value.strip() == "":
+            return None
+        return value
+
+    def process_result_value(self, value: str | None, dialect: Dialect) -> str | None:
+        """Called when loading from DB - return as-is."""
+        return value
+
 
 class UtcTimestampError(Exception, DontWrapMixin):
     pass
@@ -122,8 +152,8 @@ class ProcessTable(BaseModel):
     last_modified_at = mapped_column(
         UtcTimestamp, server_default=text("current_timestamp()"), onupdate=nowtz, nullable=False
     )
-    failed_reason = mapped_column(Text())
-    traceback = mapped_column(Text())
+    failed_reason = mapped_column(String(FAILED_REASON_LENGTH))
+    traceback = mapped_column(String(TRACEBACK_LENGTH))
     created_by = mapped_column(String(255), nullable=True)
     is_task = mapped_column(Boolean, nullable=False, server_default=text("false"), index=True)
 
@@ -223,7 +253,7 @@ class ProductTable(BaseModel):
 
     product_id = mapped_column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True)
     name = mapped_column(String(), nullable=False, unique=True)
-    description = mapped_column(Text(), nullable=False)
+    description = mapped_column(String(DESCRIPTION_LENGTH), nullable=False)
     product_type = mapped_column(String(255), nullable=False)
     tag = mapped_column(String(TAG_LENGTH), nullable=False, index=True)
     status = mapped_column(String(STATUS_LENGTH), nullable=False)
@@ -302,7 +332,7 @@ class ProductBlockTable(BaseModel):
 
     product_block_id = mapped_column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True)
     name = mapped_column(String(), nullable=False, unique=True)
-    description = mapped_column(Text(), nullable=False)
+    description = mapped_column(String(DESCRIPTION_LENGTH), nullable=False)
     tag = mapped_column(String(TAG_LENGTH))
     status = mapped_column(String(STATUS_LENGTH))
     created_at = mapped_column(UtcTimestamp, nullable=False, server_default=text("current_timestamp()"))
@@ -401,7 +431,7 @@ class ResourceTypeTable(BaseModel):
 
     resource_type_id = mapped_column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True)
     resource_type = mapped_column(String(510), nullable=False, unique=True)
-    description = mapped_column(Text())
+    description = mapped_column(String(DESCRIPTION_LENGTH))
 
     product_blocks = relationship(
         "ProductBlockTable", secondary=product_block_resource_type_association, back_populates="resource_types"
@@ -414,7 +444,7 @@ class WorkflowTable(BaseModel):
     workflow_id = mapped_column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True)
     name = mapped_column(String(), nullable=False, unique=True)
     target = mapped_column(String(), nullable=False)
-    description = mapped_column(Text(), nullable=False)
+    description = mapped_column(String(DESCRIPTION_LENGTH), nullable=False)
     created_at = mapped_column(UtcTimestamp, nullable=False, server_default=text("current_timestamp()"))
     deleted_at = mapped_column(UtcTimestamp, deferred=True)
 
@@ -452,7 +482,7 @@ class SubscriptionInstanceRelationTable(BaseModel):
 
     # Needed to make sure subscription instance is populated in the right domain model attribute, if more than one
     # attribute uses the same product block model.
-    domain_model_attr = Column(Text())
+    domain_model_attr = Column(String(DOMAIN_MODEL_ATTR_LENGTH))
 
     in_use_by: Mapped[SubscriptionInstanceTable] = relationship(
         "SubscriptionInstanceTable", back_populates="depends_on_block_relations", foreign_keys=[in_use_by_id]
@@ -559,7 +589,7 @@ class SubscriptionInstanceValueTable(BaseModel):
     resource_type_id = mapped_column(
         UUIDType, ForeignKey("resource_types.resource_type_id"), nullable=False, index=True
     )
-    value = mapped_column(Text(), nullable=False)
+    value = mapped_column(String(RESOURCE_VALUE_LENGTH), nullable=False)
 
     resource_type = relationship("ResourceTypeTable", lazy="subquery")
     subscription_instance = relationship("SubscriptionInstanceTable", back_populates="values")
@@ -587,7 +617,7 @@ class SubscriptionCustomerDescriptionTable(BaseModel):
         index=True,
     )
     customer_id = mapped_column(String, nullable=False, index=True)
-    description = mapped_column(Text(), nullable=False)
+    description = mapped_column(String(DESCRIPTION_LENGTH), nullable=False)
     created_at = mapped_column(UtcTimestamp, nullable=False, server_default=text("current_timestamp()"))
     version = mapped_column(Integer, nullable=False, server_default="1")
 
@@ -600,14 +630,14 @@ class SubscriptionTable(BaseModel):
     subscription_id = mapped_column(
         UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True, nullable=False
     )
-    description = mapped_column(Text(), nullable=False)
+    description = mapped_column(String(DESCRIPTION_LENGTH), nullable=False)
     status = mapped_column(String(STATUS_LENGTH), nullable=False, index=True)
     product_id = mapped_column(UUIDType, ForeignKey("products.product_id"), nullable=False, index=True)
     customer_id = mapped_column(String, index=True, nullable=False)
     insync = mapped_column(Boolean(), nullable=False)
     start_date = mapped_column(UtcTimestamp, nullable=True)
     end_date = mapped_column(UtcTimestamp)
-    note = mapped_column(Text())
+    note = mapped_column(StringThatAutoConvertsToNullWhenEmpty(NOTE_LENGTH))
     version = mapped_column(Integer, nullable=False, server_default="1")
 
     product = relationship("ProductTable", foreign_keys=[product_id])

orchestrator_core-4.8.0rc1/orchestrator/migrations/versions/schema/2026-01-12_d69e10434a04_convert_unlimited_text_fields_to_.py

@@ -0,0 +1,32 @@
+"""Convert unlimited text fields to limited nullable strings and normalize empty subscription notes.
+
+Revision ID: d69e10434a04
+Revises: 9736496e3eba
+Create Date: 2026-01-12 14:17:58.255515
+
+"""
+
+from pathlib import Path
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d69e10434a04"
+down_revision = "9736496e3eba"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    revision_file_path = Path(__file__.replace(".py", "_upgrade.sql"))
+    with open(revision_file_path) as f:
+        conn.execute(sa.text(f.read()))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    revision_file_path = Path(__file__.replace(".py", "_downgrade.sql"))
+    with open(revision_file_path) as f:
+        conn.execute(sa.text(f.read()))

orchestrator_core-4.8.0rc1/orchestrator/migrations/versions/schema/2026-01-12_d69e10434a04_convert_unlimited_text_fields_to__downgrade.sql

@@ -0,0 +1,138 @@
+-- Downgrade: Revert limited nullable strings back to unlimited text fields.
+-- Revision ID: d69e10434a04
+
+-- Drop triggers first
+DROP TRIGGER IF EXISTS fi_refresh_search ON fixed_inputs;
+DROP TRIGGER IF EXISTS products_refresh_search ON products;
+DROP TRIGGER IF EXISTS sub_cust_desc_refresh_search ON subscription_customer_descriptions;
+DROP TRIGGER IF EXISTS siv_refresh_search ON subscription_instance_values;
+DROP TRIGGER IF EXISTS sub_refresh_search ON subscriptions;
+
+-- Drop the refresh function
+DROP FUNCTION IF EXISTS refresh_subscriptions_search_view();
+
+-- Drop the materialized view (CASCADE will drop indexes)
+DROP MATERIALIZED VIEW IF EXISTS subscriptions_search CASCADE;
+
+-- Alter column types back from VARCHAR to TEXT
+ALTER TABLE subscriptions ALTER COLUMN note TYPE TEXT;
+ALTER TABLE subscriptions ALTER COLUMN description TYPE TEXT;
+ALTER TABLE processes ALTER COLUMN failed_reason TYPE TEXT;
+ALTER TABLE processes ALTER COLUMN traceback TYPE TEXT;
+ALTER TABLE products ALTER COLUMN description TYPE TEXT;
+ALTER TABLE product_blocks ALTER COLUMN description TYPE TEXT;
+ALTER TABLE resource_types ALTER COLUMN description TYPE TEXT;
+ALTER TABLE workflows ALTER COLUMN description TYPE TEXT;
+ALTER TABLE subscription_customer_descriptions ALTER COLUMN description TYPE TEXT;
+ALTER TABLE subscription_instance_values ALTER COLUMN value TYPE TEXT;
+ALTER TABLE subscription_instance_relations ALTER COLUMN domain_model_attr TYPE TEXT;
+
+-- Recreate the materialized view
+CREATE MATERIALIZED VIEW subscriptions_search AS
+WITH rt_info AS (SELECT s.subscription_id,
+                        concat_ws(
+                                ' ',
+                                string_agg(rt.resource_type || ' ' || siv.value, ' '),
+                                string_agg(distinct 'subscription_instance_id' || ':' || si.subscription_instance_id, ' ')
+                        ) AS rt_info
+                 FROM subscription_instance_values siv
+                          JOIN resource_types rt ON siv.resource_type_id = rt.resource_type_id
+                          JOIN subscription_instances si ON siv.subscription_instance_id = si.subscription_instance_id
+                          JOIN subscriptions s ON si.subscription_id = s.subscription_id
+                 GROUP BY s.subscription_id),
+     sub_prod_info AS (SELECT s.subscription_id,
+                              array_to_string(
+                                      ARRAY ['subscription_id:' || s.subscription_id,
+                                          'status:' || s.status,
+                                          'insync:' || s.insync,
+                                          'subscription_description:' || s.description,
+                                          'note:' || coalesce(s.note, ''),
+                                          'customer_id:' || s.customer_id,
+                                          'product_id:' || s.product_id],
+                                      ' '
+                              ) AS sub_info,
+                              array_to_string(
+                                      ARRAY ['product_name:' || p.name,
+                                          'product_description:' || p.description,
+                                          'tag:' || p.tag,
+                                          'product_type:', p.product_type],
+                                      ' '
+                              ) AS prod_info
+                       FROM subscriptions s
+                                JOIN products p ON s.product_id = p.product_id),
+     fi_info AS (SELECT s.subscription_id,
+                        string_agg(fi.name || ':' || fi.value, ' ') AS fi_info
+                 FROM subscriptions s
+                          JOIN products p ON s.product_id = p.product_id
+                          JOIN fixed_inputs fi ON p.product_id = fi.product_id
+                 GROUP BY s.subscription_id),
+     cust_info AS (SELECT s.subscription_id,
+                          string_agg('customer_description: ' || scd.description, ' ') AS cust_info
+                   FROM subscriptions s
+                            JOIN subscription_customer_descriptions scd ON s.subscription_id = scd.subscription_id
+                   GROUP BY s.subscription_id)
+-- to_tsvector handles parsing of hyphened words in a peculiar way and is inconsistent with how to_tsquery parses it in Postgres <14
+-- Replacing all hyphens with underscores makes the parsing more predictable and removes some issues arising when searching for subscription ids for example
+-- See: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=0c4f355c6a5fd437f71349f2f3d5d491382572b7
+SELECT s.subscription_id,
+       to_tsvector(
+               'simple',
+               replace(
+                       concat_ws(
+                               ' ',
+                               spi.sub_info,
+                               spi.prod_info,
+                               fi.fi_info,
+                               rti.rt_info,
+                               ci.cust_info,
+                               md.metadata::text
+                       ),
+                       '-', '_')
+       ) as tsv
+FROM subscriptions s
+         LEFT JOIN sub_prod_info spi ON s.subscription_id = spi.subscription_id
+         LEFT JOIN fi_info fi ON s.subscription_id = fi.subscription_id
+         LEFT JOIN rt_info rti ON s.subscription_id = rti.subscription_id
+         LEFT JOIN cust_info ci ON s.subscription_id = ci.subscription_id
+         LEFT JOIN subscription_metadata md ON s.subscription_id = md.subscription_id;
+
+-- Create indexes
+CREATE INDEX subscriptions_search_tsv_idx ON subscriptions_search USING GIN (tsv);
+CREATE UNIQUE INDEX subscriptions_search_subscription_id_idx ON subscriptions_search (subscription_id);
+
+-- Create refresh function with epoch-based throttling
+CREATE OR REPLACE FUNCTION refresh_subscriptions_search_view()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+AS
+$$
+DECLARE
+    should_refresh     bool;
+    current_epoch      int;
+    last_refresh_epoch int;
+    comment_sql        text;
+BEGIN
+    SELECT extract(epoch from now())::int INTO current_epoch;
+    SELECT coalesce(pg_catalog.obj_description('subscriptions_search'::regclass)::int, 0) INTO last_refresh_epoch;
+
+    SELECT (current_epoch - last_refresh_epoch) > 120 INTO should_refresh;
+
+    IF should_refresh THEN
+        REFRESH MATERIALIZED VIEW CONCURRENTLY subscriptions_search;
+
+        comment_sql := 'COMMENT ON MATERIALIZED VIEW subscriptions_search IS ' || quote_literal(current_epoch);
+        EXECUTE comment_sql;
+    END IF;
+    RETURN NULL;
+END;
+$$;
+
+-- Create triggers
+CREATE CONSTRAINT TRIGGER fi_refresh_search AFTER UPDATE ON fixed_inputs DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER products_refresh_search AFTER UPDATE ON products DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER sub_cust_desc_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscription_customer_descriptions DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER siv_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscription_instance_values DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER sub_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscriptions DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+
+-- Refresh the materialized view
+REFRESH MATERIALIZED VIEW subscriptions_search;

orchestrator_core-4.8.0rc1/orchestrator/migrations/versions/schema/2026-01-12_d69e10434a04_convert_unlimited_text_fields_to__upgrade.sql

@@ -0,0 +1,153 @@
+-- Convert unlimited text fields to limited nullable strings and normalize empty subscription notes.
+-- Revision ID: d69e10434a04
+
+-- Field length limits chosen based on expected usage patterns
+-- These values are intended to be reasonable, but give lots of wiggle room
+-- If these values are updated, they also need to be updated in orchestrator-core/orchestrator/db/models.py
+-- NOTE_LENGTH = 5000
+-- DESCRIPTION_LENGTH = 2000
+-- FAILED_REASON_LENGTH = 10000
+-- TRACEBACK_LENGTH = 50000
+-- RESOURCE_VALUE_LENGTH = 10000
+-- DOMAIN_MODEL_ATTR_LENGTH = 255
+
+-- Drop triggers first
+DROP TRIGGER IF EXISTS fi_refresh_search ON fixed_inputs;
+DROP TRIGGER IF EXISTS products_refresh_search ON products;
+DROP TRIGGER IF EXISTS sub_cust_desc_refresh_search ON subscription_customer_descriptions;
+DROP TRIGGER IF EXISTS siv_refresh_search ON subscription_instance_values;
+DROP TRIGGER IF EXISTS sub_refresh_search ON subscriptions;
+
+-- Drop the refresh function
+DROP FUNCTION IF EXISTS refresh_subscriptions_search_view();
+
+-- Drop the materialized view (CASCADE will drop indexes)
+DROP MATERIALIZED VIEW IF EXISTS subscriptions_search CASCADE;
+
+-- Normalize empty strings to NULL before altering column types
+UPDATE subscriptions
+SET note = NULL
+WHERE note IS NOT NULL AND TRIM(note) = '';
+
+-- Alter column types from TEXT to VARCHAR with limits
+ALTER TABLE subscriptions ALTER COLUMN note TYPE VARCHAR(5000);
+ALTER TABLE subscriptions ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE processes ALTER COLUMN failed_reason TYPE VARCHAR(10000);
+ALTER TABLE processes ALTER COLUMN traceback TYPE VARCHAR(50000);
+ALTER TABLE products ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE product_blocks ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE resource_types ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE workflows ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE subscription_customer_descriptions ALTER COLUMN description TYPE VARCHAR(2000);
+ALTER TABLE subscription_instance_values ALTER COLUMN value TYPE VARCHAR(10000);
+ALTER TABLE subscription_instance_relations ALTER COLUMN domain_model_attr TYPE VARCHAR(255);
+
+-- Recreate the materialized view
+CREATE MATERIALIZED VIEW subscriptions_search AS
+WITH rt_info AS (SELECT s.subscription_id,
+                        concat_ws(
+                                ' ',
+                                string_agg(rt.resource_type || ' ' || siv.value, ' '),
+                                string_agg(distinct 'subscription_instance_id' || ':' || si.subscription_instance_id, ' ')
+                        ) AS rt_info
+                 FROM subscription_instance_values siv
+                          JOIN resource_types rt ON siv.resource_type_id = rt.resource_type_id
+                          JOIN subscription_instances si ON siv.subscription_instance_id = si.subscription_instance_id
+                          JOIN subscriptions s ON si.subscription_id = s.subscription_id
+                 GROUP BY s.subscription_id),
+     sub_prod_info AS (SELECT s.subscription_id,
+                              array_to_string(
+                                      ARRAY ['subscription_id:' || s.subscription_id,
+                                          'status:' || s.status,
+                                          'insync:' || s.insync,
+                                          'subscription_description:' || s.description,
+                                          'note:' || coalesce(s.note, ''),
+                                          'customer_id:' || s.customer_id,
+                                          'product_id:' || s.product_id],
+                                      ' '
+                              ) AS sub_info,
+                              array_to_string(
+                                      ARRAY ['product_name:' || p.name,
+                                          'product_description:' || p.description,
+                                          'tag:' || p.tag,
+                                          'product_type:', p.product_type],
+                                      ' '
+                              ) AS prod_info
+                       FROM subscriptions s
+                                JOIN products p ON s.product_id = p.product_id),
+     fi_info AS (SELECT s.subscription_id,
+                        string_agg(fi.name || ':' || fi.value, ' ') AS fi_info
+                 FROM subscriptions s
+                          JOIN products p ON s.product_id = p.product_id
+                          JOIN fixed_inputs fi ON p.product_id = fi.product_id
+                 GROUP BY s.subscription_id),
+     cust_info AS (SELECT s.subscription_id,
+                          string_agg('customer_description: ' || scd.description, ' ') AS cust_info
+                   FROM subscriptions s
+                            JOIN subscription_customer_descriptions scd ON s.subscription_id = scd.subscription_id
+                   GROUP BY s.subscription_id)
+-- to_tsvector handles parsing of hyphened words in a peculiar way and is inconsistent with how to_tsquery parses it in Postgres <14
+-- Replacing all hyphens with underscores makes the parsing more predictable and removes some issues arising when searching for subscription ids for example
+-- See: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=0c4f355c6a5fd437f71349f2f3d5d491382572b7
+SELECT s.subscription_id,
+       to_tsvector(
+               'simple',
+               replace(
+                       concat_ws(
+                               ' ',
+                               spi.sub_info,
+                               spi.prod_info,
+                               fi.fi_info,
+                               rti.rt_info,
+                               ci.cust_info,
+                               md.metadata::text
+                       ),
+                       '-', '_')
+       ) as tsv
+FROM subscriptions s
+         LEFT JOIN sub_prod_info spi ON s.subscription_id = spi.subscription_id
+         LEFT JOIN fi_info fi ON s.subscription_id = fi.subscription_id
+         LEFT JOIN rt_info rti ON s.subscription_id = rti.subscription_id
+         LEFT JOIN cust_info ci ON s.subscription_id = ci.subscription_id
+         LEFT JOIN subscription_metadata md ON s.subscription_id = md.subscription_id;
+
+-- Create indexes
+CREATE INDEX subscriptions_search_tsv_idx ON subscriptions_search USING GIN (tsv);
+CREATE UNIQUE INDEX subscriptions_search_subscription_id_idx ON subscriptions_search (subscription_id);
+
+-- Create refresh function with epoch-based throttling
+CREATE OR REPLACE FUNCTION refresh_subscriptions_search_view()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+AS
+$$
+DECLARE
+    should_refresh     bool;
+    current_epoch      int;
+    last_refresh_epoch int;
+    comment_sql        text;
+BEGIN
+    SELECT extract(epoch from now())::int INTO current_epoch;
+    SELECT coalesce(pg_catalog.obj_description('subscriptions_search'::regclass)::int, 0) INTO last_refresh_epoch;
+
+    SELECT (current_epoch - last_refresh_epoch) > 120 INTO should_refresh;
+
+    IF should_refresh THEN
+        REFRESH MATERIALIZED VIEW CONCURRENTLY subscriptions_search;
+
+        comment_sql := 'COMMENT ON MATERIALIZED VIEW subscriptions_search IS ' || quote_literal(current_epoch);
+        EXECUTE comment_sql;
+    END IF;
+    RETURN NULL;
+END;
+$$;
+
+-- Create triggers
+CREATE CONSTRAINT TRIGGER fi_refresh_search AFTER UPDATE ON fixed_inputs DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER products_refresh_search AFTER UPDATE ON products DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER sub_cust_desc_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscription_customer_descriptions DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER siv_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscription_instance_values DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+CREATE CONSTRAINT TRIGGER sub_refresh_search AFTER INSERT OR UPDATE OR DELETE ON subscriptions DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_subscriptions_search_view();
+
+-- Refresh the materialized view
+REFRESH MATERIALIZED VIEW subscriptions_search;

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/schemas/product.py

@@ -12,10 +12,12 @@
 # limitations under the License.
 
 from datetime import datetime
+from typing import Annotated
 from uuid import UUID
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
+from orchestrator.db.models import DESCRIPTION_LENGTH
 from orchestrator.domain.lifecycle import ProductLifecycle
 from orchestrator.schemas.base import OrchestratorBaseModel
 from orchestrator.schemas.fixed_input import FixedInputSchema
@@ -44,4 +46,4 @@ class ProductSchema(ProductBaseSchema):
 
 
 class ProductPatchSchema(OrchestratorBaseModel):
-    description: str | None = None
+    description: Annotated[str, Field(max_length=DESCRIPTION_LENGTH)] | None = None

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/schemas/product_block.py

@@ -12,10 +12,12 @@
 # limitations under the License.
 
 from datetime import datetime
+from typing import Annotated
 from uuid import UUID
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
+from orchestrator.db.models import DESCRIPTION_LENGTH
 from orchestrator.domain.lifecycle import ProductLifecycle
 from orchestrator.schemas.base import OrchestratorBaseModel
 from orchestrator.schemas.resource_type import ResourceTypeBaseSchema, ResourceTypeSchema
@@ -40,4 +42,4 @@ class ProductBlockSchema(ProductBlockBaseSchema):
 
 
 class ProductBlockPatchSchema(OrchestratorBaseModel):
-    description: str | None = None
+    description: Annotated[str, Field(max_length=DESCRIPTION_LENGTH)] | None = None

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/schemas/resource_type.py

@@ -11,10 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Annotated
 from uuid import UUID
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
+from orchestrator.db.models import DESCRIPTION_LENGTH
 from orchestrator.schemas.base import OrchestratorBaseModel
 
 
@@ -30,4 +32,4 @@ class ResourceTypeSchema(ResourceTypeBaseSchema):
 
 
 class ResourceTypePatchSchema(OrchestratorBaseModel):
-    description: str | None = None
+    description: Annotated[str, Field(max_length=DESCRIPTION_LENGTH)] | None = None

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/schemas/search.py

@@ -34,10 +34,17 @@ class ProductSchema(BaseModel):
     product_type: str
 
 
+class CursorInfoSchema(BaseModel):
+    total_items: int | None = None
+    start_cursor: int | None = None
+    end_cursor: int | None = None
+
+
 class SearchResultsSchema(BaseModel, Generic[T]):
     data: list[T] = Field(default_factory=list)
     page_info: PageInfoSchema = Field(default_factory=PageInfoSchema)
     search_metadata: SearchMetadata | None = None
+    cursor: CursorInfoSchema | None = None
 
 
 class PathsResponse(BaseModel):

{orchestrator_core-4.7.1 → orchestrator_core-4.8.0rc1}/orchestrator/schemas/workflow.py

@@ -12,11 +12,12 @@
 # limitations under the License.
 
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 from uuid import UUID
 
 from pydantic import ConfigDict, Field
 
+from orchestrator.db.models import DESCRIPTION_LENGTH
 from orchestrator.schemas.base import OrchestratorBaseModel
 from orchestrator.targets import Target
 
@@ -65,4 +66,4 @@ class SubscriptionWorkflowListsSchema(OrchestratorBaseModel):
 
 
 class WorkflowPatchSchema(OrchestratorBaseModel):
-    description: str | None = None
+    description: Annotated[str, Field(max_length=DESCRIPTION_LENGTH)] | None = None