haiway 0.19.3__py3-none-any.whl → 0.19.5__py3-none-any.whl

haiway/__init__.py

@@ -22,6 +22,7 @@ from haiway.helpers import (
     LoggerObservability,
     asynchronous,
     cache,
+    process_concurrently,
     retry,
     throttle,
     timeout,
@@ -112,6 +113,7 @@ __all__ = (
     "mimic_function",
     "noop",
     "not_missing",
+    "process_concurrently",
     "retry",
     "setup_logging",
     "throttle",

haiway/helpers/__init__.py

@@ -1,5 +1,6 @@
 from haiway.helpers.asynchrony import asynchronous, wrap_async
 from haiway.helpers.caching import CacheMakeKey, CacheRead, CacheWrite, cache
+from haiway.helpers.concurrent import process_concurrently
 from haiway.helpers.observability import LoggerObservability
 from haiway.helpers.retries import retry
 from haiway.helpers.throttling import throttle
@@ -13,6 +14,7 @@ __all__ = (
     "LoggerObservability",
     "asynchronous",
     "cache",
+    "process_concurrently",
     "retry",
     "throttle",
     "timeout",

haiway/helpers/concurrent.py

@@ -0,0 +1,74 @@
+from asyncio import FIRST_COMPLETED, CancelledError, Task, wait
+from collections.abc import AsyncIterator, Callable, Coroutine
+from concurrent.futures import ALL_COMPLETED
+from typing import Any
+
+from haiway.context import ctx
+
+__all__ = ("process_concurrently",)
+
+
+async def process_concurrently[Element](  # noqa: C901
+    source: AsyncIterator[Element],
+    /,
+    handler: Callable[[Element], Coroutine[Any, Any, None]],
+    *,
+    concurrent_tasks: int = 2,
+    ignore_exceptions: bool = False,
+) -> None:
+    """Process elements from an async iterator concurrently.
+
+    Parameters
+    ----------
+    source: AsyncIterator[Element]
+        An async iterator providing elements to process.
+
+    handler: Callable[[Element], Coroutine[Any, Any, None]]
+        A coroutine function that processes each element.
+
+    concurrent_tasks: int
+        Maximum number of concurrent tasks (must be > 0), default is 2.
+
+    ignore_exceptions: bool
+        If True, exceptions from tasks will be logged but not propagated,
+         default is False.
+
+    """
+    assert concurrent_tasks > 0  # nosec: B101
+    running: set[Task[None]] = set()
+    try:
+        while element := await anext(source, None):
+            if len(running) < concurrent_tasks:
+                running.add(ctx.spawn(handler, element))
+                continue  # keep spawning tasks
+
+            completed, running = await wait(running, return_when=FIRST_COMPLETED)
+
+            for task in completed:
+                if exc := task.exception():
+                    if not ignore_exceptions:
+                        raise exc
+
+                    ctx.log_error(
+                        f"Concurrent processing error - {type(exc)}: {exc}",
+                        exception=exc,
+                    )
+
+    except CancelledError as exc:
+        # Cancel all running tasks
+        for task in running:
+            task.cancel()
+
+        raise exc
+
+    finally:
+        completed, _ = await wait(running, return_when=ALL_COMPLETED)
+        for task in completed:
+            if exc := task.exception():
+                if not ignore_exceptions:
+                    raise exc
+
+                ctx.log_error(
+                    f"Concurrent processing error - {type(exc)}: {exc}",
+                    exception=exc,
+                )

haiway/helpers/observability.py

@@ -218,9 +218,13 @@ def LoggerObservability(  # noqa: C901, PLR0915
         )
 
         # try complete parent scopes
-        parent_id: str = scope.parent_id
-        while scopes[parent_id].try_complete():
-            parent_id = scopes[parent_id].identifier.parent_id
+        if scope != root_scope:
+            parent_id: str = scope.parent_id
+            while scopes[parent_id].try_complete():
+                if scopes[parent_id].identifier == root_scope:
+                    break
+
+                parent_id = scopes[parent_id].identifier.parent_id
 
         # check for root completion
         if scopes[root_scope.scope_id].completed:

haiway/opentelemetry/observability.py

@@ -1,17 +1,17 @@
 import os
 from collections.abc import Mapping
-from typing import Any, Self, cast, final
+from typing import Any, ClassVar, Self, cast, final
 
 from opentelemetry import metrics, trace
 from opentelemetry._logs import get_logger, set_logger_provider
 from opentelemetry._logs._internal import Logger
 from opentelemetry._logs.severity import SeverityNumber
-from opentelemetry.context import Context
+from opentelemetry.context import Context, attach, detach, get_current
 from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.metrics._internal import Meter
-from opentelemetry.metrics._internal.instrument import Counter, Histogram
+from opentelemetry.metrics._internal.instrument import Counter
 from opentelemetry.sdk._logs import LoggerProvider
 from opentelemetry.sdk._logs._internal import LogRecord
 from opentelemetry.sdk._logs._internal.export import (
@@ -21,10 +21,7 @@ from opentelemetry.sdk._logs._internal.export import (
 )
 from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider
 from opentelemetry.sdk.metrics._internal.export import MetricExporter
-from opentelemetry.sdk.metrics.export import (
-    ConsoleMetricExporter,
-    PeriodicExportingMetricReader,
-)
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter, SpanExporter
@@ -44,8 +41,8 @@ class ScopeStore:
         "_completed",
         "_counters",
         "_exited",
-        "_histograms",
-        "_span_context",
+        "_token",
+        "context",
         "identifier",
         "logger",
         "meter",
@@ -57,6 +54,7 @@ class ScopeStore:
         self,
         identifier: ScopeIdentifier,
         /,
+        context: Context,
         span: Span,
         meter: Meter,
         logger: Logger,
@@ -64,13 +62,16 @@ class ScopeStore:
         self.identifier: ScopeIdentifier = identifier
         self.nested: list[ScopeStore] = []
         self._counters: dict[str, Counter] = {}
-        self._histograms: dict[str, Histogram] = {}
         self._exited: bool = False
         self._completed: bool = False
         self.span: Span = span
-        self._span_context: SpanContext = span.get_span_context()
         self.meter: Meter = meter
         self.logger: Logger = logger
+        self.context: Context = trace.set_span_in_context(
+            span,
+            context,
+        )
+        self._token: Any = attach(self.context)
 
     @property
     def exited(self) -> bool:
@@ -96,6 +97,8 @@ class ScopeStore:
 
         self._completed = True
         self.span.end()
+
+        detach(self._token)
         return True  # successfully completed
 
     def record_log(
@@ -104,11 +107,12 @@ class ScopeStore:
         /,
         level: ObservabilityLevel,
     ) -> None:
+        span_context: SpanContext = self.span.get_span_context()
         self.logger.emit(
             LogRecord(
-                span_id=self._span_context.span_id,
-                trace_id=self._span_context.trace_id,
-                trace_flags=self._span_context.trace_flags,
+                span_id=span_context.span_id,
+                trace_id=span_context.trace_id,
+                trace_flags=span_context.trace_flags,
                 body=message,
                 severity_text=level.name,
                 severity_number=SEVERITY_MAPPING[level],
@@ -174,14 +178,15 @@ class ScopeStore:
             },
         )
 
-    def record_attribute(
+    def record_attributes(
         self,
-        name: str,
+        attributes: Mapping[str, ObservabilityAttribute],
         /,
-        *,
-        value: ObservabilityAttribute,
     ) -> None:
-        if value is not None and value is not MISSING:
+        for name, value in attributes.items():
+            if value is None or value is MISSING:
+                continue
+
             self.span.set_attribute(
                 name,
                 value=cast(Any, value),
@@ -190,6 +195,9 @@ class ScopeStore:
 
 @final
 class OpenTelemetry:
+    service: ClassVar[str]
+    environment: ClassVar[str]
+
     @classmethod
     def configure(
         cls,
@@ -199,9 +207,11 @@ class OpenTelemetry:
         environment: str,
         otlp_endpoint: str | None = None,
         insecure: bool = True,
-        export_interval_millis: int = 10000,
+        export_interval_millis: int = 5000,
         attributes: Mapping[str, Any] | None = None,
     ) -> type[Self]:
+        cls.service = service
+        cls.environment = environment
         # Create shared resource for both metrics and traces
         resource: Resource = Resource.create(
             {
@@ -274,7 +284,7 @@ class OpenTelemetry:
         cls,
         level: ObservabilityLevel = ObservabilityLevel.INFO,
     ) -> Observability:
-        tracer: Tracer | None = None
+        tracer: Tracer = trace.get_tracer(cls.service)
         meter: Meter | None = None
         root_scope: ScopeIdentifier | None = None
         scopes: dict[str, ScopeStore] = {}
@@ -355,17 +365,12 @@ class OpenTelemetry:
             if not attributes:
                 return
 
-            for attribute, value in attributes.items():
-                scopes[scope.scope_id].record_attribute(
-                    attribute,
-                    value=value,
-                )
+            scopes[scope.scope_id].record_attributes(attributes)
 
         def scope_entering[Metric: State](
             scope: ScopeIdentifier,
             /,
         ) -> None:
-            nonlocal tracer
             assert scope.scope_id not in scopes  # nosec: B101
 
             nonlocal root_scope
@@ -373,17 +378,14 @@ class OpenTelemetry:
 
             scope_store: ScopeStore
             if root_scope is None:
-                tracer = trace.get_tracer(scope.trace_id)
                 meter = metrics.get_meter(scope.trace_id)
+                context: Context = get_current()
                 scope_store = ScopeStore(
                     scope,
+                    context=context,
                     span=tracer.start_span(
                         name=scope.label,
-                        context=Context(
-                            trace_id=scope.trace_id,
-                            scope_id=scope.scope_id,
-                            parent_id=scope.parent_id,
-                        ),
+                        context=context,
                         attributes={
                             "context.trace_id": scope.trace_id,
                             "context.scope_id": scope.scope_id,
@@ -396,21 +398,13 @@ class OpenTelemetry:
                 root_scope = scope
 
             else:
-                assert tracer is not None  # nosec: B101
                 assert meter is not None  # nosec: B101
-
                 scope_store = ScopeStore(
                     scope,
+                    context=scopes[scope.parent_id].context,
                     span=tracer.start_span(
                         name=scope.label,
-                        context=trace.set_span_in_context(
-                            scopes[scope.parent_id].span,
-                            Context(
-                                trace_id=scope.trace_id,
-                                scope_id=scope.scope_id,
-                                parent_id=scope.parent_id,
-                            ),
-                        ),
+                        context=scopes[scope.parent_id].context,
                         attributes={
                             "context.trace_id": scope.trace_id,
                             "context.scope_id": scope.scope_id,
@@ -432,6 +426,7 @@ class OpenTelemetry:
         ) -> None:
             nonlocal root_scope
             nonlocal scopes
+            nonlocal meter
             assert root_scope is not None  # nosec: B101
             assert scope.scope_id in scopes  # nosec: B101
 
@@ -446,14 +441,19 @@ class OpenTelemetry:
                 return  # not completed yet or already completed
 
             # try complete parent scopes
-            parent_id: str = scope.parent_id
-            while scopes[parent_id].try_complete():
-                parent_id = scopes[parent_id].identifier.parent_id
+            if scope != root_scope:
+                parent_id: str = scope.parent_id
+                while scopes[parent_id].try_complete():
+                    if scopes[parent_id].identifier == root_scope:
+                        break
+
+                    parent_id = scopes[parent_id].identifier.parent_id
 
             # check for root completion
             if scopes[root_scope.scope_id].completed:
                 # finished root - cleanup state
                 root_scope = None
+                meter = None
                 scopes = {}
 
         return Observability(

{haiway-0.19.3.dist-info → haiway-0.19.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiway
-Version: 0.19.3
+Version: 0.19.5
 Summary: Framework for dependency injection and state management within structured concurrency model.
 Project-URL: Homepage, https://miquido.com
 Project-URL: Repository, https://github.com/miquido/haiway.git

{haiway-0.19.3.dist-info → haiway-0.19.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-haiway/__init__.py,sha256=UjaJeNa5lQGaeOguf2COSakrBDc8sd5zB9ioHiK4uOw,2353
+haiway/__init__.py,sha256=bFRmunbEmBzJy5iH_2WoMzvwhApRk3eo37b23zBe6Bc,2407
 haiway/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 haiway/context/__init__.py,sha256=eoxqxUmFtkWLhc_gH6tqax9m90tSDie-jiP1BHruTdk,1102
 haiway/context/access.py,sha256=vTh5BlduAGBjqnVyTgTUt08DG6SrnAakweAfIDBqx-0,18838
@@ -8,16 +8,17 @@ haiway/context/observability.py,sha256=u9NPejZGyGupqx3eH9IQKbcVcCFpZrg2ZAiov2FQi
 haiway/context/state.py,sha256=0oq7ctNO0urJd7rVzwwNtgpguoXuI6Tp1exfCsxrS2M,5981
 haiway/context/tasks.py,sha256=QOxFdjmMp4IYff0ihHElKLCQrcVksSJmxqTlOKfoH4o,2907
 haiway/context/types.py,sha256=WulPvpqUbI1vYyny-s2NItldDnk3zh1O-n_hGibFZRY,142
-haiway/helpers/__init__.py,sha256=dYqwWSBk8ss9XyXEF6YHZNPrCwU8VDM4nKtcz1tsvZI,583
+haiway/helpers/__init__.py,sha256=_A-JYAPemd-2o4NxxchaZENu8gANL8QDxK5ADzInXUU,670
 haiway/helpers/asynchrony.py,sha256=k_A0yCWUKSFfzYZ8WvqK4wqTMljv6ykMivmERrDLHIU,6266
 haiway/helpers/caching.py,sha256=N60FBFfSoOdSUI6S49l47G6T1j-y9tc3qhU5msSRdPM,13221
-haiway/helpers/observability.py,sha256=6UZMDR56kljGpou7qeDFl5XK-NkxZY0ob2nSFesO9TY,7843
+haiway/helpers/concurrent.py,sha256=04pWpS7qOQDD1pa2ifhFwedPcl4sRinhe-KgyDe5GF0,2296
+haiway/helpers/observability.py,sha256=hahA7jgGj5TH6mYBm3bKVwI22Fseuh-IX6X487gueaU,7977
 haiway/helpers/retries.py,sha256=unssUKBDOENvquh6R4Ud65TuSKl4mTHgZ5N_b7mAYa4,7533
 haiway/helpers/throttling.py,sha256=U6HJvSzffw47730VeiXxXSW4VVxpDx48k0oIAOpL-O4,4115
 haiway/helpers/timeouted.py,sha256=_M8diuD_GN49pl5KQA5fMKn4iUHsUuhkDSatAwWXiK8,3331
 haiway/helpers/tracing.py,sha256=6724MnXDqVcdWiQh0ggZRC3s5ZZSp6LEtLh3uze8SS0,4264
 haiway/opentelemetry/__init__.py,sha256=TV-1C14mDAtcHhFZ29ActFQdrGH6x5KuGV9w-JlKYJg,91
-haiway/opentelemetry/observability.py,sha256=ac0Jl0YFkpb4rbmwg3smPAs-QV_pcMjxSzelGhbwcsc,15332
+haiway/opentelemetry/observability.py,sha256=_VIhPRk8zfpww5YB-Tvct9R-J7oge-tTs20Zlj_0Dto,15245
 haiway/state/__init__.py,sha256=AaMqlMhO4zKS_XNevy3A7BHh5PxmguA-Sk_FnaNDY1Q,355
 haiway/state/attributes.py,sha256=p6jUBzg62bOl0zAYTCa7NIllsaNY2Kt68IooQ9tb-y8,23311
 haiway/state/path.py,sha256=-IpbUpF2QHWg3hEITkWYHJ6ZPoRVixu-SOSuWk-bbBY,21318
@@ -39,7 +40,7 @@ haiway/utils/mimic.py,sha256=L5AS4WEL2aPMZAQZlvLvRzHl0cipI7ivky60_eL4iwY,1822
 haiway/utils/noop.py,sha256=f54PSLHGEwCQNYXQHkPAW5NDE-tk5yjzkNL1pZj0TJQ,344
 haiway/utils/queue.py,sha256=YTvCn3wgSwLJiLqolMx44sa3304Xkv3tJG77gvfWnZs,4114
 haiway/utils/stream.py,sha256=Mjhy2S-ZDR1g_NsgS_nuBA8AgVbhrGXKvG3wjJ5mCJQ,2826
-haiway-0.19.3.dist-info/METADATA,sha256=U7KxwEJ85Smk5GOBJyPlCTjf_9x39y3-D9m74QARekU,4527
-haiway-0.19.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiway-0.19.3.dist-info/licenses/LICENSE,sha256=3phcpHVNBP8jsi77gOO0E7rgKeDeu99Pi7DSnK9YHoQ,1069
-haiway-0.19.3.dist-info/RECORD,,
+haiway-0.19.5.dist-info/METADATA,sha256=9zl0LfMAvGxzSzK7owry2SMQfSnitr3LUH6_UeqBqkI,4527
+haiway-0.19.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiway-0.19.5.dist-info/licenses/LICENSE,sha256=3phcpHVNBP8jsi77gOO0E7rgKeDeu99Pi7DSnK9YHoQ,1069
+haiway-0.19.5.dist-info/RECORD,,