pycfdp 0.2.2__py3-none-any.whl

cfdp/shell/effect_executor.py

@@ -0,0 +1,285 @@
+"""The effect executor and its file table (design D7/D10).
+
+A pure machine never touches the world; it returns effects. The
+:class:`EffectExecutor` interprets those effects against the real filestore,
+transport, and indication/message plumbing, and owns the one place where open
+OS file handles live: the ``tid -> handle`` :class:`FileTable` (design D10).
+
+The executor is deliberately *thin logic, real I/O*: each ``execute`` call maps
+exactly one :class:`~cfdp.pure.outputs.Output` effect to its side effect. The
+:class:`~cfdp.shell.dispatcher.Dispatcher` owns the exception boundary around
+these calls (design D4/D7), so the executor itself just does the work and lets
+exceptions propagate.
+"""
+
+from .. import logger
+from ..event import EventType
+from ..filestore.base import FileHandle
+from ..meta import execute_filestore_requests
+from ..pure.outputs import (
+    CancelTimer,
+    CloseFile,
+    EmitIndication,
+    ExecuteFilestoreRequests,
+    Finalize,
+    InternalEvent,
+    OpenSource,
+    OpenTemp,
+    Output,
+    PostUserMessage,
+    RequestChecksum,
+    ResumeTimer,
+    ScheduleEvent,
+    SendPdu,
+    StartTimer,
+    SuspendTimer,
+    TimerKind,
+    WriteSegment,
+)
+from ..pure.transaction import Transaction, TransactionId
+
+# Which EventType a given timer kind injects on expiry (design D6). The pure
+# machine emits ``StartTimer(kind, interval)`` without naming the event; the
+# shell owns this fixed kind -> event mapping so the machine stays agnostic of
+# how expiry re-enters the FIFO.
+_TIMER_EXPIRY_EVENT = {
+    TimerKind.INACTIVITY: EventType.E27_INACTIVITY_TIMEOUT,
+    TimerKind.ACK: EventType.E25_ACK_TIMEOUT,
+    TimerKind.NAK: EventType.E26_NAK_TIMEOUT,
+}
+
+
+class FileTable:
+    """The ``tid -> open handle`` map (design D10).
+
+    A machine that never opened a file has no entry, so cleanup is a no-op for
+    it — this kills the class of ``AttributeError`` the old ``shutdown()`` could
+    raise (design D10 / issue 01).
+
+    Thread-safety invariant (issue #21): ``get(tid)`` is read by the async
+    checksum worker thread while the event thread calls ``set``/``pop``/
+    ``cleanup``. This is lock-free-safe because the receive-side lifecycle
+    guarantees the worker's read never overlaps a close: the handle stays in the
+    table until the transaction is reaped, and reaping happens only after the
+    checksum result (E42) has been processed, i.e. after the worker has finished.
+    """
+
+    def __init__(self) -> None:
+        self._handles: dict[TransactionId, FileHandle] = {}
+
+    def get(self, tid: TransactionId) -> FileHandle | None:
+        return self._handles.get(tid)
+
+    def set(self, tid: TransactionId, handle) -> None:
+        self._handles[tid] = handle
+
+    def pop(self, tid: TransactionId):
+        return self._handles.pop(tid, None)
+
+    def __contains__(self, tid: TransactionId) -> bool:
+        return tid in self._handles
+
+    def close_all(self) -> None:
+        """Close and forget every open handle (entity shutdown / reaping)."""
+        for tid, handle in list(self._handles.items()):
+            try:
+                if handle is not None:
+                    handle.close()
+            except Exception:
+                logger.exception("Error closing file handle for %s", tid)
+            self._handles.pop(tid, None)
+
+    def cleanup(self, tid: TransactionId) -> None:
+        """Close and forget the handle for one transaction, if any."""
+        handle = self._handles.pop(tid, None)
+        if handle is not None:
+            try:
+                handle.close()
+            except Exception:
+                logger.exception("Error closing file handle for %s", tid)
+
+
+class EffectExecutor:
+    """Interprets effect outputs against real I/O (design D7).
+
+    Owns the :class:`FileTable`; opens/writes/finalizes/closes files, encodes and
+    sends PDUs, and fans indications and user messages out through the entity's
+    existing indication methods and message queue.
+    """
+
+    def __init__(
+        self, entity, file_table=None, timer_service=None, checksum_service=None
+    ):
+        self.entity = entity
+        self.files = file_table if file_table is not None else FileTable()
+        # The wall-clock timer table (design D6 / issue #17). Owns real timers
+        # per (tid, kind) and injects the expiry event onto the entity FIFO.
+        self.timers = timer_service
+        # The async-checksum worker (design R1 / issue #21). Computes a whole-file
+        # checksum off the event thread and injects E42_CHECKSUM_READY. Optional
+        # so the Class-1 shell-seam unit tests (which never request a checksum via
+        # the executor) can construct an executor without one.
+        self.checksums = checksum_service
+        # Direct-to-destination bookkeeping (write_to_destination_directly).
+        # ``_direct_dest`` maps a tid to the destination filename its handle was
+        # opened on (so a failed transfer's partial file can be deleted);
+        # ``_direct_committed`` marks tids whose data was verified/Finalized, so
+        # the following CloseFile keeps the file instead of deleting it.
+        self._direct_dest: dict[TransactionId, str] = {}
+        self._direct_committed: set[TransactionId] = set()
+
+    # -- the single execute door ------------------------------------------- #
+
+    def execute(
+        self, tid: TransactionId, transaction: Transaction, effect: Output
+    ) -> None:
+        """Execute one effect for the machine owning ``tid``.
+
+        Timer effects are handled by the dispatcher/timer service, not here;
+        internal events are re-enqueued by the dispatcher. This method covers
+        only the world-touching effects the Class-1 machines emit.
+        """
+        if isinstance(effect, SendPdu):
+            self.entity.transport.send(effect.pdu.encode())
+
+        elif isinstance(effect, OpenSource):
+            self.files.set(tid, self.entity.filestore.open(transaction.source_filename))
+
+        elif isinstance(effect, OpenTemp):
+            dest_name = transaction.destination_filename
+            if self.entity.write_to_destination_directly and dest_name:
+                # Direct-to-destination mode: write incoming data straight into
+                # the target file instead of a temp file, so a remote-service
+                # filestore can show download progress. Open read/write (``w+b``)
+                # so the async checksum can read the file back before Finished.
+                # If the destination is not yet known (Class-2 recovery with a
+                # lost Metadata, where only the machine holds the resolved name),
+                # fall back to a temp file and the normal Finalize copy.
+                self.files.set(tid, self.entity.filestore.open(dest_name, "w+b"))
+                self._direct_dest[tid] = dest_name
+            else:
+                self.files.set(tid, self.entity.filestore.open_tempfile())
+
+        elif isinstance(effect, WriteSegment):
+            handle = self.files.get(tid)
+            handle.seek(effect.offset)
+            handle.write(effect.data)
+
+        elif isinstance(effect, ExecuteFilestoreRequests):
+            if effect.requests:
+                execute_filestore_requests(self.entity, effect.requests)
+
+        elif isinstance(effect, Finalize):
+            self._finalize(tid, transaction, effect.destination_filename)
+
+        elif isinstance(effect, CloseFile):
+            handle = self.files.get(tid)
+            if handle is not None:
+                handle.close()
+            if tid in self._direct_dest:
+                # Direct-to-destination mode: the target file was written in
+                # place. If the transfer never reached Finalize (checksum/size
+                # fault, cancel, abandon), delete the partial file so a failed
+                # transfer never leaves a corrupt file behind. Pop the tracking
+                # so the second CloseFile on the happy path is a plain no-op.
+                dest_name = self._direct_dest.pop(tid)
+                if tid not in self._direct_committed:
+                    try:
+                        self.entity.filestore.delete_file(dest_name)
+                    except Exception:
+                        logger.exception(
+                            "Error deleting partial direct-write file for %s", tid
+                        )
+                self._direct_committed.discard(tid)
+            # Mirror the old machine: close_tempfile does NOT drop the handle,
+            # so a double CloseFile on the happy EOF path is a no-op the second
+            # time (the file object tolerates a repeat close). The handle is
+            # only dropped from the table when the transaction is reaped, so the
+            # QueryPort can still read the temp file between close and reap.
+
+        elif isinstance(effect, EmitIndication):
+            self._emit_indication(effect)
+
+        elif isinstance(effect, PostUserMessage):
+            self.entity._messages_to_user_queue.put(effect.message)
+
+        elif isinstance(effect, StartTimer):
+            self.timers.start(
+                tid, effect.kind, effect.interval, _TIMER_EXPIRY_EVENT[effect.kind]
+            )
+
+        elif isinstance(effect, CancelTimer):
+            self.timers.cancel(tid, effect.kind)
+
+        elif isinstance(effect, SuspendTimer):
+            self.timers.suspend(tid, effect.kind)
+
+        elif isinstance(effect, ResumeTimer):
+            self.timers.resume(tid, effect.kind)
+
+        elif isinstance(effect, RequestChecksum):
+            # Compute the whole-file checksum off the event thread (issue #21);
+            # the result returns as an injected E42_CHECKSUM_READY event.
+            self.checksums.request(
+                tid, transaction, effect.checksum_type, effect.from_temp_handle
+            )
+
+        elif isinstance(effect, (InternalEvent, ScheduleEvent)):
+            # Should be routed by the dispatcher, never reach the executor.
+            raise TypeError("internal events are not executor effects")
+
+        else:
+            raise TypeError(f"unknown effect: {effect!r}")
+
+    # -- file lifecycle ---------------------------------------------------- #
+
+    def _finalize(
+        self,
+        tid: TransactionId,
+        transaction: Transaction,
+        destination_filename=None,
+    ) -> None:
+        """Copy the completed temp file to its destination (receive side).
+
+        ``destination_filename`` from the :class:`Finalize` effect wins when the
+        machine resolved it from a late-recovered Metadata (Class-2 recovery);
+        otherwise fall back to the frozen transaction's filename (the Class-1
+        path, where Metadata arrives first).
+        """
+        if tid in self._direct_dest:
+            # Direct-to-destination mode: the data is already in the target
+            # file, so there is nothing to copy. Mark it committed so the
+            # following CloseFile keeps the file instead of deleting it.
+            self._direct_committed.add(tid)
+            return
+        dest_name = destination_filename or transaction.destination_filename
+        handle = self.files.get(tid)
+        if handle is None:
+            # No temp handle for this transaction (issue #30): raise a clean,
+            # descriptive error BEFORE opening dest, so the dispatcher's exception
+            # boundary faults cleanly rather than AttributeError-ing on ``None``
+            # (and no destination fd is opened, so none can leak).
+            raise RuntimeError(
+                f"cannot finalize {tid}: no open temp file handle to copy from"
+            )
+        dest = self.entity.filestore.open(dest_name, "wb")
+        try:
+            handle.seek(0)
+            chunk_size = 65536
+            while True:
+                chunk = handle.read(chunk_size)
+                if not chunk:
+                    break
+                dest.write(chunk)
+        finally:
+            # Always close the destination handle, on every path (a mid-copy
+            # read/write error included), so a failed finalize never leaks an fd.
+            dest.close()
+
+    # -- indication fan-out ------------------------------------------------ #
+
+    def _emit_indication(self, effect):
+        method = getattr(self.entity, "_indication_" + effect.kind, None)
+        if method is None:
+            raise TypeError(f"unknown indication kind: {effect.kind!r}")
+        method(*effect.args)

cfdp/shell/machine_registry.py

@@ -0,0 +1,165 @@
+"""The ``tid -> machine`` registry and machine creation (design D7/D9).
+
+Owns the map of live pure machines, builds them (folding config into a frozen
+:class:`~cfdp.pure.config.TransactionConfig` snapshot, design D9, and injecting a
+per-transaction :class:`~cfdp.shell.query_port.EntityQueryPort`, design D3), and
+tracks completion for TTL reaping.
+
+Two creation doors:
+
+* :meth:`create_sender` — from a local ``put`` request.
+* :meth:`create_receiver` — from an inbound Metadata/Filedata/EOF toward the
+  receiver.
+
+Both Class-1 (``UNACKNOWLEDGED``) and Class-2 (``ACKNOWLEDGED``) pure machines
+are built here (issue #20 wired Class-2 through the shell). These are the only
+machines (the old machines were removed at the cutover, issue #22).
+"""
+
+from ..constants import TransmissionMode
+from ..pure import (
+    EntityDefaults,
+    PutOverrides,
+    TransactionConfig,
+)
+from ..pure import (
+    Transaction as PureTransaction,
+)
+from ..pure.machines import Receiver1 as PureReceiver1
+from ..pure.machines import Receiver2 as PureReceiver2
+from ..pure.machines import Sender1 as PureSender1
+from ..pure.machines import Sender2 as PureSender2
+from ..pure.transaction import TransactionId
+from .query_port import EntityQueryPort
+
+
+class MachineRegistry:
+    """The live pure-machine map plus creation and reaping (design D7)."""
+
+    def __init__(self, entity, file_table, machines=None):
+        self._entity = entity
+        self._files = file_table
+        # Share the entity's machines dict so every lookup path
+        # (cancel/suspend/resume/report/freeze/thaw, is_complete, reaping) sees
+        # the machines uniformly.
+        self.machines = machines if machines is not None else {}
+
+    # -- membership -------------------------------------------------------- #
+
+    def get(self, tid: TransactionId):
+        return self.machines.get(tid)
+
+    def __contains__(self, tid: TransactionId) -> bool:
+        return tid in self.machines
+
+    # -- config snapshot (design D9) --------------------------------------- #
+
+    def _entity_defaults(self):
+        return EntityDefaults(
+            transaction_inactivity_limit=self._entity.transaction_inactivity_limit,
+            ack_timer_interval=self._entity.positive_ack_timer_interval,
+            ack_timer_expiration_limit=self._entity.positive_ack_timer_expiration_limit,
+            nak_timer_interval=self._entity.nak_timer_interval,
+            nak_timer_expiration_limit=self._entity.nak_timer_expiration_limit,
+            maximum_file_segment_length=self._entity.maximum_file_segment_length,
+            inter_pdu_delay=self._entity.inter_pdu_delay,
+            fault_handlers=dict(self._entity.default_fault_handlers),
+        )
+
+    def _resolve_config(self, remote_config, put_overrides=None):
+        return TransactionConfig.resolve(
+            self._entity_defaults(), remote_config, put_overrides
+        )
+
+    # -- creation doors ---------------------------------------------------- #
+
+    def create_sender(
+        self,
+        *,
+        seq_number,
+        destination_id,
+        source_filename,
+        destination_filename,
+        checksum_type,
+        transmission_mode,
+        maximum_file_segment_length,
+        messages_to_user=None,
+        filestore_requests=None,
+        fault_handler_overrides=None,
+        remote_config=None,
+    ):
+        """Build a pure sender for a local ``put`` and register it.
+
+        Picks :class:`PureSender1` for ``UNACKNOWLEDGED`` and :class:`PureSender2`
+        for ``ACKNOWLEDGED`` (issue #20); both take the same
+        ``(transaction, config, query_port, messages_to_user, filestore_requests)``
+        constructor.
+        """
+        transaction = PureTransaction(
+            source_entity_id=self._entity.entity_id,
+            seq_number=seq_number,
+            destination_entity_id=destination_id,
+            transmission_mode=transmission_mode,
+            source_filename=source_filename,
+            destination_filename=destination_filename,
+            checksum_type=checksum_type,
+        )
+        put_overrides = PutOverrides(
+            maximum_file_segment_length=maximum_file_segment_length,
+            fault_handler_overrides=fault_handler_overrides,
+        )
+        config = self._resolve_config(remote_config, put_overrides)
+        query_port = EntityQueryPort(self._entity, transaction, self._files)
+        sender_cls = (
+            PureSender2
+            if transmission_mode == TransmissionMode.ACKNOWLEDGED
+            else PureSender1
+        )
+        machine = sender_cls(
+            transaction,
+            config,
+            query_port,
+            messages_to_user=messages_to_user,
+            filestore_requests=filestore_requests,
+        )
+        self.machines[transaction.id] = machine
+        return machine, transaction
+
+    def create_receiver(self, pdu, remote_config=None):
+        """Build a pure receiver from an inbound toward-receiver PDU.
+
+        Picks :class:`PureReceiver1` for ``UNACKNOWLEDGED`` and
+        :class:`PureReceiver2` for ``ACKNOWLEDGED`` (issue #20), keyed off the
+        inbound PDU header's transmission mode.
+        """
+        source_entity_id = pdu.pdu_header.source_entity_id
+        seq_number = pdu.pdu_header.transaction_seq_number
+        destination_entity_id = pdu.pdu_header.destination_entity_id
+        transmission_mode = pdu.pdu_header.transmission_mode
+        # Metadata carries the filenames + checksum type; Filedata/EOF-first do
+        # not, matching the old core._run_action_for_incoming_pdu split.
+        source_filename = getattr(pdu, "source_filename", None)
+        destination_filename = getattr(pdu, "destination_filename", None)
+        checksum_type = getattr(pdu, "checksum_type", None)
+        kwargs = {}
+        if checksum_type is not None:
+            kwargs["checksum_type"] = checksum_type
+        transaction = PureTransaction(
+            source_entity_id=source_entity_id,
+            seq_number=seq_number,
+            destination_entity_id=destination_entity_id,
+            transmission_mode=transmission_mode,
+            source_filename=source_filename,
+            destination_filename=destination_filename,
+            **kwargs,
+        )
+        config = self._resolve_config(remote_config)
+        query_port = EntityQueryPort(self._entity, transaction, self._files)
+        receiver_cls = (
+            PureReceiver2
+            if transmission_mode == TransmissionMode.ACKNOWLEDGED
+            else PureReceiver1
+        )
+        machine = receiver_cls(transaction, config, query_port)
+        self.machines[transaction.id] = machine
+        return machine, transaction

cfdp/shell/query_port.py

@@ -0,0 +1,70 @@
+"""The synchronous, local-only query port implementation (design D3/D5/D10).
+
+A pure machine calls its injected :class:`~cfdp.pure.query_port.QueryPort`
+synchronously mid-decision for local + fast reads only: source size/checksum,
+sequential source segments, temp-file checksum read-back, and transport
+readiness. :class:`EntityQueryPort` backs those against the entity's real
+filestore/transport and the shell-held file handle (the send-side source handle
+or the receive-side temp handle) in the :class:`~cfdp.shell.effect_executor.FileTable`.
+
+One port instance is bound per transaction: it captures the ``tid`` and the
+frozen :class:`~cfdp.pure.transaction.Transaction` so the machine's calls resolve
+to that transaction's filenames and handle.
+"""
+
+from ..pure.transaction import Transaction
+
+
+class EntityQueryPort:
+    """A :class:`~cfdp.pure.query_port.QueryPort` bound to one transaction.
+
+    ``read_segment``/temp ``checksum`` read the handle the
+    :class:`~cfdp.shell.effect_executor.EffectExecutor` opened for this ``tid``
+    (design D10: the shell owns the fd, the machine holds only positions).
+    ``size``/source ``checksum`` open the source afresh, mirroring the old
+    ``Transaction`` read helpers exactly.
+    """
+
+    def __init__(self, entity, transaction: Transaction, file_table) -> None:
+        self._entity = entity
+        self._transaction = transaction
+        self._files = file_table
+        self._tid = transaction.id
+
+    def is_file(self, filename: str) -> bool:
+        return self._entity.filestore.is_file(filename)
+
+    def size(self, filename: str) -> int:
+        try:
+            return self._entity.filestore.get_size(filename)
+        except FileNotFoundError:
+            return 0
+
+    def checksum(self, checksum_type: int) -> int:
+        """Checksum the transaction's current file.
+
+        Receive side: read back the shell-held temp handle (design D10). Send
+        side (no temp handle in the table): open the source afresh, exactly as
+        the old ``Transaction.get_file_checksum`` did.
+        """
+        handle = self._files.get(self._tid)
+        if handle is not None:
+            return handle.calculate_checksum(checksum_type)
+        source = self._transaction.source_filename
+        if not source:
+            return 0
+        fh = self._entity.filestore.open(source)
+        try:
+            return fh.calculate_checksum(checksum_type)
+        finally:
+            fh.close()
+
+    def read_segment(self, offset: int, length: int) -> tuple[int, bytes]:
+        handle = self._files.get(self._tid)
+        if handle is None:
+            raise ValueError("File handle not defined")
+        handle.seek(offset)
+        return offset, handle.read(length)
+
+    def transport_ready(self) -> bool:
+        return self._entity.transport.is_ready()