quapp-merlin 0.0.1.dev1__py3-none-any.whl

quapp_merlin/__init__.py

@@ -0,0 +1,8 @@
+from .data.merlin_job_spec import MerlinJobSpec, TrainingSpec
+from .factory.merlin_handler_factory import MerlinHandlerFactory
+
+# Imported for the side effect: register_rules() has to run before the
+# first job fails, so this import cannot be made lazy.
+from . import error_rules as _error_rules  # noqa: E402,F401
+
+__all__ = ['MerlinJobSpec', 'TrainingSpec', 'MerlinHandlerFactory']

quapp_merlin/component/backend/merlin_invocation.py

@@ -0,0 +1,82 @@
+#  Quapp Platform Project
+#  merlin_invocation.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+"""Wiring MerLin into the invocation pipeline.
+
+The pipeline carries whatever ``processing()`` returned without looking at it --
+except in one place, ``_get_qubit_amount``, which DeviceSelection uses to pick a
+machine. That is the only spot where a MerLin job has to explain itself, and the
+answer is a **mode count**, not a qubit count: see the note on the method.
+"""
+
+from quapp_common.component.backend.invocation import Invocation
+from quapp_common.data.request.invocation_request import InvocationRequest
+from quapp_common.model.provider.provider import Provider
+
+from ...factory.merlin_device_factory import MerlinDeviceFactory
+from ...factory.merlin_provider_factory import MerlinProviderFactory
+from ...util.single_machine import quantum_leaves
+
+
+class MerlinInvocation(Invocation):
+
+    def __init__(self, request_data: InvocationRequest):
+        super().__init__(request_data)
+
+    def _export_circuit(self, circuit):
+        # Only QISKIT and BRAKET export circuits (EXPORT_CIRCUIT_SDK in
+        # quapp-common). A MerLin job is a torch module, not a drawable circuit.
+        return None
+
+    def _create_provider(self):
+        self.logger.debug(
+                f'Creating provider: '
+                f'provider_tag={getattr(self.backend_information, "provider_tag", None)}')
+        provider = MerlinProviderFactory.create_provider(
+                provider_type=self.backend_information.provider_tag,
+                sdk=self.sdk,
+                authentication=self.backend_information.authentication)
+        self.logger.info(f'Provider created: {type(provider).__name__}')
+        return provider
+
+    def _create_device(self, provider: Provider):
+        device_specification = getattr(self.backend_information, 'device_name',
+                                       None)
+        self.logger.debug(f'Creating device: {device_specification}')
+        device = MerlinDeviceFactory.create_device(
+                provider=provider,
+                device_specification=device_specification,
+                authentication=self.backend_information.authentication,
+                sdk=self.sdk)
+        self.logger.info(f'Device created: {type(device).__name__}')
+        return device
+
+    def _get_qubit_amount(self, circuit):
+        """How many optical **modes** the job needs.
+
+        DeviceSelection asks for a qubit count because every other SDK is
+        gate-based. Photonic hardware is sized in modes, and MerLin's layers
+        carry a Perceval circuit whose ``m`` is exactly that. Reporting modes
+        here is the closest true answer; a device catalogue that stores qubits
+        for a Quandela machine has to be read the same way.
+
+        Returns 0 rather than raising when the model hides its circuit: failing
+        device selection over a metric nobody uses downstream would turn a
+        runnable job into a failed one.
+        """
+        for _, leaf in quantum_leaves(getattr(circuit, 'model', circuit)):
+            for attribute in ('m', 'n_modes', 'modes'):
+                modes = getattr(leaf, attribute, None)
+                if isinstance(modes, int) and modes > 0:
+                    return modes
+
+            inner = getattr(leaf, 'circuit', None)
+            modes = getattr(inner, 'm', None)
+            if isinstance(modes, int) and modes > 0:
+                return modes
+
+        self.logger.warning(
+                'Could not read a mode count from the model; reporting 0 to '
+                'device selection.')
+        return 0

quapp_merlin/data/merlin_job_spec.py

@@ -0,0 +1,93 @@
+#  Quapp Platform Project
+#  merlin_job_spec.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+"""What a MerLin handler returns instead of a circuit.
+
+Every other SDK in the platform has ``processing()`` return a circuit, and the
+whole pipeline carries that object around opaquely. MerLin has no circuit to
+return: the unit of work is a ``torch.nn.Module`` (normally one holding an
+``merlin.QuantumLayer``) plus the input tensor to run it on.
+
+Returning a bare Module would not be enough -- the model alone says nothing about
+WHAT to run it on, and the device would have to guess which key of
+``invocation_input`` holds the data. A spec makes the contract explicit and lets
+the device reject a malformed job with a specific error code instead of a
+``TypeError`` deep inside torch.
+
+A bare Module is still accepted as shorthand; see ``coerce()``.
+"""
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class TrainingSpec:
+    """Local-only training.
+
+    MerLin's remote processor cannot backpropagate -- the documentation is
+    explicit: *"No gradient can flow through the MerLin processor."* So a spec
+    carrying ``training`` is refused on a remote device and only runs on the
+    local simulator.
+    """
+
+    targets: object
+    epochs: int = 10
+    learning_rate: float = 0.01
+    loss: str = 'mse'
+
+
+@dataclass
+class MerlinJobSpec:
+    """One MerLin job: one model, one input batch, one machine.
+
+    @param model: the torch Module to run. Must contain exactly one quantum
+                  layer -- see the single-machine rules in
+                  ``model/device/merlin_remote_device.py``.
+    @param inputs: the input batch, shape ``[batch, input_size]``. Anything
+                   ``torch.as_tensor`` accepts is fine (list, ndarray, tensor).
+    @param shots: sample count. ``None`` or ``0`` means exact probabilities --
+                  MerLin does not invent a default, and neither do we.
+    @param sampling_method: e.g. ``'multinomial'``; ``None`` leaves MerLin's
+                            default alone.
+    @param training: present only for local training runs.
+    @param metadata: free-form, copied into the job result for the handler's own
+                     post_processing().
+    """
+
+    model: object
+    inputs: object
+    shots: int = None
+    sampling_method: str = None
+    training: TrainingSpec = None
+    metadata: dict = field(default_factory=dict)
+
+
+SPEC_HELP = (
+    'processing() must return a MerlinJobSpec, or a torch.nn.Module when the '
+    'input batch is in invocation_input["inputs"].')
+
+
+def coerce(prepared, invocation_input: dict = None) -> MerlinJobSpec:
+    """Turn what ``processing()`` returned into a MerlinJobSpec.
+
+    A bare Module is accepted because the common case -- one layer, data already
+    in the request -- should not need the import. The data then has to be in
+    ``invocation_input["inputs"]``; anything else is rejected here rather than
+    surfacing later as a shape error inside torch.
+    """
+    if isinstance(prepared, MerlinJobSpec):
+        return prepared
+
+    if hasattr(prepared, 'forward') and callable(prepared):
+        inputs = (invocation_input or {}).get('inputs')
+        if inputs is None:
+            raise ValueError(
+                    'processing() returned a model but no input batch: put it '
+                    'in invocation_input["inputs"], or return a MerlinJobSpec '
+                    'with inputs=... instead.')
+        return MerlinJobSpec(model=prepared, inputs=inputs,
+                             shots=(invocation_input or {}).get('shots'))
+
+    raise ValueError(
+            f'processing() returned {type(prepared).__name__}. {SPEC_HELP}')

quapp_merlin/error_rules.py

@@ -0,0 +1,190 @@
+#  Quapp Platform Project
+#  error_rules.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+"""MerLin-specific error classification rules.
+
+Registered into ``quapp_common.util.error_classifier`` when ``quapp_merlin`` is
+imported, so ``quapp-common`` never has to depend on MerLin, torch or Perceval.
+
+Matching is done on the message text or on the class **NAME as a string**, never
+``isinstance``, and nothing is imported from ``merlin`` / ``torch`` / ``perceval``:
+the classifier has to work even when the SDKs are absent, and importing torch
+here would make every ``import quapp_merlin`` pay for it.
+
+The first group keys off messages **this lib itself** raises. They are the rules
+that matter most, because every single-machine constraint is enforced by raising
+one of them -- without a rule they would all land on a generic
+``PREPROCESSING_FAILED`` and tell the user nothing about what to change:
+
+    util/single_machine.py    'Handler attached its own quantum backend'
+                              'has N quantum layers'
+                              'has no quantum layer'
+                              'Input batch has N rows'
+    data/merlin_job_spec.py   'processing() returned ...'
+    model/device/merlin_remote_device.py
+                              'Training cannot run on a remote device'
+                              'shots=N exceeds'
+    model/provider/quandela_merlin_provider.py
+                              'No Quandela Cloud token'
+"""
+
+from quapp_common.enum.error_category import ErrorCategory
+from quapp_common.util.error_classifier import Rule, register_rules
+
+DOC_MACHINE = 'errors/merlin-single-machine'
+DOC_SPEC = 'errors/merlin-job-spec'
+DOC_DEVICE = 'errors/merlin-device'
+DOC_AUTH = 'errors/quandela-auth'
+
+
+def _message(exception) -> str:
+    return str(exception).lower()
+
+
+def _name(exception) -> str:
+    return type(exception).__name__
+
+
+register_rules(
+        # ----------------- raised by quapp-merlin itself (one job, one machine)
+        Rule(match=lambda e: 'attached its own quantum backend' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_FOREIGN_BACKEND',
+             resolution='The handler built its own quantum backend. A job runs '
+                        'on exactly one machine, and the platform chooses it.',
+             steps=('Remove the MerlinProcessor / RemoteProcessor / Session '
+                    'from the model returned by processing()',
+                    'Pick the machine through the device of the job instead'),
+             doc_slug=DOC_MACHINE),
+
+        Rule(match=lambda e: 'quantum layers' in _message(e)
+                             and 'one invocation runs one' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_MULTI_LAYER_NOT_ALLOWED',
+             resolution='The model holds more than one quantum layer; a single '
+                        'invocation runs one.',
+             steps=('Split the layers across separate invocations',
+                    'Or run the whole model on the local simulator'),
+             doc_slug=DOC_MACHINE),
+
+        Rule(match=lambda e: 'no quantum layer' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_NO_QUANTUM_LAYER',
+             resolution='The model has no quantum layer, so there is nothing '
+                        'for a quantum device to run.',
+             steps=('Add a merlin.QuantumLayer to the model',
+                    'A purely classical model does not need a quantum job'),
+             doc_slug=DOC_SPEC),
+
+        Rule(match=lambda e: 'input batch has' in _message(e)
+                             and 'rows' in _message(e),
+             category=ErrorCategory.INVALID_INPUT,
+             code='MERLIN_BATCH_TOO_LARGE',
+             resolution='The input batch is large enough that MerLin would '
+                        'split it into several cloud jobs.',
+             steps=('Send fewer rows per invocation',
+                    'Or run the batch on the local simulator'),
+             doc_slug=DOC_MACHINE),
+
+        Rule(match=lambda e: 'processing() returned' in _message(e)
+                             or 'must return a merlinjobspec' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_INVALID_JOB_SPEC',
+             resolution='processing() did not return something MerLin can run.',
+             steps=('Return a MerlinJobSpec(model=..., inputs=...)',
+                    'Or return a torch model and put the batch in '
+                    'invocation_input["inputs"]'),
+             doc_slug=DOC_SPEC),
+
+        Rule(match=lambda e: 'training cannot run on a remote device' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_TRAINING_NOT_REMOTE',
+             resolution='Training cannot run on a remote machine: no gradient '
+                        'flows through the MerLin processor.',
+             steps=('Train on the local simulator',
+                    'Send the trained model to the QPU for inference only'),
+             doc_slug=DOC_MACHINE),
+
+        Rule(match=lambda e: 'exceeds the' in _message(e)
+                             and 'single cloud call' in _message(e),
+             category=ErrorCategory.INVALID_INPUT,
+             code='MERLIN_SHOTS_TOO_MANY',
+             resolution='The requested shot count is above what one cloud call '
+                        'accepts, and would be silently clamped.',
+             steps=('Lower shots for this job',
+                    'Or split the run across several invocations'),
+             doc_slug=DOC_MACHINE),
+
+        # ------------------------------------------------- Quandela Cloud auth
+        Rule(match=lambda e: 'no quandela cloud token' in _message(e),
+             category=ErrorCategory.AUTHENTICATION,
+             code='QUANDELA_TOKEN_MISSING',
+             resolution='No Quandela Cloud token is configured for this '
+                        'workspace.',
+             steps=('Open Workspace -> Providers -> Quandela',
+                    'Add the API token from cloud.quandela.com'),
+             doc_slug=DOC_AUTH,
+             retryable=False),
+
+        Rule(match=lambda e: 'unsupported device' in _message(e)
+                             or 'unsupported provider' in _message(e),
+             category=ErrorCategory.PROVIDER_DEVICE,
+             code='MERLIN_DEVICE_UNSUPPORTED',
+             resolution="This device or provider is not supported by Quapp's "
+                        'MerLin lib yet.',
+             steps=('Check the device selected for the job',
+                    'quapp-merlin serves QUAO_QUANTUM_SIMULATOR and QUANDELA'),
+             doc_slug=DOC_DEVICE),
+
+        # ------------------------------------------------------ torch failures
+        # A Fock space grows combinatorially with modes and photons, so the
+        # usual way to run out of memory here is an over-ambitious layer, not a
+        # platform fault -- the user is the one who can fix it.
+        Rule(match=lambda e: _name(e) == 'OutOfMemoryError'
+                             or 'out of memory' in _message(e)
+                             or 'cannot allocate' in _message(e),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_OUT_OF_MEMORY',
+             resolution='The simulation ran out of memory: the state space of '
+                        'this circuit is too large for the function.',
+             steps=('Use fewer modes or fewer photons',
+                    'Reduce the batch size',
+                    'Consider no_bunching or a smaller computation space')),
+
+        Rule(match=lambda e: _name(e) == 'RuntimeError'
+                             and ('shapes cannot be multiplied' in _message(e)
+                                  or 'size mismatch' in _message(e)
+                                  or 'must match the size' in _message(e)
+                                  or 'expected input' in _message(e)),
+             category=ErrorCategory.USER_CODE,
+             code='MERLIN_SHAPE_MISMATCH',
+             resolution='The input tensor does not match what the model '
+                        'expects.',
+             steps=('Check input_size of the quantum layer against the width '
+                    'of your input batch',
+                    'inputs must have shape [batch, input_size]'),
+             doc_slug=DOC_SPEC),
+
+        # ------------------------------------------- remote execution failures
+        Rule(match=lambda e: _name(e) in {'TimeoutError', 'FuturesTimeoutError'}
+                             or 'timed out' in _message(e),
+             category=ErrorCategory.TIMEOUT,
+             code='MERLIN_REMOTE_TIMEOUT',
+             resolution='The remote machine did not return a result in time.',
+             steps=('The QPU queue may be long -- try again later',
+                    'Reduce shots or batch size to shorten the run'),
+             retryable=True,
+             retry_after=300),
+
+        # Placed last: matching is by NAME, so only these exact names land here.
+        Rule(match=lambda e: _name(e) in {'PercevalError', 'ProcessorError',
+                                          'RPCError', 'CircuitError'},
+             category=ErrorCategory.SDK_RUNTIME,
+             code='MERLIN_SDK_ERROR',
+             resolution='Perceval failed while running the circuit behind '
+                        'MerLin. See the stack trace below.',
+             steps=('Read the rootCause to see which step was rejected',
+                    'Check the mode count and photon count the platform '
+                    'supports')),
+)

quapp_merlin/factory/merlin_device_factory.py

@@ -0,0 +1,42 @@
+#  Quapp Platform Project
+#  merlin_device_factory.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+from quapp_common.config.logging_config import logger
+from quapp_common.enum.provider_tag import ProviderTag
+from quapp_common.enum.sdk import Sdk
+from quapp_common.factory.device_factory import DeviceFactory
+from quapp_common.model.provider.provider import Provider
+
+from ..model.device.merlin_local_device import MerlinLocalDevice
+from ..model.device.merlin_remote_device import MerlinRemoteDevice
+
+logger = logger.bind(context='MerlinDeviceFactory')
+
+_MERLIN_SDK = getattr(Sdk, 'MERLIN', None)
+
+
+class MerlinDeviceFactory(DeviceFactory):
+
+    @staticmethod
+    def create_device(provider: Provider, device_specification: str,
+            authentication: dict, sdk: Sdk):
+        logger.debug(f'Create Merlin device: {device_specification}')
+
+        if _MERLIN_SDK is not None and not _MERLIN_SDK.__eq__(sdk):
+            raise Exception('Unsupported device!')
+
+        provider_type = ProviderTag.resolve(
+                provider.get_provider_type().value)
+
+        # One device, one machine: the remote device receives the
+        # RemoteProcessor the provider just built and uses nothing else.
+        if provider_type is ProviderTag.QUANDELA:
+            logger.info('Created remote Merlin device')
+            return MerlinRemoteDevice(provider, device_specification)
+
+        if provider_type is ProviderTag.QUAO_QUANTUM_SIMULATOR:
+            logger.info('Created local Merlin device')
+            return MerlinLocalDevice(provider, device_specification)
+
+        raise Exception('Unsupported device!')

quapp_merlin/factory/merlin_handler_factory.py

@@ -0,0 +1,35 @@
+#  Quapp Platform Project
+#  merlin_handler_factory.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+from quapp_common.config.logging_config import job_logger
+from quapp_common.factory.handler_factory import HandlerFactory
+from quapp_common.handler.handler import Handler
+
+from ..handler.invocation_handler import InvocationHandler
+
+
+class MerlinHandlerFactory(HandlerFactory):
+
+    @staticmethod
+    def create_handler(event, circuit_preparation_fn,
+            post_processing_fn) -> Handler:
+        request_data = event.json()
+
+        # event.json() returns a DICT, so getattr(..., 'jobId') would always
+        # give None and the job_id would never get bound. Read it with .get().
+        logger = job_logger(request_data.get('jobId'))
+        logger.debug('Create Merlin Handler')
+
+        provider_job_id = request_data.get('providerJobId')
+
+        if provider_job_id is None:
+            logger.debug('providerJobId is None. Create InvocationHandler')
+            return InvocationHandler(
+                    request_data=request_data,
+                    circuit_preparation_fn=circuit_preparation_fn,
+                    post_processing_fn=post_processing_fn)
+
+        raise ValueError(
+                'MerLin has no job-fetching handler: a run is synchronous and '
+                'its result is returned by the invocation that started it.')

quapp_merlin/factory/merlin_provider_factory.py

@@ -0,0 +1,36 @@
+#  Quapp Platform Project
+#  merlin_provider_factory.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+from quapp_common.config.logging_config import logger
+from quapp_common.enum.provider_tag import ProviderTag
+from quapp_common.enum.sdk import Sdk
+from quapp_common.factory.provider_factory import ProviderFactory
+
+from ..model.provider.quandela_merlin_provider import QuandelaMerlinProvider
+from ..model.provider.quapp_merlin_provider import QuappMerlinProvider
+
+logger = logger.bind(context='MerlinProviderFactory')
+
+_MERLIN_SDK = getattr(Sdk, 'MERLIN', None)
+
+
+class MerlinProviderFactory(ProviderFactory):
+
+    @staticmethod
+    def create_provider(provider_type: ProviderTag, sdk: Sdk,
+            authentication: dict):
+        logger.debug(f'Create Merlin provider for {provider_type}')
+
+        if _MERLIN_SDK is not None and not _MERLIN_SDK.__eq__(sdk):
+            raise Exception('Unsupported provider!')
+
+        if provider_type is ProviderTag.QUANDELA:
+            logger.debug('Create Quandela Merlin Provider')
+            return QuandelaMerlinProvider(authentication=authentication)
+
+        if provider_type is ProviderTag.QUAO_QUANTUM_SIMULATOR:
+            logger.debug('Create Quapp Merlin Provider')
+            return QuappMerlinProvider(authentication=authentication)
+
+        raise Exception('Unsupported provider!')

quapp_merlin/handler/invocation_handler.py

@@ -0,0 +1,59 @@
+#  Quapp Platform Project
+#  invocation_handler.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+"""Entry point for a MerLin invocation.
+
+One thing differs from the other libs: the handler's ``processing()`` returns a
+model, not a circuit, so it is wrapped here to produce a validated
+``MerlinJobSpec`` before anything else touches it.
+
+The wrapper runs inside ``Invocation.__prepare_circuit``, which means inside
+``with phase(PREPROCESSING)`` -- so a malformed return value is reported as a
+preprocessing error belonging to the user, which is exactly what it is.
+"""
+
+from quapp_common.data.request.invocation_request import InvocationRequest
+from quapp_common.handler.handler import Handler
+
+from ..component.backend.merlin_invocation import MerlinInvocation
+from ..data.merlin_job_spec import coerce
+
+
+class InvocationHandler(Handler):
+    def __init__(self, request_data: dict, circuit_preparation_fn,
+                 post_processing_fn):
+        super().__init__(request_data, post_processing_fn)
+        self.circuit_preparation_fn = circuit_preparation_fn
+
+    def _prepare_spec(self):
+        """``processing()``, with its return value normalised to a job spec."""
+
+        def prepare(invocation_input):
+            prepared = self.circuit_preparation_fn(invocation_input)
+            spec = coerce(prepared, invocation_input)
+            self.logger.debug(
+                    f'Job spec ready: model={type(spec.model).__name__}, '
+                    f'shots={spec.shots}, training={spec.training is not None}')
+            return spec
+
+        return prepare
+
+    def handle(self):
+        self.logger.debug('Handle Invocation: start')
+        try:
+            invocation_request = InvocationRequest(self.request_data)
+            self.logger.debug(f'Invocation request created successfully: '
+                              f'{invocation_request.__dict__.keys()}')
+
+            backend = MerlinInvocation(invocation_request)
+            self.logger.info('Submitting job via MerlinInvocation')
+
+            backend.submit_job(
+                    circuit_preparation_fn=self._prepare_spec(),
+                    post_processing_fn=self.post_processing_fn)
+
+            self.logger.debug('Job submission triggered successfully')
+        except Exception as exception:
+            self.logger.exception(f'Invocation handling failed: {exception}')
+            raise