ocrd 3.5.1__py3-none-any.whl → 3.7.0__py3-none-any.whl

ocrd_network/cli/client.py

@@ -1,14 +1,36 @@
+import sys
 import click
 from json import dumps
 from typing import List, Optional, Tuple
+from urllib.parse import urlparse
+from tempfile import NamedTemporaryFile
+
 from ocrd.decorators.parameter_option import parameter_option, parameter_override_option
 from ocrd_network.constants import JobState
 from ocrd_utils import DEFAULT_METS_BASENAME
 from ocrd_utils.introspect import set_json_key_value_overrides
 from ocrd_utils.str import parse_json_string_or_file
 from ..client import Client
+from requests import RequestException
+
+
+ADDRESS_HELP = 'The URL of the Processing Server. If not provided, ' + \
+    'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" environment variable is used by default'
 
 
+class URLType(click.types.StringParamType):
+    name = "url"
+    def convert(self, value, param, ctx):
+        try:
+            parsed = urlparse(value)
+            if parsed.scheme not in ("http", "https"):
+                self.fail(f"invalid URL scheme ({parsed.scheme}): only HTTP allowed",
+                          param, ctx)
+            return value
+        except ValueError as err:
+            self.fail(err, param, ctx)
+URL = URLType()
+
 @click.group('client')
 def client_cli():
     """
@@ -27,30 +49,40 @@ def discovery_cli():
 
 
 @discovery_cli.command('processors')
-@click.option('--address',
-              help='The address of the Processing Server. If not provided, '
-                   'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 def check_deployed_processors(address: Optional[str]):
     """
-    Get a list of deployed processing workers/processor servers.
+    Get a list of deployed processing workers.
     Each processor is shown only once regardless of the amount of deployed instances.
     """
     client = Client(server_addr_processing=address)
-    processors_list = client.check_deployed_processors()
+    try:
+        processors_list = client.check_deployed_processors()
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(dumps(processors_list, indent=4))
 
 
 @discovery_cli.command('processor')
-@click.option('--address',
-              help='The address of the Processing Server. If not provided, '
-                   'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 @click.argument('processor_name', required=True, type=click.STRING)
 def check_processor_ocrd_tool(address: Optional[str], processor_name: str):
     """
     Get the json tool of a deployed processor specified with `processor_name`
     """
     client = Client(server_addr_processing=address)
-    ocrd_tool = client.check_deployed_processor_ocrd_tool(processor_name=processor_name)
+    try:
+        ocrd_tool = client.check_deployed_processor_ocrd_tool(processor_name=processor_name)
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(dumps(ocrd_tool, indent=4))
 
 
@@ -63,39 +95,46 @@ def processing_cli():
 
 
 @processing_cli.command('check-log')
-@click.option('--address',
-              help='The address of the Processing Server. If not provided, '
-                   'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 @click.option('-j', '--processing-job-id', required=True)
-def check_processing_job_status(address: Optional[str], processing_job_id: str):
+def check_processing_job_log(address: Optional[str], processing_job_id: str):
     """
     Check the log of a previously submitted processing job.
     """
     client = Client(server_addr_processing=address)
-    response = client.check_job_log(job_id=processing_job_id)
+    try:
+        response = client.check_job_log(job_id=processing_job_id)
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(response._content.decode(encoding='utf-8'))
 
 
 @processing_cli.command('check-status')
-@click.option('--address',
-              help='The address of the Processing Server. If not provided, '
-                   'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 @click.option('-j', '--processing-job-id', required=True)
 def check_processing_job_status(address: Optional[str], processing_job_id: str):
     """
     Check the status of a previously submitted processing job.
     """
     client = Client(server_addr_processing=address)
-    job_status = client.check_job_status(processing_job_id)
-    assert job_status
+    try:
+        job_status = client.check_job_status(processing_job_id)
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(f"Processing job status: {job_status}")
 
 
 @processing_cli.command('run')
 @click.argument('processor_name', required=True, type=click.STRING)
-@click.option('--address',
-              help='The address of the Processing Server. If not provided, '
-                   'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 @click.option('-m', '--mets', required=True, default=DEFAULT_METS_BASENAME)
 @click.option('-I', '--input-file-grp', default='OCR-D-INPUT')
 @click.option('-O', '--output-file-grp', default='OCR-D-OUTPUT')
@@ -104,7 +143,6 @@ def check_processing_job_status(address: Optional[str], processing_job_id: str):
 @parameter_override_option
 @click.option('--result-queue-name')
 @click.option('--callback-url')
-@click.option('--agent-type', default='worker')
 @click.option('-b', '--block', default=False, is_flag=True,
               help='If set, the client will block till job timeout, fail or success.')
 @click.option('-p', '--print-state', default=False, is_flag=True,
@@ -120,9 +158,6 @@ def send_processing_job_request(
     parameter_override: List[Tuple[str, str]],
     result_queue_name: Optional[str],
     callback_url: Optional[str],
-    # TODO: This is temporally available to toggle
-    #  between the ProcessingWorker/ProcessorServer
-    agent_type: Optional[str],
     block: Optional[bool],
     print_state: Optional[bool]
 ):
@@ -133,7 +168,6 @@ def send_processing_job_request(
         "path_to_mets": mets,
         "description": "OCR-D Network client request",
         "input_file_grps": input_file_grp.split(','),
-        "agent_type": agent_type
     }
     if output_file_grp:
         req_params["output_file_grps"] = output_file_grp.split(',')
@@ -145,12 +179,26 @@ def send_processing_job_request(
     if callback_url:
         req_params["callback_url"] = callback_url
     client = Client(server_addr_processing=address)
-    processing_job_id = client.send_processing_job_request(
-        processor_name=processor_name, req_params=req_params)
-    assert processing_job_id
+    try:
+        processing_job_id = client.send_processing_job_request(
+            processor_name=processor_name, req_params=req_params)
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(f"Processing job id: {processing_job_id}")
+
     if block:
-        client.poll_job_status(job_id=processing_job_id, print_state=print_state)
+        try:
+            client.poll_job_status(job_id=processing_job_id, print_state=print_state)
+        except RequestException as e:
+            print(
+                getattr(e, 'detail_message', str(e)),
+                f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+            )
+            sys.exit(1)
 
 
 @client_cli.group('workflow')
@@ -162,58 +210,82 @@ def workflow_cli():
 
 
 @workflow_cli.command('check-status')
-@click.option('--address', help='The address of the Processing Server. If not provided, '
-                                'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
+@click.option('--address', type=URL, help=ADDRESS_HELP)
 @click.option('-j', '--workflow-job-id', required=True)
 def check_workflow_job_status(address: Optional[str], workflow_job_id: str):
     """
     Check the status of a previously submitted workflow job.
     """
     client = Client(server_addr_processing=address)
-    job_status = client.check_workflow_status(workflow_job_id)
-    assert job_status
+    try:
+        job_status = client.check_workflow_status(workflow_job_id)
+    except RequestException as e:
+        print(
+            getattr(e, 'detail_message', str(e)),
+            f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+        )
+        sys.exit(1)
     print(f"Workflow job status: {job_status}")
 
 
 @workflow_cli.command('run')
-@click.option('--address', help='The address of the Processing Server. If not provided, '
-                                'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
-@click.option('-m', '--path-to-mets', required=True)
-@click.option('-w', '--path-to-workflow', required=True)
-@click.option('--page-wise/--no-page-wise', is_flag=True, default=False, help="Whether to generate per-page jobs")
+@click.option('--address', type=URL, help=ADDRESS_HELP)
+@click.option('-m', '--path-to-mets', required=True, help="path to METS file of workspace to be processed (server-side path)")
+@click.option('-w', '--path-to-workflow', required=False, help="path to workflow file (server- or client-side path)")
+@click.option('--page-wise', is_flag=True, default=False, help="Whether to generate per-page jobs")
 @click.option('-b', '--block', default=False, is_flag=True,
               help='If set, the client will block till job timeout, fail or success.')
 @click.option('-p', '--print-state', default=False, is_flag=True,
               help='If set, the client will print job states by each iteration.')
+@click.argument('tasks', nargs=-1)
 def send_workflow_job_request(
     address: Optional[str],
     path_to_mets: str,
-    path_to_workflow: str,
+    path_to_workflow: Optional[str],
     page_wise: bool,
     block: bool,
-    print_state: bool
+    print_state: bool,
+    tasks: List[str]
 ):
     """
     Submit a workflow job to the processing server.
+
+    Provide workflow either via `tasks` arguments (same syntax
+    as in ``ocrd process`` tasks arguments), or via `-w` file path
+    (same syntax, but newline separated).
     """
+    if (path_to_workflow) != bool(len(tasks)):
+        raise ValueError("either -w/path-to-workflow or task argument(s) is required")
+
     client = Client(server_addr_processing=address)
-    workflow_job_id = client.send_workflow_job_request(
-        path_to_wf=path_to_workflow,
-        path_to_mets=path_to_mets,
-        page_wise=page_wise,
-    )
-    assert workflow_job_id
+    with NamedTemporaryFile() as workflow_file:
+        for task in tasks:
+            workflow_file.write((task + '\n').encode('utf-8'))
+        workflow_file.flush()
+        workflow_job_id = client.send_workflow_job_request(
+            path_to_wf=path_to_workflow or workflow_file.name,
+            path_to_mets=path_to_mets,
+            page_wise=page_wise,
+        )
     print(f"Workflow job id: {workflow_job_id}")
     if block:
         print(f"Polling state of workflow job {workflow_job_id}")
-        state = client.poll_workflow_status(job_id=workflow_job_id, print_state=print_state)
+        try:
+            state = client.poll_workflow_status(job_id=workflow_job_id, print_state=print_state)
+        except RequestException as e:
+            print(
+                getattr(e, 'detail_message', str(e)),
+                f"Requested URL: {getattr(getattr(e, 'response', ''), 'url', '')}"
+            )
+            sys.exit(1)
         if state != JobState.success:
             print(f"Workflow failed with {state}")
             exit(1)
         else:
-            print(f"Workflow succeeded")
+            print("Workflow succeeded")
             exit(0)
 
+
 @client_cli.group('workspace')
 def workspace_cli():
     """

ocrd_network/cli/processing_server.py

@@ -12,8 +12,7 @@ from ocrd_network import ProcessingServer, ServerAddressParamType
 def processing_server_cli(path_to_config, address: str):
     """
     Start the Processing Server
-    (proxy between the user and the
-    Processing Worker(s) / Processor Server(s))
+    (proxy between the user and the Processing Worker(s))
     """
 
     # Note, the address is already validated with the type field

ocrd_network/client.py

@@ -1,5 +1,5 @@
 from typing import Optional
-from ocrd_utils import config, getLogger, LOG_FORMAT
+from ocrd_utils import config, getLogger
 from .client_utils import (
     get_ps_deployed_processors,
     get_ps_deployed_processor_ocrd_tool,
@@ -21,7 +21,7 @@ class Client:
         timeout: int = config.OCRD_NETWORK_CLIENT_POLLING_TIMEOUT,
         wait: int = config.OCRD_NETWORK_CLIENT_POLLING_SLEEP
     ):
-        self.log = getLogger(f"ocrd_network.client")
+        self.log = getLogger("ocrd_network.client")
         if not server_addr_processing:
             server_addr_processing = config.OCRD_NETWORK_SERVER_ADDR_PROCESSING
         self.server_addr_processing = server_addr_processing

ocrd_network/client_utils.py

@@ -1,10 +1,13 @@
 import json
-from requests import get as request_get, post as request_post
+import os
+from requests import get as request_get, post as request_post, RequestException, Response
+from requests.exceptions import JSONDecodeError
 from time import sleep
 from .constants import JobState, NETWORK_PROTOCOLS
 
 
-def _poll_endpoint_status(ps_server_host: str, job_id: str, job_type: str, tries: int, wait: int, print_state: bool = False) -> JobState:
+def _poll_endpoint_status(ps_server_host: str, job_id: str, job_type: str, tries: int, wait: int,
+                          print_state: bool = False) -> JobState:
     if job_type not in ["workflow", "processor"]:
         raise ValueError(f"Unknown job type '{job_type}', expected 'workflow' or 'processor'")
     job_state = JobState.unset
@@ -22,6 +25,19 @@ def _poll_endpoint_status(ps_server_host: str, job_id: str, job_type: str, tries
     return job_state
 
 
+def _raise_if_error(response: Response) -> None:
+    """Check the requests-response and raise an exception if its status code indicates an error"""
+    try:
+        response.raise_for_status()
+    except RequestException as e:
+        try:
+            message = response.json()["detail"]
+        except JSONDecodeError:
+            message = response.text
+        e.detail_message = message
+        raise e
+
+
 def poll_job_status_till_timeout_fail_or_success(
     ps_server_host: str, job_id: str, tries: int, wait: int, print_state: bool = False) -> JobState:
     return _poll_endpoint_status(ps_server_host, job_id, "processor", tries, wait, print_state)
@@ -35,14 +51,14 @@ def poll_wf_status_till_timeout_fail_or_success(
 def get_ps_deployed_processors(ps_server_host: str):
     request_url = f"{ps_server_host}/processor"
     response = request_get(url=request_url, headers={"accept": "application/json; charset=utf-8"})
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     return response.json()
 
 
 def get_ps_deployed_processor_ocrd_tool(ps_server_host: str, processor_name: str):
     request_url = f"{ps_server_host}/processor/info/{processor_name}"
     response = request_get(url=request_url, headers={"accept": "application/json; charset=utf-8"})
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     return response.json()
 
 
@@ -55,17 +71,18 @@ def get_ps_processing_job_log(ps_server_host: str, processing_job_id: str):
 def get_ps_processing_job_status(ps_server_host: str, processing_job_id: str) -> JobState:
     request_url = f"{ps_server_host}/processor/job/{processing_job_id}"
     response = request_get(url=request_url, headers={"accept": "application/json; charset=utf-8"})
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     job_state = response.json()["state"]
-    assert job_state
+    assert job_state, "Propery 'state' is expected to always have a value"
     return getattr(JobState, job_state.lower())
 
+
 def get_ps_workflow_job_status(ps_server_host: str, workflow_job_id: str) -> JobState:
     request_url = f"{ps_server_host}/workflow/job-simple/{workflow_job_id}"
     response = request_get(url=request_url, headers={"accept": "application/json; charset=utf-8"})
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     job_state = response.json()["state"]
-    assert job_state
+    assert job_state, "Property 'state' is expected to always have a value"
     return getattr(JobState, job_state.lower())
 
 
@@ -76,9 +93,9 @@ def post_ps_processing_request(ps_server_host: str, processor: str, job_input: d
         headers={"accept": "application/json; charset=utf-8"},
         json=job_input
     )
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     processing_job_id = response.json()["job_id"]
-    assert processing_job_id
+    assert processing_job_id, "Property 'job_id' is expected to always have a value"
     return processing_job_id
 
 
@@ -92,16 +109,16 @@ def post_ps_workflow_request(
     response = request_post(
         url=request_url,
         headers={"accept": "application/json; charset=utf-8"},
-        files={"workflow": open(path_to_wf, "rb")}
+        files={"workflow": open(path_to_wf, "rb") if os.path.exists(path_to_wf) else path_to_wf}
     )
     # print(response.json())
     # print(response.__dict__)
     json_resp_raw = response.text
     # print(f'post_ps_workflow_request >> {response.status_code}')
     # print(f'post_ps_workflow_request >> {json_resp_raw}')
-    assert response.status_code == 200, f"Processing server: {request_url}, {response.status_code}"
+    _raise_if_error(response)
     wf_job_id = json.loads(json_resp_raw)["job_id"]
-    assert wf_job_id
+    assert wf_job_id, "Property 'job_id' is expected to always have a value"
     return wf_job_id
 
 

ocrd_network/constants.py

@@ -15,10 +15,6 @@ class StrEnum(str, Enum):
     def __str__(self):
         return self.value
 
-class AgentType(StrEnum):
-    PROCESSING_WORKER = "worker"
-    PROCESSOR_SERVER = "server"
-
 
 class DeployType(StrEnum):
     # Deployed by the Processing Server config file
@@ -39,7 +35,7 @@ class JobState(StrEnum):
     failed = "FAILED"
     # The processing job is queued inside the RabbitMQ
     queued = "QUEUED"
-    # Processing job is currently running in a Worker or Processor Server
+    # Processing job is currently running on a Worker
     running = "RUNNING"
     # Processing job finished successfully
     success = "SUCCESS"
@@ -52,7 +48,6 @@ class NetworkLoggingDirs(StrEnum):
     PROCESSING_JOBS = "processing_jobs"
     PROCESSING_SERVERS = "processing_servers"
     PROCESSING_WORKERS = "processing_workers"
-    PROCESSOR_SERVERS = "processor_servers"
 
 
 class ServerApiTags(StrEnum):

ocrd_network/database.py

@@ -65,7 +65,7 @@ async def sync_db_create_workspace(mets_path: str) -> DBWorkspace:
 async def db_get_workspace(workspace_id: str = None, workspace_mets_path: str = None) -> DBWorkspace:
     workspace = None
     if not workspace_id and not workspace_mets_path:
-        raise ValueError(f'Either `workspace_id` or `workspace_mets_path` field must be used as a search key')
+        raise ValueError('Either `workspace_id` or `workspace_mets_path` field must be used as a search key')
     if workspace_id:
         workspace = await DBWorkspace.find_one(
             DBWorkspace.workspace_id == workspace_id
@@ -89,7 +89,7 @@ async def sync_db_get_workspace(workspace_id: str = None, workspace_mets_path: s
 async def db_update_workspace(workspace_id: str = None, workspace_mets_path: str = None, **kwargs) -> DBWorkspace:
     workspace = None
     if not workspace_id and not workspace_mets_path:
-        raise ValueError(f'Either `workspace_id` or `workspace_mets_path` field must be used as a search key')
+        raise ValueError('Either `workspace_id` or `workspace_mets_path` field must be used as a search key')
     if workspace_id:
         workspace = await DBWorkspace.find_one(DBWorkspace.workspace_id == workspace_id)
         if not workspace:
@@ -274,4 +274,4 @@ def verify_mongodb_available(mongo_url: str) -> None:
         client = MongoClient(mongo_url, serverSelectionTimeoutMS=60000.0)
         client.admin.command("ismaster")
     except Exception:
-        raise RuntimeError(f'Cannot connect to MongoDB: {re_sub(r":[^@]+@", ":****@", mongo_url)}')
+        raise RuntimeError(f'Cannot connect to MongoDB: {re_sub(r":[^@]+@", ":****@", mongo_url)}')

ocrd_network/logging_utils.py

@@ -2,7 +2,7 @@ from logging import FileHandler, Formatter, Logger
 from pathlib import Path
 
 from ocrd_utils import config, LOG_FORMAT, safe_filename
-from .constants import AgentType, NetworkLoggingDirs
+from .constants import NetworkLoggingDirs
 
 
 def configure_file_handler_with_formatter(logger: Logger, log_file: Path, mode: str = "a") -> None:
@@ -54,10 +54,5 @@ def get_processing_server_logging_file_path(pid: int) -> Path:
 
 
 def get_processing_worker_logging_file_path(processor_name: str, pid: int) -> Path:
-    log_file: str = f"{AgentType.PROCESSING_WORKER}.{pid}.{processor_name}.log"
+    log_file: str = f"worker.{pid}.{processor_name}.log"
     return Path(get_root_logging_dir(NetworkLoggingDirs.PROCESSING_WORKERS), log_file)
-
-
-def get_processor_server_logging_file_path(processor_name: str, pid: int) -> Path:
-    log_file: str = f"{AgentType.PROCESSOR_SERVER}.{pid}.{processor_name}.log"
-    return Path(get_root_logging_dir(NetworkLoggingDirs.PROCESSOR_SERVERS), log_file)

ocrd_network/models/__init__.py

@@ -10,13 +10,11 @@ __all__ = [
     'DBWorkflowScript',
     'PYJobInput',
     'PYJobOutput',
-    'PYOcrdTool',
     'PYResultMessage',
     'PYWorkflowJobOutput'
 ]
 
 from .job import DBProcessorJob, DBWorkflowJob, PYJobInput, PYJobOutput, PYWorkflowJobOutput
 from .messages import PYResultMessage
-from .ocrd_tool import PYOcrdTool
 from .workspace import DBWorkspace
 from .workflow import DBWorkflowScript

ocrd_network/models/job.py

@@ -2,7 +2,7 @@ from beanie import Document
 from datetime import datetime
 from pydantic import BaseModel
 from typing import Dict, List, Optional
-from ..constants import AgentType, JobState
+from ..constants import JobState
 
 
 class PYJobInput(BaseModel):
@@ -13,30 +13,28 @@ class PYJobInput(BaseModel):
     workspace_id: Optional[str] = None
     description: Optional[str] = None
     input_file_grps: List[str]
-    output_file_grps: Optional[List[str]]
+    output_file_grps: Optional[List[str]] = None
     page_id: Optional[str] = None
     parameters: dict = {}  # Always set to empty dict when None, otherwise it fails ocr-d-validation
     result_queue_name: Optional[str] = None
     callback_url: Optional[str] = None
-    # Used to toggle between sending requests to different network agents
-    agent_type: AgentType = AgentType.PROCESSING_WORKER
-    # Auto generated by the Processing Server when forwarding to the Processor Server
+    # Auto generated by the Processing Server when forwarding to the Processing-Worker
     job_id: Optional[str] = None
     # If set, specifies a list of job ids this job depends on
     depends_on: Optional[List[str]] = None
 
-    class Config:
-        schema_extra = {
+    model_config = {
+        'json_schema_extra': {
             'example': {
                 'path_to_mets': '/path/to/mets.xml',
                 'description': 'The description of this execution',
                 'input_file_grps': ['DEFAULT'],
                 'output_file_grps': ['OCR-D-BIN'],
-                'agent_type': AgentType.PROCESSING_WORKER,
                 'page_id': 'PHYS_0001..PHYS_0003',
                 'parameters': {}
             }
         }
+    }
 
 
 class PYJobOutput(BaseModel):
@@ -45,12 +43,12 @@ class PYJobOutput(BaseModel):
     job_id: str
     processor_name: str
     state: JobState = JobState.unset
-    path_to_mets: Optional[str]
-    workspace_id: Optional[str]
+    path_to_mets: Optional[str] = None
+    workspace_id: Optional[str] = None
     input_file_grps: List[str]
-    output_file_grps: Optional[List[str]]
+    output_file_grps: Optional[List[str]] = None
     page_id: Optional[str] = None
-    log_file_path: Optional[str]
+    log_file_path: Optional[str] = None
 
 
 class DBProcessorJob(Document):
@@ -58,22 +56,22 @@ class DBProcessorJob(Document):
     """
     job_id: str
     processor_name: str
-    path_to_mets: Optional[str]
-    workspace_id: Optional[str]
-    description: Optional[str]
+    path_to_mets: Optional[str] = None
+    workspace_id: Optional[str] = None
+    description: Optional[str] = None
     state: JobState = JobState.unset
     input_file_grps: List[str]
-    output_file_grps: Optional[List[str]]
-    page_id: Optional[str]
-    parameters: Optional[dict]
-    depends_on: Optional[List[str]]
-    result_queue_name: Optional[str]
-    callback_url: Optional[str]
-    internal_callback_url: Optional[str]
-    start_time: Optional[datetime]
-    end_time: Optional[datetime]
-    exec_time: Optional[str]
-    log_file_path: Optional[str]
+    output_file_grps: Optional[List[str]] = None
+    page_id: Optional[str] = None
+    parameters: Optional[dict] = None
+    depends_on: Optional[List[str]] = None
+    result_queue_name: Optional[str] = None
+    callback_url: Optional[str] = None
+    internal_callback_url: Optional[str] = None
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    exec_time: Optional[str] = None
+    log_file_path: Optional[str] = None
 
     class Settings:
         use_enum_values = True
@@ -102,9 +100,9 @@ class PYWorkflowJobOutput(BaseModel):
     page_id: str
     page_wise: bool = False
     job_id: str
-    path_to_mets: Optional[str]
-    workspace_id: Optional[str]
-    description: Optional[str]
+    path_to_mets: Optional[str] = None
+    workspace_id: Optional[str] = None
+    description: Optional[str] = None
 
 
 class DBWorkflowJob(Document):
@@ -117,10 +115,10 @@ class DBWorkflowJob(Document):
     # key: page_id
     # value: List of and processing job ids sorted in dependency order
     processing_job_ids: Dict
-    path_to_mets: Optional[str]
-    workspace_id: Optional[str]
-    description: Optional[str]
-    workflow_callback_url: Optional[str]
+    path_to_mets: Optional[str] = None
+    workspace_id: Optional[str] = None
+    description: Optional[str] = None
+    workflow_callback_url: Optional[str] = None
 
     class Settings:
         use_enum_values = True