ocrd 3.0.0b5__py3-none-any.whl → 3.0.0b7__py3-none-any.whl

ocrd/processor/helpers.py

@@ -2,6 +2,7 @@
 Helper methods for running and documenting processors
 """
 from time import perf_counter, process_time
+from os import times
 from functools import lru_cache
 import json
 import inspect
@@ -94,6 +95,7 @@ def run_processor(
     log.debug("Processor instance %s (%s doing %s)", processor, name, otherrole)
     t0_wall = perf_counter()
     t0_cpu = process_time()
+    t0_os = times()
     if any(x in config.OCRD_PROFILE for x in ['RSS', 'PSS']):
         backend = 'psutil_pss' if 'PSS' in config.OCRD_PROFILE else 'psutil'
         from memory_profiler import memory_usage # pylint: disable=import-outside-toplevel
@@ -123,7 +125,13 @@ def run_processor(
 
     t1_wall = perf_counter() - t0_wall
     t1_cpu = process_time() - t0_cpu
-    logProfile.info("Executing processor '%s' took %fs (wall) %fs (CPU)( [--input-file-grp='%s' --output-file-grp='%s' --parameter='%s' --page-id='%s']" % (
+    t1_os = times()
+    # add CPU time from child processes (page worker etc)
+    t1_cpu += t1_os.children_user - t0_os.children_user
+    t1_cpu += t1_os.children_system - t0_os.children_system
+    logProfile.info(
+        "Executing processor '%s' took %fs (wall) %fs (CPU)( "
+        "[--input-file-grp='%s' --output-file-grp='%s' --parameter='%s' --page-id='%s']",
         ocrd_tool['executable'],
         t1_wall,
         t1_cpu,
@@ -131,7 +139,7 @@ def run_processor(
         processor.output_file_grp or '',
         json.dumps(processor.parameter) or '',
         processor.page_id or ''
-    ))
+    )
     workspace.mets.add_agent(
         name=name,
         _type='OTHER',

{ocrd-3.0.0b5.dist-info → ocrd-3.0.0b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b5
+Version: 3.0.0b7
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -24,6 +24,7 @@ Requires-Dist: frozendict>=2.3.4
 Requires-Dist: gdown
 Requires-Dist: httpx>=0.22.0
 Requires-Dist: jsonschema>=4
+Requires-Dist: loky
 Requires-Dist: lxml
 Requires-Dist: memory-profiler>=0.58.0
 Requires-Dist: numpy

{ocrd-3.0.0b5.dist-info → ocrd-3.0.0b7.dist-info}/RECORD

@@ -18,16 +18,17 @@ ocrd/cli/ocrd_tool.py,sha256=qaJgt-LNH0tXkaupMNrEKXasxcgsabHdfLdYESEsomk,7035
 ocrd/cli/process.py,sha256=8KD0i7LT01H9u5CC1vktYMEVpS67da_rp_09_EOECmw,1233
 ocrd/cli/resmgr.py,sha256=bTE-MpF7RRCHhgAbknqZUFHgHScIK6FR3S4h4DEAets,10080
 ocrd/cli/validate.py,sha256=nvageDaHCETcE71X5lu7i_4JKpgo9MrvJKinVPLYUTI,5727
-ocrd/cli/workspace.py,sha256=InIQ5rfQWPn4Qsd1s_xA6AC6ndZLCsuyhoAEiqP8bK4,39479
+ocrd/cli/workspace.py,sha256=KTbSzIUrba5WoYETvM9ElRZVsDUHCGVvjoFgBGZS2nU,40468
 ocrd/cli/zip.py,sha256=MMJLw3OXWiJVfVtrdJcBkbB8vA1IzSautluazZRuCQ0,5910
-ocrd/decorators/__init__.py,sha256=IJlA1XcdVBO6Hxm9rNDya7QYcqeWcaXXuLtGjfjcen8,7596
+ocrd/decorators/__init__.py,sha256=NWqZUTSBve6yIBgTN3miGmz4w1IFefLhG92qXx5ADtw,7620
 ocrd/decorators/loglevel_option.py,sha256=tgipROEu3t4hkwWvFssd80k2SbTBwBIC4WNE6Gc-XAg,798
 ocrd/decorators/mets_find_options.py,sha256=d4oATKMP6bFQHNqOK6nLqgUiWF2FYdkPvzkTVRMYpKo,635
 ocrd/decorators/ocrd_cli_options.py,sha256=hr2EugwAY_-GJ7F7g77Od9o9eAqhfLBHSpfmCql2OCU,2665
 ocrd/decorators/parameter_option.py,sha256=n8hYw7XVTd3i3tvpK8F1Jx_CqRp6EGF9qJVH95yj92Q,1076
 ocrd/processor/__init__.py,sha256=39ymNwYRdc-b_OJzzKmWCvo2ga3KdsGSYDHE1Hzkn_w,274
-ocrd/processor/base.py,sha256=_TvaxKf_oMaIfuUcaFWas8YimhZ-l1d3RWGELlBJfy8,50307
-ocrd/processor/helpers.py,sha256=vPYUri6ucuhdTNrideywriJ0fCa8UE2QyBXOmS-7RcQ,10232
+ocrd/processor/base.py,sha256=8kFrYLd1cSHBaBolHjXdIVHwmV8muKgBCBrAYw7kWTQ,59154
+ocrd/processor/concurrent.py,sha256=IMMBFGDGqfpgm7Rp6J_dnXVckyBKntwARwcKDnxadHQ,38217
+ocrd/processor/helpers.py,sha256=8ngrqAJ01BSoSJNsIoK_YfA8QdryM5y0MqaqA9f7ELM,10483
 ocrd/processor/ocrd_page_result.py,sha256=eDkpyVHcpaBzTHXiGrcNk9PP9Xr-XZru2w_uoX_ZeNA,510
 ocrd/processor/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ocrd/processor/builtin/dummy_processor.py,sha256=iWiw_jJXOqwr7-hFjdkmTCCo1xGr6MLGOshx81PTu-8,3548
@@ -40,7 +41,7 @@ ocrd_models/mets-empty.xml,sha256=dFixfbxSXrgjZx9BfdIKWHX-khNmp7dNYaFe2qQSwCY,12
 ocrd_models/ocrd_agent.py,sha256=E9OtDhz9UfKb6ou2qvsuCL9NlO1V6zMb0s8nVq8dVos,5609
 ocrd_models/ocrd_exif.py,sha256=5BRLjvB6jg36V68i8jvVnT2SSNnpqLbhLsaMuP51Scw,4583
 ocrd_models/ocrd_file.py,sha256=7lyHezuNnl2FEYV1lV35-QTCrgYAL-3wO2ulFUNq2Ak,9717
-ocrd_models/ocrd_mets.py,sha256=h3y_WI5fVLsbBoUIRNH2ebjuO1-_P6T3BMIULX-ZOIs,42514
+ocrd_models/ocrd_mets.py,sha256=cRBEnvRNmRUhcrKouqygTQqYA-XdgyA-FNLwt86V0vg,42878
 ocrd_models/ocrd_page.py,sha256=sVIvvMeBT8eZnOfW0DTjQUNyu62-llz0v_Ga5Xo-tUM,5393
 ocrd_models/ocrd_page_generateds.py,sha256=wfx3vESMAi08rl6-16zNVJe4E3B6APIvL6RCr1roAzg,774092
 ocrd_models/ocrd_xml_base.py,sha256=OW57mXLlwm1nH8CNefvXmwLRws9KL9zSrb-3vH--mX8,1641
@@ -92,8 +93,8 @@ ocrd_utils/constants.py,sha256=ImbG1d8t2MW3uuFi-mN6aY90Zn74liAKZBKlfuKN86w,3278
 ocrd_utils/deprecate.py,sha256=4i50sZsA3Eevqn5D-SL5yGf9KEZfGCV4A5Anzn1GRMs,1026
 ocrd_utils/image.py,sha256=zNNX1cnRy6yvrxx8mnYQiqWraAh5-i4a1AOfCCg4SmI,24781
 ocrd_utils/introspect.py,sha256=gfBlmeEFuRmRUSgdSK0jOxRpYqDRXl2IAE6gv2MZ6as,1977
-ocrd_utils/logging.py,sha256=5_-5T5OWSYicNk8SQyjVqdRj2bVl-gDK1Th-C7oW_HE,8248
-ocrd_utils/ocrd_logging.conf,sha256=kl9x9JS1d8h8F0QZabvrjZtW1iApIaChvkImYafKO5g,3623
+ocrd_utils/logging.py,sha256=sHNfih9kBfvKsHdLqMK_ew9Pld1GsRyYlrZHIySujnw,7313
+ocrd_utils/ocrd_logging.conf,sha256=JlWmA_5vg6HnjPGjTC4mA5vFHqmnEinwllSTiOw5CCo,3473
 ocrd_utils/os.py,sha256=acRRdDBI8L6BK0Mf773yKEzwdpZSFRBJEKB2crL4EjU,9865
 ocrd_utils/str.py,sha256=cRgqYILDGOAqWr0qrCrV52I3y4wvpwDVtnBGEUjXNS4,10116
 ocrd_validators/__init__.py,sha256=ZFc-UqRVBk9o1YesZFmr9lOepttNJ_NKx1Zdb7g_YsU,972
@@ -118,9 +119,9 @@ ocrd_validators/xlink.xsd,sha256=8fW7YAMWXN2PbB_MMvj9H5ZeFoEBDzuYBtlGC8_6ijw,318
 ocrd_validators/xsd_mets_validator.py,sha256=4GWfLyqkmca0x7osDuXuExYuM0HWVrKoqn0S35sFhHU,467
 ocrd_validators/xsd_page_validator.py,sha256=BNz_9u-Ek4UCeyZu3KxSQoolfW9lvuaSR9nIu1XXxeE,467
 ocrd_validators/xsd_validator.py,sha256=6HrVAf6SzCvfUIuQdIzz9bOq4V-zhyii9yrUPoK2Uvo,2094
-ocrd-3.0.0b5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ocrd-3.0.0b5.dist-info/METADATA,sha256=eIpkAoobj7QocP9VXYASLqE82wN35JlvYuYjSBLk30o,10397
-ocrd-3.0.0b5.dist-info/WHEEL,sha256=5Mi1sN9lKoFv_gxcPtisEVrJZihrm_beibeg5R6xb4I,91
-ocrd-3.0.0b5.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
-ocrd-3.0.0b5.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
-ocrd-3.0.0b5.dist-info/RECORD,,
+ocrd-3.0.0b7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ocrd-3.0.0b7.dist-info/METADATA,sha256=lc6oZVb9SPl97qvspPigal4yb_1DBBRZwE3GQUTAK_o,10417
+ocrd-3.0.0b7.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+ocrd-3.0.0b7.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
+ocrd-3.0.0b7.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
+ocrd-3.0.0b7.dist-info/RECORD,,

{ocrd-3.0.0b5.dist-info → ocrd-3.0.0b7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.0.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ocrd_models/ocrd_mets.py

@@ -599,7 +599,16 @@ class OcrdMets(OcrdXmlDocument):
         If return_divs is set, returns div memory objects instead of strings of ids
         """
         if for_fileIds is None and for_pageIds is None:
+            if return_divs:
+                if self._cache_flag:
+                    return list(self._page_cache[METS_PAGE_DIV_ATTRIBUTE.ID].values())
+
+                return [x for x in self._tree.getroot().xpath(
+                    'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]',
+                    namespaces=NS)]
+
             return self.physical_pages
+
         # log = getLogger('ocrd.models.ocrd_mets.get_physical_pages')
         if for_pageIds is not None:
             ret = []

ocrd_utils/logging.py

@@ -46,13 +46,8 @@ __all__ = [
     'setOverrideLogLevel',
 ]
 
-# These are the loggers we add handlers to
-ROOT_OCRD_LOGGERS = [
-    'ocrd',
-    'ocrd_network'
-]
-
 LOGGING_DEFAULTS = {
+    '': logging.WARNING,
     'ocrd': logging.INFO,
     'ocrd_network': logging.INFO,
     # 'ocrd.resolver': logging.INFO,
@@ -113,18 +108,15 @@ def setOverrideLogLevel(lvl, silent=not config.OCRD_LOGGING_DEBUG):
         lvl (string): Log level name.
         silent (boolean): Whether to log the override call
     """
-    if not _initialized_flag:
-        initLogging(silent=silent)
-    ocrd_logger = logging.getLogger('ocrd')
-
-    if lvl is None:
-        if not silent:
-            print('[LOGGING] Reset log level override', file=sys.stderr)
-        ocrd_logger.setLevel(logging.NOTSET)
-    else:
-        if not silent:
-            print(f'[LOGGING] Overriding ocrd log level to {lvl}', file=sys.stderr)
-        ocrd_logger.setLevel(lvl)
+    if lvl is not None:
+        lvl = getLevelName(lvl)
+        if not _initialized_flag:
+            initLogging(silent=silent)
+        # affect all configured loggers
+        for logger_name in logging.root.manager.loggerDict:
+            if not silent:
+                print(f'[LOGGING] Overriding {logger_name} log level to {lvl}', file=sys.stderr)
+            logging.getLogger(logger_name).setLevel(lvl)
 
 def get_logging_config_files():
     """
@@ -158,20 +150,11 @@ def initLogging(builtin_only=False, force_reinit=False, silent=not config.OCRD_L
         - silent (bool): Whether to log logging behavior by printing to stderr
     """
     global _initialized_flag
-    if _initialized_flag and not force_reinit:
-        return
-    # disableLogging()
-
-    # https://docs.python.org/3/library/logging.html#logging.disable
-    # If logging.disable(logging.NOTSET) is called, it effectively removes this
-    # overriding level, so that logging output again depends on the effective
-    # levels of individual loggers.
-    logging.disable(logging.NOTSET)
-
-    # remove all handlers for the ocrd root loggers
-    for logger_name in ROOT_OCRD_LOGGERS:
-        for handler in logging.getLogger(logger_name).handlers[:]:
-            logging.getLogger(logger_name).removeHandler(handler)
+    if _initialized_flag:
+        if force_reinit:
+            disableLogging(silent=silent)
+        else:
+            return
 
     config_file = None
     if not builtin_only:
@@ -190,8 +173,8 @@ def initLogging(builtin_only=False, force_reinit=False, silent=not config.OCRD_L
         ocrd_handler = logging.StreamHandler(stream=sys.stderr)
         ocrd_handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT, datefmt=LOG_TIMEFMT))
         ocrd_handler.setLevel(logging.DEBUG)
-        for logger_name in ROOT_OCRD_LOGGERS:
-            logging.getLogger(logger_name).addHandler(ocrd_handler)
+        root_logger = logging.getLogger('')
+        root_logger.addHandler(ocrd_handler)
         for logger_name, logger_level in LOGGING_DEFAULTS.items():
             logging.getLogger(logger_name).setLevel(logger_level)
     _initialized_flag = True
@@ -207,24 +190,16 @@ def disableLogging(silent=not config.OCRD_LOGGING_DEBUG):
     if _initialized_flag and not silent:
         print("[LOGGING] Disabling logging", file=sys.stderr)
     _initialized_flag = False
-    # logging.basicConfig(level=logging.CRITICAL)
-    # logging.disable(logging.ERROR)
-    # remove all handlers for the ocrd logger
-    for logger_name in ROOT_OCRD_LOGGERS + ['']:
-        for handler in logging.getLogger(logger_name).handlers[:]:
-            logging.getLogger(logger_name).removeHandler(handler)
-    for logger_name in LOGGING_DEFAULTS:
-        logging.getLogger(logger_name).setLevel(logging.NOTSET)
+    # remove all handlers we might have added (via initLogging on builtin or file config)
+    for logger_name in logging.root.manager.loggerDict:
+        if not silent:
+            print(f'[LOGGING] Resetting {logger_name} log level and handlers')
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(logging.NOTSET)
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
     # Python default log level is WARNING
     logging.root.setLevel(logging.WARNING)
 
-# Initializing stream handlers at module level
-# would cause message output in all runtime contexts,
-# including those which are already run for std output
-# (--dump-json, --version, ocrd-tool, bashlib etc).
-# So this needs to be an opt-in from the CLIs/decorators:
-#initLogging()
-# Also, we even have to block log output for libraries
-# (like matplotlib/tensorflow) which set up logging
-# themselves already:
-disableLogging()

ocrd_utils/ocrd_logging.conf

@@ -34,7 +34,7 @@ keys=defaultFormatter,detailedFormatter
 # default logger "root" using consoleHandler
 #
 [logger_root]
-level=INFO
+level=WARNING
 handlers=consoleHandler,fileHandler
 
 
@@ -56,22 +56,22 @@ handlers=consoleHandler,fileHandler
 # ocrd loggers
 [logger_ocrd]
 level=INFO
-handlers=consoleHandler,fileHandler
+handlers=
 qualname=ocrd
-propagate=0
 
 [logger_ocrd_network]
 level=INFO
-handlers=consoleHandler,processingServerHandler
+#handlers=consoleHandler,processingServerHandler
+handlers=processingServerHandler
 qualname=ocrd_network
-propagate=0
+#propagate=0
 
 #
 # logger tensorflow
 #
 [logger_ocrd_tensorflow]
 level=ERROR
-handlers=consoleHandler
+handlers=
 qualname=tensorflow
 
 #
@@ -79,7 +79,7 @@ qualname=tensorflow
 #
 [logger_ocrd_shapely_geos]
 level=ERROR
-handlers=consoleHandler
+handlers=
 qualname=shapely.geos
 
 
@@ -88,7 +88,7 @@ qualname=shapely.geos
 #
 [logger_ocrd_PIL]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=PIL
 
 #
@@ -96,34 +96,32 @@ qualname=PIL
 #
 [logger_paramiko]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=paramiko
-propagate=0
 
 [logger_paramiko_transport]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=paramiko.transport
-propagate=0
 
 #
 # uvicorn loggers
 #
 [logger_uvicorn]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=uvicorn
 [logger_uvicorn_access]
 level=WARN
-handlers=consoleHandler
+handlers=
 qualname=uvicorn.access
 [logger_uvicorn_error]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=uvicorn.error
 [logger_multipart]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=multipart