queuemo 1.27.6__py3-none-any.whl

queuemo/__init__.py

@@ -0,0 +1,24 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+import queuemo.__patch__
+from queuemo.runtime.api import RecipeIO
+from queuemo.runtime.scheduler import isrequired
+from queuemo.runtime.scheduler import required
+
+__version__ = importlib.metadata.version('queuemo')
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+PROCESS = 'queuemo'
+
+INTERACTIVE = False  # if True, pause on every scheduler step

queuemo/__main__.py

@@ -0,0 +1,14 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Evaluate cmd-args and execute commands"""
+
+import queuemo.cli.main
+
+if __name__ == "__main__":
+    queuemo.cli.main.main()

queuemo/__patch__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

queuemo/cli/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

queuemo/cli/main.py

@@ -0,0 +1,185 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import collections
+
+import configos
+import qvlink
+import serializeraw
+import utilo
+import utilo.cli
+
+import queuemo
+import queuemo.cli.utils
+import queuemo.runnable
+
+RUNTIME = 'runtime'
+WAITING = 'waiting'
+JOBS = 'jobs'
+VERBOSE = 'verbose'
+INTERACTIVE = 'interactive'
+
+Config = collections.namedtuple(
+    'Config',
+    ('interactive failfast showdebug clean runtime waiting '
+     'jobs verbose http skip_unsupported webconfig'),
+)
+
+
+@utilo.saveme
+def main():
+    config = evaluate_cli()
+    if config == utilo.INVALID_COMMAND:
+        return utilo.INVALID_COMMAND
+    # run showdebug
+    if config.showdebug:
+        return showdebug()
+    configure(config)
+    closed = worker_startup(config)
+    return closed
+
+
+def configure(config: Config):
+    queuemo.INTERACTIVE = config.interactive
+    queuemo.cli.utils.logger_start()
+    if config.interactive:
+        utilo.log('use interactive mode')
+    if not utilo.testing():
+        # ensure that all test directories are created
+        configos.makedirs()
+    # check startup
+    configos.check_startup()
+    if config.clean:
+        # remove outdated maybe failed jobs
+        qvlink.remove_outdated()
+    # run http interface if required
+    if config.http:
+        http_startup()
+
+
+def worker_startup(config: Config):
+    # start scheduler
+    import queuemo.worker.startup  # pylint:disable=W0621,C0415
+    closed = queuemo.worker.startup.run(
+        failfast=config.failfast,
+        jobs=config.jobs,
+        runtime=config.runtime,
+        skip_unsupported=config.skip_unsupported,
+        verbose=config.verbose,
+        waiting=config.waiting,
+        webconfig=config.webconfig,
+    )
+    return closed
+
+
+def http_startup():
+    # lower queuemo start time, flask requires a second of init time
+    import queuemo.runtime.control  # pylint:disable=W0621,C0415
+    queuemo.runtime.control.start()
+
+
+CMDS = [
+    utilo.Parameter(
+        longcut=RUNTIME,
+        message='runtime in ticks',
+        args={
+            'default': 100,
+        },
+    ),
+    utilo.Parameter(
+        longcut=WAITING,
+        message='time to wait for the next job in miliseconds',
+        args={
+            'default': 1000,
+        },
+    ),
+    utilo.Parameter(
+        longcut=JOBS,
+        message='maximal number of executed jobs',
+        args={
+            'default': 1000,
+        },
+    ),
+    utilo.Flag(
+        longcut=INTERACTIVE,
+        message='pause on every recipe step and ask to continue',
+    ),
+    utilo.Flag(
+        longcut='failfast',
+        message='fail on first error',
+    ),
+    utilo.Flag(
+        longcut='http',
+        message='start http interface',
+    ),
+    utilo.Flag(
+        longcut='showdebug',
+        message='show list of installed runnable and return',
+    ),
+    utilo.Flag(
+        longcut='clean',
+        message='clean outdated jobs',
+    ),
+    utilo.Flag(
+        longcut='skip_unsupported',
+        message='do not analyze not supported documents',
+    ),
+    utilo.Parameter(
+        longcut='webconfig',
+        message='select msgids of published findings',
+    ),
+]
+
+
+def evaluate_cli() -> Config:
+    parser = utilo.cli.create_parser(
+        todo=CMDS,
+        config=utilo.ParserConfiguration(
+            waitingflag=False,
+            cacheflag=False,
+            pages=False,
+        ),
+        prog=queuemo.PROCESS,
+        version=queuemo.__version__,
+    )
+    args = utilo.parse(parser)
+    try:
+        webconfig = args.get('webconfig', None)
+        if webconfig:
+            if not utilo.exists(webconfig):
+                utilo.error(f'webconfig does not exists: {webconfig}')
+                return utilo.INVALID_COMMAND
+            utilo.debug(f'use webconfig: {webconfig}')
+            webconfig = serializeraw.load_webconfig(webconfig)
+        config = Config(
+            args[INTERACTIVE],
+            args.get('failfast', False),
+            args.get('showdebug', False),
+            args.get('clean', False),
+            int(args[RUNTIME]),
+            int(args[WAITING]),
+            int(args[JOBS]),
+            int(args[VERBOSE]) if args[VERBOSE] else 0,
+            args.get('http', False),
+            args.get('skip_unsupported', False),
+            webconfig=webconfig,
+        )
+    except ValueError:
+        msg = (f'invalid start parameter: `{args[RUNTIME]}`, `{args[WAITING]}`,'
+               f' `{args[VERBOSE]}`, `{args[JOBS]}`')
+        utilo.error(args)
+        utilo.error(msg)
+        return utilo.INVALID_COMMAND
+    utilo.level_setup(config.verbose)
+    return config
+
+
+def showdebug():
+    queuemo.runnable.check()
+    return utilo.SUCCESS

queuemo/cli/single.py

@@ -0,0 +1,107 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Run queuemo for selected pdf file."""
+
+import os
+
+import configos
+import qvlink
+import utilo
+import utilo.cli
+import utilo.logger
+
+import queuemo
+import queuemo.cli.utils
+import queuemo.runtime.api
+import queuemo.runtime.execute
+import queuemo.worker.startup
+
+COMMANDS = [
+    utilo.Flag(longcut='create'),
+]
+DESCRIPTION = """\
+Run queuemo recipe for given -i input and write to -o folder.
+"""
+
+
+@utilo.saveme
+def main() -> int:
+    inpath, outpath, verbose, create = eval_cli()
+    utilo.logger.level_setup(utilo.Level(verbose))
+    queuemo.cli.utils.logger_start()
+    if os.path.isfile(inpath):
+        if create:
+            inpath = create_todo(inpath, outpath)
+            if inpath == utilo.FAILURE:
+                return utilo.FAILURE
+        else:
+            utilo.error(f'require directory: {inpath} or use --create flag')
+            return utilo.INVALID_COMMAND
+    outpath = inpath
+    utilo.log(f'write result to: {outpath}')
+    # patch todo outpath path. TODO: INVESTIGATE HERE?
+    todopath = os.path.normpath(os.path.join(inpath, os.path.pardir))
+    configos.todo = lambda _=False: todopath
+    configos.ready = lambda _=False: todopath
+    # analyze single document
+    tmpdir = utilo.tmpdir(configos.tmp())
+    jobid = queuemo.worker.startup.jobid(inpath)
+    queuemo.runtime.execute.run_document(
+        workspace=tmpdir,
+        documentid=jobid,
+        outpath=outpath,
+        config=queuemo.runtime.api.RecipeConfig(verbose=verbose),
+    )
+    return utilo.SUCCESS
+
+
+def eval_cli() -> tuple:
+    parser = utilo.cli.create_parser(
+        description=DESCRIPTION,
+        todo=COMMANDS,
+        config=utilo.ParserConfiguration(
+            inputparameter=True,
+            outputparameter=True,
+        ),
+        prog='queuemo_single',
+        version=queuemo.__version__,
+    )
+    args = utilo.parse(parser)
+    inpath, outpath, verbose = utilo.sources(  # pylint:disable=W0632
+        args,
+        singleinput=True,
+        use_cwd=False,
+        verbose=True,
+    )
+    inpath = inpath[0]
+    create = args.get('create', False)
+    return inpath, outpath, verbose, create
+
+
+def create_todo(inpath, outpath):
+    filename = str(os.path.split(inpath)[1])
+    if outpath:
+        outpath, outname = os.path.split(outpath)
+    else:
+        outpath = os.getcwd()
+        # remove file name
+        outname = filename.replace('.pdf', '')
+    try:
+        outpath = qvlink.create_todo(
+            file=inpath,
+            filename=filename,
+            todoname=outname,
+            todopath=outpath,
+            exist_ok=True,
+        )
+    except OSError:
+        utilo.error(f'{os.path.join(outpath, outname)} already exists')
+        return utilo.FAILURE
+    utilo.log(f'create job description in {outpath}')
+    return outpath

queuemo/cli/state.py

@@ -0,0 +1,104 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import argparse
+import sys
+
+import configos
+import qvlink
+import utilo
+
+
+@utilo.saveme
+def main() -> int:
+    action = eval_cli()
+    if action.kill:
+        kill(action.kill)
+        return utilo.SUCCESS
+    utilo.log('verify state ...')
+    utilo.log(f'    todo: {configos.todo()}')
+    print_running()
+    print_waiting()
+    return utilo.SUCCESS
+
+
+def print_running():
+    print_documents(qvlink.State.RUNNING)
+
+
+def print_waiting():
+    print_documents(qvlink.State.WAITING)
+
+
+def print_documents(state):
+    # TODO: PASS READY AND TODO TO AVOID REQUIRED COMMON PARENT
+    common = utilo.path_parent(configos.ready())
+    documents = qvlink.load_documents(
+        common=common,
+        owner=None,
+        state=state,
+    )
+    if not documents:
+        utilo.log(f'no {state.name} documents')
+        return
+    utilo.log(f'\n{state.name} documents({len(documents)}):')
+    for document in documents:
+        print_document(document)
+
+
+def print_document(document):
+    pages = str(document.get('pages')).zfill(3)
+    title = document.get('title')
+    date = document.get('date')
+    name = document.get('name')
+    utilo.log(f'   - {name} {date} {pages} {title}')
+
+
+def kill(documentid: str):
+    if documentid == 'all':
+        todos = collect_all()
+        if not todos:
+            utilo.error('nothing to kill')
+            sys.exit(utilo.INVALID_COMMAND)
+        for jobinfo in todos:
+            kill(jobinfo.name)
+        return
+    if not qvlink.document(documentid=documentid):
+        utilo.exitx(f'invalid documentid: {documentid}', utilo.INVALID_COMMAND)
+    if qvlink.deleted(documentid):
+        utilo.exitx(f'already deleted: {documentid}', utilo.INVALID_COMMAND)
+        return
+    if qvlink.delete(documentid):
+        utilo.log(f'killed: {documentid}')
+        return
+    utilo.error(f'could not kill: {documentid}')
+    sys.exit(utilo.INVALID_COMMAND)
+
+
+def collect_all() -> list:
+    todos = qvlink.collect_jobs()[0]
+    result = []
+    for todo in todos:
+        process = qvlink.current(todo.documentid)
+        current: qvlink.State = qvlink.State.fromstate(process)
+        if current not in (qvlink.State.WAITING, qvlink.State.RUNNING):
+            continue
+        result.append(todo)
+    return result
+
+
+def eval_cli():
+    parser = argparse.ArgumentParser(prog='queuemo_state',)
+    parser.add_argument(
+        '--kill',
+        nargs='?',
+        help='stop single job, use all to stop all pending jobs',
+    )
+    result = parser.parse_args()
+    return result

queuemo/cli/utils.py

@@ -0,0 +1,26 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+
+import configos
+import utilo.logger
+
+
+def logger_start():
+    tmp = configos.tmp()
+    os.makedirs(tmp, exist_ok=True)
+    # create log folder
+    queuemo_tmp = os.path.join(tmp, 'queuemo/log/')
+    os.makedirs(queuemo_tmp, exist_ok=True)
+    # determine current file name
+    logfile = f'{utilo.filetime()}_{utilo.tmpname(10)}.log'
+    outpath = os.path.join(queuemo_tmp, logfile)
+    utilo.debug(f'write global log to {outpath}')
+    utilo.outfile_setup(outpath)

queuemo/recipes/__init__.py

@@ -0,0 +1,59 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Composition of recipes
+======================
+
+.. note :: outdated approach, TODO: UPDATE
+
+DataMining: PDF ->
+    RawMaker        -> text.yaml ->
+    GroupMe         -> chapter.yaml ->
+
+Analysis/Reporting:
+
+    Statistica      -> statistic.yaml ->
+
+Decider:
+
+    MathStyle       -> style.yaml                   WordStyle
+
+Feedback:
+
+    Feedback        -> feedback.yaml                Feedback
+
+Presentation:
+
+    HTML VIEW
+"""
+
+import utilo
+
+import queuemo
+import queuemo.runtime.scheduler
+
+RECIPES = utilo.join(queuemo.ROOT, 'queuemo', 'recipes', exist=True)
+
+
+def create(name: str):
+    path = utilo.join(RECIPES, name, exist=True)
+    loaded = queuemo.runtime.scheduler.load(path)
+    return loaded
+
+
+VALIDATION = create('validation')
+FASTVIEW = create('fastview')
+RESULT = create('result')
+PUBLISH = create('publish')
+
+ORDER = [
+    VALIDATION,
+    FASTVIEW,
+    RESULT,
+    PUBLISH,
+]

queuemo/recipes/fastview/__init__.py

@@ -0,0 +1,26 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""The fast view aims to produce very fast results to avoid that a user
+leaves the page cause of long waiting time.
+
+Requirements:
+
+* The first result must be reached in lower than **five seconds**.
+* A complete fast view must be produced in lower than **thirty seconds**.
+
+What can be archived very fast?
+
+* Title Page Analysis
+* Table of Content Analysis
+* (?)Border Violation
+* (?)Font Analysis
+
+How does the fast view work?
+----------------------------
+"""

queuemo/recipes/fastview/after.py

@@ -0,0 +1,21 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import qvlink
+
+import queuemo
+import queuemo.runtime.api
+
+
+@queuemo.required
+def run(flow: queuemo.runtime.api.RecipeIO) -> list:
+    """Create a file named close to signal update hander that process is
+    completed."""
+    # TODO: ADD STATUS INFORMATION LATER
+    qvlink.finish_fastview(flow.document)

queuemo/recipes/fastview/before.py

@@ -0,0 +1,76 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import qvlink
+import utilo
+
+import queuemo
+import queuemo.recipes.utils
+import queuemo.runnable
+import queuemo.runtime.api
+
+TIMEOUT_FASTVIEW_TABLERO = configos.HV_INT_PLUS(default=60)
+
+FIRST_PAGES = configos.HV_STR(default='0:10')
+
+
+@queuemo.required
+def before(flow: queuemo.runtime.api.RecipeIO):
+    qvlink.start_analysis(flow.document)
+
+
+@queuemo.required
+def run(flow: queuemo.runtime.api.RecipeIO) -> list:
+    unpack_pdf(flow)
+
+    queuemo.recipes.utils.determine_sections(flow, pages='0:20')
+
+    # TODO: DO WE REQUIRE TABLES FOR FIRST 10 PAGES?
+    # queuemo.runnable.groupme(
+    #     inpath=flow.tmppath,
+    #     outpath=flow.tmppath,
+    #     pagenumbers=True,
+    #     footer=True,
+    #     content=True,
+    # )
+    # queuemo.runnable.tablero(
+    #     inpath=flow.tmppath,
+    #     outpath=flow.tmppath,
+    #     table=flow.sourcepath,
+    #     timeout=TIMEOUT_FASTVIEW_TABLERO.value,
+    #     pages=FIRST_PAGES.value,
+    # )
+
+
+def unpack_pdf(flow: queuemo.runtime.api.RecipeIO):
+    pdf = flow.sourcepath
+    with utilo.GeorgFork(process=False) as runtime:
+        runtime.fork(
+            queuemo.runnable.rawmaker,
+            inpath=pdf,
+            outpath=flow.tmppath,
+            all=True,
+            images=False,
+            figures=False,
+            pages=FIRST_PAGES,
+        )
+        runtime.fork(
+            queuemo.runnable.rawmaker,
+            inpath=pdf,
+            outpath=flow.tmppath,
+            prefix='oneline',
+            boxes_flow=1.0,
+            char_margin=100,
+            line_margin=0.1,
+            text=True,
+            fonts=True,
+            pages='0:5',
+        )
+    assert runtime.result == utilo.SUCCESS, 'could not extract pdf'