streamline 1.2.0__py3-none-any.whl

streamline/__init__.py

@@ -0,0 +1 @@
+

streamline/cli.py

@@ -0,0 +1,326 @@
+import argparse
+import asyncio
+import logging
+import yaml
+import sys
+import os
+import re
+
+from . import utils
+from . import executors
+from . import generators
+from . import consumers
+from . import streamers
+from .core import pipe
+
+logger = logging.getLogger(__file__)
+
+SHORTHAND_PATTERN = r'^(?P<target_attr>[-\[\]_\w]+=)?(?P<streamer>[-\[\]_.\w]+)\((?P<input_extract>[-\[\]._*\w]+), ?(?P<output_extract>[-._*\w\[\]]+)\)$'
+
+
+class OptionsProcessor():
+    def __init__(self, args):
+        self.args = args
+
+    def parse(self, parser):
+        # Support '--' separator between commands
+        if self.args and self.args[0] == '--':
+            self.args = self.args[1:]
+
+        parsed_args, remaining = parser.parse_known_args(self.args)
+        self.args = remaining
+        return parsed_args
+
+    def has_remaining_args(self):
+        return len(self.args) > 0
+
+    def remaining_args(self):
+        return self.args.copy()
+
+
+def _parser_main():
+    cmd_parser = argparse.ArgumentParser(prog='streamline', add_help=False)
+    cmd_parser.add_argument(
+        '--generator',
+        help='Entry Generator Module',
+    )
+    cmd_parser.add_argument(
+        '--consumer',
+        help='Entry Consumer/Writer Module',
+    )
+    cmd_parser.add_argument(
+        '-s', '--streamers',
+        help='Additional streamers to apply (-s is optional)',
+        nargs='*',
+    )
+    cmd_parser.add_argument(
+        '-h', '--help',
+        action='store_true',
+        help='Print help',
+    )
+    cmd_parser.add_argument(
+        '-p', '--progress',
+        choices=['buffer', 'stream-output', 'streaming'],
+        help='Print progress to stdout. ("buffer": buffers input and output, "stream-output" buffers only input, "stream" for no buffering at all)',
+    )
+    cmd_parser.add_argument(
+        '-w', '--workers',
+        type=int,
+        help='Number of concurrent workers for any one async execution module to have',
+    )
+    cmd_parser.add_argument(
+        '-y', '--yaml',
+        help='Take options from a yaml or json config file',
+    )
+    return cmd_parser
+
+def _generator_parser(generator):
+    if not hasattr(generator, 'args'):
+        return None
+    
+    generator_parser = argparse.ArgumentParser(
+        prog=generator.__name__,
+        add_help=False,
+        usage='streamline [streamers...] --generator %(prog)s [generator options]',
+    )
+    generator.args(generator_parser)
+    return generator_parser
+
+def _consumer_parser(consumer):
+    if not hasattr(consumer, 'args'):
+        return None
+    
+    consumer_parser = argparse.ArgumentParser(
+        prog=consumer.__name__,
+        add_help=False,
+        usage='streamline [streamers...] --consumer %(prog)s [consumer options]',
+    )
+    consumer.args(consumer_parser)
+    return consumer_parser
+
+def _streamer_parser(Streamer):
+    streamer_parser = argparse.ArgumentParser(
+        prog=Streamer.__name__,
+        add_help=False,
+        usage='streamline -s %(prog)s -- [options]',
+    )
+    if hasattr(Streamer, 'args'):
+        Streamer.args(streamer_parser)
+    return streamer_parser
+
+
+def load_config(*sources, ignore_nulls=True):
+    config = {}
+    for source in sources:
+        if source is None:
+            continue
+        if not isinstance(source, dict):
+            source = source.__dict__
+        if ignore_nulls:
+            source = utils.strip_nulls(source)
+        config.update(source)
+    return config
+
+def wrap_streamer(streamer, input_path=None, output_path=None, target=None):
+    if not (input_path or output_path or target):
+        return [streamer]
+
+    if target and not (input_path or output_path):
+        combiner = streamers.Combiner(path=target, target=-2, source=-1)
+        return [streamer, combiner]
+
+    subpipe = []
+    # Compose the sub-command
+    subpipe.append(streamers.history_push)
+    if input_path != 'value':
+        input_extract = streamers.ExtractionStreamer(
+            selector=input_path,
+        )
+        subpipe.append(input_extract.stream)
+    subpipe.append(streamer)
+    if output_path != 'value':
+        output_extract = streamers.ExtractionStreamer(
+            selector=output_path
+        )
+        subpipe.append(output_extract.stream)
+    subpipe.append(streamers.history_pop)
+    if target:
+        combiner = streamers.Combiner(path=target, target=-2, source=-1)
+        subpipe.append(combiner.stream)
+    return subpipe
+
+def streamline_command(args):
+    if args and '-s' not in args:
+        args.insert(0, '-s'),
+    options_processor = OptionsProcessor(args)
+    main_parser = _parser_main()
+    main_args = options_processor.parse(main_parser)
+
+    yaml_config = {}
+    use_yaml = main_args.yaml is not None
+    if main_args.yaml:
+        with open(main_args.yaml, 'r') as f:
+            yaml_config = yaml.safe_load(f.read())
+
+    yaml_generator_config = yaml_config.get('generator', {})
+    yaml_consumer_config = yaml_config.get('consumer', {})
+    yaml_streamers = yaml_config.get('streamers', [])
+    command_config = load_config(
+        {
+            'generator': 'file',
+            'consumer': 'file',
+            'workers': streamers.AsyncExecutor.DEFAULT_WORKERS,
+            'streamers': [],
+        },
+        {
+            'generator': yaml_generator_config.get('name', None),
+            'consumer': yaml_consumer_config.get('name', None),
+        },
+        main_args,
+        ignore_nulls=True,
+    )
+    ae_args = {'workers': command_config.get('workers')}
+        
+    # Load Generator & Consumer
+    Generator = generators.load_generator(command_config['generator'])
+    if 'options' in yaml_generator_config:
+        generator_options = yaml_generator_config.get('options')
+    else:
+        generator_options = options_processor.parse(_generator_parser(Generator)).__dict__
+    generator = Generator(**generator_options)
+
+    Consumer = consumers.load_consumer(command_config['consumer'])
+    if 'options' in yaml_consumer_config:
+        consumer_options = yaml_consumer_config.get('options')
+    else:
+        consumer_options = options_processor.parse(_consumer_parser(Consumer)).__dict__
+    consumer = Consumer(**consumer_options)
+
+    if main_args.help:
+        print('=' * 15 + ' Streamline ' + '=' * 15)
+        print('\n')
+        main_parser.print_help()
+        streamer_list = command_config.get('streamers', [])
+        if streamer_list:
+            for streamer_name in streamer_list:
+                print('\n\n')
+                print('=' * 15 + ' Streamer::{} '.format(streamer_name) + '=' * 15)
+                print('\n')
+                load_streamer(streamer_name, print_help=True, ae_args=ae_args)
+        else:
+            print('\n')
+            print('=' * 15 + ' Streamers ' + '=' * 15)
+            for streamer_name, streamer in streamers.STREAMERS.items():
+                description = getattr(streamer, '_arg_description', 'An undocumented module')
+                example = getattr(streamer, '_arg_example', '')
+                invocation = 'streamline -s {} --'.format(streamer_name)
+                print('\n::{}::\n\tDescription: {}\n\tExample: {} {}'.format(
+                    streamer_name,
+                    description,
+                    invocation,
+                    example,
+                ))
+                
+        return
+
+    # Parse streamers
+    command_streamers = []
+    if use_yaml:
+        for streamer_conf in yaml_streamers:
+            streamer = load_streamer(
+                streamer_conf['name'],
+                options=streamer_conf.get('options', {}),
+                ae_args=ae_args,
+            )
+            sub_pipeline = wrap_streamer(
+                streamer,
+                input_path=streamer_conf.get('input'),
+                output_path=streamer_conf.get('output'),
+                target=streamer_conf.get('target'),
+            )
+            command_streamers.extend(sub_pipeline)
+    else:
+        for streamer_name in command_config.get('streamers',[]):
+            # Support shorthand syntax "attr=streamer(input.path, output.path)"
+            shorthand_match = re.match(SHORTHAND_PATTERN, streamer_name)
+            if shorthand_match:
+                shorthand_options = shorthand_match.groupdict()
+
+                # Load the main streamer
+                streamer_name = shorthand_options.get('streamer')
+                main_streamer = load_streamer(shorthand_options['streamer'], options_processor, ae_args=ae_args)
+
+                target_attr = shorthand_options.get('target_attr', '')
+                if target_attr:
+                    target_attr = target_attr[:-1]
+                    
+                command_streamers.extend(wrap_streamer(
+                    main_streamer,
+                    input_path=shorthand_options.get('input_extract', None),
+                    output_path=shorthand_options.get('output_extract', None),
+                    target=target_attr,
+                ))
+            else:
+                streamer = load_streamer(streamer_name, options_processor, ae_args=ae_args)
+                command_streamers.append(streamer)
+
+    # Ensure we don't have any extra arguments
+    if options_processor.has_remaining_args():
+        sys.stderr.write('Extra arguments found: {}\n'.format(' '.join(options_processor.remaining_args())))
+        sys.exit(2)
+
+    # Override streamers
+    progress_option = command_config.get('progress', None)
+    if command_config.get('progress', None):
+        buffer_start = progress_option in ('buffer', 'stream-output')
+        buffer_end = progress_option in ('buffer')
+        progress = streamers.ProgressStreamer(buffer_start=buffer_start, buffer_end=buffer_end)
+        command_streamers = [progress.streamer_start, *command_streamers, progress.streamer_end]
+
+    future = pipe(generator.stream(), command_streamers, consumer=consumer.stream)
+    asyncio.run(future)
+
+def load_streamer(path, options_processor=None, options=None, print_help=False, ae_args=None):
+    kwargs = {}
+    if options:
+        kwargs.update(options)
+    if path is None:
+        return None
+    elif '.' in path:
+        Streamer = utils.import_obj(path)
+    else:
+        Streamer = streamers.STREAMERS.get(path)
+    if Streamer is None:
+        raise ValueError('Invalid streamer: {}'.format(path))
+
+    if print_help:
+        _streamer_parser(Streamer).print_help()
+        return None
+
+    if options_processor:
+        streamer_parser = _streamer_parser(Streamer)
+        streamer_args = options_processor.parse(streamer_parser)
+        kwargs.update(load_config(streamer_args.__dict__))
+
+
+    if hasattr(Streamer, 'async_handler'):
+        # This is really a handler that needs wrapped with AsyncExecutor
+        Executor = Streamer
+        if Executor and hasattr(Executor, 'handle'):
+            executor = Executor(**kwargs).handle
+        else:
+            executor = Executor
+
+        # Now build the wrapper
+        ae = streamers.AsyncExecutor(executor, **ae_args)
+        return ae.stream
+    else:
+        if type(Streamer) == type:
+            return Streamer(**kwargs).stream
+        return Streamer
+
+
+def main():
+    """Target entry point for standard pyproject.toml scripts."""
+    args = sys.argv[1:]
+    streamline_command(args)

streamline/consumers.py

@@ -0,0 +1,169 @@
+import json
+import csv
+import os
+
+from . import utils 
+
+def stringify_all(source):
+    return [utils.force_string(v) for v in source]
+
+class FileWriter():
+    DEFAULT_OUTPUT = '-'
+    DELIMITER = '\n'
+
+    @classmethod
+    def args(cls, parser):
+        parser.add_argument(
+            '--output',
+            default=cls.DEFAULT_OUTPUT,
+            help='Set target of output (Default stdout)',
+        )
+
+    def __init__(self, output=None):
+        self.target_name = output
+        self.target = None
+        self.target_template = None
+        self.first_written = False
+
+        self.force_closing_newline = os.environ.get('STREAMLINE_CLOSING_NEWLINE')
+
+    def _output_value(self, entry):
+        return utils.force_string(entry.value)
+
+    async def stream(self, source):
+        output = ''
+        if '{' in self.target_name:
+            self.target_template = self.target_name
+        else:
+            self.target = utils.get_file_io(self.target_name, write=True)
+        async for entry in source:
+            output = self._output_value(entry)
+            if self.target_template:
+
+                file_name = self.target_template.format(input=entry.original_value, index=entry.index)
+                with open(file_name, 'w') as target_file:
+                    target_file.write(output)
+            else:
+                if self.force_closing_newline:
+                    output = output + self.DELIMITER
+                elif self.first_written:
+                    output = self.DELIMITER + output
+                else:
+                    self.first_written = True
+                self.target.write(output)
+            if hasattr(self.target, 'flush'):
+                self.target.flush()
+
+        if self.target and hasattr(self.target, 'close'):
+            self.target.close()
+
+class CSVWriter():
+    DEFAULT_OUTPUT = '-'
+
+    @classmethod
+    def args(cls, parser):
+        parser.add_argument('--output', help='Set target of output (Default stdout)', default=cls.DEFAULT_OUTPUT)
+        parser.add_argument(
+            '--input-column',
+            action='store_true',
+            default=False,
+            help='Automatically add a column for input value',
+        )
+
+    def __init__(self, output=DEFAULT_OUTPUT, input_column=False):
+        self.target_name = output
+        self.input_column = input_column
+
+    def _parse_fields(self, entry):
+        if not isinstance(entry.value, dict):
+            return None
+        else:
+            return entry.value.keys()
+    
+    def _get_values(self, entry, fields):
+        if fields is None:
+            return stringify_all((entry.original_value, entry.value))
+
+        # Return blanks for all invalid rows
+        if not isinstance(entry.value, dict):
+            return [''] * len(fields)
+
+        values = [entry.value.get(field, '') for field in fields]
+        if self.input_column:
+            values.insert(0, entry.original_value)
+        return stringify_all(values)
+
+    async def stream(self, source):
+        self.target = utils.get_file_io(self.target_name, write=True)
+        writer = csv.writer(self.target, 'unix', quoting=csv.QUOTE_MINIMAL)
+
+        fields = None
+        header_written = False
+        async for entry in source:
+            # Write csv column names
+            if not header_written:
+                fields = self._parse_fields(entry)
+                if fields is None:
+                    writer.writerow(['input', 'value'])
+                else:
+                    header = list(fields)
+                    if self.input_column:
+                        header.insert(0, 'input')
+                    writer.writerow(header)
+                header_written = True
+
+            # Write values
+            entry_values = self._get_values(entry, fields)
+            writer.writerow(entry_values)
+
+        if hasattr(self.target, 'close'):
+            self.target.close()
+
+class JsonWriter():
+    DEFAULT_OUTPUT = '-'
+
+    @classmethod
+    def args(cls, parser):
+        parser.add_argument(
+            '--output',
+            default=cls.DEFAULT_OUTPUT,
+            help='Set target of output (Default stdout)',
+        )
+
+    def __init__(self, output=DEFAULT_OUTPUT):
+        self.target_name = output
+
+    async def stream(self, source):
+        self.target = utils.get_file_io(self.target_name, write=True)
+        self.target.write('[\n')
+        first = True
+        async for entry in source:
+            if not first:
+                self.target.write(',\n    ')
+            else:
+                self.target.write('    ')
+            self.target.write(json.dumps(entry.value))
+            first = False
+        self.target.write('\n]')
+
+        if hasattr(self.target, 'close'):
+            self.target.close()
+
+
+CONSUMERS = {
+    'file': FileWriter,
+    'csv': CSVWriter,
+    'json': JsonWriter,
+}
+
+def load_consumer(path):
+    if path is None:
+        return None
+    elif '.' in path:
+        Consumer = utils.import_obj(path)
+    else:
+        Consumer = CONSUMERS.get(path)
+    if Consumer is None:
+        raise ValueError('Invalid consumer: {}'.format(path))
+    return Consumer
+

streamline/core.py

@@ -0,0 +1,45 @@
+import asyncio
+
+async def drain(generator):
+    """ A no-op drain of a generator """
+    items = []
+    async for item in generator:
+        items.append(item)
+    return items
+
+async def transync(synchronous_iter):
+    """
+        This is a converter which takes a synchronous iterator and yields the items
+        asynchronously which allows a standard pipe to convert to an async pipe.
+    """
+    for item in synchronous_iter:
+        yield item
+
+async def static_pipe(stream, inputs):
+    source = transync(inputs)
+    outputs = await drain(stream(source))
+    return outputs
+
+async def pipe(generator, streamers, consumer=None):
+    """
+        The "pipe" function is the central piece of a stream, the pipe connects
+        the generators together in a chain allowing each to pass the result of the
+        previous to the next.
+
+        The first generator is special as it does not take an input stream. 
+
+        The last piece of the stream is the consumer which is not a generator but 
+        is expected to drain the generator given to it, writing any valued output.
+    """
+    if consumer is None:
+        consumer = drain
+
+    pipe = generator
+    for streamer in streamers:
+        pipe = streamer(pipe)
+
+    await consumer(pipe)
+
+def sync_exec(future):
+    result = asyncio.run(future)
+    return result

streamline/entries.py

@@ -0,0 +1,71 @@
+import copy
+
+class EntryFactory():
+    def __init__(self, error_value=None):
+        self.error_value = error_value
+        self.index = 0
+
+    def __call__(self, value):
+        entry = Entry(
+            value,
+            error_value=self.error_value,
+            index=self.index
+        )
+        self.index += 1
+        return entry
+
+class Entry():
+    def __init__(self, value=None, index=None, error_value=None):
+        self.index = index
+        self.history = [[value]]
+        self.errors = []
+        self.error_value = error_value
+
+    def push(self, value=None):
+        if value is None:
+            value = self.value
+        self.history.append([value])
+
+    def pop(self):
+        if len(self.history) == 1:
+            raise ValueError('Attempted to pop an entry history that is only 1 level deep!')
+        value = self.value
+        self.history = self.history[:-1]
+        self.value = value
+
+    def reset(self):
+        self.history = [[self.history[0][0]]]
+
+    def collapse(self):
+        self.history[-1] = [self.history[-1][-1]]
+
+    def get_history(self):
+        return self.history[-1]
+
+    @property
+    def original_value(self):
+        return self.history[-1][0]
+
+    def get_value(self):
+        return self.history[-1][-1]
+
+    def set_value(self, new_value):
+        self.history[-1].append(new_value)
+
+    def error(self, e):
+        self.errors.append(e)
+        self.value = self.error_value
+
+    def clone(self):
+        new_clone = Entry(index=self.index, error_value=self.error_value)
+        new_clone.errors = self.errors.copy()
+        new_clone.history = [h.copy() for h in self.history]
+        return new_clone
+
+    value = property(get_value, set_value)
+
+def entry_wrap(items):
+    return [Entry(item) for item in items]
+
+def entry_unwrap(entries):
+    return [entry.value for entry in entries]