celus-nibbler 10.1.3__tar.gz

celus_nibbler-10.1.3/LICENSE

@@ -0,0 +1,19 @@
+Copyright 2022 BigDigData s.r.o.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

celus_nibbler-10.1.3/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: celus-nibbler
+Version: 10.1.3
+Summary: Counter-like data reader and processor.
+License: MIT
+Keywords: parsing,counter5
+Author: Stepan Henek
+Author-email: stepan@bigdigdata.com
+Requires-Python: >=3.8.9,<4.0.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Provides-Extra: xls
+Requires-Dist: celus-nigiri (>=1.3.3,<1.4.0)
+Requires-Dist: diskcache (>=5.6.0,<5.7.0)
+Requires-Dist: isbnlib (>=3.10.14,<3.11.0)
+Requires-Dist: nltk (>=3.8.1,<3.9.0)
+Requires-Dist: openpyxl (>=3.1.2,<3.2.0)
+Requires-Dist: pydantic (>=2.4.2,<2.5.0)
+Requires-Dist: python-dateutil (>=2.8.2,<2.9.0)
+Requires-Dist: typing-extensions (>=4.8.0,<4.9.0)
+Requires-Dist: unidecode (>=1.3.7,<1.4.0)
+Requires-Dist: xlrd (>=2.0.1,<2.1.0) ; extra == "xls"
+Project-URL: Documentation, https://gitlab.com/big-dig-data/celus-nibbler/-/blob/master/README.md

celus_nibbler-10.1.3/celus_nibbler/__init__.py

@@ -0,0 +1,28 @@
+from .eat_and_poop import Poop, PoopOrganizationStats, PoopStats, eat
+from .errors import (
+    MultipleParsersFound,
+    NibblerError,
+    NoParserForPlatformFound,
+    NoParserFound,
+    NoParserMatchesHeuristics,
+    RecordError,
+    TableException,
+    WrongFileFormatError,
+)
+from .parsers import get_supported_platforms
+
+__all__ = [
+    'eat',
+    'get_supported_platforms',
+    'Poop',
+    'PoopStats',
+    'PoopOrganizationStats',
+    'MultipleParsersFound',
+    'NibblerError',
+    'NoParserFound',
+    'NoParserForPlatformFound',
+    'NoParserMatchesHeuristics',
+    'RecordError',
+    'TableException',
+    'WrongFileFormatError',
+]

celus_nibbler-10.1.3/celus_nibbler/__main__.py

@@ -0,0 +1,241 @@
+import argparse
+import csv
+import json
+import logging
+import logging.config
+import pathlib
+import sys
+import typing
+from contextlib import nullcontext
+from copy import deepcopy
+from datetime import date
+from importlib.metadata import distribution
+
+from celus_nigiri import CounterRecord
+from unidecode import unidecode
+
+from celus_nibbler import Poop, eat
+from celus_nibbler.aggregator import CounterOrdering
+from celus_nibbler.definitions import Definition
+from celus_nibbler.parsers import available_parsers
+from celus_nibbler.parsers.dynamic import gen_parser
+from celus_nibbler.utils import profile
+
+
+def read_definition(path: str) -> Definition:
+    with pathlib.Path(path).open() as f:
+        definition_data = json.load(f)
+    return Definition.parse(definition_data)
+
+
+def gen_description(parsers: typing.List[str]) -> str:
+    parsers_list = "\n".join([f"  {e}" for e in parsers])
+
+    return f"""Available Parsers:
+{parsers_list}
+"""
+
+
+def write_batch(
+    writer,
+    records: typing.List[CounterRecord],
+    dimensions: typing.List[str],
+    title_ids: typing.List[str],
+    months: typing.List[date],
+):
+    months_dict = {e: "" for e in months}
+    for r in records:
+        months_dict[r.start] = r.value
+    writer.writerow(
+        [r.title, r.organization]
+        + [r.title_ids[ti] for ti in title_ids]
+        + [r.dimension_data[d] for d in dimensions]
+        + [r.metric]
+        + [months_dict[k] for k in sorted(months)]
+    )
+
+
+def gen_argument_parser(parsers: typing.List[str]) -> argparse.ArgumentParser:
+    dist = distribution("celus-nibbler")
+    version = dist.version if dist else "?"
+    description = gen_description(parsers)
+    parser = argparse.ArgumentParser(
+        prog="nibbler-eat",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=description,
+    )
+    parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False)
+    parser.add_argument("--version", action="version", version=version)
+    parser.add_argument(
+        "-s",
+        "--skip-heuristics",
+        dest="skip_heuristics",
+        action="store_true",
+        default=False,
+        help="skip heuristic checks",
+    )
+    parser.add_argument(
+        "-p",
+        "--parser",
+        action="append",
+        default=[],
+        help="limit parsers which can be used",
+    )
+    parser.add_argument(
+        "-S",
+        "--show-summary",
+        action="store_true",
+        default=False,
+        help="show per sheet summary",
+    )
+    parser.add_argument(
+        "-D",
+        "--definition",
+        action="append",
+        default=[],
+        help="Reads a definition of a dynamic parser from a file",
+        type=read_definition,
+    )
+    parser.add_argument(
+        "-P",
+        "--platform",
+        default="",
+        help="Limit platform that can be used",
+    )
+    parser.add_argument(
+        "-c",
+        "--counter-like-output",
+        action="store_true",
+        default=False,
+        help="Display processed data in counter like format",
+    )
+    parser.add_argument(
+        "-N",
+        "--no-output",
+        action="store_true",
+        default=False,
+        help="Parses the entire file without producing any data output (useful for benchmarks)",
+    )
+    parser.add_argument("file", nargs='*')
+    parser.add_argument("--profile", dest="profile", action="store_true", default=False)
+
+    return parser
+
+
+def parse(options, platform, dynamic_parsers):
+    for file in options.file:
+        if poops := eat(
+            pathlib.Path(file),
+            platform or "void",
+            parsers=options.parser or None,
+            check_platform=bool(platform),
+            use_heuristics=not options.skip_heuristics,
+            dynamic_parsers=dynamic_parsers,
+        ):
+            for idx, poop in enumerate(poops):
+                if not isinstance(poop, Poop):
+                    print(f"Failed to pick parser for sheet {idx}", file=sys.stderr)
+                    continue
+
+                stat_dict = poop.get_stats().dict()
+                if options.show_summary:
+                    print(f"Parsing sheet {idx}", file=sys.stderr)
+                    print(f"Months: {stat_dict['months']}", file=sys.stderr)
+                    print(f"Metrics: {stat_dict['metrics']}", file=sys.stderr)
+                    print(f"Dimensions: {stat_dict['dimensions']}", file=sys.stderr)
+                    print(f"Title ids: {stat_dict['title_ids']}", file=sys.stderr)
+
+                if options.no_output:
+                    header = None
+                elif options.counter_like_output:
+                    header = (
+                        ["title", "organization"]
+                        + poop.title_ids
+                        + poop.dimensions
+                        + ["metric"]
+                        + poop.months
+                    )
+                else:
+                    header = [
+                        "start",
+                        "end",
+                        "organization",
+                        "title",
+                        "metric",
+                        "dimensions",
+                        "title_ids",
+                        "value",
+                    ]
+
+                writer = csv.writer(sys.stdout, dialect="unix")
+                if header:
+                    writer.writerow(header)
+                records = (
+                    CounterOrdering().aggregate(poop.records_basic())
+                    if options.counter_like_output
+                    else poop.records_basic()
+                )
+                last_rec = None
+                batch = []
+                for idx, record in records:
+                    if options.no_output:
+                        # Just go through the records and suppress any output
+                        pass
+                    elif options.counter_like_output:
+                        this_rec = deepcopy(record)
+                        this_rec.value = -1
+                        this_rec.start = date(2020, 1, 1)
+                        this_rec.end = date(2020, 1, 31)
+
+                        if not last_rec or last_rec == this_rec:
+                            batch.append(record)
+                        elif batch:
+                            write_batch(
+                                writer,
+                                batch,
+                                stat_dict["dimensions"].keys(),
+                                stat_dict["title_ids"],
+                                stat_dict["months"].keys(),
+                            )
+                            batch = [record]
+                        last_rec = this_rec
+                    else:
+                        writer.writerow(record.as_csv())
+
+                if batch:
+                    write_batch(
+                        writer,
+                        batch,
+                        stat_dict["dimensions"].keys(),
+                        stat_dict["title_ids"],
+                        stat_dict["months"].keys(),
+                    )
+
+
+def main():
+    parser = gen_argument_parser(available_parsers())
+    options = parser.parse_args()
+
+    # init logging
+    logging.basicConfig(level=logging.DEBUG if options.debug else logging.WARNING)
+    logger = logging.getLogger(__name__)
+    logger.debug("Logging is configured.")
+
+    platform = unidecode(options.platform)
+    dynamic_parsers = [gen_parser(e) for e in options.definition]
+
+    if not options.file:
+        # Just print help if not parsers specified
+        # Enrich with dynamic parsers
+        parser = gen_argument_parser(available_parsers(dynamic_parsers))
+        parser.print_help()
+        return
+
+    context_man = nullcontext if not options.profile else profile
+
+    with context_man():
+        parse(options, platform, dynamic_parsers)
+
+
+if __name__ == "__main__":
+    main()

celus_nibbler-10.1.3/celus_nibbler/aggregator.py

@@ -0,0 +1,162 @@
+import abc
+import copy
+import tempfile
+import typing
+from collections import deque
+from datetime import date
+
+from celus_nigiri import CounterRecord
+from diskcache import Cache
+
+from celus_nibbler.errors import NegativeValueInOutput, SameRecordsInOutput
+
+
+class BaseAggregator(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        pass
+
+    def __or__(self, other):
+        return PippedAggregator(self, other)
+
+
+class NoAggregator(BaseAggregator):
+    """Doesn't aggregate anything just passes the data"""
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        yield from records
+
+
+class SameAggregator(BaseAggregator):
+    """Aggregates the same records - sums values"""
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with Cache(tmpdir) as db, db.transact():
+                # Read all records
+                for record in records:
+                    # Derive key
+                    new_record = copy.deepcopy(record)
+                    new_record.value = -1
+                    key = new_record.as_csv()
+
+                    # Aggregate
+                    if existing_record := db.get(key):
+                        existing_record.value += record.value
+                        db[key] = existing_record
+                    else:
+                        db[key] = record
+
+                for key in db.iterkeys():
+                    yield db[key]
+
+
+class CheckConflictingRecordsAggregator(BaseAggregator):
+    """
+    Checks whether last n records doesn't contain same conflicting records e.g.
+
+    Title1,Dim1,Metric1,2021-01-01,2021-01-31,999
+    Title2,Dim1,Metric1,2021-01-01,2021-01-31,888
+    Title1,Dim1,Metric1,2021-01-01,2021-01-31,999
+
+    is wrong, because it contains same Title, Dimensions and metric for 2020-01 twice
+
+    """
+
+    def __init__(self, size: int = 100):
+        self.hash_buffer: typing.Deque[int, int] = deque(maxlen=size)
+
+    @staticmethod
+    def make_record_hash(record: CounterRecord):
+        return hash(tuple(e for e in record.as_csv()[:-1]))  # skip value (last in csv)
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        idx = 0
+        for record in records:
+            hsh = self.make_record_hash(record)
+            for e in self.hash_buffer:
+                if e[0] == hsh:
+                    raise SameRecordsInOutput(e[1], idx, record)
+            self.hash_buffer.append((hsh, idx))
+            yield record
+            idx += 1
+
+
+class CounterOrdering(BaseAggregator):
+    """
+    Extracts same records with different dates together e.g.
+
+    Title1,Dim1,Metric1,2021-01-01,2021-01-31,999
+    Title2,Dim1,Metric1,2021-01-01,2021-01-31,888
+    Title1,Dim1,Metric1,2021-02-01,2021-02-28,777
+    Title2,Dim1,Metric1,2021-02-01,2021-02-28,666
+
+    ->
+
+    Title1,Dim1,Metric1,2021-01-01,2021-01-31,999
+    Title1,Dim1,Metric1,2021-02-01,2021-02-28,777
+    Title2,Dim1,Metric1,2021-01-01,2021-01-31,888
+    Title2,Dim1,Metric1,2021-02-01,2021-02-28,666
+    """
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with Cache(tmpdir) as db, db.transact():
+                # Read all records
+                for record in records:
+                    # Derive key
+                    new_record = copy.deepcopy(record)
+                    new_record.value = -1
+                    # Date aggregation -> fix date in key
+                    new_record.start = date(2020, 1, 1)
+                    new_record.end = date(2020, 1, 31)
+                    key = new_record.as_csv()
+
+                    # Aggregate
+                    record_dict: typing.Dict[date, CounterRecord] = db.get(key, {})
+                    if rec := record_dict.get(record.start):
+                        rec.value += record.value
+                    else:
+                        record_dict[record.start] = record
+                    db[key] = record_dict
+
+                for db_key in db.iterkeys():
+                    record_dict = db[db_key]
+                    yield from [record_dict[key] for key in sorted(record_dict.keys())]
+
+
+class CheckNonNegativeValues(BaseAggregator):
+    """Checks whether outout contains negative values and raises and error if so"""
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        idx = 0
+        for record in records:
+            if record.value < 0:
+                raise NegativeValueInOutput(idx, record)
+            yield record
+            idx += 1
+
+
+class PippedAggregator(BaseAggregator):
+    """Pipes output from one aggregator to another aggregator"""
+
+    def __init__(self, a1: BaseAggregator, a2: BaseAggregator):
+        self.a1 = a1
+        self.a2 = a2
+
+    def aggregate(
+        self, records: typing.Generator[CounterRecord, None, None]
+    ) -> typing.Generator[CounterRecord, None, None]:
+        return self.a2.aggregate(self.a1.aggregate(records))