flow.record 3.21.dev3__tar.gz → 3.21.dev5__tar.gz

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.21.dev3
+Version: 3.21.dev5
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -37,16 +37,21 @@ Requires-Dist: duckdb; extra == "duckdb"
 Requires-Dist: pytz; extra == "duckdb"
 Provides-Extra: splunk
 Requires-Dist: httpx; extra == "splunk"
+Provides-Extra: xlsx
+Requires-Dist: openpyxl; extra == "xlsx"
 Provides-Extra: test
 Requires-Dist: flow.record[compression]; extra == "test"
 Requires-Dist: flow.record[avro]; extra == "test"
 Requires-Dist: flow.record[elastic]; extra == "test"
+Requires-Dist: flow.record[xlsx]; extra == "test"
 Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 Requires-Dist: tqdm; extra == "test"
+Requires-Dist: structlog; extra == "test"
 Provides-Extra: full
 Requires-Dist: flow.record[compression]; extra == "full"
 Requires-Dist: tqdm; extra == "full"
+Requires-Dist: structlog; extra == "full"
 Dynamic: license-file
 
 # flow.record

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/examples/filesystem.py

@@ -1,10 +1,15 @@
-import os
-import stat
+from __future__ import annotations
 
-from datetime import datetime
+import stat
+from pathlib import Path
+from typing import TYPE_CHECKING
 
 from flow.record import RecordDescriptor, RecordWriter
 
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
 descriptor = """
 filesystem/unix/entry
     string path;
@@ -22,34 +27,32 @@ filesystem/unix/entry
 FilesystemFile = RecordDescriptor(descriptor)
 
 
-def hash_file(path, t):
-    f = open(path, "rb")
-    while 1:
-        d = f.read(4096)
-        if d == "":
-            break
-    f.close()
+def hash_file(path: str | Path) -> None:
+    with Path(path).open("rb") as f:
+        while True:
+            d = f.read(4096)
+            if not d:
+                break
 
 
 class FilesystemIterator:
     basepath = None
 
-    def __init__(self, basepath):
+    def __init__(self, basepath: str | None):
         self.basepath = basepath
         self.recordType = FilesystemFile
 
-    def classify(self, source, classification):
+    def classify(self, source: str, classification: str) -> None:
         self.recordType = FilesystemFile.base(_source=source, _classification=classification)
 
-    def iter(self, path):
-        path = os.path.abspath(path)
-        return self._iter(path)
+    def iter(self, path: str | Path) -> Iterator[FilesystemFile]:
+        return self._iter(Path(path).resolve())
 
-    def _iter(self, path):
-        if path.startswith("/proc"):
+    def _iter(self, path: Path) -> Iterator[FilesystemFile]:
+        if path.is_relative_to("/proc"):
             return
 
-        st = os.lstat(path)
+        st = path.lstat()
 
         abspath = path
         if self.basepath and abspath.startswith(self.basepath):
@@ -59,7 +62,7 @@ class FilesystemIterator:
 
         link = None
         if ifmt == stat.S_IFLNK:
-            link = os.readlink(path)
+            link = path.readlink()
 
         yield self.recordType(
             path=abspath,
@@ -69,20 +72,16 @@ class FilesystemIterator:
             size=st.st_size,
             uid=st.st_uid,
             gid=st.st_gid,
-            ctime=datetime.fromtimestamp(st.st_ctime),
-            mtime=datetime.fromtimestamp(st.st_mtime),
-            atime=datetime.fromtimestamp(st.st_atime),
+            ctime=st.st_ctime,
+            mtime=st.st_mtime,
+            atime=st.st_atime,
             link=link,
         )
 
         if ifmt == stat.S_IFDIR:
-            for i in os.listdir(path):
-                if i in (".", ".."):
-                    continue
-
-                fullpath = os.path.join(path, i)
-                for e in self.iter(fullpath):
-                    yield e
+            for i in path.iterdir():
+                fullpath = path.joinpath(i)
+                yield from self.iter(fullpath)
 
 
 chunk = []

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/examples/passivedns.py

@@ -1,18 +1,21 @@
 #!/usr/bin/env pypy
-import record
+from __future__ import annotations
+
 import sys
-import datetime
+from datetime import datetime, timezone
 
 import net.ipv4
-
+import record
 from fileprocessing import DirectoryProcessor
 
+UTC_TIMEZONE = timezone.utc
+
 
-def ts(s):
-    return datetime.datetime.fromtimestamp(float(s))
+def ts(s: float) -> datetime:
+    return datetime.fromtimestamp(float(s), tz=UTC_TIMEZONE)
 
 
-def ip(s):
+def ip(s: str) -> net.ipv4.Address:
     return net.ipv4.Address(s)
 
 
@@ -21,7 +24,7 @@ class SeparatedFile:
     seperator = None
     format = None
 
-    def __init__(self, fp, seperator, format):
+    def __init__(self, fp: list[str], seperator: str | None, format: list[tuple]):
         self.fp = fp
         self.seperator = seperator
         self.format = format
@@ -46,7 +49,7 @@ class SeparatedFile:
             yield recordtype(**r)
 
 
-def PassiveDnsFile(fp):
+def PassiveDnsFile(fp: list[str]) -> SeparatedFile:
     return SeparatedFile(fp, "||", PASSIVEDNS_FORMAT)
 
 
@@ -63,7 +66,7 @@ PASSIVEDNS_FORMAT = [
 ]
 
 
-def main():
+def main() -> None:
     rs = record.RecordOutput(sys.stdout)
     for r in DirectoryProcessor(sys.argv[1], PassiveDnsFile, r"\.log\.gz"):
         rs.write(r)

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/examples/tcpconn.py

@@ -1,8 +1,10 @@
 import random
+from datetime import datetime, timezone
 
-from datetime import datetime
 from flow import record
 
+UTC_TIMEZONE = timezone.utc
+
 descriptor = """
 network/traffic/tcp/connection
     datetime ts;
@@ -32,9 +34,9 @@ port_list = [
 
 rs = record.RecordWriter()
 
-for i in range(500):
+for _ in range(500):
     r = conn(
-        ts=datetime.now(),
+        ts=datetime.now(tz=UTC_TIMEZONE),
         src=random.choice(ip_list),
         srcport=random.choice(port_list),
         dst=random.choice(ip_list),

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/adapter/elastic.py

@@ -4,8 +4,11 @@ import hashlib
 import logging
 import queue
 import threading
+from contextlib import suppress
 from typing import TYPE_CHECKING
 
+import urllib3
+
 try:
     import elasticsearch
     import elasticsearch.helpers
@@ -77,6 +80,8 @@ class ElasticWriter(AbstractWriter):
         http_compress = boolean_argument(http_compress)
         self.hash_record = boolean_argument(hash_record)
         queue_size = int(queue_size)
+        request_timeout = int(request_timeout)
+        self.max_retries = int(max_retries)
 
         if not uri.lower().startswith(("http://", "https://")):
             uri = "http://" + uri
@@ -93,7 +98,7 @@ class ElasticWriter(AbstractWriter):
             api_key=api_key,
             request_timeout=request_timeout,
             retry_on_timeout=True,
-            max_retries=max_retries,
+            max_retries=self.max_retries,
         )
 
         self.json_packer = JsonRecordPacker()
@@ -103,8 +108,6 @@ class ElasticWriter(AbstractWriter):
 
         if not verify_certs:
             # Disable InsecureRequestWarning of urllib3, caused by the verify_certs flag.
-            import urllib3
-
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
         self.metadata_fields = {}
@@ -113,10 +116,9 @@ class ElasticWriter(AbstractWriter):
                 self.metadata_fields[arg_key[6:]] = arg_val
 
     def excepthook(self, exc: threading.ExceptHookArgs, *args, **kwargs) -> None:
-        log.error("Exception in thread: %s", exc)
         self.exception = getattr(exc, "exc_value", exc)
+        self.exception = enrich_elastic_exception(self.exception)
         self.event.set()
-        self.close()
 
     def record_to_document(self, record: Record, index: str) -> dict:
         """Convert a record to a Elasticsearch compatible document dictionary"""
@@ -169,13 +171,13 @@ class ElasticWriter(AbstractWriter):
             - https://elasticsearch-py.readthedocs.io/en/v8.17.1/helpers.html#elasticsearch.helpers.streaming_bulk
             - https://github.com/elastic/elasticsearch-py/blob/main/elasticsearch/helpers/actions.py#L362
         """
+
         for _ok, _item in elasticsearch.helpers.streaming_bulk(
             self.es,
             self.document_stream(),
             raise_on_error=True,
             raise_on_exception=True,
-            # Some settings have to be redefined because streaming_bulk does not inherit them from the self.es instance.
-            max_retries=3,
+            max_retries=self.max_retries,
         ):
             pass
 
@@ -191,13 +193,17 @@ class ElasticWriter(AbstractWriter):
         pass
 
     def close(self) -> None:
-        self.queue.put(StopIteration)
-        self.event.wait()
+        if hasattr(self, "queue"):
+            self.queue.put(StopIteration)
+
+        if hasattr(self, "event"):
+            self.event.wait()
 
         if hasattr(self, "es"):
-            self.es.close()
+            with suppress(Exception):
+                self.es.close()
 
-        if self.exception:
+        if hasattr(self, "exception") and self.exception:
             raise self.exception
 
 
@@ -219,6 +225,8 @@ class ElasticReader(AbstractReader):
         self.selector = selector
         verify_certs = boolean_argument(verify_certs)
         http_compress = boolean_argument(http_compress)
+        request_timeout = int(request_timeout)
+        max_retries = int(max_retries)
 
         if not uri.lower().startswith(("http://", "https://")):
             uri = "http://" + uri
@@ -235,8 +243,6 @@ class ElasticReader(AbstractReader):
 
         if not verify_certs:
             # Disable InsecureRequestWarning of urllib3, caused by the verify_certs flag.
-            import urllib3
-
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
     def __iter__(self) -> Iterator[Record]:
@@ -255,3 +261,32 @@ class ElasticReader(AbstractReader):
     def close(self) -> None:
         if hasattr(self, "es"):
             self.es.close()
+
+
+def enrich_elastic_exception(exception: Exception) -> Exception:
+    """Extend the exception with error information from Elastic.
+
+    Resources:
+        - https://elasticsearch-py.readthedocs.io/en/v8.17.1/exceptions.html
+    """
+    errors = set()
+    if hasattr(exception, "errors"):
+        try:
+            for error in exception.errors:
+                index_dict = error.get("index", {})
+                status = index_dict.get("status")
+                error_dict = index_dict.get("error", {})
+                error_type = error_dict.get("type")
+                error_reason = error_dict.get("reason", "")
+
+                errors.add(f"({status} {error_type} {error_reason})")
+        except Exception:
+            errors.add("unable to extend errors")
+
+    # append errors to original exception message
+    error_str = ", ".join(errors)
+    original_message = exception.args[0] if exception.args else ""
+    new_message = f"{original_message} {error_str}"
+    exception.args = (new_message,) + exception.args[1:]
+
+    return exception

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/adapter/splunk.py

@@ -35,7 +35,7 @@ Write usage: rdump -w splunk+[PROTOCOL]://[IP]:[PORT]?tag=[TAG]&token=[TOKEN]&so
 [SSL_VERIFY]: Whether to verify the server certificate when sending data over HTTPS. Defaults to True.
 """
 
-log = logging.getLogger(__package__)
+log = logging.getLogger(__name__)
 
 # Amount of records to bundle into a single request when sending data over HTTP(S).
 RECORD_BUFFER_LIMIT = 20

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/base.py

@@ -64,7 +64,7 @@ if TYPE_CHECKING:
 
     from flow.record.adapter import AbstractReader, AbstractWriter
 
-log = logging.getLogger(__package__)
+log = logging.getLogger(__name__)
 _utcnow = functools.partial(datetime.now, timezone.utc)
 
 RECORD_VERSION = 1
@@ -186,6 +186,7 @@ class Record:
         return OrderedDict((k, getattr(self, k)) for k in self.__slots__ if k not in exclude)
 
     if TYPE_CHECKING:
+
         def __getattr__(self, name: str) -> Any: ...
 
     def __setattr__(self, k: str, v: Any) -> None:

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/jsonpacker.py

@@ -11,7 +11,7 @@ from flow.record.base import Record, RecordDescriptor
 from flow.record.exceptions import RecordDescriptorNotFound
 from flow.record.utils import EventHandler
 
-log = logging.getLogger(__package__)
+log = logging.getLogger(__name__)
 
 
 class JsonRecordPacker:

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/stream.py

@@ -15,14 +15,14 @@ from flow.record.base import Record, RecordDescriptor, RecordReader
 from flow.record.fieldtypes import fieldtype_for_value
 from flow.record.packer import RecordPacker
 from flow.record.selector import make_selector
-from flow.record.utils import is_stdout
+from flow.record.utils import LOGGING_TRACE_LEVEL, is_stdout
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
     from flow.record.adapter import AbstractWriter
 
-log = logging.getLogger(__package__)
+log = logging.getLogger(__name__)
 
 aRepr = reprlib.Repr()
 aRepr.maxother = 255
@@ -146,8 +146,11 @@ class RecordStreamReader:
 def record_stream(sources: list[str], selector: str | None = None) -> Iterator[Record]:
     """Return a Record stream generator from the given Record sources.
 
-    Exceptions in a Record source will be caught so the stream is not interrupted.
+    If there are multiple sources, exceptions are caught and logged, and the stream continues with the next source.
     """
+
+    trace = log.isEnabledFor(LOGGING_TRACE_LEVEL)
+
     log.debug("Record stream with selector: %r", selector)
     for src in sources:
         # Inform user that we are reading from stdin
@@ -161,12 +164,20 @@ def record_stream(sources: list[str], selector: str | None = None) -> Iterator[R
             yield from reader
             reader.close()
         except IOError as e:
-            log.exception("%s(%r): %s", reader, src, e)  # noqa: TRY401
+            if len(sources) == 1:
+                raise
+            else:
+                log.error("%s(%r): %s", reader, src, e)
+                if trace:
+                    log.exception("Full traceback")
         except KeyboardInterrupt:
             raise
         except Exception as e:
-            log.warning("Exception in %r for %r: %s -- skipping to next reader", reader, src, aRepr.repr(e))
-            continue
+            if len(sources) == 1:
+                raise
+            else:
+                log.warning("Exception in %r for %r: %s -- skipping to next reader", reader, src, aRepr.repr(e))
+                continue
 
 
 class PathTemplateWriter:

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/tools/rdump.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from __future__ import annotations
 
+import argparse
 import logging
 import sys
 from importlib import import_module
@@ -14,7 +15,7 @@ import flow.record.adapter
 from flow.record import RecordWriter, iter_timestamped_records, record_stream
 from flow.record.selector import make_selector
 from flow.record.stream import RecordFieldRewriter
-from flow.record.utils import catch_sigpipe
+from flow.record.utils import LOGGING_TRACE_LEVEL, catch_sigpipe
 
 try:
     from flow.record.version import version
@@ -29,6 +30,15 @@ try:
 except ImportError:
     HAS_TQDM = False
 
+try:
+    import structlog
+
+    HAS_STRUCTLOG = True
+
+except ImportError:
+    HAS_STRUCTLOG = False
+
+
 log = logging.getLogger(__name__)
 
 
@@ -69,8 +79,6 @@ def list_adapters() -> None:
 
 @catch_sigpipe
 def main(argv: list[str] | None = None) -> int:
-    import argparse
-
     parser = argparse.ArgumentParser(
         description="Record dumper, a tool that can read, write and filter records",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -130,6 +138,11 @@ def main(argv: list[str] | None = None) -> int:
         action="store_true",
         help="Show progress bar (requires tqdm)",
     )
+    output.add_argument(
+        "--stats",
+        action="store_true",
+        help="Show count of processed records",
+    )
 
     advanced = parser.add_argument_group("advanced")
     advanced.add_argument(
@@ -196,10 +209,30 @@ def main(argv: list[str] | None = None) -> int:
 
     args = parser.parse_args(argv)
 
-    levels = [logging.WARNING, logging.INFO, logging.DEBUG]
+    levels = [logging.WARNING, logging.INFO, logging.DEBUG, LOGGING_TRACE_LEVEL]
     level = levels[min(len(levels) - 1, args.verbose)]
     logging.basicConfig(level=level, format="%(asctime)s %(levelname)s %(message)s")
 
+    if HAS_STRUCTLOG:
+        # We have structlog, configure Python logging to use it for rendering
+        console_renderer = structlog.dev.ConsoleRenderer()
+        handler = logging.StreamHandler()
+        handler.setFormatter(
+            structlog.stdlib.ProcessorFormatter(
+                processor=console_renderer,
+                foreign_pre_chain=[
+                    structlog.stdlib.add_logger_name,
+                    structlog.stdlib.add_log_level,
+                    structlog.processors.TimeStamper(fmt="iso"),
+                ],
+            )
+        )
+
+        # Clear existing handlers and add our structlog handler
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+        root_logger.addHandler(handler)
+
     fields_to_exclude = args.exclude.split(",") if args.exclude else []
     fields = args.fields.split(",") if args.fields else []
 
@@ -253,6 +286,7 @@ def main(argv: list[str] | None = None) -> int:
 
     count = 0
     record_writer = None
+    ret = 0
 
     try:
         record_writer = RecordWriter(uri)
@@ -280,14 +314,33 @@ def main(argv: list[str] | None = None) -> int:
                 else:
                     record_writer.write(rec)
 
+    except Exception as e:
+        print_error(e)
+
+        # Prevent throwing an exception twice when deconstructing the record writer.
+        if hasattr(record_writer, "exception") and record_writer.exception is e:
+            record_writer.exception = None
+
+        ret = 1
+
     finally:
         if record_writer:
-            record_writer.__exit__()
+            # Exceptions raised in threads can be thrown when deconstructing the writer.
+            try:
+                record_writer.__exit__()
+            except Exception as e:
+                print_error(e)
+
+    if (args.list or args.stats) and not args.progress:
+        print(f"Processed {count} records", file=sys.stdout if args.list else sys.stderr)
+
+    return ret
 
-    if args.list:
-        print(f"Processed {count} records")
 
-    return 0
+def print_error(e: Exception) -> None:
+    log.error("rdump encountered a fatal error: %s", e)
+    if log.isEnabledFor(LOGGING_TRACE_LEVEL):
+        log.exception("Full traceback")
 
 
 if __name__ == "__main__":

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/utils.py

@@ -7,6 +7,8 @@ import warnings
 from functools import wraps
 from typing import Any, BinaryIO, Callable, TextIO
 
+LOGGING_TRACE_LEVEL = 5
+
 
 def get_stdout(binary: bool = False) -> TextIO | BinaryIO:
     """Return the stdout stream as binary or text stream.

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow/record/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.21.dev3'
-__version_tuple__ = version_tuple = (3, 21, 'dev3')
+__version__ = version = '3.21.dev5'
+__version_tuple__ = version_tuple = (3, 21, 'dev5')

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow.record.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.21.dev3
+Version: 3.21.dev5
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -37,16 +37,21 @@ Requires-Dist: duckdb; extra == "duckdb"
 Requires-Dist: pytz; extra == "duckdb"
 Provides-Extra: splunk
 Requires-Dist: httpx; extra == "splunk"
+Provides-Extra: xlsx
+Requires-Dist: openpyxl; extra == "xlsx"
 Provides-Extra: test
 Requires-Dist: flow.record[compression]; extra == "test"
 Requires-Dist: flow.record[avro]; extra == "test"
 Requires-Dist: flow.record[elastic]; extra == "test"
+Requires-Dist: flow.record[xlsx]; extra == "test"
 Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 Requires-Dist: tqdm; extra == "test"
+Requires-Dist: structlog; extra == "test"
 Provides-Extra: full
 Requires-Dist: flow.record[compression]; extra == "full"
 Requires-Dist: tqdm; extra == "full"
+Requires-Dist: structlog; extra == "full"
 Dynamic: license-file
 
 # flow.record

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow.record.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ MANIFEST.in
 README.md
 pyproject.toml
 tox.ini
+examples/__init__.py
 examples/filesystem.py
 examples/passivedns.py
 examples/records.json

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/flow.record.egg-info/requires.txt

@@ -23,6 +23,7 @@ elasticsearch
 [full]
 flow.record[compression]
 tqdm
+structlog
 
 [geoip]
 maxminddb
@@ -34,8 +35,13 @@ httpx
 flow.record[compression]
 flow.record[avro]
 flow.record[elastic]
+flow.record[xlsx]
 tqdm
+structlog
 
 [test:platform_python_implementation != "PyPy" and python_version < "3.12"]
 duckdb
 pytz
+
+[xlsx]
+openpyxl

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/pyproject.toml

@@ -56,17 +56,23 @@ duckdb = [
 splunk = [
     "httpx",
 ]
+xlsx = [
+    "openpyxl",
+]
 test = [
     "flow.record[compression]",
     "flow.record[avro]",
     "flow.record[elastic]",
+    "flow.record[xlsx]",
     "duckdb; platform_python_implementation != 'PyPy' and python_version < '3.12'", # duckdb
     "pytz; platform_python_implementation != 'PyPy' and python_version < '3.12'", # duckdb
     "tqdm",
+    "structlog",
 ]
 full = [
     "flow.record[compression]",
     "tqdm",
+    "structlog",
 ]
 
 [project.scripts]
@@ -116,7 +122,7 @@ select = [
   "FURB",
   "RUF",
 ]
-ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003"]
+ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003", "TRY400"]
 
 [tool.ruff.lint.per-file-ignores]
 "tests/docs/**" = ["INP001"]

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tests/standalone_test.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import traceback
 from typing import Callable
 
 
@@ -14,6 +15,5 @@ def main(glob: dict[str, Callable[..., None]]) -> None:
             print("PASSED")
         except Exception:
             print("FAILED")
-            import traceback
 
             traceback.print_exc()

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tests/test_fieldtypes.py

@@ -376,15 +376,21 @@ def test_uri_type() -> None:
     assert r.path.protocol == "http"
     assert r.path.hostname == "example.com"
 
-    with pytest.warns(DeprecationWarning):
+    with pytest.warns(
+        DeprecationWarning, match=r"Do not use class uri\(...\) for filesystem paths, use class path\(...\)"
+    ):
         r = TestRecord(uri.from_windows(r"c:\windows\program files\Fox-IT B.V\flow.exe"))
     assert r.path.filename == "flow.exe"
 
     r = TestRecord()
-    with pytest.warns(DeprecationWarning):
+    with pytest.warns(
+        DeprecationWarning, match=r"Do not use class uri\(...\) for filesystem paths, use class path\(...\)"
+    ):
         r.path = uri.normalize(r"c:\Users\Fox-IT\Downloads\autoruns.exe")
     assert r.path.filename == "autoruns.exe"
-    with pytest.warns(DeprecationWarning):
+    with pytest.warns(
+        DeprecationWarning, match=r"Do not use class uri\(...\) for filesystem paths, use class path\(...\)"
+    ):
         assert r.path.dirname == uri.normalize(r"\Users\Fox-IT\Downloads")
     assert r.path.dirname == "/Users/Fox-IT/Downloads"
 

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tests/test_packer.py

@@ -22,7 +22,7 @@ def test_uri_packing() -> None:
         ],
     )
 
-    # construct with an url
+    # Construct with an url
     record = TestRecord("http://www.google.com/evil.bin")
     data = packer.pack(record)
     record = packer.unpack(data)
@@ -30,8 +30,9 @@ def test_uri_packing() -> None:
     assert record.path.filename == "evil.bin"
     assert record.path.dirname == "/"
 
-    # construct from uri() -> for windows=True
-    with pytest.warns(DeprecationWarning):
+    with pytest.warns(
+        DeprecationWarning, match=r"Do not use class uri\(...\) for filesystem paths, use class path\(...\)"
+    ):
         path = uri.from_windows(r"c:\Program Files\Fox-IT\flow is awesome.exe")
     record = TestRecord(path)
     data = packer.pack(record)
@@ -40,8 +41,9 @@ def test_uri_packing() -> None:
     assert record.path.filename == "flow is awesome.exe"
     assert record.path.dirname == "/Program Files/Fox-IT"
 
-    # construct using uri.from_windows()
-    with pytest.warns(DeprecationWarning):
+    with pytest.warns(
+        DeprecationWarning, match=r"Do not use class uri\(...\) for filesystem paths, use class path\(...\)"
+    ):
         path = uri.from_windows(r"c:\Users\Hello World\foo.bar.exe")
     record = TestRecord(path)
     data = packer.pack(record)

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tests/test_rdump.py

@@ -17,6 +17,7 @@ import pytest
 
 import flow.record.fieldtypes
 from flow.record import RecordDescriptor, RecordReader, RecordWriter
+from flow.record.adapter.line import field_types_for_record_descriptor
 from flow.record.fieldtypes import flow_record_tz
 from flow.record.tools import rdump
 
@@ -681,8 +682,6 @@ def test_rdump_line_verbose(tmp_path: Path, capsys: pytest.CaptureFixture, rdump
         writer.write(TestRecord(counter=2))
         writer.write(TestRecord(counter=3))
 
-    from flow.record.adapter.line import field_types_for_record_descriptor
-
     field_types_for_record_descriptor.cache_clear()
     assert field_types_for_record_descriptor.cache_info().currsize == 0
     rdump.main([str(record_path), *rdump_params])
@@ -721,4 +720,3 @@ def test_rdump_list_progress(tmp_path: Path, capsys: pytest.CaptureFixture) -> N
 
     # stdout should contain the RecordDescriptor definition and count
     assert "# <RecordDescriptor test/rdump/progress, hash=eeb21156>" in captured.out
-    assert "Processed 100 records" in captured.out

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tests/test_xlsx_adapter.py

@@ -9,6 +9,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from flow.record import fieldtypes
+from flow.record.adapter.xlsx import sanitize_fieldvalues
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -27,8 +28,6 @@ def mock_openpyxl_package(monkeypatch: pytest.MonkeyPatch) -> Iterator[MagicMock
 
 
 def test_sanitize_field_values(mock_openpyxl_package: MagicMock) -> None:
-    from flow.record.adapter.xlsx import sanitize_fieldvalues
-
     assert list(
         sanitize_fieldvalues(
             [

{flow_record-3.21.dev3 → flow_record-3.21.dev5}/tox.ini

@@ -35,6 +35,7 @@ package = skip
 deps =
     ruff==0.9.2
 commands =
+    ruff check --fix flow tests
     ruff format flow tests
 
 [testenv:lint]