flow.record 3.20.dev1__tar.gz → 3.21__tar.gz

flow_record-3.21/.gitattributes

@@ -0,0 +1 @@
+tests/_data/** filter=lfs diff=lfs merge=lfs -text

{flow_record-3.20.dev1 → flow_record-3.21}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: flow.record
-Version: 3.20.dev1
+Version: 3.21
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
-License: Affero General Public License v3
+License-Expression: AGPL-3.0-or-later
 Project-URL: homepage, https://dissect.tools
 Project-URL: documentation, https://docs.dissect.tools/en/latest/projects/flow.record
 Project-URL: repository, https://github.com/fox-it/flow.record
@@ -11,12 +11,11 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
-Classifier: License :: OSI Approved
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Utilities
-Requires-Python: ~=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: COPYRIGHT
@@ -24,29 +23,25 @@ Requires-Dist: msgpack>=0.5.2
 Requires-Dist: tzdata; platform_system == "Windows"
 Provides-Extra: compression
 Requires-Dist: lz4; extra == "compression"
-Requires-Dist: zstandard; extra == "compression"
+Requires-Dist: zstandard; platform_python_implementation != "PyPy" and extra == "compression"
 Provides-Extra: elastic
 Requires-Dist: elasticsearch; extra == "elastic"
 Provides-Extra: geoip
 Requires-Dist: maxminddb; extra == "geoip"
 Provides-Extra: avro
-Requires-Dist: cramjam<2.8.4; (platform_python_implementation == "PyPy" and python_version == "3.9") and extra == "avro"
 Requires-Dist: fastavro[snappy]; extra == "avro"
 Provides-Extra: duckdb
-Requires-Dist: duckdb; extra == "duckdb"
-Requires-Dist: pytz; extra == "duckdb"
+Requires-Dist: duckdb; platform_python_implementation != "PyPy" and extra == "duckdb"
+Requires-Dist: pytz; platform_python_implementation != "PyPy" and extra == "duckdb"
 Provides-Extra: splunk
 Requires-Dist: httpx; extra == "splunk"
-Provides-Extra: test
-Requires-Dist: flow.record[compression]; extra == "test"
-Requires-Dist: flow.record[avro]; extra == "test"
-Requires-Dist: flow.record[elastic]; extra == "test"
-Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
-Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
-Requires-Dist: tqdm; extra == "test"
+Provides-Extra: xlsx
+Requires-Dist: openpyxl; extra == "xlsx"
 Provides-Extra: full
 Requires-Dist: flow.record[compression]; extra == "full"
 Requires-Dist: tqdm; extra == "full"
+Requires-Dist: structlog; extra == "full"
+Dynamic: license-file
 
 # flow.record
 

{flow_record-3.20.dev1 → flow_record-3.21}/examples/filesystem.py

@@ -1,10 +1,15 @@
-import os
-import stat
+from __future__ import annotations
 
-from datetime import datetime
+import stat
+from pathlib import Path
+from typing import TYPE_CHECKING
 
 from flow.record import RecordDescriptor, RecordWriter
 
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
 descriptor = """
 filesystem/unix/entry
     string path;
@@ -22,34 +27,32 @@ filesystem/unix/entry
 FilesystemFile = RecordDescriptor(descriptor)
 
 
-def hash_file(path, t):
-    f = open(path, "rb")
-    while 1:
-        d = f.read(4096)
-        if d == "":
-            break
-    f.close()
+def hash_file(path: str | Path) -> None:
+    with Path(path).open("rb") as f:
+        while True:
+            d = f.read(4096)
+            if not d:
+                break
 
 
 class FilesystemIterator:
     basepath = None
 
-    def __init__(self, basepath):
+    def __init__(self, basepath: str | None):
         self.basepath = basepath
         self.recordType = FilesystemFile
 
-    def classify(self, source, classification):
+    def classify(self, source: str, classification: str) -> None:
         self.recordType = FilesystemFile.base(_source=source, _classification=classification)
 
-    def iter(self, path):
-        path = os.path.abspath(path)
-        return self._iter(path)
+    def iter(self, path: str | Path) -> Iterator[FilesystemFile]:
+        return self._iter(Path(path).resolve())
 
-    def _iter(self, path):
-        if path.startswith("/proc"):
+    def _iter(self, path: Path) -> Iterator[FilesystemFile]:
+        if path.is_relative_to("/proc"):
             return
 
-        st = os.lstat(path)
+        st = path.lstat()
 
         abspath = path
         if self.basepath and abspath.startswith(self.basepath):
@@ -59,7 +62,7 @@ class FilesystemIterator:
 
         link = None
         if ifmt == stat.S_IFLNK:
-            link = os.readlink(path)
+            link = path.readlink()
 
         yield self.recordType(
             path=abspath,
@@ -69,20 +72,16 @@ class FilesystemIterator:
             size=st.st_size,
             uid=st.st_uid,
             gid=st.st_gid,
-            ctime=datetime.fromtimestamp(st.st_ctime),
-            mtime=datetime.fromtimestamp(st.st_mtime),
-            atime=datetime.fromtimestamp(st.st_atime),
+            ctime=st.st_ctime,
+            mtime=st.st_mtime,
+            atime=st.st_atime,
             link=link,
         )
 
         if ifmt == stat.S_IFDIR:
-            for i in os.listdir(path):
-                if i in (".", ".."):
-                    continue
-
-                fullpath = os.path.join(path, i)
-                for e in self.iter(fullpath):
-                    yield e
+            for i in path.iterdir():
+                fullpath = path.joinpath(i)
+                yield from self.iter(fullpath)
 
 
 chunk = []

{flow_record-3.20.dev1 → flow_record-3.21}/examples/passivedns.py

@@ -1,18 +1,21 @@
 #!/usr/bin/env pypy
-import record
+from __future__ import annotations
+
 import sys
-import datetime
+from datetime import datetime, timezone
 
 import net.ipv4
-
+import record
 from fileprocessing import DirectoryProcessor
 
+UTC_TIMEZONE = timezone.utc
+
 
-def ts(s):
-    return datetime.datetime.fromtimestamp(float(s))
+def ts(s: float) -> datetime:
+    return datetime.fromtimestamp(float(s), tz=UTC_TIMEZONE)
 
 
-def ip(s):
+def ip(s: str) -> net.ipv4.Address:
     return net.ipv4.Address(s)
 
 
@@ -21,7 +24,7 @@ class SeparatedFile:
     seperator = None
     format = None
 
-    def __init__(self, fp, seperator, format):
+    def __init__(self, fp: list[str], seperator: str | None, format: list[tuple]):
         self.fp = fp
         self.seperator = seperator
         self.format = format
@@ -46,7 +49,7 @@ class SeparatedFile:
             yield recordtype(**r)
 
 
-def PassiveDnsFile(fp):
+def PassiveDnsFile(fp: list[str]) -> SeparatedFile:
     return SeparatedFile(fp, "||", PASSIVEDNS_FORMAT)
 
 
@@ -63,7 +66,7 @@ PASSIVEDNS_FORMAT = [
 ]
 
 
-def main():
+def main() -> None:
     rs = record.RecordOutput(sys.stdout)
     for r in DirectoryProcessor(sys.argv[1], PassiveDnsFile, r"\.log\.gz"):
         rs.write(r)

{flow_record-3.20.dev1 → flow_record-3.21}/examples/tcpconn.py

@@ -1,8 +1,10 @@
 import random
+from datetime import datetime, timezone
 
-from datetime import datetime
 from flow import record
 
+UTC_TIMEZONE = timezone.utc
+
 descriptor = """
 network/traffic/tcp/connection
     datetime ts;
@@ -32,9 +34,9 @@ port_list = [
 
 rs = record.RecordWriter()
 
-for i in range(500):
+for _ in range(500):
     r = conn(
-        ts=datetime.now(),
+        ts=datetime.now(tz=UTC_TIMEZONE),
         src=random.choice(ip_list),
         srcport=random.choice(port_list),
         dst=random.choice(ip_list),

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/avro.py

@@ -9,6 +9,7 @@ import fastavro
 
 from flow import record
 from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.selector import make_selector
 from flow.record.utils import is_stdout
 
@@ -113,6 +114,8 @@ class AvroReader(AbstractReader):
         }
 
     def __iter__(self) -> Iterator[record.Record]:
+        ctx = get_app_context()
+        selector = self.selector
         for obj in self.reader:
             # Convert timestamp-micros fields back to datetime fields
             for field_name in self.datetime_fields:
@@ -121,7 +124,7 @@ class AvroReader(AbstractReader):
                     obj[field_name] = EPOCH + timedelta(microseconds=value)
 
             rec = self.desc.recordType(**obj)
-            if not self.selector or self.selector.match(rec):
+            if match_record_with_context(rec, selector, ctx):
                 yield rec
 
     def close(self) -> None:

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/broker.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from flow.broker import Publisher, Subscriber
+
 from flow.record.adapter import AbstractReader, AbstractWriter
 
 if TYPE_CHECKING:

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/csvfile.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import csv
 import sys
 from pathlib import Path
@@ -8,8 +9,9 @@ from typing import TYPE_CHECKING
 from flow.record import RecordDescriptor
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.base import Record, normalize_fieldname
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.selector import make_selector
-from flow.record.utils import is_stdout
+from flow.record.utils import boolean_argument, is_stdout
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -17,11 +19,12 @@ if TYPE_CHECKING:
 __usage__ = """
 Comma-separated values (CSV) adapter
 ---
-Write usage: rdump -w csvfile://[PATH]?lineterminator=[TERMINATOR]
+Write usage: rdump -w csvfile://[PATH]?lineterminator=[TERMINATOR]&header=[HEADER]
 Read usage: rdump csvfile://[PATH]?fields=[FIELDS]
 [PATH]: path to file. Leave empty or "-" to output to stdout
 
 Optional parameters:
+    [HEADER]: if set to false, it will not print the CSV header (default: true)
     [TERMINATOR]: line terminator, default is \\r\\n
     [FIELDS]: comma-separated list of CSV fields (in case of missing CSV header)
 """
@@ -34,6 +37,7 @@ class CsvfileWriter(AbstractWriter):
         fields: str | list[str] | None = None,
         exclude: str | list[str] | None = None,
         lineterminator: str = "\r\n",
+        header: str = "true",
         **kwargs,
     ):
         self.fp = None
@@ -52,13 +56,16 @@ class CsvfileWriter(AbstractWriter):
             self.fields = self.fields.split(",")
         if isinstance(self.exclude, str):
             self.exclude = self.exclude.split(",")
+        self.header = boolean_argument(header)
 
     def write(self, r: Record) -> None:
         rdict = r._asdict(fields=self.fields, exclude=self.exclude)
         if not self.desc or self.desc != r._desc:
             self.desc = r._desc
             self.writer = csv.DictWriter(self.fp, rdict, lineterminator=self.lineterminator)
-            self.writer.writeheader()
+            if self.header:
+                # Write header only if it is requested
+                self.writer.writeheader()
         self.writer.writerow(rdict)
 
     def flush(self) -> None:
@@ -84,7 +91,8 @@ class CsvfileReader(AbstractReader):
 
         self.dialect = "excel"
         if self.fp.seekable():
-            self.dialect = csv.Sniffer().sniff(self.fp.read(1024))
+            with contextlib.suppress(csv.Error):
+                self.dialect = csv.Sniffer().sniff(self.fp.read(1024))
             self.fp.seek(0)
         self.reader = csv.reader(self.fp, dialect=self.dialect)
 
@@ -107,8 +115,10 @@ class CsvfileReader(AbstractReader):
         self.fp = None
 
     def __iter__(self) -> Iterator[Record]:
+        ctx = get_app_context()
+        selector = self.selector
         for row in self.reader:
-            rdict = dict(zip(self.fields, row))
+            rdict = dict(zip(self.fields, row, strict=False))
             record = self.desc.init_from_dict(rdict)
-            if not self.selector or self.selector.match(record):
+            if match_record_with_context(record, selector, ctx):
                 yield record

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/elastic.py

@@ -4,8 +4,11 @@ import hashlib
 import logging
 import queue
 import threading
+from contextlib import suppress
 from typing import TYPE_CHECKING
 
+import urllib3
+
 try:
     import elasticsearch
     import elasticsearch.helpers
@@ -17,8 +20,10 @@ except ImportError:
 
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.base import Record, RecordDescriptor
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.fieldtypes import fieldtype_for_value
 from flow.record.jsonpacker import JsonRecordPacker
+from flow.record.utils import boolean_argument
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -72,10 +77,12 @@ class ElasticWriter(AbstractWriter):
 
         self.index = index
         self.uri = uri
-        verify_certs = str(verify_certs).lower() in ("1", "true")
-        http_compress = str(http_compress).lower() in ("1", "true")
-        self.hash_record = str(hash_record).lower() in ("1", "true")
+        verify_certs = boolean_argument(verify_certs)
+        http_compress = boolean_argument(http_compress)
+        self.hash_record = boolean_argument(hash_record)
         queue_size = int(queue_size)
+        request_timeout = int(request_timeout)
+        self.max_retries = int(max_retries)
 
         if not uri.lower().startswith(("http://", "https://")):
             uri = "http://" + uri
@@ -92,7 +99,7 @@ class ElasticWriter(AbstractWriter):
             api_key=api_key,
             request_timeout=request_timeout,
             retry_on_timeout=True,
-            max_retries=max_retries,
+            max_retries=self.max_retries,
         )
 
         self.json_packer = JsonRecordPacker()
@@ -102,8 +109,6 @@ class ElasticWriter(AbstractWriter):
 
         if not verify_certs:
             # Disable InsecureRequestWarning of urllib3, caused by the verify_certs flag.
-            import urllib3
-
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
         self.metadata_fields = {}
@@ -112,10 +117,9 @@ class ElasticWriter(AbstractWriter):
                 self.metadata_fields[arg_key[6:]] = arg_val
 
     def excepthook(self, exc: threading.ExceptHookArgs, *args, **kwargs) -> None:
-        log.error("Exception in thread: %s", exc)
         self.exception = getattr(exc, "exc_value", exc)
+        self.exception = enrich_elastic_exception(self.exception)
         self.event.set()
-        self.close()
 
     def record_to_document(self, record: Record, index: str) -> dict:
         """Convert a record to a Elasticsearch compatible document dictionary"""
@@ -168,13 +172,13 @@ class ElasticWriter(AbstractWriter):
             - https://elasticsearch-py.readthedocs.io/en/v8.17.1/helpers.html#elasticsearch.helpers.streaming_bulk
             - https://github.com/elastic/elasticsearch-py/blob/main/elasticsearch/helpers/actions.py#L362
         """
+
         for _ok, _item in elasticsearch.helpers.streaming_bulk(
             self.es,
             self.document_stream(),
             raise_on_error=True,
             raise_on_exception=True,
-            # Some settings have to be redefined because streaming_bulk does not inherit them from the self.es instance.
-            max_retries=3,
+            max_retries=self.max_retries,
         ):
             pass
 
@@ -190,13 +194,17 @@ class ElasticWriter(AbstractWriter):
         pass
 
     def close(self) -> None:
-        self.queue.put(StopIteration)
-        self.event.wait()
+        if hasattr(self, "queue"):
+            self.queue.put(StopIteration)
+
+        if hasattr(self, "event"):
+            self.event.wait()
 
         if hasattr(self, "es"):
-            self.es.close()
+            with suppress(Exception):
+                self.es.close()
 
-        if self.exception:
+        if hasattr(self, "exception") and self.exception:
             raise self.exception
 
 
@@ -216,8 +224,10 @@ class ElasticReader(AbstractReader):
         self.index = index
         self.uri = uri
         self.selector = selector
-        verify_certs = str(verify_certs).lower() in ("1", "true")
-        http_compress = str(http_compress).lower() in ("1", "true")
+        verify_certs = boolean_argument(verify_certs)
+        http_compress = boolean_argument(http_compress)
+        request_timeout = int(request_timeout)
+        max_retries = int(max_retries)
 
         if not uri.lower().startswith(("http://", "https://")):
             uri = "http://" + uri
@@ -234,11 +244,11 @@ class ElasticReader(AbstractReader):
 
         if not verify_certs:
             # Disable InsecureRequestWarning of urllib3, caused by the verify_certs flag.
-            import urllib3
-
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
     def __iter__(self) -> Iterator[Record]:
+        ctx = get_app_context()
+        selector = self.selector
         res = self.es.search(index=self.index)
         log.debug("ElasticSearch returned %u hits", res["hits"]["total"]["value"])
         for hit in res["hits"]["hits"]:
@@ -248,9 +258,38 @@ class ElasticReader(AbstractReader):
             fields = [(fieldtype_for_value(val, "string"), key) for key, val in source.items()]
             desc = RecordDescriptor("elastic/record", fields)
             obj = desc(**source)
-            if not self.selector or self.selector.match(obj):
+            if match_record_with_context(obj, selector, ctx):
                 yield obj
 
     def close(self) -> None:
         if hasattr(self, "es"):
             self.es.close()
+
+
+def enrich_elastic_exception(exception: Exception) -> Exception:
+    """Extend the exception with error information from Elastic.
+
+    Resources:
+        - https://elasticsearch-py.readthedocs.io/en/v8.17.1/exceptions.html
+    """
+    errors = set()
+    if hasattr(exception, "errors"):
+        try:
+            for error in exception.errors:
+                index_dict = error.get("index", {})
+                status = index_dict.get("status")
+                error_dict = index_dict.get("error", {})
+                error_type = error_dict.get("type")
+                error_reason = error_dict.get("reason", "")
+
+                errors.add(f"({status} {error_type} {error_reason})")
+        except Exception:
+            errors.add("unable to extend errors")
+
+    # append errors to original exception message
+    error_str = ", ".join(errors)
+    original_message = exception.args[0] if exception.args else ""
+    new_message = f"{original_message} {error_str}"
+    exception.args = (new_message, *exception.args[1:])
+
+    return exception

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/jsonfile.py

@@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, BinaryIO
 from flow import record
 from flow.record import JsonRecordPacker
 from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.fieldtypes import fieldtype_for_value
 from flow.record.selector import make_selector
-from flow.record.utils import is_stdout
+from flow.record.utils import boolean_argument, is_stdout
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -33,7 +34,7 @@ class JsonfileWriter(AbstractWriter):
     def __init__(
         self, path: str | Path | BinaryIO, indent: str | int | None = None, descriptors: bool = True, **kwargs
     ):
-        self.descriptors = str(descriptors).lower() in ("true", "1")
+        self.descriptors = boolean_argument(descriptors)
         self.fp = record.open_path_or_stream(path, "w")
         if isinstance(indent, str):
             indent = int(indent)
@@ -75,10 +76,12 @@ class JsonfileReader(AbstractReader):
         self.fp = None
 
     def __iter__(self) -> Iterator[Record]:
+        ctx = get_app_context()
+        selector = self.selector
         for line in self.fp:
             obj = self.packer.unpack(line)
             if isinstance(obj, record.Record):
-                if not self.selector or self.selector.match(obj):
+                if match_record_with_context(obj, selector, ctx):
                     yield obj
             elif isinstance(obj, record.RecordDescriptor):
                 pass
@@ -90,5 +93,5 @@ class JsonfileReader(AbstractReader):
                 ]
                 desc = record.RecordDescriptor("json/record", fields)
                 obj = desc(**jd)
-                if not self.selector or self.selector.match(obj):
+                if match_record_with_context(obj, selector, ctx):
                     yield obj

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/mongo.py

@@ -7,6 +7,7 @@ from pymongo import MongoClient
 
 from flow import record
 from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.selector import make_selector
 
 if TYPE_CHECKING:
@@ -91,6 +92,8 @@ class MongoReader(AbstractReader):
 
     def __iter__(self) -> Iterator[Record]:
         desc = None
+        ctx = get_app_context()
+        selector = self.selector
         for r in self.collection.find():
             if r["_type"] not in self.descriptors:
                 packed_desc = self.coll_descriptors.find({"name": r["_type"]})[0]["descriptor"]
@@ -106,5 +109,5 @@ class MongoReader(AbstractReader):
                     r[k] = int(r[k])
 
             obj = desc(**r)
-            if not self.selector or self.selector.match(obj):
+            if match_record_with_context(obj, selector, ctx):
                 yield obj

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/splunk.py

@@ -18,7 +18,7 @@ except ImportError:
 
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.jsonpacker import JsonRecordPacker
-from flow.record.utils import to_base64, to_bytes, to_str
+from flow.record.utils import boolean_argument, to_base64, to_bytes, to_str
 
 if TYPE_CHECKING:
     from flow.record.base import Record
@@ -35,7 +35,7 @@ Write usage: rdump -w splunk+[PROTOCOL]://[IP]:[PORT]?tag=[TAG]&token=[TOKEN]&so
 [SSL_VERIFY]: Whether to verify the server certificate when sending data over HTTPS. Defaults to True.
 """
 
-log = logging.getLogger(__package__)
+log = logging.getLogger(__name__)
 
 # Amount of records to bundle into a single request when sending data over HTTP(S).
 RECORD_BUFFER_LIMIT = 20
@@ -218,7 +218,7 @@ class SplunkWriter(AbstractWriter):
                 self.token = f"Splunk {self.token}"
 
             # Assume verify=True unless specified otherwise.
-            self.verify = str(ssl_verify).lower() not in ("0", "false")
+            self.verify = boolean_argument(ssl_verify)
             if not self.verify:
                 log.warning("Certificate verification is disabled")
 

{flow_record-3.20.dev1 → flow_record-3.21}/flow/record/adapter/sqlite.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
 from flow.record import Record, RecordDescriptor
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.base import RESERVED_FIELDS, normalize_fieldname
+from flow.record.context import get_app_context, match_record_with_context
 from flow.record.selector import Selector, make_selector
 
 if TYPE_CHECKING:
@@ -191,14 +192,16 @@ class SqliteReader(AbstractReader):
                             row[idx] = None
                         elif isinstance(value, str):
                             row[idx] = value.encode(errors="surrogateescape")
-                yield descriptor_cls.init_from_dict(dict(zip(fnames, row)))
+                yield descriptor_cls.init_from_dict(dict(zip(fnames, row, strict=False)))
 
     def __iter__(self) -> Iterator[Record]:
         """Iterate over all tables in the database and yield records."""
+        ctx = get_app_context()
+        selector = self.selector
         for table_name in self.table_names():
             self.logger.debug("Reading table: %s", table_name)
             for record in self.read_table(table_name):
-                if not self.selector or self.selector.match(record):
+                if match_record_with_context(record, selector, ctx):
                     yield record