detextive 2.0__tar.gz → 3.1__tar.gz

detextive-3.1/.gitignore

@@ -0,0 +1,7 @@
+.env
+*.so
+.*.swp
+__pycache__/
+bugs/
+build/
+dist/

{detextive-2.0 → detextive-3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detextive
-Version: 2.0
+Version: 3.1
 Summary: Detects textual content.
 Project-URL: Homepage, https://github.com/emcd/python-detextive
 Project-URL: Documentation, https://emcd.github.io/python-detextive
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Topic :: Software Development
@@ -27,7 +28,7 @@ Requires-Dist: absence~=1.1
 Requires-Dist: accretive~=4.1
 Requires-Dist: chardet
 Requires-Dist: dynadoc~=1.4
-Requires-Dist: frigid~=4.1
+Requires-Dist: frigid~=4.2
 Requires-Dist: puremagic
 Requires-Dist: typing-extensions
 Provides-Extra: all
@@ -306,6 +307,9 @@ Other Projects by This Author 🌟
 * `python-icecream-truck <https://github.com/emcd/python-icecream-truck>`_ (`icecream-truck <https://pypi.org/project/icecream-truck/>`_ on PyPI)
 
   🍦 **Flavorful Debugging** - A Python library which enhances the powerful and well-known ``icecream`` package with flavored traces, configuration hierarchies, customized outputs, ready-made recipes, and more.
+* `python-librovore <https://github.com/emcd/python-librovore>`_ (`librovore <https://pypi.org/project/librovore/>`_ on PyPI)
+
+  🐲 **Documentation Search Engine** - An intelligent documentation search and extraction tool that provides both a command-line interface for humans and an MCP (Model Context Protocol) server for AI agents. Search across Sphinx and MkDocs sites with fuzzy matching, extract clean markdown content, and integrate seamlessly with AI development workflows.
 * `python-mimeogram <https://github.com/emcd/python-mimeogram>`_ (`mimeogram <https://pypi.org/project/mimeogram/>`_ on PyPI)
 
   📨 A command-line tool for **exchanging collections of files with Large Language Models** - bundle multiple files into a single clipboard-ready document while preserving directory structure and metadata... good for code reviews, project sharing, and LLM interactions.

{detextive-2.0 → detextive-3.1}/README.rst

@@ -263,6 +263,9 @@ Other Projects by This Author 🌟
 * `python-icecream-truck <https://github.com/emcd/python-icecream-truck>`_ (`icecream-truck <https://pypi.org/project/icecream-truck/>`_ on PyPI)
 
   🍦 **Flavorful Debugging** - A Python library which enhances the powerful and well-known ``icecream`` package with flavored traces, configuration hierarchies, customized outputs, ready-made recipes, and more.
+* `python-librovore <https://github.com/emcd/python-librovore>`_ (`librovore <https://pypi.org/project/librovore/>`_ on PyPI)
+
+  🐲 **Documentation Search Engine** - An intelligent documentation search and extraction tool that provides both a command-line interface for humans and an MCP (Model Context Protocol) server for AI agents. Search across Sphinx and MkDocs sites with fuzzy matching, extract clean markdown content, and integrate seamlessly with AI development workflows.
 * `python-mimeogram <https://github.com/emcd/python-mimeogram>`_ (`mimeogram <https://pypi.org/project/mimeogram/>`_ on PyPI)
 
   📨 A command-line tool for **exchanging collections of files with Large Language Models** - bundle multiple files into a single clipboard-ready document while preserving directory structure and metadata... good for code reviews, project sharing, and LLM interactions.

{detextive-2.0 → detextive-3.1}/pyproject.toml

@@ -15,14 +15,14 @@ license = 'Apache-2.0'
 readme = { 'file' = 'README.rst', 'content-type' = 'text/x-rst' }
 requires-python = '>= 3.10'
 dependencies = [
-  'absence~=1.1',
   'accretive~=4.1',
   'chardet',
-  'dynadoc~=1.4',
-  'frigid~=4.1',
   'puremagic',
   'typing-extensions',
   # --- BEGIN: Injected by Copier ---
+  'absence~=1.1',
+  'dynadoc~=1.4',
+  'frigid~=4.2',
   # --- END: Injected by Copier ---
 ]
 classifiers = [ # https://pypi.org/classifiers
@@ -35,6 +35,7 @@ classifiers = [ # https://pypi.org/classifiers
   'Programming Language :: Python :: 3.11',
   'Programming Language :: Python :: 3.12',
   'Programming Language :: Python :: 3.13',
+  'Programming Language :: Python :: 3.14',
   'Programming Language :: Python :: Implementation :: CPython',
   'Programming Language :: Python :: Implementation :: PyPy',
   # --- END: Injected by Copier ---
@@ -105,13 +106,13 @@ strict-naming = false
 python = '3.10'
 [tool.hatch.envs.develop]
 description = ''' Development environment. '''
+builder = true
 dependencies = [
-  'Jinja2',
   'coverage[toml]',
   'detextive[all]',
   'furo',
   'isort',
-  'packaging',
+  'myst-parser',
   'pre-commit',
   'pyright',
   'pytest',
@@ -129,6 +130,7 @@ post-install-commands = [
   # --- END: Injected by Copier ---
 ]
 [tool.hatch.envs.develop.env-vars]
+PYTHONIOENCODING = 'utf-8' # TODO: Only for coverage/doctest.
 PYTHONUNBUFFERED = 'TRUE' # TODO: Only for coverage/pytest.
 # --- BEGIN: Injected by Copier ---
 # --- END: Injected by Copier ---
@@ -141,10 +143,10 @@ docsgen = [
 ]
 linters = [
   """ruff check --quiet sources documentation tests""",
+  """vibelinter check""",
   # --- BEGIN: Injected by Copier ---
   # --- END: Injected by Copier ---
   """isort --check-only --diff sources tests""",
-  """vulture""",
   """pyright sources""",
 ]
 packagers = [
@@ -186,7 +188,9 @@ python = [
     '3.11',
     '3.12',
     '3.13',
+    '3.14',
     'pypy3.10',
+    'pypy3.11',
 ]
 [tool.hatch.version]
 path = 'sources/detextive/__init__.py'
@@ -359,6 +363,10 @@ directory = 'repair'
 name = 'Repairs'
 showcontent = true
 
+[tool.vibelinter]
+context = 3
+exclude_paths = [ '.auxiliary/**', '.venv/**', 'tests/**',  ]
+
 [tool.vulture]
 paths = [ '.auxiliary/configuration/vulturefood.py', 'sources' ]
 min_confidence = 60

{detextive-2.0 → detextive-3.1}/sources/detextive/__/imports.py

@@ -24,8 +24,10 @@
 
 import collections.abc as   cabc
 import                      codecs
+import contextlib as        ctxl
 import dataclasses as       dcls
 import                      enum
+import functools as         funct
 import                      locale
 import                      mimetypes
 import                      os
@@ -36,11 +38,13 @@ import                      unicodedata
 from pathlib import Path
 
 import accretive as         accret
-import dynadoc as           ddoc
-import frigid as            immut
 import typing_extensions as typx
 
-from absence import Absential, absent, is_absent
+# --- BEGIN: Injected by Copier ---
+import dynadoc as           ddoc
+import frigid as            immut
+# --- END: Injected by Copier ---
 
 # --- BEGIN: Injected by Copier ---
+from absence import Absential, absent, is_absent
 # --- END: Injected by Copier ---

{detextive-2.0 → detextive-3.1}/sources/detextive/__/nomina.py

@@ -30,8 +30,3 @@ PositionalArguments: __.typx.TypeAlias = __.cabc.Sequence[ __.typx.Any ]
 
 
 package_name = __name__.split( '.', maxsplit = 1 )[ 0 ]
-
-
-def is_public_identifier( name: str ) -> bool:
-    ''' Is Python identifier public? '''
-    return not name.startswith( '_' )

{detextive-2.0 → detextive-3.1}/sources/detextive/__init__.py

@@ -37,7 +37,7 @@ from . import exceptions
 # --- END: Injected by Copier ---
 
 
-__version__ = '2.0'
+__version__ = '3.1'
 
 
 __.immut.finalize_module( __name__, recursive = True )

detextive-3.1/sources/detextive/charsets.py

@@ -0,0 +1,201 @@
+# vim: set filetype=python fileencoding=utf-8:
+# -*- coding: utf-8 -*-
+
+#============================================================================#
+#                                                                            #
+#  Licensed under the Apache License, Version 2.0 (the "License");           #
+#  you may not use this file except in compliance with the License.          #
+#  You may obtain a copy of the License at                                   #
+#                                                                            #
+#      http://www.apache.org/licenses/LICENSE-2.0                            #
+#                                                                            #
+#  Unless required by applicable law or agreed to in writing, software       #
+#  distributed under the License is distributed on an "AS IS" BASIS,         #
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
+#  See the License for the specific language governing permissions and       #
+#  limitations under the License.                                            #
+#                                                                            #
+#============================================================================#
+
+
+''' Management of bytes array decoding via trial character sets. '''
+
+
+from . import __
+from . import core as _core
+from . import exceptions as _exceptions
+from . import nomina as _nomina
+
+from .core import ( # isort: skip
+    BEHAVIORS_DEFAULT as        _BEHAVIORS_DEFAULT,
+    BehaviorTristate as         _BehaviorTristate,
+    Behaviors as                _Behaviors,
+    CharsetResult as            _CharsetResult,
+    CodecSpecifiers as          _CodecSpecifiers,
+)
+
+
+def attempt_decodes( # noqa: PLR0913
+    content: _nomina.Content, /, *,
+    behaviors: _Behaviors = _BEHAVIORS_DEFAULT,
+    inference: __.Absential[ str ] = __.absent,
+    supplement: __.Absential[ str ] = __.absent,
+    location: __.Absential[ _nomina.Location ] = __.absent,
+    validator: __.Absential[
+        __.cabc.Callable[ [ str, _CharsetResult ], None ]
+    ] = __.absent,
+) -> tuple[ str, _CharsetResult ]:
+    ''' Attempts to decode content with various character sets.
+
+        Will try character sets in the order specified by the trial codecs
+        listed on the behaviors object.
+    '''
+    confidence = _core.confidence_from_bytes_quantity(
+        content, behaviors = behaviors )
+    trial_codecs = _collect_trial_codecs(
+        content,
+        behaviors = behaviors,
+        inference = inference,
+        supplement = supplement )
+    trials: set[ str ] = set( )
+    for trial_codec in trial_codecs:
+        try: text = content.decode(
+            trial_codec, errors = behaviors.on_decode_error )
+        except UnicodeDecodeError: continue
+        finally: trials.add( trial_codec )
+        result = _CharsetResult(
+            charset = normalize_charset_for_content( content, trial_codec ),
+            confidence = confidence )
+        if not __.is_absent( validator ):
+            try: validator( text, result )
+            except _exceptions.TextInvalidity: continue
+        return text, result
+    raise _exceptions.ContentDecodeFailure(
+        charset = tuple( trials ), location = location )
+
+
+def discover_os_charset_default( ) -> str:
+    ''' Discovers default character set encoding from operating system. '''
+    discoverer = getattr(
+        __.locale, 'getencoding', __.sys.getfilesystemencoding )
+    return normalize_charset( discoverer( ) )
+
+
+def normalize_charset( charset: str, bom_cognizant: bool = False ) -> str:
+    ''' Normalizes character set encoding names. '''
+    charset_ = __.codecs.lookup( charset ).name
+    if bom_cognizant and charset_ == 'utf-8': return 'utf-8-sig'
+    return charset_
+
+
+def normalize_charset_for_content(
+    content: _nomina.Content, charset: str
+) -> str:
+    ''' Normalizes charset reporting based on byte-order mark provenance. '''
+    charset_ = normalize_charset( charset )
+    bom_charset = _discover_utf_bom_charset( content )
+    if charset_ in ( 'utf-8', 'utf-8-sig' ):
+        if bom_charset == 'utf-8-sig': return 'utf-8-sig'
+        return 'utf-8'
+    for family in ( 'utf-16', 'utf-32' ):
+        if not charset_.startswith( family ): continue
+        if bom_charset in ( f"{family}-le", f"{family}-be" ):
+            return family
+        return charset_
+    return charset_
+
+
+def trial_decode_as_confident( # noqa: PLR0913
+    content: _nomina.Content, /, *,
+    behaviors: _Behaviors = _BEHAVIORS_DEFAULT,
+    inference: __.Absential[ str ] = __.absent,
+    confidence: float = 0.0,
+    supplement: __.Absential[ str ] = __.absent,
+    location: __.Absential[ _nomina.Location ] = __.absent,
+) -> _CharsetResult:
+    ''' Performs trial decode of content.
+
+        Considers desired trial decode behavior and detection confidence.
+    '''
+    nomargs: __.NominativeArguments = dict(
+        behaviors = behaviors,
+        inference = inference,
+        supplement = supplement,
+        location = location )
+    should_decode = False
+    match behaviors.trial_decode:
+        case _BehaviorTristate.Always: should_decode = True
+        case _BehaviorTristate.AsNeeded:
+            should_decode = confidence < behaviors.trial_decode_confidence
+        case _BehaviorTristate.Never: pass
+    if should_decode:
+        _, result = attempt_decodes( content, **nomargs )
+        return result
+    if __.is_absent( inference ):
+        raise _exceptions.CharsetDetectFailure( location = location )
+    return _CharsetResult( charset = inference, confidence = confidence )
+
+
+def _collect_trial_codecs(
+    content: _nomina.Content, /, *,
+    behaviors: _Behaviors,
+    inference: __.Absential[ str ],
+    supplement: __.Absential[ str ],
+) -> tuple[ str, ... ]:
+    codecs: list[ str ] = [ ]  # No set needed; this candidate list is tiny.
+    for codec in behaviors.trial_codecs:
+        charset = _resolve_trial_codec(
+            codec, inference = inference, supplement = supplement )
+        if __.is_absent( charset ): continue
+        charset = normalize_charset( charset )
+        if _is_ambiguous_utf_trial( content, charset, behaviors ): continue
+        if behaviors.remove_bom and charset == 'utf-8': charset = 'utf-8-sig'
+        if charset in codecs: continue
+        codecs.append( charset )
+    return tuple( codecs )
+
+
+def _discover_utf_bom_charset(
+    content: _nomina.Content
+) -> __.typx.Optional[ str ]:
+    # Must check UTF-32 markers first, since they prefix-match UTF-16 markers.
+    if content.startswith( __.codecs.BOM_UTF32_LE ): return 'utf-32-le'
+    if content.startswith( __.codecs.BOM_UTF32_BE ): return 'utf-32-be'
+    if content.startswith( __.codecs.BOM_UTF8 ): return 'utf-8-sig'
+    if content.startswith( __.codecs.BOM_UTF16_LE ): return 'utf-16-le'
+    if content.startswith( __.codecs.BOM_UTF16_BE ): return 'utf-16-be'
+    return None
+
+
+def _is_ambiguous_utf_trial(
+    content: _nomina.Content, charset: str, behaviors: _Behaviors
+) -> bool:
+    if not behaviors.utf_16_32_requires_byte_order: return False
+    match charset:
+        case 'utf-16':
+            bom_charset = _discover_utf_bom_charset( content )
+            return bom_charset not in ( 'utf-16-le', 'utf-16-be' )
+        case 'utf-32':
+            bom_charset = _discover_utf_bom_charset( content )
+            return bom_charset not in ( 'utf-32-le', 'utf-32-be' )
+        case _: return False
+
+
+def _resolve_trial_codec(
+    codec: __.typx.Any, /, *,
+    inference: __.Absential[ str ],
+    supplement: __.Absential[ str ],
+) -> __.Absential[ str ]:
+    charset: __.Absential[ str ] = __.absent
+    match codec:
+        case _CodecSpecifiers.FromInference:
+            if not __.is_absent( inference ): charset = inference
+        case _CodecSpecifiers.OsDefault:
+            charset = discover_os_charset_default( )
+        case _CodecSpecifiers.PythonDefault:
+            charset = __.locale.getpreferredencoding( )
+        case _CodecSpecifiers.UserSupplement:
+            if not __.is_absent( supplement ): charset = supplement
+        case str( ): charset = codec
+        case _: pass
+    return charset

{detextive-2.0 → detextive-3.1}/sources/detextive/core.py

@@ -22,15 +22,10 @@
 
 
 from . import __
+from . import exceptions as _exceptions
 from . import nomina as _nomina
 
 
-_STANDARD_CHARSET_PROMOTIONS = (
-    ( 'ascii', 'utf-8-sig' ),
-    ( 'utf-8', 'utf-8-sig' ),
-)
-
-
 CHARSET_DEFAULT = 'utf-8'
 MIMETYPE_DEFAULT = 'application/octet-stream'
 
@@ -68,9 +63,9 @@ class Behaviors( __.immut.DataclassObject ):
             ''' Minimum number of bytes for full detection confidence. ''' ),
     ] = 1024
     charset_detect: __.typx.Annotated[
-        BehaviorTristate,
-        __.ddoc.Doc( ''' When to detect charset from content. ''' ),
-    ] = BehaviorTristate.AsNeeded
+        bool,
+        __.ddoc.Doc( ''' Whether to detect charset from content. ''' ),
+    ] = True
     charset_detectors_order: __.typx.Annotated[
         __.cabc.Sequence[ str ],
         __.ddoc.Doc(
@@ -80,20 +75,10 @@ class Behaviors( __.immut.DataclassObject ):
         DetectFailureActions,
         __.ddoc.Doc( ''' Action to take on charset detection failure. ''' ),
     ] = DetectFailureActions.Default
-    charset_promotions: __.typx.Annotated[
-        __.cabc.Mapping[ str, str ],
-        __.ddoc.Doc(
-            ''' Which detected charsets to promote to other charsets.
-
-                E.g., 7-bit ASCII to UTF-8.
-            ''' ),
-    ] = __.dcls.field(
-        default_factory = (
-            lambda: __.immut.Dictionary( _STANDARD_CHARSET_PROMOTIONS ) ) )
     mimetype_detect: __.typx.Annotated[
-        BehaviorTristate,
-        __.ddoc.Doc( ''' When to detect MIME type from content. ''' ),
-    ] = BehaviorTristate.AsNeeded
+        bool,
+        __.ddoc.Doc( ''' Whether to detect MIME type from content. ''' ),
+    ] = True
     mimetype_detectors_order: __.typx.Annotated[
         __.cabc.Sequence[ str ],
         __.ddoc.Doc(
@@ -114,6 +99,9 @@ class Behaviors( __.immut.DataclassObject ):
                 'codecs' module.
             ''' ),
     ] = 'strict'
+    remove_bom: __.typx.Annotated[
+        bool, __.ddoc.Doc( ''' Remove byte-ordering mark? ''' )
+    ] = True
     text_validate: __.typx.Annotated[
         BehaviorTristate,
         __.ddoc.Doc( ''' When to validate text. ''' ),
@@ -125,7 +113,13 @@ class Behaviors( __.immut.DataclassObject ):
     trial_codecs: __.typx.Annotated[
         __.cabc.Sequence[ str | CodecSpecifiers ],
         __.ddoc.Doc( ''' Sequence of codec names or specifiers. ''' ),
-    ] = ( CodecSpecifiers.FromInference, CodecSpecifiers.UserSupplement )
+    ] = (
+        CodecSpecifiers.UserSupplement,
+        'utf-8',
+        CodecSpecifiers.FromInference,
+        CodecSpecifiers.OsDefault,
+        CodecSpecifiers.PythonDefault,
+    )
     trial_decode: __.typx.Annotated[
         BehaviorTristate,
         __.ddoc.Doc(
@@ -134,6 +128,23 @@ class Behaviors( __.immut.DataclassObject ):
     trial_decode_confidence: __.typx.Annotated[
         float, __.ddoc.Doc( ''' Minimum confidence to skip trial decode. ''')
     ] = 0.80
+    utf_16_32_requires_byte_order: __.typx.Annotated[
+        bool,
+        __.ddoc.Doc(
+            ''' Require explicit byte order for BOM-less generic UTF-16/32? '''
+        ),
+    ] = False
+
+    def __post_init__( self ) -> None:
+        if not isinstance( self.charset_detect, bool ):
+            raise _exceptions.BehaviorsInvalidity(
+                'charset_detect', 'a boolean' )
+        if not isinstance( self.mimetype_detect, bool ):
+            raise _exceptions.BehaviorsInvalidity(
+                'mimetype_detect', 'a boolean' )
+        if not isinstance( self.utf_16_32_requires_byte_order, bool ):
+            raise _exceptions.BehaviorsInvalidity(
+                'utf_16_32_requires_byte_order', 'a boolean' )
 
 
 BehaviorsArgument: __.typx.TypeAlias = __.typx.Annotated[