commodutil 4.0.1__tar.gz → 4.2.0__tar.gz

{commodutil-4.0.1 → commodutil-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: commodutil
-Version: 4.0.1
+Version: 4.2.0
 Summary: common commodity/oil analytics utils
 Author-email: aeorxc <author@example.com>
 Project-URL: Homepage, https://dev.azure.com/RWEST-MFI-TE/Oil/_git/commodutil

{commodutil-4.0.1 → commodutil-4.2.0}/commodutil/standards/commodities.py

@@ -21,6 +21,9 @@ commodutil's commodity lists.
 
 from __future__ import annotations
 
+import re
+from typing import Optional
+
 
 COMMODITY_KEYWORDS = [
     ("Brent", "Crude Oil", ["brent"]),
@@ -29,7 +32,20 @@ COMMODITY_KEYWORDS = [
     # NB: 'Natural Gasoline' MUST come before 'Natural Gas' — the substring
     # "natural gas" is contained in "natural gasoline" and would otherwise win.
     ("Natural Gasoline", "NGL", ["natural gasoline"]),
-    ("Natural Gas", "Natural Gas", ["natural gas", "nat gas", "natgas"]),
+    (
+        "Natural Gas",
+        "Natural Gas",
+        [
+            "natural gas",
+            "nat gas",
+            "natgas",
+            "jkm",
+            "ttf",
+            "nbp",
+            "henry hub",
+            "henry",
+        ],
+    ),
     ("Jet", "Refined Products", ["jet fuel", "jet"]),
     ("Diesel", "Refined Products", ["diesel", "ulsd", "gasoil", "heating oil"]),
     ("Gasoline", "Refined Products", ["gasoline", "rbob", "cbob", "mogas", "eurobob"]),
@@ -154,48 +170,92 @@ def normalize_commodity_for_conversion(commodity: Optional[str]) -> Optional[str
     return text.replace(" ", "_")
 
 
+_EXCHANGE_SYMBOL_TOKEN_SPLIT = re.compile(r"[_:.\-\s/]+")
+
+# Token sets for ``infer_commodity_from_exchange_symbol`` — matched via
+# whole-token equality after splitting on ``_ : . - whitespace /``. Token sets
+# are checked in order; first hit wins. All tokens are lower-case.
+#
+# We deliberately AVOID short ambiguous 2-char tokens (``cl`` / ``rb`` /
+# ``ho`` / ``ng``) that used to live here as substrings — they caused
+# false-positives across real feed-prefixed identifiers (e.g.
+# ``Ice_ClearedGas:JKM`` → 'crude' via ``cl``; ``Singapore_Spot:Naphtha`` →
+# 'natgas' via ``ng``; ``Hong_Kong:HKD`` → 'gasoil' via ``ho``).
+#
+# Natgas tokens include LNG/European/US hub acronyms (jkm/ttf/nbp/hh/henry)
+# so feed-prefixed gas symbols classify correctly instead of falling through
+# to ``cl``-style false matches.
+_EXCHANGE_SYMBOL_TOKENS: list[tuple[str, frozenset[str]]] = [
+    # Order matters only when token sets could overlap; they don't here.
+    ("crude", frozenset({"wti", "brent", "brn"})),
+    ("gasoline", frozenset({"rbob", "gasoline", "mogas"})),
+    ("gasoil", frozenset({"gasoil", "diesel", "heating"})),
+    (
+        "natgas",
+        frozenset(
+            {
+                "natural",
+                "natgas",
+                "jkm",
+                "ttf",
+                "nbp",
+                "hh",
+                "henry",
+            }
+        ),
+    ),
+]
+
+
 def infer_commodity_from_exchange_symbol(symbol: Optional[str]) -> Optional[str]:
-    """Infer commodity from a raw exchange symbol name (loose substring match).
+    """Infer commodity from a raw exchange symbol name (token-based match).
 
     Last-resort fallback when description-based ``infer_commodity_and_group``
     fails (no Description, or Description didn't match COMMODITY_KEYWORDS).
-    Mirrors legacy substring-fallback logic that lived inline in
-    ``pyoilprice.conversion`` and then in ``curvemetadata.taxonomy``. Patterns
-    are SHORT substrings (cl, rb, ho, ng) matched anywhere in the input —
-    ``"close_value"`` will match ``cl`` and return ``"crude"``. This is
-    acceptable on raw exchange-symbol identifiers (which are short and
-    predictable) but **UNSAFE on free-text inputs** — use
-    ``infer_commodity_and_group()`` for descriptions or product names.
+    The symbol is lower-cased and split on ``_ : . - whitespace /`` into
+    tokens; commodity is inferred via WHOLE-TOKEN equality against
+    ``_EXCHANGE_SYMBOL_TOKENS``.
+
+    This replaces an older substring-based implementation that used short
+    2-char tokens (``cl`` / ``rb`` / ``ho`` / ``ng``); those caused
+    false-positives on feed-prefixed identifiers — e.g.
+    ``"Ice_ClearedGas:JKM"`` matched ``cl`` and returned 'crude' instead of
+    'natgas'; ``"Singapore_Spot:Naphtha"`` matched ``ng`` and returned
+    'natgas' instead of 'naphtha' (no match). The token-equality rewrite
+    eliminates the substring leak.
 
     Returns:
         Canonical commodity name ('crude' / 'gasoline' / 'gasoil' / 'natgas')
-        or None if no match.
+        or None if no match. Symbols that don't match any known token return
+        ``None`` — callers should treat ``None`` as "skip / unknown" rather
+        than guess.
 
-    Examples (raw exchange symbols only):
-        >>> infer_commodity_from_exchange_symbol("CL_Mar25")
-        'crude'
+    Examples:
         >>> infer_commodity_from_exchange_symbol("ICE_EuroFutures:BRN")
         'crude'
         >>> infer_commodity_from_exchange_symbol("RBOB_Apr25")
         'gasoline'
-        >>> infer_commodity_from_exchange_symbol("HO_May25")
-        'gasoil'
-        >>> infer_commodity_from_exchange_symbol("NG_Jun25")
+        >>> infer_commodity_from_exchange_symbol("Ice_ClearedGas:JKM")
+        'natgas'
+        >>> infer_commodity_from_exchange_symbol("Ice_ClearedGas:TTF")
         'natgas'
+        >>> infer_commodity_from_exchange_symbol("Singapore_Spot:Naphtha") is None
+        True
+        >>> infer_commodity_from_exchange_symbol("Hong_Kong:HKD") is None
+        True
+        >>> infer_commodity_from_exchange_symbol("LME_Copper:Long") is None
+        True
         >>> infer_commodity_from_exchange_symbol("XYZ_Spot") is None
         True
     """
     if not symbol:
         return None
-    s = str(symbol).lower()
-    if any(x in s for x in ["cl", "wti", "brent", "brn"]):
-        return "crude"
-    if any(x in s for x in ["rb", "gasoline", "mogas"]):
-        return "gasoline"
-    if any(x in s for x in ["ho", "diesel", "gasoil"]):
-        return "gasoil"
-    if any(x in s for x in ["ng", "natural"]):
-        return "natgas"
+    tokens = {t for t in _EXCHANGE_SYMBOL_TOKEN_SPLIT.split(str(symbol).lower()) if t}
+    if not tokens:
+        return None
+    for commodity, token_set in _EXCHANGE_SYMBOL_TOKENS:
+        if tokens & token_set:
+            return commodity
     return None
 
 

{commodutil-4.0.1 → commodutil-4.2.0}/commodutil.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: commodutil
-Version: 4.0.1
+Version: 4.2.0
 Summary: common commodity/oil analytics utils
 Author-email: aeorxc <author@example.com>
 Project-URL: Homepage, https://dev.azure.com/RWEST-MFI-TE/Oil/_git/commodutil

{commodutil-4.0.1 → commodutil-4.2.0}/tests/test_standards_commodities.py

@@ -141,27 +141,101 @@ def test_normalize_commodity_for_conversion_unknown_falls_back_to_slug():
 @pytest.mark.parametrize(
     "symbol,expected",
     [
-        ("CL_Mar25", "crude"),
+        # Crude — token matches on wti / brent / brn. Raw NYMEX `cl` is
+        # NOT a token any more (false-positive risk in 'Cleared' / 'Close');
+        # the canonical Brent token is 'brn' and full forms hit free-text.
         ("ICE_EuroFutures:BRN", "crude"),
         ("brent forward", "crude"),
         ("wti", "crude"),
-        ("CME_NymexFutures_EOD:RB", "gasoline"),
+        # Gasoline — token matches on rbob / gasoline / mogas. `rb` alone
+        # is no longer a token (false positives in 'Carbon').
         ("RBOB_Apr25", "gasoline"),
-        ("HO_May25", "gasoil"),
+        # Gasoil — token matches on gasoil / diesel / heating. `ho` alone
+        # is no longer a token (false positives in 'Hong' / 'HKD').
         ("diesel europe", "gasoil"),
-        ("NG_Jun25", "natgas"),
+        ("heating oil", "gasoil"),
+        # Natgas — natural / natgas / hub acronyms. `ng` alone is no
+        # longer a token (false positives in 'Long' / 'Naphtha').
         ("natural gas", "natgas"),
+        ("JKM_M1", "natgas"),
+        ("ICE_TTF", "natgas"),
+        # No-match — returns None and the caller (e.g. pyoilprice) should
+        # skip with a WARN rather than guess.
         ("XYZ_Spot", None),
         (None, None),
         ("", None),
+        # Dropped legacy 2-char tokens — these now return None.
+        ("CL_Mar25", None),
+        ("CME_NymexFutures_EOD:RB", None),
+        ("HO_May25", None),
+        ("NG_Jun25", None),
     ],
 )
 def test_infer_commodity_from_exchange_symbol(symbol, expected):
     assert infer_commodity_from_exchange_symbol(symbol) == expected
 
 
-def test_infer_commodity_from_exchange_symbol_loose_match_documented():
-    # INTENTIONAL loose-match behaviour: "close_value" contains "cl" so the
-    # function returns "crude". This is the documented short-substring
-    # fallback for raw exchange symbols (NOT free-form text).
-    assert infer_commodity_from_exchange_symbol("close_value") == "crude"
+def test_infer_commodity_from_exchange_symbol_no_loose_substring_match():
+    # GUARD against the old substring-based behaviour: previously
+    # ``"close_value"`` contained the substring "cl" and the function
+    # returned "crude". The token-based rewrite splits the symbol and
+    # matches whole tokens only — "close" is not "cl" and we get None.
+    # The docstring previously called this loose match "INTENTIONAL"; it
+    # caused false-positives across feed-prefixed identifiers
+    # (JKM/TTF/Naphtha/HKD/Copper) and is now considered a bug.
+    assert infer_commodity_from_exchange_symbol("close_value") is None
+
+
+# Regression suite — these symbols previously misclassified under the
+# substring-based implementation. See task #68 diagnosis + pyoilprice
+# PR #20928 (JKM unit-conversion bug).
+@pytest.mark.parametrize(
+    "symbol,expected",
+    [
+        # Gas hubs that previously matched 'cl' (ClearedGas) -> 'crude'.
+        # Token-based: 'jkm' / 'ttf' / 'nbp' / 'hh' tokens hit 'natgas'.
+        ("Ice_ClearedGas:JKM", "natgas"),
+        ("Ice_ClearedGas:TTF", "natgas"),
+        ("Ice_ClearedGas:NBP", "natgas"),
+        ("Ice_ClearedGas:HH", "natgas"),
+        ("Ice_ClearedGas:Henry", "natgas"),
+        # 'Naphtha' contains 'ng' substring; previously matched 'natgas'.
+        # Token-based: 'naphtha' is its own token, not 'ng' -> None.
+        ("Singapore_Spot:Naphtha", None),
+        # 'HKD' contains 'ho' substring; previously matched 'gasoil'.
+        ("Hong_Kong:HKD", None),
+        # 'Carbon' contains 'rb' substring; previously matched 'gasoline'.
+        ("Carbon:EUA", None),
+        # 'Long' contains 'ng' substring; previously matched 'natgas'.
+        ("LME_Copper:Long", None),
+    ],
+)
+def test_infer_commodity_from_exchange_symbol_regression_no_substring_leak(
+    symbol, expected
+):
+    assert infer_commodity_from_exchange_symbol(symbol) == expected
+
+
+def test_infer_commodity_from_exchange_symbol_dropped_2char_tokens():
+    """The 2-char ambiguous tokens cl/rb/ho/ng must NOT match when embedded
+    in larger words. The legacy substring behaviour was the root cause of
+    all the regressions above; this test pins the removal."""
+    # 'cl' inside 'Cleared' / 'Close' must not return 'crude'.
+    assert infer_commodity_from_exchange_symbol("Cleared") is None
+    assert infer_commodity_from_exchange_symbol("close") is None
+    # 'rb' inside 'Carbon' must not return 'gasoline'.
+    assert infer_commodity_from_exchange_symbol("Carbon") is None
+    # 'ho' inside 'HKD' / 'Hong' must not return 'gasoil'.
+    assert infer_commodity_from_exchange_symbol("Hong") is None
+    # 'ng' inside 'Long' / 'Naphtha' must not return 'natgas'.
+    assert infer_commodity_from_exchange_symbol("Long") is None
+    assert infer_commodity_from_exchange_symbol("Naphtha") is None
+
+
+def test_infer_commodity_from_exchange_symbol_token_separators():
+    """Tokeniser must split on _ : . - whitespace and /."""
+    assert infer_commodity_from_exchange_symbol("ICE:BRN") == "crude"
+    assert infer_commodity_from_exchange_symbol("ICE.BRN") == "crude"
+    assert infer_commodity_from_exchange_symbol("ICE-BRN") == "crude"
+    assert infer_commodity_from_exchange_symbol("ICE BRN") == "crude"
+    assert infer_commodity_from_exchange_symbol("ICE/BRN") == "crude"