data-manipulation 0.50__tar.gz → 0.53__tar.gz

{data_manipulation-0.50 → data_manipulation-0.53}/.gitignore

@@ -51,15 +51,15 @@ coverage.xml
 # Django stuff:
 *.log
 
-# Sphinx documentation
-# docs/_build/
-!docs/build
+# MkDocs documentation
+/site/
 
 # PyBuilder
 target/
 
 # DotEnv configuration
 .env
+.pypirc
 
 # Database
 *.db
@@ -93,7 +93,8 @@ target/
 .mypy_cache/
 
 # custom
+.opencode
+.specstory
 build.sh
-install_packages.sh
 docs/README*
-.opencode
+install_packages.sh

data_manipulation-0.53/CHANGELOG.md

@@ -0,0 +1,81 @@
+# Changelog
+
+## [Unreleased]
+
+### Removed
+
+- **`pandas_.clean_none()`** — function has been removed.
+  Migrate to pandas built-ins:
+  ```python
+  df = df.replace(r"^\s*$", np.nan, regex=True)
+  df = df.where(pd.notnull(df), None)
+  ```
+
+- **`pandas_.config_pandas_display()`** — function has been removed.
+  Set display options directly:
+  ```python
+  pd.set_option("display.max_columns", 500)
+  pd.set_option("display.max_colwidth", 500)
+  pd.set_option("display.expand_frame_repr", True)
+  ```
+
+- **`prometheus_` module** — was an empty placeholder; removed from
+  the public API and from `__init__` imports.
+
+### Changed
+
+- **`base.get_none_variation()`** — now returns a `frozenset` of 7
+  lowercase canonical strings instead of a 72-item list of case
+  permutations. Use `.casefold()` for membership checks:
+  ```python
+  # before
+  if value in get_none_variation():
+      ...
+  # after
+  if isinstance(value, str) and value.casefold() in get_none_variation():
+      ...
+  ```
+
+- **`pyspark_.group_count()`** — the `percent` column is now computed
+  as a native Spark column expression instead of a Python UDF.
+  Behaviour is identical; performance improves on large datasets.
+
+- **`pyspark_.column_into_list()`** — raises `ValueError` (with the
+  list of available columns) when the requested column does not exist.
+  Previously returned `None` silently.
+
+- **`pandas_.print_dataframe_overview()`** — now returns a
+  `Dict[str, Dict[str, Any]]` of per-column stats (`unique`,
+  `null_count`, `value_counts`). Still prints to stdout as before;
+  existing call sites are unaffected.
+
+- **`pandas_.compare_dataframes()`** — now returns a comparison
+  summary dict (`same_length`, `df1_length`, `df2_length`, `columns`).
+  Still prints to stdout as before; existing call sites are unaffected.
+
+- **`pyspark_` module** — all `print()` calls replaced with
+  `logger.info()` / `logger.debug()`. Output is now routed through
+  the standard logging pipeline and can be silenced or redirected.
+
+- **`sqlalchemy_.create_sqlalchemy_engine()`** — duplicate
+  `if/elif` timeout branches (both setting the same value) consolidated
+  into a single condition.
+
+- **`django_.django_validate_phone()`** — now has an explicit
+  `return None` on failure instead of a bare `return`.
+
+### Fixed (previous pass — reference)
+
+- `pyspark_.columns_statistics()` — always-true condition in empty-column
+  detection (third clause was truthy for any non-empty string).
+- `pandas_.dtypes_dictionary()` — `.iteritems()` removed in pandas 2.0;
+  replaced with `.items()`.
+- Bare `except:` replaced with `except Exception:` in `pandas_` and
+  `django_`.
+- `base.string_boolean_to_int()` — removed (used `distutils`, dropped
+  in Python 3.12).
+- `flask_.py` — deleted (was entirely commented-out dead code).
+- `beautifulsoup_.preprocess()` — fixed mixed tabs/spaces and invalid
+  `\s` escape sequences.
+- All modules now guard third-party imports with `try/except`; the
+  package imports cleanly even when optional dependencies are missing.

{data_manipulation-0.50 → data_manipulation-0.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data_manipulation
-Version: 0.50
+Version: 0.53
 Summary: Powerful data manipulation
 Author-email: Shawn Ng <shawn.coding.acc@gmail.com>
 License-Expression: BSD-3-Clause
@@ -43,6 +43,7 @@ Dynamic: license-file
   - [Where to get it](#where-to-get-it)
   - [Dependencies](#dependencies)
   - [License](#license)
+  - [Maintainer Release](#maintainer-release)
   - [Getting Help](#getting-help)
   - [Discussion and Development](#discussion-and-development)
   - [Contributing to data manipulation](#contributing-to-data-manipulation)
@@ -69,6 +70,10 @@ The dependencies will be installed automatically along with this package, using
 
 [BSD 3](LICENSE)
 
+## Maintainer Release
+
+Release and deployment steps are documented in [RELEASE.md](RELEASE.md).
+
 ## Getting Help
 
 For usage questions, the best place to go to is StackOverflow.

{data_manipulation-0.50 → data_manipulation-0.53}/README.md

@@ -5,6 +5,7 @@
   - [Where to get it](#where-to-get-it)
   - [Dependencies](#dependencies)
   - [License](#license)
+  - [Maintainer Release](#maintainer-release)
   - [Getting Help](#getting-help)
   - [Discussion and Development](#discussion-and-development)
   - [Contributing to data manipulation](#contributing-to-data-manipulation)
@@ -31,6 +32,10 @@ The dependencies will be installed automatically along with this package, using
 
 [BSD 3](LICENSE)
 
+## Maintainer Release
+
+Release and deployment steps are documented in [RELEASE.md](RELEASE.md).
+
 ## Getting Help
 
 For usage questions, the best place to go to is StackOverflow.

data_manipulation-0.53/RELEASE.md

@@ -0,0 +1,84 @@
+# Release Guide
+
+This guide is for maintainers publishing documentation to GitHub Pages and
+package artifacts to PyPI.
+
+## Preconditions
+
+- Work from a clean `master` branch.
+- Release versions are derived from git tags by `setuptools-scm`.
+- Never build or upload release artifacts from a dirty worktree.
+- Keep PyPI credentials out of git.
+
+Check the release state:
+
+```bash
+git checkout master
+git pull --ff-only origin master
+git status
+```
+
+`git status` should report no changes before continuing.
+
+## Tag The Release
+
+Use the next release version, for example `0.50`:
+
+```bash
+git tag -a 0.50 -m "Release 0.50"
+git push origin master
+git push origin 0.50
+```
+
+The tag must point at the exact commit being released.
+
+## Publish Documentation
+
+Build the MkDocs documentation:
+
+```bash
+mkdocs build
+```
+
+Publish the generated HTML to GitHub Pages:
+
+```bash
+mkdocs gh-deploy
+```
+
+This publishes the contents of `site/` to the `gh-pages` branch.
+
+## Publish To PyPI
+
+Build and validate the distribution artifacts:
+
+```bash
+./setup.sh cleanup
+```
+
+Upload the artifacts:
+
+```bash
+./setup.sh upload
+```
+
+When prompted by `twine`, enter the PyPI API token. The token should start with
+`pypi-`. It will not be displayed while typing or pasting.
+
+## Verify The Release
+
+Install the published package in a fresh environment and confirm the version:
+
+```bash
+python3 -m pip install --upgrade data-manipulation
+python3 -c "import data_manipulation; print(data_manipulation.__version__)"
+```
+
+## Security Notes
+
+- Do not commit PyPI API tokens, passwords, or `.pypirc` files with credentials.
+- Do not add tokens to shell history as command-line arguments.
+- Prefer entering the PyPI token at the `twine` prompt or using a local secret
+  store.
+- If a token is accidentally committed or exposed, revoke it in PyPI immediately
+  and create a new one.

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/__init__.py

@@ -4,14 +4,12 @@ from data_manipulation import (
     boto3_,
     cryptography_,
     django_,
-    flask_,
     geopandas_,
     kerberos_,
     mysql_connector_python_,
     openldap_,
     pandas_,
     postgres_,
-    prometheus_,
     pyspark_,
     smtplib_,
     sqlalchemy_,

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.50'
-__version_tuple__ = version_tuple = (0, 50)
+__version__ = version = '0.53'
+__version_tuple__ = version_tuple = (0, 53)
 
-__commit_id__ = commit_id = 'g3a0e189a0'
+__commit_id__ = commit_id = 'g2446c213a'

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/base.py

@@ -7,7 +7,11 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 from urllib.parse import urlencode, urlparse, urlunparse
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 
 def deprecated(func):
@@ -82,27 +86,24 @@ def get_string_case_combination(str_: str) -> List[str]:
     return list(map("".join, itertools.product(*zip(str_.upper(), str_.lower()))))
 
 
-def get_none_variation() -> List[Union[None, str]]:
-    """Returns a list of common variations of None/null values.
+def get_none_variation() -> frozenset:
+    """Returns a frozenset of lowercase None/null string representations.
+
+    Use with .casefold() for case-insensitive membership tests:
+        item.casefold() in get_none_variation()
 
     Returns:
-        list: List containing None and various string representations of null values.
+        frozenset: Frozenset of lowercase null string representations.
 
     Examples:
-        >>> get_none_variation()
-        [None, 'NONE', 'NONe', 'NOnE', 'NOne', 'NoNE', 'NoNe', 'NonE', 'None', 'nONE', 'nONe', 'nOnE', 'nOne', 'noNE', 'noNe', 'nonE', 'none', 'NULL', 'NULl', 'NUlL', 'NUll', 'NuLL', 'NuLl', 'NulL', 'Null', 'nULL', 'nULl', 'nUlL', 'nUll', 'nuLL', 'nuLl', 'nulL', 'null', 'NA', 'Na', 'nA', 'na', 'N.A', 'N.a', 'N.A', 'N.a', 'n.A', 'n.a', 'n.A', 'n.a', 'N.A.', 'N.A.', 'N.a.', 'N.a.', 'N.A.', 'N.A.', 'N.a.', 'N.a.', 'n.A.', 'n.A.', 'n.a.', 'n.a.', 'n.A.', 'n.A.', 'n.a.', 'n.a.', 'NAN', 'NAn', 'NaN', 'Nan', 'nAN', 'nAn', 'naN', 'nan', 'NIL', 'NIl', 'NiL', 'Nil', 'nIL', 'nIl', 'niL', 'nil']
+        >>> "none" in get_none_variation()
+        True
+        >>> "NONE".casefold() in get_none_variation()
+        True
+        >>> "valid".casefold() in get_none_variation()
+        False
     """
-    variations = (
-        [None]
-        + get_string_case_combination("none")
-        + get_string_case_combination("null")
-        + get_string_case_combination("na")
-        + get_string_case_combination("n.a")
-        + get_string_case_combination("n.a.")
-        + get_string_case_combination("nan")
-        + get_string_case_combination("nil")
-    )
-    return variations
+    return frozenset({"none", "null", "na", "n.a", "n.a.", "nan", "nil"})
 
 
 def get_country_name_variation() -> dict:
@@ -172,35 +173,17 @@ def list_tuple_without_none(list_tuple: Union[List, Tuple]) -> Union[List, Tuple
 
     none_variations = get_none_variation()
 
-    if isinstance(list_tuple, list):
-        return [item for item in list_tuple if item and item not in none_variations]
-    return tuple(item for item in list_tuple if item and item not in none_variations)
-
-
-@deprecated
-def string_boolean_to_int(boolean_str_rep: str) -> int:
-    """Converts string boolean representations to integers.
-
-    Args:
-        boolean_str_rep (str): String representation of boolean value.
+    def _keep(item: Any) -> bool:
+        if not item:
+            return False
+        if isinstance(item, str):
+            return item.casefold() not in none_variations
+        return True
 
-    Returns:
-        int: 1 for true values, 0 for false values.
-
-    Examples:
-        >>> string_boolean_to_int("true")
-        1
-        >>> string_boolean_to_int("True")
-        1
-    """
-    try:
-        from distutils.util import strtobool
+    if isinstance(list_tuple, list):
+        return [item for item in list_tuple if _keep(item)]
+    return tuple(item for item in list_tuple if _keep(item))
 
-        return strtobool(string_str_to_str(boolean_str_rep))
-    except ValueError as e:
-        raise ValueError(
-            f"Invalid boolean string representation: {boolean_str_rep}"
-        ) from e
 
 
 def string_dlt_to_dlt(dlt_str_rep: str) -> Union[Dict, List, Tuple]:
@@ -417,9 +400,14 @@ def parse_ps_aux(ps_aux_commands: str) -> List[List[str]]:
             check=True,
         )
         lines = output.stdout.strip().split("\n")
-        if not lines:
+        if not lines or not lines[0]:
             return []
 
+        if "PID" not in lines[0] and "USER" not in lines[0]:
+            logger.warning(
+                "ps output header looks unexpected; "
+                "column split may be incorrect"
+            )
         n_columns = len(lines[0].split()) - 1
         return [line.split(None, n_columns) for line in lines if line]
     except subprocess.SubprocessError as e:

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/beautifulsoup_.py

@@ -1,7 +1,11 @@
 import re
 from typing import TYPE_CHECKING, Optional
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
     from bs4 import BeautifulSoup
@@ -31,15 +35,11 @@ def preprocess(html: str) -> Optional[str]:
         Reference: https://stackoverflow.com/questions/23241641
     """
 
-    # remove leading and trailing whitespaces
-    pattern = re.compile("(^[\s]+)|([\s]+$)", re.MULTILINE)
+    pattern = re.compile(r"(^[\s]+)|([\s]+$)", re.MULTILINE)
     html = re.sub(pattern, "", html)
-    # convert newlines to spaces, this preserves newline delimiters
     html = re.sub("\n", " ", html)
-    # remove whitespaces before opening tags
-    html = re.sub("[\s]+<", "<", html)
-    # remove whitespaces after closing tags
-    html = re.sub(">[\s]+", ">", html)
+    html = re.sub(r"[\s]+<", "<", html)
+    html = re.sub(r">[\s]+", ">", html)
     return html
 
 

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/boto3_.py

@@ -1,7 +1,11 @@
 import os
 from typing import Any, Dict, List, Optional, Union
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 # Constants - now serving as defaults
 DEFAULT_MAX_ATTACHMENT_SIZE = 10 * 1024 * 1024  # 10MB

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/cryptography_.py

@@ -2,8 +2,17 @@ import os
 from pathlib import Path
 from typing import Union
 
-from cryptography.fernet import Fernet, InvalidToken
-from loguru import logger
+try:
+    from cryptography.fernet import Fernet, InvalidToken
+    HAS_CRYPTOGRAPHY = True
+except ImportError:
+    HAS_CRYPTOGRAPHY = False
+
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 
 def generate_fernet_key(

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/django_.py

@@ -2,7 +2,11 @@ import os
 import sys
 from typing import Dict, List, Optional, Tuple
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 
 def init_django(
@@ -151,12 +155,12 @@ def django_validate_phone(
 
     try:
         return PhoneNumber.from_string(phone).as_e164
-    except:
+    except Exception:
         try:
             return PhoneNumber.from_string(phone, region=region).as_e164
         except Exception as e:
             logger.error(e)
-            return
+            return None
 
 
 if __name__ == "__main__":

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/geopandas_.py

@@ -1,10 +1,16 @@
-from typing import TypeVar
+from __future__ import annotations
 
-import numpy as np
-from numpy.typing import NDArray
+from typing import TypeVar
 
-# Type variable for numeric types (float or numpy array)
-NumericType = TypeVar("NumericType", float, NDArray[np.float64])
+try:
+    import numpy as np
+    from numpy.typing import NDArray
+    HAS_NUMPY = True
+    NumericType = TypeVar("NumericType", float, NDArray[np.float64])
+except ImportError:
+    np = None
+    HAS_NUMPY = False
+    NumericType = TypeVar("NumericType", float, float)
 
 # Earth's radius in kilometers
 EARTH_RADIUS_KM = 6371.0

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/kerberos_.py

@@ -2,7 +2,11 @@ import os
 import shlex
 import subprocess
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 
 def keytab_valid(

{data_manipulation-0.50 → data_manipulation-0.53}/data_manipulation/mysql_connector_python_.py

@@ -1,7 +1,11 @@
 from contextlib import contextmanager
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union
 
-from loguru import logger
+try:
+    from loguru import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
     from mysql.connector.connection import MySQLConnection