pinky-core 0.1.0.dev1__py3-none-any.whl

pinky_core/__init__.py

@@ -0,0 +1,149 @@
+"""
+pinky_core
+=============
+Core Python utilities for Snowflake development.
+
+Modules
+-------
+fmt       Pure formatting helpers (text, dates, numbers, FR addresses).
+files     File pattern normalization for S3 and MFT path analysis.
+schedule  Business-day calendar (FR) and cron utilities for SPs and DAGs.
+security  Path and identifier sanitisation for safe dynamic SQL.
+sql       SQL parsing helpers powered by sqlglot (Snowflake dialect).
+validate  French/international data validation (email, phone, SIRET, NIR, IBAN/BIC).
+xml       XML utilities for Snowflake VARIANT — UDF handlers and helpers.
+
+No Snowflake connection required — importable in any context (CI, local, UDF).
+Optional deps for validate: email-validator, phonenumbers, python-stdnum.
+Optional deps for address:  usaddress.
+Optional deps for schedule: croniter (next_schedule, is_in_schedule).
+"""
+
+from __future__ import annotations
+
+# files — pure file pattern normalization, always available
+from pinky_core.files import (
+    FileDetection,
+    detect_file_pattern,
+)
+
+# fmt — pure formatting, always available
+from pinky_core.fmt import (
+    AddressComponents,
+    AfnorAddress,
+    deduplicate_headers,
+    format_boolean,
+    format_date,
+    format_datetime,
+    format_duration,
+    format_fraction,
+    format_number,
+    format_period,
+    format_stars,
+    format_trend,
+    iso3_to_iso2,
+    normalize_city_fr,
+    normalize_text,
+    parse_address,
+    period_to_date,
+    to_camel_case,
+    to_upper_snake_case,
+)
+
+# schedule — pure calendar + cron, always available (croniter optional)
+from pinky_core.schedule import (
+    business_days_between,
+    cron_to_human,
+    is_business_day,
+    is_in_schedule,
+    next_business_day,
+    next_schedule,
+    public_holidays_fr,
+)
+
+# security — pure sanitisation, always available
+from pinky_core.security import (
+    safe_filepath,
+    validate_identifier,
+)
+
+# sql — pure SQL parsing, always available
+from pinky_core.sql import (
+    is_select,
+    split_sql_statements,
+)
+
+# validate — lazy optional deps (email-validator, phonenumbers, python-stdnum)
+from pinky_core.validate import (
+    EmailResult,
+    FrIdResult,
+    IbanBicResult,
+    NirResult,
+    PhoneResult,
+    VatResult,
+    standardize_email,
+    standardize_phone,
+    validate_iban_bic,
+    validate_nir,
+    validate_siret,
+    validate_vat_eu,
+)
+
+# xml — XML/VARIANT navigation UDF handlers, always available (lxml optional for xml_path_str)
+from pinky_core.xml import xml_path, xml_path_str
+
+__all__ = [
+    # files
+    "FileDetection",
+    "detect_file_pattern",
+    # fmt
+    "AddressComponents",
+    "AfnorAddress",
+    "deduplicate_headers",
+    "iso3_to_iso2",
+    "normalize_city_fr",
+    "normalize_text",
+    "format_boolean",
+    "format_date",
+    "format_datetime",
+    "format_duration",
+    "format_fraction",
+    "format_number",
+    "format_period",
+    "format_stars",
+    "format_trend",
+    "parse_address",
+    "period_to_date",
+    "to_camel_case",
+    "to_upper_snake_case",
+    # schedule
+    "public_holidays_fr",
+    "is_business_day",
+    "next_business_day",
+    "business_days_between",
+    "cron_to_human",
+    "next_schedule",
+    "is_in_schedule",
+    # security
+    "safe_filepath",
+    "validate_identifier",
+    # sql
+    "is_select",
+    "split_sql_statements",
+    # xml
+    "xml_path",
+    "xml_path_str",
+    # validate
+    "EmailResult",
+    "FrIdResult",
+    "IbanBicResult",
+    "NirResult",
+    "PhoneResult",
+    "standardize_email",
+    "standardize_phone",
+    "VatResult",
+    "validate_iban_bic",
+    "validate_nir",
+    "validate_siret",
+    "validate_vat_eu",
+]

pinky_core/files.py

@@ -0,0 +1,178 @@
+"""File pattern normalization utilities for S3 and MFT path analysis."""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+# ISO 3166-1 alpha-2 codes (pipe-separated, used in regex alternation)
+_ISO2 = (
+    "AD|AE|AF|AG|AI|AL|AM|AO|AQ|AR|AS|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BJ|BL|"
+    "BM|BN|BO|BQ|BR|BS|BT|BV|BW|BY|BZ|CA|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|CR|CU|CV|"
+    "CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EE|EG|EH|ER|ES|ET|FI|FJ|FK|FM|FO|FR|GA|GB|GD|"
+    "GE|GF|GG|GH|GI|GL|GM|GN|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|"
+    "IN|IO|IQ|IR|IS|IT|JE|JM|JO|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|"
+    "LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MF|MG|MH|MK|ML|MM|MN|MO|MP|MQ|MR|MS|MT|MU|MV|MW|"
+    "MX|MY|MZ|NA|NC|NE|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|"
+    "PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|SS|"
+    "ST|SV|SX|SY|SZ|TC|TD|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TR|TT|TV|TW|TZ|UA|UG|UM|US|UY|"
+    "UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|YE|YT|ZA|ZM|ZW|UK"
+)
+
+# ISO 3166-1 alpha-3 codes (pipe-separated, used in regex alternation)
+_ISO3 = (
+    "ABW|AFG|AGO|AIA|ALA|ALB|AND|ARE|ARG|ARM|ASM|ATA|ATF|ATG|AUS|AUT|AZE|BDI|BEL|BEN|"
+    "BES|BFA|BGD|BGR|BHR|BHS|BIH|BLM|BLR|BLZ|BMU|BOL|BRA|BRB|BRN|BTN|BVT|BWA|CAF|CAN|"
+    "CCK|CHE|CHL|CHN|CIV|CMR|COD|COG|COK|COL|COM|CPV|CRI|CUB|CUW|CXR|CYM|CYP|CZE|DEU|"
+    "DJI|DMA|DNK|DOM|DZA|ECU|EGY|ERI|ESH|ESP|EST|ETH|FIN|FJI|FLK|FRA|FRO|FSM|GAB|GBR|"
+    "GEO|GGY|GHA|GIB|GIN|GLP|GMB|GNB|GNQ|GRC|GRD|GRL|GTM|GUF|GUM|GUY|HKG|HMD|HND|HRV|"
+    "HTI|HUN|IDN|IMN|IND|IOT|IRL|IRN|IRQ|ISL|ISR|ITA|JAM|JEY|JOR|JPN|KAZ|KEN|KGZ|KHM|"
+    "KIR|KNA|KOR|KWT|LAO|LBN|LBR|LBY|LCA|LIE|LKA|LSO|LTU|LUX|LVA|MAC|MAF|MAR|MCO|MDA|"
+    "MDG|MDV|MEX|MHL|MKD|MLI|MLT|MMR|MNE|MNG|MNP|MOZ|MRT|MSR|MTQ|MUS|MWI|MYS|MYT|NAM|"
+    "NCL|NER|NFK|NGA|NIC|NIU|NLD|NOR|NPL|NRU|NZL|OMN|PAK|PAN|PCN|PER|PHL|PLW|PNG|POL|"
+    "PRI|PRK|PRT|PRY|PSE|PYF|QAT|REU|ROU|RUS|RWA|SAU|SDN|SEN|SGP|SGS|SHN|SJM|SLB|SLE|"
+    "SLV|SMR|SOM|SPM|SRB|SSD|STP|SUR|SVK|SVN|SWE|SWZ|SXM|SYC|SYR|TCA|TCD|TGO|THA|TJK|"
+    "TKL|TKM|TLS|TON|TTO|TUN|TUR|TUV|TWN|TZA|UGA|UKR|UMI|URY|USA|UZB|VAT|VCT|VEN|VGB|"
+    "VIR|VNM|VUT|WLF|WSM|YEM|ZAF|ZMB|ZWE"
+)
+
+
+@dataclass
+class FileDetection:
+    """A custom detection rule for :func:`detect_file_pattern`.
+
+    Attributes:
+        pattern: Regex applied to the working string (after prior substitutions).
+        replacement: Replacement string, e.g. ``"{COST_CENTER}"`` or ``"<SAP>"``.
+        substring: If given, the rule is skipped when this literal is absent from
+            the *original* basename — cheap fast-path for context-specific rules.
+    """
+
+    pattern: str
+    replacement: str
+    substring: str | None = None
+
+
+def detect_file_pattern(
+    filepath: str,
+    custom_detections: list[FileDetection] | None = None,
+) -> str:
+    """Normalize a filename to a canonical pattern by replacing variable segments.
+
+    Replaces dates, timestamps, UUIDs, ISO country codes, structured business
+    identifiers (SIRET, SIREN, IBAN, BIC, VAT), and generic numeric IDs with
+    named placeholders — enabling grouping of structurally similar filenames
+    regardless of the specific value they carry.
+
+    Detection is purely structural: no business-context hint is required.
+    Substitutions run most-specific first to avoid double-replacement.
+
+    Placeholder conventions:
+
+    - ``{...}`` — temporal or generic numeric variable (date, timestamp, UUID, ID)
+    - ``<...>`` — typed structured identifier (country code, SIRET, IBAN…)
+
+    Substitution order:
+
+    1. Timestamps (most digits, most specific)
+    2. UUIDs
+    3. Dates (delimited, then bare YYYYMMDD)
+    4. Year-month combinations
+    5. ISO country codes (alpha-3 before alpha-2)
+    6. Structured business identifiers (IBAN, SIRET, SIREN, VAT_FR)
+    7. Generic numeric segments (least specific)
+
+    Note: BIC/SWIFT codes (e.g. ``BNPAFRPP``) are structurally indistinguishable
+    from 8-letter uppercase words and cannot be detected reliably without a bank
+    directory. BIC detection is intentionally excluded.
+
+    Args:
+        filepath: Full file path or bare filename to normalize.  The basename
+            (last ``/``-separated segment) is extracted automatically before
+            substitution, so both ``"EMPLOYEES_20240115.csv"`` and
+            ``"s3://bucket/feed/EMPLOYEES_20240115.csv"`` produce the same output.
+        custom_detections: Optional list of :class:`FileDetection` rules applied
+            *before* built-in substitutions. Each rule's ``substring`` filter is
+            tested against the **full** ``filepath`` (not just the basename) so
+            that directory-level context (e.g. ``"workday"`` in the S3 prefix) can
+            gate project-specific patterns.
+
+    Returns:
+        Pattern string for the basename, with variable segments replaced by
+        placeholders.
+
+    Examples:
+        >>> detect_file_pattern("EMPLOYEES_20240115.csv")
+        'EMPLOYEES_{DATE}.csv'
+        >>> detect_file_pattern("s3://bucket/feed/EMPLOYEES_20240115.csv")
+        'EMPLOYEES_{DATE}.csv'
+        >>> detect_file_pattern("PAYROLL_55208131700027.csv")
+        'PAYROLL_{SIRET}.csv'
+        >>> detect_file_pattern("TRANSFER_FR7630004000031234567890143.csv")
+        'TRANSFER_{IBAN}.csv'
+    """
+    basename = os.path.basename(filepath) if "/" in filepath else filepath
+    r = basename
+
+    # 0. Caller-supplied rules — run before all built-ins
+    # substring is tested against the original filepath so directory context is available
+    if custom_detections:
+        for rule in custom_detections:
+            if rule.substring is None or rule.substring in filepath:
+                r = re.sub(rule.pattern, rule.replacement, r)
+
+    # 1. Timestamps — anchored to avoid matching inside longer digit sequences
+    r = re.sub(r"(?<![0-9])[0-9]{8}[-._/:][0-9]{6}(?![0-9])", "{TIMESTAMP}", r)
+    r = re.sub(r"(?<![0-9])(19|20)[0-9]{12}(?![0-9])", "{TIMESTAMP_YYYYMMDDHHMMSS}", r)
+    r = re.sub(r"(?<![0-9])(19|20)[0-9]{10}(?![0-9])", "{TIMESTAMP_YYYYMMDDHHMM}", r)
+
+    # 2. UUIDs (self-anchoring via hyphens)
+    r = re.sub(
+        r"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}",
+        "{UUID}",
+        r,
+    )
+
+    # 3. Dates — delimited formats before bare YYYYMMDD, all anchored
+    r = re.sub(r"(?<![0-9])[0-9]{4}[-._/][0-9]{2}[-._/][0-9]{2}(?![0-9])", "{DATE}", r)
+    r = re.sub(r"(?<![0-9])[0-9]{2}[-._/][0-9]{2}[-._/][0-9]{4}(?![0-9])", "{DATE}", r)
+    r = re.sub(r"(?<![0-9])(19|20)[0-9]{6}(?![0-9])", "{DATE}", r)
+
+    # 4. Year-month — delimited before bare YYYYMM, all anchored
+    r = re.sub(
+        r"(?<![0-9])(19|20)[0-9]{2}[-._/](0[1-9]|1[0-2])(?![0-9])", "{YEAR_MONTH}", r
+    )
+    r = re.sub(
+        r"(?<![0-9])(0[1-9]|1[0-2])[-._/](19|20)[0-9]{2}(?![0-9])", "{YEAR_MONTH}", r
+    )
+    r = re.sub(r"(?<![0-9])(19|20)[0-9]{2}(0[1-9]|1[0-2])(?![0-9])", "{YEAR_MONTH}", r)
+    r = re.sub(r"(?<![0-9])(0[1-9]|1[0-2])(19|20)[0-9]{2}(?![0-9])", "{YEAR_MONTH}", r)
+
+    # 5. ISO country codes — alpha-3 before alpha-2 (more specific first)
+    r = re.sub(f"_({_ISO3})_", "_<ISO3_COUNTRY>_", r)
+    r = re.sub(f"^({_ISO3})_", "<ISO3_COUNTRY>_", r)
+    r = re.sub(f"_({_ISO2})_", "_<ISO2_COUNTRY>_", r)
+    r = re.sub(f"^({_ISO2})_", "<ISO2_COUNTRY>_", r)
+
+    # 6. Structured business identifiers — context-free, anchored by non-alphanumeric boundaries
+    # IBAN: 2-letter country + 2-digit check + 11-30 alphanum (run before VAT_FR)
+    r = re.sub(r"(?<![A-Z0-9])[A-Z]{2}[0-9]{2}[A-Z0-9]{11,30}(?![A-Z0-9])", "{IBAN}", r)
+    # FR VAT number: FR + 2 alphanum + 9 digits (run after IBAN to avoid partial match)
+    r = re.sub(r"(?<![A-Z0-9])FR[A-Z0-9]{2}[0-9]{9}(?![A-Z0-9])", "{VAT_FR}", r)
+    # SIRET: exactly 14 digits — timestamps starting with 19/20 already consumed above
+    r = re.sub(r"(?<![0-9])[0-9]{14}(?![0-9])", "{SIRET}", r)
+    # SIREN: exactly 9 digits
+    r = re.sub(r"(?<![0-9])[0-9]{9}(?![0-9])", "{SIREN}", r)
+
+    # 7. Generic numeric segments (least specific, run last)
+    r = re.sub(r"_([0-9]{6,})_", "_{ID}_", r)
+    r = re.sub(r"_([0-9]+)_", "_{NUM}_", r)
+    r = re.sub(r"_([0-9]+)\.", "_{NUM}.", r)
+    r = re.sub(r"^([0-9]+)_", "{NUM}_", r)
+    # Numbers immediately after a closing placeholder brace
+    r = re.sub(r"\}([0-9]+)", "}{NUM}", r)
+    # Trailing number before extension (not preceded by > or })
+    r = re.sub(r"([^}>])([0-9]+)\.([a-z]+)$", r"\1{NUM}.\3", r)
+
+    return r