acuity-framework 1.0.0__py3-none-any.whl

acuity/__init__.py

@@ -0,0 +1,27 @@
+"""
+ACUITY Framework
+================
+
+**A**utomated **C**ommunity **U**nstructured **I**nformation to **T**argeted visibilit**Y**
+
+A machine learning framework for extracting, verifying, and recommending
+local micro-enterprise profiles from unstructured community posts.
+
+Modules:
+    - acuity.extraction: NLP pipeline (preprocessing → NER → rule-based → profile building)
+    - acuity.recommendation: TF-IDF + cosine similarity + Haversine proximity ranking
+    - acuity.verification: Business legitimacy verification via fuzzy matching against registries
+    - acuity.scraper: Facebook community group post scraper (optional)
+
+Quick Start:
+    >>> from acuity.extraction import ExtractionPipeline
+    >>> from acuity.recommendation import RecommendationEngine
+    >>> from acuity.verification import BPLOVerifier
+"""
+
+__version__ = "1.0.0"
+__author__ = "ACUITY Research Team"
+
+from .config import AcuityConfig
+
+__all__ = ["AcuityConfig"]

acuity/config.py

@@ -0,0 +1,47 @@
+"""
+ACUITY Framework Configuration
+
+Provides a centralized dataclass for all configurable parameters.
+No hardcoded paths — users supply their own paths and thresholds.
+"""
+from dataclasses import dataclass
+
+
+@dataclass
+class AcuityConfig:
+    """Configuration for the ACUITY framework.
+
+    All paths default to None, requiring the user to set them
+    based on their own project structure.
+
+    Attributes:
+        ner_model_path: Path to a trained NER model (CRF .pkl or HuggingFace directory).
+        ner_backend: Which NER backend to use: ``"crf"`` or ``"transformer"``.
+        completeness_threshold: Minimum number of populated detail fields
+            for a profile to be considered complete enough to keep.
+        relevance_weight: Weight for textual relevance in recommendation ranking [0, 1].
+        proximity_weight: Weight for geographic proximity in recommendation ranking [0, 1].
+        default_top_k: Default number of results returned by the recommendation engine.
+        fuzzy_match_threshold_verified: Levenshtein ratio threshold (0–1) to mark
+            a business as "Verified" against a BPLO registry.
+        fuzzy_match_threshold_pending: Levenshtein ratio threshold (0–1) to mark
+            a business as "Pending Verification".
+        max_flags_threshold: Number of user flags before a profile is auto-deactivated.
+    """
+
+    # NLP / Pipeline settings
+    ner_model_path: str | None = None
+    ner_backend: str = "crf"  # "crf" or "transformer"
+    completeness_threshold: int = 2
+
+    # Recommendation settings
+    relevance_weight: float = 0.6
+    proximity_weight: float = 0.4
+    default_top_k: int = 10
+
+    # BPLO Verification settings
+    fuzzy_match_threshold_verified: float = 0.8
+    fuzzy_match_threshold_pending: float = 0.6
+
+    # Business Profile limits
+    max_flags_threshold: int = 3

acuity/extraction/__init__.py

@@ -0,0 +1,22 @@
+"""
+ACUITY Framework — Extraction Module
+
+NLP pipeline for extracting structured business information from
+unstructured community group posts (e.g., Facebook, forums).
+
+Components:
+    - preprocessing: Text cleaning and normalisation
+    - ner: Named Entity Recognition (CRF or Transformer backend)
+    - rules: Regex-based structured field extraction
+    - postprocessing: Profile construction from extraction outputs
+    - pipeline: End-to-end orchestrator
+
+Quick Start:
+    >>> from acuity.extraction import ExtractionPipeline
+    >>> pipeline = ExtractionPipeline()
+    >>> profiles = pipeline.extract_from_texts(["Mang Juan's Bakery, Mamatid, open 8am-5pm, 0917-123-4567"])
+"""
+
+from .pipeline import ExtractionPipeline
+
+__all__ = ["ExtractionPipeline"]

acuity/extraction/ner_crf.py

@@ -0,0 +1,164 @@
+"""
+ACUITY Framework — Named Entity Recognition (CRF Backend)
+
+Extracts business-related entities from preprocessed post text using
+a trained CRF (Conditional Random Field) model with BIO tagging:
+  - BUSINESS_NAME
+  - SERVICE_CATEGORY
+  - LOCATION
+
+The CRF model path must be supplied via ``AcuityConfig.ner_model_path``.
+"""
+from __future__ import annotations
+
+import pickle
+from typing import Any
+
+try:
+    import nltk  # type: ignore
+
+    try:
+        nltk.data.find("taggers/averaged_perceptron_tagger_eng")
+    except LookupError:
+        nltk.download("averaged_perceptron_tagger_eng", quiet=True)
+
+    _NLTK_AVAILABLE = True
+except ImportError:
+    nltk: Any = None
+    _NLTK_AVAILABLE = False
+
+
+def _extract_features(tokens: list[str], pos_tags: list[tuple[str, str]], i: int) -> dict[str, Any]:
+    """Build the feature dictionary for token at index *i*."""
+    word = tokens[i]
+    postag = pos_tags[i][1]
+
+    features: dict[str, Any] = {
+        "bias": 1.0,
+        "word.lower()": word.lower(),
+        "word[-3:]": word[-3:],
+        "word[-2:]": word[-2:],
+        "word[:3]": word[:3],
+        "word[:2]": word[:2],
+        "word.isupper()": word.isupper(),
+        "word.istitle()": word.istitle(),
+        "word.isdigit()": word.isdigit(),
+        "postag": postag,
+        "postag[:2]": postag[:2],
+    }
+
+    if i > 0:
+        word1 = tokens[i - 1]
+        postag1 = pos_tags[i - 1][1]
+        features.update({
+            "-1:word.lower()": word1.lower(),
+            "-1:word.istitle()": word1.istitle(),
+            "-1:word.isupper()": word1.isupper(),
+            "-1:postag": postag1,
+            "-1:postag[:2]": postag1[:2],
+        })
+    else:
+        features["BOS"] = True
+
+    if i < len(tokens) - 1:
+        word1 = tokens[i + 1]
+        postag1 = pos_tags[i + 1][1]
+        features.update({
+            "+1:word.lower()": word1.lower(),
+            "+1:word.istitle()": word1.istitle(),
+            "+1:word.isupper()": word1.isupper(),
+            "+1:postag": postag1,
+            "+1:postag[:2]": postag1[:2],
+        })
+    else:
+        features["EOS"] = True
+
+    return features
+
+
+def _sent2features(tokens: list[str]) -> list[dict[str, Any]]:
+    """Convert a sentence (list of tokens) into a list of feature dicts."""
+    if not _NLTK_AVAILABLE or nltk is None:
+        return []
+    pos_tags = nltk.pos_tag(tokens)
+    return [_extract_features(tokens, pos_tags, i) for i in range(len(tokens))]
+
+
+def load_crf_model(model_path: str):
+    """Load a pickled CRF model from disk.
+
+    Args:
+        model_path: Absolute path to the ``.pkl`` file.
+
+    Returns:
+        The loaded CRF model object, or ``None`` on failure.
+    """
+    try:
+        with open(model_path, "rb") as f:
+            return pickle.load(f)
+    except Exception as e:
+        print(f"Warning: Could not load CRF model from {model_path}: {e}")
+        return None
+
+
+def extract_entities_crf(text: str, model: Any) -> dict:
+    """Extract named entities from *text* using a CRF model.
+
+    Args:
+        text: Preprocessed post text.
+        model: A loaded CRF model (e.g., from ``load_crf_model``).
+
+    Returns:
+        dict with keys: ``business_name``, ``categories``, ``locations``.
+        Each value is a list of extracted strings.
+    """
+    extracted: dict[str, list[str]] = {
+        "business_name": [],
+        "categories": [],
+        "locations": [],
+    }
+
+    if not model or not _NLTK_AVAILABLE:
+        return extracted
+
+    tokens = text.split()
+    if not tokens:
+        return extracted
+
+    features = _sent2features(tokens)
+    predictions = model.predict([features])[0]
+
+    # Reconstruct entities from BIO tags
+    current_entity_type: str | None = None
+    current_entity_tokens: list[str] = []
+
+    def _save_entity() -> None:
+        if current_entity_type and current_entity_tokens:
+            entity_text = " ".join(current_entity_tokens)
+            if current_entity_type == "BUSINESS_NAME":
+                extracted["business_name"].append(entity_text)
+            elif current_entity_type == "SERVICE_CATEGORY":
+                extracted["categories"].append(entity_text)
+            elif current_entity_type == "LOCATION":
+                extracted["locations"].append(entity_text)
+
+    for token, tag in zip(tokens, predictions):
+        if tag.startswith("B-"):
+            _save_entity()
+            current_entity_type = tag[2:]
+            current_entity_tokens = [token]
+        elif tag.startswith("I-"):
+            if current_entity_type == tag[2:]:
+                current_entity_tokens.append(token)
+            else:
+                _save_entity()
+                current_entity_type = tag[2:]
+                current_entity_tokens = [token]
+        else:
+            _save_entity()
+            current_entity_type = None
+            current_entity_tokens = []
+
+    _save_entity()
+
+    return extracted

acuity/extraction/ner_transformer.py

@@ -0,0 +1,76 @@
+"""
+ACUITY Framework — Named Entity Recognition (Transformer Backend)
+
+Extracts business-related entities using a fine-tuned HuggingFace
+transformer model with BIO tagging:
+  - BUSINESS_NAME
+  - SERVICE_CATEGORY
+  - LOCATION
+
+Requires ``transformers`` and ``torch`` (install with ``pip install acuity-framework[transformers]``).
+"""
+from __future__ import annotations
+
+from typing import Any
+
+
+def load_transformer_model(model_path: str):
+    """Load a HuggingFace NER pipeline from a local directory.
+
+    Args:
+        model_path: Path to the fine-tuned model directory.
+
+    Returns:
+        A HuggingFace ``pipeline`` object, or ``None`` on failure.
+    """
+    try:
+        from transformers import pipeline as hf_pipeline  # type: ignore
+        print(f"Loading Fine-Tuned NER model from {model_path}...")
+        return hf_pipeline("ner", model=model_path, aggregation_strategy="simple")
+    except Exception as e:
+        print(f"Warning: Could not load NER transformer model: {e}")
+        return None
+
+
+def extract_entities_transformer(text: str, model: Any) -> dict:
+    """Extract named entities from *text* using a HuggingFace NER model.
+
+    Args:
+        text: Preprocessed post text.
+        model: A loaded HuggingFace NER pipeline (from ``load_transformer_model``).
+
+    Returns:
+        dict with keys: ``business_name``, ``categories``, ``locations``.
+        Each value is a list of extracted strings.
+    """
+    extracted: dict[str, list[str]] = {
+        "business_name": [],
+        "categories": [],
+        "locations": [],
+    }
+
+    if not model:
+        return extracted
+
+    # Truncate to ~2000 chars to stay within 512 token limit
+    truncated_text = text[:2000]
+
+    hf_results = model(truncated_text)
+
+    for entity in hf_results:
+        label = entity.get("entity_group")
+        word = entity.get("word", "").strip()
+        score = float(entity.get("score", 0))
+
+        # Only keep confident extractions
+        if score < 0.30:
+            continue
+
+        if label == "BUSINESS_NAME":
+            extracted["business_name"].append(word)
+        elif label == "LOCATION":
+            extracted["locations"].append(word)
+        elif label == "SERVICE_CATEGORY":
+            extracted["categories"].append(word)
+
+    return extracted

acuity/extraction/pipeline.py

@@ -0,0 +1,171 @@
+"""
+ACUITY Framework — Extraction Pipeline Orchestrator
+
+Coordinates the full extraction flow:
+    Raw post text → preprocessing → NER → rule-based extraction → profile construction
+
+This is the main public API for the extraction module. It is fully decoupled
+from any web framework, database, or frontend — it takes text in and returns
+structured profiles out.
+
+Usage:
+    >>> from acuity.extraction import ExtractionPipeline
+    >>> pipeline = ExtractionPipeline()
+    >>> profiles = pipeline.extract_from_texts([
+    ...     "Mang Juan's Bakery, Mamatid, open 8am-5pm, 0917-123-4567",
+    ... ])
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from .preprocessing import preprocess
+from .rules import extract_structured_fields
+from .postprocessing import build_business_profile
+from ..config import AcuityConfig
+
+
+class ExtractionPipeline:
+    """End-to-end extraction pipeline for community post text.
+
+    This class coordinates preprocessing, NER, rule-based extraction,
+    and profile construction into a single callable pipeline.
+
+    Args:
+        config: An ``AcuityConfig`` instance. If ``None``, uses defaults.
+        ner_model: A pre-loaded NER model object. If provided, the pipeline
+            will use this model directly instead of loading from ``config.ner_model_path``.
+    """
+
+    def __init__(self, config: AcuityConfig | None = None, ner_model: Any = None):
+        self.config = config or AcuityConfig()
+        self._ner_model = ner_model
+        self._ner_extract_fn = None
+        self._setup_ner()
+
+    def _setup_ner(self) -> None:
+        """Initialise the NER backend based on configuration."""
+        backend = self.config.ner_backend
+
+        if backend == "crf":
+            from .ner_crf import extract_entities_crf, load_crf_model
+
+            if self._ner_model is None and self.config.ner_model_path:
+                self._ner_model = load_crf_model(self.config.ner_model_path)
+
+            self._ner_extract_fn = lambda text: extract_entities_crf(text, self._ner_model)
+
+        elif backend == "transformer":
+            from .ner_transformer import extract_entities_transformer, load_transformer_model
+
+            if self._ner_model is None and self.config.ner_model_path:
+                self._ner_model = load_transformer_model(self.config.ner_model_path)
+
+            self._ner_extract_fn = lambda text: extract_entities_transformer(text, self._ner_model)
+
+        else:
+            raise ValueError(f"Unknown NER backend: {backend!r}. Use 'crf' or 'transformer'.")
+
+    def extract_single(
+        self,
+        text: str,
+        metadata: dict | None = None,
+        poster_name: str | None = None,
+    ) -> dict | None:
+        """Run the full extraction pipeline on a single post.
+
+        Args:
+            text: Raw post text.
+            metadata: Optional metadata dict (e.g., source_index, scraped_at).
+            poster_name: Optional poster name to use as fallback business name.
+
+        Returns:
+            A business profile dict, or ``None`` if not enough info was extracted.
+        """
+        # Step 1: Preprocess (clean, normalise)
+        cleaned = preprocess(text)
+
+        # Step 2: Named Entity Recognition
+        entities = self._ner_extract_fn(cleaned) if self._ner_extract_fn else {
+            "business_name": [], "categories": [], "locations": []
+        }
+
+        # Step 3: Rule-based extraction (contacts, hours, address patterns)
+        structured = extract_structured_fields(cleaned)
+
+        # Step 4: Build business profile
+        profile = build_business_profile(
+            raw_text=text,
+            entities=entities,
+            structured_fields=structured,
+            metadata=metadata,
+            poster_name=poster_name,
+        )
+
+        return profile
+
+    def extract_from_texts(
+        self,
+        texts: list[str | dict],
+        completeness_threshold: int | None = None,
+        deduplicate: bool = True,
+    ) -> list[dict]:
+        """Run extraction on a batch of texts and return quality-filtered profiles.
+
+        Args:
+            texts: A list of raw text strings, or a list of dicts with at least
+                a ``"text"`` key (and optionally ``"poster"`` and ``"scraped_at"``).
+            completeness_threshold: Minimum number of populated detail fields
+                (categories, locations, phones, prices, hours) to keep a profile.
+                Defaults to ``config.completeness_threshold``.
+            deduplicate: If True, deduplicate profiles by business name.
+
+        Returns:
+            List of extracted business profile dicts.
+        """
+        threshold = completeness_threshold if completeness_threshold is not None else self.config.completeness_threshold
+
+        profiles = []
+        for i, item in enumerate(texts):
+            # Accept either plain strings or dicts
+            if isinstance(item, str):
+                raw_text = item
+                poster = None
+                scraped_at = None
+            else:
+                raw_text = item.get("text", "")
+                poster = item.get("poster")
+                scraped_at = item.get("scraped_at")
+
+            profile = self.extract_single(
+                text=raw_text,
+                metadata={"source_index": i, "scraped_at": scraped_at},
+                poster_name=poster,
+            )
+
+            if profile:
+                # Filter weak profiles by counting populated detail fields
+                detail_lists = [
+                    profile.get("categories", []),
+                    profile.get("locations", []),
+                    profile.get("phones", []),
+                    profile.get("prices", []),
+                    profile.get("hours", []),
+                ]
+                populated = sum(1 for field in detail_lists if len(field) > 0)
+
+                if populated >= threshold:
+                    profiles.append(profile)
+
+        # Deduplicate by business name
+        if deduplicate:
+            seen_names: set[str] = set()
+            unique_profiles = []
+            for p in profiles:
+                name = p.get("business_name")
+                if name and name not in seen_names:
+                    unique_profiles.append(p)
+                    seen_names.add(name)
+            profiles = unique_profiles
+
+        return profiles

acuity/extraction/postprocessing.py

@@ -0,0 +1,74 @@
+"""
+ACUITY Framework — Post-processing & Profile Construction
+
+Merges NER outputs and rule-based fields into a unified business profile,
+performs validation, and assigns confidence scores.
+"""
+from __future__ import annotations
+
+import re
+
+
+def format_business_name(name: str) -> str:
+    """Normalise and title-case a raw business name string.
+
+    Args:
+        name: Raw business name extracted from a post.
+
+    Returns:
+        Cleaned and title-cased name.
+    """
+    if not name:
+        return name
+    cleaned = re.sub(r'[#.,!_]', ' ', name)
+    cleaned = re.sub(r'([a-z])([A-Z])', r'\1 \2', cleaned)
+    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
+    return cleaned.title()
+
+
+def build_business_profile(
+    raw_text: str,
+    entities: dict,
+    structured_fields: dict,
+    metadata: dict | None = None,
+    poster_name: str | None = None,
+) -> dict | None:
+    """Construct a business profile dict from extraction outputs.
+
+    Args:
+        raw_text: The original post text.
+        entities: Output from NER (``business_name``, ``categories``, ``locations``).
+        structured_fields: Output from rule-based extraction (``phones``, ``prices``, ``hours``).
+        metadata: Optional metadata (``source_index``, ``scraped_at``, etc.).
+        poster_name: Optional poster name to fall back to if business_name is missing.
+
+    Returns:
+        A business profile dict, or ``None`` if insufficient information was extracted.
+    """
+    b_names = entities.get("business_name", [])
+    raw_business_name = " ".join(b_names) if b_names else None
+    if not raw_business_name and poster_name:
+        raw_business_name = poster_name
+
+    business_name = format_business_name(raw_business_name) if raw_business_name else None
+
+    profile = {
+        "business_name": business_name,
+        "categories": entities.get("categories", []),
+        "locations": entities.get("locations", []),
+        "phones": structured_fields.get("phones", []),
+        "prices": structured_fields.get("prices", []),
+        "hours": structured_fields.get("hours", []),
+        "description": raw_text,
+        "metadata": metadata or {},
+    }
+
+    # Basic validation: require at least some extractable information
+    has_info = (
+        profile["business_name"]
+        or profile["categories"]
+        or profile["phones"]
+        or profile["prices"]
+    )
+
+    return profile if has_info else None

acuity/extraction/preprocessing.py

@@ -0,0 +1,57 @@
+"""
+ACUITY Framework — Text Preprocessing
+
+Cleans and normalises raw social media post text for downstream NLP tasks.
+Handles Taglish (Tagalog-English code-switching), informal spelling,
+and social-media-specific noise.
+"""
+import re
+import unicodedata
+
+
+def preprocess(text: str) -> str:
+    """Full preprocessing pipeline for a single post.
+
+    Args:
+        text: Raw post text.
+
+    Returns:
+        Cleaned and normalised text.
+    """
+    text = normalise_unicode(text)
+    text = remove_urls(text)
+    text = remove_emojis(text)
+    text = normalise_whitespace(text)
+    return text.strip()
+
+
+def normalise_unicode(text: str) -> str:
+    """Normalise Unicode characters (e.g. fullwidth → ASCII)."""
+    return unicodedata.normalize("NFKC", text)
+
+
+def remove_urls(text: str) -> str:
+    """Strip URLs from the text."""
+    return re.sub(r"https?://\S+", "", text)
+
+
+def remove_emojis(text: str) -> str:
+    """Remove common emoji / symbol characters."""
+    emoji_pattern = re.compile(
+        "["
+        "\U0001f600-\U0001f64f"  # emoticons
+        "\U0001f300-\U0001f5ff"  # symbols & pictographs
+        "\U0001f680-\U0001f6ff"  # transport & map
+        "\U0001f1e0-\U0001f1ff"  # flags
+        "\U00002700-\U000027bf"  # dingbats
+        "\U0000fe00-\U0000fe0f"  # variation selectors
+        "\U0000200d"             # zero width joiner
+        "]+",
+        flags=re.UNICODE,
+    )
+    return emoji_pattern.sub("", text)
+
+
+def normalise_whitespace(text: str) -> str:
+    """Collapse multiple whitespace characters into single spaces."""
+    return re.sub(r"\s+", " ", text)

acuity/extraction/rules.py

@@ -0,0 +1,49 @@
+"""
+ACUITY Framework — Rule-Based Extraction
+
+Complements the NER module by extracting structured fields that follow
+predictable patterns in community posts:
+  - Phone numbers (PH mobile formats)
+  - Addresses / barangay references
+  - Operating hours
+  - Price mentions
+"""
+import re
+
+
+# ---------------------------------------------------------------------------
+# Pattern definitions
+# ---------------------------------------------------------------------------
+
+# Philippine mobile: 09XX-XXX-XXXX or +639XX-XXX-XXXX
+PHONE_PATTERN = re.compile(
+    r"(?:\+63|0)9\d{2}[\s\-]?\d{3}[\s\-]?\d{4}"
+)
+
+# Simple price patterns (₱, PHP, P followed by digits)
+PRICE_PATTERN = re.compile(
+    r"(?:[₱Pp](?:HP)?)\s?\d[\d,]*(?:\.\d{2})?"
+)
+
+# Operating hours heuristic (e.g. "open 8am-5pm", "available 24/7")
+HOURS_PATTERN = re.compile(
+    r"\b(?:open|available|hours?)\b.*?\d{1,2}\s?(?:am|pm|AM|PM)",
+    re.IGNORECASE,
+)
+
+
+def extract_structured_fields(text: str) -> dict:
+    """Extract structured information from *text* using regex patterns.
+
+    Args:
+        text: Preprocessed post text.
+
+    Returns:
+        dict with keys: ``phones``, ``prices``, ``hours``.
+        Each value is a list of matched strings.
+    """
+    return {
+        "phones": PHONE_PATTERN.findall(text),
+        "prices": PRICE_PATTERN.findall(text),
+        "hours": HOURS_PATTERN.findall(text),
+    }