orga 0.1.0__py3-none-any.whl

orga/__init__.py

@@ -0,0 +1,4 @@
+from .model import OrganizationProfile, Document, OrgaConfig
+from .pipeline import OrgaPipeline
+
+__version__ = "0.1.0"

orga/cli/main.py

@@ -0,0 +1,216 @@
+import typer
+import asyncio
+import json
+import yaml
+from pathlib import Path
+from typing import Optional, List
+from pydantic import ValidationError
+
+from orga.pipeline import OrgaPipeline
+from orga.model import OrgaConfig
+from orga.registry import registry
+
+# Ensure all default strategies are registered
+import orga.fetch.httpx_fetcher
+import orga.discover
+import orga.parse.fields.parsers
+import orga.parse.fields.classifier
+import orga.merge.processor
+
+app = typer.Typer(help="ORGA - Organization Profile Extractor CLI")
+
+def load_config(config_path: Optional[Path]) -> OrgaConfig:
+    """
+    Load OrgaConfig from a YAML or JSON file.
+    """
+    if config_path is None:
+        return OrgaConfig()
+    
+    if not config_path.exists():
+        typer.secho(f"Error: Config file {config_path} not found.", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1)
+    
+    try:
+        content = config_path.read_text(encoding="utf-8")
+        if config_path.suffix in [".yaml", ".yml"]:
+            data = yaml.safe_load(content)
+        else:
+            data = json.loads(content)
+        
+        return OrgaConfig.model_validate(data or {})
+    except ValidationError as e:
+        typer.secho(f"Error: Invalid configuration in {config_path}:", fg=typer.colors.RED, err=True)
+        typer.echo(str(e), err=True)
+        raise typer.Exit(code=1)
+    except Exception as e:
+        typer.secho(f"Error loading config: {str(e)}", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1)
+
+@app.command()
+def parse(
+    url: str = typer.Argument(..., help="The URL to parse"),
+    config: Optional[Path] = typer.Option(None, "--config", "-c", help="Path to configuration file (YAML/JSON)"),
+    output: Optional[Path] = typer.Option(None, "--output", "-o", help="Path to save the output JSON"),
+    pretty: bool = typer.Option(True, help="Pretty print JSON output"),
+    debug: bool = typer.Option(False, "--debug", "-d", help="Enable debug output (internal evidence, filtered links)")
+):
+    """
+    Parse an organization profile from a single URL.
+    """
+    orga_config = load_config(config)
+    pipeline = OrgaPipeline(orga_config)
+    
+    async def _run():
+        typer.echo(f"Fetching and parsing: {url} ...", err=True)
+        profile = await pipeline.run_from_url(url)
+        return profile
+
+    profile = asyncio.run(_run())
+    
+    indent = 2 if pretty else None
+    
+    # Configure export based on debug flag
+    exclude_fields = {}
+    if not debug:
+        # Hide internal/rejected evidence and debug info in standard mode
+        exclude_fields = {"internal_evidence", "debug_info"}
+        
+    # We use model_dump then json.dumps because model_dump_json doesn't support complex exclude logic as easily with nested models 
+    # (actually it does, but manual control is safer here). 
+    # Wait, model_dump_json supports `exclude`.
+    # However, `internal_evidence` is also inside `Contact` and `Location`. 
+    # We need to exclude it recursively. Pydantic excludes are usually recursive if field names match.
+    
+    # Pydantic v2 recursive exclusion:
+    # exclude={"internal_evidence", "debug_info", "phones": {"__all__": {"internal_evidence"}}, ...}
+    # This is getting complicated.
+    # Simpler: If we defined `internal_evidence` in models, we can just pass `exclude` if it works recursively.
+    # Or, we update the models to use `Field(exclude=True)`? No, that's permanent.
+    
+    # Let's try explicit recursive exclusion for top-level + common nested fields
+    if not debug:
+        exclude_set = {
+            "internal_evidence": True,
+            "debug_info": True,
+            "locations": {"__all__": {"internal_evidence": True}},
+            "phones": {"__all__": {"internal_evidence": True}},
+            "emails": {"__all__": {"internal_evidence": True}},
+            "social_links": {"__all__": {"internal_evidence": True}}
+        }
+        json_output = profile.model_dump_json(indent=indent, exclude=exclude_set)
+    else:
+        json_output = profile.model_dump_json(indent=indent)
+    
+    if output:
+        output.write_text(json_output, encoding="utf-8")
+        typer.echo(f"Saved output to {output}", err=True)
+    else:
+        typer.echo(json_output)
+
+@app.command()
+def parse_batch(
+    input_file: Path = typer.Argument(..., help="Text file containing URLs (one per line)"),
+    config: Optional[Path] = typer.Option(None, "--config", "-c", help="Path to configuration file"),
+    output: Optional[Path] = typer.Option(None, "--output", "-o", help="Path to save the output JSONL file"),
+    pretty: bool = typer.Option(False, "--pretty", help="Pretty print JSON output (best for stdout debugging)"),
+    debug: bool = typer.Option(False, "--debug", "-d", help="Enable debug output in JSONL")
+):
+    """
+    Batch parse multiple URLs.
+    Default output is JSONL (one JSON per line).
+    Use --pretty for readable output on stdout.
+    """
+    orga_config = load_config(config)
+    pipeline = OrgaPipeline(orga_config)
+    
+    if not input_file.exists():
+        typer.secho(f"Error: Input file {input_file} not found.", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1)
+    
+    urls = [line.strip() for line in input_file.read_text().splitlines() if line.strip()]
+    
+    async def _run_batch():
+        results = []
+        for i, url in enumerate(urls):
+            # Log progress to stderr so stdout remains clean for piping
+            typer.echo(f"[{i+1}/{len(urls)}] Processing {url}...", err=True)
+            try:
+                profile = await pipeline.run_from_url(url)
+                
+                # Determine exclusion set
+                exclude_set = {}
+                if not debug:
+                    exclude_set = {
+                        "internal_evidence": True,
+                        "debug_info": True,
+                        "locations": {"__all__": {"internal_evidence": True}},
+                        "phones": {"__all__": {"internal_evidence": True}},
+                        "emails": {"__all__": {"internal_evidence": True}},
+                        "social_links": {"__all__": {"internal_evidence": True}}
+                    }
+                
+                # Format output
+                indent = 2 if pretty and not output else None
+                json_str = profile.model_dump_json(indent=indent, exclude=exclude_set if not debug else None)
+                results.append(json_str)
+                    
+            except Exception as e:
+                typer.secho(f"Failed to process {url}: {str(e)}", fg=typer.colors.YELLOW, err=True)
+        return results
+
+    json_lines = asyncio.run(_run_batch())
+    
+    if output:
+        # File output is always JSONL (no pretty print to ensure validity)
+        if pretty:
+             typer.secho("Warning: --pretty is ignored when writing to file to maintain valid JSONL format.", fg=typer.colors.YELLOW, err=True)
+        
+        with output.open("w", encoding="utf-8") as f:
+            for line in json_lines:
+                # If we computed pretty strings above, we must flatten them for JSONL file
+                if pretty:
+                    # Parse back and dump as single line
+                    import json
+                    line = json.dumps(json.loads(line))
+                f.write(line + "\n")
+        typer.secho(f"Successfully processed {len(json_lines)} URLs. Output: {output}", fg=typer.colors.GREEN, err=True)
+    else:
+        # Stdout output
+        for line in json_lines:
+            typer.echo(line)
+
+@app.command()
+def list_strategies():
+    """
+    List all registered strategies.
+    """
+    kinds = ["fetcher", "discoverer", "parser", "category_classifier", "merger"]
+    
+    typer.echo("Registered ORGA Strategies:")
+    for kind in kinds:
+        strategies = registry.list(kind)
+        typer.echo(f"\n[{kind.upper()}]")
+        if not strategies:
+            typer.echo("  (None)")
+        for name in strategies:
+            impl = registry.get(kind, name)
+            typer.echo(f"  - {name}: {impl.__name__}")
+
+@app.command()
+def validate_config(config_path: Path):
+    """
+    Validate an ORGA configuration file.
+    """
+    load_config(config_path)
+    typer.secho(f"Configuration is valid: {config_path}", fg=typer.colors.GREEN)
+
+@app.command()
+def inspect_signals(url: str):
+    """
+    Inspect raw signals extracted from a URL (Debug mode).
+    """
+    typer.echo(f"Inspecting signals for {url}...")
+    typer.echo("This feature is planned for a future milestone (M4).", err=True)
+
+if __name__ == "__main__":
+    app()

orga/discover/__init__.py

@@ -0,0 +1,96 @@
+from abc import ABC, abstractmethod
+from typing import List, Set, Dict
+from urllib.parse import urljoin, urlparse
+from selectolax.parser import HTMLParser
+import re
+from orga.model import Document
+from orga.registry import registry
+
+class DiscoveryStrategy(ABC):
+    """
+    Base class for page discovery strategies.
+    Used to discover more high-value links from an initial document.
+    """
+    @abstractmethod
+    def discover(self, entry_doc: Document) -> List[str]:
+        pass
+
+class HeuristicDiscoveryStrategy(DiscoveryStrategy):
+    """
+    Heuristic-based page discovery strategy.
+    Finds high-value links using keyword matching and domain filtering.
+    """
+    
+    DEFAULT_KEYWORDS = {
+        "contact": 10,
+        "contact-us": 10,
+        "about": 8,
+        "about-us": 8,
+        "location": 9,
+        "locations": 9,
+        "store": 7,
+        "clinic": 7,
+        "team": 5,
+        "find-us": 9,
+        "support": 6
+    }
+
+    def __init__(self, max_pages: int = 5, keywords: Dict[str, int] = None):
+        self.max_pages = max_pages
+        self.keywords = keywords or self.DEFAULT_KEYWORDS
+
+    def discover(self, entry_doc: Document) -> List[str]:
+        if not entry_doc.content:
+            return []
+
+        tree = HTMLParser(entry_doc.content)
+        entry_url_parsed = urlparse(entry_doc.url)
+        entry_domain = entry_url_parsed.netloc
+        
+        candidates = []
+        seen_urls = {entry_doc.url.rstrip('/')}
+
+        for node in tree.css("a[href]"):
+            # href might be None if <a href> (boolean attribute style)
+            href_val = node.attributes.get("href")
+            href = (href_val or "").strip()
+            
+            if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+                continue
+            
+            abs_url = urljoin(entry_doc.url, href)
+            normalized_url = abs_url.rstrip('/')
+            
+            if normalized_url in seen_urls:
+                continue
+            
+            parsed_candidate = urlparse(abs_url)
+            if parsed_candidate.netloc != entry_domain:
+                continue
+            
+            score = self._score_url(abs_url, node.text() or "")
+            if score > 0:
+                candidates.append((score, abs_url))
+                seen_urls.add(normalized_url)
+
+        candidates.sort(key=lambda x: x[0], reverse=True)
+        return [url for score, url in candidates[:self.max_pages]]
+
+    def _score_url(self, url: str, link_text: str) -> int:
+        score = 0
+        path = urlparse(url).path.lower()
+        link_text = link_text.lower()
+        
+        for kw, weight in self.keywords.items():
+            if kw in path:
+                score += weight
+            if kw in link_text:
+                score += weight
+                
+        if score == 0:
+            score = 1
+            
+        return score
+
+# Register strategy
+registry.register("discoverer", "heuristic", HeuristicDiscoveryStrategy)

orga/fetch/__init__.py

@@ -0,0 +1,4 @@
+from orga.fetch.httpx_fetcher import HttpxFetcher
+from orga.fetch.strategy import FetchStrategy
+
+__all__ = ["HttpxFetcher", "FetchStrategy"]

orga/fetch/httpx_fetcher.py

@@ -0,0 +1,119 @@
+import httpx
+import asyncio
+from typing import Optional, List, Dict
+import tenacity
+from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
+from aiolimiter import AsyncLimiter
+from urllib.parse import urlparse
+
+from orga.model import Document, OrgaConfig, Warning, WarningSeverity, SourceKind
+from orga.registry import registry
+
+class FetchError(Exception):
+    """Custom exception for fetch operations."""
+    pass
+
+class HttpxFetcher:
+    """
+    Default fetch strategy using HTTPX.
+    Implements retries, timeout control, concurrency limits, and rate limiting.
+    """
+    
+    # Static pool of semaphores and limiters to share across fetcher instances 
+    # if using the same configuration.
+    _semaphores: Dict[int, asyncio.Semaphore] = {}
+    _limiters: Dict[int, AsyncLimiter] = {}
+
+    def __init__(self, config: OrgaConfig):
+        self.config = config.fetch
+        
+        # Initialize global concurrency control
+        # We use the config hash or id to share resources among identical configs
+        config_id = id(self.config)
+        if config_id not in self._semaphores:
+            self._semaphores[config_id] = asyncio.Semaphore(self.config.concurrency)
+            # Default rate limit: concurrency * 2 requests per second if not specified
+            # For now, let's use a conservative 5 req/s as default
+            self._limiters[config_id] = AsyncLimiter(10, 1) # 10 requests per second
+
+        self._semaphore = self._semaphores[config_id]
+        self._limiter = self._limiters[config_id]
+
+    async def fetch(self, url: str) -> Document:
+        """
+        Fetch a single URL asynchronously with concurrency and rate limit control.
+        """
+        headers = {"User-Agent": self.config.user_agent}
+        
+        # Apply rate limiting and concurrency control
+        async with self._limiter:
+            async with self._semaphore:
+                try:
+                    return await self._fetch_with_retry(url, headers)
+                except tenacity.RetryError as e:
+                    last_exception = e.last_attempt.exception()
+                    message = f"Connection timed out (Max retries reached): {str(last_exception)}"
+                    return self._create_error_document(url, "FETCH_TIMEOUT", message, WarningSeverity.ERROR)
+                except httpx.TimeoutException:
+                     return self._create_error_document(url, "FETCH_TIMEOUT", "Connection timed out", WarningSeverity.ERROR)
+                except Exception as e:
+                     return self._create_error_document(url, "FETCH_ERROR", str(e), WarningSeverity.ERROR)
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        retry=retry_if_exception_type((httpx.NetworkError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.HTTPStatusError))
+    )
+    async def _fetch_with_retry(self, url: str, headers: Dict[str, str]) -> Document:
+        # Note: AsyncClient is instantiated inside the retry block to ensure fresh connection pool if needed,
+        # but for high performance we might want to share one client. 
+        # Design Doc 7.2 suggests default fetcher should be a 'convenience layer'.
+        async with httpx.AsyncClient(timeout=self.config.timeout, follow_redirects=True) as client:
+            response = await client.get(url, headers=headers)
+            
+            # Retry on 5xx errors
+            if response.status_code >= 500:
+                response.raise_for_status()
+            
+            # Handle 404 gracefully without retry
+            if response.status_code == 404:
+                return self._create_document_from_response(response, warnings=[
+                    Warning(code="HTTP_404", message="Page not found", severity=WarningSeverity.WARNING)
+                ])
+            
+            # Check for non-HTML content
+            content_type = response.headers.get("content-type", "").lower()
+            warnings = []
+            if "text/html" not in content_type:
+                 warnings.append(Warning(
+                     code="NON_HTML_CONTENT", 
+                     message=f"Content-Type is {content_type}, expected text/html", 
+                     severity=WarningSeverity.WARNING
+                 ))
+
+            return self._create_document_from_response(response, warnings=warnings)
+
+    def _create_document_from_response(self, response: httpx.Response, warnings: List[Warning] = None) -> Document:
+        return Document(
+            url=str(response.url),
+            content=response.text,
+            content_type=response.headers.get("content-type", "application/octet-stream"),
+            status_code=response.status_code,
+            headers_summary={k: v for k, v in response.headers.items() if k.lower() in ["content-type", "server", "date"]},
+            fetch_warnings=warnings or [],
+            source_kind=SourceKind.HTTP_FETCH
+        )
+
+    def _create_error_document(self, url: str, code: str, message: str, severity: WarningSeverity) -> Document:
+        return Document(
+            url=url,
+            content="[FETCH FAILED]", 
+            status_code=0,
+            fetch_warnings=[
+                Warning(code=code, message=message, severity=severity, source_url=url)
+            ],
+            source_kind=SourceKind.HTTP_FETCH
+        )
+
+# Register strategy
+registry.register("fetcher", "httpx", HttpxFetcher)

orga/fetch/strategy.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+from typing import List
+from orga.model import Document
+
+class FetchStrategy(ABC):
+    @abstractmethod
+    async def fetch(self, url: str) -> Document:
+        pass

orga/governance/__init__.py

@@ -0,0 +1,152 @@
+from typing import List, Dict, Any, Optional
+from orga.model import (
+    OrganizationProfile, Evidence, Warning, 
+    WarningSeverity, Confidence, Location, ContactKind
+)
+
+# Default Weights and Reliabilities for Evidence Types
+# Defined in Design Doc 12.2
+SOURCE_TYPE_METRICS = {
+    "jsonld_address": {"weight": 1.0, "reliability": 1.0},
+    "jsonld_org_name": {"weight": 1.0, "reliability": 1.0},
+    "html_attr_tel": {"weight": 0.9, "reliability": 0.95},
+    "html_attr_mailto": {"weight": 0.9, "reliability": 0.95},
+    "html_attr_social": {"weight": 0.8, "reliability": 0.9},
+    "regex_text_validated": {"weight": 0.5, "reliability": 0.7},
+    "parsed_address": {"weight": 0.7, "reliability": 0.8},
+    "heuristic_text": {"weight": 0.3, "reliability": 0.5},
+    "text_matcher_validated": {"weight": 0.6, "reliability": 0.7},
+}
+
+class ScoringEngine:
+    """
+    Implements mathematical scoring formulas from Design Document Section 12.3.
+    """
+
+    def calculate_field_score(self, evidences: List[Evidence]) -> float:
+        """
+        Calculate field-level score using weighted average formula:
+        score = sum(w_i * r_i) / sum(w_i)
+        """
+        if not evidences:
+            return 0.0
+        
+        total_weighted_reliability = 0.0
+        total_weight = 0.0
+        
+        for ev in evidences:
+            metrics = SOURCE_TYPE_METRICS.get(ev.source_type, {"weight": 0.5, "reliability": 0.5})
+            w = metrics["weight"]
+            r = ev.confidence_score if ev.confidence_score > 0 else metrics["reliability"]
+            
+            total_weighted_reliability += w * r
+            total_weight += w
+            
+        return round(total_weighted_reliability / total_weight, 2) if total_weight > 0 else 0.0
+
+    def calculate_profile_score(self, profile: OrganizationProfile) -> float:
+        """
+        Calculate profile-level score with completeness penalty:
+        score_profile = (sum(alpha_f * score_f)) * beta_completeness
+        """
+        # alpha_f: relative importance of fields
+        weights = {
+            "name": 0.3,
+            "locations": 0.4,
+            "contacts": 0.3 # phones + emails + socials
+        }
+        
+        sum_weighted_scores = 0.0
+        
+        # 1. Name score
+        if profile.name:
+            sum_weighted_scores += weights["name"] * 1.0
+            
+        # 2. Locations score
+        if profile.locations:
+            loc_scores = [loc.confidence for loc in profile.locations]
+            avg_loc_score = max(loc_scores) if loc_scores else 0.0 # Use max for profile strength
+            sum_weighted_scores += weights["locations"] * avg_loc_score
+            
+        # 3. Contacts score
+        all_contacts = profile.phones + profile.emails + profile.social_links
+        if all_contacts:
+            contact_scores = [c.confidence for c in all_contacts]
+            avg_contact_score = max(contact_scores) if contact_scores else 0.0
+            sum_weighted_scores += weights["contacts"] * avg_contact_score
+            
+        # beta_completeness: Penalty for missing critical fields (Design Doc 12.3.2)
+        beta = 1.0
+        if not profile.locations:
+            beta *= 0.7
+        if not profile.phones and not profile.emails:
+            beta *= 0.8
+        if not profile.categories:
+            beta *= 0.9
+            
+        return round(sum_weighted_scores * beta, 2)
+
+class WarningRegistry:
+    """
+    Standardized Warning Codes implementation (Design Doc 12.4.1).
+    """
+
+    def scan_for_warnings(self, profile: OrganizationProfile) -> List[Warning]:
+        """
+        Scan a profile and generate standardized warnings according to the contract.
+        """
+        warnings = []
+        
+        # EMPTY_PROFILE
+        if not profile.name and not profile.locations and not profile.phones and not profile.emails:
+            warnings.append(Warning(
+                code="EMPTY_PROFILE",
+                message="No significant profile data extracted",
+                severity=WarningSeverity.ERROR
+            ))
+            return warnings
+
+        # NO_LOCATION_FOUND
+        if not profile.locations:
+            warnings.append(Warning(
+                code="NO_LOCATION_FOUND",
+                message="No physical address found in documentation",
+                severity=WarningSeverity.WARNING
+            ))
+        else:
+            # ADDRESS_PARTIALLY_PARSED
+            for loc in profile.locations:
+                if loc.address.raw and not (loc.address.street or loc.address.city):
+                    warnings.append(Warning(
+                        code="ADDRESS_PARTIALLY_PARSED",
+                        message="Address found but only raw string was extracted",
+                        severity=WarningSeverity.WARNING,
+                        related_field="locations"
+                    ))
+                    break
+
+        # NO_CONTACT_FOUND
+        if not profile.phones and not profile.emails and not profile.social_links:
+            warnings.append(Warning(
+                code="NO_CONTACT_FOUND",
+                message="No telephone, email or social media links found",
+                severity=WarningSeverity.WARNING
+            ))
+
+        # CLASSIFICATION_LOW_CONFIDENCE (Aligned with Design Doc 12.4.1)
+        if not profile.categories:
+            warnings.append(Warning(
+                code="CLASSIFICATION_LOW_CONFIDENCE",
+                message="Business classification confidence is low or no categories found",
+                severity=WarningSeverity.WARNING
+            ))
+
+        # LOW_CONFIDENCE_FIELD
+        if profile.confidence and profile.confidence.overall_score < 0.4:
+             warnings.append(Warning(
+                code="LOW_CONFIDENCE_FIELD",
+                message=f"Overall profile confidence is low ({profile.confidence.overall_score})",
+                severity=WarningSeverity.WARNING
+            ))
+
+        return warnings