codetour-cli 0.1.0__py3-none-any.whl

codetour_cli/review_parser.py

@@ -0,0 +1,348 @@
+"""
+Review report parser using schema-driven parsing.
+
+Parses migration review reports to extract user corrections and decisions.
+"""
+
+import re
+import structlog
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+from dataclasses import dataclass
+
+from codetour_cli.schema_models import ParsingSchema, FieldSchema, FieldType, ValidationRule
+
+log = structlog.get_logger()
+
+
+@dataclass
+class StepReview:
+    """Parsed review data for a single step."""
+    step_number: int
+    status_checked: bool
+    tentative_file: str
+    tentative_line: Optional[int]  # None if "-" (deletion marker)
+    confidence: Optional[int]
+    description_edited: bool = False
+    edited_description: Optional[str] = None
+
+    def is_marked_for_deletion(self) -> bool:
+        """Check if step is marked for deletion (file and line both = '-')."""
+        return self.tentative_line is None and self.tentative_file == "-"
+
+
+@dataclass
+class ReviewReport:
+    """Parsed review report with all step corrections."""
+    tour_name: str
+    source_commit: str
+    target_commit: str
+    mode: str
+    step_reviews: List[StepReview]
+
+    def get_review(self, step_number: int) -> Optional[StepReview]:
+        """Get review for specific step number."""
+        for review in self.step_reviews:
+            if review.step_number == step_number:
+                return review
+        return None
+
+    def get_corrected_steps(self) -> List[StepReview]:
+        """Get only steps that have been reviewed (checkbox checked)."""
+        return [r for r in self.step_reviews if r.status_checked]
+
+    def get_deletion_steps(self) -> List[StepReview]:
+        """Get steps marked for deletion."""
+        return [r for r in self.step_reviews if r.is_marked_for_deletion()]
+
+
+def extract_frontmatter(content: str) -> Dict[str, Any]:
+    """
+    Extract YAML frontmatter from markdown content.
+
+    Returns:
+        Dictionary of frontmatter values (empty if no frontmatter)
+    """
+    import yaml
+
+    # Look for YAML frontmatter delimited by ---
+    match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
+    if not match:
+        return {}
+
+    try:
+        frontmatter = yaml.safe_load(match.group(1))
+        return frontmatter or {}
+    except yaml.YAMLError as e:
+        log.warning("failed_to_parse_frontmatter", error=str(e))
+        return {}
+
+
+def load_parsing_schema(report_content: str) -> ParsingSchema:
+    """
+    Load parsing schema using smart defaults.
+
+    Implements the refined Iteration 2 approach:
+    1. Extract frontmatter to get schema version/language
+    2. Load default schema for that version
+    3. Merge with custom end-schema if present
+    """
+    # 1. Extract frontmatter
+    frontmatter = extract_frontmatter(report_content)
+    schema_version = frontmatter.get('schema_version', '1.0')
+    language = frontmatter.get('language', 'en')
+
+    # 2. Load default schema for version
+    schemas_dir = Path(__file__).parent / 'schemas'
+
+    # Try language-specific schema first
+    if language != 'en':
+        lang_schema_path = schemas_dir / f"review_report_v{schema_version}_{language}.yaml"
+        if lang_schema_path.exists():
+            return ParsingSchema.from_yaml(lang_schema_path)
+
+    # Fall back to default English schema
+    default_schema_path = schemas_dir / f"review_report_v{schema_version}.yaml"
+
+    if not default_schema_path.exists():
+        log.warning(
+            "schema_version_not_found",
+            requested=schema_version,
+            fallback="1.0"
+        )
+        default_schema_path = schemas_dir / "review_report_v1.0.yaml"
+
+    schema = ParsingSchema.from_yaml(default_schema_path)
+
+    # 3. Check for custom schema override (future: internationalization)
+    # For now, we only use built-in schemas
+
+    return schema
+
+
+def parse_checkbox(value: str) -> bool:
+    """Parse checkbox value: [x] = True, [ ] = False."""
+    return value.strip().lower() == 'x'
+
+
+def parse_integer_or_dash(value: str) -> Optional[int]:
+    """Parse integer or dash. Returns None if dash."""
+    value = value.strip()
+    if value == '-':
+        return None
+    try:
+        return int(value)
+    except ValueError:
+        raise ValueError(f"Expected integer or '-', got: {value}")
+
+
+def validate_field_value(value: Any, field: FieldSchema) -> Any:
+    """
+    Validate and convert field value according to schema rules.
+
+    Args:
+        value: Raw extracted value
+        field: Field schema with validation rules
+
+    Returns:
+        Validated and converted value
+
+    Raises:
+        ValueError: If validation fails
+    """
+    # Apply type conversions
+    if field.type == FieldType.CHECKBOX:
+        value = parse_checkbox(value)
+    elif field.type == FieldType.INTEGER:
+        value = int(value)
+    elif field.type == FieldType.STRING_OR_DASH:
+        # Keep as string initially, will validate below
+        pass
+    elif field.type == FieldType.MULTILINE:
+        # Extract quoted lines
+        lines = value.strip().split('\n')
+        cleaned_lines = []
+        for line in lines:
+            line = line.strip()
+            if line.startswith('>'):
+                cleaned_lines.append(line[1:].strip())
+        value = '\n'.join(cleaned_lines)
+
+    # Apply validation rules
+    for rule in field.validation:
+        if rule == ValidationRule.INTEGER_OR_DASH:
+            value = parse_integer_or_dash(value)
+        elif rule == ValidationRule.POSITIVE_IF_INTEGER:
+            if isinstance(value, int) and value <= 0:
+                raise ValueError(f"Value must be positive, got: {value}")
+        elif rule == ValidationRule.NOT_EMPTY:
+            if not value or (isinstance(value, str) and not value.strip()):
+                raise ValueError("Value cannot be empty")
+
+    return value
+
+
+def parse_step_section(step_content: str, step_number: int, schema: ParsingSchema) -> StepReview:
+    """
+    Parse a single step section from the review report.
+
+    Args:
+        step_content: Markdown content of the step section
+        step_number: Step number (1-indexed)
+        schema: Parsing schema to use
+
+    Returns:
+        StepReview with extracted values
+
+    Raises:
+        ValueError: If required fields are missing or invalid
+    """
+    extracted = {}
+
+    # Extract each field according to schema
+    for field in schema.fields:
+        # Skip optional fields based on version
+        if field.since_version:
+            # For now, we parse all fields; version filtering can be added later
+            pass
+
+        # Try to extract field value
+        pattern = re.compile(field.pattern, re.MULTILINE | re.DOTALL)
+        match = pattern.search(step_content)
+
+        if match:
+            raw_value = match.group(1)
+
+            # Check conditional fields
+            if field.conditional:
+                # Only parse if the conditional field is checked
+                conditional_field = schema.get_field(field.conditional)
+                if conditional_field and conditional_field.logical_name in extracted:
+                    if not extracted[conditional_field.logical_name]:
+                        # Conditional not met, skip this field
+                        continue
+
+            # Validate and convert value
+            try:
+                validated_value = validate_field_value(raw_value, field)
+                extracted[field.logical_name] = validated_value
+            except ValueError as e:
+                log.warning(
+                    "field_validation_failed",
+                    step=step_number,
+                    field=field.logical_name,
+                    error=str(e)
+                )
+                if field.required:
+                    raise
+        else:
+            # Field not found
+            if field.required:
+                raise ValueError(
+                    f"Required field '{field.logical_name}' not found in step {step_number}"
+                )
+
+    # Build StepReview from extracted values
+    return StepReview(
+        step_number=step_number,
+        status_checked=extracted.get('status', False),
+        tentative_file=extracted.get('tentative_file', ''),
+        tentative_line=extracted.get('tentative_line'),
+        confidence=extracted.get('confidence'),
+        description_edited=extracted.get('description_edited', False),
+        edited_description=extracted.get('edited_description')
+    )
+
+
+def parse_review_report(report_path: Path) -> ReviewReport:
+    """
+    Parse a review report markdown file.
+
+    Args:
+        report_path: Path to the review report markdown file
+
+    Returns:
+        ReviewReport with all parsed step reviews
+
+    Raises:
+        FileNotFoundError: If report file doesn't exist
+        ValueError: If report format is invalid
+    """
+    if not report_path.exists():
+        raise FileNotFoundError(f"Review report not found: {report_path}")
+
+    content = report_path.read_text()
+
+    # Load schema
+    schema = load_parsing_schema(content)
+
+    log.info(
+        "parsing_review_report",
+        path=str(report_path),
+        schema_version=schema.version
+    )
+
+    # Extract metadata from header (simple regex for now)
+    tour_name_match = re.search(r'\*\*Tour\*\*:\s*(.+)', content)
+    source_match = re.search(r'\*\*Source commit\*\*:\s*(\w+)', content)
+    target_match = re.search(r'\*\*Target commit\*\*:\s*(?:HEAD,\s*)?(\w+)', content)
+    mode_match = re.search(r'\*\*Mode\*\*:\s*(.+)', content)
+
+    tour_name = tour_name_match.group(1) if tour_name_match else "unknown"
+    source_commit = source_match.group(1) if source_match else "unknown"
+    target_commit = target_match.group(1) if target_match else "unknown"
+    mode = mode_match.group(1) if mode_match else "unknown"
+
+    # Split into step sections
+    step_pattern = re.compile(schema.step_marker_pattern)
+    step_sections = []
+
+    # Find all step headers
+    for match in step_pattern.finditer(content):
+        step_num = int(match.group(1))
+        start = match.start()
+        step_sections.append((step_num, start))
+
+    # Extract content for each step
+    step_reviews = []
+    for i, (step_num, start) in enumerate(step_sections):
+        # Get content until next step or end
+        if i < len(step_sections) - 1:
+            end = step_sections[i + 1][1]
+        else:
+            # Last step - go until appendix or end
+            appendix_match = re.search(r'^## Appendix:', content[start:], re.MULTILINE)
+            if appendix_match:
+                end = start + appendix_match.start()
+            else:
+                end = len(content)
+
+        step_content = content[start:end]
+
+        try:
+            review = parse_step_section(step_content, step_num, schema)
+            step_reviews.append(review)
+            log.debug(
+                "parsed_step",
+                step=step_num,
+                checked=review.status_checked,
+                file=review.tentative_file,
+                line=review.tentative_line
+            )
+        except ValueError as e:
+            log.error("failed_to_parse_step", step=step_num, error=str(e))
+            # Continue parsing other steps
+
+    log.info(
+        "parsed_review_report",
+        total_steps=len(step_reviews),
+        corrected=len([r for r in step_reviews if r.status_checked])
+    )
+
+    return ReviewReport(
+        tour_name=tour_name,
+        source_commit=source_commit,
+        target_commit=target_commit,
+        mode=mode,
+        step_reviews=step_reviews
+    )

codetour_cli/schema_models.py

@@ -0,0 +1,203 @@
+"""
+Pydantic models for parsing schema structure.
+
+Provides type-safe, validated schema definitions for review report parsing.
+"""
+
+import re
+from enum import Enum
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+
+from pydantic import BaseModel, Field, field_validator, ConfigDict
+
+
+class FieldType(str, Enum):
+    """Supported field types for parseable fields."""
+    CHECKBOX = "checkbox"
+    STRING = "string"
+    INTEGER = "integer"
+    STRING_OR_DASH = "string_or_dash"
+    MULTILINE = "multiline"
+
+
+class ValidationRule(str, Enum):
+    """Supported validation rules for field values."""
+    INTEGER_OR_DASH = "integer_or_dash"
+    POSITIVE_IF_INTEGER = "positive_if_integer"
+    NOT_EMPTY = "not_empty"
+
+
+class FieldSchema(BaseModel):
+    """Schema definition for a single parseable field in review reports."""
+
+    logical_name: str = Field(
+        description="Internal field name used for parsing (constant across languages)"
+    )
+    display_label: str = Field(
+        description="Human-readable label shown in report (can be translated)"
+    )
+    pattern: str = Field(
+        description="Regex pattern to extract field value from markdown"
+    )
+    type: FieldType = Field(
+        description="Data type of the field value"
+    )
+    required: bool = Field(
+        default=True,
+        description="Whether this field must be present in all reports"
+    )
+    since_version: Optional[str] = Field(
+        default=None,
+        description="Schema version that introduced this field (for backwards compatibility)"
+    )
+    conditional: Optional[str] = Field(
+        default=None,
+        description="Logical name of field that must be checked for this field to be parsed"
+    )
+    validation: List[ValidationRule] = Field(
+        default_factory=list,
+        description="Validation rules to apply to extracted values"
+    )
+
+    model_config = ConfigDict(use_enum_values=True)
+
+    @field_validator('pattern')
+    @classmethod
+    def validate_pattern(cls, v: str) -> str:
+        """Ensure regex pattern is valid and compiles."""
+        try:
+            re.compile(v)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern: {e}")
+        return v
+
+
+class ParsingSchema(BaseModel):
+    """Complete parsing schema for review reports."""
+
+    version: str = Field(
+        pattern=r'^\d+\.\d+$',
+        description="Schema version in X.Y format"
+    )
+    language: str = Field(
+        default="en",
+        description="Language code (en, fr, es, etc.)"
+    )
+    step_marker_pattern: str = Field(
+        default=r'### Step (\d+):',
+        description="Regex pattern to identify step sections"
+    )
+    fields: List[FieldSchema] = Field(
+        description="List of parseable field definitions"
+    )
+
+    model_config = ConfigDict(use_enum_values=True)
+
+    @field_validator('version')
+    @classmethod
+    def validate_version(cls, v: str) -> str:
+        """Ensure version follows semantic versioning format."""
+        if not re.match(r'^\d+\.\d+$', v):
+            raise ValueError('Version must be in format X.Y (e.g., 1.0, 1.1)')
+        return v
+
+    def get_field(self, logical_name: str) -> Optional[FieldSchema]:
+        """
+        Get field schema by logical name.
+
+        Args:
+            logical_name: Internal field name to look up
+
+        Returns:
+            FieldSchema if found, None otherwise
+        """
+        for field in self.fields:
+            if field.logical_name == logical_name:
+                return field
+        return None
+
+    def get_required_fields(self) -> List[FieldSchema]:
+        """Get list of required fields."""
+        return [f for f in self.fields if f.required]
+
+    def get_optional_fields(self) -> List[FieldSchema]:
+        """Get list of optional fields."""
+        return [f for f in self.fields if not f.required]
+
+    def merge(self, override: 'ParsingSchema') -> 'ParsingSchema':
+        """
+        Merge with override schema (override takes precedence).
+
+        Args:
+            override: Schema to merge in (takes precedence over self)
+
+        Returns:
+            New ParsingSchema with merged field definitions
+        """
+        # Start with base schema fields
+        merged_fields: Dict[str, FieldSchema] = {
+            f.logical_name: f for f in self.fields
+        }
+
+        # Override with custom schema fields
+        for field in override.fields:
+            merged_fields[field.logical_name] = field
+
+        return ParsingSchema(
+            version=override.version,
+            language=override.language or self.language,
+            step_marker_pattern=override.step_marker_pattern or self.step_marker_pattern,
+            fields=list(merged_fields.values())
+        )
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> 'ParsingSchema':
+        """
+        Load schema from YAML file.
+
+        Args:
+            path: Path to YAML file containing schema definition
+
+        Returns:
+            ParsingSchema instance
+
+        Raises:
+            FileNotFoundError: If schema file doesn't exist
+            ValidationError: If schema YAML is invalid
+        """
+        import yaml
+
+        if not path.exists():
+            raise FileNotFoundError(f"Schema file not found: {path}")
+
+        data = yaml.safe_load(path.read_text())
+        return cls(**data)
+
+    def to_yaml(self, path: Path) -> None:
+        """
+        Save schema to YAML file.
+
+        Args:
+            path: Path where to save the schema
+        """
+        import yaml
+
+        # Convert to dict for YAML serialization
+        data = self.dict(exclude_none=True)
+
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(yaml.dump(data, sort_keys=False, allow_unicode=True))
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ParsingSchema':
+        """
+        Create schema from dictionary (useful for parsing embedded schemas).
+
+        Args:
+            data: Dictionary containing schema definition
+
+        Returns:
+            ParsingSchema instance
+        """
+        return cls(**data)

codetour_cli/tour/__init__.py

@@ -0,0 +1,6 @@
+"""
+CodeTour schema and manipulation module.
+
+Handles parsing, validation, and updating of .tour files according to the
+CodeTour specification.
+"""