mdb-cli 0.1.0__py3-none-any.whl

mdb/models.py

@@ -0,0 +1,81 @@
+"""Data model classes for mdb tools."""
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class FeedMarker:
+    """A parsed feed marker found in a markdown file."""
+    line_number: int
+    scope: str
+    raw_query: str
+    table_name: str | None
+    direction: str = "🌀"
+    end_line_number: int | None = None
+
+
+@dataclass
+class MarkdownTable:
+    """A parsed markdown table immediately following a query marker."""
+    columns: list[str]
+    rows: list[list[str]]
+    start_line: int
+    end_line: int
+    column_types: list[str] | None = None
+
+
+@dataclass
+class FeedTask:
+    """A marker and its associated table, ready for database execution."""
+    marker: FeedMarker
+    table: MarkdownTable
+    resolved_db_path: str
+
+
+@dataclass
+class FeedResult:
+    """The outcome of executing a single feed task."""
+    task: FeedTask
+    success: bool
+    error: str | None = None
+    rows_written: int = 0
+    skipped: bool = False
+
+
+@dataclass
+class TapMarker:
+    """A parsed tap marker found in a markdown file."""
+    line_number: int
+    scope: str
+    raw_query: str
+    direction: str = "💎"
+    end_line_number: int | None = None
+
+
+@dataclass
+class TapResult:
+    """The outcome of executing a single tap query."""
+    marker: TapMarker
+    success: bool
+    columns: list[str] = field(default_factory=list)
+    rows: list[list[str]] = field(default_factory=list)
+    error: str | None = None
+
+
+@dataclass
+class FileMarkers:
+    """All parsed markers and content for a single markdown file."""
+    filepath: str
+    content: str
+    lines: list[str]
+    tap_markers: list = field(default_factory=list)
+    feed_markers: list = field(default_factory=list)
+
+
+@dataclass
+class FeedConflict:
+    """A detected cross-file feed conflict."""
+    scope: str
+    resolved_path: str
+    table_name: str
+    conflicting: list  # list of (filepath, line_number) tuples

mdb/parser.py

@@ -0,0 +1,609 @@
+"""Markdown parsing: markers, tables, table name extraction."""
+
+import hashlib
+import os
+import re
+import sys
+
+from mdb.models import FeedMarker, MarkdownTable, TapMarker
+
+MDB_DIR = ".mdb"
+IMPLICIT_DB_FILENAME = "_.db"
+
+
+def compute_include_hash(include_paths: str) -> str:
+    """Compute a deterministic MD5 hash from the --include argument value.
+
+    Splits on whitespace, sorts alphabetically, joins with single space,
+    and returns the MD5 hex digest. Duplicates are preserved (no dedup).
+    Caller must ensure include_paths is non-empty and non-whitespace-only.
+    """
+    tokens = include_paths.split()
+    tokens.sort()
+    normalized = " ".join(tokens)
+    return hashlib.md5(normalized.encode("utf-8")).hexdigest()
+
+
+def resolve_scope_path(scope: str, md_filepath: str, include_paths: str | None = None) -> str:
+    """Resolve a database path from a named scope and optional include paths.
+
+    Produces a two-segment filename <include-part>-<scope-part>.db in the
+    .mdb/ directory under the current working directory.
+
+    Filename resolution:
+      - Both unset:        _.db
+      - Scope only:        _-<md5(scope)>.db
+      - Include only:      <include-hash>-_.db
+      - Both set:          <include-hash>-<md5(scope)>.db
+
+    The md_filepath parameter is retained for API compatibility but is not
+    used for resolution. Empty/whitespace include_paths is treated as unset.
+    """
+    has_scope = bool(scope.strip())
+    has_include = bool(include_paths and include_paths.strip())
+
+    if not has_scope and not has_include:
+        filename = IMPLICIT_DB_FILENAME
+    elif has_scope and not has_include:
+        scope_hash = hashlib.md5(scope.encode("utf-8")).hexdigest()
+        filename = f"_-{scope_hash}.db"
+    elif not has_scope and has_include:
+        include_hash = compute_include_hash(include_paths)
+        filename = f"{include_hash}-_.db"
+    else:
+        include_hash = compute_include_hash(include_paths)
+        scope_hash = hashlib.md5(scope.encode("utf-8")).hexdigest()
+        filename = f"{include_hash}-{scope_hash}.db"
+
+    return os.path.normpath(os.path.join(os.getcwd(), MDB_DIR, filename))
+
+
+# Regex for FROM <table_name> (case-insensitive)
+_FROM_RE = re.compile(r'\bFROM\s+(\w+)', re.IGNORECASE)
+
+# Regex for feed marker (🌀): `💾 [scope] 🌀 <query>`
+# Includes the table below as part of the dataset (table required -- defines column schema)
+# scope group is optional to support implicit scope (e.g. `💾 🌀 SELECT ...`)
+_MARKER_RE = re.compile(
+    r'^`💾\s*(.*?)\s*🌀\s*(.+?)`\s*$'
+)
+
+# Regex for tap marker (💎): `💾 [scope] 💎 <query>`
+# Executes query on dataset and renders results as the table below (table optional -- auto-generated if absent)
+# scope group is optional to support implicit scope (e.g. `💾 💎 SELECT ...`)
+_READ_MARKER_RE = re.compile(
+    r'^`💾\s*(.*?)\s*💎\s*(.+?)`\s*$'
+)
+
+# Regex for bare 💾 marker header (inside codeblock, not backtick-wrapped)
+# scope group is optional to support implicit scope (e.g. `💾 💎`)
+_MARKER_HEADER_RE = re.compile(
+    r'^💾\s*(.*?)\s*(💎|🌀)\s*(.*?)\s*$'
+)
+
+# Regex for opening fence: 0-3 spaces indent, 3+ backticks or tildes, optional language tag
+_OPENING_FENCE_RE = re.compile(
+    r'^( {0,3})((`{3,})|~{3,})([^`]*)?$'
+)
+
+def extract_table_name(query: str) -> str | None:
+    """Extract the target table name from a SELECT query string.
+
+    Matches FROM <identifier> pattern (case-insensitive).
+    Returns the first identifier after FROM, or None if unparseable.
+    """
+    match = _FROM_RE.search(query)
+    if match:
+        return match.group(1)
+    return None
+
+
+def _parse_marker_header(line: str) -> tuple[str, str, str] | None:
+    """Parse a bare 💾 marker header line (not backtick-wrapped).
+
+    Returns (scope, direction, sql_tail) or None.
+    Direction is exactly '💎' or '🌀'.
+    """
+    match = _MARKER_HEADER_RE.match(line.strip())
+    if match:
+        scope = match.group(1).strip()
+        direction = match.group(2)
+        sql_tail = match.group(3).strip()
+        return (scope, direction, sql_tail)
+    return None
+
+
+def _is_opening_fence(line: str) -> tuple[str, int, int] | None:
+    """Check if a line is a fenced codeblock opening fence.
+
+    Returns (fence_char, fence_length, indent) or None.
+    Matches ``` or ~~~ with 3+ chars, 0-3 spaces indent, optional language tag.
+    """
+    match = _OPENING_FENCE_RE.match(line.rstrip())
+    if not match:
+        return None
+    indent_str = match.group(1)
+    fence_str = match.group(2)
+    fence_char = fence_str[0]
+    fence_length = len(fence_str)
+    return (fence_char, fence_length, len(indent_str))
+
+
+def _is_closing_fence(line: str, fence_char: str, fence_length: int) -> bool:
+    """Check if a line is a closing fence matching the opening fence.
+
+    Closing fence: same character, >= same length, 0-3 spaces indent, no other content.
+    """
+    stripped = line.rstrip()
+    # Count leading spaces (0-3 allowed)
+    content = stripped.lstrip(' ')
+    indent = len(stripped) - len(content)
+    if indent > 3:
+        return False
+    # Must be only fence chars
+    if not content:
+        return False
+    if not all(c == fence_char for c in content):
+        return False
+    if len(content) < fence_length:
+        return False
+    return True
+
+
+def _find_all_markers(content: str) -> list[TapMarker | FeedMarker]:
+    """Scan content for both single-line and codeblock markers in a single pass.
+
+    Returns all markers (read and write) in document order.
+    """
+    lines = content.splitlines()
+    markers = []
+
+    # State machine
+    state = "SCANNING"
+    fence_char = None
+    fence_length = 0
+    fence_start_line = 0  # 1-based
+    header_info = None  # (scope, direction, sql_tail)
+    sql_lines = []
+
+    for line_num_0, line in enumerate(lines):
+        line_num = line_num_0 + 1  # 1-based
+
+        if state == "SCANNING":
+            # Check for opening fence
+            fence_info = _is_opening_fence(line)
+            if fence_info is not None:
+                fence_char, fence_length, _ = fence_info
+                fence_start_line = line_num
+                state = "CHECK_FIRST_LINE"
+                header_info = None
+                sql_lines = []
+                continue
+
+            # Check for single-line backtick markers
+            stripped = line.strip()
+            read_match = _READ_MARKER_RE.match(stripped)
+            if read_match:
+                scope = read_match.group(1).strip()
+                raw_query = read_match.group(2).strip()
+                markers.append(TapMarker(
+                    line_number=line_num,
+                    scope=scope,
+                    raw_query=raw_query,
+                    direction="💎",
+                ))
+                continue
+
+            write_match = _MARKER_RE.match(stripped)
+            if write_match:
+                scope = write_match.group(1).strip()
+                raw_query = write_match.group(2).strip()
+                table_name = extract_table_name(raw_query)
+                markers.append(FeedMarker(
+                    line_number=line_num,
+                    scope=scope,
+                    raw_query=raw_query,
+                    table_name=table_name,
+                    direction="🌀",
+                ))
+                continue
+
+        elif state == "CHECK_FIRST_LINE":
+            # Check if this line is a closing fence (empty codeblock)
+            if _is_closing_fence(line, fence_char, fence_length):
+                state = "SCANNING"
+                continue
+
+            # Check if first content line is a 💾 marker header
+            header_info = _parse_marker_header(line)
+            if header_info is not None:
+                sql_lines = []
+                if header_info[2]:  # sql_tail
+                    sql_lines.append(header_info[2])
+                state = "IN_CODEBLOCK"
+            else:
+                # Not an mdb codeblock - skip until closing fence
+                state = "SKIP_CODEBLOCK"
+
+        elif state == "IN_CODEBLOCK":
+            if _is_closing_fence(line, fence_char, fence_length):
+                # Assemble query and emit marker
+                assembled = "\n".join(sql_lines).strip()
+                if not assembled:
+                    print(
+                        f"Warning: Empty query in codeblock at line {fence_start_line}, skipping",
+                        file=sys.stderr,
+                    )
+                else:
+                    scope, direction, _ = header_info
+                    if direction == "💎":
+                        markers.append(TapMarker(
+                            line_number=fence_start_line,
+                            scope=scope,
+                            raw_query=assembled,
+                            direction="💎",
+                            end_line_number=line_num,
+                        ))
+                    elif direction == "🌀":
+                        table_name = extract_table_name(assembled)
+                        markers.append(FeedMarker(
+                            line_number=fence_start_line,
+                            scope=scope,
+                            raw_query=assembled,
+                            table_name=table_name,
+                            direction="🌀",
+                            end_line_number=line_num,
+                        ))
+                state = "SCANNING"
+            else:
+                sql_lines.append(line)
+
+        elif state == "SKIP_CODEBLOCK":
+            if _is_closing_fence(line, fence_char, fence_length):
+                state = "SCANNING"
+
+    # Handle unclosed codeblock at EOF
+    if state == "IN_CODEBLOCK":
+        print(
+            f"Warning: Unclosed codeblock at line {fence_start_line}, skipping",
+            file=sys.stderr,
+        )
+
+    return markers
+
+
+def parse_markdown_table(lines: list[str], start_index: int) -> MarkdownTable | None:
+    """Parse a markdown table starting from a given line index.
+
+    The line at start_index must be a valid pipe-delimited header row.
+    The next line must be a valid separator row.
+    Returns MarkdownTable or None if no valid table found.
+    """
+    if start_index >= len(lines):
+        return None
+
+    header_line = lines[start_index]
+    if '|' not in header_line:
+        return None
+
+    # Parse header
+    columns = _parse_pipe_row(header_line)
+    if not columns:
+        return None
+
+    # Check separator row
+    sep_index = start_index + 1
+    if sep_index >= len(lines):
+        return None
+
+    sep_line = lines[sep_index]
+    if not _is_separator_row(sep_line):
+        return None
+
+    # Parse data rows
+    rows = []
+    i = sep_index + 1
+    while i < len(lines):
+        line = lines[i]
+        if '|' not in line:
+            break
+        cells = _parse_pipe_row(line)
+        # Pad or truncate to match column count
+        while len(cells) < len(columns):
+            cells.append("")
+        cells = cells[:len(columns)]
+        rows.append(cells)
+        i += 1
+
+    start_line = start_index + 1  # 1-based
+    end_line = (i if rows else sep_index + 1)  # 1-based
+
+    return MarkdownTable(
+        columns=columns,
+        rows=rows,
+        start_line=start_line,
+        end_line=end_line,
+    )
+
+
+def _parse_pipe_row(line: str) -> list[str]:
+    """Parse a pipe-delimited row, returning trimmed cell values."""
+    stripped = line.strip()
+    if stripped.startswith('|'):
+        stripped = stripped[1:]
+    if stripped.endswith('|'):
+        stripped = stripped[:-1]
+    return [cell.strip() for cell in stripped.split('|')]
+
+
+def _is_separator_row(line: str) -> bool:
+    """Check if a line is a valid markdown table separator row."""
+    stripped = line.strip()
+    if '|' not in stripped:
+        return False
+    cells = _parse_pipe_row(stripped)
+    for cell in cells:
+        cleaned = cell.strip().strip(':')
+        if not cleaned or not all(c == '-' for c in cleaned):
+            return False
+    return True
+
+
+def find_tap_markers(content: str) -> list[TapMarker]:
+    """Scan markdown content for tap markers (💎 TapMarker).
+
+    Tap markers execute a query and render results as the table below.
+    Table is optional -- auto-generated from DB if absent.
+    Matches both single-line backtick and fenced codeblock patterns.
+    Returns markers in document order.
+    """
+    return [m for m in _find_all_markers(content) if isinstance(m, TapMarker)]
+
+
+def find_feed_markers(content: str) -> list[FeedMarker]:
+    """Scan markdown content for feed markers (🌀 FeedMarker).
+
+    Feed markers include the table below as part of the dataset.
+    Table is required -- defines column schema (source of truth).
+    Matches both single-line backtick and fenced codeblock patterns.
+    Returns markers in document order.
+    """
+    return [m for m in _find_all_markers(content) if isinstance(m, FeedMarker)]
+
+
+def find_adjacent_table(lines, marker_line_number, end_line_number=None):
+    """Find the markdown table adjacent to a marker.
+
+    Searches forward from the marker (or closing fence) for the first
+    non-blank line and attempts to parse it as a markdown table.
+    Returns (MarkdownTable, line_index) or (None, None).
+    """
+    search_after = end_line_number if end_line_number else marker_line_number
+    start = search_after  # 0-based index of line after marker/closing fence
+    for i in range(start, len(lines)):
+        if lines[i].strip():
+            table = parse_markdown_table(lines, i)
+            return (table, i) if table is not None else (None, None)
+    return (None, None)
+
+
+# ---------------------------------------------------------------------------
+# Type annotation parsing (consolidated from type_parser.py)
+# ---------------------------------------------------------------------------
+
+VALID_TYPES: frozenset[str] = frozenset({"TEXT", "INTEGER", "REAL", "NUMERIC", "BLOB"})
+
+
+def parse_header_types(header_cells: list[str]) -> tuple[list[str], list[str], list[str], list[str]]:
+    """Parse type annotations from markdown table header cells.
+
+    Uses last-colon delimiter to split name from type.
+    Returns (column_names, column_types, warnings, errors).
+    """
+    column_names: list[str] = []
+    column_types: list[str] = []
+    warnings: list[str] = []
+    errors: list[str] = []
+
+    for cell in header_cells:
+        cell = cell.strip()
+        colon_idx = cell.rfind(":")
+        if colon_idx == -1:
+            # No colon: whole cell is the name, type defaults to TEXT
+            column_names.append(cell)
+            column_types.append("TEXT")
+        else:
+            name = cell[:colon_idx].strip()
+            type_str = cell[colon_idx + 1:].strip()
+            if not type_str:
+                # Empty type annotation (e.g., "id:")
+                column_names.append(name)
+                column_types.append("TEXT")
+                warnings.append(f"Empty type annotation on column '{name}'")
+            elif type_str.upper() in VALID_TYPES:
+                column_names.append(name)
+                column_types.append(type_str.upper())
+            else:
+                # Invalid type
+                column_names.append(name)
+                column_types.append("TEXT")
+                errors.append(f"Invalid type '{type_str}' on column '{name}'")
+
+    return column_names, column_types, warnings, errors
+
+
+def parse_header_types_with_annotation_flags(
+    header_cells: list[str],
+) -> tuple[list[str], list[str], list[bool], list[str], list[str]]:
+    """Extended version that also returns which columns were explicitly annotated.
+
+    Returns (column_names, column_types, was_annotated, warnings, errors).
+    """
+    column_names: list[str] = []
+    column_types: list[str] = []
+    was_annotated: list[bool] = []
+    warnings: list[str] = []
+    errors: list[str] = []
+
+    for cell in header_cells:
+        cell = cell.strip()
+        colon_idx = cell.rfind(":")
+        if colon_idx == -1:
+            column_names.append(cell)
+            column_types.append("TEXT")
+            was_annotated.append(False)
+        else:
+            name = cell[:colon_idx].strip()
+            type_str = cell[colon_idx + 1:].strip()
+            if not type_str:
+                column_names.append(name)
+                column_types.append("TEXT")
+                was_annotated.append(False)
+                warnings.append(f"Empty type annotation on column '{name}'")
+            elif type_str.upper() in VALID_TYPES:
+                column_names.append(name)
+                column_types.append(type_str.upper())
+                was_annotated.append(True)
+            else:
+                column_names.append(name)
+                column_types.append("TEXT")
+                was_annotated.append(False)
+                errors.append(f"Invalid type '{type_str}' on column '{name}'")
+
+    return column_names, column_types, was_annotated, warnings, errors
+
+
+# Regex to match cast(... as TYPE) - case insensitive
+_CAST_RE = re.compile(r"cast\s*\(\s*.+?\s+as\s+(\w+)\s*\)", re.IGNORECASE)
+
+
+def parse_cast_types(raw_query: str) -> tuple[dict[int, str], list[str]]:
+    """Parse cast(col as type) expressions from a query marker's SELECT list.
+
+    Returns (cast_types, errors) where cast_types maps 0-based position to type.
+    """
+    cast_types: dict[int, str] = {}
+    errors: list[str] = []
+
+    query_upper = raw_query.strip().upper()
+    if query_upper.startswith("SELECT") and "*" in raw_query.split("FROM", 1)[0] if "FROM" in raw_query.upper() else True:
+        # Check for select * (no individual columns)
+        select_part = raw_query.strip()
+        # Extract portion between SELECT and FROM
+        select_match = re.match(r"select\s+(.*?)\s+from\s+", select_part, re.IGNORECASE | re.DOTALL)
+        if select_match is None:
+            return cast_types, errors
+        select_list = select_match.group(1).strip()
+        if select_list == "*":
+            return cast_types, errors
+    else:
+        select_match = re.match(r"select\s+(.*?)\s+from\s+", raw_query.strip(), re.IGNORECASE | re.DOTALL)
+        if select_match is None:
+            return cast_types, errors
+        select_list = select_match.group(1).strip()
+
+    # Split by commas, respecting parentheses
+    columns_expr = _split_select_list(select_list)
+
+    for i, expr in enumerate(columns_expr):
+        expr = expr.strip()
+        cast_match = _CAST_RE.match(expr)
+        if cast_match:
+            type_str = cast_match.group(1).strip()
+            if type_str.upper() in VALID_TYPES:
+                cast_types[i] = type_str.upper()
+            else:
+                errors.append(f"Invalid type '{type_str}' in cast at position {i}")
+
+    return cast_types, errors
+
+
+def _split_select_list(select_list: str) -> list[str]:
+    """Split a SELECT column list by commas, respecting parentheses nesting."""
+    parts: list[str] = []
+    current: list[str] = []
+    depth = 0
+    for ch in select_list:
+        if ch == "(":
+            depth += 1
+            current.append(ch)
+        elif ch == ")":
+            depth -= 1
+            current.append(ch)
+        elif ch == "," and depth == 0:
+            parts.append("".join(current))
+            current = []
+        else:
+            current.append(ch)
+    if current:
+        parts.append("".join(current))
+    return parts
+
+
+def resolve_column_types(
+    header_types: list[str],
+    cast_types: dict[int, str],
+    num_columns: int,
+    was_annotated: list[bool] | None = None,
+) -> list[str]:
+    """Merge header and cast type sources with header precedence.
+
+    For each column position:
+    - If header was explicitly annotated: use header type
+    - Else if position in cast_types: use cast type
+    - Else: TEXT
+    """
+    result: list[str] = []
+    for i in range(num_columns):
+        if was_annotated is not None and i < len(was_annotated) and was_annotated[i]:
+            # Explicitly annotated in header -- header wins
+            result.append(header_types[i] if i < len(header_types) else "TEXT")
+        elif i in cast_types:
+            result.append(cast_types[i])
+        elif i < len(header_types):
+            result.append(header_types[i])
+        else:
+            result.append("TEXT")
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Post-pull-tap-query parser (027)
+# ---------------------------------------------------------------------------
+
+SCOPE_DELIMITER = " \U0001f48e "  # " 💎 "
+
+
+def parse_post_pull_tap_query(arg: str) -> tuple[str, str]:
+    """Parse post-pull tap query argument into (scope_name, sql_query).
+
+    If arg contains ' 💎 ', split on first occurrence.
+    Otherwise, scope_name is empty string.
+    """
+    if SCOPE_DELIMITER in arg:
+        idx = arg.index(SCOPE_DELIMITER)
+        scope_name = arg[:idx].strip()
+        sql_query = arg[idx + len(SCOPE_DELIMITER):].strip()
+        return (scope_name, sql_query)
+    return ("", arg.strip())
+
+
+# ---------------------------------------------------------------------------
+# Pre-push-feed-query parser (028)
+# ---------------------------------------------------------------------------
+
+PUSH_SCOPE_DELIMITER = " \U0001f300 "  # " 🌀 "
+
+
+def parse_pre_push_feed_query(arg: str) -> tuple[str, str]:
+    """Parse pre-push feed query argument into (scope_name, sql_query).
+
+    If arg contains ' 🌀 ', split on first occurrence.
+    Otherwise, scope_name is empty string.
+    """
+    if PUSH_SCOPE_DELIMITER in arg:
+        idx = arg.index(PUSH_SCOPE_DELIMITER)
+        scope_name = arg[:idx].strip()
+        sql_query = arg[idx + len(PUSH_SCOPE_DELIMITER):].strip()
+        return (scope_name, sql_query)
+    return ("", arg.strip())