lare 0.1.0__py3-none-any.whl

lare/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'

lare/cli.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import webbrowser
+from pathlib import Path
+
+import click
+
+from lare.console import console, log_error, log_step, log_success
+
+
+@click.group()
+@click.version_option(package_name='lare')
+def main():
+    '''LARE — Label And Review Engine'''
+    pass
+
+
+@main.command()
+@click.argument('csv_file', required=False)
+@click.option('--config', 'config_path', type=click.Path(exists=True), help='Path to lare_config.toml')
+@click.option('--no-project', is_flag=True, help='Author config only, skip database creation')
+def create(csv_file: str | None, config_path: str | None, no_project: bool):
+    '''Initialize a new LARE project from a CSV dataset.'''
+    import questionary
+    from rich.progress import Progress
+
+    from lare.config import LareConfig, load_config, config_to_dict
+    from lare.console import print_config_table, print_creation_summary
+    from lare.database import create_database
+    from lare.interactive import run_wizard
+    from lare.pipeline import load_and_process
+
+    config: LareConfig | None = None
+
+    try:
+        if config_path:
+            log_step(f'Loading config from {config_path}')
+            config = load_config(config_path)
+        elif Path('lare_config.toml').exists():
+            use_existing = questionary.confirm(
+                'Found existing lare_config.toml. Use this to initialize the project?',
+                default=True,
+            ).ask()
+            if use_existing is None:
+                console.print('\n[yellow]Wizard cancelled.[/yellow]')
+                raise SystemExit(1)
+            if use_existing:
+                config = load_config('lare_config.toml')
+            else:
+                config = run_wizard()
+        else:
+            config = run_wizard()
+    except KeyboardInterrupt:
+        console.print('\n[yellow]Wizard cancelled.[/yellow]')
+        raise SystemExit(1)
+
+    if config is None:
+        log_error('No configuration provided')
+
+    print_config_table(config_to_dict(config))
+
+    if no_project:
+        log_success('Configuration authored — skipping database creation (--no-project)')
+        return
+
+    project_path = config.paths.project
+    if Path(project_path).exists():
+        log_error(
+            f'File risk: {project_path} already exists. '
+            'Overwriting is blocked.'
+        )
+
+    source_csv = csv_file or config.paths.source_csv
+
+    if not source_csv or not Path(source_csv).exists():
+        log_error(f'Source CSV not found: {source_csv}')
+
+    with Progress(console=console) as progress:
+        task = progress.add_task('Processing pipeline...', total=None)
+        df, rows_before = load_and_process(config, source_csv)
+        progress.update(task, completed=True)
+
+    log_step(f'Creating database: {project_path}')
+
+    with Progress(console=console) as progress:
+        task = progress.add_task('Writing database...', total=None)
+        create_database(project_path, config, df)
+        progress.update(task, completed=True)
+
+    print_creation_summary(
+        rows_before=rows_before,
+        rows_after=df.height,
+        label_count=len(config.labels),
+        project_path=project_path,
+    )
+
+
+@main.command()
+@click.argument('project', type=click.Path(exists=True))
+@click.option('--port', default=8741, type=int, help='Server port (default: 8741)')
+def run(project: str, port: int):
+    '''Launch the LARE review interface.'''
+    import uvicorn
+    from lare.server import create_app
+
+    log_step(f'Opening project: {project}')
+
+    app = create_app(project)
+
+    url = f'http://127.0.0.1:{port}'
+    log_step(f'Starting server at {url}')
+
+    webbrowser.open(url)
+
+    uvicorn.run(app, host='127.0.0.1', port=port, log_level='warning')
+
+
+@main.command()
+@click.argument('project', type=click.Path(exists=True))
+@click.option('--strategy', type=click.Choice(['csv', 'copy']), required=True, help='Export strategy')
+@click.option('--output', required=True, type=click.Path(), help='Output path')
+def export(project: str, strategy: str, output: str):
+    '''Export classified entries from a LARE project.'''
+    from lare.export import export_copy, export_csv
+
+    log_step(f'Exporting from {project} using strategy: {strategy}')
+
+    if strategy == 'csv':
+        export_csv(project, output)
+    elif strategy == 'copy':
+        export_copy(project, output)
+
+
+if __name__ == '__main__':
+    main()

lare/config.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import sys
+from typing import Literal
+
+from pydantic import BaseModel, field_validator, model_validator
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+import tomli_w
+
+
+class PathsConfig(BaseModel):
+    project: str = 'project.lare'
+    source_csv: str
+    image_directory: str | None = None
+
+
+class RulesConfig(BaseModel):
+    filter_method: str | None = None
+    ranking_score: str
+    id_column: str
+    id_display_method: Literal['raw', 'basename', 'stem'] = 'raw'
+
+
+class ImageConfig(BaseModel):
+    column: str
+    title: str
+    format: Literal['fits', 'png', 'jpg', 'jpeg', 'tiff']
+    stretching: str | None = None
+    min_threshold: float = 0.5
+    max_threshold: float = 99.5
+    subdirectory: str | None = None
+
+    @field_validator('stretching')
+    @classmethod
+    def stretching_requires_fits(cls, v: str | None, info) -> str | None:
+        if v is not None and info.data.get('format') != 'fits':
+            raise ValueError('stretching is only valid for FITS format images')
+        return v
+
+    @field_validator('subdirectory')
+    @classmethod
+    def subdirectory_needs_image_directory(cls, v: str | None) -> str | None:
+        return v
+
+
+class LabelConfig(BaseModel):
+    label: str
+    title: str
+    shortcut: str
+    is_emergent: bool = False
+
+    @field_validator('shortcut')
+    @classmethod
+    def validate_shortcut(cls, v: str) -> str:
+        if len(v) != 1 or not v.isalpha():
+            raise ValueError('shortcut must be a single alphabetic character')
+        return v.lower()
+
+
+class LareConfig(BaseModel):
+    paths: PathsConfig
+    rules: RulesConfig
+    images: list[ImageConfig]
+    labels: list[LabelConfig]
+
+    @model_validator(mode='after')
+    def validate_shortcuts_unique(self) -> LareConfig:
+        shortcuts = [lb.shortcut for lb in self.labels]
+        if len(shortcuts) != len(set(shortcuts)):
+            dupes = [s for s in shortcuts if shortcuts.count(s) > 1]
+            raise ValueError(
+                f'duplicate keyboard shortcuts detected: {set(dupes)}'
+            )
+        return self
+
+    @model_validator(mode='after')
+    def validate_subdirectory_requires_image_dir(self) -> LareConfig:
+        if self.paths.image_directory is None:
+            for img in self.images:
+                if img.subdirectory is not None:
+                    raise ValueError(
+                        f'image "{img.title}" has subdirectory set but '
+                        f'paths.image_directory is not configured'
+                    )
+        return self
+
+    @model_validator(mode='after')
+    def validate_fits_extras(self) -> LareConfig:
+        has_fits = any(img.format == 'fits' for img in self.images)
+        if has_fits:
+            try:
+                import astropy  # noqa: F401
+            except ImportError:
+                raise ValueError(
+                    'FITS image support requires extra dependencies. '
+                    'Install with: pip install lare[fits]'
+                )
+        return self
+
+
+def load_config(path: str) -> LareConfig:
+    with open(path, 'rb') as f:
+        raw = tomllib.load(f)
+    return parse_config_dict(raw)
+
+
+def parse_config_dict(raw: dict) -> LareConfig:
+    return LareConfig(
+        paths=PathsConfig(**raw.get('paths', {})),
+        rules=RulesConfig(**raw.get('rules', {})),
+        images=[ImageConfig(**img) for img in raw.get('images', [])],
+        labels=[LabelConfig(**lb) for lb in raw.get('labels', [])],
+    )
+
+
+def config_to_dict(config: LareConfig) -> dict:
+    return config.model_dump(mode='python')
+
+
+def write_config(config: LareConfig, path: str) -> None:
+    d = config.model_dump(mode='python', exclude_none=True)
+    toml_dict = {
+        'paths': d['paths'],
+        'rules': d['rules'],
+        'images': d['images'],
+        'labels': d['labels'],
+    }
+    with open(path, 'wb') as f:
+        tomli_w.dump(toml_dict, f)

lare/console.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+console = Console()
+
+
+def log_step(msg: str) -> None:
+    console.print(f'\n[bold cyan]ℹ[/] {msg}')
+
+
+def log_success(msg: str) -> None:
+    console.print(f'\n[bold green]✔[/] {msg}')
+
+
+def log_warning(msg: str) -> None:
+    console.print(f'\n[bold yellow]⚠[/] {msg}')
+
+
+def log_error(msg: str) -> None:
+    console.print(f'\n[bold red]✖[/] {msg}')
+    raise click.Abort()
+
+
+def print_config_table(config_dict: dict) -> None:
+    from rich import box
+    
+    console.print()
+    table = Table(title='Project Configuration', show_lines=False, box=box.SIMPLE, show_edge=False)
+    table.add_column('Parameter', style='cyan')
+    table.add_column('Value', style='white')
+
+    def _flatten(d: dict, prefix: str = '') -> None:
+        for key, val in d.items():
+            full_key = f'{prefix}.{key}' if prefix else key
+            if isinstance(val, dict):
+                _flatten(val, full_key)
+            elif isinstance(val, list):
+                for i, item in enumerate(val):
+                    if isinstance(item, dict):
+                        _flatten(item, f'{full_key}[{i}]')
+                    else:
+                        table.add_row(f'{full_key}[{i}]', str(item))
+            else:
+                table.add_row(full_key, str(val))
+
+    _flatten(config_dict)
+    console.print(table)
+
+
+def print_audit_banner(labeled: int, total: int) -> None:
+    from rich import box
+    
+    console.print()
+    if labeled < total:
+        console.print(Panel(
+            f'Exported {labeled} entries.\n'
+            f'[yellow]Alert:[/] You have only audited '
+            f'{labeled}/{total} filtered entries in this project database.',
+            title='[bold yellow]⚠ Audit Incomplete[/]',
+            border_style='yellow',
+            box=box.ROUNDED,
+            padding=(1, 2)
+        ))
+    else:
+        console.print(Panel(
+            f'Exported {labeled}/{total} entries — full audit complete.',
+            title='[bold green]✔ Audit Complete[/]',
+            border_style='green',
+            box=box.ROUNDED,
+            padding=(1, 2)
+        ))
+    console.print()
+
+
+def print_creation_summary(
+    rows_before: int,
+    rows_after: int,
+    label_count: int,
+    project_path: str,
+) -> None:
+    from rich import box
+    
+    console.print()
+    table = Table(title='Project Created', show_lines=False, box=box.SIMPLE, show_edge=False)
+    table.add_column('Metric', style='cyan')
+    table.add_column('Value', style='white')
+    table.add_row('Source rows', str(rows_before))
+    table.add_row('Queued rows (after filter)', str(rows_after))
+    table.add_row('Labels registered', str(label_count))
+    table.add_row('Database path', project_path)
+    console.print(table)
+    console.print()

lare/database.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import uuid
+from pathlib import Path
+
+import polars as pl
+
+from lare.config import LareConfig
+
+
+POLARS_TO_SQLITE = {
+    pl.Float64: 'REAL',
+    pl.Float32: 'REAL',
+    pl.Int64: 'INTEGER',
+    pl.Int32: 'INTEGER',
+    pl.Int16: 'INTEGER',
+    pl.Int8: 'INTEGER',
+    pl.UInt64: 'INTEGER',
+    pl.UInt32: 'INTEGER',
+    pl.UInt16: 'INTEGER',
+    pl.UInt8: 'INTEGER',
+    pl.Boolean: 'INTEGER',
+    pl.Utf8: 'TEXT',
+    pl.String: 'TEXT',
+}
+
+
+class LareDatabase:
+    def __init__(self, path: str | Path):
+        self.path = str(path)
+        self._lock = threading.Lock()
+        self._conn: sqlite3.Connection | None = None
+
+    def connect(self) -> LareDatabase:
+        self._conn = sqlite3.connect(self.path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute('PRAGMA journal_mode=WAL')
+        self._conn.execute('PRAGMA busy_timeout=5000')
+        return self
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+    def __enter__(self) -> LareDatabase:
+        return self.connect()
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    @property
+    def conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            raise RuntimeError('Database not connected')
+        return self._conn
+
+    def _write(self, sql: str, params: tuple = ()) -> sqlite3.Cursor:
+        with self._lock:
+            cursor = self._conn.execute(sql, params)
+            self._conn.commit()
+            return cursor
+
+    def _write_many(self, sql: str, rows: list[tuple]) -> None:
+        with self._lock:
+            self._conn.executemany(sql, rows)
+            self._conn.commit()
+
+    def get_entry(self, queue_order: int) -> dict | None:
+        row = self.conn.execute(
+            'SELECT * FROM labeling_queue WHERE queue_order = ?',
+            (queue_order,)
+        ).fetchone()
+        return dict(row) if row else None
+
+    def get_next_unlabeled(self, after: int = -1) -> dict | None:
+        row = self.conn.execute(
+            'SELECT * FROM labeling_queue '
+            'WHERE (final_label IS NULL OR TRIM(final_label) = \'\') AND queue_order > ? '
+            'ORDER BY queue_order ASC LIMIT 1',
+            (after,)
+        ).fetchone()
+        return dict(row) if row else None
+
+    def set_label(self, entry_id: str, label: str) -> None:
+        self._write(
+            'UPDATE labeling_queue SET final_label = ? WHERE entry_id = ?',
+            (label.strip(), entry_id),
+        )
+
+    def clear_label(self, entry_id: str) -> None:
+        self._write(
+            'UPDATE labeling_queue SET final_label = NULL WHERE entry_id = ?',
+            (entry_id,),
+        )
+
+    def search_display_id(self, query: str) -> list[dict]:
+        rows = self.conn.execute(
+            'SELECT queue_order, entry_id, display_id, final_label '
+            'FROM labeling_queue WHERE display_id LIKE ? ORDER BY queue_order',
+            (f'%{query}%',)
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_entry_by_display_id(self, display_id: str) -> list[dict]:
+        rows = self.conn.execute(
+            'SELECT * FROM labeling_queue WHERE display_id = ? '
+            'ORDER BY queue_order',
+            (display_id,)
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_audit_stats(self) -> tuple[int, int]:
+        row = self.conn.execute(
+            'SELECT '
+            'COUNT(CASE WHEN final_label IS NOT NULL AND TRIM(final_label) != \'\' THEN 1 END) AS labeled, '
+            'COUNT(*) AS total '
+            'FROM labeling_queue'
+        ).fetchone()
+        return row['labeled'], row['total']
+
+    def get_config(self) -> dict:
+        row = self.conn.execute(
+            'SELECT config_json FROM project_metadata LIMIT 1'
+        ).fetchone()
+        if row is None:
+            raise RuntimeError('No project metadata found in database')
+        return json.loads(row['config_json'])
+
+    def get_all_labeled(self) -> list[dict]:
+        rows = self.conn.execute(
+            'SELECT * FROM labeling_queue '
+            'WHERE final_label IS NOT NULL AND TRIM(final_label) != \'\' '
+            'ORDER BY queue_order'
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_total_count(self) -> int:
+        row = self.conn.execute(
+            'SELECT COUNT(*) AS cnt FROM labeling_queue'
+        ).fetchone()
+        return row['cnt']
+
+
+def create_database(
+    path: str | Path,
+    config: LareConfig,
+    df: pl.DataFrame,
+) -> None:
+    path = str(path)
+    conn = sqlite3.connect(path)
+    conn.execute('PRAGMA journal_mode=WAL')
+
+    conn.execute(
+        'CREATE TABLE project_metadata ('
+        '    project_id TEXT PRIMARY KEY,'
+        '    created_at TEXT DEFAULT CURRENT_TIMESTAMP,'
+        '    config_json TEXT NOT NULL'
+        ')'
+    )
+
+    fixed_cols = [
+        'queue_order INTEGER PRIMARY KEY',
+        'entry_id TEXT UNIQUE NOT NULL',
+        'display_id TEXT NOT NULL',
+        'final_label TEXT DEFAULT NULL',
+    ]
+
+    skip_cols = {'queue_order', 'entry_id', 'display_id', 'final_label'}
+    dynamic_cols = []
+    for name, dtype in zip(df.columns, df.dtypes):
+        if name in skip_cols:
+            continue
+        sqlite_type = POLARS_TO_SQLITE.get(dtype, 'TEXT')
+        dynamic_cols.append(f'"{name}" {sqlite_type}')
+
+    all_cols = fixed_cols + dynamic_cols
+    col_defs = ', '.join(all_cols)
+
+    conn.execute(f'CREATE TABLE labeling_queue ({col_defs})')
+    conn.execute('CREATE INDEX idx_display_id ON labeling_queue(display_id)')
+    conn.execute('CREATE INDEX idx_final_label ON labeling_queue(final_label)')
+
+    ordered_columns = ['queue_order', 'entry_id', 'display_id']
+    ordered_columns.append('final_label')
+    for name in df.columns:
+        if name not in skip_cols:
+            ordered_columns.append(name)
+
+    select_cols = []
+    for col in ordered_columns:
+        if col == 'final_label':
+            select_cols.append(col)
+        elif col in df.columns:
+            select_cols.append(col)
+
+    insert_df = df.select([c for c in ordered_columns if c in df.columns])
+
+    if 'final_label' not in insert_df.columns:
+        insert_df = insert_df.with_columns(
+            pl.lit(None).cast(pl.Utf8).alias('final_label')
+        )
+
+    final_order = ['queue_order', 'entry_id', 'display_id', 'final_label']
+    for c in insert_df.columns:
+        if c not in final_order:
+            final_order.append(c)
+    insert_df = insert_df.select(final_order)
+
+    placeholders = ', '.join(['?'] * len(insert_df.columns))
+    insert_sql = f'INSERT INTO labeling_queue VALUES ({placeholders})'
+
+    rows = insert_df.rows()
+    conn.executemany(insert_sql, rows)
+
+    project_id = str(uuid.uuid4())
+    config_json = json.dumps(config.model_dump(mode='python'), default=str)
+    conn.execute(
+        'INSERT INTO project_metadata (project_id, config_json) VALUES (?, ?)',
+        (project_id, config_json),
+    )
+
+    conn.commit()
+    conn.close()
+
+
+def open_database(path: str | Path) -> LareDatabase:
+    return LareDatabase(path)

lare/export.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import csv as csv_module
+import shutil
+from pathlib import Path
+
+from lare.config import LareConfig, parse_config_dict
+from lare.console import console, log_step, log_success, print_audit_banner
+from lare.database import open_database
+
+
+def export_csv(db_path: str, output_path: str) -> None:
+    db = open_database(db_path)
+    with db:
+        labeled = db.get_all_labeled()
+        labeled_count = len(labeled)
+        total = db.get_total_count()
+
+        if not labeled:
+            log_step('No labeled entries to export')
+            print_audit_banner(0, total)
+            return
+
+        log_step(f'Exporting {labeled_count} labeled entries to CSV')
+
+        fieldnames = list(labeled[0].keys())
+        with open(output_path, 'w', newline='') as f:
+            writer = csv_module.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(labeled)
+
+        log_success(f'CSV written to {output_path}')
+        print_audit_banner(labeled_count, total)
+
+
+def export_copy(db_path: str, output_path: str) -> None:
+    from rich.progress import Progress
+
+    db = open_database(db_path)
+    with db:
+        config_dict = db.get_config()
+        config = parse_config_dict(config_dict)
+
+        labeled = db.get_all_labeled()
+        labeled_count = len(labeled)
+        total = db.get_total_count()
+
+        if not labeled:
+            log_step('No labeled entries to export')
+            print_audit_banner(0, total)
+            return
+
+        log_step(f'Copying {labeled_count} labeled entries')
+        output_root = Path(output_path)
+
+        with Progress(console=console) as progress:
+            task = progress.add_task('Copying files...', total=labeled_count)
+            copied = 0
+
+            for entry in labeled:
+                label = entry['final_label']
+                for img_config in config.images:
+                    col_value = entry.get(img_config.column)
+                    if not col_value:
+                        continue
+
+                    if config.paths.image_directory:
+                        base = Path(config.paths.image_directory)
+                        basename = Path(col_value).name
+                        if img_config.subdirectory:
+                            source = base / img_config.subdirectory / basename
+                        else:
+                            source = base / basename
+                    else:
+                        source = Path(col_value)
+
+                    if not source.exists():
+                        continue
+
+                    target_dir = output_root / label
+                    target_dir.mkdir(parents=True, exist_ok=True)
+                    target = target_dir / source.name
+
+                    if target.exists():
+                        stem = target.stem
+                        suffix = target.suffix
+                        counter = 1
+                        while target.exists():
+                            target = target_dir / f'{stem}_{counter}{suffix}'
+                            counter += 1
+
+                    shutil.copy2(str(source), str(target))
+                    copied += 1
+
+                progress.advance(task)
+
+        log_success(f'Copied {copied} files to {output_path}')
+        print_audit_banner(labeled_count, total)