libreqda 1.2.0__py3-none-any.whl

taguette/extract.py

@@ -0,0 +1,287 @@
+from bs4 import BeautifulSoup, NavigableString
+import opentelemetry.trace
+import prometheus_client
+
+
+tracer = opentelemetry.trace.get_tracer(__name__)
+
+
+BUCKETS = [0.001, 0.002, 0.003, 0.004,
+           0.006, 0.008, 0.010,
+           0.015, 0.020, 0.025, 0.030, 0.035, 0.04, 0.045, 0.05,
+           0.06, 0.07, 0.08, 0.09, 0.10, 0.15, 0.20, 0.5, 1.0]
+PROM_EXTRACT_TIME = prometheus_client.Histogram(
+    'html_extract_seconds',
+    "Time to extract part of an HTML document (extract.extract())",
+    buckets=BUCKETS,
+)
+PROM_HIGHLIGHT_TIME = prometheus_client.Histogram(
+    'html_highlight_seconds',
+    "Time to add highlight tags to an HTML document (extract.highlight())",
+    buckets=BUCKETS,
+)
+
+
+def split_utf8(s, pos):
+    """Split a string at a given UTF-8 byte position.
+
+    If the position falls inside of a codepoint, the string will be split after
+    that codepoint.
+    """
+    s = s.encode('utf-8')
+    while pos < len(s) and 0x80 <= s[pos] <= 0xBF:
+        pos += 1
+    return s[:pos].decode('utf-8'), s[pos:].decode('utf-8')
+
+
+def find_pos(node, offset, after):
+    """Find a position in an HTML document.
+    """
+    indices = []
+    while True:
+        if getattr(node, 'contents', None):
+            indices.append(0)
+            node = node.contents[0]
+        else:
+            if isinstance(node, NavigableString):
+                nb = len(node.string.encode('utf-8'))
+                if (after and nb > offset) or (not after and nb >= offset):
+                    break
+                else:
+                    offset -= nb
+            while not node.next_sibling:
+                indices.pop()
+                node = node.parent
+            indices[-1] += 1
+            node = node.next_sibling
+    return node, offset, indices
+
+
+def delete_left(node, indices):
+    """Delete the left part of an HTML tree.
+    """
+    for idx in indices:
+        for child in node.contents[:idx]:
+            child.extract()
+        node = node.contents[0]
+
+
+def delete_right(node, indices):
+    """Delete the right part of an HTML tree.
+    """
+    for idx in indices:
+        del node.contents[idx + 1:]
+        node = node.contents[idx]
+
+
+@tracer.start_as_current_span('taguette/extract')
+@PROM_EXTRACT_TIME.time()
+def extract(html, start, end):
+    """Extract a snippet out of an HTML document.
+
+    Locations are computed over UTF-8 bytes, and doesn't count HTML tags.
+
+    Extraction is aware of tags, so:
+
+    >>> '<p><u>Hello</u> there <i>World</i></p>'[17:27]
+    'here <i>Wo'
+    >>> extract('<p><u>Hello</u> there <i>World</i></p>', 7, 14)
+    '<p>here <i>Wo</i></p>'
+    """
+    soup = BeautifulSoup(html, 'html5lib')
+
+    # Trim the right side first, because that doesn't mess our start position
+    if end is not None:
+        e = find_pos(soup, end, False)
+        e[0].replace_with(NavigableString(split_utf8(e[0].string, e[1])[0]))
+        delete_right(soup, e[2])
+
+    # Trim the left side
+    if start is not None:
+        s = find_pos(soup, start, True)
+        s[0].replace_with(NavigableString(split_utf8(s[0].string, s[1])[1]))
+        delete_left(soup, s[2])
+
+    # Remove everything but body
+    body = soup.body
+    soup.clear()
+    soup.append(body)
+
+    # Remove the body tag itself to only have the contents
+    soup.body.unwrap()
+
+    # Back to text
+    return str(soup)
+
+
+def byte_to_str_index(string, byte_index):
+    """Converts a byte index in the UTF-8 string into a codepoint index.
+
+    If the index falls inside of a codepoint unit, it will be rounded down.
+    """
+    for idx, char in enumerate(string):
+        char_size = len(char.encode('utf-8'))
+        if char_size > byte_index:
+            return idx
+        byte_index -= char_size
+    return len(string)
+
+
+@tracer.start_as_current_span('taguette/highlight')
+@PROM_HIGHLIGHT_TIME.time()
+def highlight(html, highlights, show_tags=False):
+    """Highlight part of an HTML documents.
+
+    :param highlights: Iterable of (start, end, tags) triples, which are
+        computed over UTF-8 bytes and don't count HTML tags
+    :param show_tags: Whether to show the tag names within brackets after each
+        highlight
+    """
+    # Build a list of starting points and ending points
+    starts = []
+    ends = []
+    for hl in highlights:
+        starts.append((hl[0], 'start', []))
+        if len(hl) == 2:
+            ends.append((hl[1], 'end', []))
+        else:
+            ends.append((hl[1], 'end', hl[2]))
+    # This relies on the fact that 'end' < 'start'
+    events = sorted(ends + starts)
+
+    events = iter(events)
+    soup = BeautifulSoup(html, 'html5lib')
+
+    pos = 0
+    node = soup
+    highlighting = 0
+    try:
+        event_pos, event_type, tags = next(events)
+    except StopIteration:
+        event_pos = event_type = tags = None
+
+    while node is not None:
+        if getattr(node, 'contents', None):
+            # Move down
+            node = node.contents[0]
+            continue
+
+        if isinstance(node, NavigableString):
+            # Move through text
+            nb = len(node.string.encode('utf-8'))
+            while event_pos is not None:
+                if event_pos == pos and event_type == 'start':
+                    # Start highlighting at beginning of text node
+                    highlighting += 1
+                    try:
+                        event_pos, event_type, tags = next(events)
+                    except StopIteration:
+                        event_pos = None
+                elif pos + nb > event_pos:
+                    # Next event falls inside of this text node
+                    if event_type == 'start' and highlighting:
+                        # Keep highlighting (can't highlight *more*)
+                        highlighting += 1
+                    elif (
+                        event_type == 'end'
+                        and not show_tags
+                        and highlighting > 1
+                    ):
+                        # Keep highlighting (no need to put labels)
+                        highlighting -= 1
+                    else:  # 'end' and (show_tags or highlighting becomes 0)
+                        # Split it
+                        char_idx = byte_to_str_index(
+                            node.string,
+                            event_pos - pos,
+                        )
+                        left = node.string[:char_idx]
+                        right = node.string[char_idx:]
+
+                        # Left part
+                        newnode = NavigableString(left)
+                        if highlighting:
+                            # Optionally highlight left part
+                            span = soup.new_tag(
+                                'span',
+                                attrs={'class': 'highlight'},
+                            )
+                            span.append(newnode)
+                            newnode = span
+                        node.replace_with(newnode)
+                        node = newnode
+
+                        if event_type == 'start':
+                            highlighting += 1
+                        else:
+                            highlighting -= 1
+                            if show_tags:
+                                # Add tag labels
+                                comment = soup.new_tag(
+                                    'span',
+                                    attrs={'class': 'taglist'},
+                                )
+                                comment.string = ' [%s]' % ', '.join(tags)
+                                node.insert_after(comment)
+                                node = comment
+
+                        # Right part
+                        newnode = NavigableString(right)
+                        node.insert_after(newnode)
+                        node = newnode
+                        nb -= event_pos - pos
+                        pos = event_pos
+                        # Next loop will highlight right part if needed
+
+                    try:
+                        event_pos, event_type, tags = next(events)
+                    except StopIteration:
+                        event_pos = None
+                elif highlighting:  # and pos + nb <= event_pos:
+                    # Highlight whole text node
+                    newnode = soup.new_tag(
+                        'span',
+                        attrs={'class': 'highlight'},
+                    )
+                    node.replace_with(newnode)
+                    newnode.append(node)
+                    node = newnode
+                    if pos + nb == event_pos and event_type == 'end':
+                        if show_tags:
+                            comment = soup.new_tag(
+                                'span',
+                                attrs={'class': 'taglist'},
+                            )
+                            comment.string = ' [%s]' % ', '.join(tags)
+                            newnode.insert_after(comment)
+                            node = comment
+                        highlighting -= 1
+                        try:
+                            event_pos, event_type, tags = next(events)
+                        except StopIteration:
+                            event_pos = None
+                    break
+                else:  # not highlighting and pos + nb <= event_pos
+                    # Skip whole text node
+                    break
+
+            pos += nb
+
+        # Move up until there's a sibling
+        while not node.next_sibling and node.parent:
+            node = node.parent
+        if not node.parent:
+            break
+        # Move to next node
+        node = node.next_sibling
+
+    # Remove everything but body
+    body = soup.body
+    soup.clear()
+    soup.append(body)
+
+    # Remove the body tag itself to only have the contents
+    soup.body.unwrap()
+
+    # Back to text
+    return str(soup)

taguette/import_codebook.py

@@ -0,0 +1,110 @@
+import chardet
+import codecs
+import csv
+
+from .utils import _f
+
+
+class InvalidCodebook(ValueError):
+    """File can't be read as a codebook.
+    """
+
+    def __init__(self, message, row=None):
+        super(InvalidCodebook, self).__init__(message)
+        self.message = message
+        self.row = row
+
+
+def list_tags_csv(reader):
+    detector = chardet.UniversalDetector()
+    chunk = reader.read(4096)
+    while chunk and not detector.done:
+        detector.feed(chunk)
+        chunk = reader.read(4096)
+    detector.close()
+    charset = detector.result['encoding'] or 'utf-8'
+    reader.seek(0, 0)
+
+    reader = codecs.getreader(charset)(reader)
+    reader = csv.reader(reader)
+
+    # Read header
+    try:
+        header = next(reader)
+    except (StopIteration, csv.Error, UnicodeDecodeError):
+        raise InvalidCodebook(_f("Invalid file: CSV expected"))
+    header = [cell.lower() for cell in header]
+
+    # Figure out which column has the tag path
+    expected = ('tag' in header) + ('name' in header) + ('path' in header)
+    if expected > 1:
+        raise InvalidCodebook(
+            _f("Not sure which column to use for tag name"),
+        )
+    elif expected == 0:
+        raise InvalidCodebook(_f("No 'tag', 'name', or 'path' column"))
+    else:
+        if 'tag' in header:
+            col_path = header.index('tag')
+        elif 'name' in header:
+            col_path = header.index('name')
+        else:
+            col_path = header.index('path')
+
+    # Is there a description?
+    try:
+        col_description = header.index('description')
+    except ValueError:
+        col_description = None
+
+    # Is there a color?
+    try:
+        col_color = header.index('color')
+    except ValueError:
+        col_color = None
+
+    # Is there a parent?
+    try:
+        col_parent = header.index('parent')
+    except ValueError:
+        col_parent = None
+
+    needed_rows = max(
+        num + 1
+        for num in (col_path, col_color, col_parent, col_description)
+        if num is not None
+    )
+
+    # Read file
+    try:
+        tags = []
+        for nb, row in enumerate(reader, 2):
+            if len(row) < needed_rows:
+                raise InvalidCodebook(_f("Not enough columns"), nb)
+            if not any(row):
+                continue  # Skip empty row
+            path = row[col_path].strip()
+            if not path:
+                raise InvalidCodebook(_f("Empty tag name"), nb)
+            if col_description is not None:
+                description = row[col_description].strip()
+            else:
+                description = ''
+            if col_color is not None:
+                color = row[col_color].strip()
+            else:
+                color = ''
+            if col_parent is not None:
+                parent = row[col_parent].strip()
+            else:
+                parent = ''
+            tags.append({'path': path, 'color': color,
+                         'parent': parent, 'description': description})
+    except (csv.Error, UnicodeDecodeError):
+        raise InvalidCodebook(_f("Invalid CSV file"))
+
+    return tags
+
+
+def list_tags(reader):
+    return list_tags_csv(reader)

taguette/loki_handler.py

@@ -0,0 +1,107 @@
+import logging
+import requests
+import time
+
+
+class LokiHandler(logging.Handler):
+
+    def __init__(self, loki_url: str, job_name: str = "libreqda"):
+        super().__init__()
+        self.loki_url = f"{loki_url}/loki/api/v1/push"
+        self.job_name = job_name
+        self.buffer = []
+        self.buffer_size = 10
+
+    def emit(self, record):
+        try:
+            message = self.format(record)
+            level = record.levelname.lower()
+
+            labels = {
+                "job": self.job_name,
+                "level": level,
+                "logger": record.name
+            }
+
+            if hasattr(record, 'user_id'):
+                labels["user_id"] = str(record.user_id)
+            if hasattr(record, 'project_id'):
+                labels["project_id"] = str(record.project_id)
+
+            timestamp = str(int(time.time() * 1000000000))
+
+            self.buffer.append({
+                "stream": labels,
+                "values": [[timestamp, message]]
+            })
+
+            if len(self.buffer) >= self.buffer_size:
+                self._send_buffer()
+
+        except Exception:
+            pass
+
+    def _send_buffer(self):
+        if not self.buffer:
+            return
+
+        try:
+            payload = {"streams": self.buffer}
+            response = requests.post(
+                self.loki_url,
+                json=payload,
+                timeout=5
+            )
+            if response.status_code == 204:
+                self.buffer.clear()
+        except Exception:
+            self.buffer.clear()
+
+    def close(self):
+        self._send_buffer()
+        super().close()
+
+
+def test_loki_connectivity(loki_url: str):
+    """Test if Loki is accessible and ready"""
+    try:
+        import requests
+        response = requests.get(f"{loki_url}/ready", timeout=5)
+        if response.status_code == 200:
+            return True, "Loki is ready"
+        else:
+            return False, f"Loki not ready (status: {response.status_code})"
+    except requests.exceptions.ConnectionError:
+        return False, f"Loki not accessible at {loki_url}"
+    except Exception as e:
+        return False, f"Loki test error: {e}"
+
+
+def setup_loki_logging(loki_url: str):
+    # Test connectivity first
+    is_ready, message = test_loki_connectivity(loki_url)
+    logger = logging.getLogger(__name__)
+
+    if is_ready:
+        logger.info(message)
+    else:
+        logger.warning(message)
+
+    loki_handler = LokiHandler(loki_url)
+    loki_handler.setLevel(logging.WARNING)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+    loki_handler.setFormatter(formatter)
+
+    # Filter out tornado.access logs from Loki
+    class ExcludeTornadoAccess(logging.Filter):
+        def filter(self, record):
+            return not record.name.startswith("tornado.access")
+    loki_handler.addFilter(ExcludeTornadoAccess())
+
+    root_logger = logging.getLogger()
+    root_logger.addHandler(loki_handler)
+
+    # Test log
+    logger.info("Loki test log - %s", loki_url)
+
+    return loki_handler