inscriptis 2.6.0__tar.gz → 2.7.0__tar.gz

inscriptis-2.7.0/.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit = tests/

inscriptis-2.7.0/.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+55fa29ca39f9ed5895f9e88b2eb0f17e4d84245f
+4a41e301851661c9e74b851ecbeafa08767cd2d0

inscriptis-2.7.0/.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+
+updates:
+
+  # Enable version updates for github actions.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every weekday
+      interval: "weekly"
+
+  # Enable version updates for Docker.
+  - package-ecosystem: "docker"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: "/"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"

inscriptis-2.7.0/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,68 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '26 5 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

inscriptis-2.7.0/.github/workflows/create-container.yml

@@ -0,0 +1,35 @@
+name: container
+
+on:
+  push:
+    tags:
+      - '*'
+    
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: get version
+      id: version
+      run: echo ::set-output name=APP_VERSION::${GITHUB_REF/refs\/tags\//}
+
+    - name: init docker build
+      uses: docker/setup-buildx-action@v2
+
+    - name: login docker
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: publish container
+      uses: docker/build-push-action@v4
+      with:
+        push: true
+        tags: |
+          ghcr.io/weblyzard/inscriptis:v${{ steps.version.outputs.APP_VERSION }}
+          ghcr.io/weblyzard/inscriptis:latest

inscriptis-2.7.0/.github/workflows/helm-release.yaml

@@ -0,0 +1,20 @@
+name: helm release
+
+on:
+  push:
+    branches:
+      - PhilippKuntschik-patch-2
+    tags:
+      - '*'
+
+jobs:
+  dispatch_helm_release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: dispatch inscriptis-helm
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.HELMREPO_ACCESS_TOKEN }}
+          repository: weblyzard/inscriptis-helm
+          event-type: tag-released
+          client-payload: '{"ref": "${{ github.ref_name }}"}'

inscriptis-2.7.0/.github/workflows/python-package.yml

@@ -0,0 +1,29 @@
+name: build
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install build environment
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install uv
+    - name: Build and test with uv.
+      run: |
+        uv run ruff check
+        uv build

inscriptis-2.7.0/.gitignore

@@ -0,0 +1,30 @@
+*.pyc
+*.pyx
+.*.swp
+*.egg-info
+__pycache__/
+benchmarking_results/
+html_cache/
+.tox
+build/
+dist/
+.cache/
+.project
+.pydevproject
+.settings/
+.pytest_cache/
+.coverage
+_build/
+.mypy_cache/
+.idea/
+venv/
+tests/converted.txt
+tests/reference.txt
+*.c
+docs/paper/*.pdf
+htmlcov/
+poetry.lock
+
+# test
+converted.txt
+reference.txt

inscriptis-2.7.0/.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

inscriptis-2.7.0/.safety-project.ini

@@ -0,0 +1,5 @@
+[project]
+id = inscriptis
+url = /codebases/inscriptis/findings
+name = inscriptis
+

inscriptis-2.7.0/CONTRIBUTING.md

@@ -0,0 +1,54 @@
+# Contributing to Inscriptis
+
+First off, thank you for considering contributing to inscriptis. 
+There are many ways how you can contribute to the project and these guidelines aim at supporting you in doing so.
+
+1. [Reporting bugs and seeking support](#reporting-bugs-and-seeking-support)
+2. [Suggesting enhancements](#suggesting-enhancements)
+3. [Pull requests](#pull-requests) (contributing code)
+4. [Python style guide](#python-style-guide)
+
+
+## Reporting bugs and seeking support
+
+Bugs and support requests are tracked as GitHub issues.
+
+To create an effective and high quality ticket, please include the following information in your
+ticket:
+
+ 1. **Use a clear and descriptive title** for the issue to identify the problem. This also helps other users to quickly locate bug reports that affect them.
+ 2. **Describe the exact steps necessary for reproducing the problem** including at least information on
+    - the affected URL
+    - the command line parameters or function arguments you used
+ 3. What would have been the **expected behavior**?
+ 4. Describe the **observed behavior**.
+ 5. Provide any additional information which might be helpful in reproducing and/or fixing this issue. 
+
+
+## Suggesting enhancements
+
+Enhancements are also tracked as GitHub issues and should contain the following information:
+
+ 1. **A clear and descriptive title** helps other people to identify enhancements they like, so that they can also add their thoughts and suggestions.
+ 2. **Provide a step-by-step description** of the suggested enhancement.
+ 3. **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
+
+
+## Pull requests
+
+1. Ensure that your code complies with our [Python style guide](#python-style-guide).
+2. Write a unit test that covers your new code and put it into the `./tests` directory.
+3. Execute `tox .` in the project's root directory to ensure that your code passes the static code analysis, coding style guidelines and security checks.
+4. In addition, please document any new API functions in the Inscriptis documentation.
+
+
+## Python style guide
+
+Inscriptis code should comply to
+- the [PEP8 Style Guide for Python Code](https://www.python.org/dev/peps/pep-0008/), and
+- to the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html)
+
+Please also ensure that 
+1. functions are properly documented with docstrings that comply to the Google Python Style Guide, and
+2. any new code is covered by unit tests.
+

inscriptis-2.7.0/Dockerfile

@@ -0,0 +1,24 @@
+#
+# Stage 1 - Install build dependencies
+#
+FROM python:3.11-slim-bullseye AS builder
+
+WORKDIR /inscriptis
+RUN python -m venv .venv && .venv/bin/python -m pip install --upgrade pip
+RUN .venv/bin/pip install --no-cache-dir inscriptis[web-service] && \
+    find /inscriptis/.venv \( -type d -a -name test -o -name tests \) -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) -exec rm -rf '{}' \+
+
+#
+# Stage 2 - Copy only necessary files to the runner stage
+#
+FROM python:3.11-slim-bullseye 
+LABEL maintainer="albert@weichselbraun.net"
+
+# Note: only copy the src directory, to prevent bloating the image with 
+#       irrelevant files from the project directory.
+WORKDIR /inscriptis
+COPY --from=builder /inscriptis /inscriptis
+
+ENV PATH="/inscriptis/.venv/bin:$PATH"
+CMD ["uvicorn", "inscriptis.service.web:app", "--port=5000", "--host=0.0.0.0"]
+EXPOSE 5000

{inscriptis-2.6.0 → inscriptis-2.7.0}/PKG-INFO

@@ -1,32 +1,32 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: inscriptis
-Version: 2.6.0
+Version: 2.7.0
 Summary: inscriptis - HTML to text converter.
-Home-page: https://github.com/weblyzard/inscriptis
-License: Apache-2.0
+Project-URL: Homepage, https://github.com/weblyzard/inscriptis
+Project-URL: Repository, https://github.com/weblyzard/inscriptis
+Project-URL: Documentation, https://inscriptis.readthedocs.io/en/latest/
+Author-email: Albert Weichselbraun <albert.weichselbraun@fhgr.ch>, Fabian Odoni <fabian.odoni@fhgr.ch>
+License-Expression: Apache-2.0
+License-File: AUTHORS
+License-File: LICENSE
 Keywords: HTML,converter,text
-Author: Albert Weichselbraun
-Author-email: albert.weichselbraun@fhgr.ch
-Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Text Processing :: Markup :: HTML
 Classifier: Topic :: Utilities
+Requires-Python: <3.15,>=3.10
+Requires-Dist: lxml<6.0.0,>=5.4.0
+Requires-Dist: requests<3.0.0,>=2.32.3
 Provides-Extra: web-service
-Requires-Dist: fastapi (>=0.115.11,<0.116.0) ; extra == "web-service"
-Requires-Dist: lxml (>=4.9.3)
-Requires-Dist: requests (>=2.32.2)
-Requires-Dist: uvicorn (>=0.34.0,<0.35.0) ; extra == "web-service"
-Project-URL: Documentation, https://inscriptis.readthedocs.io/en
-Project-URL: Repository, https://github.com/weblyzard/inscriptis
+Requires-Dist: fastapi<1.0.0,>=0.115.11; extra == 'web-service'
+Requires-Dist: uvicorn<1.0.0,>=0.34.0; extra == 'web-service'
 Description-Content-Type: text/x-rst
 
 ==================================================================================
@@ -37,10 +37,6 @@ inscriptis -- HTML to text conversion library, command line client and Web servi
    :target: https://badge.fury.io/py/inscriptis
    :alt: Supported python versions
 
-.. image:: https://api.codeclimate.com/v1/badges/f8ed73f8a764f2bc4eba/maintainability
-   :target: https://codeclimate.com/github/weblyzard/inscriptis/maintainability
-   :alt: Maintainability
-
 .. image:: https://codecov.io/gh/weblyzard/inscriptis/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/weblyzard/inscriptis/
    :alt: Coverage
@@ -523,41 +519,112 @@ be used within a program:
   print("Text:", output['text'])
   print("Annotations:", output['label'])
 
-Fine tuning
------------
+Fine-tuning the HTML rendering
+------------------------------
 
-The following options are available for fine tuning inscriptis' HTML rendering:
+Inscriptis provides the ``ParserConfig`` class to fine-tune the HTML rendering 
+(`see documentation <https://inscriptis.readthedocs.io/en/latest/api.html#inscriptis.model.config.ParserConfig>`_).
 
-1. **More rigorous indentation:** call ``inscriptis.get_text()`` with the
-   parameter ``indentation='extended'`` to also use indentation for tags such as
-   ``<div>`` and ``<span>`` that do not provide indentation in their standard
-   definition. This strategy is the default in ``inscript`` and many other
-   tools such as Lynx. If you do not want extended indentation you can use the
-   parameter ``indentation='standard'`` instead.
+It allows modifying the interpretation of HTML-tags and setting parameters that control the rendering of anchors,
+captions, images and links.
 
-2. **Overwriting the default CSS definition:** inscriptis uses CSS definitions
-   that are maintained in ``inscriptis.css.CSS`` for rendering HTML tags. You can
-   override these definitions (and therefore change the rendering) as outlined
-   below:
+1. **Firefox-like whitespace handling:** Use the more standard-conform `strict` CSS_PROFILE to render the page. 
+   (``<div>`` and ``<span>`` do not add whitespaces in the `strict` profile. Many text-based browsers such 
+   as Lynx and ``inscript``, add whitespaces per default to reduce the likelihood of words getting glued together).
 
-.. code-block:: python
+   .. code-block:: python
+   
+           from lxml.html import fromstring
+           
+           from inscriptis import Inscriptis
+           from inscriptis.css_profiles import CSS_PROFILES
+           from inscriptis.model.config import ParserConfig
+           
+           # create a ParserConfig that uses the strict CSS rendering profile
+           css = CSS_PROFILES['strict']
+           config = ParserConfig(css=css)
+           
+           html_tree = fromstring(html)
+           parser = Inscriptis(html_tree, config)
+           text = parser.get_text()
+
+2. **Firefox-like whitespace handling and fine-tuning of link handling:** Use the strict profile 
+   together with inline links and anchor URLs.
+
+   .. code-block:: python
+   
+           from lxml.html import fromstring
+           
+           from inscriptis import Inscriptis
+           from inscriptis.css_profiles import CSS_PROFILES
+           from inscriptis.model.config import ParserConfig
+           
+           # uses the strict CSS rendering profile and fine-tune link handling.
+           css = CSS_PROFILES['strict']
+           config = ParserConfig(css=css, display_links=True, 
+                                 display_anchors=True)
+           
+           html_tree = fromstring(html)
+           parser = Inscriptis(html_tree, config)
+           text = parser.get_text()
+
+
+3. **Overwriting the default CSS definition:** inscriptis uses CSS definitions
+   that are maintained in ``inscriptis.css_profiles_CSS_PROFILES`` for 
+   rendering HTML tags. You can override these definitions (and therefore 
+   change the rendering) as outlined below:
+
+   .. code-block:: python
+   
+           from lxml.html import fromstring
+           
+           from inscriptis import Inscriptis
+           from inscriptis.css_profiles import CSS_PROFILES
+           from inscriptis.html_properties import Display
+           from inscriptis.model.config import ParserConfig
+           from inscriptis.model.html_element import HtmlElement
+           
+           # Create a custom CSS based on the default style sheet and change the
+           # rendering of `div` and `span` elements.
+           css = CSS_PROFILES['strict'].copy()
+           css['div'] = HtmlElement(display=Display.block, padding=2)
+           css['span'] = HtmlElement(prefix=' ', suffix=' ')
+           
+           html_tree = fromstring(html)
+           # create a parser using a custom css
+           config = ParserConfig(css=css)
+           parser = Inscriptis(html_tree, config)
+           text = parser.get_text()
+
+4. **Ignore elements during parsing:**
+   Overwriting the default CSS profile also allows changing the rendering of selected elements. 
+   The snippet below, for example, removes forms from the parsed text by setting the definition of the ``form`` tag to ``Display.none``.
+   
+   .. code-block:: python
+   
+         from inscriptis import get_text
+         from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+         from inscriptis.html_properties import Display
+         from inscriptis.model.config import ParserConfig
+   
+         # create a custom CSS based on the default style sheet and change the
+         # rendering of `div` and `span` elements
+         css = CSS_PROFILES['strict'].copy()
+         css['form'] = HtmlElement(display=Display.none)
+   
+         # create a parser configuration using a custom css
+         html = """First line. 
+                   <form>
+                     User data
+                     <label for="name">Name:</label><br>
+                     <input type="text" id="name" name="name"><br>
+                     <label for="pass">Password:</label><br>
+                     <input type="hidden" id="pass" name="pass">
+                   </form>"""
+         config = ParserConfig(css=css)
+         text = get_text(html, config)
+         print(text)
 
-      from lxml.html import fromstring
-      from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
-      from inscriptis.html_properties import Display
-      from inscriptis.model.config import ParserConfig
-      
-      # create a custom CSS based on the default style sheet and change the
-      # rendering of `div` and `span` elements
-      css = CSS_PROFILES['strict'].copy()
-      css['div'] = HtmlElement(display=Display.block, padding=2)
-      css['span'] = HtmlElement(prefix=' ', suffix=' ')
-      
-      html_tree = fromstring(html)
-      # create a parser using a custom css
-      config = ParserConfig(css=css)
-      parser = Inscriptis(html_tree, config)
-      text = parser.get_text()
 
 
 Custom HTML tag handling
@@ -601,55 +668,6 @@ The following code mitigates this problem on Unix systems by manually forcing lx
       return libc.malloc_trim(0)
 
 
-Examples
-========
-
-Strict indentation handling
----------------------------
-
-The following example demonstrates modifying ``ParserConfig`` for strict indentation handling.
-
-.. code-block:: python
-
-   from inscriptis import get_text
-   from inscriptis.css_profiles import CSS_PROFILES
-   from inscriptis.model.config import ParserConfig
-
-   config = ParserConfig(css=CSS_PROFILES['strict'].copy())
-   text = get_text('fi<span>r</span>st', config)
-   print(text)
-
-Ignore elements during parsing 
-------------------------------
-
-Overwriting the default CSS profile also allows changing the rendering of selected elements. 
-The snippet below, for example, removes forms from the parsed text by setting the definition of the ``form`` tag to ``Display.none``.
-
-.. code-block:: python
-
-      from inscriptis import get_text
-      from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
-      from inscriptis.html_properties import Display
-      from inscriptis.model.config import ParserConfig
-
-      # create a custom CSS based on the default style sheet and change the
-      # rendering of `div` and `span` elements
-      css = CSS_PROFILES['strict'].copy()
-      css['form'] = HtmlElement(display=Display.none)
-
-      # create a parser configuration using a custom css
-      html = """First line. 
-                <form>
-                  User data
-                  <label for="name">Name:</label><br>
-                  <input type="text" id="name" name="name"><br>
-                  <label for="pass">Password:</label><br>
-                  <input type="hidden" id="pass" name="pass">
-                </form>"""
-      config = ParserConfig(css=css)
-      text = get_text(html, config)
-      print(text)
-
 
 Citation
 ========
@@ -678,4 +696,3 @@ Changelog
 A full list of changes can be found in the
 `release notes <https://github.com/weblyzard/inscriptis/releases>`_.
 
-