PyPI - inscriptis - Versions diffs - 2.3.2__tar.gz → 2.4.0.1__tar.gz - Mend

inscriptis 2.3.2tar.gz → 2.4.0.1tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (40) hide show

{inscriptis-2.3.2/src/inscriptis.egg-info → inscriptis-2.4.0.1}/PKG-INFO RENAMED Viewed

@@ -1,29 +1,33 @@
 Metadata-Version: 2.1
 Name: inscriptis
-Version: 2.3.2
+Version: 2.4.0.1
 Summary: inscriptis - HTML to text converter.
 Home-page: https://github.com/weblyzard/inscriptis
-Author: Albert Weichselbraun, Fabian Odoni
-Author-email: albert.weichselbraun@fhgr.ch, fabian.odoni@fhgr.ch
-License: Apache 2.0
+License: Apache-2.0
 Keywords: HTML,converter,text
-Platform: UNKNOWN
+Author: Albert Weichselbraun
+Author-email: albert.weichselbraun@fhgr.ch
+Requires-Python: >=3.8,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Topic :: Text Processing
-Classifier: Topic :: Text Processing :: Markup :: HTML
-Classifier: Topic :: Utilities
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.6
-License-File: LICENSE
-License-File: AUTHORS
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Utilities
+Provides-Extra: web-service
+Requires-Dist: fastapi (>=0.109.0,<0.110.0) ; extra == "web-service"
+Requires-Dist: lxml (>=4.9.3)
+Requires-Dist: requests (>=2.31.0)
+Requires-Dist: uvicorn (>=0.25.0,<0.26.0) ; extra == "web-service"
+Project-URL: Documentation, https://inscriptis.readthedocs.io/en
+Project-URL: Repository, https://github.com/weblyzard/inscriptis
+Description-Content-Type: text/x-rst
 ==================================================================================
 inscriptis -- HTML to text conversion library, command line client and Web service
@@ -158,9 +162,9 @@ the corresponding text representation.
 Command line parameters
 -----------------------
-The inscript.py command line client supports the following parameters::
+The inscript command line client supports the following parameters::
-    usage: inscript.py [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR] [--indentation INDENTATION]
+    usage: inscript [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR] [--indentation INDENTATION]
                        [--table-cell-separator TABLE_CELL_SEPARATOR] [-v]
                        [input]
@@ -199,19 +203,19 @@ HTML to text conversion
 -----------------------
 convert the given page to text and output the result to the screen::
-  $ inscript.py https://www.fhgr.ch
+  $ inscript https://www.fhgr.ch
 convert the file to text and save the output to fhgr.txt::
-  $ inscript.py fhgr.html -o fhgr.txt
+  $ inscript fhgr.html -o fhgr.txt
 convert the file using strict indentation (i.e., minimize indentation and extra spaces) and save the output to fhgr-layout-optimized.txt::
-  $ inscript.py --indentation strict fhgr.html -o fhgr-layout-optimized.txt
+  $ inscript --indentation strict fhgr.html -o fhgr-layout-optimized.txt
 convert HTML provided via stdin and save the output to output.txt::
-  $ echo "<body><p>Make it so!</p></body>" | inscript.py -o output.txt
+  $ echo "<body><p>Make it so!</p></body>" | inscript -o output.txt
 HTML to annotated text conversion
@@ -220,7 +224,7 @@ convert and annotate HTML from a Web page using the provided annotation rules.
 Download the example `annotation-profile.json <https://github.com/weblyzard/inscriptis/blob/master/examples/annotation-profile.json>`_ and save it to your working directory::
-  $ inscript.py https://www.fhgr.ch -r annotation-profile.json
+  $ inscript https://www.fhgr.ch -r annotation-profile.json
 The annotation rules are specified in `annotation-profile.json`:
@@ -268,7 +272,7 @@ Annotation postprocessors enable the post processing of annotations to formats
 that are suitable for your particular application. Post processors can be
 specified with the ``-p`` or ``--postprocessor`` command line argument::
-  $ inscript.py https://www.fhgr.ch \
+  $ inscript https://www.fhgr.ch \
           -r ./examples/annotation-profile.json \
           -p surface
@@ -313,7 +317,7 @@ Currently, inscriptis supports the following postprocessors:
    .. code-block:: bash
-      inscript.py --annotation-rules ./wikipedia.json \
+      inscript --annotation-rules ./wikipedia.json \
                   --postprocessor html \
                   https://en.wikipedia.org/wiki/Chur.html
@@ -338,14 +342,18 @@ Currently, inscriptis supports the following postprocessors:
 Web Service
 ===========
-The Flask Web Service translates HTML pages to the corresponding plain text.
+A FastAPI-based Web Service that uses Inscriptis for translating HTML pages to plain text.
 Run the Web Service on your host system
 ---------------------------------------
-Provide additional requirement `python3-flask <https://flask.palletsprojects.com/en/2.2.x/>`_, then start the inscriptis Web service with the following command::
+Install the optional feature `web-service` for inscriptis::
+  $ pip install inscriptis[web-service]
+Start the Inscriptis Web service with the following command::
+  $ uvicorn inscriptis.service.web:app --port 5000 --host 127.0.0.1
-  $ export FLASK_APP="inscriptis.service.web"
-  $ python3 -m flask run
 Run the Web Service with Docker
 -------------------------------
@@ -526,7 +534,7 @@ The following options are available for fine tuning inscriptis' HTML rendering:
 1. **More rigorous indentation:** call ``inscriptis.get_text()`` with the
    parameter ``indentation='extended'`` to also use indentation for tags such as
    ``<div>`` and ``<span>`` that do not provide indentation in their standard
-   definition. This strategy is the default in ``inscript.py`` and many other
+   definition. This strategy is the default in ``inscript`` and many other
    tools such as Lynx. If you do not want extended indentation you can use the
    parameter ``indentation='standard'`` instead.
@@ -551,36 +559,7 @@ The following options are available for fine tuning inscriptis' HTML rendering:
       html_tree = fromstring(html)
       # create a parser using a custom css
       config = ParserConfig(css=css)
-      parser = Inscriptis(html_tree, config)  usage: inscript.py [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR]
-                     [--indentation INDENTATION] [-v]
-                     [input]
-  Convert the given HTML document to text.
-  positional arguments:
-    input                 Html input either from a file or a URL (default:stdin).
-  optional arguments:
-    -h, --help            show this help message and exit
-    -o OUTPUT, --output OUTPUT
-                          Output file (default:stdout).
-    -e ENCODING, --encoding ENCODING
-                          Input encoding to use (default:utf-8 for files; detected server encoding for Web URLs).
-    -i, --display-image-captions
-                          Display image captions (default:false).
-    -d, --deduplicate-image-captions
-                          Deduplicate image captions (default:false).
-    -l, --display-link-targets
-                          Display link targets (default:false).
-    -a, --display-anchor-urls
-                          Display anchor URLs (default:false).
-    -r ANNOTATION_RULES, --annotation-rules ANNOTATION_RULES
-                          Path to an optional JSON file containing rules for annotating the retrieved text.
-    -p POSTPROCESSOR, --postprocessor POSTPROCESSOR
-                          Optional component for postprocessing the result (html, surface, xml).
-    --indentation INDENTATION
-                          How to handle indentation (extended or strict; default: extended).
-    -v, --version         display version information
+      parser = Inscriptis(html_tree, config)
       text = parser.get_text()
@@ -616,6 +595,56 @@ The following code mitigates this problem on Unix systems by manually forcing lx
       return libc.malloc_trim(0)
+Examples
+========
+Strict indentation handling
+---------------------------
+The following example demonstrates modifying ``ParserConfig`` for strict indentation handling.
+.. code-block:: python
+   from inscriptis import get_text
+   from inscriptis.css_profiles import CSS_PROFILES
+   from inscriptis.model.config import ParserConfig
+   config = ParserConfig(css=CSS_PROFILES['strict'].copy())
+   text = get_text('fi<span>r</span>st', config)
+   print(text)
+Ignore elements during parsing
+------------------------------
+Overwriting the default CSS profile also allows changing the rendering of selected elements.
+The snippet below, for example, removes forms from the parsed text by setting the definition of the ``form`` tag to ``Display.none``.
+.. code-block:: python
+      from inscriptis import get_text
+      from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+      from inscriptis.html_properties import Display
+      from inscriptis.model.config import ParserConfig
+      # create a custom CSS based on the default style sheet and change the
+      # rendering of `div` and `span` elements
+      css = CSS_PROFILES['strict'].copy()
+      css['form'] = HtmlElement(display=Display.none)
+      # create a parser configuration using a custom css
+      html = """First line.
+                <form>
+                  User data
+                  <label for="name">Name:</label><br>
+                  <input type="text" id="name" name="name"><br>
+                  <label for="pass">Password:</label><br>
+                  <input type="hidden" id="pass" name="pass">
+                </form>"""
+      config = ParserConfig(css=css)
+      text = get_text(html, config)
+      print(text)
 Citation
 ========
@@ -644,4 +673,3 @@ A full list of changes can be found in the
 `release notes <https://github.com/weblyzard/inscriptis/releases>`_.

{inscriptis-2.3.2 → inscriptis-2.4.0.1}/README.rst RENAMED Viewed

@@ -131,9 +131,9 @@ the corresponding text representation.
 Command line parameters
 -----------------------
-The inscript.py command line client supports the following parameters::
+The inscript command line client supports the following parameters::
-    usage: inscript.py [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR] [--indentation INDENTATION]
+    usage: inscript [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR] [--indentation INDENTATION]
                        [--table-cell-separator TABLE_CELL_SEPARATOR] [-v]
                        [input]
@@ -172,19 +172,19 @@ HTML to text conversion
 -----------------------
 convert the given page to text and output the result to the screen::
-  $ inscript.py https://www.fhgr.ch
+  $ inscript https://www.fhgr.ch
 convert the file to text and save the output to fhgr.txt::
-  $ inscript.py fhgr.html -o fhgr.txt
+  $ inscript fhgr.html -o fhgr.txt
 convert the file using strict indentation (i.e., minimize indentation and extra spaces) and save the output to fhgr-layout-optimized.txt::
-  $ inscript.py --indentation strict fhgr.html -o fhgr-layout-optimized.txt
+  $ inscript --indentation strict fhgr.html -o fhgr-layout-optimized.txt
 convert HTML provided via stdin and save the output to output.txt::
-  $ echo "<body><p>Make it so!</p></body>" | inscript.py -o output.txt
+  $ echo "<body><p>Make it so!</p></body>" | inscript -o output.txt
 HTML to annotated text conversion
@@ -193,7 +193,7 @@ convert and annotate HTML from a Web page using the provided annotation rules.
 Download the example `annotation-profile.json <https://github.com/weblyzard/inscriptis/blob/master/examples/annotation-profile.json>`_ and save it to your working directory::
-  $ inscript.py https://www.fhgr.ch -r annotation-profile.json
+  $ inscript https://www.fhgr.ch -r annotation-profile.json
 The annotation rules are specified in `annotation-profile.json`:
@@ -241,7 +241,7 @@ Annotation postprocessors enable the post processing of annotations to formats
 that are suitable for your particular application. Post processors can be
 specified with the ``-p`` or ``--postprocessor`` command line argument::
-  $ inscript.py https://www.fhgr.ch \
+  $ inscript https://www.fhgr.ch \
           -r ./examples/annotation-profile.json \
           -p surface
@@ -286,7 +286,7 @@ Currently, inscriptis supports the following postprocessors:
    .. code-block:: bash
-      inscript.py --annotation-rules ./wikipedia.json \
+      inscript --annotation-rules ./wikipedia.json \
                   --postprocessor html \
                   https://en.wikipedia.org/wiki/Chur.html
@@ -311,14 +311,18 @@ Currently, inscriptis supports the following postprocessors:
 Web Service
 ===========
-The Flask Web Service translates HTML pages to the corresponding plain text.
+A FastAPI-based Web Service that uses Inscriptis for translating HTML pages to plain text.
 Run the Web Service on your host system
 ---------------------------------------
-Provide additional requirement `python3-flask <https://flask.palletsprojects.com/en/2.2.x/>`_, then start the inscriptis Web service with the following command::
+Install the optional feature `web-service` for inscriptis::
+  $ pip install inscriptis[web-service]
+Start the Inscriptis Web service with the following command::
+  $ uvicorn inscriptis.service.web:app --port 5000 --host 127.0.0.1
-  $ export FLASK_APP="inscriptis.service.web"
-  $ python3 -m flask run
 Run the Web Service with Docker
 -------------------------------
@@ -499,7 +503,7 @@ The following options are available for fine tuning inscriptis' HTML rendering:
 1. **More rigorous indentation:** call ``inscriptis.get_text()`` with the
    parameter ``indentation='extended'`` to also use indentation for tags such as
    ``<div>`` and ``<span>`` that do not provide indentation in their standard
-   definition. This strategy is the default in ``inscript.py`` and many other
+   definition. This strategy is the default in ``inscript`` and many other
    tools such as Lynx. If you do not want extended indentation you can use the
    parameter ``indentation='standard'`` instead.
@@ -524,36 +528,7 @@ The following options are available for fine tuning inscriptis' HTML rendering:
       html_tree = fromstring(html)
       # create a parser using a custom css
       config = ParserConfig(css=css)
-      parser = Inscriptis(html_tree, config)  usage: inscript.py [-h] [-o OUTPUT] [-e ENCODING] [-i] [-d] [-l] [-a] [-r ANNOTATION_RULES] [-p POSTPROCESSOR]
-                     [--indentation INDENTATION] [-v]
-                     [input]
-  Convert the given HTML document to text.
-  positional arguments:
-    input                 Html input either from a file or a URL (default:stdin).
-  optional arguments:
-    -h, --help            show this help message and exit
-    -o OUTPUT, --output OUTPUT
-                          Output file (default:stdout).
-    -e ENCODING, --encoding ENCODING
-                          Input encoding to use (default:utf-8 for files; detected server encoding for Web URLs).
-    -i, --display-image-captions
-                          Display image captions (default:false).
-    -d, --deduplicate-image-captions
-                          Deduplicate image captions (default:false).
-    -l, --display-link-targets
-                          Display link targets (default:false).
-    -a, --display-anchor-urls
-                          Display anchor URLs (default:false).
-    -r ANNOTATION_RULES, --annotation-rules ANNOTATION_RULES
-                          Path to an optional JSON file containing rules for annotating the retrieved text.
-    -p POSTPROCESSOR, --postprocessor POSTPROCESSOR
-                          Optional component for postprocessing the result (html, surface, xml).
-    --indentation INDENTATION
-                          How to handle indentation (extended or strict; default: extended).
-    -v, --version         display version information
+      parser = Inscriptis(html_tree, config)
       text = parser.get_text()
@@ -589,6 +564,56 @@ The following code mitigates this problem on Unix systems by manually forcing lx
       return libc.malloc_trim(0)
+Examples
+========
+Strict indentation handling
+---------------------------
+The following example demonstrates modifying ``ParserConfig`` for strict indentation handling.
+.. code-block:: python
+   from inscriptis import get_text
+   from inscriptis.css_profiles import CSS_PROFILES
+   from inscriptis.model.config import ParserConfig
+   config = ParserConfig(css=CSS_PROFILES['strict'].copy())
+   text = get_text('fi<span>r</span>st', config)
+   print(text)
+Ignore elements during parsing
+------------------------------
+Overwriting the default CSS profile also allows changing the rendering of selected elements.
+The snippet below, for example, removes forms from the parsed text by setting the definition of the ``form`` tag to ``Display.none``.
+.. code-block:: python
+      from inscriptis import get_text
+      from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+      from inscriptis.html_properties import Display
+      from inscriptis.model.config import ParserConfig
+      # create a custom CSS based on the default style sheet and change the
+      # rendering of `div` and `span` elements
+      css = CSS_PROFILES['strict'].copy()
+      css['form'] = HtmlElement(display=Display.none)
+      # create a parser configuration using a custom css
+      html = """First line.
+                <form>
+                  User data
+                  <label for="name">Name:</label><br>
+                  <input type="text" id="name" name="name"><br>
+                  <label for="pass">Password:</label><br>
+                  <input type="hidden" id="pass" name="pass">
+                </form>"""
+      config = ParserConfig(css=css)
+      text = get_text(html, config)
+      print(text)
 Citation
 ========

inscriptis-2.4.0.1/pyproject.toml ADDED Viewed

@@ -0,0 +1,63 @@
+[tool.poetry]
+name = "inscriptis"
+version = "2.4.0.1"
+authors = ["Albert Weichselbraun <albert.weichselbraun@fhgr.ch>", "Fabian Odoni <fabian.odoni@fhgr.ch>"]
+description = "inscriptis - HTML to text converter."
+keywords = ["HTML", "converter", "text"]
+classifiers = [
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Topic :: Text Processing',
+        'Topic :: Text Processing :: Markup :: HTML',
+        'Topic :: Utilities',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+    ]
+homepage = "https://github.com/weblyzard/inscriptis"
+repository = "https://github.com/weblyzard/inscriptis"
+documentation = "https://inscriptis.readthedocs.io/en"
+license = "Apache-2.0"
+readme = "README.rst"
+packages = [
+   {include = "inscriptis", from="src"},
+]
+[tool.poetry.scripts]
+inscript = "inscriptis.cli.inscript:cli"
+inscriptis-api = "inscriptis.service.web:start"
+[tool.poetry.extras]
+web-service = ["fastapi", "uvicorn"]
+[tool.poetry.dependencies]
+python = "^3.8 || ^3.9 || ^3.10 || ^3.11 || ^3.12"
+requests = ">=2.31.0"
+lxml = ">=4.9.3"
+# optional dependencies
+fastapi = { version = "^0.109.0", optional = true }
+uvicorn = { version = "^0.25.0", optional = true }
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+# code formatting with black
+[tool.black]
+line-length = 88
+target-version = ["py38", "py39", "py310", "py311", "py312"]
+extend-exclude = '\.html$|\.json$|\.txt$|/a$|/b$'
+include = '''
+  ^/src/|^/tests/|^/benchmarking/
+'''

{inscriptis-2.3.2 → inscriptis-2.4.0.1}/src/inscriptis/__init__.py RENAMED Viewed

@@ -68,7 +68,7 @@ from typing import Dict, Optional, Any
 from inscriptis.model.config import ParserConfig
 from inscriptis.html_engine import Inscriptis
-RE_STRIP_XML_DECLARATION = re.compile(r'^<\?xml [^>]+?\?>')
+RE_STRIP_XML_DECLARATION = re.compile(r"^<\?xml [^>]+?\?>")
 def _get_html_tree(html_content: str) -> Optional[HtmlElement]:
@@ -85,13 +85,13 @@ def _get_html_tree(html_content: str) -> Optional[HtmlElement]:
         return None
     # strip XML declaration, if necessary
-    if html_content.startswith('<?xml '):
-        html_content = RE_STRIP_XML_DECLARATION.sub('', html_content, count=1)
+    if html_content.startswith("<?xml "):
+        html_content = RE_STRIP_XML_DECLARATION.sub("", html_content, count=1)
     try:
         return fromstring(html_content)
     except ParserError:
-        return fromstring('<pre>' + html_content + '</pre>')
+        return fromstring("<pre>" + html_content + "</pre>")
 def get_text(html_content: str, config: ParserConfig = None) -> str:
@@ -105,12 +105,12 @@ def get_text(html_content: str, config: ParserConfig = None) -> str:
       The text representation of the HTML content.
     """
     html_tree = _get_html_tree(html_content)
-    return Inscriptis(html_tree, config).get_text() if html_tree is not None \
-        else ''
+    return Inscriptis(html_tree, config).get_text() if html_tree is not None else ""
-def get_annotated_text(html_content: str,
-                       config: ParserConfig = None) -> Dict[str, Any]:
+def get_annotated_text(
+    html_content: str, config: ParserConfig = None
+) -> Dict[str, Any]:
     """Return a dictionary of the extracted text and annotations.
     Notes:
@@ -132,7 +132,5 @@ def get_annotated_text(html_content: str,
         return {}
     inscriptis = Inscriptis(html_tree, config)
-    labels = [(a.start, a.end, a.metadata)
-              for a in inscriptis.get_annotations()]
-    return {'text': inscriptis.get_text(),
-            'label': labels}
+    labels = [(a.start, a.end, a.metadata) for a in inscriptis.get_annotations()]
+    return {"text": inscriptis.get_text(), "label": labels}

{inscriptis-2.3.2 → inscriptis-2.4.0.1}/src/inscriptis/annotation/__init__.py RENAMED Viewed

@@ -29,9 +29,13 @@ class Annotation(NamedTuple):
     """a tuple of tags to be attached to the annotation."""
-def horizontal_shift(annotations: List[Annotation], content_width: int,
-                     line_width: int, align: HorizontalAlignment,
-                     shift: int = 0) -> List[Annotation]:
+def horizontal_shift(
+    annotations: List[Annotation],
+    content_width: int,
+    line_width: int,
+    align: HorizontalAlignment,
+    shift: int = 0,
+) -> List[Annotation]:
     r"""Shift annotations based on the given line's formatting.
     Adjusts the start and end indices of annotations based on the line's
@@ -56,5 +60,6 @@ def horizontal_shift(annotations: List[Annotation], content_width: int,
     else:
         h_align = shift + (line_width - content_width) // 2
-    return [Annotation(a.start + h_align, a.end + h_align, a.metadata)
-            for a in annotations]
+    return [
+        Annotation(a.start + h_align, a.end + h_align, a.metadata) for a in annotations
+    ]

inscriptis 2.3.2__tar.gz → 2.4.0.1__tar.gz

inscriptis 2.3.2tar.gz → 2.4.0.1tar.gz