PyPI - js-vars-extractor - Versions diffs - 0.1.6__tar.gz - Mend

js-vars-extractor 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (6) hide show

js_vars_extractor-0.1.6/PKG-INFO +57 -0
js_vars_extractor-0.1.6/README.md +46 -0
js_vars_extractor-0.1.6/pyproject.toml +110 -0
js_vars_extractor-0.1.6/src/js_vars_extractor/__init__.py +1 -0
js_vars_extractor-0.1.6/src/js_vars_extractor/_js_vars.py +124 -0
js_vars_extractor-0.1.6/src/js_vars_extractor/py.typed +0 -0

js_vars_extractor-0.1.6/PKG-INFO ADDED Viewed

@@ -0,0 +1,57 @@
+Metadata-Version: 2.3
+Name: js-vars-extractor
+Version: 0.1.6
+Summary: Extract JavaScript variables from HTML
+Author: Avazart
+Requires-Dist: chompjs>=1.4.1
+Requires-Dist: esprima>=4.0.1
+Requires-Dist: selectolax>=0.4.11
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+# av-js-vars-extractor
+A lightweight Python library designed to extract JavaScript variables, objects, and configurations from `<script>` tags within HTML documents
+## Installation
+Using [`uv`](https://github.com/astral-sh/uv):
+```bash
+uv add av-js-vars-extractor
+```
+### Quick Start
+```python
+from selectolax.lexbor import LexborHTMLParser
+from av_js_vars_extractor import find_js_var
+html = """
+<html>
+    <head>
+        <script>
+            window.APP_CONFIG = {
+                env: "production",
+                apiUrl: "[https://api.example.com](https://api.example.com)",
+                features: ["auth", "payments"]
+            };
+        </script>
+    </head>
+</html>
+"""
+parser = LexborHTMLParser(html)
+config = find_js_var("window.APP_CONFIG", parser)
+print(config)
+```
+```
+# Output:
+{
+   'env': 'production',
+   'apiUrl': '[https://api.example.com](https://api.example.com)',
+   'features': ['auth', 'payments']
+ }
+```

js_vars_extractor-0.1.6/README.md ADDED Viewed

@@ -0,0 +1,46 @@
+# av-js-vars-extractor
+A lightweight Python library designed to extract JavaScript variables, objects, and configurations from `<script>` tags within HTML documents
+## Installation
+Using [`uv`](https://github.com/astral-sh/uv):
+```bash
+uv add av-js-vars-extractor
+```
+### Quick Start
+```python
+from selectolax.lexbor import LexborHTMLParser
+from av_js_vars_extractor import find_js_var
+html = """
+<html>
+    <head>
+        <script>
+            window.APP_CONFIG = {
+                env: "production",
+                apiUrl: "[https://api.example.com](https://api.example.com)",
+                features: ["auth", "payments"]
+            };
+        </script>
+    </head>
+</html>
+"""
+parser = LexborHTMLParser(html)
+config = find_js_var("window.APP_CONFIG", parser)
+print(config)
+```
+```
+# Output:
+{
+   'env': 'production',
+   'apiUrl': '[https://api.example.com](https://api.example.com)',
+   'features': ['auth', 'payments']
+ }
+```

js_vars_extractor-0.1.6/pyproject.toml ADDED Viewed

@@ -0,0 +1,110 @@
+[project]
+name = "js_vars_extractor"
+version = "0.1.6"
+description = "Extract JavaScript variables from HTML"
+readme = "README.md"
+authors = [
+    { name = "Avazart"}
+]
+requires-python = ">=3.12"
+dependencies = [
+    "chompjs>=1.4.1",
+    "esprima>=4.0.1",
+    "selectolax>=0.4.11",
+]
+[build-system]
+requires = ["uv_build>=0.8.21,<0.9.0"]
+build-backend = "uv_build"
+[dependency-groups]
+dev = [
+    "mypy>=2.3.1",
+    "pre-commit>=4.6.2",
+    "pytest>=9.1.1",
+    "ruff>=0.16.6",
+    "vulture>=2.16",
+]
+[tool.ruff]
+exclude = [
+    ".git",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "venv",
+    ".venv",
+    ".env",
+    ".idea",
+    ".vscode",
+]
+line-length = 79
+target-version = "py311"
+[tool.ruff.format]
+quote-style = "double"
+skip-magic-trailing-comma = false
+[tool.ruff.lint]
+select = [
+    # pycodestyle
+    #"E",
+    # Pyflakes
+    "F",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-simplify
+    #"SIM",
+    # isort
+    "I",
+]
+ignore = []
+[tool.ruff.lint.extend-per-file-ignores]
+"__init__.py" = ["F401"]
+[tool.mypy]
+strict = false
+exclude = [
+    "^venv/.*$",
+    "^migrations/.*$",
+    "^tests/.*$",
+    "^scripts/.*$",
+    "^research/.*$",
+]
+plugins = []
+#[[tool.mypy.overrides]]
+# module = "aiogram.*,apscheduler.*,gspread.*,oauth2client.*"
+# ignore_missing_imports = true
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]
+python_files = "test_*.py"
+pythonpath = ["."]
+log_cli = true
+log_cli_level = "DEBUG"
+[tool.pylint.messages_control]
+disable = [
+  "C0114",
+  "C0115",
+  "C0116",
+  "R0903",
+  "R0913",
+  "R0914",
+  "R0917",
+]
+[tool.pylint.master]
+ignore = [
+   ".venv",
+   "migrations",
+   "tests",
+   "scripts",
+]

js_vars_extractor-0.1.6/src/js_vars_extractor/__init__.py ADDED Viewed

	@@ -0,0 +1 @@
1	+ from ._js_vars import extract_js_vars, find_js_var, js_vars_iter

js_vars_extractor-0.1.6/src/js_vars_extractor/_js_vars.py ADDED Viewed

@@ -0,0 +1,124 @@
+import enum
+import json
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+import chompjs
+import esprima
+from esprima import Error
+from esprima.nodes import ExpressionStatement, Node, VariableDeclaration
+from selectolax.lexbor import LexborHTMLParser
+logger = logging.getLogger(Path(__name__).parent.name)
+class StatementType(enum.StrEnum):
+    EXPRESSION_STATEMENT = "ExpressionStatement"
+    VARIABLE_DECLARATION = "VariableDeclaration"
+    FUNCTION_DECLARATION = "FunctionDeclaration"
+    CLASS_DECLARATION = "ClassDeclaration"
+class ExpressionType(enum.StrEnum):
+    LITERAL = "Literal"
+    OBJECT_EXPRESSION = "ObjectExpression"
+    ARRAY_EXPRESSION = "ArrayExpression"
+    MEMBER_EXPRESSION = "MemberExpression"  # window.CONFIG, obj['prop']
+    UNARY_EXPRESSION = "UnaryExpression"  # -100, !true, void 0
+    BINARY_EXPRESSION = "BinaryExpression"  # "a" + "b", 10 + 20
+    IDENTIFIER = "Identifier"  # undefined, NaN, configVar
+    CALL_EXPRESSION = "CallExpression"  # JSON.parse("...")
+    TEMPLATE_LITERAL = "TemplateLiteral"  # `template_string`
+    LOGICAL_EXPRESSION = "LogicalExpression"  # a || b
+    CONDITIONAL_EXPRESSION = "ConditionalExpression"  # a ? b : c
+def _extract_var_name(left: Node) -> str | None:
+    if left.type == ExpressionType.MEMBER_EXPRESSION:
+        obj_name = (
+            left.object.name
+            if left.object.type == ExpressionType.IDENTIFIER
+            else ""
+        )
+        prop_name = (
+            left.property.name
+            if left.property.type == ExpressionType.IDENTIFIER
+            else ""
+        )
+        return f"{obj_name}.{prop_name}" if obj_name else prop_name
+    elif left.type == ExpressionType.IDENTIFIER:
+        return left.name
+    return None
+def _extract_var_value(
+    expr,
+    script_text: str,
+) -> int | float | str | dict | list | None:
+    if expr.type == ExpressionType.LITERAL:
+        if isinstance(expr.value, str):
+            try:
+                return json.loads(expr.value)
+            except json.JSONDecodeError:
+                pass
+        return expr.value
+    elif expr.type in (
+        ExpressionType.OBJECT_EXPRESSION,
+        ExpressionType.ARRAY_EXPRESSION,
+    ):
+        text = script_text[expr.range[0] : expr.range[1]]  # noqa: E203
+        value = chompjs.parse_js_object(text)
+        return value
+    return None
+def _parse_assignment(
+    statement: ExpressionStatement,
+    scrtipt_text: str,
+    data: dict[str, object],
+) -> None:
+    expr = statement.expression
+    if expr.left and (var_name := _extract_var_name(expr.left)):
+        if expr.right.type == ExpressionType.LITERAL:
+            data[var_name] = _extract_var_value(expr.right, scrtipt_text)
+def _parse_variable_declaration(
+    statement: VariableDeclaration,
+    script_text: str,
+    data: dict[str, object],
+) -> None:
+    for decl in statement.declarations:
+        if decl.init:
+            data[decl.id.name] = _extract_var_value(decl.init, script_text)
+def extract_js_vars(script_text: str) -> dict[str, object]:
+    data: dict[str, object] = {}
+    ast = esprima.parseScript(script_text, {"tolerant": True, "range": True})
+    for statement in ast.body:
+        if statement.type == StatementType.EXPRESSION_STATEMENT:
+            _parse_assignment(statement, script_text, data)
+        elif statement.type == StatementType.VARIABLE_DECLARATION:
+            _parse_variable_declaration(statement, script_text, data)
+    return data
+def js_vars_iter(parser: LexborHTMLParser) -> Iterator[dict[str, object]]:
+    for script_el in parser.css("script:not([type='application/ld+json'])"):
+        if text := script_el.text():
+            try:
+                if script_data := extract_js_vars(text):
+                    yield script_data
+            except Error as e:
+                logger.debug("%s %s", type(e), e)
+def find_js_var(var_name: str, parser: LexborHTMLParser) -> Any | None:
+    for variables in js_vars_iter(parser):
+        if var_name in variables:
+            return variables[var_name]
+    return None

js_vars_extractor-0.1.6/src/js_vars_extractor/py.typed ADDED Viewed

File without changes