ssc_codegen 0.20.0__tar.gz → 0.21.0__tar.gz

{ssc_codegen-0.20.0 → ssc_codegen-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.20.0
+Version: 0.21.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -28,6 +28,9 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: soupsieve>=2.6
 Requires-Dist: typer>=0.15.1
 Requires-Dist: typing-extensions; python_version < '3.11'
+Provides-Extra: repl
+Requires-Dist: httpx>=0.28.1; extra == 'repl'
+Requires-Dist: ipython>=8.39.0; extra == 'repl'
 Description-Content-Type: text/markdown
 
 # ssc-codegen

{ssc_codegen-0.20.0 → ssc_codegen-0.21.0}/pyproject.toml

@@ -1,113 +1,119 @@
-[project]
-name = "ssc_codegen"
-version = "0.20.0"
-description = "Python-dsl code converter to html parser for web scraping "
-readme = "README.md"
-requires-python = ">=3.10"
-dependencies = [
-    "bs4>=0.0.2",
-    "colorama>=0.4.6 ; sys_platform == 'win32'",
-    "cssselect>=1.2.0",
-    "lxml>=5.3.0",
-    "soupsieve>=2.6",
-    "typer>=0.15.1",
-    "typing_extensions;python_version<'3.11'",
-    # https://github.com/fastapi/typer/discussions/1215
-    # https://github.com/fastapi/typer/pull/1145
-    "click<8.2.0",
-    "pyyaml>=6.0",
-]
-
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "Environment :: Console",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
-    "Topic :: Software Development :: Code Generators",
-    "Topic :: Text Processing :: Markup :: HTML",
-    "Topic :: Utilities",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-]
-
-[project.urls]
-Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
-Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
-Source = "https://github.com/vypivshiy/selector_schema_codegen"
-Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
-
-
-[project.scripts]
-ssc-gen = 'ssc_codegen.main:main'
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.sdist]
-include = [
-  "ssc_codegen/**",
-]
-
-[tool.hatch.build.targets.wheel]
-packages = ["ssc_codegen"]
-
-[dependency-groups]
-dev = [
-    "bs4>=0.0.2",
-    "coverage>=7.6.12",
-    "httpx>=0.28.1",
-    "hypothesis>=6.151.9",
-    "lxml>=5.3.0",
-    "mypy>=1.14.1",
-    "parsel>=1.9.1",
-    "pytest>=8.3.4",
-    "ruff>=0.9.3",
-    "selectolax>=0.3.27",
-]
-
-[tool.ruff]
-target-version = "py310"
-line-length = 80
-exclude = ["__init__.py"]
-
-[tool.mypy]
-python_version = "3.10"
-pretty = true
-ignore_missing_imports = true
-disallow_untyped_defs = true
-show_error_codes = true
-no_implicit_optional = true
-mypy_path = 'ssc_codegen'
-exclude = [
-    "converters/.*\\.py$",  # to tired typing
-    "examples/.*\\.py$"
-]
-[[tool.mypy.overrides]]
-module="tests.*"
-disallow_untyped_defs = false
-
-
-[tool.coverage.report]
-exclude_also = [
-    'def __repr__',
-    'if self.debug:',
-    'if settings.DEBUG',
-    'raise AssertionError',
-    'raise NotImplementedError',
-    'if 0:',
-    'if __name__ == .__main__.:',
-    'if TYPE_CHECKING:',
-    'class .*\bProtocol\):',
-    '@(abc\.)?abstractmethod',
-]
-
-[tool.coverage.run]
-source = ["ssc_codegen"]
-omit = [
-    "*/.venv/*",
-    "*/scripts/*",
-    "*/tests/*"
-]
+[project]
+name = "ssc_codegen"
+version = "0.21.0"
+description = "Python-dsl code converter to html parser for web scraping "
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "bs4>=0.0.2",
+    "colorama>=0.4.6 ; sys_platform == 'win32'",
+    "cssselect>=1.2.0",
+    "lxml>=5.3.0",
+    "soupsieve>=2.6",
+    "typer>=0.15.1",
+    "typing_extensions;python_version<'3.11'",
+    # https://github.com/fastapi/typer/discussions/1215
+    # https://github.com/fastapi/typer/pull/1145
+    "click<8.2.0",
+    "pyyaml>=6.0",
+]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Utilities",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.urls]
+Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
+Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
+Source = "https://github.com/vypivshiy/selector_schema_codegen"
+Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
+
+
+[project.scripts]
+ssc-gen = 'ssc_codegen.main:main'
+
+[project.optional-dependencies]
+repl = [
+    "httpx>=0.28.1",
+    "ipython>=8.39.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "ssc_codegen/**",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["ssc_codegen"]
+
+[dependency-groups]
+dev = [
+    "bs4>=0.0.2",
+    "coverage>=7.6.12",
+    "httpx>=0.28.1",
+    "hypothesis>=6.151.9",
+    "lxml>=5.3.0",
+    "mypy>=1.14.1",
+    "parsel>=1.9.1",
+    "pytest>=8.3.4",
+    "ruff>=0.9.3",
+    "selectolax>=0.3.27",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 80
+exclude = ["__init__.py"]
+
+[tool.mypy]
+python_version = "3.10"
+pretty = true
+ignore_missing_imports = true
+disallow_untyped_defs = true
+show_error_codes = true
+no_implicit_optional = true
+mypy_path = 'ssc_codegen'
+exclude = [
+    "converters/.*\\.py$",  # to tired typing
+    "examples/.*\\.py$"
+]
+[[tool.mypy.overrides]]
+module="tests.*"
+disallow_untyped_defs = false
+
+
+[tool.coverage.report]
+exclude_also = [
+    'def __repr__',
+    'if self.debug:',
+    'if settings.DEBUG',
+    'raise AssertionError',
+    'raise NotImplementedError',
+    'if 0:',
+    'if __name__ == .__main__.:',
+    'if TYPE_CHECKING:',
+    'class .*\bProtocol\):',
+    '@(abc\.)?abstractmethod',
+]
+
+[tool.coverage.run]
+source = ["ssc_codegen"]
+omit = [
+    "*/.venv/*",
+    "*/scripts/*",
+    "*/tests/*"
+]

{ssc_codegen-0.20.0 → ssc_codegen-0.21.0}/ssc_codegen/main.py

@@ -660,7 +660,152 @@ def health(
         raise typer.Exit(code=1)
 
 
-@app.command(name="openapi-to-ssc", help="Experimental converter swagger openapi 3.0 to kdl DSL")
+@app.command()
+def shell(
+    schema: Annotated[
+        str | None,
+        typer.Argument(
+            help="Schema target in format 'path/to/schema.kdl:StructName'. "
+            "If omitted, starts empty REPL.",
+        ),
+    ] = None,
+    target: Annotated[
+        _PyTarget,
+        typer.Option(
+            "--target",
+            "-t",
+            help="Python backend for the REPL.",
+            case_sensitive=False,
+        ),
+    ] = _PyTarget.PY_BS4,
+    input_file: Annotated[
+        Path | None,
+        typer.Option(
+            "--input",
+            "-i",
+            help="HTML input file.",
+            exists=True,
+            file_okay=True,
+            dir_okay=False,
+            readable=True,
+        ),
+    ] = None,
+    url: Annotated[
+        str | None,
+        typer.Option(
+            "--url",
+            help="URL to fetch HTML from.",
+        ),
+    ] = None,
+    http_client: Annotated[
+        str | None,
+        typer.Option(
+            "--http-client",
+            help="HTTP client for REST structs: httpx (default) or requests.",
+        ),
+    ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Print generated code and enable DEBUG logging.",
+        ),
+    ] = False,
+    css_to_xpath: Annotated[
+        bool,
+        typer.Option(
+            "--css-to-xpath",
+            help="Convert CSS selectors to XPath.",
+        ),
+    ] = False,
+) -> None:
+    """Launch an interactive REPL shell for testing KDL schema parsers.
+
+    \b
+    Examples:
+        ssc-gen shell examples/booksToScrape.kdl:Book --url https://books.toscrape.com/
+        ssc-gen shell schema.kdl:Product -i page.html -t py-lxml
+        ssc-gen shell examples/restApiLike.kdl:DummyJsonApi --http-client httpx
+        ssc-gen shell
+    """
+    from ssc_codegen.repl import Repl, ReplState
+
+    if verbose:
+        setup_debug_logging()
+
+    state = ReplState(
+        target=target.value,
+        http_client=http_client or "httpx",
+        verbose=verbose,
+        css_to_xpath=css_to_xpath,
+    )
+
+    if input_file is not None:
+        state.html = input_file.read_text(encoding="utf-8")
+
+    if schema is not None:
+        if ":" in schema:
+            path_part, struct_name = schema.rsplit(":", 1)
+            kdl_path = Path(path_part)
+        else:
+            kdl_path = Path(schema)
+            struct_name = ""
+        if not kdl_path.is_file():
+            typer.echo(f"ERROR: file not found: {kdl_path}", err=True)
+            raise typer.Exit(code=1)
+        from ssc_codegen import parse_ast
+        from ssc_codegen.ast import Struct as StructNode
+
+        try:
+            state.module_ast = parse_ast(
+                path=str(kdl_path), css_to_xpath=css_to_xpath
+            )
+        except Exception as exc:
+            if verbose:
+                typer.echo(traceback.format_exc(), err=True)
+            else:
+                typer.echo(
+                    f"ERROR: failed to parse {kdl_path}: {exc}", err=True
+                )
+            raise typer.Exit(code=1)
+
+        state.schema_path = kdl_path
+        state.kdl_source = kdl_path.read_text(encoding="utf-8-sig")
+
+        structs = [
+            n for n in state.module_ast.body if isinstance(n, StructNode)
+        ]
+        if not struct_name:
+            if structs:
+                struct_name = structs[0].name
+        struct_names = [s.name for s in structs]
+        if struct_name and struct_name not in struct_names:
+            typer.echo(
+                f"ERROR: struct '{struct_name}' not found. "
+                f"Available: {', '.join(struct_names)}",
+                err=True,
+            )
+            raise typer.Exit(code=1)
+        state.struct_name = struct_name
+
+    if url is not None and not state.html:
+        try:
+            from ssc_codegen.repl import _fetch_html
+
+            state.html = _fetch_html(url)
+        except Exception as exc:
+            typer.echo(f"ERROR fetching URL: {exc}", err=True)
+            raise typer.Exit(code=1)
+
+    repl = Repl(state)
+    repl.cmdloop()
+
+
+@app.command(
+    name="openapi-to-ssc",
+    help="Experimental converter swagger openapi 3.0 to kdl DSL",
+)
 def openapi_to_ssc(
     spec: Annotated[
         Path,