pyhtml2md 1.6.6__tar.gz → 1.7.0__tar.gz

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             files: |
               build/packages/html2md*.deb
               build/packages/html2md*.tar.gz

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/.gitignore

@@ -19,3 +19,4 @@ DerivedData/
 .swiftpm/configuration/registries.json
 .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
 .netrc
+*.pyc

pyhtml2md-1.7.0/.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "cmake.configureArgs": [
+        "-DBUILD_TEST=ON",
+        "-DPYTHON_BINDINGS=ON",
+        "-G=Ninja",        
+    ]
+}

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 [TOC]
 
+## 1.7.0
+- Added API to add/remove HTML symbol conversions (see #158)
+- Added HTML symbol conversion API to Python bindings
+- Fixed attribute parsing with whitespace around equals sign (see #159)
+- Added full support for uppercase tags/attributes
+- Supported whitespace around tag names
+- Fixed issues with self-closing tags
+- Improved performance
+- Improved blockquote handling (see #157)
+
 ## 1.6.6
 - Python: Fix wheels not uploaded
 

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/CMakeLists.txt

@@ -1,5 +1,5 @@
-cmake_minimum_required(VERSION 3.8)
-project(html2md VERSION 1.6.6 LANGUAGES CXX)
+cmake_minimum_required(VERSION 3.8...3.31)
+project(html2md VERSION 1.7.0 LANGUAGES CXX)
 
 set(PROJECT_HOMEPAGE_URL "https://tim-gromeyer.github.io/html2md/")
 set(html2md_HOMEPAGE_URL "${PROJECT_HOMEPAGE_URL}")

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyhtml2md
-Version: 1.6.6
+Version: 1.7.0
 Summary: Transform your HTML into clean, easy-to-read markdown with pyhtml2md.
 Keywords: html,markdown,html-to-markdown,python3,cpp17,cpp-library,html2markdown,html2md
 Author-Email: Tim Gromeyer <sakul8826@gmail.com>

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/include/html2md.h

@@ -170,7 +170,7 @@ public:
    * You can use appendToMd() to append something to the beginning of the
    * generated output.
    */
-  explicit inline Converter(std::string &html,
+  explicit inline Converter(const std::string &html,
                             struct Options *options = nullptr) {
     *this = Converter(&html, options);
   }
@@ -217,6 +217,39 @@ public:
    */
   Converter *appendBlank();
 
+  /*!
+    * \brief Add an HTML symbol conversion
+    * \param htmlSymbol The HTML symbol to convert
+    * \param replacement The replacement string
+    * \note This is useful for converting HTML entities to their Markdown
+    * equivalents. For example, you can add a conversion for " " to
+    * " " (space) or "&lt;" to "<" (less than).
+    * \note This is not a standard feature of the Converter class, but it can
+    * be added to the class to allow for more flexibility in the conversion
+    * process. You can use this feature to add custom conversions for any HTML
+    * symbol that you want to convert to a specific Markdown representation.
+   */
+  void addHtmlSymbolConversion(const std::string &htmlSymbol,
+                               const std::string &replacement) {
+    htmlSymbolConversions_[htmlSymbol] = replacement;
+  }
+
+  /*!
+   * \brief Remove an HTML symbol conversion
+   * \param htmlSymbol The HTML symbol to remove
+   * \note This is useful for removing custom conversions that you have added
+   * previously.
+   */
+  void removeHtmlSymbolConversion(const std::string &htmlSymbol) {
+    htmlSymbolConversions_.erase(htmlSymbol);
+  }
+
+  /*!
+   * \brief Clear all HTML symbol conversions
+   * \note This is useful for clearing the conversion map (it's empty afterwards).
+   */
+  void clearHtmlSymbolConversions() { htmlSymbolConversions_.clear(); }
+
   /*!
    * \brief Checks if everything was closed properly(in the HTML).
    * \return Returns false if there is a unclosed tag.
@@ -347,6 +380,10 @@ private:
 
   Options option;
 
+  std::unordered_map<std::string, std::string> htmlSymbolConversions_ = {
+      {"&quot;", "\""}, {"&lt;", "<"},   {"&gt;", ">"},
+      {"&amp;", "&"},   {"&nbsp;", " "}, {"&rarr;", "→"}};
+
   // Tag: base class for tag types
   struct Tag {
     virtual void OnHasLeftOpeningTag(Converter *c) = 0;
@@ -511,7 +548,7 @@ private:
 
   std::unordered_map<std::string, std::shared_ptr<Tag>> tags_;
 
-  explicit Converter(std::string *html, struct Options *options);
+  explicit Converter(const std::string *html, struct Options *options);
 
   void CleanUpMarkdown();
 
@@ -594,7 +631,7 @@ private:
  * \param ok Optional: Pass a reference to a local bool to store the output of
  * Converter::ok() \return Returns the by Converter generated Markdown
  */
-inline std::string Convert(std::string &html, bool *ok = nullptr) {
+inline std::string Convert(const std::string &html, bool *ok = nullptr) {
   Converter c(html);
   auto md = c.convert();
   if (ok != nullptr)
@@ -603,7 +640,7 @@ inline std::string Convert(std::string &html, bool *ok = nullptr) {
 }
 
 #ifndef PYTHON_BINDINGS
-inline std::string Convert(std::string &&html, bool *ok = nullptr) {
+inline std::string Convert(const std::string &&html, bool *ok = nullptr) {
   return Convert(html, ok);
 }
 #endif

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/pyproject.toml

@@ -11,7 +11,7 @@ description = "Transform your HTML into clean, easy-to-read markdown with pyhtml
 readme = "python/README.md"
 requires-python = ">=3.7"
 license = { text = "MIT" }
-version = "1.6.6"
+version = "1.7.0"
 classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
@@ -73,7 +73,7 @@ filterwarnings = ["error"]
 testpaths = ["tests"]
 
 [tool.cibuildwheel]
-test-command = "pytest {project}/tests"
+test-command = "pytest {project}/tests/python"
 test-extras = ["test"]
 test-skip = ["*universal2:arm64"]
 # Setuptools bug causes collision between pypy and cpython artifacts

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/python/bindings.cpp

@@ -34,9 +34,19 @@ PYBIND11_MODULE(pyhtml2md, m) {
            "This function actually converts the HTML into Markdown.")
       .def("ok", &html2md::Converter::ok,
            "Checks if everything was closed properly(in the HTML).")
+      .def("add_html_symbol_conversion",
+           &html2md::Converter::addHtmlSymbolConversion,
+           "Add or modify an HTML symbol conversion", py::arg("html_symbol"),
+           py::arg("replacement"))
+      .def("remove_html_symbol_conversion",
+           &html2md::Converter::removeHtmlSymbolConversion,
+           "Remove an HTML symbol conversion", py::arg("html_symbol"))
+      .def("clear_html_symbol_conversions",
+           &html2md::Converter::clearHtmlSymbolConversions,
+           "Clear all HTML symbol conversions")
       .def("__call__", &html2md::Converter::operator bool);
 
   m.def("convert", &html2md::Convert,
         "Static wrapper around the Converter class", py::arg("html"),
         py::arg("ok") = py::none());
-}
+}

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/src/html2md.cpp

@@ -74,14 +74,25 @@ string Repeat(const string &str, size_t amount) {
 
   return out;
 }
+
+string toLower(const string &str) {
+  string lower;
+  lower.reserve(str.size());
+  for (char ch : str) {
+    lower += tolower(ch);
+  }
+  return lower;
+}
+
 } // namespace
 
 namespace html2md {
 
-Converter::Converter(string *html, Options *options) : html_(*html) {
+Converter::Converter(const string *html, Options *options) : html_(*html) {
   if (options)
     option = *options;
 
+  md_.reserve(html->size() * 0.8);
   tags_.reserve(41);
 
   // non-printing tags
@@ -144,22 +155,43 @@ Converter::Converter(string *html, Options *options) : html_(*html) {
 
 void Converter::CleanUpMarkdown() {
   TidyAllLines(&md_);
+  std::string buffer;
+  buffer.reserve(md_.size());
+
+  // Replace HTML symbols during the initial pass
+  for (size_t i = 0; i < md_.size();) {
+    bool replaced = false;
+
+    // C++11 compatible iteration over htmlSymbolConversions_
+    for (const auto &symbol_replacement : htmlSymbolConversions_) {
+      const std::string &symbol = symbol_replacement.first;
+      const std::string &replacement = symbol_replacement.second;
+
+      if (md_.compare(i, symbol.size(), symbol) == 0) {
+        buffer.append(replacement);
+        i += symbol.size();
+        replaced = true;
+        break;
+      }
+    }
 
-  ReplaceAll(&md_, " , ", ", ");
+    if (!replaced) {
+      buffer.push_back(md_[i++]);
+    }
+  }
 
-  ReplaceAll(&md_, "\n.\n", ".\n");
-  ReplaceAll(&md_, "\n↵\n", " ↵\n");
-  ReplaceAll(&md_, "\n*\n", "\n");
-  ReplaceAll(&md_, "\n. ", ".\n");
+  // Use swap instead of move assignment for better pre-C++11 compatibility
+  md_.swap(buffer);
 
-  ReplaceAll(&md_, "&quot;", '"');
-  ReplaceAll(&md_, "&lt;", "<");
-  ReplaceAll(&md_, "&gt;", ">");
-  ReplaceAll(&md_, "&amp;", '&');
-  ReplaceAll(&md_, "&nbsp;", ' ');
-  ReplaceAll(&md_, "&rarr;", "→");
+  // Optimized replacement sequence
+  const char *replacements[][2] = {
+      {" , ", ", "},   {"\n.\n", ".\n"},   {"\n↵\n", " ↵\n"}, {"\n*\n", "\n"},
+      {"\n. ", ".\n"}, {"\t\t  ", "\t\t"},
+  };
 
-  ReplaceAll(&md_, "\t\t  ", "\t\t");
+  for (const auto &replacement : replacements) {
+    ReplaceAll(&md_, replacement[0], replacement[1]);
+  }
 }
 
 Converter *Converter::appendToMd(char ch) {
@@ -286,9 +318,10 @@ void Converter::TidyAllLines(string *str) {
 string Converter::ExtractAttributeFromTagLeftOf(const string &attr) {
   // Extract the whole tag from current offset, e.g. from '>', backwards
   auto tag = html_.substr(offset_lt_, index_ch_in_html_ - offset_lt_);
+  string lowerTag = toLower(tag); // Convert tag to lowercase for comparison
 
-  // locate given attribute
-  auto offset_attr = tag.find(attr);
+  // locate given attribute (case-insensitive)
+  auto offset_attr = lowerTag.find(attr);
 
   if (offset_attr == string::npos)
     return "";
@@ -387,6 +420,7 @@ string Converter::convert() {
 void Converter::OnHasEnteredTag() {
   offset_lt_ = index_ch_in_html_;
   is_in_tag_ = true;
+  is_closing_tag_ = false;
   prev_tag_ = current_tag_;
   current_tag_ = "";
 
@@ -407,28 +441,55 @@ Converter *Converter::UpdatePrevChFromMd() {
 }
 
 bool Converter::ParseCharInTag(char ch) {
+  static bool skipping_leading_whitespace = true;
+
   if (ch == '/' && !is_in_attribute_value_) {
     is_closing_tag_ = current_tag_.empty();
     is_self_closing_tag_ = !is_closing_tag_;
-
+    skipping_leading_whitespace = true; // Reset for next tag
     return true;
   }
 
-  if (ch == '>')
-    return OnHasLeftTag();
+  if (ch == '>') {
+    // Trim trailing whitespace by removing characters from current_tag_
+    while (!current_tag_.empty() && std::isspace(current_tag_.back())) {
+      current_tag_.pop_back();
+    }
+    skipping_leading_whitespace = true; // Reset for next tag
+    if (!is_self_closing_tag_)
+      return OnHasLeftTag();
+    else {
+      OnHasLeftTag();
+      is_self_closing_tag_ = false;
+      is_closing_tag_ = true;
+      return OnHasLeftTag();
+    }
+  }
 
   if (ch == '"') {
     if (is_in_attribute_value_) {
       is_in_attribute_value_ = false;
-    } else if (current_tag_[current_tag_.length() - 1] == '=') {
-      is_in_attribute_value_ = true;
+    } else {
+      size_t pos = current_tag_.length();
+      while (pos > 0 && isspace(current_tag_[pos - 1])) {
+        pos--;
+      }
+      if (pos > 0 && current_tag_[pos - 1] == '=') {
+        is_in_attribute_value_ = true;
+      }
     }
-
+    skipping_leading_whitespace = false; // Stop skipping after attribute
     return true;
   }
 
-  current_tag_ += ch;
+  // Handle whitespace: skip leading whitespace, keep others
+  if (isspace(ch) && skipping_leading_whitespace) {
+    return true; // Ignore leading whitespace
+  }
 
+  // Once we encounter a non-whitespace character, stop skipping
+  skipping_leading_whitespace = false;
+  current_tag_ += tolower(ch);
   return false;
 }
 
@@ -455,7 +516,7 @@ bool Converter::OnHasLeftTag() {
   if (!is_closing_tag_) {
     tag->OnHasLeftOpeningTag(this);
   }
-  if (is_closing_tag_ || is_self_closing_tag_) {
+  else {
     is_closing_tag_ = false;
 
     tag->OnHasLeftClosingTag(this);
@@ -962,11 +1023,17 @@ void Converter::TagTableData::OnHasLeftClosingTag(Converter *c) {}
 
 void Converter::TagBlockquote::OnHasLeftOpeningTag(Converter *c) {
   ++c->index_blockquote;
+  c->appendToMd("\n");
+  c->appendToMd(Repeat("> ", c->index_blockquote));
 }
 
 void Converter::TagBlockquote::OnHasLeftClosingTag(Converter *c) {
   --c->index_blockquote;
-  c->ShortenMarkdown(2); // Remove the '> '
+  // Only shorten if a "> " was added (i.e., a newline was processed in the blockquote)
+  if (!c->md_.empty() && c->md_.length() >= 2 &&
+      c->md_.substr(c->md_.length() - 2) == "> ") {
+    c->ShortenMarkdown(2); // Remove the '> ' only if it exists
+  }
 }
 
 void Converter::reset() {

{pyhtml2md-1.6.6 → pyhtml2md-1.7.0}/tests/CMakeLists.txt

@@ -2,14 +2,11 @@ project(tests LANGUAGES C CXX)
 
 if (NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/md4c/src/)
     include(FindGit)
-
     if(NOT GIT_FOUND)
         message(WARNING "git not found. Please download md4c manually or disable tests.")
         return()
     endif()
-
     get_directory_property(dir PARENT_DIRECTORY)
-
     execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --depth=1
                     WORKING_DIRECTORY ${dir})
 endif()
@@ -26,12 +23,19 @@ set(MD4C_FILES
 add_library(md4c-html STATIC ${MD4C_FILES})
 target_include_directories(md4c-html PUBLIC md4c/src)
 
+# Existing test executable
 add_executable(test-exe main.cpp)
 target_link_libraries(test-exe md4c-html html2md-static)
 target_compile_definitions(test-exe PUBLIC DIR="${CMAKE_CURRENT_LIST_DIR}")
 set_target_properties(test-exe PROPERTIES OUTPUT_NAME "tests")
-target_compile_features(test-exe PUBLIC cxx_std_17) # Require at least c++17
+target_compile_features(test-exe PUBLIC cxx_std_17)
 
+# New benchmark executable
+add_executable(benchmark-exe benchmark.cpp)
+target_link_libraries(benchmark-exe md4c-html html2md-static)
+target_compile_definitions(benchmark-exe PUBLIC DIR="${CMAKE_CURRENT_LIST_DIR}")
+set_target_properties(benchmark-exe PROPERTIES OUTPUT_NAME "benchmarks")
+target_compile_features(benchmark-exe PUBLIC cxx_std_17)
 
 if (CMAKE_VERSION VERSION_LESS 3.11.0)
     return()
@@ -39,6 +43,12 @@ endif()
 
 add_custom_target(test
     COMMAND $<TARGET_FILE:test-exe>
-    COMMENT Runing tests..
+    COMMENT Running tests..
     DEPENDS test-exe
 )
+
+add_custom_target(benchmark
+    COMMAND $<TARGET_FILE:benchmark-exe>
+    COMMENT Running benchmarks..
+    DEPENDS benchmark-exe
+)

pyhtml2md-1.7.0/tests/benchmark.cpp

@@ -0,0 +1,184 @@
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <filesystem>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "html2md.h"
+#include "md4c-html.h"
+#include "table.h"
+
+using std::cerr;
+using std::cout;
+using std::ifstream;
+using std::string;
+using std::stringstream;
+using std::vector;
+using std::chrono::duration;
+using std::chrono::high_resolution_clock;
+using std::chrono::microseconds;
+namespace fs = std::filesystem;
+
+// Markdown and HTML utility functions
+namespace markdown {
+void captureHtmlFragment(const MD_CHAR *data, const MD_SIZE data_size,
+                         void *userData) {
+  auto *str = static_cast<stringstream *>(userData);
+  str->write(data, data_size);
+}
+
+string toHTML(const string &md) {
+  stringstream html;
+  static MD_TOC_OPTIONS options;
+  md_html(md.c_str(), md.size(), &captureHtmlFragment, &html, MD_DIALECT_GITHUB,
+          MD_HTML_FLAG_SKIP_UTF8_BOM, &options);
+  return html.str();
+}
+
+string fromHTML(string &html) {
+  static html2md::Options options;
+  options.splitLines = false;
+  html2md::Converter c(html, &options);
+  return c.convert();
+}
+} // namespace markdown
+
+// Benchmark result structure
+struct BenchmarkResult {
+  string test_name;
+  double avg_time_us;     // Average time in microseconds
+  double std_dev_us;      // Standard deviation in microseconds
+  size_t input_size;      // Input size in bytes
+  double throughput_mbps; // Throughput in megabytes per second
+};
+
+// Benchmark runner class
+class BenchmarkRunner {
+public:
+  void addTest(const string &name, const string &input, bool is_markdown) {
+    tests_.push_back({name, input, is_markdown});
+  }
+
+  void run(int iterations) {
+    auto start_total = high_resolution_clock::now(); // Start total timer
+    for (const auto &test : tests_) {
+      runTest(test, iterations);
+    }
+    auto end_total = high_resolution_clock::now(); // End total timer
+    total_duration_ms_ =
+        duration<double, std::milli>(end_total - start_total).count();
+    printSummary();
+  }
+
+private:
+  struct Test {
+    string name;
+    string input;
+    bool is_markdown; // false for HTML-to-Markdown
+  };
+
+  vector<Test> tests_;
+  vector<BenchmarkResult> results_;
+  double total_duration_ms_ = 0.0; // Total duration in milliseconds
+
+  void runTest(const Test &test, int iterations) {
+    vector<double> times_us(iterations);
+    size_t input_size = test.input.size();
+
+    for (int i = 0; i < iterations; ++i) {
+      auto start = high_resolution_clock::now();
+      if (test.is_markdown) {
+        markdown::toHTML(test.input);
+      } else {
+        string input_copy = test.input; // fromHTML modifies input
+        markdown::fromHTML(input_copy);
+      }
+      auto end = high_resolution_clock::now();
+      times_us[i] = duration<double, std::micro>(end - start).count();
+    }
+
+    // Calculate average and standard deviation
+    double sum = 0.0;
+    for (double t : times_us)
+      sum += t;
+    double avg_time_us = sum / iterations;
+
+    double variance_sum = 0.0;
+    for (double t : times_us) {
+      variance_sum += (t - avg_time_us) * (t - avg_time_us);
+    }
+    double std_dev_us = std::sqrt(variance_sum / iterations);
+
+    // Calculate throughput (MB/s)
+    double avg_time_s = avg_time_us / 1e6;
+    double throughput_mbps = (input_size / (1024.0 * 1024.0)) / avg_time_s;
+
+    results_.push_back(
+        {test.name, avg_time_us, std_dev_us, input_size, throughput_mbps});
+  }
+
+  void printSummary() {
+    cout << "\n=== Benchmark Summary ===\n";
+    cout << std::left << std::setw(30) << "Test Name" << std::setw(15)
+         << "Avg Time (us)" << std::setw(15) << "Std Dev (us)" << std::setw(15)
+         << "Input Size (B)" << std::setw(15) << "Throughput (MB/s)\n";
+    cout << std::string(90, '-') << "\n";
+
+    for (const auto &result : results_) {
+      cout << std::left << std::setw(30) << result.test_name << std::fixed
+           << std::setprecision(2) << std::setw(15) << result.avg_time_us
+           << std::setw(15) << result.std_dev_us << std::setw(15)
+           << result.input_size << std::setw(15) << result.throughput_mbps
+           << "\n";
+    }
+
+    cout << "\nTotal Benchmark Duration: " << std::fixed << std::setprecision(2)
+         << total_duration_ms_ << " ms\n";
+  }
+};
+
+namespace file {
+string readAll(const string &name) {
+  ifstream in(name);
+  stringstream buffer;
+  buffer << in.rdbuf();
+  return buffer.str();
+}
+} // namespace file
+
+int main() {
+  using namespace markdown;
+  BenchmarkRunner runner;
+  const int iterations = 10000; // Number of iterations per test
+
+  // Add tests for Markdown files in the directory
+  vector<string> files;
+  static vector<string> markdownExtensions = {".md", ".markdown", ".mkd"};
+  for (const auto &p : fs::recursive_directory_iterator(DIR)) {
+    if (std::find(markdownExtensions.begin(), markdownExtensions.end(),
+                  p.path().extension()) != markdownExtensions.end() &&
+        p.path().parent_path() == DIR) {
+      files.emplace_back(p.path().string());
+    }
+  }
+  std::sort(files.begin(), files.end());
+
+  for (const auto &file : files) {
+    string md = file::readAll(file);
+    string html = markdown::toHTML(md);
+    string filename = fs::path(file).filename().string();
+    runner.addTest(filename, html, false);
+  }
+
+  // Run benchmarks
+  cout << "Running benchmarks with " << iterations
+       << " iterations per test...\n";
+  runner.run(iterations);
+
+  return 0;
+}