docstring-generator-ext 2.0.12__tar.gz → 2.0.13b1__tar.gz

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docstring_generator_ext
-Version: 2.0.12
+Version: 2.0.13b1
 Summary: Generate Docstrings with type-hint information.
 Author-email: FelixTheC <fberndt87@gmail.com>
 Classifier: Environment :: Console

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/docstring_generator_ext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docstring_generator_ext
-Version: 2.0.12
+Version: 2.0.13b1
 Summary: Generate Docstrings with type-hint information.
 Author-email: FelixTheC <fberndt87@gmail.com>
 Classifier: Environment :: Console

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/docstring_generator_ext.egg-info/SOURCES.txt

@@ -21,5 +21,8 @@ src/docComparator.cpp
 src/docComparator.hpp
 src/docstringFormat.cpp
 src/docstringFormat.hpp
+src/existingDocstringExtractor.cpp
+src/existingDocstringExtractor.hpp
+src/formatStyles.hpp
 src/parser.cpp
 src/parser.hpp

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "docstring_generator_ext"
-version = "2.0.12"
+version = "2.0.13b1"
 description = "Generate Docstrings with type-hint information."
 authors = [
     { name = "FelixTheC", email = "fberndt87@gmail.com" },

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/setup.py

@@ -47,6 +47,7 @@ ext_modules = [
             "src/NumpyDocstring.cpp",
             "src/ReStructuredDocstring.cpp",
             "src/docComparator.cpp",
+            "src/existingDocstringExtractor.cpp",
         ],
         include_dirs=["src"],
         cxx_std=20,

{docstring_generator_ext-2.0.12 → docstring_generator_ext-2.0.13b1}/src/FunctionFormat.cpp

@@ -1,8 +1,10 @@
 #include "FunctionFormat.hpp"
 
+#include <complex>
 #include <map>
 #include <ranges>
 
+#include "existingDocstringExtractor.hpp"
 #include "parser.hpp"
 
 const std::map<std::string, ParameterKind> ParameterKinds = {
@@ -97,6 +99,16 @@ void FunctionParameter::update_rest_description(const std::string &descr) noexce
     description = remove_whitespace(remove_trailing_whitespace(new_description));
 }
 
+void FunctionParameter::erase() {
+  name.clear();
+  default_value.clear();
+  type.clear();
+  kind = ParameterKind::ARG;
+  line_no = 0;
+  description.clear();
+  skip = true;
+}
+
 [[nodiscard]] int FunctionInfo::get_file_write_position() const noexcept {
     if (!docstring.docstring.empty()) {
         return docstring.start_line;
@@ -117,32 +129,25 @@ void FunctionParameter::update_rest_description(const std::string &descr) noexce
 }
 
 void FunctionInfo::update_descriptions(const DocstringFormatStyle &formatStyle) noexcept {
-    for (size_t idx = 0; idx < args.size(); ++idx) {
-        size_t start_pos;
 
-        if (formatStyle == DocstringFormatStyle::reST) {
-            start_pos = docstring.docstring.find(":param " + args[idx].name + ":");
-        } else {
-            start_pos = docstring.docstring.find(args[idx].name);
-        }
+  const auto extractor = ExistingDocstringExtractor::parse_text(docstring.docstring, formatStyle);
+  if (!extractor.get_return_info().empty())
+    returns.description = extractor.get_return_info()[0].description;
 
-        if (start_pos < std::string::npos) {
-            size_t end_pos = docstring.docstring.size() - 1;
-
-            if (idx < args.size() - 1) {
-                end_pos = docstring.docstring.find(args[idx + 1].name);
-            } else if (idx == args.size() - 1 && docstring.docstring.find("Returns") < std::string::npos) {
-                if (const auto return_pos = docstring.docstring.find("Returns"); return_pos != std::string::npos) {
-                    end_pos = return_pos - 1;
-                }
-            }
-
-            if (end_pos != std::string::npos) {
-                std::string part_of_interest = docstring.docstring.substr(start_pos, end_pos - start_pos);
-                args[idx].update_description(part_of_interest, formatStyle);
-            }
-        }
+  if (!extractor.get_exception_info().empty()) {
+    for (const auto &exception: extractor.get_exception_info()) {
+      exceptions.emplace(exception.type, exception.description);
     }
+  }
+
+  docstring.docstring = extractor.get_reduced_docstring();
+  size_t param_idx = 0;
+  // we expect that the params are in the same order, cause otherwise we had somewhere a violation agains a PEP
+  for (const auto &param: extractor.get_extracted_parameters()) {
+    // args[param_idx].default_value = param.default_val;
+    args[param_idx].description = param.description;
+    ++param_idx;
+  }
 }
 
 bool FunctionInfo::summary_only() const noexcept {
@@ -243,7 +248,7 @@ void FunctionDocstring::detect_format_style() noexcept {
     if (end_pos == -1 || static_cast<size_t>(end_pos) > lines.size())
         return -1;
 
-    std::string tmp = "";
+    std::string tmp;
     lines.clear();
     for (const auto&line : std::views::split(docstring, '\n') | std::views::transform([](const auto &obj){ return std::string(obj.begin(), obj.end()); })) {
         lines.emplace_back(line);

docstring_generator_ext-2.0.13b1/src/FunctionFormat.hpp

@@ -0,0 +1,123 @@
+#ifndef DOCSTRING_GENERATOR_EXT_FUNCTIONFORMAT_HPP
+#define DOCSTRING_GENERATOR_EXT_FUNCTIONFORMAT_HPP
+
+#include "formatStyles.hpp"
+
+#include <format>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <pybind11/pybind11.h>
+namespace py = pybind11;
+
+enum class ParameterKind {
+  ARG,
+  POS_ONLY,
+  KW_ONLY,
+  VARIADIC_ARG,
+  KEYWORD_ARG,
+};
+
+using StyleSections = std::array<std::string_view, 3>;
+constexpr std::array<std::pair<std::string_view, StyleSections>, 3> TypicalFormatStyle = {
+      {
+        {"numpy", {"Parameters\n", "Returns\n", "Raises\n"}},
+        {"google", {"Args:", "Returns:", "Raises:"}},
+        {"rest", {":param", ":returns:", ":raises"}}
+      }
+    };
+
+constexpr std::array<std::pair<std::string_view, StyleSections>, 3> TypicalSectionStyle = {
+      {
+        {"numpy", {"Parameters", "Returns", "Raises"}},
+        {"google", {"Args:", "Returns:", "Raises:"}},
+        {"rest", {":param", ":returns:", ":raises"}}
+      }
+    };
+
+ParameterKind from_str(const std::string &kind);
+
+std::ostream &operator<<(std::ostream &out, ParameterKind const &obj) noexcept;
+
+namespace parameter_kind {
+  inline std::string to_str(const ParameterKind &kind) {
+    switch (kind) {
+      case ParameterKind::ARG: return "Argument";
+      case ParameterKind::POS_ONLY: return "Positional only argument";
+      case ParameterKind::KW_ONLY: return "Keyword only argument";
+      case ParameterKind::VARIADIC_ARG: return "Variadic arguments";
+      case ParameterKind::KEYWORD_ARG: return "Keyword arguments";
+    }
+    return "UNKNOWN";
+  }
+}
+
+template<>
+struct std::formatter<ParameterKind> : std::formatter<std::string> {
+  auto format(const ParameterKind &kind, std::format_context &ctx) const {
+    return std::formatter<std::string>::format(parameter_kind::to_str(kind), ctx);
+  }
+};
+
+struct FunctionParameter {
+  std::string name;
+  std::string default_value;
+  std::string type;
+  ParameterKind kind;
+  uint32_t line_no;
+  std::string description{};
+  bool skip {false};
+
+  void update_description(const std::string &descr, const DocstringFormatStyle &formatStyle) noexcept;
+
+  void update_description(const std::string &descr) noexcept;
+
+  void update_rest_description(const std::string &descr) noexcept;
+
+  void erase();
+};
+
+struct FunctionReturn {
+  uint32_t line_no{};
+  std::string type;
+  std::string description;
+};
+
+struct FunctionDocstring {
+  uint32_t start_line{};
+  size_t end_line{};
+  std::string docstring;
+  std::optional<DocstringFormatStyle> detected_format_style{std::nullopt};
+
+  void detect_format_style() noexcept;
+
+  [[nodiscard]] int remove_old_format_style() noexcept;
+};
+
+FunctionDocstring get_docstring(const py::object &obj, const py::module &ast_module) noexcept;
+
+struct FunctionException {
+  std::string name;
+  std::string description;
+
+  auto operator<=>(const FunctionException &) const = default;
+};
+
+struct FunctionInfo {
+  std::vector<FunctionParameter> args{};
+  uint32_t offset{};
+  uint64_t body_start{};
+  FunctionDocstring docstring;
+  FunctionReturn returns;
+  std::set<FunctionException> exceptions{};
+
+  [[nodiscard]] int get_file_write_position() const noexcept;
+
+  void update_descriptions(const DocstringFormatStyle &formatStyle) noexcept;
+
+  [[nodiscard]] bool summary_only() const noexcept;
+};
+
+#endif //DOCSTRING_GENERATOR_EXT_FUNCTIONFORMAT_HPP

docstring_generator_ext-2.0.13b1/src/GoogleDocstring.cpp

@@ -0,0 +1,153 @@
+#include "GoogleDocstring.hpp"
+#include "parser.hpp"
+
+void GoogleDocstring::check_current_docstring() noexcept {
+  auto current_py_tab = get_tabs();
+  auto google_args_begin = functionInfo.docstring.docstring.find("Args:");
+
+  if (google_args_begin != std::string::npos) {
+    const auto strip_len = current_py_tab.size() + 1;
+    const auto trim_pos = google_args_begin > strip_len ? google_args_begin - strip_len : 0;
+    functionInfo.docstring.docstring = functionInfo.docstring.docstring.substr(
+      0,
+      trim_pos
+    );
+    functionInfo.docstring.end_line = functionInfo.docstring.start_line + google_args_begin;
+  }
+}
+
+std::string GoogleDocstring::cleanArgsDescription(const std::string &arg, const std::string &description) noexcept {
+  std::string res = std::string{description};
+
+  if (const auto startPos = description.find("Args:\n"); startPos != std::string::npos) {
+    res = res.substr(startPos, res.size() - startPos);
+  }
+
+  if (const auto startPos = res.find(arg); startPos != std::string::npos) {
+    const auto pos = startPos + arg.size() + 1;
+    if (pos > res.size())
+      return res;
+    res = res.substr(pos, res.size() - pos);
+  }
+
+  return res;
+}
+
+std::string GoogleDocstring::docstringSummary() noexcept {
+  std::string res = functionInfo.docstring.docstring;
+  if (res.empty())
+    return res;
+  if (!functionInfo.summary_only()) {
+    if (!res.ends_with('\n')) {
+      res += '\n';
+      res += '\n';
+    }
+  }
+  return res;
+}
+
+std::string GoogleDocstring::docstringArgs() noexcept {
+  if (functionInfo.args.empty())
+    return "";
+
+  auto current_py_tab = get_tabs();
+
+  std::string result;
+
+  if (PY_TAB != current_py_tab) {
+    result += current_py_tab;
+    current_py_tab += PY_TAB;
+  } else {
+    result += PY_TAB;
+    current_py_tab = PY_TAB + PY_TAB;
+  }
+  result += "Args:\n";
+  for (const auto &val: functionInfo.args) {
+    result += std::format("{}{}", current_py_tab, val.name);
+    if (!val.type.empty())
+      result += std::format(" ({}", val.type);
+    if (!val.default_value.empty()) {
+      result += ", optional";
+    }
+    if (!val.type.empty() || !val.default_value.empty())
+      result += ')';
+
+    result += ": ";
+
+    if (val.kind != ParameterKind::ARG) {
+      if (val.description.empty() || val.description.find(parameter_kind::to_str(val.kind)) == std::string::npos)
+        result += std::format("{}. ", parameter_kind::to_str(val.kind));
+    }
+
+    if (!val.description.empty()) {
+      result += std::format("{}", cleanArgsDescription(val.name, val.description));
+      if (!result.ends_with('.'))
+        result += '.';
+      result += ' ';
+    }
+
+    if (!val.default_value.empty()) {
+      if (auto tmp = std::format("Defaults to {}.", val.default_value);
+        val.description.empty() || val.description.find(tmp) == std::string::npos)
+        result += tmp;
+    }
+
+    if (result.ends_with(' '))
+      result.pop_back();
+    result += '\n';
+  }
+  return result;
+}
+
+std::string GoogleDocstring::docstringReturn() noexcept {
+  std::string result;
+
+  auto current_py_tab = get_tabs();
+
+  if (!functionInfo.returns.description.empty() || !functionInfo.returns.type.empty()) {
+    result += current_py_tab;
+    result += "Returns:\n";
+    current_py_tab += PY_TAB;
+    result += current_py_tab;
+
+    if (!functionInfo.returns.type.empty()) {
+      result += std::format("{}: ", functionInfo.returns.type);
+    }
+    if (!functionInfo.returns.description.empty()) {
+      result += functionInfo.returns.description;
+    }
+    result += '\n';
+  } else {
+    result = "";
+  }
+
+  return result;
+}
+
+std::string GoogleDocstring::docstringExceptions() noexcept {
+  if (functionInfo.exceptions.empty())
+    return "\n";
+
+  std::string result;
+  auto current_py_tab = get_tabs();
+
+  result += current_py_tab;
+
+  current_py_tab += PY_TAB;
+
+  result += "Raises:\n";
+
+  for (const auto &[name, description]: functionInfo.exceptions) {
+    result += current_py_tab;
+    if (!name.empty()) {
+      result += std::format("{}: ", name);
+    }
+    if (!description.empty()) {
+      result += description;
+    }
+
+    result += '\n';
+  }
+
+  return result;
+}

docstring_generator_ext-2.0.13b1/src/IDocstringFormat.cpp

@@ -0,0 +1,55 @@
+#include "IDocstringFormat.hpp"
+#include "parser.hpp"
+
+bool onlyQuotesAndWhitespace(const std::string &s) {
+  return std::ranges::all_of(
+    s,
+    [](char c) {
+      return c == '"' || std::isspace(static_cast<unsigned char>(c));
+    }
+  );
+}
+
+[[ nodiscard ]] std::string DocstringFormat::docstring() noexcept {
+  std::string result;
+  std::string tmp{};
+  result.reserve(1024); // rough upfront guess for typical arg count
+  const auto current_pytab = get_tabs();
+
+  if (const auto argTxt = docstringArgs(); !argTxt.empty()) {
+    tmp += argTxt;
+  }
+  if (const auto returnTxt = docstringReturn(); !returnTxt.empty()) {
+    tmp += returnTxt;
+  }
+  if (const auto exceptionTxt = docstringExceptions(); !exceptionTxt.empty() && exceptionTxt != "\n") {
+    tmp += exceptionTxt;
+  }
+
+  result += current_pytab;
+  result += R"(""")";
+  if (const auto summaryTxt = docstringSummary(); !summaryTxt.empty()) {
+    result += summaryTxt;
+    if (!result.empty() && !result.ends_with('\n') && !tmp.empty())
+      result += '\n';
+  } else {
+    result += '\n';
+  }
+  result += tmp;
+  if (result.ends_with('\n'))
+    result += current_pytab;
+  result += R"(""")";
+
+  if (onlyQuotesAndWhitespace(result))
+    return "";
+  return result;
+}
+
+[[ nodiscard ]] std::string DocstringFormat::get_tabs() const noexcept {
+  auto current_py_tab = PY_TAB;
+  for (uint32_t idx = 0; idx < (functionInfo.offset / 4); ++idx) {
+    current_py_tab += PY_TAB;
+  }
+
+  return current_py_tab;
+}

docstring_generator_ext-2.0.13b1/src/NumpyDocstring.cpp

@@ -0,0 +1,140 @@
+#include "NumpyDocstring.hpp"
+
+#include "parser.hpp"
+
+void NumpyDocstring::check_current_docstring() noexcept {
+  const auto current_py_tab = get_tabs();
+  const auto numpy_args_begin = functionInfo.docstring.docstring.find("Parameters");
+
+  if (numpy_args_begin != std::string::npos) {
+    const auto strip_len = current_py_tab.size() + 1;
+    const auto trim_pos = numpy_args_begin > strip_len ? numpy_args_begin - strip_len : 0;
+    functionInfo.docstring.docstring = functionInfo.docstring.docstring.substr(
+      0,
+      trim_pos
+    );
+    functionInfo.docstring.end_line = functionInfo.docstring.start_line + numpy_args_begin;
+  }
+}
+
+std::string NumpyDocstring::docstringSummary() noexcept {
+  std::string res = functionInfo.docstring.docstring;
+  if (res.empty())
+    return res;
+  if (!functionInfo.summary_only()) {
+    res += '\n';
+    res += '\n';
+  }
+  return res;
+}
+
+std::string NumpyDocstring::docstringArgs() noexcept {
+  std::string docstream;
+  const auto current_py_tab = get_tabs();
+
+  if (functionInfo.args.empty()) {
+    return docstream;
+  }
+
+  docstream.reserve(1024);
+
+  docstream += std::format("{}Parameters\n", current_py_tab);
+  docstream += std::format("{}----------\n", current_py_tab);
+  std::ranges::for_each(
+    functionInfo.args,
+    [&docstream, &current_py_tab](const FunctionParameter &val) {
+      if (val.skip)
+        return;
+
+      std::string result;
+      result.reserve(256);
+
+      if (val.name.empty())
+        result += std::format("{}PLACEHOLDER", current_py_tab);
+      else
+        result += std::format("{}{}", current_py_tab, val.name);
+
+      if (!val.type.empty()) {
+        result += std::format(" : {}", val.type);
+
+        if (!val.default_value.empty()) {
+          result += ", optional";
+          result += std::format(", default {}", val.default_value);
+        }
+      }
+
+      if (val.kind != ParameterKind::ARG) {
+        if (!val.description.empty() && val.description.find(parameter_kind::to_str(val.kind)) == std::string::npos) {
+          result += '\n';
+          result += std::format("{}{}{}", current_py_tab, PY_TAB, parameter_kind::to_str(val.kind));
+        }
+      }
+
+      if (!val.description.empty()) {
+        result += '\n';
+        result += std::format("{}{}{}", current_py_tab, PY_TAB, remove_whitespace(val.description));
+        if (!result.ends_with('.'))
+          result += '.';
+      }
+
+      result = remove_trailing_whitespace(result);
+
+      docstream += std::format("{}\n", result);
+    }
+  );
+  docstream += '\n';
+  return docstream;
+}
+
+std::string NumpyDocstring::docstringReturn() noexcept {
+  std::string result;
+  const auto current_py_tab = get_tabs();
+  if (functionInfo.returns.type.empty() and functionInfo.returns.description.empty())
+    return "";
+
+  result.reserve(256);
+
+  if (!functionInfo.returns.type.empty()) {
+    result += std::format("{}Returns\n", current_py_tab);
+    result += std::format("{}-------\n", current_py_tab);
+
+    result += std::format("{}{}", current_py_tab, functionInfo.returns.type);
+  }
+
+  if (!functionInfo.returns.description.empty()) {
+    result += '\n';
+    result += std::format("{}{}{}", current_py_tab, PY_TAB, functionInfo.returns.description);
+  }
+
+  result += '\n';
+  return result;
+}
+
+std::string NumpyDocstring::docstringExceptions() noexcept {
+  if (functionInfo.exceptions.empty())
+    return "";
+
+  std::string result;
+  result.reserve(256);
+
+  if (!functionInfo.returns.type.empty())
+    result += '\n';
+
+  const auto current_py_tab = get_tabs();
+
+  result += std::format("{}Raises\n", current_py_tab);
+  result += std::format("{}------\n", current_py_tab);
+
+  for (const auto &[name, description]: functionInfo.exceptions) {
+    if (!name.empty()) {
+      result += std::format("{}{}", current_py_tab, name);
+      result += '\n';
+    }
+    if (!description.empty()) {
+      result += std::format("{}{}{}", current_py_tab, PY_TAB, description);
+    }
+    result += '\n';
+  }
+
+  return result;
+}