wikinator 0.1.0__tar.gz → 0.2.0__tar.gz

{wikinator-0.1.0 → wikinator-0.2.0}/Makefile

@@ -29,8 +29,10 @@ dist:
 	uv publish
 
 clean:
-	rm -rf __pycache__
 	rm -fr .ruff_cache/
 	rm -fr dist/
+	rm -fr .venv/
 	rm -f dpytest_*.dat
-	find . -type f -name ‘*.pyc’ -delete
+	rm -fr .pytest_cache/
+	find . -type f -name ‘*.pyc’ -delete
+	find . -name __pycache__  | xargs rm -rf

{wikinator-0.1.0 → wikinator-0.2.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: wikinator
-Version: 0.1.0
+Version: 0.2.0
 Summary: Convert Google docs to markdown
 Project-URL: Documentation, https://github.com/philion/wikinator#readme
 Project-URL: Issues, https://github.com/philion/wikinator/issues
 Project-URL: Source, https://github.com/philion/wikinator
 License-File: LICENSE
 Requires-Python: >=3.13
+Requires-Dist: docx2markdown>=0.1.1
 Requires-Dist: dotenv>=0.9.9
 Requires-Dist: google-api-python-client>=2.175.0
 Requires-Dist: google-auth-httplib2>=0.2.0
@@ -52,12 +53,12 @@ uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql
 ## Build & Test
 1. Clone
     ```
-    git clone ...
+    git clone https://github.com/philion/wikinator.git
     cd wikinator
     ```
 2. Run, with uv
     ```
-    uv run -m wikinator [options]
+    uv run wikinator [options]
     ```
 3. Test, with pytest
     ```
@@ -66,13 +67,42 @@ uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql
 
 ## Development Log
 
+### 2025-07-05
+Starting work on image preservation.
+
+Looking first at https://github.com/haesleinhuepf/docx2markdown for images.
+
+Created a `Docx2MarkdownConverter` which almost works: images are put in the wrong path in the MD (`s/images/` instead of just `images`).
+There's probably an easy fix, but lets try a pandoc version.
+
+Creating `PandocConverter` to try and compare output.
+```
+pandoc {indoc} -f docx -t markdown --wrap=none --markdown-headings=atx --extract-media=images -o {outdoc}
+```
+
+Neither produces desired results.
+
+Trying a literal hack of docx2markdown, to see how quickly I can fix the little problems I saw.
+
+Got it working quickly, removed a little bug, got the images.
+
+Now looking over the DOCX XML format to see how much I can scrape out.
+
+https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.wordprocessing.fontsizecomplexscript?view=openxml-3.0.1
+
+Added detection for strikethru and Courier New (as "code font").
+
+This is good enough for v0.2!
+```
+
+
 ### 2025-07-04
 Let's make a project! Today's goals:
 - [x] clean up code and README
 - [x] add CLI options, using type (not all implemented)
 - [x] initial commit to github
 - [ ] add image handling
-- [ ] upload to pypi and confirm uvx commands
+- [x] upload to pypi and confirm uvx commands
 
 Cruft removed. README updated. (author waves, breaking 4th wall)
 
@@ -80,6 +110,10 @@ Moving on the main() cleanup and adding support for https://github.com/fastapi/t
 
 Added simple CLI options for src and dest. Got end-to-end tree processing.
 
+Added Makefile to help with release management. Got PyPI setup: https://pypi.org/project/wikinator/
+
+`uvx wikinator` is working.
+
 Let's go for git and call it a day!
 
 ### 2025-07-03

{wikinator-0.1.0 → wikinator-0.2.0}/README.md

@@ -34,12 +34,12 @@ uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql
 ## Build & Test
 1. Clone
     ```
-    git clone ...
+    git clone https://github.com/philion/wikinator.git
     cd wikinator
     ```
 2. Run, with uv
     ```
-    uv run -m wikinator [options]
+    uv run wikinator [options]
     ```
 3. Test, with pytest
     ```
@@ -48,13 +48,42 @@ uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql
 
 ## Development Log
 
+### 2025-07-05
+Starting work on image preservation.
+
+Looking first at https://github.com/haesleinhuepf/docx2markdown for images.
+
+Created a `Docx2MarkdownConverter` which almost works: images are put in the wrong path in the MD (`s/images/` instead of just `images`).
+There's probably an easy fix, but lets try a pandoc version.
+
+Creating `PandocConverter` to try and compare output.
+```
+pandoc {indoc} -f docx -t markdown --wrap=none --markdown-headings=atx --extract-media=images -o {outdoc}
+```
+
+Neither produces desired results.
+
+Trying a literal hack of docx2markdown, to see how quickly I can fix the little problems I saw.
+
+Got it working quickly, removed a little bug, got the images.
+
+Now looking over the DOCX XML format to see how much I can scrape out.
+
+https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.wordprocessing.fontsizecomplexscript?view=openxml-3.0.1
+
+Added detection for strikethru and Courier New (as "code font").
+
+This is good enough for v0.2!
+```
+
+
 ### 2025-07-04
 Let's make a project! Today's goals:
 - [x] clean up code and README
 - [x] add CLI options, using type (not all implemented)
 - [x] initial commit to github
 - [ ] add image handling
-- [ ] upload to pypi and confirm uvx commands
+- [x] upload to pypi and confirm uvx commands
 
 Cruft removed. README updated. (author waves, breaking 4th wall)
 
@@ -62,6 +91,10 @@ Moving on the main() cleanup and adding support for https://github.com/fastapi/t
 
 Added simple CLI options for src and dest. Got end-to-end tree processing.
 
+Added Makefile to help with release management. Got PyPI setup: https://pypi.org/project/wikinator/
+
+`uvx wikinator` is working.
+
 Let's go for git and call it a day!
 
 ### 2025-07-03

{wikinator-0.1.0 → wikinator-0.2.0}/pyproject.toml

@@ -1,10 +1,11 @@
 [project]
 name = "wikinator"
-version = "0.1.0"
+version = "0.2.0"
 description = "Convert Google docs to markdown"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "docx2markdown>=0.1.1",
     "dotenv>=0.9.9",
     "google-api-python-client>=2.175.0",
     "google-auth-httplib2>=0.2.0",

{wikinator-0.1.0 → wikinator-0.2.0}/src/wikinator/__about__.py

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: MIT
 """project version"""
-__version__ = "0.1.0"
+__version__ = "0.2.0"

{wikinator-0.1.0 → wikinator-0.2.0}/src/wikinator/app.py

@@ -2,11 +2,15 @@ import os
 from pathlib import Path, PurePath
 import logging
 import json
+import subprocess
 
 from markitdown import MarkItDown
+import docx2markdown
 
 import typer
 
+from wikinator import docxit
+
 class Page:
     """
     Specific class for page, to help with validation, as graphql is strict about params
@@ -101,6 +105,7 @@ class Converter:
                     logging.debug(f"No processor for {ext}, skipping {full_path}")
 
 
+# remove
 class MarkitdownConverter(Converter):
     def _convert(self, filepath:Path) -> Page:
         """
@@ -126,21 +131,106 @@ class MarkitdownConverter(Converter):
             isPrivate = True,
         )
 
-        # trying to make everything async
-        # skipping for now
-        # start workers
-        # with multiprocessing.Pool(3) as worker_pool:
-        #     # create a list the same size for starmap
-        #     args = [(doc, outroot) for doc in docx_queue]
-        #     # run the workers
-        #     results = worker_pool.starmap(convert_docx, args)
-        #     # check result success
-        #     for result in results:
-        #         print(result)
+# remove
+class Docx2MarkdownConverter(Converter):
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        """
+        Converts a docx file into markdown using docx2markdown
+        """
+        rel_path = PurePath(os.path.relpath(infile, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, infile.stem + ".md")
+        out_file = Path(outroot, rel_file)
+
+        #print(f"in: {infile}, out: {out_file}")
+
+        # FIXME deconstruct docx_to_markdown to seperate convert from write
+        # assure required dirs exist
+        out_file.parent.mkdir(parents=True, exist_ok=True)
+        docx2markdown.docx_to_markdown(infile, out_file)
+
+        return Page(
+            title =  infile.stem,
+            path = str(rel_file),
+            content = "", ## FIXME
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {infile}",
+            isPublished = False,
+            isPrivate = True,
+        )
+
+# remove?
+class PandocConverter(Converter):
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        """
+        Converts a docx file into markdown using docx2markdown
+        """
+        rel_path = PurePath(os.path.relpath(infile, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, infile.stem + ".md")
+        out_file = Path(outroot, rel_file)
+
+        media_path = Path(outroot, "images")
+
+        # assure required dirs exist
+        out_file.parent.mkdir(parents=True, exist_ok=True)
+
+        # pandoc {indoc} -f docx -t markdown --wrap=none --markdown-headings=atx --extract-media=images -o {outdoc}
+        command = ["pandoc", infile,
+                   "-f", "docx",
+                   "-t", "markdown",
+                   "--wrap=none",
+                   "--markdown-headings=atx",
+                   "--extract-media=" + str(media_path),
+                   "-o", out_file]
+        subprocess.run(command, check=True)
+
+        return Page(
+            title =  infile.stem,
+            path = str(rel_file),
+            content = "", ## FIXME
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {infile}",
+            isPublished = False,
+            isPrivate = True,
+        )
+
+
+class DocxitConverter(Converter):
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        """
+        Converts a docx file into markdown using docxit
+        """
+        rel_path = PurePath(os.path.relpath(infile, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, infile.stem + ".md")
+        out_file = Path(outroot, rel_file)
+
+        # FIXME deconstruct docx_to_markdown to seperate convert from write
+        # assure required dirs exist
+        out_file.parent.mkdir(parents=True, exist_ok=True)
+        docxit.docx_to_markdown(infile, out_file)
+
+        return Page(
+            title =  infile.stem,
+            path = str(rel_file),
+            content = "", ## FIXME
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {infile}",
+            isPublished = False,
+            isPrivate = True,
+        )
 
 
 def do_convert(src: str, dest: str) -> None:
-    MarkitdownConverter().convert_directory(src, dest)
+    #MarkitdownConverter().convert_directory(src, dest)
+    #Docx2MarkdownConverter().convert_directory(src, dest)
+    #PandocConverter().convert_directory(src, dest)
+    DocxitConverter().convert_directory(src, dest)
+
 
 
 def main() -> None:

wikinator-0.2.0/src/wikinator/docxit.py

@@ -0,0 +1,197 @@
+# This file is original taken from https://github.com/haesleinhuepf/docx2markdown
+# - src/docx2markdown/_docx_to_markdown.py
+# It is being altered here to provide results needed for this project.
+# original license: https://github.com/haesleinhuepf/docx2markdown/blob/main/LICENSE
+# included here for completeness
+#---
+# BSD 3-Clause License
+# Copyright (c) 2024, Robert Haase, ScaDS.AI, Uni Leipzig
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#---
+# All changes in the this version are Copyright (c) 2025, Paul Philion, Acme Rocket Company
+# under the provided MIT license.
+
+import docx
+import os
+from lxml import etree
+from pathlib import Path
+
+def docx_to_markdown(docx_file, output_md):
+    """Convert a .docx file to a Markdown file and a subfolder of images."""
+
+    folder = str(Path(output_md).parent)
+    image_folder = str(Path(output_md).parent / "images")
+
+    doc = docx.Document(docx_file)
+
+    paragraphs = list(doc.paragraphs)
+    tables = list(doc.tables)
+    markdown = []
+
+    # save all images
+    images = {}
+    for rel in doc.part.rels.values():
+        if "image" in rel.reltype:
+            image_filename = save_image(rel.target_part, image_folder)
+            images[rel.rId] = image_filename[len(folder)+1:]
+            #print("image file", image_filename, "-->", images[rel.rId])
+
+    #print("images", images)
+
+    for block in doc.element.body:
+        if block.tag.endswith('p'):  # Handle paragraphs
+            paragraph = paragraphs.pop(0)  # Match current paragraph
+            md_paragraph = ""
+
+            ### switching on paragraph.style.name
+
+            style_name = paragraph.style.name
+
+            if "List" in style_name:
+                prefix = get_bullet_point_prefix(paragraph)
+                md_paragraph = prefix  # Markdown syntax for bullet points
+            elif "Heading 1" in style_name:
+                md_paragraph = "# "
+            elif "Heading 2" in style_name:
+                md_paragraph = "## "
+            elif "Heading 3" in style_name:
+                md_paragraph = "### "
+            elif "Heading 4" in style_name:
+                md_paragraph = "#### "
+            elif "Heading 5" in style_name:
+                md_paragraph = "##### "
+
+            elif "Normal" or "normal" in style_name:
+                md_paragraph = ""
+            else:
+                print("Unsupported style:", style_name)
+
+            md_paragraph += parse_run(paragraph, images)
+
+            markdown.append(md_paragraph)
+
+        elif block.tag.endswith('tbl'):  # Handle tables (if present)
+            table = tables.pop(0)  # Match current table
+            table_text = ""
+            for i, row in enumerate(table.rows):
+                table_text += "| " + " | ".join(cell.text.strip() for cell in row.cells) + " |\n"
+                if i == 0:
+                    table_text += "| " + " | ".join("---" for _ in row.cells) + " |\n"
+
+            markdown.append(table_text)
+
+        elif block.tag.endswith('sectPr') or block.tag.endswith('sdt'):
+            # ignore
+            pass
+        else:
+            print("Unsupported block:", docx_file, block.tag)
+
+    # Write to Markdown file
+    with open(output_md, "w", encoding="utf-8") as md_file:
+        md_file.write("\n\n".join(markdown))
+
+
+def extract_r_embed(xml_string):
+    """
+    Extract the value of r:embed from the given XML string.
+
+    :param xml_string: The XML content as a string.
+    :return: The value of r:embed or None if not found.
+    """
+    # Parse the XML
+    root = etree.fromstring(xml_string)
+
+    # Define the namespaces
+    namespaces = {
+        'a': "http://schemas.openxmlformats.org/drawingml/2006/main",
+        'r': "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+        'pic': "http://schemas.openxmlformats.org/drawingml/2006/picture",
+    }
+
+    # Use XPath to find the <a:blip> element with r:embed
+    blip = root.find(".//a:blip", namespaces=namespaces)
+
+    # Extract the r:embed attribute value
+    if blip is not None:
+        return blip.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
+    return None
+
+def save_image(image_part, output_folder):
+    """Save an image to the output folder and return the filename."""
+    os.makedirs(output_folder, exist_ok=True)
+    image_filename = os.path.join(output_folder, os.path.basename(image_part.partname))
+    with open(image_filename, "wb") as img_file:
+        img_file.write(image_part.blob)
+    return str(image_filename).replace("\\", "/")
+
+
+def get_list_level(paragraph):
+    """Determine the level of a bullet point or numbered list item."""
+    # Access the raw XML of the paragraph
+    p = paragraph._element
+    numPr = p.find(".//w:numPr", namespaces=p.nsmap)
+    if numPr is not None:
+        ilvl = numPr.find(".//w:ilvl", namespaces=p.nsmap)
+        if ilvl is not None:
+            return int(ilvl.get("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"))
+    return 0
+
+def get_bullet_point_prefix(paragraph):
+    """Determine the Markdown prefix for a bullet point based on its indentation level."""
+    level = get_list_level(paragraph)
+    return "  " * level + "- "  # Use Markdown syntax for nested lists
+
+def parse_run(run, images):
+    """Go through document objects recursively and return markdown."""
+    sub_parts = list(run.iter_inner_content())
+    text = ""
+    for s in sub_parts:
+        if isinstance(s, str):
+            text += s
+        elif isinstance(s, docx.text.run.Run):
+            text += parse_run(s, images)
+        elif isinstance(s, docx.text.hyperlink.Hyperlink):
+            text += f"[{s.text}]({s.address})"
+        elif isinstance(s, docx.drawing.Drawing):
+            rId = extract_r_embed(s._element.xml)
+            image_url = images[rId]
+            text += f"![]({image_url})"
+        else:
+            print("unknown run type", s)
+
+    if isinstance(run, docx.text.run.Run):
+        if run.bold:
+            text = f"**{text}**"
+        if run.italic:
+            text = f"*{text}*"
+        if run.underline:
+            text = f"__{text}__"
+        if run.font.strike:
+            text = f"~~{text}~~"
+        # check .font for monospacing
+        # check style
+        if run.font.name == "Courier New":
+            print(text)
+            text = f"`{text}`"
+            #print(run.font.name)
+
+    return text

{wikinator-0.1.0 → wikinator-0.2.0}/uv.lock

@@ -347,6 +347,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" },
 ]
 
+[[package]]
+name = "docx2markdown"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+    { name = "python-docx" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/be/692425e45455b1bbff7ad0b9fa2fab31ac3f9da13f210ded06c0c99158f5/docx2markdown-0.1.1.tar.gz", hash = "sha256:177e7ca0c998139b9afb9ffbe9a3be05a27d60664ac602b38de643e7b343593d", size = 8256, upload-time = "2024-11-23T21:11:45.015Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d2/f5/970f7c2f64331891ff8b8c1863a793779e4fc784dbd210e8ae3712bcfd0e/docx2markdown-0.1.1-py3-none-any.whl", hash = "sha256:a97d413e6a225d4a0932cc8660b92f10f5de28f0436bc18ee12ef3961cd4b2b1", size = 8393, upload-time = "2024-11-23T21:11:43.452Z" },
+]
+
 [[package]]
 name = "dotenv"
 version = "0.9.9"
@@ -1141,6 +1154,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
 ]
 
+[[package]]
+name = "python-docx"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" },
+]
+
 [[package]]
 name = "python-dotenv"
 version = "1.1.1"
@@ -1364,6 +1390,7 @@ name = "wikinator"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "docx2markdown" },
     { name = "dotenv" },
     { name = "google-api-python-client" },
     { name = "google-auth-httplib2" },
@@ -1375,6 +1402,7 @@ dependencies = [
 
 [package.metadata]
 requires-dist = [
+    { name = "docx2markdown", specifier = ">=0.1.1" },
     { name = "dotenv", specifier = ">=0.9.9" },
     { name = "google-api-python-client", specifier = ">=2.175.0" },
     { name = "google-auth-httplib2", specifier = ">=0.2.0" },