wikinator 0.2.0__tar.gz → 0.4.0__tar.gz

{wikinator-0.2.0 → wikinator-0.4.0}/PKG-INFO

@@ -1,19 +1,18 @@
 Metadata-Version: 2.4
 Name: wikinator
-Version: 0.2.0
+Version: 0.4.0
 Summary: Convert Google docs to markdown
 Project-URL: Documentation, https://github.com/philion/wikinator#readme
 Project-URL: Issues, https://github.com/philion/wikinator/issues
 Project-URL: Source, https://github.com/philion/wikinator
 License-File: LICENSE
 Requires-Python: >=3.13
-Requires-Dist: docx2markdown>=0.1.1
 Requires-Dist: dotenv>=0.9.9
 Requires-Dist: google-api-python-client>=2.175.0
 Requires-Dist: google-auth-httplib2>=0.2.0
 Requires-Dist: google-auth-oauthlib>=1.2.2
 Requires-Dist: gql[all]>=3.5.3
-Requires-Dist: markitdown[all]>=0.1.2
+Requires-Dist: python-docx>=1.2.0
 Requires-Dist: typer-slim>=0.16.0
 Description-Content-Type: text/markdown
 
@@ -93,7 +92,46 @@ https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.wordprocessi
 Added detection for strikethru and Courier New (as "code font").
 
 This is good enough for v0.2!
+
+Oops. minor bug. fixing with v0.3
+
+Noticed when working on strikethru that nested lists didn't seem to be working.
+
+Next tasks:
+- [x] List handling
+- [x] Code cleanup (remove unused libs)
+- [ ] restructure docxit for in-memory
+- [ ] simple testing.
+- [x] recognize and handle single file
+- [x] default output to local dir
+
+Looking over the raw XML, it looks like...
+```xml
+<w:numPr>
+    <w:ilvl w:val="0" />
+    <w:numId w:val="2" />
+</w:numPr>
 ```
+It looks like:
+* `w:ilvl` is the zero-based indent level
+* `w:numId` is an ID from numbering.xml, roughly mapping to:
+    - val=1 ordered list: `1. `
+    - val=2 checklist: `- [ ]`
+    - val=3 bullet: `* `
+
+Restructured and cleaned up. Removed unneeded code and libraried.
+
+Created a simple docx doc for testing.
+
+Far enough that a new release feels right. v0.4!
+
+Let's combine testing lists with a simple test:
+- load a file with a list
+- convert
+- confirm it contains the correct list
+
+
+
 
 
 ### 2025-07-04

{wikinator-0.2.0 → wikinator-0.4.0}/README.md

@@ -74,7 +74,46 @@ https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.wordprocessi
 Added detection for strikethru and Courier New (as "code font").
 
 This is good enough for v0.2!
+
+Oops. minor bug. fixing with v0.3
+
+Noticed when working on strikethru that nested lists didn't seem to be working.
+
+Next tasks:
+- [x] List handling
+- [x] Code cleanup (remove unused libs)
+- [ ] restructure docxit for in-memory
+- [ ] simple testing.
+- [x] recognize and handle single file
+- [x] default output to local dir
+
+Looking over the raw XML, it looks like...
+```xml
+<w:numPr>
+    <w:ilvl w:val="0" />
+    <w:numId w:val="2" />
+</w:numPr>
 ```
+It looks like:
+* `w:ilvl` is the zero-based indent level
+* `w:numId` is an ID from numbering.xml, roughly mapping to:
+    - val=1 ordered list: `1. `
+    - val=2 checklist: `- [ ]`
+    - val=3 bullet: `* `
+
+Restructured and cleaned up. Removed unneeded code and libraried.
+
+Created a simple docx doc for testing.
+
+Far enough that a new release feels right. v0.4!
+
+Let's combine testing lists with a simple test:
+- load a file with a list
+- convert
+- confirm it contains the correct list
+
+
+
 
 
 ### 2025-07-04

{wikinator-0.2.0 → wikinator-0.4.0}/pyproject.toml

@@ -1,17 +1,16 @@
 [project]
 name = "wikinator"
-version = "0.2.0"
+version = "0.4.0"
 description = "Convert Google docs to markdown"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
-    "docx2markdown>=0.1.1",
     "dotenv>=0.9.9",
     "google-api-python-client>=2.175.0",
     "google-auth-httplib2>=0.2.0",
     "google-auth-oauthlib>=1.2.2",
     "gql[all]>=3.5.3",
-    "markitdown[all]>=0.1.2",
+    "python-docx>=1.2.0",
     "typer-slim>=0.16.0",
 ]
 
@@ -34,9 +33,6 @@ exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 [tool.hatch]
 
-[tool.hatch.version]
-path = "src/wikinator/__about__.py"
-
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

wikinator-0.4.0/src/wikinator/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+log = logging.getLogger(__name__)

wikinator-0.4.0/src/wikinator/app.py

@@ -0,0 +1,148 @@
+import os
+import json
+from pathlib import Path, PurePath
+
+import typer
+
+from . import docxit, log
+
+class Page:
+    """
+    Specific class for page, to help with validation, as graphql is strict about params
+    """
+    content: str
+    editor: str
+    isPublished: bool
+    isPrivate: bool
+    locale: str
+    path: str
+    tags: list[str]
+    title: str
+    description: str
+
+    def __init__(self, content: str, editor: str, isPublished: bool, isPrivate: bool,
+                locale: str, path: str, tags: list[str], title: str, description: str):
+        self.content = content
+        self.editor = editor
+        self.isPublished = isPublished
+        self.isPrivate = isPrivate
+        self.locale = locale
+        self.path = path
+        self.tags = tags
+        self.title = title
+        self.description = description
+
+    @classmethod
+    def load(cls, params: dict[str,any]):
+        return cls(
+            content = params["content"],
+            editor  = params["editor"],
+            isPublished = params["isPublished"],
+            isPrivate = params["isPrivate"],
+            locale = params["locale"],
+            path = params["path"],
+            tags = params["tags"],
+            title = params["title"],
+            description = params["description"],
+        )
+
+    @classmethod
+    def load_json(cls, json_str:str):
+        return cls.load(cls, json.loads(json_str))
+
+    def write(self, root:str) -> None:
+        """
+        Output the converted document to the specified directory `root`.
+        Use the stored path to output relative to the provided root.
+        """
+        filename = self.path + '.md'
+        target = Path(root, filename)
+        # assure required dirs exist
+        target.parent.mkdir(parents=True, exist_ok=True)
+        # write the content
+        with open(target, 'w') as output_file:
+            # TODO write yaml-based meta data?
+            output_file.write(self.content)
+
+
+class Converter:
+    root: Path # Root for file walk, and to resolve rol paths
+
+    def _convert(self, infile:Path) -> Page:
+        raise NotImplementedError
+
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        # translate with subclass
+        new_doc = self._convert(infile)
+
+        #print("converted", infile, new_doc.path)
+
+        # write!
+        new_doc.write(outroot)
+
+        return new_doc
+
+
+    def convert_file(self, full_path:Path, outroot:str):
+        ext = full_path.suffix.lower() # TODO strip first char, '.'
+
+        # TODO: generic mapping to queue based on ext.
+        if ext == ".docx":
+            #docx_queue.append(full_path) # TODO async!
+            self.convert(full_path, outroot)
+        else:
+            log.debug(f"No processor for {ext}, skipping {full_path}")
+
+
+    def convert_directory(self, inpath:str, outroot:str):
+        # load queues
+        #docx_queue: list[Path] = []
+
+        self.root = Path(inpath)
+
+        if os.path.isfile(inpath):
+            return self.convert_file(Path(inpath), outroot)
+
+        for root, dirs, files in os.walk(self.root):
+            for file in files:
+                full_path = Path(root, file)
+                self.convert_file(full_path, outroot)
+
+
+class DocxitConverter(Converter):
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        """
+        Converts a docx file into markdown using docxit
+        """
+        rel_path = PurePath(os.path.relpath(infile, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, infile.stem + ".md")
+        out_file = Path(outroot, rel_file)
+
+        # FIXME deconstruct docx_to_markdown to seperate convert from write
+        # assure required dirs exist
+        out_file.parent.mkdir(parents=True, exist_ok=True)
+        docxit.docx_to_markdown(infile, out_file)
+
+        return Page(
+            title =  infile.stem,
+            path = str(rel_file),
+            content = "", ## FIXME
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {infile}",
+            isPublished = False,
+            isPrivate = True,
+        )
+
+
+def do_convert(src: str, dest: str = ".") -> None:
+    DocxitConverter().convert_directory(src, dest)
+
+
+def main() -> None:
+    typer.run(do_convert)
+
+
+if __name__ == "__main__":
+    main()

{wikinator-0.2.0 → wikinator-0.4.0}/src/wikinator/docxit.py

@@ -35,6 +35,8 @@ import os
 from lxml import etree
 from pathlib import Path
 
+from . import log
+
 def docx_to_markdown(docx_file, output_md):
     """Convert a .docx file to a Markdown file and a subfolder of images."""
 
@@ -55,7 +57,6 @@ def docx_to_markdown(docx_file, output_md):
             images[rel.rId] = image_filename[len(folder)+1:]
             #print("image file", image_filename, "-->", images[rel.rId])
 
-    #print("images", images)
 
     for block in doc.element.body:
         if block.tag.endswith('p'):  # Handle paragraphs
@@ -63,13 +64,10 @@ def docx_to_markdown(docx_file, output_md):
             md_paragraph = ""
 
             ### switching on paragraph.style.name
-
             style_name = paragraph.style.name
+            #print("STYLE:", style_name)
 
-            if "List" in style_name:
-                prefix = get_bullet_point_prefix(paragraph)
-                md_paragraph = prefix  # Markdown syntax for bullet points
-            elif "Heading 1" in style_name:
+            if "Heading 1" in style_name:
                 md_paragraph = "# "
             elif "Heading 2" in style_name:
                 md_paragraph = "## "
@@ -79,14 +77,19 @@ def docx_to_markdown(docx_file, output_md):
                 md_paragraph = "#### "
             elif "Heading 5" in style_name:
                 md_paragraph = "##### "
-
             elif "Normal" or "normal" in style_name:
                 md_paragraph = ""
+                if is_list(paragraph):
+                    md_paragraph = get_bullet_point_prefix(paragraph)
             else:
                 print("Unsupported style:", style_name)
 
-            md_paragraph += parse_run(paragraph, images)
+            content = parse_run(paragraph, images)
 
+            #print("---> ", md_paragraph, content[:40])
+
+            #if len(content.strip()) > 0:
+            md_paragraph += content
             markdown.append(md_paragraph)
 
         elif block.tag.endswith('tbl'):  # Handle tables (if present)
@@ -107,7 +110,7 @@ def docx_to_markdown(docx_file, output_md):
 
     # Write to Markdown file
     with open(output_md, "w", encoding="utf-8") as md_file:
-        md_file.write("\n\n".join(markdown))
+        md_file.write("\n".join(markdown))
 
 
 def extract_r_embed(xml_string):
@@ -135,6 +138,7 @@ def extract_r_embed(xml_string):
         return blip.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
     return None
 
+
 def save_image(image_part, output_folder):
     """Save an image to the output folder and return the filename."""
     os.makedirs(output_folder, exist_ok=True)
@@ -155,10 +159,42 @@ def get_list_level(paragraph):
             return int(ilvl.get("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"))
     return 0
 
+
+def get_list_marker(paragraph):
+    p = paragraph._element
+    numPr = p.find(".//w:numPr", namespaces=p.nsmap)
+    ilvl = numPr.find(".//w:numId", namespaces=p.nsmap)
+    type_id = int(ilvl.get("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"))
+
+    match type_id:
+        case 1:
+            return '1. '
+        case 2:
+            return '- [ ] '
+        case 3:
+            return '* '
+        case _:
+            log.info(f"Unknown list type id: {type_id}")
+
+    # by default
+    return "* "
+
+
+def is_list(paragraph) -> bool:
+    p = paragraph._element
+    numPr = p.find(".//w:numPr", namespaces=p.nsmap)
+    return numPr is not None
+
+
 def get_bullet_point_prefix(paragraph):
-    """Determine the Markdown prefix for a bullet point based on its indentation level."""
+    """
+    Determine the Markdown prefix for a bullet point
+    based on its indentation level.
+    """
     level = get_list_level(paragraph)
-    return "  " * level + "- "  # Use Markdown syntax for nested lists
+    marker = get_list_marker(paragraph)
+    return "  " * level + marker
+
 
 def parse_run(run, images):
     """Go through document objects recursively and return markdown."""
@@ -176,7 +212,7 @@ def parse_run(run, images):
             image_url = images[rId]
             text += f"![]({image_url})"
         else:
-            print("unknown run type", s)
+            log.warning("unknown run type", s)
 
     if isinstance(run, docx.text.run.Run):
         if run.bold:
@@ -189,9 +225,7 @@ def parse_run(run, images):
             text = f"~~{text}~~"
         # check .font for monospacing
         # check style
-        if run.font.name == "Courier New":
-            print(text)
-            text = f"`{text}`"
-            #print(run.font.name)
+        if run.font.name == "Courier New": # more fonts!
+            text = f"`{text}`<br>"
 
     return text