wikinator 0.1.0__tar.gz

wikinator-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,28 @@
+name: Release
+
+on:
+  push:
+    tags:
+      # Publish on any tag starting with a `v`, e.g. v1.2.3
+      - v*
+
+jobs:
+  pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    # Environment and permissions trusted publishing.
+    environment:
+      # Create this environment in the GitHub repository under Settings -> Environments
+      name: release
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv build
+      # Check that basic features work and we didn't miss to include crucial files
+      - name: Smoke test (wheel)
+        run: uv run --isolated --no-project -p 3.13 --with dist/*.whl tests/smoke_test.py
+      - name: Smoke test (source distribution)
+        run: uv run --isolated --no-project -p 3.13 --with dist/*.tar.gz tests/smoke_test.py
+      - run: uv publish --trusted-publishing always

wikinator-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.python-version
+
+# Virtual environments
+.venv
+
+# env and secure files
+.env
+.envrc
+token.json
+credentials.json
+
+# Apple dingles
+.DS_Store

wikinator-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Paul Philion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wikinator-0.1.0/Makefile

@@ -0,0 +1,36 @@
+.PHONY: all run clean
+
+# Simple makefile to help me remember uv tasks
+# Targets are:
+# - lint     : run ruff linter
+# - fix      : ... with fixes
+# - test     : run test suite
+# - build    : build
+# - publish  : publish
+# - dist     : clean, build, publish
+# - clean    : remove anything built
+
+
+lint:
+	uvx ruff check
+
+fix:
+	uvx ruff check --fix
+
+test:
+	uv run pytest
+
+build:
+	uv build
+
+dist:
+	rm -fr dist/
+	uv build
+	uv publish
+
+clean:
+	rm -rf __pycache__
+	rm -fr .ruff_cache/
+	rm -fr dist/
+	rm -f dpytest_*.dat
+	find . -type f -name ‘*.pyc’ -delete

wikinator-0.1.0/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.4
+Name: wikinator
+Version: 0.1.0
+Summary: Convert Google docs to markdown
+Project-URL: Documentation, https://github.com/philion/wikinator#readme
+Project-URL: Issues, https://github.com/philion/wikinator/issues
+Project-URL: Source, https://github.com/philion/wikinator
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: google-api-python-client>=2.175.0
+Requires-Dist: google-auth-httplib2>=0.2.0
+Requires-Dist: google-auth-oauthlib>=1.2.2
+Requires-Dist: gql[all]>=3.5.3
+Requires-Dist: markitdown[all]>=0.1.2
+Requires-Dist: typer-slim>=0.16.0
+Description-Content-Type: text/markdown
+
+# wikinator
+
+Convert a Google drive download into a markdown-based wiki.
+
+**Note**: This is a work in progress, and not all features will be supported or working properly.
+
+## tl;dr
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+Given a directory, convert supported file types into markdown-based files while maintaining names and directory structure. This can then be uploaded into various wiki systems.
+
+### Supported File Types
+- DOCX files (default for GDocs) are converted to markdown
+- images are extracted, uploaded and embedded in the markdown
+- `text` and code file types are wrapped in markdown code blocks
+- CSV and XSLT are converted to markdown tables
+- for any document that is converted to markdown, a copy of the original is uploaded and attached
+
+### Supported Wiki Import
+- wiki.js (and other GraphQL-based wikis)
+- Obsidian
+
+The development log will be kept here until the 1.0 release.
+
+## Usage
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+## Build & Test
+1. Clone
+    ```
+    git clone ...
+    cd wikinator
+    ```
+2. Run, with uv
+    ```
+    uv run -m wikinator [options]
+    ```
+3. Test, with pytest
+    ```
+    uv run pytest
+    ```
+
+## Development Log
+
+### 2025-07-04
+Let's make a project! Today's goals:
+- [x] clean up code and README
+- [x] add CLI options, using type (not all implemented)
+- [x] initial commit to github
+- [ ] add image handling
+- [ ] upload to pypi and confirm uvx commands
+
+Cruft removed. README updated. (author waves, breaking 4th wall)
+
+Moving on the main() cleanup and adding support for https://github.com/fastapi/typer
+
+Added simple CLI options for src and dest. Got end-to-end tree processing.
+
+Let's go for git and call it a day!
+
+### 2025-07-03
+Next steps are testing different document converters and accessing google drive via API.
+
+#### Markdown conversion libraries
+- pandoc, see https://docs.asciidoctor.org/asciidoctor/latest/migrate/ms-word/
+- markitdown, https://github.com/microsoft/markitdown
+- docx2markdown, https://github.com/haesleinhuepf/docx2markdown
+- docx2md, https://github.com/mattn/docx2md
+
+Reference:
+- https://www.docstomarkdown.pro/convert-word-or-docs-to-markdown-using-pandoc/
+
+#### Google Drive API
+Starting with https://developers.google.com/workspace/drive/api/quickstart/python
+
+> Note: Follow those Google directions for setting up everything. It's complicated compared to simply generating a service token. Your intrepid author made different tokens in different accounts and couldn't access anything! And get permissions right! Document specific needs in intstall docs.
+
+Further aside: There are two versions of the tool: file-based and google-takeout. The google related stuff will always be a bear to setup.
+
+Made suffienct progress to feel like there a seperate CLI tool here. Set aside for now, and focus on:
+1. Build file-based output
+2. Generate and link images
+3. Clean up for initial 0.1 version
+
+### 2025-07-02
+Initial time-boxed work started to examine what would be required to migrate our existing GoogleDocs-based info repo into a wiki, with wiki.js being targeted.
+
+Initial proof-of-concept goals:
+- Convert a docx page to md or asciidoc
+- Upload test pages to wiki.js
+
+I was able to get this working in sample code in a few hours.

wikinator-0.1.0/README.md

@@ -0,0 +1,98 @@
+# wikinator
+
+Convert a Google drive download into a markdown-based wiki.
+
+**Note**: This is a work in progress, and not all features will be supported or working properly.
+
+## tl;dr
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+Given a directory, convert supported file types into markdown-based files while maintaining names and directory structure. This can then be uploaded into various wiki systems.
+
+### Supported File Types
+- DOCX files (default for GDocs) are converted to markdown
+- images are extracted, uploaded and embedded in the markdown
+- `text` and code file types are wrapped in markdown code blocks
+- CSV and XSLT are converted to markdown tables
+- for any document that is converted to markdown, a copy of the original is uploaded and attached
+
+### Supported Wiki Import
+- wiki.js (and other GraphQL-based wikis)
+- Obsidian
+
+The development log will be kept here until the 1.0 release.
+
+## Usage
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+## Build & Test
+1. Clone
+    ```
+    git clone ...
+    cd wikinator
+    ```
+2. Run, with uv
+    ```
+    uv run -m wikinator [options]
+    ```
+3. Test, with pytest
+    ```
+    uv run pytest
+    ```
+
+## Development Log
+
+### 2025-07-04
+Let's make a project! Today's goals:
+- [x] clean up code and README
+- [x] add CLI options, using type (not all implemented)
+- [x] initial commit to github
+- [ ] add image handling
+- [ ] upload to pypi and confirm uvx commands
+
+Cruft removed. README updated. (author waves, breaking 4th wall)
+
+Moving on the main() cleanup and adding support for https://github.com/fastapi/typer
+
+Added simple CLI options for src and dest. Got end-to-end tree processing.
+
+Let's go for git and call it a day!
+
+### 2025-07-03
+Next steps are testing different document converters and accessing google drive via API.
+
+#### Markdown conversion libraries
+- pandoc, see https://docs.asciidoctor.org/asciidoctor/latest/migrate/ms-word/
+- markitdown, https://github.com/microsoft/markitdown
+- docx2markdown, https://github.com/haesleinhuepf/docx2markdown
+- docx2md, https://github.com/mattn/docx2md
+
+Reference:
+- https://www.docstomarkdown.pro/convert-word-or-docs-to-markdown-using-pandoc/
+
+#### Google Drive API
+Starting with https://developers.google.com/workspace/drive/api/quickstart/python
+
+> Note: Follow those Google directions for setting up everything. It's complicated compared to simply generating a service token. Your intrepid author made different tokens in different accounts and couldn't access anything! And get permissions right! Document specific needs in intstall docs.
+
+Further aside: There are two versions of the tool: file-based and google-takeout. The google related stuff will always be a bear to setup.
+
+Made suffienct progress to feel like there a seperate CLI tool here. Set aside for now, and focus on:
+1. Build file-based output
+2. Generate and link images
+3. Clean up for initial 0.1 version
+
+### 2025-07-02
+Initial time-boxed work started to examine what would be required to migrate our existing GoogleDocs-based info repo into a wiki, with wiki.js being targeted.
+
+Initial proof-of-concept goals:
+- Convert a docx page to md or asciidoc
+- Upload test pages to wiki.js
+
+I was able to get this working in sample code in a few hours.

wikinator-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[project]
+name = "wikinator"
+version = "0.1.0"
+description = "Convert Google docs to markdown"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "dotenv>=0.9.9",
+    "google-api-python-client>=2.175.0",
+    "google-auth-httplib2>=0.2.0",
+    "google-auth-oauthlib>=1.2.2",
+    "gql[all]>=3.5.3",
+    "markitdown[all]>=0.1.2",
+    "typer-slim>=0.16.0",
+]
+
+[project.urls]
+Documentation = "https://github.com/philion/wikinator#readme"
+Issues = "https://github.com/philion/wikinator/issues"
+Source = "https://github.com/philion/wikinator"
+
+[tool.coverage.run]
+source_pkgs = ["wikinator", "tests"]
+branch = true
+parallel = true
+
+[tool.coverage.paths]
+clw = ["src/wikinator", "*/wikinator/src/wikinator"]
+tests = ["tests", "*/wikinator/tests"]
+
+[tool.coverage.report]
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
+
+[tool.hatch]
+
+[tool.hatch.version]
+path = "src/wikinator/__about__.py"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/wikinator"]
+
+[project.scripts]
+wikinator = "wikinator.app:main"

wikinator-0.1.0/src/wikinator/__about__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025-present Paul Philion <philion@acmerocket.com>
+#
+# SPDX-License-Identifier: MIT
+"""project version"""
+__version__ = "0.1.0"

wikinator-0.1.0/src/wikinator/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from wikinator.app import main
+
+if __name__ == "__main__":
+    main()

wikinator-0.1.0/src/wikinator/app.py

@@ -0,0 +1,151 @@
+import os
+from pathlib import Path, PurePath
+import logging
+import json
+
+from markitdown import MarkItDown
+
+import typer
+
+class Page:
+    """
+    Specific class for page, to help with validation, as graphql is strict about params
+    """
+    content: str
+    editor: str
+    isPublished: bool
+    isPrivate: bool
+    locale: str
+    path: str
+    tags: list[str]
+    title: str
+    description: str
+
+    def __init__(self, content: str, editor: str, isPublished: bool, isPrivate: bool,
+                locale: str, path: str, tags: list[str], title: str, description: str):
+        self.content = content
+        self.editor = editor
+        self.isPublished = isPublished
+        self.isPrivate = isPrivate
+        self.locale = locale
+        self.path = path
+        self.tags = tags
+        self.title = title
+        self.description = description
+
+    @classmethod
+    def load(cls, params: dict[str,any]):
+        return cls(
+            content = params["content"],
+            editor  = params["editor"],
+            isPublished = params["isPublished"],
+            isPrivate = params["isPrivate"],
+            locale = params["locale"],
+            path = params["path"],
+            tags = params["tags"],
+            title = params["title"],
+            description = params["description"],
+        )
+
+    @classmethod
+    def load_json(cls, json_str:str):
+        return cls.load(cls, json.loads(json_str))
+
+    def write(self, root:str) -> None:
+        """
+        Output the converted document to the specified directory `root`.
+        Use the stored path to output relative to the provided root.
+        """
+        filename = self.path + '.md'
+        target = Path(root, filename)
+        # assure required dirs exist
+        target.parent.mkdir(parents=True, exist_ok=True)
+        # write the content
+        with open(target, 'w') as output_file:
+            # TODO write yaml-based meta data?
+            output_file.write(self.content)
+
+
+class Converter:
+    root: Path # Root for file walk, and to resolve rol paths
+
+    def _convert(self, infile:Path) -> Page:
+        raise NotImplementedError
+
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        # translate with subclass
+        new_doc = self._convert(infile)
+
+        #print("converted", infile, new_doc.path)
+
+        # write!
+        new_doc.write(outroot)
+
+        return new_doc
+
+    def convert_directory(self, inpath:str, outroot:str):
+        # load queues
+        #docx_queue: list[Path] = []
+        self.root = Path(inpath)
+
+        for root, dirs, files in os.walk(self.root):
+            for file in files:
+                full_path = Path(root, file)
+                ext = full_path.suffix.lower() # TODO strip first char, '.'
+
+                # TODO: generic mapping to queue based on ext.
+                if ext == ".docx":
+                    #docx_queue.append(full_path) # TODO async!
+                    self.convert(full_path, outroot)
+                else:
+                    logging.debug(f"No processor for {ext}, skipping {full_path}")
+
+
+class MarkitdownConverter(Converter):
+    def _convert(self, filepath:Path) -> Page:
+        """
+        Converts a markdown file into a struct to add to the wiki
+        """
+        md = MarkItDown(enable_plugins=False)
+        result = md.convert(filepath)
+
+        # determine the relative file path
+        # this *includes* the top-level directly name
+        rel_path = PurePath(os.path.relpath(filepath, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, filepath.stem)
+
+        return Page(
+            title = result.title if result.title else filepath.stem,
+            path = str(rel_file),
+            content = result.text_content,
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {filepath}",
+            isPublished = False,
+            isPrivate = True,
+        )
+
+        # trying to make everything async
+        # skipping for now
+        # start workers
+        # with multiprocessing.Pool(3) as worker_pool:
+        #     # create a list the same size for starmap
+        #     args = [(doc, outroot) for doc in docx_queue]
+        #     # run the workers
+        #     results = worker_pool.starmap(convert_docx, args)
+        #     # check result success
+        #     for result in results:
+        #         print(result)
+
+
+def do_convert(src: str, dest: str) -> None:
+    MarkitdownConverter().convert_directory(src, dest)
+
+
+def main() -> None:
+    typer.run(do_convert)
+
+
+if __name__ == "__main__":
+    main()

wikinator-0.1.0/src/wikinator/quickstart.py

@@ -0,0 +1,66 @@
+import os.path
+
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# If modifying these scopes, delete the file token.json.
+SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"]
+
+
+def main():
+    """Shows basic usage of the Drive v3 API.
+    Prints the names and ids of the first 10 files the user has access to.
+    """
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    if os.path.exists("token.json"):
+        creds = Credentials.from_authorized_user_file("token.json", SCOPES)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(
+                "credentials.json", SCOPES
+            )
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open("token.json", "w") as token:
+            token.write(creds.to_json())
+
+    try:
+        service = build("drive", "v3", credentials=creds)
+
+        # Call the Drive v3 API
+        files = service.files()
+        request = files.list(pageSize=1, fields="nextPageToken, files(*)")
+
+        #while request is not None:
+        results = request.execute()
+
+        items = results.get("files", [])
+        if not items:
+            print("No files found.")
+            return
+        for item in items:
+            #print(json.dumps(item, indent=4))
+            print(item['name'], item['mimeType'])
+
+            # download
+            md_doc = files.export(fileId=item['id'], mimeType="text/markdown").execute()
+            print(md_doc)
+
+        #    request = files.list_next(request, results)
+
+    except HttpError as error:
+        # TODO(developer) - Handle errors from drive API.
+        print(f"An error occurred: {error}")
+
+
+if __name__ == "__main__":
+    main()