wikinator 0.1.0__py3-none-any.whl

wikinator/__about__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025-present Paul Philion <philion@acmerocket.com>
+#
+# SPDX-License-Identifier: MIT
+"""project version"""
+__version__ = "0.1.0"

wikinator/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from wikinator.app import main
+
+if __name__ == "__main__":
+    main()

wikinator/app.py

@@ -0,0 +1,151 @@
+import os
+from pathlib import Path, PurePath
+import logging
+import json
+
+from markitdown import MarkItDown
+
+import typer
+
+class Page:
+    """
+    Specific class for page, to help with validation, as graphql is strict about params
+    """
+    content: str
+    editor: str
+    isPublished: bool
+    isPrivate: bool
+    locale: str
+    path: str
+    tags: list[str]
+    title: str
+    description: str
+
+    def __init__(self, content: str, editor: str, isPublished: bool, isPrivate: bool,
+                locale: str, path: str, tags: list[str], title: str, description: str):
+        self.content = content
+        self.editor = editor
+        self.isPublished = isPublished
+        self.isPrivate = isPrivate
+        self.locale = locale
+        self.path = path
+        self.tags = tags
+        self.title = title
+        self.description = description
+
+    @classmethod
+    def load(cls, params: dict[str,any]):
+        return cls(
+            content = params["content"],
+            editor  = params["editor"],
+            isPublished = params["isPublished"],
+            isPrivate = params["isPrivate"],
+            locale = params["locale"],
+            path = params["path"],
+            tags = params["tags"],
+            title = params["title"],
+            description = params["description"],
+        )
+
+    @classmethod
+    def load_json(cls, json_str:str):
+        return cls.load(cls, json.loads(json_str))
+
+    def write(self, root:str) -> None:
+        """
+        Output the converted document to the specified directory `root`.
+        Use the stored path to output relative to the provided root.
+        """
+        filename = self.path + '.md'
+        target = Path(root, filename)
+        # assure required dirs exist
+        target.parent.mkdir(parents=True, exist_ok=True)
+        # write the content
+        with open(target, 'w') as output_file:
+            # TODO write yaml-based meta data?
+            output_file.write(self.content)
+
+
+class Converter:
+    root: Path # Root for file walk, and to resolve rol paths
+
+    def _convert(self, infile:Path) -> Page:
+        raise NotImplementedError
+
+    def convert(self, infile:Path, outroot:Path) -> Page:
+        # translate with subclass
+        new_doc = self._convert(infile)
+
+        #print("converted", infile, new_doc.path)
+
+        # write!
+        new_doc.write(outroot)
+
+        return new_doc
+
+    def convert_directory(self, inpath:str, outroot:str):
+        # load queues
+        #docx_queue: list[Path] = []
+        self.root = Path(inpath)
+
+        for root, dirs, files in os.walk(self.root):
+            for file in files:
+                full_path = Path(root, file)
+                ext = full_path.suffix.lower() # TODO strip first char, '.'
+
+                # TODO: generic mapping to queue based on ext.
+                if ext == ".docx":
+                    #docx_queue.append(full_path) # TODO async!
+                    self.convert(full_path, outroot)
+                else:
+                    logging.debug(f"No processor for {ext}, skipping {full_path}")
+
+
+class MarkitdownConverter(Converter):
+    def _convert(self, filepath:Path) -> Page:
+        """
+        Converts a markdown file into a struct to add to the wiki
+        """
+        md = MarkItDown(enable_plugins=False)
+        result = md.convert(filepath)
+
+        # determine the relative file path
+        # this *includes* the top-level directly name
+        rel_path = PurePath(os.path.relpath(filepath, self.root.parent)) # remove .parent to remove top dirname
+        rel_file = PurePath(rel_path.parent, filepath.stem)
+
+        return Page(
+            title = result.title if result.title else filepath.stem,
+            path = str(rel_file),
+            content = result.text_content,
+            editor = "markdown",
+            locale = "en",
+            tags = None,
+            description = f"generated from: {filepath}",
+            isPublished = False,
+            isPrivate = True,
+        )
+
+        # trying to make everything async
+        # skipping for now
+        # start workers
+        # with multiprocessing.Pool(3) as worker_pool:
+        #     # create a list the same size for starmap
+        #     args = [(doc, outroot) for doc in docx_queue]
+        #     # run the workers
+        #     results = worker_pool.starmap(convert_docx, args)
+        #     # check result success
+        #     for result in results:
+        #         print(result)
+
+
+def do_convert(src: str, dest: str) -> None:
+    MarkitdownConverter().convert_directory(src, dest)
+
+
+def main() -> None:
+    typer.run(do_convert)
+
+
+if __name__ == "__main__":
+    main()

wikinator/quickstart.py

@@ -0,0 +1,66 @@
+import os.path
+
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# If modifying these scopes, delete the file token.json.
+SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"]
+
+
+def main():
+    """Shows basic usage of the Drive v3 API.
+    Prints the names and ids of the first 10 files the user has access to.
+    """
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    if os.path.exists("token.json"):
+        creds = Credentials.from_authorized_user_file("token.json", SCOPES)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(
+                "credentials.json", SCOPES
+            )
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open("token.json", "w") as token:
+            token.write(creds.to_json())
+
+    try:
+        service = build("drive", "v3", credentials=creds)
+
+        # Call the Drive v3 API
+        files = service.files()
+        request = files.list(pageSize=1, fields="nextPageToken, files(*)")
+
+        #while request is not None:
+        results = request.execute()
+
+        items = results.get("files", [])
+        if not items:
+            print("No files found.")
+            return
+        for item in items:
+            #print(json.dumps(item, indent=4))
+            print(item['name'], item['mimeType'])
+
+            # download
+            md_doc = files.export(fileId=item['id'], mimeType="text/markdown").execute()
+            print(md_doc)
+
+        #    request = files.list_next(request, results)
+
+    except HttpError as error:
+        # TODO(developer) - Handle errors from drive API.
+        print(f"An error occurred: {error}")
+
+
+if __name__ == "__main__":
+    main()

wikinator/wiki.py

@@ -0,0 +1,281 @@
+import os
+
+from dotenv import load_dotenv
+
+
+from gql import Client, gql
+from gql.transport.aiohttp import AIOHTTPTransport
+
+
+
+def init_client() -> Client:
+    """
+    Initialize the GraphQL client with the credentials found in the system ENV:
+    - GRAPH_DB : The full URL for requests to the graph DB
+    - AUTH_TOKEN : Security token to authorize session
+    """
+    load_dotenv()
+    db_url = os.getenv("GRAPH_DB")
+    token = os.getenv("AUTH_TOKEN")
+    transport = AIOHTTPTransport(url=db_url, headers={'Authorization': f'Bearer {token}'}, ssl=True)
+    return Client(transport=transport)
+
+
+def store_page(client: Client, params: dict):
+    # store in file?
+    query = gql(
+        '''
+        mutation Page (
+                $content: String!,
+                $description: String!,
+                $editor:String!,
+                $isPublished:Boolean!,
+                $isPrivate:Boolean!,
+                $locale:String!,
+                $path:String!,
+                $tags:[String]!,
+                $title:String!) {
+            pages {
+                create (
+                    content:$content,
+                    description:$description,
+                    editor: $editor,
+                    isPublished: $isPublished,
+                    isPrivate: $isPrivate,
+                    locale: $locale,
+                    path:$path,
+                    tags: $tags,
+                    title:$title
+                ) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        slug
+                        message
+                    }
+                    page {
+                        id
+                        path
+                        title
+                    }
+                }
+            }
+        }
+        '''
+    )
+    print('---', params['path'])
+    result = client.execute(query, variable_values=params)
+    print(result)
+    return result
+
+# EXTS = {
+#     ".docx":
+#  32
+# .pdf 87
+# .xlsx 15
+# .rsc 4
+# .drawio 4
+# .deb 8
+# .bin 1
+# .mp4 28
+# .png 49
+# .PDF 7
+# .pptx 1
+# .apk 10
+# .jpg 5
+# .HEIC 21
+# .tgz 1
+# .php 2
+# .yaml 2
+# .gz 1
+# .toml 1
+# .service 1
+# .jpeg 2
+# .txt 1
+# .csv 1
+# }
+
+### Design Notes
+# multiprocessing.JoinableQueue for queues, using both get() and task_done()
+
+#running = False
+#docx_queue = multiprocessing.JoinableQueue()
+
+# def docx_worker(client: Client):
+#     while running:
+#         try:
+#             item = docx_queue.get(timeout=0.01)
+#             if item is None:
+#                 continue
+
+#             try:
+#                 process_docx(client, item)
+#             finally:
+#                 docx_queue.task_done()
+
+#         except queue.Empty:
+#             pass
+#         except:
+#             logging.exception('error while processing item')
+
+CLIENT: Client = init_client()
+
+
+
+#def main():
+    #CLIENT = init_client()
+    #process_directory("./data")
+
+    # client = init_client()
+
+    # with open("sample.adoc", 'r') as file:
+    #     file_content = file.read()
+
+    # params = {
+    #     "title": "This is a test",
+    #     "path": "test/test-43",
+    #     "content": file_content,
+    #     "editor": "asciidoc",
+    #     "locale": "en",
+    #     "tags": "",
+    #     "description": "generated from googledoc/...",
+    #     "isPublished": False,
+    #     "isPrivate": True,
+    # }
+
+    # page = create_page(client, params)
+    # print(json.dumps(page, indent=4))
+
+
+
+
+
+
+
+### other queries. remove when stable
+    # query = gql(
+    #     """
+    #     {
+    #         pages {
+    #             list (orderBy: TITLE) {
+    #                 id
+    #                 path
+    #                 title
+    #                 contentType
+    #                 createdAt
+    #                 description
+    #                 isPrivate
+    #                 isPublished
+    #                 updatedAt
+    #             }
+    #         }
+    #     }
+    #     """
+    # )
+
+    # query = gql(
+    #     """
+    #     {
+    #         users {
+    #             list {
+    #                 id
+    #                name
+    #                email
+    #             }
+    #         }
+    #     }
+    #     """
+    # )
+
+    # insert into "pages" (
+        # "authorId",
+        # "content",
+        # "contentType",
+        # "createdAt",
+        # "creatorId",
+        # "description",
+        # "editorKey",
+        # "extra",
+        # "hash",
+        # "isPrivate",
+        # "isPublished",
+        # "localeCode",
+        # "path",
+        # "publishEndDate",
+        # "publishStartDate",
+        # "title",
+        # "toc",
+        # "updatedAt")
+
+    # query = gql(
+    #     """
+    #     mutation Page ($content: String!, $description: String!, $editor:String!, $isPublished:Boolean!, $isPrivate:Boolean!, $locale:String!, $path:String!,$tags:[String]!, $title:String!) {
+    #         pages {
+    #             create (content:$content, description:$description, editor: $editor, isPublished: $isPublished, isPrivate: $isPrivate, locale: $locale, path:$path, tags: $tags, title:$title) {
+    #                 responseResult {
+    #                     succeeded,
+    #                     errorCode,
+    #                     slug,
+    #                     message
+    #                 },
+    #                 page {
+    #                     id,
+    #                     path,
+    #                     title
+    #                 }
+    #             }
+    #         }
+    #     }
+    #     """
+    # )
+
+    # query = gql(
+    #     '''
+    #     mutation Page (
+    #             $content: String!,
+    #             $description: String!,
+    #             $editor:String!,
+    #             $isPublished:Boolean!,
+    #             $isPrivate:Boolean!,
+    #             $locale:String!,
+    #             $path:String!,
+    #             $tags:[String]!,
+    #             $title:String!) {
+    #         pages {
+    #             create (
+    #                 content:$content,
+    #                 description:$description,
+    #                 editor: $editor,
+    #                 isPublished: $isPublished,
+    #                 isPrivate: $isPrivate,
+    #                 locale: $locale,
+    #                 path:$path,
+    #                 tags: $tags,
+    #                 title:$title
+    #             ) {
+    #                 responseResult {
+    #                     succeeded
+    #                     errorCode
+    #                     slug
+    #                     message
+    #                 }
+    #                 page {
+    #                     id
+    #                     path
+    #                     title
+    #                 }
+    #             }
+    #         }
+    #     }
+    #     '''
+    # )
+
+    # query = gql(
+    #     """
+    #     query getContinentName ($code: ID!) {
+    #     continent (code: $code) {
+    #         name
+    #       }
+    #     }
+    # """
+    # )

wikinator-0.1.0.dist-info/METADATA

@@ -0,0 +1,116 @@
+Metadata-Version: 2.4
+Name: wikinator
+Version: 0.1.0
+Summary: Convert Google docs to markdown
+Project-URL: Documentation, https://github.com/philion/wikinator#readme
+Project-URL: Issues, https://github.com/philion/wikinator/issues
+Project-URL: Source, https://github.com/philion/wikinator
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: google-api-python-client>=2.175.0
+Requires-Dist: google-auth-httplib2>=0.2.0
+Requires-Dist: google-auth-oauthlib>=1.2.2
+Requires-Dist: gql[all]>=3.5.3
+Requires-Dist: markitdown[all]>=0.1.2
+Requires-Dist: typer-slim>=0.16.0
+Description-Content-Type: text/markdown
+
+# wikinator
+
+Convert a Google drive download into a markdown-based wiki.
+
+**Note**: This is a work in progress, and not all features will be supported or working properly.
+
+## tl;dr
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+Given a directory, convert supported file types into markdown-based files while maintaining names and directory structure. This can then be uploaded into various wiki systems.
+
+### Supported File Types
+- DOCX files (default for GDocs) are converted to markdown
+- images are extracted, uploaded and embedded in the markdown
+- `text` and code file types are wrapped in markdown code blocks
+- CSV and XSLT are converted to markdown tables
+- for any document that is converted to markdown, a copy of the original is uploaded and attached
+
+### Supported Wiki Import
+- wiki.js (and other GraphQL-based wikis)
+- Obsidian
+
+The development log will be kept here until the 1.0 release.
+
+## Usage
+```
+uvx wikinator some/dir another_dir
+uvx wikinator some/dir -graphql https://wiki.example.com/graphql -token 'graphql-auth-token'
+```
+
+## Build & Test
+1. Clone
+    ```
+    git clone ...
+    cd wikinator
+    ```
+2. Run, with uv
+    ```
+    uv run -m wikinator [options]
+    ```
+3. Test, with pytest
+    ```
+    uv run pytest
+    ```
+
+## Development Log
+
+### 2025-07-04
+Let's make a project! Today's goals:
+- [x] clean up code and README
+- [x] add CLI options, using type (not all implemented)
+- [x] initial commit to github
+- [ ] add image handling
+- [ ] upload to pypi and confirm uvx commands
+
+Cruft removed. README updated. (author waves, breaking 4th wall)
+
+Moving on the main() cleanup and adding support for https://github.com/fastapi/typer
+
+Added simple CLI options for src and dest. Got end-to-end tree processing.
+
+Let's go for git and call it a day!
+
+### 2025-07-03
+Next steps are testing different document converters and accessing google drive via API.
+
+#### Markdown conversion libraries
+- pandoc, see https://docs.asciidoctor.org/asciidoctor/latest/migrate/ms-word/
+- markitdown, https://github.com/microsoft/markitdown
+- docx2markdown, https://github.com/haesleinhuepf/docx2markdown
+- docx2md, https://github.com/mattn/docx2md
+
+Reference:
+- https://www.docstomarkdown.pro/convert-word-or-docs-to-markdown-using-pandoc/
+
+#### Google Drive API
+Starting with https://developers.google.com/workspace/drive/api/quickstart/python
+
+> Note: Follow those Google directions for setting up everything. It's complicated compared to simply generating a service token. Your intrepid author made different tokens in different accounts and couldn't access anything! And get permissions right! Document specific needs in intstall docs.
+
+Further aside: There are two versions of the tool: file-based and google-takeout. The google related stuff will always be a bear to setup.
+
+Made suffienct progress to feel like there a seperate CLI tool here. Set aside for now, and focus on:
+1. Build file-based output
+2. Generate and link images
+3. Clean up for initial 0.1 version
+
+### 2025-07-02
+Initial time-boxed work started to examine what would be required to migrate our existing GoogleDocs-based info repo into a wiki, with wiki.js being targeted.
+
+Initial proof-of-concept goals:
+- Convert a docx page to md or asciidoc
+- Upload test pages to wiki.js
+
+I was able to get this working in sample code in a few hours.

wikinator-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+wikinator/__about__.py,sha256=IdVXYY7Z3ydcf0n-hgfyqAau2-a22qBsSNMM5FZtHes,154
+wikinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wikinator/__main__.py,sha256=TRhJB-hjf9CD9mFn4TlXG2qPRpVNZ_wArjpvm6jiZdQ,93
+wikinator/app.py,sha256=JzHYmgLLWoTJCBsw-u71qLzJZTS7nTZo8FjcfwvSnCU,4628
+wikinator/quickstart.py,sha256=FHx_b1mPA8H2-5Pw0FbnOP6UCmARAs0DsxHJ6y3BmFc,2255
+wikinator/wiki.py,sha256=bdM9nGtylfywlWkGD1dAqTdVnPEQPwR20Y62wM6yPVM,6905
+wikinator-0.1.0.dist-info/METADATA,sha256=xrVcgY4sH8-8gs2LbUBqy9TyxsQcMm_RaB9be40_0Ng,4059
+wikinator-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+wikinator-0.1.0.dist-info/entry_points.txt,sha256=6NGOQQ5sXSESmFtd4j0Dy_Yv6fcNwE-NUxPp10xG4cI,49
+wikinator-0.1.0.dist-info/licenses/LICENSE,sha256=Mmlntf1TLxyV5T4YFPW_x1oKKyb9-OC2pT4dJHC4ebQ,1069
+wikinator-0.1.0.dist-info/RECORD,,

wikinator-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

wikinator-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wikinator = wikinator.app:main

wikinator-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Paul Philion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.