mokosh 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mokosh-0.1.0/LICENSE +21 -0
- mokosh-0.1.0/PKG-INFO +239 -0
- mokosh-0.1.0/README.md +217 -0
- mokosh-0.1.0/mokosh/__init__.py +11 -0
- mokosh-0.1.0/mokosh/__main__.py +3 -0
- mokosh-0.1.0/mokosh/cli.py +440 -0
- mokosh-0.1.0/mokosh/db_manager.py +404 -0
- mokosh-0.1.0/mokosh/exporter.py +256 -0
- mokosh-0.1.0/mokosh/git_connector.py +151 -0
- mokosh-0.1.0/mokosh/schema.py +45 -0
- mokosh-0.1.0/mokosh/task_extractor.py +141 -0
- mokosh-0.1.0/mokosh/task_fetcher.py +121 -0
- mokosh-0.1.0/mokosh/text.py +103 -0
- mokosh-0.1.0/mokosh/tokens.py +91 -0
- mokosh-0.1.0/mokosh/trackers/__init__.py +20 -0
- mokosh-0.1.0/mokosh/trackers/github.py +83 -0
- mokosh-0.1.0/mokosh/trackers/gitlab.py +54 -0
- mokosh-0.1.0/mokosh/trackers/jira_api.py +34 -0
- mokosh-0.1.0/mokosh/trackers/jira_html.py +30 -0
- mokosh-0.1.0/mokosh/trackers/jira_selenium.py +41 -0
- mokosh-0.1.0/mokosh/trackers/youtrack.py +49 -0
- mokosh-0.1.0/mokosh/ui/__init__.py +0 -0
- mokosh-0.1.0/mokosh/ui/server.py +315 -0
- mokosh-0.1.0/mokosh/ui/webui/app.js +203 -0
- mokosh-0.1.0/mokosh/ui/webui/i18n.js +39 -0
- mokosh-0.1.0/mokosh/ui/webui/index.html +120 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/de.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/en.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/es.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/fr.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/pt.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/lang/uk.js +71 -0
- mokosh-0.1.0/mokosh/ui/webui/style.css +251 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/beach.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/dark.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/light.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/mono-dark.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/mono-light.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/ocean.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/soft.css +19 -0
- mokosh-0.1.0/mokosh/ui/webui/themes/summer.css +19 -0
- mokosh-0.1.0/mokosh.egg-info/PKG-INFO +239 -0
- mokosh-0.1.0/mokosh.egg-info/SOURCES.txt +55 -0
- mokosh-0.1.0/mokosh.egg-info/dependency_links.txt +1 -0
- mokosh-0.1.0/mokosh.egg-info/entry_points.txt +2 -0
- mokosh-0.1.0/mokosh.egg-info/requires.txt +13 -0
- mokosh-0.1.0/mokosh.egg-info/top_level.txt +1 -0
- mokosh-0.1.0/pyproject.toml +37 -0
- mokosh-0.1.0/setup.cfg +4 -0
- mokosh-0.1.0/tests/test_change_type.py +98 -0
- mokosh-0.1.0/tests/test_fetch_empty.py +62 -0
- mokosh-0.1.0/tests/test_github_url.py +25 -0
- mokosh-0.1.0/tests/test_ingest.py +156 -0
- mokosh-0.1.0/tests/test_masks.py +47 -0
- mokosh-0.1.0/tests/test_text.py +61 -0
- mokosh-0.1.0/tests/test_tokens.py +51 -0
- mokosh-0.1.0/tests/test_ui.py +78 -0
mokosh-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stanislav Zholobetskyi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
mokosh-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mokosh
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Gathers the dataset a code-search study starts from: the commits of a git repository and the tasks of its tracker (Jira, GitHub, GitLab, YouTrack) in one SQLite database. No GPU, no LLM.
|
|
5
|
+
Project-URL: Homepage, https://github.com/szholobetsky/mokosh
|
|
6
|
+
Project-URL: Repository, https://github.com/szholobetsky/mokosh
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: gitpython>=3.1
|
|
11
|
+
Requires-Dist: requests>=2.28
|
|
12
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
13
|
+
Requires-Dist: pyyaml>=6.0
|
|
14
|
+
Requires-Dist: tqdm>=4.60
|
|
15
|
+
Requires-Dist: flask>=2.3
|
|
16
|
+
Requires-Dist: waitress>=2.1
|
|
17
|
+
Provides-Extra: selenium
|
|
18
|
+
Requires-Dist: selenium>=4.0; extra == "selenium"
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
# mokosh
|
|
26
|
+
|
|
27
|
+
**The dataset a code-search study starts from.** mokosh reads the commits of a git repository,
|
|
28
|
+
finds the task each commit belongs to (`PROJ-123 fix login`), fetches those tasks from the
|
|
29
|
+
tracker — Jira, GitHub, GitLab or YouTrack — and writes both into one SQLite database:
|
|
30
|
+
|
|
31
|
+
- `TASK` — `NAME`, `TITLE`, `DESCRIPTION`, `COMMENTS`, in plain text (Jira ADF trees and wiki
|
|
32
|
+
markup are converted);
|
|
33
|
+
- `COMMIT` — one row per (commit, file): `SHA`, author, `CMT_DATE`, `MESSAGE`, `PATH`, `DIFF`,
|
|
34
|
+
`TASK_NAME`, `CHANGE_TYPE` — what the commit did to the file: `A` added, `M` modified,
|
|
35
|
+
`D` deleted, `R` renamed (`PATH` is the new name). A database made before `CHANGE_TYPE` gets the
|
|
36
|
+
column on the next `mokosh ingest`, filled from `git log --name-status` of the repository.
|
|
37
|
+
`DIFF` is the patch parent → commit: a line the commit added starts with `+`.
|
|
38
|
+
|
|
39
|
+
mokosh before 2026-10-10 wrote `DIFF` the other way round (commit → parent: added lines with `-`).
|
|
40
|
+
`mokosh ingest` turns such a database the right way before it adds anything; for a database that is
|
|
41
|
+
not ingested again, `mokosh fixdiff <db> --repo <repo>` does the same. Either runs once — the
|
|
42
|
+
database is marked (`MOKOSH_META`) — and leaves the root commits alone (their patch against the
|
|
43
|
+
empty tree was already right).
|
|
44
|
+
|
|
45
|
+
Everything else in the toolkit reads that database: **simargl** embeds the tasks, **gamayun**
|
|
46
|
+
builds an ontology from them, **1bcoder** exports them for a glossary. mokosh is the only one
|
|
47
|
+
that writes it. No GPU, no LLM, no embeddings: git, HTTP and SQLite.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
pip install mokosh # gitpython, requests, beautifulsoup4, pyyaml, tqdm, flask, waitress
|
|
53
|
+
pip install mokosh[selenium] # only for Jira instances that need a real browser
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
simargl and gamayun depend on it, so installing either of them installs mokosh too.
|
|
57
|
+
|
|
58
|
+
## Use
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
cd C:\Projects\Data\sonarqube
|
|
62
|
+
mokosh init # repository, branch, history since, tracker -> .mokosh/project.yaml
|
|
63
|
+
mokosh ingest # [1/2] commits [2/2] task keys from messages + task details
|
|
64
|
+
mokosh status # tasks, commits, how many are linked to a task, date range
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`ingest` is incremental. Run it again a month later and it adds only the commits whose SHA is not
|
|
68
|
+
in the database yet (new rows get IDs after the existing ones; old rows are never touched), links
|
|
69
|
+
only the commits that have no task yet, and asks the tracker only about the tasks that have no
|
|
70
|
+
details yet — so each task is fetched once, and a failed one is tried again on the next run. The
|
|
71
|
+
same makes an interrupted fetch of ten thousand tasks continue where it stopped. An empty answer of the
|
|
72
|
+
tracker (404, no access, a rate limit, a wrong owner/repo) is not stored as a task with no text: the
|
|
73
|
+
task stays unfetched and is asked again next time, and when the first 20 tasks all come back empty
|
|
74
|
+
the fetch stops and says which settings to check. What each run
|
|
75
|
+
added is kept in `.mokosh/ingest_state.yaml`. `--phase git|tasks` runs one phase; `--force`
|
|
76
|
+
links every commit to a task key again (after a change of `commit_mask`). History rewritten
|
|
77
|
+
upstream (rebase, force push) leaves the old commits in the database and adds the new ones.
|
|
78
|
+
Tokens never go into the file. `mokosh init` (and the Setup page of `mokosh ui`) put the token you enter into an
|
|
79
|
+
environment variable — `GITHUB_TOKEN`, `JIRA_TOKEN`, `GITLAB_TOKEN`, `YOUTRACK_TOKEN`, or
|
|
80
|
+
`MOKOSH_<PROJECT>_GITHUB_TOKEN` when the plain name already holds another token — and project.yaml keeps
|
|
81
|
+
only `github_token: ${GITHUB_TOKEN}`. On Windows the variable is saved for your user (`setx`; mokosh also
|
|
82
|
+
reads it from there in a terminal opened before); elsewhere init prints the `export` line for your shell
|
|
83
|
+
profile.
|
|
84
|
+
|
|
85
|
+
A monthly refresh is two commands; gamayun rebuilds from scratch in minutes:
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
mokosh ingest
|
|
89
|
+
gamayun index --db units.db --repo . --project sonar
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Then build on it:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
simargl index units units.db # semantic search over the tasks
|
|
96
|
+
gamayun index --db units.db --repo . --project sonar # statistical ontology
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Other commands:
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
mokosh plaintext units.db [--dry-run] # clean tracker markup in a database from an older ingest
|
|
103
|
+
mokosh export units.db --out DIR # the rows as text files, for /flow glossary (see below)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
A folder set up earlier with `simargl init` (`.simargl/project.yaml`) is used as it is — no new
|
|
107
|
+
`init`, no ingest from scratch. (simargl no longer gathers anything: it only reads the database.)
|
|
108
|
+
|
|
109
|
+
### Web UI
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
mokosh ui # http://127.0.0.1:8093/ (--port, --host 0.0.0.0, --root FOLDER)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Flask + waitress, like the other tools of the toolkit (eight themes, six languages):
|
|
116
|
+
|
|
117
|
+
- **Setup** — the form `mokosh init` asks for: folder, database, repository and branch, tracker,
|
|
118
|
+
the commit mask with every preset tried on the repository's last 300 commits. A token goes to an
|
|
119
|
+
environment variable, as with `mokosh init`.
|
|
120
|
+
- **Ingest** — `mokosh ingest` (both phases, or git / tasks only; `--force`) with its log.
|
|
121
|
+
- **Status** — what the database holds, `project.yaml` (tokens masked), `ingest_state.yaml`.
|
|
122
|
+
|
|
123
|
+
Projects are the folders with `.mokosh/project.yaml` (or an earlier `.simargl/project.yaml`) under
|
|
124
|
+
the folder it was started in, and those remembered in `~/.mokosh/projects.json` (`mokosh init`
|
|
125
|
+
and the UI remember them).
|
|
126
|
+
|
|
127
|
+
### Export for a glossary
|
|
128
|
+
|
|
129
|
+
`/flow glossary` (1bcoder, vyrii) reads folders of text files, never SQLite; `mokosh export` writes
|
|
130
|
+
them:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
mokosh export units.db --out DIR # --all (default): DIR/tasks/ and DIR/commits/ apart
|
|
134
|
+
mokosh export units.db --out DIR --join # one file per task with its commits (DIR/joined/)
|
|
135
|
+
mokosh export units.db --out DIR --task # tasks only (--commits: commits only)
|
|
136
|
+
mokosh export units.db --out DIR --level title # TITLE | desc: + DESCRIPTION (default) | comments: + COMMENTS
|
|
137
|
+
mokosh export units.db --out DIR --diff-mode full # the whole stored diff (default changed: only @/+/- lines)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
`--join` writes each commit as its own `--- commit <sha> ---` block (message + per-file diff); a
|
|
141
|
+
task with no commit still gets its file, and commits with no task are counted, not guessed at.
|
|
142
|
+
`--all` keeps tasks and commit diffs as separate corpora, to be indexed as separate glossary
|
|
143
|
+
projects. `desc` is the default level: title + description beat title alone and the noisier
|
|
144
|
+
+comments in the retrieval experiments; `changed` drops the untouched context lines of a diff.
|
|
145
|
+
|
|
146
|
+
### Trackers
|
|
147
|
+
|
|
148
|
+
| `source` | Settings in `tasks:` |
|
|
149
|
+
|---|---|
|
|
150
|
+
| `jira` | `jira_url`, `jira_project`, `jira_connector: api \| html \| selenium`, `jira_token` |
|
|
151
|
+
| `github` | `github_owner`, `github_repo`, `github_token` — `mokosh init` asks for the repository URL `https://github.com/<USER>/<REPO>` and writes both names |
|
|
152
|
+
| `gitlab` | `gitlab_url`, `gitlab_project`, `gitlab_token` |
|
|
153
|
+
| `youtrack` | `youtrack_url`, `youtrack_project`, `youtrack_token` |
|
|
154
|
+
|
|
155
|
+
How a task key is found in a commit message is `commit_mask` in `tasks:`, for any tracker:
|
|
156
|
+
|
|
157
|
+
| `commit_mask` | Finds |
|
|
158
|
+
|---|---|
|
|
159
|
+
| `simple` (default) | `KAFKA-123` at the start of the message |
|
|
160
|
+
| `bracketed` | `[KAFKA-123]` anywhere |
|
|
161
|
+
| `django` | `Fixed #123` |
|
|
162
|
+
| `generic` | `fixes #123`, `closed: #123`, `resolves #123` |
|
|
163
|
+
| `broad` | any `#123` |
|
|
164
|
+
| `regex-key` | the project's own key anywhere, as a whole word: `Fixed: invoice total (OFBIZ-12345)`. Built from `jira_project` (or `youtrack_project`); a team that uses the keys of several projects writes them all: `jira_project: ABC,ABD,ABK` |
|
|
165
|
+
| any regex | your own pattern; the first group is the key, if there is one |
|
|
166
|
+
|
|
167
|
+
Own regexes, for example:
|
|
168
|
+
|
|
169
|
+
| regex | finds |
|
|
170
|
+
|---|---|
|
|
171
|
+
| `(OFBIZ-\d+)` | one project's key anywhere |
|
|
172
|
+
| `((?:ABC\|ABD\|ABK)-\d+)` | the keys of several projects anywhere |
|
|
173
|
+
| `\b((?:ABC\|ABD\|ABK)-\d+)\b` | the same, not inside a longer word (`XABC-1`) — what `regex-key` builds |
|
|
174
|
+
|
|
175
|
+
A pattern like `([A-Z]+-\d+)` (any key) also catches `UTF-8`, `SHA-256`, `ISO-8859`; name the keys.
|
|
176
|
+
|
|
177
|
+
`mokosh init` asks for it with any tracker. It prints this table, then tries every mask on the repository's
|
|
178
|
+
last 300 commits and offers the one that links the most:
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
On the last 300 commits of C:/Projects/Data/flink:
|
|
182
|
+
simple links 0 (0%) -
|
|
183
|
+
bracketed links 282 (94%) FLINK-40293, FLINK-38262, FLINK-40830
|
|
184
|
+
...
|
|
185
|
+
Commit mask [Enter = bracketed]:
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`mokosh ui` (**Setup**) shows the same table and the same counts.
|
|
189
|
+
|
|
190
|
+
Without a tracker mokosh stores the commits only (`TASK` stays empty); simargl can still index
|
|
191
|
+
them as commit units.
|
|
192
|
+
|
|
193
|
+
### Table names
|
|
194
|
+
|
|
195
|
+
New databases have singular `TASK` and `COMMIT`. Older simargl ingests have `TASKS` / `COMMITS`,
|
|
196
|
+
and the codeXtract datasets (`sonar.db`, `flink.db`) mix them; `mokosh.schema` resolves whatever a
|
|
197
|
+
file has, and simargl and gamayun read through it.
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Part of the SIMARGL toolkit
|
|
202
|
+
|
|
203
|
+
mokosh is one of ten tools that together form an **intellectual development support system**:
|
|
204
|
+
|
|
205
|
+
| Tool | Role |
|
|
206
|
+
|---|---|
|
|
207
|
+
| **[simargl](https://github.com/szholobetsky/simargl)** | Task-to-code retrieval — given a task description, finds which files and modules are likely affected, using semantic similarity over git history |
|
|
208
|
+
| **[svitovyd](https://github.com/szholobetsky/svitovyd)** | Project map — scans any codebase and produces a structural map of definitions and cross-file dependencies; queried from the CLI, a web UI with a layered map, or an MCP server |
|
|
209
|
+
| **[1bcoder](https://github.com/szholobetsky/1bcoder)** | AI coding assistant for small local models — surgical context management, agents, parallel inference, proc scripts |
|
|
210
|
+
| **[yasna](https://github.com/szholobetsky/yasna)** | Session memory — indexes conversations from all AI agents so you can find what was discussed, when, and where |
|
|
211
|
+
| **[radogast](https://github.com/szholobetsky/radogast)** | Context drift monitor — measures how far an AI agent's conversation has drifted from the original task |
|
|
212
|
+
| **[vyrii](https://github.com/szholobetsky/vyrii)** | Local AI web UI — chat, translate, web research, RAG and file management in the browser (Flask; Gradio optional); powered by Ollama or any OpenAI-compatible backend |
|
|
213
|
+
| **[syryn](https://github.com/szholobetsky/syryn)** | Bluetooth identity beacon for headless devices — the device's IP is its Bluetooth name; hostname, mDNS and network interfaces on request |
|
|
214
|
+
| **[alkonost](https://github.com/szholobetsky/alkonost)** | Browser cockpit for 1bcoder's flows — the `deepagent_*` spec pipeline (plan tree → specs → task board) and the glossary (build, browse, search), tuned for small local models |
|
|
215
|
+
| **[gamayun](https://github.com/szholobetsky/gamayun)** | Statistical ontology without an LLM — builds triples of terms, identifiers, files and modules from task history or a folder of documents and code; every triple carries its significance test |
|
|
216
|
+
| **[mokosh](https://github.com/szholobetsky/mokosh)** | Dataset gathering — commits of a git repository and the tasks of its tracker (Jira, GitHub, GitLab, YouTrack) in one SQLite database that the other tools read |
|
|
217
|
+
|
|
218
|
+
- **simargl** finds the code related to a task by vector search over the task history (embeddings).
|
|
219
|
+
- **svitovyd** answers: *how is the code structured and what depends on what?*
|
|
220
|
+
- **1bcoder** answers: *how do I work with local models efficiently?*
|
|
221
|
+
- **yasna** answers: *where did I already discuss this?*
|
|
222
|
+
- **radogast** answers: *is the AI agent still on track toward the goal?*
|
|
223
|
+
- **vyrii** answers: *how do I access all of this through a browser?*
|
|
224
|
+
- **syryn** answers: *what is the address of this headless device?*
|
|
225
|
+
- **alkonost** answers: *how do I turn one task description into a reviewed, working plan — and see the project's glossary — without a big model or a big context window?*
|
|
226
|
+
- **gamayun** finds the code related to a task through an ontology of significant links between words, files and modules (no LLM).
|
|
227
|
+
- **mokosh** prepares the initial dataset from version control and the issue tracker.
|
|
228
|
+
|
|
229
|
+
Together they cover the full development loop: gather the history, understand the codebase, find relevant history, work with AI locally, remember what was decided, and verify the context stays on target.
|
|
230
|
+
|
|
231
|
+
The name comes from Slavic mythology. Mokosh (Мокош) is the goddess of the harvest and of spinning: she spins the threads of fate and watches over the yield of the land. A fitting name for the tool that spins tasks and commits into one yarn — no good ingest, no harvest.
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## About
|
|
236
|
+
|
|
237
|
+
(c) 2026 Stanislav Zholobetskyi, Oleh Andriichuk
|
|
238
|
+
Institute for Information Recording, National Academy of Sciences of Ukraine, Kyiv
|
|
239
|
+
PhD research: «Intelligent Technology for Software Development and Maintenance Support»
|
mokosh-0.1.0/README.md
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
# mokosh
|
|
4
|
+
|
|
5
|
+
**The dataset a code-search study starts from.** mokosh reads the commits of a git repository,
|
|
6
|
+
finds the task each commit belongs to (`PROJ-123 fix login`), fetches those tasks from the
|
|
7
|
+
tracker — Jira, GitHub, GitLab or YouTrack — and writes both into one SQLite database:
|
|
8
|
+
|
|
9
|
+
- `TASK` — `NAME`, `TITLE`, `DESCRIPTION`, `COMMENTS`, in plain text (Jira ADF trees and wiki
|
|
10
|
+
markup are converted);
|
|
11
|
+
- `COMMIT` — one row per (commit, file): `SHA`, author, `CMT_DATE`, `MESSAGE`, `PATH`, `DIFF`,
|
|
12
|
+
`TASK_NAME`, `CHANGE_TYPE` — what the commit did to the file: `A` added, `M` modified,
|
|
13
|
+
`D` deleted, `R` renamed (`PATH` is the new name). A database made before `CHANGE_TYPE` gets the
|
|
14
|
+
column on the next `mokosh ingest`, filled from `git log --name-status` of the repository.
|
|
15
|
+
`DIFF` is the patch parent → commit: a line the commit added starts with `+`.
|
|
16
|
+
|
|
17
|
+
mokosh before 2026-10-10 wrote `DIFF` the other way round (commit → parent: added lines with `-`).
|
|
18
|
+
`mokosh ingest` turns such a database the right way before it adds anything; for a database that is
|
|
19
|
+
not ingested again, `mokosh fixdiff <db> --repo <repo>` does the same. Either runs once — the
|
|
20
|
+
database is marked (`MOKOSH_META`) — and leaves the root commits alone (their patch against the
|
|
21
|
+
empty tree was already right).
|
|
22
|
+
|
|
23
|
+
Everything else in the toolkit reads that database: **simargl** embeds the tasks, **gamayun**
|
|
24
|
+
builds an ontology from them, **1bcoder** exports them for a glossary. mokosh is the only one
|
|
25
|
+
that writes it. No GPU, no LLM, no embeddings: git, HTTP and SQLite.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
pip install mokosh # gitpython, requests, beautifulsoup4, pyyaml, tqdm, flask, waitress
|
|
31
|
+
pip install mokosh[selenium] # only for Jira instances that need a real browser
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
simargl and gamayun depend on it, so installing either of them installs mokosh too.
|
|
35
|
+
|
|
36
|
+
## Use
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
cd C:\Projects\Data\sonarqube
|
|
40
|
+
mokosh init # repository, branch, history since, tracker -> .mokosh/project.yaml
|
|
41
|
+
mokosh ingest # [1/2] commits [2/2] task keys from messages + task details
|
|
42
|
+
mokosh status # tasks, commits, how many are linked to a task, date range
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`ingest` is incremental. Run it again a month later and it adds only the commits whose SHA is not
|
|
46
|
+
in the database yet (new rows get IDs after the existing ones; old rows are never touched), links
|
|
47
|
+
only the commits that have no task yet, and asks the tracker only about the tasks that have no
|
|
48
|
+
details yet — so each task is fetched once, and a failed one is tried again on the next run. The
|
|
49
|
+
same makes an interrupted fetch of ten thousand tasks continue where it stopped. An empty answer of the
|
|
50
|
+
tracker (404, no access, a rate limit, a wrong owner/repo) is not stored as a task with no text: the
|
|
51
|
+
task stays unfetched and is asked again next time, and when the first 20 tasks all come back empty
|
|
52
|
+
the fetch stops and says which settings to check. What each run
|
|
53
|
+
added is kept in `.mokosh/ingest_state.yaml`. `--phase git|tasks` runs one phase; `--force`
|
|
54
|
+
links every commit to a task key again (after a change of `commit_mask`). History rewritten
|
|
55
|
+
upstream (rebase, force push) leaves the old commits in the database and adds the new ones.
|
|
56
|
+
Tokens never go into the file. `mokosh init` (and the Setup page of `mokosh ui`) put the token you enter into an
|
|
57
|
+
environment variable — `GITHUB_TOKEN`, `JIRA_TOKEN`, `GITLAB_TOKEN`, `YOUTRACK_TOKEN`, or
|
|
58
|
+
`MOKOSH_<PROJECT>_GITHUB_TOKEN` when the plain name already holds another token — and project.yaml keeps
|
|
59
|
+
only `github_token: ${GITHUB_TOKEN}`. On Windows the variable is saved for your user (`setx`; mokosh also
|
|
60
|
+
reads it from there in a terminal opened before); elsewhere init prints the `export` line for your shell
|
|
61
|
+
profile.
|
|
62
|
+
|
|
63
|
+
A monthly refresh is two commands; gamayun rebuilds from scratch in minutes:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
mokosh ingest
|
|
67
|
+
gamayun index --db units.db --repo . --project sonar
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Then build on it:
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
simargl index units units.db # semantic search over the tasks
|
|
74
|
+
gamayun index --db units.db --repo . --project sonar # statistical ontology
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Other commands:
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
mokosh plaintext units.db [--dry-run] # clean tracker markup in a database from an older ingest
|
|
81
|
+
mokosh export units.db --out DIR # the rows as text files, for /flow glossary (see below)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
A folder set up earlier with `simargl init` (`.simargl/project.yaml`) is used as it is — no new
|
|
85
|
+
`init`, no ingest from scratch. (simargl no longer gathers anything: it only reads the database.)
|
|
86
|
+
|
|
87
|
+
### Web UI
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
mokosh ui # http://127.0.0.1:8093/ (--port, --host 0.0.0.0, --root FOLDER)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Flask + waitress, like the other tools of the toolkit (eight themes, six languages):
|
|
94
|
+
|
|
95
|
+
- **Setup** — the form `mokosh init` asks for: folder, database, repository and branch, tracker,
|
|
96
|
+
the commit mask with every preset tried on the repository's last 300 commits. A token goes to an
|
|
97
|
+
environment variable, as with `mokosh init`.
|
|
98
|
+
- **Ingest** — `mokosh ingest` (both phases, or git / tasks only; `--force`) with its log.
|
|
99
|
+
- **Status** — what the database holds, `project.yaml` (tokens masked), `ingest_state.yaml`.
|
|
100
|
+
|
|
101
|
+
Projects are the folders with `.mokosh/project.yaml` (or an earlier `.simargl/project.yaml`) under
|
|
102
|
+
the folder it was started in, and those remembered in `~/.mokosh/projects.json` (`mokosh init`
|
|
103
|
+
and the UI remember them).
|
|
104
|
+
|
|
105
|
+
### Export for a glossary
|
|
106
|
+
|
|
107
|
+
`/flow glossary` (1bcoder, vyrii) reads folders of text files, never SQLite; `mokosh export` writes
|
|
108
|
+
them:
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
mokosh export units.db --out DIR # --all (default): DIR/tasks/ and DIR/commits/ apart
|
|
112
|
+
mokosh export units.db --out DIR --join # one file per task with its commits (DIR/joined/)
|
|
113
|
+
mokosh export units.db --out DIR --task # tasks only (--commits: commits only)
|
|
114
|
+
mokosh export units.db --out DIR --level title # TITLE | desc: + DESCRIPTION (default) | comments: + COMMENTS
|
|
115
|
+
mokosh export units.db --out DIR --diff-mode full # the whole stored diff (default changed: only @/+/- lines)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`--join` writes each commit as its own `--- commit <sha> ---` block (message + per-file diff); a
|
|
119
|
+
task with no commit still gets its file, and commits with no task are counted, not guessed at.
|
|
120
|
+
`--all` keeps tasks and commit diffs as separate corpora, to be indexed as separate glossary
|
|
121
|
+
projects. `desc` is the default level: title + description beat title alone and the noisier
|
|
122
|
+
+comments in the retrieval experiments; `changed` drops the untouched context lines of a diff.
|
|
123
|
+
|
|
124
|
+
### Trackers
|
|
125
|
+
|
|
126
|
+
| `source` | Settings in `tasks:` |
|
|
127
|
+
|---|---|
|
|
128
|
+
| `jira` | `jira_url`, `jira_project`, `jira_connector: api \| html \| selenium`, `jira_token` |
|
|
129
|
+
| `github` | `github_owner`, `github_repo`, `github_token` — `mokosh init` asks for the repository URL `https://github.com/<USER>/<REPO>` and writes both names |
|
|
130
|
+
| `gitlab` | `gitlab_url`, `gitlab_project`, `gitlab_token` |
|
|
131
|
+
| `youtrack` | `youtrack_url`, `youtrack_project`, `youtrack_token` |
|
|
132
|
+
|
|
133
|
+
How a task key is found in a commit message is `commit_mask` in `tasks:`, for any tracker:
|
|
134
|
+
|
|
135
|
+
| `commit_mask` | Finds |
|
|
136
|
+
|---|---|
|
|
137
|
+
| `simple` (default) | `KAFKA-123` at the start of the message |
|
|
138
|
+
| `bracketed` | `[KAFKA-123]` anywhere |
|
|
139
|
+
| `django` | `Fixed #123` |
|
|
140
|
+
| `generic` | `fixes #123`, `closed: #123`, `resolves #123` |
|
|
141
|
+
| `broad` | any `#123` |
|
|
142
|
+
| `regex-key` | the project's own key anywhere, as a whole word: `Fixed: invoice total (OFBIZ-12345)`. Built from `jira_project` (or `youtrack_project`); a team that uses the keys of several projects writes them all: `jira_project: ABC,ABD,ABK` |
|
|
143
|
+
| any regex | your own pattern; the first group is the key, if there is one |
|
|
144
|
+
|
|
145
|
+
Own regexes, for example:
|
|
146
|
+
|
|
147
|
+
| regex | finds |
|
|
148
|
+
|---|---|
|
|
149
|
+
| `(OFBIZ-\d+)` | one project's key anywhere |
|
|
150
|
+
| `((?:ABC\|ABD\|ABK)-\d+)` | the keys of several projects anywhere |
|
|
151
|
+
| `\b((?:ABC\|ABD\|ABK)-\d+)\b` | the same, not inside a longer word (`XABC-1`) — what `regex-key` builds |
|
|
152
|
+
|
|
153
|
+
A pattern like `([A-Z]+-\d+)` (any key) also catches `UTF-8`, `SHA-256`, `ISO-8859`; name the keys.
|
|
154
|
+
|
|
155
|
+
`mokosh init` asks for it with any tracker. It prints this table, then tries every mask on the repository's
|
|
156
|
+
last 300 commits and offers the one that links the most:
|
|
157
|
+
|
|
158
|
+
```
|
|
159
|
+
On the last 300 commits of C:/Projects/Data/flink:
|
|
160
|
+
simple links 0 (0%) -
|
|
161
|
+
bracketed links 282 (94%) FLINK-40293, FLINK-38262, FLINK-40830
|
|
162
|
+
...
|
|
163
|
+
Commit mask [Enter = bracketed]:
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`mokosh ui` (**Setup**) shows the same table and the same counts.
|
|
167
|
+
|
|
168
|
+
Without a tracker mokosh stores the commits only (`TASK` stays empty); simargl can still index
|
|
169
|
+
them as commit units.
|
|
170
|
+
|
|
171
|
+
### Table names
|
|
172
|
+
|
|
173
|
+
New databases have singular `TASK` and `COMMIT`. Older simargl ingests have `TASKS` / `COMMITS`,
|
|
174
|
+
and the codeXtract datasets (`sonar.db`, `flink.db`) mix them; `mokosh.schema` resolves whatever a
|
|
175
|
+
file has, and simargl and gamayun read through it.
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## Part of the SIMARGL toolkit
|
|
180
|
+
|
|
181
|
+
mokosh is one of ten tools that together form an **intellectual development support system**:
|
|
182
|
+
|
|
183
|
+
| Tool | Role |
|
|
184
|
+
|---|---|
|
|
185
|
+
| **[simargl](https://github.com/szholobetsky/simargl)** | Task-to-code retrieval — given a task description, finds which files and modules are likely affected, using semantic similarity over git history |
|
|
186
|
+
| **[svitovyd](https://github.com/szholobetsky/svitovyd)** | Project map — scans any codebase and produces a structural map of definitions and cross-file dependencies; queried from the CLI, a web UI with a layered map, or an MCP server |
|
|
187
|
+
| **[1bcoder](https://github.com/szholobetsky/1bcoder)** | AI coding assistant for small local models — surgical context management, agents, parallel inference, proc scripts |
|
|
188
|
+
| **[yasna](https://github.com/szholobetsky/yasna)** | Session memory — indexes conversations from all AI agents so you can find what was discussed, when, and where |
|
|
189
|
+
| **[radogast](https://github.com/szholobetsky/radogast)** | Context drift monitor — measures how far an AI agent's conversation has drifted from the original task |
|
|
190
|
+
| **[vyrii](https://github.com/szholobetsky/vyrii)** | Local AI web UI — chat, translate, web research, RAG and file management in the browser (Flask; Gradio optional); powered by Ollama or any OpenAI-compatible backend |
|
|
191
|
+
| **[syryn](https://github.com/szholobetsky/syryn)** | Bluetooth identity beacon for headless devices — the device's IP is its Bluetooth name; hostname, mDNS and network interfaces on request |
|
|
192
|
+
| **[alkonost](https://github.com/szholobetsky/alkonost)** | Browser cockpit for 1bcoder's flows — the `deepagent_*` spec pipeline (plan tree → specs → task board) and the glossary (build, browse, search), tuned for small local models |
|
|
193
|
+
| **[gamayun](https://github.com/szholobetsky/gamayun)** | Statistical ontology without an LLM — builds triples of terms, identifiers, files and modules from task history or a folder of documents and code; every triple carries its significance test |
|
|
194
|
+
| **[mokosh](https://github.com/szholobetsky/mokosh)** | Dataset gathering — commits of a git repository and the tasks of its tracker (Jira, GitHub, GitLab, YouTrack) in one SQLite database that the other tools read |
|
|
195
|
+
|
|
196
|
+
- **simargl** finds the code related to a task by vector search over the task history (embeddings).
|
|
197
|
+
- **svitovyd** answers: *how is the code structured and what depends on what?*
|
|
198
|
+
- **1bcoder** answers: *how do I work with local models efficiently?*
|
|
199
|
+
- **yasna** answers: *where did I already discuss this?*
|
|
200
|
+
- **radogast** answers: *is the AI agent still on track toward the goal?*
|
|
201
|
+
- **vyrii** answers: *how do I access all of this through a browser?*
|
|
202
|
+
- **syryn** answers: *what is the address of this headless device?*
|
|
203
|
+
- **alkonost** answers: *how do I turn one task description into a reviewed, working plan — and see the project's glossary — without a big model or a big context window?*
|
|
204
|
+
- **gamayun** finds the code related to a task through an ontology of significant links between words, files and modules (no LLM).
|
|
205
|
+
- **mokosh** prepares the initial dataset from version control and the issue tracker.
|
|
206
|
+
|
|
207
|
+
Together they cover the full development loop: gather the history, understand the codebase, find relevant history, work with AI locally, remember what was decided, and verify the context stays on target.
|
|
208
|
+
|
|
209
|
+
The name comes from Slavic mythology. Mokosh (Мокош) is the goddess of the harvest and of spinning: she spins the threads of fate and watches over the yield of the land. A fitting name for the tool that spins tasks and commits into one yarn — no good ingest, no harvest.
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## About
|
|
214
|
+
|
|
215
|
+
(c) 2026 Stanislav Zholobetskyi, Oleh Andriichuk
|
|
216
|
+
Institute for Information Recording, National Academy of Sciences of Ukraine, Kyiv
|
|
217
|
+
PhD research: «Intelligent Technology for Software Development and Maintenance Support»
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""mokosh — the dataset a code-search study starts from: commits of a git repository and the
|
|
2
|
+
tasks of its tracker (Jira, GitHub, GitLab, YouTrack), in one SQLite database (TASK, COMMIT).
|
|
3
|
+
simargl, gamayun and 1bcoder read that database; mokosh is the only one that writes it."""
|
|
4
|
+
__version__ = "0.1.0"
|
|
5
|
+
|
|
6
|
+
from .db_manager import DatabaseManager # noqa: E402
|
|
7
|
+
from .git_connector import GitConnector # noqa: E402
|
|
8
|
+
from .task_extractor import TaskExtractor # noqa: E402
|
|
9
|
+
from .task_fetcher import TaskFetcher # noqa: E402
|
|
10
|
+
|
|
11
|
+
__all__ = ["DatabaseManager", "GitConnector", "TaskExtractor", "TaskFetcher", "__version__"]
|