edu-ai-suite 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. edu_ai_suite-0.0.1/.gitignore +10 -0
  2. edu_ai_suite-0.0.1/AGENTS.md +123 -0
  3. edu_ai_suite-0.0.1/GLOSSARY.md +243 -0
  4. edu_ai_suite-0.0.1/PKG-INFO +293 -0
  5. edu_ai_suite-0.0.1/README.md +275 -0
  6. edu_ai_suite-0.0.1/edu_ai_suite/__init__.py +4 -0
  7. edu_ai_suite-0.0.1/edu_ai_suite/_holdout.py +36 -0
  8. edu_ai_suite-0.0.1/edu_ai_suite/assets/__init__.py +11 -0
  9. edu_ai_suite-0.0.1/edu_ai_suite/assets/__main__.py +5 -0
  10. edu_ai_suite-0.0.1/edu_ai_suite/assets/britllm.py +26 -0
  11. edu_ai_suite-0.0.1/edu_ai_suite/assets/catalogue.py +17 -0
  12. edu_ai_suite-0.0.1/edu_ai_suite/assets/cli.py +84 -0
  13. edu_ai_suite-0.0.1/edu_ai_suite/assets/clouds.py +67 -0
  14. edu_ai_suite-0.0.1/edu_ai_suite/assets/hf.py +319 -0
  15. edu_ai_suite-0.0.1/edu_ai_suite/assets/mnist.py +72 -0
  16. edu_ai_suite-0.0.1/edu_ai_suite/images/__init__.py +7 -0
  17. edu_ai_suite-0.0.1/edu_ai_suite/images/decode.py +74 -0
  18. edu_ai_suite-0.0.1/edu_ai_suite/images/features/__init__.py +49 -0
  19. edu_ai_suite-0.0.1/edu_ai_suite/images/features/_conventions.py +65 -0
  20. edu_ai_suite-0.0.1/edu_ai_suite/images/features/_measurements.py +166 -0
  21. edu_ai_suite-0.0.1/edu_ai_suite/images/features/_preprocessing.py +75 -0
  22. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/__init__.py +161 -0
  23. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/anti_diagonal_profile_peaks.py +34 -0
  24. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/autocorrelation_half_width.py +53 -0
  25. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/blue_minus_red_mean.py +34 -0
  26. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/bluish_fraction.py +36 -0
  27. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/bright_fraction.py +36 -0
  28. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/centre_of_mass_x.py +30 -0
  29. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/centre_of_mass_y.py +31 -0
  30. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/coarse_texture.py +33 -0
  31. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/coarse_to_fine_texture_ratio.py +34 -0
  32. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/column_profile_peaks.py +34 -0
  33. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/column_profile_roughness.py +30 -0
  34. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/column_profile_trend.py +35 -0
  35. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/diagonal_profile_peaks.py +34 -0
  36. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/edge_density.py +40 -0
  37. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/elongation.py +37 -0
  38. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/fine_texture.py +31 -0
  39. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/foreground_area_fraction.py +32 -0
  40. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/greenery_fraction.py +35 -0
  41. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/hole_count.py +43 -0
  42. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/horizontal_spread.py +30 -0
  43. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/intensity_std.py +33 -0
  44. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/kurtosis.py +41 -0
  45. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/left_right_symmetry.py +35 -0
  46. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/mean_chroma.py +36 -0
  47. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/mean_gradient_magnitude.py +33 -0
  48. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/mean_intensity.py +33 -0
  49. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/profile_median_crossing_count.py +39 -0
  50. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/profile_row_column_correlation.py +34 -0
  51. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/quadrant_self_similarity.py +43 -0
  52. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/radial_gradient.py +48 -0
  53. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/robust_contrast.py +34 -0
  54. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/row_profile_peaks.py +35 -0
  55. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/row_profile_roughness.py +31 -0
  56. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/row_profile_trend.py +36 -0
  57. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/skewness.py +38 -0
  58. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/stroke_crossing_count.py +34 -0
  59. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/top_bottom_symmetry.py +34 -0
  60. edu_ai_suite-0.0.1/edu_ai_suite/images/features/catalogue/vertical_spread.py +29 -0
  61. edu_ai_suite-0.0.1/edu_ai_suite/images/features/description.py +72 -0
  62. edu_ai_suite-0.0.1/edu_ai_suite/images/features/extraction.py +105 -0
  63. edu_ai_suite-0.0.1/edu_ai_suite/images/features/feature.py +76 -0
  64. edu_ai_suite-0.0.1/edu_ai_suite/images/features/groups.py +219 -0
  65. edu_ai_suite-0.0.1/edu_ai_suite/images/features/guide.py +46 -0
  66. edu_ai_suite-0.0.1/edu_ai_suite/images/features/options.py +71 -0
  67. edu_ai_suite-0.0.1/edu_ai_suite/images/flatten.py +25 -0
  68. edu_ai_suite-0.0.1/edu_ai_suite/images/pixels.py +18 -0
  69. edu_ai_suite-0.0.1/edu_ai_suite/images/resize.py +37 -0
  70. edu_ai_suite-0.0.1/edu_ai_suite/lm/__init__.py +23 -0
  71. edu_ai_suite-0.0.1/edu_ai_suite/lm/architecture.py +154 -0
  72. edu_ai_suite-0.0.1/edu_ai_suite/lm/language_model.py +255 -0
  73. edu_ai_suite-0.0.1/edu_ai_suite/lm/pairs.py +133 -0
  74. edu_ai_suite-0.0.1/edu_ai_suite/lm/stats.py +26 -0
  75. edu_ai_suite-0.0.1/edu_ai_suite/lm/tokenizer.py +356 -0
  76. edu_ai_suite-0.0.1/edu_ai_suite/lm/training.py +111 -0
  77. edu_ai_suite-0.0.1/edu_ai_suite/lora/__init__.py +37 -0
  78. edu_ai_suite-0.0.1/edu_ai_suite/lora/artifact.py +91 -0
  79. edu_ai_suite-0.0.1/edu_ai_suite/lora/data.py +125 -0
  80. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_child.py +141 -0
  81. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_layout.py +73 -0
  82. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_process.py +183 -0
  83. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_protocol.py +74 -0
  84. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_runtime.py +125 -0
  85. edu_ai_suite-0.0.1/edu_ai_suite/lora/mlx_spec.py +30 -0
  86. edu_ai_suite-0.0.1/edu_ai_suite/lora/model.py +140 -0
  87. edu_ai_suite-0.0.1/edu_ai_suite/lora/run.py +228 -0
  88. edu_ai_suite-0.0.1/edu_ai_suite/text/__init__.py +25 -0
  89. edu_ai_suite-0.0.1/edu_ai_suite/text/_clean.py +126 -0
  90. edu_ai_suite-0.0.1/edu_ai_suite/text/_qa.py +85 -0
  91. edu_ai_suite-0.0.1/edu_ai_suite/text/_split.py +64 -0
  92. edu_ai_suite-0.0.1/examples/CREDITS.md +16 -0
  93. edu_ai_suite-0.0.1/examples/build_britllm_4bit.py +269 -0
  94. edu_ai_suite-0.0.1/examples/cirrus.jpg +0 -0
  95. edu_ai_suite-0.0.1/examples/image_features.py +110 -0
  96. edu_ai_suite-0.0.1/examples/lm_from_scratch.py +91 -0
  97. edu_ai_suite-0.0.1/examples/lora_generate.py +88 -0
  98. edu_ai_suite-0.0.1/examples/lora_inspect.py +162 -0
  99. edu_ai_suite-0.0.1/examples/lora_train.py +126 -0
  100. edu_ai_suite-0.0.1/examples/lora_walkthrough.sh +91 -0
  101. edu_ai_suite-0.0.1/examples/low_cloud.jpg +0 -0
  102. edu_ai_suite-0.0.1/examples/volutus.jpg +0 -0
  103. edu_ai_suite-0.0.1/pyproject.toml +72 -0
  104. edu_ai_suite-0.0.1/tests/synth.py +105 -0
  105. edu_ai_suite-0.0.1/tests/test_assets.py +350 -0
  106. edu_ai_suite-0.0.1/tests/test_colour.py +76 -0
  107. edu_ai_suite-0.0.1/tests/test_decode.py +68 -0
  108. edu_ai_suite-0.0.1/tests/test_docs.py +72 -0
  109. edu_ai_suite-0.0.1/tests/test_extract.py +198 -0
  110. edu_ai_suite-0.0.1/tests/test_feature_contract.py +126 -0
  111. edu_ai_suite-0.0.1/tests/test_flatten.py +40 -0
  112. edu_ai_suite-0.0.1/tests/test_guide.py +37 -0
  113. edu_ai_suite-0.0.1/tests/test_holdout.py +71 -0
  114. edu_ai_suite-0.0.1/tests/test_independence.py +120 -0
  115. edu_ai_suite-0.0.1/tests/test_layout.py +209 -0
  116. edu_ai_suite-0.0.1/tests/test_lm/test_language_model.py +411 -0
  117. edu_ai_suite-0.0.1/tests/test_lm/test_pairs.py +290 -0
  118. edu_ai_suite-0.0.1/tests/test_lm/test_public_api.py +95 -0
  119. edu_ai_suite-0.0.1/tests/test_lm/test_tokenizer.py +313 -0
  120. edu_ai_suite-0.0.1/tests/test_lm/test_tokenizer_stats.py +96 -0
  121. edu_ai_suite-0.0.1/tests/test_lora/test_facade.py +42 -0
  122. edu_ai_suite-0.0.1/tests/test_lora/test_lora.py +428 -0
  123. edu_ai_suite-0.0.1/tests/test_lora/test_mlx_child.py +168 -0
  124. edu_ai_suite-0.0.1/tests/test_lora/test_mlx_contract.py +119 -0
  125. edu_ai_suite-0.0.1/tests/test_lora/test_model_stream.py +68 -0
  126. edu_ai_suite-0.0.1/tests/test_lora/test_train_run.py +468 -0
  127. edu_ai_suite-0.0.1/tests/test_photometry.py +55 -0
  128. edu_ai_suite-0.0.1/tests/test_pixels.py +65 -0
  129. edu_ai_suite-0.0.1/tests/test_preprocess.py +86 -0
  130. edu_ai_suite-0.0.1/tests/test_profiles.py +83 -0
  131. edu_ai_suite-0.0.1/tests/test_redundancy.py +41 -0
  132. edu_ai_suite-0.0.1/tests/test_resize.py +58 -0
  133. edu_ai_suite-0.0.1/tests/test_spatial.py +73 -0
  134. edu_ai_suite-0.0.1/tests/test_stability.py +108 -0
  135. edu_ai_suite-0.0.1/tests/test_symmetry.py +74 -0
  136. edu_ai_suite-0.0.1/tests/test_text.py +269 -0
  137. edu_ai_suite-0.0.1/tests/test_texture.py +64 -0
  138. edu_ai_suite-0.0.1/tests/test_topology.py +57 -0
  139. edu_ai_suite-0.0.1/uv.lock +2303 -0
@@ -0,0 +1,10 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ data/
6
+ build/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .DS_Store
@@ -0,0 +1,123 @@
1
+ # Working on this codebase
2
+
3
+ How a change is made here: its words, its layers, its prose, its evidence. The
4
+ questions are written as questions because each one is answered with a fact about
5
+ the code. "I think so" is not an answer, and a question you cannot answer is the
6
+ finding.
7
+
8
+ Use it twice — before starting, so the shape is chosen rather than discovered, and
9
+ before merging, so the shape is checked rather than assumed.
10
+
11
+ ## Words
12
+
13
+ - Does any word carry more than one meaning? List the nouns the code uses and give
14
+ each exactly one meaning per layer. Where two layers need different meanings, they
15
+ need different words.
16
+ - Does any name describe the material rather than the product? Names like *data*,
17
+ *values*, *pixels*, *grid*, *utils*, *lib* distinguish nothing in a directory of
18
+ similar files. Say what comes out in one sentence: if the name does not predict
19
+ that sentence, it is the wrong name.
20
+ - Does a name have to cover more than one verb? A name that needs "and" is a missing
21
+ split, not a naming problem.
22
+ - Is each technical term the one the formula implements? Prose and arithmetic must
23
+ use the same word, including units and qualifiers.
24
+ - Does the same concept appear under two names? Synonyms are as costly as
25
+ homonyms, and harder to notice.
26
+
27
+ ## Docstrings
28
+
29
+ - Does the module docstring say what the module *is* — one sentence naming what
30
+ comes out, then at most two sentences of what it holds — in the vocabulary of its
31
+ own names? A docstring arguing for the module's shape has taken the commit
32
+ message's job. A module of definitions is a noun; a script is a verb, so its first
33
+ line is what it does: "Inspect and print information about a LoRA adapter", not
34
+ "What a directory is, as far as LoRA is concerned".
35
+ - Does the first line of a function or class docstring say what the call returns or
36
+ does? A second paragraph is only for what the body cannot be read for: a
37
+ precondition, an invariant, a unit or range, ownership of a file or directory, or
38
+ what `None` means.
39
+ - Has a subject been turned into a `what ...` clause? "What the artifact records",
40
+ "what it leaves", "what the config says" put the actor in a noun; write "the
41
+ artifact records the run", or "the run is recorded by the artifact".
42
+ - Does a file, a record or a number speak? An artifact records, names, holds; it does
43
+ not say, tell, know, want, agree or refuse. Name the thing and give it its own verb:
44
+ the record names the package, the config holds the rank, the loader rejects the
45
+ other package's file.
46
+ - Is a metaphor standing in for the noun — a "front door", a module that "lives" in a
47
+ directory, a group that "owns" a feature? Say function, directory, field.
48
+ - Does it tell the reader what things were like before it existed? "Before this one
49
+ existed", "used to", "no longer", "was spread over", "previously", "originally",
50
+ "this replaces". The reader has no past: say what is.
51
+ - Does it defend the design? "so the example and the tutorial cannot drift", "the point
52
+ is", "one spelling", "which is why it can be tested". That is a commit message, and
53
+ it goes stale on a different clock from the code.
54
+ - Does it retell the body? A docstring that walks the statements one by one is a
55
+ comment that aged into prose. Order the caller must know (brightness before
56
+ resampling) is an invariant and belongs; order only the implementation cares about
57
+ does not.
58
+
59
+ ## Steps and boundaries
60
+
61
+ - Is the mechanism in the library and the purpose in the caller? A caller composes
62
+ steps; the library supplies them. A caller that reimplements a step is missing a
63
+ function.
64
+ - Does any function hide a pipeline? If its steps have their own failure modes or
65
+ their own tests, they are functions of their own.
66
+ - Is anything shared at the wrong level? If two packages use something from inside a
67
+ third, it belongs a level higher, where both can reach it as a sibling.
68
+ - Does any file serve two layers at once? If you cannot name the single concept it
69
+ serves, either it is two files, or the concept is "the arguments of one call" — and
70
+ then the docstring should say exactly that.
71
+ - Is any policy spelled twice? Tolerances, interpolation, ordering, naming, rounding.
72
+ One spelling is a decision; two is drift waiting to happen.
73
+ - Does any default decide something on the caller's behalf? A default is an
74
+ invisible decision. Either require the argument or make the default explicit and
75
+ documented.
76
+
77
+ ## Layout and dependency
78
+
79
+ - Build the table first, then read it instead of the code: file, one-line purpose,
80
+ imports, imported by, when it runs.
81
+ - Where does an arrow point "upward" — a sibling reaching into a subpackage, a leaf
82
+ importing its parent, a cycle? Each is either a misplaced file or a missing module.
83
+ - Which files have exactly one caller? A helper of one caller belongs with that
84
+ caller — unless it is a product with its own tests. Decide which, and say so.
85
+ - Are the private/public marks consistent? The mechanical test: does anything outside
86
+ the package import it? Then public; otherwise private. Enforce it with a test rather
87
+ than a convention.
88
+ - What axis organises each file — pipeline stage, deliverable, view, or reuse? A
89
+ reader should be able to tell from the directory. A file organised by reuse needs
90
+ to say so, or it reads as "miscellaneous".
91
+ - Does the tree match the rest of the project? A domain that is split by abstraction
92
+ level in one place and by subject in another makes both harder to find.
93
+ - Is anything maintained twice — a page beside a catalogue, a list beside a
94
+ directory? Generate it or test it, never both.
95
+
96
+ ## Truth and evidence
97
+
98
+ - For every claim of the form "always" or "never", read the branch. Intent written as
99
+ fact is a defect, and it makes every neighbouring sentence need checking.
100
+ - Does every branch exist in a test? And does the test pin the policy or only one
101
+ outcome? A branch with no test is a branch nobody has decided about.
102
+ - Is the code testable in a plain Python process? Anything that needs a GUI to be
103
+ tested has been put in the wrong layer.
104
+ - If a recommendation changes behaviour, quantify it before recommending: what moves,
105
+ by how much, and would any test notice? An unquantified claim is an opinion, and the
106
+ measurement is usually shorter than the argument.
107
+ - For a pure move or rename, is there a diff proving nothing changed? For an
108
+ intentional change, is the measured effect named in the commit message? Without
109
+ one, a refactor and a regression look identical.
110
+ - Is any correspondence held only by convention — names beside values, a roster
111
+ beside a directory, membership beside a catalogue? Make it structural, so one call
112
+ produces both, or test it. Never neither.
113
+
114
+ ## Finishing
115
+
116
+ - Can each decision be reverted alone? One commit per structural step. A rename
117
+ riding on a move cannot be reviewed.
118
+ - What runs at import time? Validation is fine; work is not. The "when it runs"
119
+ column of the table catches this, and catches accidental caching with it.
120
+ - Is the prose still true after the move — paths, module references, generated pages,
121
+ tutorial text? Generated content cannot go stale; authored content needs a sweep.
122
+ - Would a newcomer reading only the file names and the docstrings arrive at the same
123
+ mental model you have? If not, the code says one thing and the words another.
@@ -0,0 +1,243 @@
1
+ # Glossary
2
+
3
+ ## Processes
4
+
5
+ - **tokenizer**: `text` → `stream` — the whole corpus as one list of ids.
6
+ - **encode**: `text` → `stream` — EOS closes every unit, and anything the
7
+ vocabulary cannot hold becomes `<unk>`.
8
+ - **decode**: `stream` → `text` — the reverse; BOS and EOS are dropped, `<unk>`
9
+ stays literal.
10
+ - **pair builder**: `stream` → `pairs` — one context and one target per prediction.
11
+ - **fit**: `architecture` + `text` → a `model` — build the pairs, then the weights
12
+ that minimise the loss.
13
+ - **model**: `context` → `logits` — one score per token in the vocabulary.
14
+ - **sampler**: `logits` → `id` — one id per step of a generation.
15
+ - **evaluate**: `model` + `text` → loss and perplexity, on any text. Report it on
16
+ text the weights were not trained on to see the in-domain / out-of-domain gap.
17
+ - **save** / **load**: `model` ↔ a directory — the weights (`model.keras`), the
18
+ tokenizer, and the run's settings and results (`meta.json`).
19
+
20
+ ## Text
21
+
22
+ - **text** — a string a person wrote.
23
+ - **corpus** — all the text being trained on.
24
+ - **unit** — one self-contained piece of text. Currently a unit is a sentence.
25
+
26
+ ## Tokens
27
+
28
+ Tokens are built by the tokenizer (`edu_ai_suite/lm/tokenizer.py`).
29
+
30
+ - **token** — the smallest piece the tokenizer emits: one character (char
31
+ tokenizer) or one word or punctuation mark (word tokenizer).
32
+ - **id** — a token's integer. Opaque: only the tokenizer's table gives it
33
+ meaning, and ids never mean anything across two tokenizers.
34
+ - **vocabulary** — the set of tokens the tokenizer knows. **`vocab_size`** — how
35
+ many, which is also how many logits the model emits.
36
+ - **stream** — the corpus as one flat list of ids. Each unit ends with EOS. The
37
+ stream is the only input to the pair builder.
38
+ - **special token** — one of BOS, `<unk>` or EOS: the tokens every vocabulary
39
+ begins with, and the only ones with no corpus count. "The specials" for short.
40
+ - **EOS** — the id that closes a unit. It travels inside the stream, and it is
41
+ itself a target.
42
+ - **BOS** — the id meaning "nothing came before". It never travels in the
43
+ stream: the pair builder opens a unit's context with it, and pads a context
44
+ out with it when the unit has fewer than `w` ids before the target.
45
+ - **count** — how often a real token occurs in the corpus the vocabulary was
46
+ built from. Zero for the specials, which are not counted.
47
+ - **rank** — a real token's position when tokens are ordered by count, the most
48
+ frequent first; equal counts are ordered by spelling. Vocabulary order is not
49
+ rank: it carries no meaning at all.
50
+ - **frequency table** — the top `n` real tokens as (rank, token, count). The Zipf
51
+ summary is built from it; the vocabulary viewer shows the whole rank order.
52
+ - **total token count** — the corpus tokens counted when the vocabulary was
53
+ built. Pruning leaves it alone, so shares stay corpus-relative rather than
54
+ being measured against the survivors.
55
+ - **pruning** — removing real tokens from a vocabulary, by `keep_top(n)` or by a
56
+ frequency floor `drop_rare(min_count=…)`. The specials always survive, and the
57
+ remaining counts and the total are preserved.
58
+ - **max vocab** — the cap on the vocabulary, applied by `keep_top`
59
+ before training; `0` keeps everything.
60
+ - **dropped occurrences** — corpus occurrences that pruning made
61
+ unrepresentable. They are the difference between the total token count and
62
+ the sum of the counts, and they encode as `<unk>`.
63
+ - **`<unk>`** — the id for anything the vocabulary cannot represent: a token
64
+ never seen when the vocabulary was built, or a real token that pruning
65
+ removed. It appears in the stream like any other id, so it can be a **target**
66
+ as well as part of a context, and a model can learn to emit it.
67
+
68
+ ## Context
69
+
70
+ - **context** — the ids the model reads to make one prediction. At most `w` of
71
+ them are ids of the text; the rest are BOS, because the unit had nothing
72
+ before the target. Left-padded to width `w`, always.
73
+ - **slot** — one place in a context, holding one id. Slots are ordered: the
74
+ **rightmost slot holds the id immediately before the target**, and moving
75
+ left goes further back.
76
+ - **context width `w`** — how many slots a context has: how far back the model
77
+ reads. A hyperparameter of `MLP` and `LSTM`, enumerable through `get_params()`,
78
+ and the constant 1 for `Bigram`, which has no such parameter: a model that
79
+ reads more history is an n-gram of higher order, a different architecture
80
+ rather than a setting.
81
+
82
+ ## Pairs
83
+
84
+ Pairs are built by the pair builder (`edu_ai_suite/lm/pairs.py`).
85
+
86
+ - **target** — the token being predicted: the answer.
87
+ - **pair** — one (context, target). One prediction. Also called an
88
+ **example**.
89
+ - **`Pairs`** — all the pairs of a corpus, held as two arrays: `contexts` and
90
+ `targets`; reached through `make_pairs`, which returns one. Not exported by the
91
+ package façade, so a caller that needs the type by name imports it from
92
+ `edu_ai_suite/lm/pairs.py`.
93
+ - **stream length** — the number of ids in the stream. Equal to the number of
94
+ predictions, and therefore to the number of pairs. Written `len(stream)` and
95
+ called **`N`**.
96
+ - **pairs per unit** — how many pairs one unit contributes: a unit of `m` tokens
97
+ contributes `m + 1` pairs, because every token is predicted once and EOS is predicted too.
98
+ - **split** — the division of pairs into training and validation sets. Made per
99
+ unit, so every pair of a unit lands on the same side. Which units are held out
100
+ is a stride over the corpus - every n-th unit, offset by the seed - so the
101
+ split follows corpus order: reordering the corpus can move a unit across, and
102
+ appending can only add to the held-out set. How it works is documented on
103
+ :meth:`Pairs.split` and `edu_ai_suite/_holdout.py`.
104
+ - **batch** — a stack of pairs handed to the model at once.
105
+
106
+ ## Shapes
107
+
108
+ As a compact representation, the `contexts` are packaged as a matrix with one row per
109
+ prediction and one column per slot.
110
+
111
+ ```
112
+ col 0 col 1 col 2 w = 3 columns = slots,
113
+ row 1 (target a) BOS BOS BOS oldest ... newest
114
+ row 2 (target b) BOS BOS a
115
+ row 3 (target EOS) BOS a b
116
+ ^ N = 3 rows = predictions
117
+ ```
118
+
119
+ - **row** — one prediction: one pair, i.e. its context.
120
+ - **column** — one slot: the same distance back from the target, in every row.
121
+
122
+ `Pairs` keeps the targets **outside** this array: `contexts` is `(N, w)` and
123
+ `targets` is `(N,)`. The target of row *i* is the last column of row *i+1*,
124
+ wherever both rows belong to the same unit - that is what the slide does. The
125
+ pair builder does not exploit it: it writes both arrays from the same
126
+ `[BOS, s1..sm, EOS]` sequence, so a row's target stays unambiguous even when its
127
+ context is padded out with BOS, or when a unit is shorter than `w`.
128
+
129
+ ## Model
130
+
131
+ - **architecture** — a buildable recipe: a Keras graph plus its own
132
+ hyperparameters (`Bigram`, `MLP`, `LSTM`). No weights, no tokenizer.
133
+ - **model** — an instance of an architecture with a tokenizer and weights: what
134
+ `fit` returns, and what generates, evaluates and saves. The architecture is
135
+ the recipe; the model is one fitted result of it.
136
+ - **run** — one fine-tuning job, seen as an iterator: iterating it is what starts
137
+ the training, each report arrives as an event, `stop()` asks it to end from any
138
+ thread, and the artifact is ready when the iteration finishes. A run is asked
139
+ for and then started, so a caller that changes its mind leaves nothing behind.
140
+ - **hyperparameter** — a setting for an architecture, e.g. `context_width`,
141
+ `embedding_dim`, or the LSTM's `units`. Enumerable through `get_params()`.
142
+ - **parameter** — a number the model learns from the data (embeddings, weights,
143
+ biases), as against a **hyperparameter**, which is chosen. `parameter_count()`
144
+ counts these.
145
+ - **embedding** — the learned vector for one token id; `embedding_dim` is how
146
+ long it is.
147
+ - **order** — how many tokens a prediction involves: its `w` ids of history plus
148
+ the target, so `order = w + 1`. A bigram conditions on one token, so its order
149
+ is fixed at 2; an `MLP` or `LSTM` order follows its context width.
150
+ - **n-gram** — a model of order `n`, conditioning on `n - 1` ids: `Bigram` is
151
+ the `n = 2` member. `MLP` and `LSTM` are trainable relatives of the same
152
+ family, whose order is chosen instead of fixed.
153
+ - **logits** — one score per token in the vocabulary, unnormalised.
154
+ - **softmax** — turns logits into probabilities: positive and summing to one. It
155
+ is applied in the loss and in the sampler, never in the Keras graph.
156
+ - **loss** — the number training minimises: the cross-entropy between the
157
+ model's distribution over the vocabulary and the actual target, averaged over
158
+ all predictions in the set being measured (train or validation).
159
+ - **optimizer** — the rule that turns the loss into weight updates. Adam here,
160
+ with the learning rate as its step size.
161
+ - **perplexity** — `e^loss`: roughly, how many tokens the model is choosing
162
+ between at each step. 1 is perfect and the vocabulary size is a uniform guess.
163
+ Reported for the training and the validation pairs separately.
164
+ - **in-domain** / **out-of-domain** — text drawn from the corpus the model was
165
+ trained on, versus text from anywhere else. The two perplexities are the
166
+ comparison.
167
+ - **learning rate** — how far one optimizer step moves the weights.
168
+ - **batch size** — how many pairs one optimizer step averages over; the `batch`
169
+ is that stack of pairs.
170
+ - **validation fraction** — the share of *units* held out for the split, not of
171
+ pairs.
172
+ - **seed** — the number a run's random choices are derived from: weight
173
+ initialisation and the split, so the same seed and inputs repeat a run.
174
+ `generate` takes a seed of its own for sampling.
175
+ - **epoch** — one pass over all the training pairs. The per-epoch stats carry the
176
+ loss, the validation loss and both perplexities, plus a sample if the caller's
177
+ hook adds one.
178
+ - **prompt** — the user's text, tokenized and trimmed to its last unit, used as
179
+ the first context of a generation.
180
+ - **text stream** — a model's text as it is produced: a generator whose chunks a
181
+ caller concatenates, and *not* the id `stream` above. Every model in the suite
182
+ has one - its `stream` method, with `generate` as the text stream joined up -
183
+ so a caller can show text arriving without knowing which model it is looking
184
+ at. Where the chunks fall is the library's business: mlx-lm slices
185
+ arbitrarily, the Keras models yield the finished unit in one piece.
186
+ - **generation** — repeat: context → logits → pick one id → append it → drop the
187
+ oldest id. It stops at EOS or at the token cap, and a sampled BOS is dropped
188
+ rather than appended.
189
+ - **temperature** — how the picking is done: `0` takes the best id, and higher
190
+ values flatten the distribution.
191
+
192
+ ## Relations
193
+
194
+ ```
195
+ w = 1 Bigram: a constant, not a parameter
196
+ w = a hyperparameter MLP, LSTM
197
+ order = w + 1 Bigram 2; the others chosen with their width
198
+ N = sum(pairs per unit) independent of w
199
+ contexts.shape = (N, w)
200
+ ```
201
+
202
+ ## Example: one small corpus, three architectures
203
+
204
+ Corpus: two units, `a b` and `c`. The `MLP` and `LSTM` are given a
205
+ context width of 3; the `Bigram` has no width to be given and reads 1.
206
+
207
+ ```
208
+ stream = [a, b, EOS, c, EOS] len(stream) = 5 = N = 5 pairs
209
+ unit "a b": seq = [BOS, a, b, EOS] pairs: 3 (2 tokens + EOS)
210
+ unit "c": seq = [BOS, c, EOS] pairs: 2 (1 token + EOS)
211
+ ```
212
+
213
+ | # | target | `Bigram` context (`w = 1`) | `MLP` context (`w = 3`) | `LSTM` context (`w = 3`) |
214
+ |---|---|---|---|---|
215
+ | 1 | `a` | `[BOS]` | `[BOS, BOS, BOS]` | `[BOS, BOS, BOS]` |
216
+ | 2 | `b` | `[a]` | `[BOS, BOS, a]` | `[BOS, BOS, a]` |
217
+ | 3 | `EOS` | `[b]` | `[BOS, a, b]` | `[BOS, a, b]` |
218
+ | 4 | `c` | `[BOS]` | `[BOS, BOS, BOS]` | `[BOS, BOS, BOS]` |
219
+ | 5 | `EOS` | `[c]` | `[BOS, BOS, c]` | `[BOS, BOS, c]` |
220
+
221
+ What the table shows:
222
+
223
+ - **Width belongs to the model.** The `MLP` and `LSTM` were set to 3, the
224
+ `Bigram` cannot be set at all; it reads 1 slot because that is what a bigram
225
+ is.
226
+ - **Order follows `w`.** These five predictions are order 2 for `Bigram` and
227
+ order 4 for the other two: the same targets, asked of different models.
228
+ - **Width changes columns only.** The five targets and the split are identical
229
+ in all three columns, so what differs is only how much history each model
230
+ reads. The bigram is fed exactly the pair a count table would use.
231
+ - **All BOS means "nothing came before".** Rows 1 and 4 are all BOS because BOS
232
+ opening a unit's context and BOS padding a short context out are the *same
233
+ id*. Those rows are trained, not discarded, so a unit's opening token is
234
+ learnable.
235
+
236
+ And why the architectures differ at all, with `V` = `vocab_size`, `d` =
237
+ `embedding_dim`, `h` = `hidden_units`:
238
+
239
+ | architecture | reads | order | parameters |
240
+ |---|---|---|---|
241
+ | `Bigram` | 1 slot | 2 | `V²`, independent of `w` |
242
+ | `MLP` | all `w` slots | `w + 1` | grows with `w` (first dense layer takes `w*d` inputs) |
243
+ | `LSTM` | all `w` slots | `w + 1` | independent of `w` (shared weights over steps) |
@@ -0,0 +1,293 @@
1
+ Metadata-Version: 2.5
2
+ Name: edu-ai-suite
3
+ Version: 0.0.1
4
+ Summary: Teaching material for image features, tiny language models and LoRA fine-tuning: plain Python, no GUI.
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.11
7
+ Requires-Dist: datasets>=5.1
8
+ Requires-Dist: huggingface-hub>=0.29
9
+ Requires-Dist: keras>=3.10.0
10
+ Requires-Dist: numpy>=1.26
11
+ Requires-Dist: opencv-python-headless<5,>=4.12
12
+ Requires-Dist: scikit-learn>=1.4
13
+ Requires-Dist: scipy>=1.11
14
+ Requires-Dist: tensorflow>=2.19.0
15
+ Provides-Extra: lora
16
+ Requires-Dist: mlx-lm[train]==0.32.2; (platform_machine == 'arm64' and sys_platform == 'darwin') and extra == 'lora'
17
+ Description-Content-Type: text/markdown
18
+
19
+ # edu-ai-suite
20
+
21
+ A plain-Python teaching library for image features, tiny language models and LoRA
22
+ fine-tuning. Each image feature turns a picture into one number - `mean_intensity`,
23
+ `edge_density`, `coarse_texture` and `left_right_symmetry` among them - each language
24
+ model is a tokenizer, an architecture and the weights that architecture fitted, and each
25
+ LoRA adapter is a base model plus a small set of trained values stored beside it.
26
+
27
+ ## What is in the box
28
+
29
+ - **`edu_ai_suite/images/`** - the image pipeline: `decode`, `resize`, `flatten` and
30
+ `pixels`, and the `features` package built on them: one class per feature, the groups
31
+ that select them, extraction, and the generated guide.
32
+ - **`edu_ai_suite/lm/`** - the character and word tokenizers with their stats, pairs,
33
+ the bigram/MLP/LSTM architectures, training options and results, and `LanguageModel`,
34
+ which carries its tokenizer, so one directory holds everything a model needs.
35
+ - **`edu_ai_suite/lora/`** - LoRA fine-tuning of an MLX base model: options, data, the
36
+ train/validation split, the run, and the adapter and model objects that read it back.
37
+ - **`edu_ai_suite/text/`** - plain text into training rows, behind one facade: cleaning,
38
+ splitting and Q/A parsing.
39
+ - **`edu_ai_suite/assets/`** - the Hugging Face assets, each behind a download function
40
+ that caches and verifies it, and the command-line interface over them.
41
+ - **`examples/`** - one runnable script per subject, and a shell walkthrough of the LoRA
42
+ scripts in the order a run happens.
43
+ - **`tests/`** - the test suite, in plain Python.
44
+ - **`GLOSSARY.md`** - the words the text and model code uses, one definition each:
45
+ tokenizer, token, context, pair, model, sampler.
46
+
47
+ ## Installation
48
+
49
+ Requirements: Python 3.11+, and macOS on Apple Silicon for LoRA training.
50
+
51
+ Everything except LoRA is a plain dependency: OpenCV for the image pipeline, Keras on a
52
+ TensorFlow backend for the from-scratch language models. Only LoRA is an extra, because
53
+ MLX runs on Apple Silicon alone:
54
+
55
+ ```bash
56
+ pip install edu-ai-suite # everything but LoRA
57
+ pip install "edu-ai-suite[lora]" # plus LoRA (Apple Silicon)
58
+ ```
59
+
60
+ Development, from the repository root:
61
+
62
+ ```bash
63
+ uv sync
64
+ uv run pytest
65
+ uv build
66
+ ```
67
+
68
+ ## Python API
69
+
70
+ ### Language models
71
+
72
+ A runnable end-to-end script lives in `examples/lm_from_scratch.py`
73
+ (`uv run python examples/lm_from_scratch.py`); the essentials are:
74
+
75
+ ```python
76
+ import edu_ai_suite.lm as lm
77
+
78
+ text = "\n".join(texts) # join documents outside this package
79
+ tokenizer = lm.tokenizer_from_text(text, "char")
80
+ tokenizer = tokenizer.keep_top(2000) # or .drop_rare(min_count=3); pure, chainable
81
+ stream = tokenizer.encode(text) # every sentence ends with <eos>
82
+ pairs = lm.make_pairs(stream, context_width=16)
83
+ train, valid = pairs.split(validation_fraction=0.1, seed=0)
84
+
85
+ # An architecture is the recipe; a model is that recipe fitted.
86
+ architecture = lm.MLP(context_width=16, embedding_dim=32, hidden_units=128)
87
+ model = lm.LanguageModel(architecture, tokenizer).fit(
88
+ text,
89
+ lm.TrainingOptions(epochs=15, validation_fraction=0.1, seed=0),
90
+ )
91
+
92
+ model.generate("the ", temperature=0.8, max_new_tokens=64)
93
+ model.evaluate(held_out_text) # {'loss': ..., 'perplexity': ...}; perplexity = e^loss
94
+ model.save("my-model") # weights + tokenizer + metadata
95
+ ```
96
+
97
+ `fit` returns the model, so the call can be chained; it builds pairs at
98
+ `architecture.context_width`, so the data and the weights cannot disagree.
99
+ `LanguageModel.load(path)` reads the directory back as one object. The training hook
100
+ (`on_epoch`) and the stop callback (`stop_requested`) are plain Python callables.
101
+
102
+ ### LoRA
103
+
104
+ ```python
105
+ import tempfile
106
+ from pathlib import Path
107
+
108
+ import edu_ai_suite.assets as assets
109
+ import edu_ai_suite.lora as lora
110
+
111
+ base = assets.get_britllm() # pinned 4-bit MLX weights, cached on first use
112
+ data = lora.LoRAData.from_texts(corpus) # or lora.LoRAData.from_pairs(pairs)
113
+ with tempfile.TemporaryDirectory() as scratch: # this run's scratch, deleted on exit
114
+ run = lora.train_lora(
115
+ base,
116
+ data,
117
+ options=lora.LoRAOptions(iters=300, num_layers=16),
118
+ workdir=scratch, # each run gets a fresh child of it
119
+ name="my-adapter", # the artifact's name is the caller's to choose
120
+ )
121
+ for event in run: # lora.Params, then TrainReport and ValReport as they arrive
122
+ print(event)
123
+ # keep the artifact somewhere you own; the scratch can then be discarded
124
+ artifact = run.adapter.save(Path("adapters") / run.adapter.name)
125
+ model = lora.LoRAModel.from_artifact(artifact)
126
+ model.generate("In the second century", temperature=0)
127
+ model.evaluate(held_out_text) # {'loss': ..., 'perplexity': ...}
128
+ model.adapter_enabled = False # the same base model, before adaptation
129
+ ```
130
+
131
+ `train_lora` returns a run rather than doing the work: nothing starts until you iterate
132
+ it, each report arrives as a typed event, and `run.adapter` is the artifact once the
133
+ iteration ends. `run.stop()` may be called from any thread - a Stop control in a caller
134
+ does exactly that - and it returns at once; the stop itself happens in the iteration,
135
+ which also waits for the training process to be gone before anything reads its directory.
136
+ mlx-lm writes a checkpoint every `save_every` iterations, so a stop adopts the newest and
137
+ `iters_completed` records how far the run got, while a stop before the first one raises
138
+ `TrainingStopped` and produces no model. The training runs in its own process - a module
139
+ whose name begins `mlx_` hides that, along with mlx-lm's API and file conventions, so the
140
+ rest of `lora/` never sees a subprocess.
141
+
142
+ ## Image feature groups
143
+
144
+ Each feature is a class in `edu_ai_suite.images.features.catalogue`, carrying its own
145
+ name, description sentence and computation. Groups are a set of membership lists over
146
+ that flat catalogue (`edu_ai_suite.images.features.groups`): membership lists used for
147
+ ablation and display, so a feature can belong to more than one group and the grouping
148
+ never touches the feature itself.
149
+
150
+ | Group | Count | What it describes |
151
+ |---|---|---|
152
+ | A | 6 | Global photometry: brightness, contrast, skewness, kurtosis, bright fraction |
153
+ | B | 10 | Row/column/diagonal profile summaries: trend, roughness, peaks, crossings |
154
+ | C | 6 | Spatial mass: centre of mass, spread, elongation, radial gradient |
155
+ | D | 6 | Texture: gradient, edge density, fine/coarse texture, autocorrelation width |
156
+ | E | 3 | Topology: hole count, foreground area, stroke crossings |
157
+ | F | 3 | Symmetry: left-right, top-bottom, quadrant similarity |
158
+ | G | 4 | Colour (opt-in): bluish fraction, the blue-red balance, colourfulness, greenery |
159
+
160
+ Groups A-F run on a preprocessed luminance image. Group G reads colour directly and is
161
+ off by default.
162
+
163
+ ```python
164
+ import edu_ai_suite.images.features as features
165
+
166
+ vec = features.extract_features(rgb_image, groups="ABCDEF")
167
+ vec["hole_count"] # a feature value
168
+ vec.to_array() # one float per selected feature
169
+ features.get_feature("hole_count").description # title, sentence, intuition, mechanics
170
+ features.all_features() # the full catalogue, in stable column order
171
+ ```
172
+
173
+ ### The feature guide
174
+
175
+ Every feature carries four descriptions: a title, a one-sentence summary, a plain-language
176
+ account of which pictures move it and which leave it cold, and a technical account of how
177
+ it is computed with links for further reading. The whole guide is generated from the
178
+ catalogue when it is asked for, so it cannot drift from the code. `feature_guide_markdown`
179
+ returns the whole page:
180
+
181
+ ```python
182
+ import edu_ai_suite.images.features as features
183
+
184
+ print(features.feature_guide_markdown())
185
+ ```
186
+
187
+ A runnable script lives in `examples/image_features.py`
188
+ (`uv run python examples/image_features.py`). It reads three real cloud photographs from
189
+ `examples/` (credits in `examples/CREDITS.md`) and shows the catalogue and its
190
+ descriptions, the extraction and ablation selections, fine texture on wispy cirrus against
191
+ coarse texture on a roll cloud, and the colour shortcut separating a blue sky from ground
192
+ vegetation.
193
+
194
+ ## Datasets
195
+
196
+ ```python
197
+ import edu_ai_suite.assets as assets
198
+
199
+ britllm = assets.get_britllm() # a 4-bit MLX snapshot, 1.40 GB
200
+ mnist = assets.get_mnist() # <root>/train, <root>/val: 28x28, 500/100 per digit
201
+ clouds = assets.get_clouds() # <root>/ac, sc, cu, cb, ci + CREDITS.md
202
+ ```
203
+
204
+ `get_britllm`, `get_mnist` and `get_clouds` each download, prepare and verify one pinned
205
+ Hub asset. The source and the prepared tree are cached; repeated calls are offline.
206
+
207
+ - `britllm-4bit` - [tcorbettclark/britllm-3b-v0.1-4bit-mlx](https://huggingface.co/tcorbettclark/britllm-3b-v0.1-4bit-mlx),
208
+ a 4-bit affine conversion of `britllm/britllm-3b-v0.1` (2.67B parameters, 1.40 GB),
209
+ verified against the SHA-256 of its weights file. It keeps the source model's ODC-BY
210
+ licence; `examples/build_britllm_4bit.py` builds and optionally publishes it.
211
+ - `mnist-digits` - [ylecun/mnist](https://huggingface.co/datasets/ylecun/mnist)
212
+ (MIT), 60k/10k parquet; `get_mnist` keeps 500/100 per digit at the source
213
+ 28x28 resolution in the folder-per-label layout.
214
+ - `ccsn-clouds` - [aduuuuuu/CCSN](https://huggingface.co/datasets/aduuuuuu/CCSN)
215
+ (MIT mirror; the CCSN database itself is CC0), the five genera Ac, Sc, Cu,
216
+ Cb and Ci at 256x256, with `CREDITS.md`. Cite Zhang et al. (2018), *CloudNet*.
217
+
218
+ The `assets` command lists each asset with its type, size, cache status and repo;
219
+ `fetch` is idempotent and prints the usable path:
220
+
221
+ ```bash
222
+ assets list
223
+ assets fetch britllm-4bit
224
+ assets fetch mnist-digits
225
+ assets fetch ccsn-clouds
226
+ ```
227
+
228
+ The same interface is available as `python -m edu_ai_suite.assets`.
229
+
230
+ ## Examples
231
+
232
+ - `examples/image_features.py` - the catalogue, extraction and the ablation selections,
233
+ for three cloud photographs.
234
+ - `examples/lm_from_scratch.py` - trains, samples and scores a tiny character-level
235
+ language model on a six-line corpus.
236
+ - `examples/lora_train.py` - one LoRA job: the rows, the settings, the events and the
237
+ adapter; `--save DIR` keeps it and `--stop` ends the run early.
238
+ - `examples/lora_generate.py` - loads an adapter and compares the base model with it,
239
+ text and perplexity both.
240
+ - `examples/lora_inspect.py` - reads an adapter and summarises it: the base model, the
241
+ rank and the blocks it reached, how many values it holds, how far the run got. Needs
242
+ no MLX and no model.
243
+ - `examples/build_britllm_4bit.py` - a maintainer script: downloads the fp16 source,
244
+ converts it to 4-bit MLX, writes the model card and the manifest, and uploads it with
245
+ `--upload`.
246
+ - `examples/lora_walkthrough.sh` - runs the three LoRA scripts in order:
247
+ `bash examples/lora_walkthrough.sh`.
248
+
249
+ The LoRA scripts run with the extra: `uv run --extra lora python examples/lora_train.py`.
250
+
251
+ ## Layout
252
+
253
+ ```
254
+ edu_ai_suite/
255
+ __init__.py
256
+ _holdout.py which positions a validation split holds out (shared)
257
+ images/ image data: decoded, resampled, then measured
258
+ decode.py RGB/luma, alpha and [0,1] conventions (shared)
259
+ resize.py one resampling policy: area shrinking, bilinear growing
260
+ flatten.py a grid as one row, with a coordinate name per cell
261
+ pixels.py the deliberately plain baseline: a grid, flattened
262
+ features/ the feature package
263
+ catalogue/ one module and one class per feature, plus the index
264
+ description.py the four descriptions, checked when a feature is created
265
+ feature.py the feature contract, checked at import
266
+ guide.py the whole catalogue as one generated Markdown page
267
+ groups.py groups as membership lists over the catalogue
268
+ extraction.py extract_features and FeatureVector
269
+ options.py FeatureOptions
270
+ _preprocessing.py the pipeline (support code)
271
+ _conventions.py numeric rules (zero, flat, net change)
272
+ _measurements.py measurements shared by more than one feature
273
+ lm/ tokenizer (with its stats), pairs, architectures,
274
+ training options/results, LanguageModel with embedded tokenizer
275
+ lora/ LoRA options, data, split, the run, and the artifact/model objects
276
+ mlx_* the boundary, marked by the prefix: mlx-lm's files, its wire
277
+ format, its child process and its in-process runtime
278
+ text/ plain text into training rows, behind one facade
279
+ _clean.py raw text -> clean text (Gutenberg, page numbers, reflow)
280
+ _split.py clean text -> table rows (whole file, paragraph, sentence)
281
+ _qa.py clean text -> Q/A prompt/completion pairs
282
+ assets/ Hugging Face assets: download cache policy, prepare, registry, CLI
283
+ examples/ one runnable script per subject
284
+ tests/ the test suite; test_lm/ and test_lora/ per domain
285
+ ```
286
+
287
+ ## License
288
+
289
+ MIT for the code in this repository.
290
+
291
+ Third-party material keeps its own terms: MNIST is MIT, the CCSN cloud database is CC0,
292
+ downloaded text keeps its own licence, and the 4-bit BritLLM artifact keeps the source
293
+ model's ODC-BY licence.