leos-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- holographic_ablate.py +358 -0
- holographic_access.py +144 -0
- holographic_accumulate.py +115 -0
- holographic_acoustic.py +122 -0
- holographic_adaptive.py +125 -0
- holographic_adaptive_cache.py +80 -0
- holographic_adaptive_sample.py +95 -0
- holographic_agent.py +156 -0
- holographic_agent_bridge.py +189 -0
- holographic_ai.py +1133 -0
- holographic_analog.py +147 -0
- holographic_anim.py +153 -0
- holographic_answer.py +140 -0
- holographic_archive.py +329 -0
- holographic_array.py +246 -0
- holographic_assembly.py +181 -0
- holographic_assetimport.py +695 -0
- holographic_assets.py +445 -0
- holographic_attributes.py +132 -0
- holographic_audio.py +140 -0
- holographic_autobump.py +207 -0
- holographic_autodisplace.py +122 -0
- holographic_automaton.py +121 -0
- holographic_backend.py +160 -0
- holographic_backwardwarp.py +72 -0
- holographic_bandwidth.py +149 -0
- holographic_blackbody.py +135 -0
- holographic_blendhome.py +138 -0
- holographic_blendpose.py +145 -0
- holographic_brdf.py +317 -0
- holographic_burn.py +150 -0
- holographic_bus.py +237 -0
- holographic_cache.py +137 -0
- holographic_cachehome.py +145 -0
- holographic_camera.py +179 -0
- holographic_cancel.py +84 -0
- holographic_catalog.py +1009 -0
- holographic_cellular.py +160 -0
- holographic_chaos.py +204 -0
- holographic_chart.py +237 -0
- holographic_checkerboard.py +151 -0
- holographic_classifier.py +129 -0
- holographic_clifford.py +188 -0
- holographic_coarsefirst.py +143 -0
- holographic_codec.py +164 -0
- holographic_codegen.py +253 -0
- holographic_coldstore.py +278 -0
- holographic_collide.py +75 -0
- holographic_colortransfer.py +109 -0
- holographic_combustion.py +212 -0
- holographic_command.py +149 -0
- holographic_compile.py +222 -0
- holographic_compose.py +168 -0
- holographic_compress.py +87 -0
- holographic_compute.py +156 -0
- holographic_computehome.py +172 -0
- holographic_condprop.py +142 -0
- holographic_conformal.py +309 -0
- holographic_coordinator.py +432 -0
- holographic_core.py +360 -0
- holographic_cosamp.py +174 -0
- holographic_cosmic.py +142 -0
- holographic_cosserat.py +264 -0
- holographic_creature.py +2333 -0
- holographic_creature_mind.py +102 -0
- holographic_curlnoise.py +142 -0
- holographic_cymatics.py +278 -0
- holographic_dedoppler.py +173 -0
- holographic_definitions.py +934 -0
- holographic_deform.py +135 -0
- holographic_deliberate.py +194 -0
- holographic_deltachain.py +205 -0
- holographic_dendrite.py +213 -0
- holographic_denoise.py +285 -0
- holographic_denoisehome.py +131 -0
- holographic_determinism.py +117 -0
- holographic_dictionary.py +265 -0
- holographic_diffuse.py +109 -0
- holographic_diffusion.py +149 -0
- holographic_directed.py +138 -0
- holographic_dirtyfield.py +128 -0
- holographic_dispatch.py +199 -0
- holographic_displace.py +167 -0
- holographic_distbus.py +208 -0
- holographic_distribute.py +250 -0
- holographic_domecache.py +194 -0
- holographic_downscale.py +131 -0
- holographic_dream.py +91 -0
- holographic_drives.py +221 -0
- holographic_dynamics.py +99 -0
- holographic_elements.py +217 -0
- holographic_em.py +188 -0
- holographic_emergence.py +279 -0
- holographic_emitter.py +107 -0
- holographic_encoders.py +444 -0
- holographic_encyclopedia.py +171 -0
- holographic_energy.py +198 -0
- holographic_equilibrium.py +170 -0
- holographic_eulerops.py +323 -0
- holographic_extras.py +333 -0
- holographic_farm.py +325 -0
- holographic_fft.py +117 -0
- holographic_fhrr.py +169 -0
- holographic_field.py +272 -0
- holographic_fieldeffect.py +192 -0
- holographic_fieldhome.py +119 -0
- holographic_fields.py +688 -0
- holographic_filemap.py +358 -0
- holographic_flatness.py +173 -0
- holographic_flow.py +295 -0
- holographic_fluid.py +290 -0
- holographic_forecast.py +125 -0
- holographic_forward.py +160 -0
- holographic_fountain.py +182 -0
- holographic_fpe.py +194 -0
- holographic_fpefield.py +273 -0
- holographic_fractal.py +169 -0
- holographic_freesurface.py +176 -0
- holographic_fsr.py +171 -0
- holographic_fuse.py +266 -0
- holographic_gas.py +175 -0
- holographic_gbuffer.py +397 -0
- holographic_generate.py +252 -0
- holographic_generation.py +181 -0
- holographic_globalillum.py +123 -0
- holographic_gltf.py +320 -0
- holographic_grainmat.py +157 -0
- holographic_grammar.py +273 -0
- holographic_graph_memory.py +158 -0
- holographic_graphql.py +225 -0
- holographic_graphsignal.py +168 -0
- holographic_groom.py +282 -0
- holographic_grouping.py +148 -0
- holographic_hairshade.py +266 -0
- holographic_hardening.py +265 -0
- holographic_harmonic.py +262 -0
- holographic_heat.py +204 -0
- holographic_history.py +139 -0
- holographic_honesty.py +223 -0
- holographic_hopfield.py +283 -0
- holographic_horizon.py +139 -0
- holographic_hypervector.py +187 -0
- holographic_iht.py +178 -0
- holographic_image.py +344 -0
- holographic_imagecompare.py +182 -0
- holographic_inclusions.py +120 -0
- holographic_index.py +145 -0
- holographic_instancing.py +227 -0
- holographic_integrate.py +152 -0
- holographic_intent.py +135 -0
- holographic_inverserender.py +176 -0
- holographic_iterate.py +191 -0
- holographic_jit.py +222 -0
- holographic_jittersplat.py +105 -0
- holographic_jobs.py +346 -0
- holographic_kan.py +115 -0
- holographic_kde.py +183 -0
- holographic_knowledge.py +225 -0
- holographic_lang.py +154 -0
- holographic_layeredmaterial.py +205 -0
- holographic_ldexplore.py +61 -0
- holographic_lens.py +143 -0
- holographic_levitate.py +172 -0
- holographic_lexicon.py +138 -0
- holographic_lightcache.py +166 -0
- holographic_lightinghome.py +116 -0
- holographic_lights.py +668 -0
- holographic_loadmemory.py +160 -0
- holographic_lod.py +170 -0
- holographic_lookahead.py +155 -0
- holographic_lowdiscrepancy.py +96 -0
- holographic_machine.py +504 -0
- holographic_manifold.py +137 -0
- holographic_market.py +290 -0
- holographic_matbake.py +147 -0
- holographic_matcompile.py +143 -0
- holographic_material.py +298 -0
- holographic_materialdata.py +353 -0
- holographic_materialindex.py +197 -0
- holographic_materialio.py +291 -0
- holographic_matlib.py +741 -0
- holographic_meaning_predict.py +197 -0
- holographic_measure.py +149 -0
- holographic_memoryhome.py +142 -0
- holographic_merge.py +123 -0
- holographic_mesh.py +624 -0
- holographic_meshbridge.py +780 -0
- holographic_meshcurvature.py +255 -0
- holographic_meshgeodesic.py +165 -0
- holographic_meshik.py +154 -0
- holographic_meshpoly.py +228 -0
- holographic_meshqem.py +362 -0
- holographic_meshseam.py +263 -0
- holographic_meshskin.py +138 -0
- holographic_meshsmooth.py +263 -0
- holographic_meshsubdiv.py +248 -0
- holographic_meshtools.py +128 -0
- holographic_meshuv.py +229 -0
- holographic_meshverbs.py +223 -0
- holographic_meshverbs2.py +225 -0
- holographic_metrology.py +213 -0
- holographic_mind.py +311 -0
- holographic_mis.py +108 -0
- holographic_misgen.py +122 -0
- holographic_mixture.py +200 -0
- holographic_mobius.py +102 -0
- holographic_modifier.py +274 -0
- holographic_modulate.py +184 -0
- holographic_moe.py +422 -0
- holographic_mpm.py +255 -0
- holographic_multimaterial.py +151 -0
- holographic_multires.py +91 -0
- holographic_navigator.py +394 -0
- holographic_ndfield.py +337 -0
- holographic_noise.py +246 -0
- holographic_nonnewtonian.py +165 -0
- holographic_nystrom.py +221 -0
- holographic_objectarchive.py +204 -0
- holographic_occlusion.py +358 -0
- holographic_octnormal.py +137 -0
- holographic_octree.py +183 -0
- holographic_opponent.py +215 -0
- holographic_optimize.py +149 -0
- holographic_orchestrator.py +584 -0
- holographic_organizer.py +689 -0
- holographic_overrides.py +99 -0
- holographic_oxidation.py +156 -0
- holographic_pack.py +214 -0
- holographic_param.py +142 -0
- holographic_partition.py +188 -0
- holographic_pathtrace.py +285 -0
- holographic_pattern.py +166 -0
- holographic_peel.py +102 -0
- holographic_perception.py +176 -0
- holographic_phase.py +221 -0
- holographic_phasemorph.py +147 -0
- holographic_photo3d.py +171 -0
- holographic_photos.py +84 -0
- holographic_physics.py +79 -0
- holographic_pipecompile.py +107 -0
- holographic_pipeline.py +744 -0
- holographic_pivot.py +146 -0
- holographic_plan.py +397 -0
- holographic_planshape.py +316 -0
- holographic_pointsplat.py +174 -0
- holographic_postfx.py +380 -0
- holographic_predictive.py +239 -0
- holographic_preview.py +149 -0
- holographic_principal.py +131 -0
- holographic_probesweep.py +302 -0
- holographic_procbridge.py +157 -0
- holographic_procgen.py +222 -0
- holographic_protocol.py +196 -0
- holographic_provenance.py +72 -0
- holographic_prt.py +164 -0
- holographic_quantities.py +437 -0
- holographic_query.py +1528 -0
- holographic_query_concurrency.py +125 -0
- holographic_query_durable.py +148 -0
- holographic_query_graph.py +114 -0
- holographic_query_history.py +221 -0
- holographic_query_programs.py +190 -0
- holographic_queryfolder.py +223 -0
- holographic_querygraph.py +129 -0
- holographic_querylock.py +126 -0
- holographic_queryprog.py +251 -0
- holographic_querytime.py +320 -0
- holographic_radiance.py +183 -0
- holographic_ratedistortion.py +215 -0
- holographic_raycoherence.py +154 -0
- holographic_raydiff.py +155 -0
- holographic_rayindex.py +498 -0
- holographic_raymarch.py +279 -0
- holographic_reanchor.py +91 -0
- holographic_reasoning.py +396 -0
- holographic_recipe.py +205 -0
- holographic_recipeops.py +233 -0
- holographic_recurrent.py +438 -0
- holographic_reference.py +171 -0
- holographic_refine.py +111 -0
- holographic_regimegate.py +75 -0
- holographic_regionfield.py +168 -0
- holographic_registry.py +140 -0
- holographic_relations.py +269 -0
- holographic_relocate.py +142 -0
- holographic_render.py +440 -0
- holographic_renderchannels.py +152 -0
- holographic_rendergraph.py +213 -0
- holographic_reservoir.py +118 -0
- holographic_residency.py +127 -0
- holographic_resolution.py +140 -0
- holographic_resonator.py +121 -0
- holographic_respond.py +119 -0
- holographic_reversible.py +149 -0
- holographic_rns.py +119 -0
- holographic_roomacoustic.py +160 -0
- holographic_route.py +191 -0
- holographic_sampler.py +222 -0
- holographic_sampling.py +133 -0
- holographic_samplinghome.py +106 -0
- holographic_sbc.py +292 -0
- holographic_scalehome.py +126 -0
- holographic_scalenode.py +120 -0
- holographic_scatterlayer.py +110 -0
- holographic_scene.py +516 -0
- holographic_scene_doc.py +408 -0
- holographic_scene_query.py +248 -0
- holographic_scene_render.py +227 -0
- holographic_scene_semantic.py +787 -0
- holographic_scenedelta.py +148 -0
- holographic_scenegraph.py +203 -0
- holographic_schedule.py +386 -0
- holographic_schema.py +813 -0
- holographic_sculpt.py +216 -0
- holographic_sdf.py +500 -0
- holographic_sdf_render.py +181 -0
- holographic_sdfbake.py +133 -0
- holographic_sdfscene.py +162 -0
- holographic_segment.py +148 -0
- holographic_semantic.py +1155 -0
- holographic_sequence.py +211 -0
- holographic_service.py +583 -0
- holographic_session.py +180 -0
- holographic_shadowhome.py +101 -0
- holographic_sharpen.py +92 -0
- holographic_signal_structure.py +132 -0
- holographic_simgraph.py +152 -0
- holographic_simreadout.py +94 -0
- holographic_simulationhome.py +144 -0
- holographic_skills.py +186 -0
- holographic_skindeform.py +181 -0
- holographic_slime.py +390 -0
- holographic_smokepresets.py +158 -0
- holographic_snap.py +130 -0
- holographic_softbody.py +574 -0
- holographic_sparsefield.py +529 -0
- holographic_spatial.py +190 -0
- holographic_spectral.py +463 -0
- holographic_spectralfield.py +265 -0
- holographic_spharm.py +103 -0
- holographic_sphere.py +123 -0
- holographic_splat.py +511 -0
- holographic_splat_archive.py +100 -0
- holographic_splatdensify.py +186 -0
- holographic_splatexport.py +288 -0
- holographic_splatprune.py +186 -0
- holographic_splatsharpen.py +86 -0
- holographic_stageplan.py +114 -0
- holographic_steering.py +114 -0
- holographic_storage.py +127 -0
- holographic_structure.py +173 -0
- holographic_subdivcurve.py +132 -0
- holographic_superposed.py +135 -0
- holographic_superres.py +82 -0
- holographic_superschedule.py +224 -0
- holographic_surface.py +196 -0
- holographic_surfaceint.py +121 -0
- holographic_svg.py +186 -0
- holographic_svgf.py +151 -0
- holographic_symbolic.py +243 -0
- holographic_sync.py +185 -0
- holographic_tear.py +181 -0
- holographic_template.py +161 -0
- holographic_temporal.py +125 -0
- holographic_tensor.py +70 -0
- holographic_terrain.py +157 -0
- holographic_text.py +867 -0
- holographic_texturegraph.py +323 -0
- holographic_texturehome.py +114 -0
- holographic_texturerender.py +286 -0
- holographic_texturesynth.py +204 -0
- holographic_thinfilm.py +184 -0
- holographic_tiling.py +249 -0
- holographic_toolclient.py +97 -0
- holographic_topology.py +287 -0
- holographic_transfer.py +162 -0
- holographic_transform.py +296 -0
- holographic_transformhome.py +168 -0
- holographic_transport.py +145 -0
- holographic_traverse.py +125 -0
- holographic_tree.py +682 -0
- holographic_twolayer.py +108 -0
- holographic_typed.py +151 -0
- holographic_unified.py +9747 -0
- holographic_uri.py +224 -0
- holographic_valuehead.py +392 -0
- holographic_verify.py +153 -0
- holographic_video.py +151 -0
- holographic_viewlut.py +102 -0
- holographic_vision.py +582 -0
- holographic_voidsynth.py +108 -0
- holographic_volint.py +166 -0
- holographic_wave.py +169 -0
- holographic_waveadaptive.py +250 -0
- holographic_wavepacket.py +230 -0
- holographic_word_index.py +173 -0
- holographic_workspace.py +199 -0
- holographic_world.py +153 -0
- holographic_wos.py +173 -0
- lecore.py +130 -0
- lecore_data/__init__.py +31 -0
- lecore_data/definitions/README.md +87 -0
- lecore_data/definitions/native/materials/enrich.json +71 -0
- lecore_data/definitions/native/materials/gases.json +138 -0
- lecore_data/definitions/standards/generic_table/minerals_sample.json +20 -0
- lecore_data/knowledge/LICENSE_WORDNET.txt +18 -0
- lecore_data/knowledge/dictionary.json.gz +0 -0
- lecore_data/knowledge/dictionary.json.xz +0 -0
- lecore_data/knowledge/manifest.json +17 -0
- leos_core-0.1.0.dist-info/METADATA +233 -0
- leos_core-0.1.0.dist-info/RECORD +414 -0
- leos_core-0.1.0.dist-info/WHEEL +5 -0
- leos_core-0.1.0.dist-info/licenses/LICENSE +21 -0
- leos_core-0.1.0.dist-info/top_level.txt +401 -0
holographic_ablate.py
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
"""The ablation table (G2): where is VSA actually load-bearing?
|
|
2
|
+
|
|
3
|
+
The README repeatedly, honestly finds the simple baseline ties the holographic one. Taken
|
|
4
|
+
together that poses a question never answered system-wide: which subsystems genuinely need
|
|
5
|
+
superposition / binding / corruption-robustness, and which are a VSA showcase where VSA
|
|
6
|
+
isn't the reason they work? This module answers it the only honest way -- for each
|
|
7
|
+
subsystem, run the DUMBEST honest non-holographic baseline on the SAME task, data, and
|
|
8
|
+
metric, measure both across seeds with the variance harness, and let the confidence
|
|
9
|
+
intervals decide the verdict:
|
|
10
|
+
|
|
11
|
+
holo lower CI > baseline upper CI -> VSA load-bearing (a real win, invest here)
|
|
12
|
+
baseline lower > holo upper CI -> baseline wins (VSA decorative here)
|
|
13
|
+
intervals overlap -> uniformity (within the noise)
|
|
14
|
+
|
|
15
|
+
USE REAL DATA -- the verdicts below run on Reuters, UDHR, and Brown; the algebra-level
|
|
16
|
+
rows (key->value, recall index) are hermetic but at real scale. Run `python
|
|
17
|
+
holographic_ablate.py` to print the live table, or import `ablation_table()`.
|
|
18
|
+
"""
|
|
19
|
+
import math
|
|
20
|
+
import re
|
|
21
|
+
from collections import defaultdict, Counter
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
from holographic_measure import measure
|
|
26
|
+
from holographic_honesty import bh_fdr
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def verdict(holo, base):
|
|
30
|
+
"""Honest verdict from two measure() stat dicts, judged by their 95% CIs."""
|
|
31
|
+
hl, hh = holo["ci"]
|
|
32
|
+
bl, bh = base["ci"]
|
|
33
|
+
if hl > bh:
|
|
34
|
+
return "VSA load-bearing"
|
|
35
|
+
if bl > hh:
|
|
36
|
+
return "baseline wins"
|
|
37
|
+
return "uniformity"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _paired_perm_p(holo_scores, base_scores):
|
|
41
|
+
"""One-sided p-value for 'holo > base' by permutation -- the engine's own threshold-free
|
|
42
|
+
instrument. When the two arms share seeds (equal length) the seeds pair, so it is a
|
|
43
|
+
SIGN-FLIP test on the paired differences (how often a random re-signing reaches the
|
|
44
|
+
observed mean difference). When the seed counts differ (an arm ran fewer seeds) the
|
|
45
|
+
scores cannot pair, so it falls back to a TWO-SAMPLE label-permutation test (how often
|
|
46
|
+
relabelling the pooled scores reaches the observed gap of means). Small families are
|
|
47
|
+
enumerated exactly; larger ones are sampled. No-difference returns 1.0."""
|
|
48
|
+
h = np.asarray(holo_scores, float)
|
|
49
|
+
b = np.asarray(base_scores, float)
|
|
50
|
+
if len(h) == len(b): # paired: sign-flip the differences
|
|
51
|
+
d = h - b
|
|
52
|
+
n = len(d)
|
|
53
|
+
if n == 0 or not np.any(d):
|
|
54
|
+
return 1.0
|
|
55
|
+
obs = d.mean()
|
|
56
|
+
if n <= 20:
|
|
57
|
+
signs = ((np.arange(2 ** n)[:, None] >> np.arange(n)) & 1) * 2 - 1
|
|
58
|
+
return float(((signs * np.abs(d)).mean(axis=1) >= obs - 1e-12).mean())
|
|
59
|
+
rng = np.random.default_rng(0)
|
|
60
|
+
flips = rng.choice([-1.0, 1.0], size=(20000, n))
|
|
61
|
+
return float(((flips * np.abs(d)).mean(axis=1) >= obs - 1e-12).mean())
|
|
62
|
+
# unpaired: permute which pooled scores are labelled 'holo'
|
|
63
|
+
obs = h.mean() - b.mean()
|
|
64
|
+
if obs == 0:
|
|
65
|
+
return 1.0
|
|
66
|
+
pool = np.concatenate([h, b])
|
|
67
|
+
nh = len(h)
|
|
68
|
+
rng = np.random.default_rng(0)
|
|
69
|
+
perms = np.array([rng.permutation(pool) for _ in range(20000)])
|
|
70
|
+
diffs = perms[:, :nh].mean(axis=1) - perms[:, nh:].mean(axis=1)
|
|
71
|
+
return float((diffs >= obs - 1e-12).mean())
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def fdr_verdicts(rows, alpha=0.1):
|
|
75
|
+
"""Apply false-discovery control across the WHOLE ablation family at once.
|
|
76
|
+
|
|
77
|
+
The per-subsystem verdict() decides on that subsystem's own 95% CIs -- but the table is a
|
|
78
|
+
SCAN over many subsystems, and scanning enough of them means one can clear a per-test bar
|
|
79
|
+
by luck. This is the exposure bh_fdr exists for. Each subsystem gets a paired permutation
|
|
80
|
+
p-value (holo > baseline); bh_fdr (Benjamini-Yekutieli, dependent=True -- the subsystems
|
|
81
|
+
share data and methodology) then holds the false-discovery rate among the surviving calls
|
|
82
|
+
at alpha across the family. Returns (augmented_rows, n_load_bearing, n_survive) where each
|
|
83
|
+
augmented row is (name, holo, base, base_name, verdict, p, survives_fdr)."""
|
|
84
|
+
testable = [(i, r) for i, r in enumerate(rows)
|
|
85
|
+
if r[1] is not None and "scores" in r[1] and "scores" in r[2]]
|
|
86
|
+
pvals = [_paired_perm_p(r[1]["scores"], r[2]["scores"]) for _, r in testable]
|
|
87
|
+
reject, _ = bh_fdr(np.array(pvals), alpha=alpha, dependent=True) if pvals \
|
|
88
|
+
else (np.zeros(0, bool), 0)
|
|
89
|
+
survive = {}
|
|
90
|
+
for (idx, _), p, rj in zip(testable, pvals, reject):
|
|
91
|
+
survive[idx] = (float(p), bool(rj))
|
|
92
|
+
out, n_lb, n_surv = [], 0, 0
|
|
93
|
+
for i, r in enumerate(rows):
|
|
94
|
+
p, rj = survive.get(i, (float("nan"), False))
|
|
95
|
+
out.append((*r, p, rj))
|
|
96
|
+
if r[4] == "VSA load-bearing":
|
|
97
|
+
n_lb += 1
|
|
98
|
+
if rj:
|
|
99
|
+
n_surv += 1
|
|
100
|
+
return out, n_lb, n_surv
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# Each subsystem: a holographic score fn and the dumbest honest baseline, on real data.
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
def _reuters_docs():
|
|
108
|
+
from nltk.corpus import reuters
|
|
109
|
+
cats = ["earn", "acq", "crude", "trade", "money-fx"]
|
|
110
|
+
docs = {c: [] for c in cats}
|
|
111
|
+
for f in reuters.fileids():
|
|
112
|
+
cs = reuters.categories(f)
|
|
113
|
+
if len(cs) == 1 and cs[0] in cats and len(docs[cs[0]]) < 60:
|
|
114
|
+
toks = [w.lower() for w in reuters.words(f) if w.isalpha()][:120]
|
|
115
|
+
if len(toks) > 20:
|
|
116
|
+
docs[cs[0]].append(toks)
|
|
117
|
+
return cats, [(t, c) for c in cats for t in docs[c]]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _reuters_split(alldocs, seed):
|
|
121
|
+
rng = np.random.default_rng(seed)
|
|
122
|
+
items = list(alldocs); rng.shuffle(items)
|
|
123
|
+
by = {}
|
|
124
|
+
for t, c in items:
|
|
125
|
+
by.setdefault(c, []).append(t)
|
|
126
|
+
tr, te = [], []
|
|
127
|
+
for c, ts in by.items():
|
|
128
|
+
k = int(len(ts) * 0.7)
|
|
129
|
+
tr += [(t, c) for t in ts[:k]]
|
|
130
|
+
te += [(t, c) for t in ts[k:]]
|
|
131
|
+
return tr, te
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def topic_classify(seeds=range(6)):
|
|
135
|
+
"""Holographic UnifiedMind classify vs bag-of-words nearest centroid (Reuters)."""
|
|
136
|
+
cats, alldocs = _reuters_docs()
|
|
137
|
+
vocab = sorted({w for t, _ in alldocs for w in t})
|
|
138
|
+
wi = {w: i for i, w in enumerate(vocab)}
|
|
139
|
+
|
|
140
|
+
def holo(seed):
|
|
141
|
+
from holographic_unified import UnifiedMind
|
|
142
|
+
tr, te = _reuters_split(alldocs, seed)
|
|
143
|
+
m = UnifiedMind(dim=1024, seed=seed)
|
|
144
|
+
m.absorb([(" ".join(t), c) for t, c in tr])
|
|
145
|
+
return np.mean([m.classify(" ".join(t))[0] == c for t, c in te])
|
|
146
|
+
|
|
147
|
+
def bow(seed):
|
|
148
|
+
tr, te = _reuters_split(alldocs, seed)
|
|
149
|
+
|
|
150
|
+
def vec(t):
|
|
151
|
+
v = np.zeros(len(vocab))
|
|
152
|
+
for w in t:
|
|
153
|
+
if w in wi:
|
|
154
|
+
v[wi[w]] += 1
|
|
155
|
+
n = np.linalg.norm(v)
|
|
156
|
+
return v / n if n else v
|
|
157
|
+
cents = {c: np.mean([vec(t) for t, cc in tr if cc == c], axis=0) for c in cats}
|
|
158
|
+
return np.mean([max(cents, key=lambda C: np.dot(vec(t), cents[C])) == c for t, c in te])
|
|
159
|
+
|
|
160
|
+
return measure(holo, seeds), measure(bow, seeds), "bag-of-words centroid"
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def language_id(seeds=range(6)):
|
|
164
|
+
"""Holographic trigram-binding profiles vs bag-of-trigrams centroid (UDHR)."""
|
|
165
|
+
from nltk.corpus import udhr
|
|
166
|
+
files = {"en": "English-Latin1", "fr": "French_Francais-Latin1",
|
|
167
|
+
"de": "German_Deutsch-Latin1", "es": "Spanish_Espanol-Latin1",
|
|
168
|
+
"it": "Italian_Italiano-Latin1", "nl": "Dutch_Nederlands-Latin1"}
|
|
169
|
+
texts = {k: re.sub(r"[^a-z ]+", " ", udhr.raw(f).lower()) for k, f in files.items()}
|
|
170
|
+
|
|
171
|
+
def split(seed):
|
|
172
|
+
rng = np.random.default_rng(seed); tr, te = {}, []
|
|
173
|
+
for k, full in texts.items():
|
|
174
|
+
ch = [full[i:i + 200] for i in range(0, len(full) - 200, 200)]
|
|
175
|
+
rng.shuffle(ch); c = int(len(ch) * 0.6)
|
|
176
|
+
tr[k] = ch[:c]; te += [(x, k) for x in ch[c:]]
|
|
177
|
+
return tr, te
|
|
178
|
+
|
|
179
|
+
def holo(seed):
|
|
180
|
+
from holographic_text import LanguageID
|
|
181
|
+
tr, te = split(seed)
|
|
182
|
+
lid = LanguageID(dim=512, seed=seed).fit(tr)
|
|
183
|
+
return np.mean([lid.identify(x) == k for x, k in te])
|
|
184
|
+
|
|
185
|
+
def bag(seed):
|
|
186
|
+
tr, te = split(seed)
|
|
187
|
+
tris = sorted({s[i:i + 3] for k in tr for s in tr[k] for i in range(len(s) - 2)})
|
|
188
|
+
ti = {t: i for i, t in enumerate(tris)}
|
|
189
|
+
|
|
190
|
+
def vec(s):
|
|
191
|
+
v = np.zeros(len(tris))
|
|
192
|
+
for i in range(len(s) - 2):
|
|
193
|
+
if s[i:i + 3] in ti:
|
|
194
|
+
v[ti[s[i:i + 3]]] += 1
|
|
195
|
+
n = np.linalg.norm(v)
|
|
196
|
+
return v / n if n else v
|
|
197
|
+
cents = {k: np.mean([vec(s) for s in tr[k]], axis=0) for k in tr}
|
|
198
|
+
return np.mean([max(cents, key=lambda K: np.dot(vec(x), cents[K])) == k for x, k in te])
|
|
199
|
+
|
|
200
|
+
return measure(holo, seeds), measure(bag, seeds), "bag-of-trigrams centroid"
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def segmentation(seeds=range(6)):
|
|
204
|
+
"""Holographic branching entropy vs EXACT count-based branching entropy (Brown)."""
|
|
205
|
+
from nltk.corpus import brown
|
|
206
|
+
from holographic_segment import Segmenter, boundary_f1
|
|
207
|
+
words = [w.lower() for w in brown.words(categories="news") if w.isalpha()][:1500]
|
|
208
|
+
spaceless = "".join(words); truth, pos = set(), -1
|
|
209
|
+
for w in words:
|
|
210
|
+
pos += len(w); truth.add(pos)
|
|
211
|
+
|
|
212
|
+
def holo(seed):
|
|
213
|
+
seg = Segmenter(dim=512, order=3, seed=seed).fit(spaceless)
|
|
214
|
+
return boundary_f1(seg.boundaries(spaceless, 70), truth)["f1"]
|
|
215
|
+
|
|
216
|
+
def exact(seed):
|
|
217
|
+
K = 3; nxt = defaultdict(Counter)
|
|
218
|
+
for i in range(1, len(spaceless)):
|
|
219
|
+
nxt[spaceless[max(0, i - K):i]][spaceless[i]] += 1
|
|
220
|
+
H = []
|
|
221
|
+
for i in range(1, len(spaceless)):
|
|
222
|
+
c = nxt[spaceless[max(0, i - K):i]]; tot = sum(c.values())
|
|
223
|
+
H.append(-sum((v / tot) * math.log2(v / tot) for v in c.values()) if tot else 0.0)
|
|
224
|
+
H = np.array(H); thr = np.percentile(H, 70)
|
|
225
|
+
pred = set(i for i in range(1, len(H) - 1)
|
|
226
|
+
if H[i] >= thr and H[i] >= H[i - 1] and H[i] >= H[i + 1])
|
|
227
|
+
return boundary_f1(pred, truth)["f1"]
|
|
228
|
+
|
|
229
|
+
return measure(holo, seeds), measure(exact, range(3)), "exact count entropy"
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def key_value_noisy(seeds=range(6), noise=0.5):
|
|
233
|
+
"""VSA bind/bundle store with cosine cleanup vs an exact Python dict, on NOISY keys
|
|
234
|
+
(the realistic case -- keys are encoded percepts, never bit-exact). The dict can't
|
|
235
|
+
match a perturbed key at all; cosine cleanup still recovers the value."""
|
|
236
|
+
from holographic_ai import bind, unbind, cosine
|
|
237
|
+
|
|
238
|
+
def setup(seed):
|
|
239
|
+
rng = np.random.default_rng(seed); dim, n = 512, 30
|
|
240
|
+
|
|
241
|
+
def rv():
|
|
242
|
+
v = rng.standard_normal(dim); return v / np.linalg.norm(v)
|
|
243
|
+
keys = [rv() for _ in range(n)]
|
|
244
|
+
vals = [rv() for _ in range(n)]
|
|
245
|
+
return rng, dim, n, keys, vals
|
|
246
|
+
|
|
247
|
+
def holo(seed):
|
|
248
|
+
rng, dim, n, keys, vals = setup(seed)
|
|
249
|
+
trace = np.sum([bind(k, v) for k, v in zip(keys, vals)], axis=0)
|
|
250
|
+
ok = 0
|
|
251
|
+
for i in range(n):
|
|
252
|
+
q = keys[i] + noise * (lambda v: v / np.linalg.norm(v))(rng.standard_normal(dim))
|
|
253
|
+
q /= np.linalg.norm(q)
|
|
254
|
+
est = unbind(trace, q)
|
|
255
|
+
ok += (int(np.argmax([cosine(est, v) for v in vals])) == i)
|
|
256
|
+
return ok / n
|
|
257
|
+
|
|
258
|
+
def dct(seed):
|
|
259
|
+
rng, dim, n, keys, vals = setup(seed)
|
|
260
|
+
D = {k.tobytes(): i for i, k in enumerate(keys)}
|
|
261
|
+
ok = 0
|
|
262
|
+
for i in range(n):
|
|
263
|
+
q = keys[i] + noise * (lambda v: v / np.linalg.norm(v))(rng.standard_normal(dim))
|
|
264
|
+
q /= np.linalg.norm(q)
|
|
265
|
+
ok += (D.get(q.tobytes(), -1) == i) # exact hash never matches a noisy key
|
|
266
|
+
return ok / n
|
|
267
|
+
|
|
268
|
+
return measure(holo, seeds), measure(dct, seeds), "exact dict (noisy keys)"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def recall_index(seeds=range(5)):
|
|
272
|
+
"""HoloForest approximate recall vs exact brute-force scan. The forest LOSES on raw
|
|
273
|
+
recall (exact scan is trivially 1.0) but reaches its recall at a FRACTION of the
|
|
274
|
+
comparisons -- so this row's honest reading is 'scale, not accuracy', reported with
|
|
275
|
+
the comparison fraction alongside."""
|
|
276
|
+
from holographic_tree import HoloForest
|
|
277
|
+
|
|
278
|
+
def make(seed, N=2000, dim=128):
|
|
279
|
+
rng = np.random.default_rng(seed)
|
|
280
|
+
items = rng.standard_normal((N, dim)); items /= np.linalg.norm(items, axis=1, keepdims=True)
|
|
281
|
+
queries = []
|
|
282
|
+
for _ in range(120):
|
|
283
|
+
i = int(rng.integers(N))
|
|
284
|
+
q = items[i] + 0.15 * rng.standard_normal(dim); q /= np.linalg.norm(q)
|
|
285
|
+
queries.append(q)
|
|
286
|
+
return items, queries
|
|
287
|
+
|
|
288
|
+
fracs = []
|
|
289
|
+
|
|
290
|
+
def holo(seed):
|
|
291
|
+
items, queries = make(seed)
|
|
292
|
+
F = HoloForest(items.shape[1], n_trees=4, leaf_size=64, seed=seed).build(items)
|
|
293
|
+
ok, comps = 0, []
|
|
294
|
+
for q in queries:
|
|
295
|
+
true = int((items @ q).argmax())
|
|
296
|
+
ok += (F.recall(q, beam=4) == true)
|
|
297
|
+
comps.append(F.last_comparisons)
|
|
298
|
+
fracs.append(np.mean(comps) / len(items))
|
|
299
|
+
return ok / len(queries)
|
|
300
|
+
|
|
301
|
+
def scan(seed):
|
|
302
|
+
return 1.0 # exact scan: trivially perfect, 100% cost
|
|
303
|
+
|
|
304
|
+
h = measure(holo, seeds)
|
|
305
|
+
b = measure(scan, range(3))
|
|
306
|
+
h["comparison_fraction"] = float(np.mean(fracs))
|
|
307
|
+
return h, b, "exact scan (100% comparisons)"
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
SUBSYSTEMS = [
|
|
311
|
+
("topic classify (Reuters)", topic_classify),
|
|
312
|
+
("language ID (UDHR)", language_id),
|
|
313
|
+
("segmentation (Brown)", segmentation),
|
|
314
|
+
("key->value, noisy keys", key_value_noisy),
|
|
315
|
+
("recall index (forest)", recall_index),
|
|
316
|
+
]
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def ablation_table(seeds=range(6)):
|
|
320
|
+
"""Run every subsystem's ablation and return a list of rows:
|
|
321
|
+
(name, holo_stats, base_stats, baseline_name, verdict)."""
|
|
322
|
+
rows = []
|
|
323
|
+
for name, fn in SUBSYSTEMS:
|
|
324
|
+
try:
|
|
325
|
+
h, b, base_name = fn()
|
|
326
|
+
rows.append((name, h, b, base_name, verdict(h, b)))
|
|
327
|
+
except Exception as e:
|
|
328
|
+
rows.append((name, None, None, str(e), "skipped"))
|
|
329
|
+
return rows
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _demo():
|
|
333
|
+
rows = ablation_table()
|
|
334
|
+
aug, n_lb, n_surv = fdr_verdicts(rows, alpha=0.1)
|
|
335
|
+
print("ABLATION TABLE -- is VSA load-bearing? (holographic vs dumbest honest baseline,")
|
|
336
|
+
print("real data, judged by 95% CIs from the variance harness)\n")
|
|
337
|
+
print(f" {'subsystem':28} {'holo':>14} {'baseline':>14} {'verdict':18} {'p':>7} {'FDR':>9}")
|
|
338
|
+
for name, h, b, base_name, v, p, rj in aug:
|
|
339
|
+
if h is None:
|
|
340
|
+
print(f" {name:28} (skipped: {base_name})")
|
|
341
|
+
continue
|
|
342
|
+
extra = ""
|
|
343
|
+
if "comparison_fraction" in h:
|
|
344
|
+
extra = f" [forest @ {h['comparison_fraction']*100:.0f}% comparisons -> the win is SCALE]"
|
|
345
|
+
tag = "survives" if rj else ("LUCK?" if v == "VSA load-bearing" else "--")
|
|
346
|
+
print(f" {name:28} {h['mean']:.3f}+/-{h['std']:.3f} {b['mean']:.3f}+/-{b['std']:.3f} "
|
|
347
|
+
f"{v:18} {p:7.4f} {tag:>9}{extra}")
|
|
348
|
+
print(f"\n{n_surv}/{n_lb} 'load-bearing' verdicts survive family-wise false-discovery control "
|
|
349
|
+
f"(BH-Yekutieli, alpha=0.1)")
|
|
350
|
+
print("Reading: 'VSA load-bearing' = superposition/binding/cleanup is the reason it works;")
|
|
351
|
+
print("'uniformity' = the simple baseline ties it (the idea works, not the VSA encoding);")
|
|
352
|
+
print("'baseline wins' = VSA is decorative here (but the forest row buys sublinear SCALE).")
|
|
353
|
+
print("FDR: the per-test CI decides one subsystem; the p-column + FDR judge the WHOLE scan,")
|
|
354
|
+
print("so a verdict that cleared its own CI by luck across many subsystems is caught.")
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
if __name__ == "__main__":
|
|
358
|
+
_demo()
|
holographic_access.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""holographic_access.py -- who may READ what: invite-gating + per-principal read grants.
|
|
2
|
+
|
|
3
|
+
leCore's database already enforces "write only your OWN namespace" at one chokepoint (query._require_writable), with
|
|
4
|
+
cross-namespace READS always allowed -- fine for a TRUSTED host. The moment others connect (guests joining a workspace,
|
|
5
|
+
a public farm), that read-any default is too open. This adds the symmetric, stricter twin for guests:
|
|
6
|
+
|
|
7
|
+
* per-principal READ GRANTS -- a guest is granted read access to SPECIFIC namespaces, and sees NOTHING else.
|
|
8
|
+
Default is nothing granted, so "choose what to share" is an explicit act, not the current read-any default.
|
|
9
|
+
* require_readable(principal, namespace) -- the read chokepoint mirroring _require_writable: raise unless the
|
|
10
|
+
namespace is in the principal's read grants.
|
|
11
|
+
* invites -- a host hands a guest an INVITE (a token) that admits it as a given kind with some initial grants;
|
|
12
|
+
redeeming the invite makes a Principal with exactly those grants and nothing more.
|
|
13
|
+
|
|
14
|
+
Writes stay own-namespace-only (unchanged). This is opt-in: a trusted host that wants read-any simply doesn't call
|
|
15
|
+
require_readable. numpy-free; stdlib only (secrets for unguessable tokens).
|
|
16
|
+
"""
|
|
17
|
+
import secrets
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AccessError(Exception):
|
|
21
|
+
"""Raised when a principal tries to read a namespace it hasn't been granted."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _read_grants(principal):
|
|
25
|
+
"""The principal's set of granted-readable namespaces (creating the container if missing)."""
|
|
26
|
+
grants = getattr(principal, "grants", None)
|
|
27
|
+
if grants is None:
|
|
28
|
+
grants = {}
|
|
29
|
+
setattr(principal, "grants", grants)
|
|
30
|
+
return grants.setdefault("read", set())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _as_list(x):
|
|
34
|
+
if x is None:
|
|
35
|
+
return []
|
|
36
|
+
return [x] if isinstance(x, str) else list(x)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def grant(principal, read=None):
|
|
40
|
+
"""Grant this principal READ access to a namespace (or list of namespaces). Returns the principal (chainable)."""
|
|
41
|
+
g = _read_grants(principal)
|
|
42
|
+
for ns in _as_list(read):
|
|
43
|
+
g.add(ns)
|
|
44
|
+
return principal
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def revoke(principal, read=None):
|
|
48
|
+
"""Stop sharing: remove READ access to a namespace (or list). Returns the principal."""
|
|
49
|
+
g = _read_grants(principal)
|
|
50
|
+
for ns in _as_list(read):
|
|
51
|
+
g.discard(ns)
|
|
52
|
+
return principal
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def can_read(principal, namespace):
|
|
56
|
+
"""True iff this principal has been granted read access to `namespace`. Its own namespace is always readable."""
|
|
57
|
+
if namespace == getattr(principal, "namespace_name", None):
|
|
58
|
+
return True
|
|
59
|
+
return namespace in _read_grants(principal)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def require_readable(principal, namespace):
|
|
63
|
+
"""The read chokepoint -- the symmetric twin of the DB's _require_writable. Raises AccessError unless `namespace`
|
|
64
|
+
is readable by `principal` (granted, or the principal's own). A guest sees nothing until the host grants it."""
|
|
65
|
+
if not can_read(principal, namespace):
|
|
66
|
+
raise AccessError("%s is not shared with %s" % (namespace, getattr(principal, "id", principal)))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class Invite:
|
|
70
|
+
"""A one-token admission: it says a guest may connect as `kind` and, on redemption, starts with these read grants
|
|
71
|
+
(and nothing else). The host creates it and hands the guest the .code; redeeming binds it to a specific actor id.
|
|
72
|
+
Single-use by default -- redeeming marks it used so a leaked code can't admit a crowd."""
|
|
73
|
+
|
|
74
|
+
def __init__(self, kind="user", grants=None, code=None, single_use=True):
|
|
75
|
+
self.kind = kind
|
|
76
|
+
self.read = set(_as_list((grants or {}).get("read")))
|
|
77
|
+
self.code = code or secrets.token_hex(8)
|
|
78
|
+
self.single_use = single_use
|
|
79
|
+
self.redeemed_by = None
|
|
80
|
+
|
|
81
|
+
def is_valid(self):
|
|
82
|
+
return not (self.single_use and self.redeemed_by is not None)
|
|
83
|
+
|
|
84
|
+
def __repr__(self):
|
|
85
|
+
return "Invite(kind=%r, code=%r, read=%s)" % (self.kind, self.code, sorted(self.read))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def invite(kind="user", grants=None, code=None):
|
|
89
|
+
"""Make an invite token admitting a guest as `kind` with initial read `grants` (e.g. {'read': ['lab/scene']})."""
|
|
90
|
+
return Invite(kind=kind, grants=grants, code=code)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def apply_invite(principal, inv):
|
|
94
|
+
"""Redeem an invite onto a freshly created principal: copy its kind's grants over and mark the invite used."""
|
|
95
|
+
for ns in inv.read:
|
|
96
|
+
grant(principal, read=ns)
|
|
97
|
+
if inv.single_use:
|
|
98
|
+
inv.redeemed_by = getattr(principal, "id", principal)
|
|
99
|
+
return principal
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _selftest():
|
|
103
|
+
# a stand-in principal with an own-namespace and a grants container
|
|
104
|
+
class _P:
|
|
105
|
+
def __init__(self, i, ns):
|
|
106
|
+
self.id, self.namespace_name, self.grants = i, ns, {"read": set()}
|
|
107
|
+
|
|
108
|
+
guest = _P("guest", "ws:lab/user:guest")
|
|
109
|
+
|
|
110
|
+
# default: a guest sees NOTHING but its own namespace
|
|
111
|
+
assert can_read(guest, guest.namespace_name) # own namespace always readable
|
|
112
|
+
assert not can_read(guest, "ws:lab/user:alice")
|
|
113
|
+
try:
|
|
114
|
+
require_readable(guest, "lab/scene")
|
|
115
|
+
assert False, "should have raised"
|
|
116
|
+
except AccessError:
|
|
117
|
+
pass
|
|
118
|
+
|
|
119
|
+
# grant one namespace -> now readable; others still not
|
|
120
|
+
grant(guest, read="lab/scene")
|
|
121
|
+
require_readable(guest, "lab/scene") # no raise
|
|
122
|
+
assert can_read(guest, "lab/scene") and not can_read(guest, "lab/notes")
|
|
123
|
+
|
|
124
|
+
# grant a list, then revoke one
|
|
125
|
+
grant(guest, read=["lab/notes", "lab/props"])
|
|
126
|
+
assert can_read(guest, "lab/notes") and can_read(guest, "lab/props")
|
|
127
|
+
revoke(guest, read="lab/notes")
|
|
128
|
+
assert not can_read(guest, "lab/notes") and can_read(guest, "lab/props")
|
|
129
|
+
|
|
130
|
+
# an invite carries initial grants and is single-use
|
|
131
|
+
inv = invite(kind="user", grants={"read": ["lab/scene", "lab/props"]}, code="TESTCODE")
|
|
132
|
+
assert inv.code == "TESTCODE" and inv.kind == "user" and inv.is_valid()
|
|
133
|
+
newcomer = _P("newcomer", "ws:lab/user:newcomer")
|
|
134
|
+
apply_invite(newcomer, inv)
|
|
135
|
+
assert can_read(newcomer, "lab/scene") and can_read(newcomer, "lab/props")
|
|
136
|
+
assert not inv.is_valid() # single-use: now spent
|
|
137
|
+
|
|
138
|
+
print("OK: holographic_access self-test passed (a guest reads only its own namespace by default; require_readable "
|
|
139
|
+
"raises until granted; grant/revoke share and un-share specific namespaces; an invite confers initial grants "
|
|
140
|
+
"and is single-use)")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
if __name__ == "__main__":
|
|
144
|
+
_selftest()
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Robust accumulation -- harmonic-weight averaging and firefly clamping for the engine's averaging paths.
|
|
2
|
+
|
|
3
|
+
WHY THIS EXISTS
|
|
4
|
+
---------------
|
|
5
|
+
Wherever the engine accumulates noisy estimates of one quantity -- consolidation over a growing store, HoloForest
|
|
6
|
+
vote-averaging across trees, any iterate-and-average -- two cheap, standard fixes from the rendering world buy
|
|
7
|
+
real robustness:
|
|
8
|
+
|
|
9
|
+
ACCUM-2, harmonic weights (TAA's lesson). A fixed-alpha exponential blend `x <- (1-a)x + a*sample` never fully
|
|
10
|
+
converges: it keeps forgetting old samples, so its variance plateaus at a*sigma^2/(2-a) instead of falling to
|
|
11
|
+
zero. The harmonic (1/n) running average `x <- x + (sample - x)/n` weights every sample seen so far equally and
|
|
12
|
+
converges -- measured error keeps dropping with N while the EMA flatlines. The honest CAVEAT: harmonic is right
|
|
13
|
+
for a STATIONARY target; for a DRIFTING one the forgetful EMA tracks better (measured), so `schedule='ema'` stays
|
|
14
|
+
available for that case.
|
|
15
|
+
|
|
16
|
+
ACCUM-3, firefly clamping (V-Ray's adaptivity clamp / TAA history rectification). A single outlier estimate -- a
|
|
17
|
+
firefly recall or a bad vote with a huge magnitude -- skews a mean badly. Clamping (winsorizing) each sample's
|
|
18
|
+
deviation from a ROBUST centre (the median) to k robust-scales keeps one bad estimate from dominating: measured
|
|
19
|
+
~100x lower error under injected fireflies, with NO loss on clean data.
|
|
20
|
+
|
|
21
|
+
The two compose: `robust_accumulate(samples, schedule='harmonic', clamp_k=2.5)` clamps the fireflies, then
|
|
22
|
+
averages with harmonic weights.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _winsorize(X, k):
|
|
29
|
+
"""Clamp each sample's deviation from the median to k robust-scales (the median deviation), in place on a
|
|
30
|
+
copy. Works on scalar samples (X shape (N,)) or vector samples (X shape (N, D, ...))."""
|
|
31
|
+
A = np.array(X, float)
|
|
32
|
+
c = np.median(A, axis=0)
|
|
33
|
+
dev = A - c
|
|
34
|
+
nrm = np.linalg.norm(dev.reshape(len(A), -1), axis=1) # deviation magnitude per sample
|
|
35
|
+
scale = np.median(nrm) + 1e-9 # robust scale: the median deviation
|
|
36
|
+
over = nrm > k * scale
|
|
37
|
+
factor = np.ones(len(A))
|
|
38
|
+
factor[over] = k * scale / nrm[over] # shrink only the outliers
|
|
39
|
+
return c + dev * factor.reshape((-1,) + (1,) * (A.ndim - 1))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def robust_accumulate(samples, schedule="harmonic", alpha=0.2, clamp_k=None):
|
|
43
|
+
"""Average a sequence of noisy estimates of one quantity, robustly. `schedule='harmonic'` uses 1/n weights
|
|
44
|
+
(ACCUM-2: converges, best for a STATIONARY target); `'ema'` uses a fixed-alpha exponential blend (best for a
|
|
45
|
+
DRIFTING target); `'mean'` is the plain mean. `clamp_k` (ACCUM-3), if set, winsorizes outlier samples to
|
|
46
|
+
clamp_k robust-scales from the median BEFORE accumulating, so one firefly can't dominate. Samples may be
|
|
47
|
+
scalars or vectors."""
|
|
48
|
+
X = np.array([np.asarray(s, float) for s in samples], float)
|
|
49
|
+
if clamp_k is not None:
|
|
50
|
+
X = _winsorize(X, clamp_k)
|
|
51
|
+
if schedule == "harmonic":
|
|
52
|
+
acc = np.zeros_like(X[0])
|
|
53
|
+
for n, s in enumerate(X, 1):
|
|
54
|
+
acc = acc + (s - acc) / n # 1/n running average -- converges
|
|
55
|
+
return acc
|
|
56
|
+
if schedule == "ema":
|
|
57
|
+
acc = X[0].copy()
|
|
58
|
+
for s in X[1:]:
|
|
59
|
+
acc = (1 - alpha) * acc + alpha * s # fixed-alpha blend -- tracks drift, plateaus
|
|
60
|
+
return acc
|
|
61
|
+
return X.mean(0)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def harmonic_accumulate(samples):
|
|
65
|
+
"""ACCUM-2 convenience: the 1/n running average (converges on a stationary target)."""
|
|
66
|
+
return robust_accumulate(samples, schedule="harmonic")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def clamped_accumulate(samples, k=2.5):
|
|
70
|
+
"""ACCUM-3 convenience: firefly-clamped mean (winsorize outliers to k robust-scales from the median)."""
|
|
71
|
+
return robust_accumulate(samples, schedule="mean", clamp_k=k)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _selftest():
|
|
75
|
+
"""CI-fast: ACCUM-2 (harmonic converges where the EMA plateaus, with the drift caveat) and ACCUM-3 (clamping
|
|
76
|
+
is robust to fireflies with no loss on clean data) on noisy vector streams."""
|
|
77
|
+
rng = np.random.default_rng(0)
|
|
78
|
+
D = 128
|
|
79
|
+
def cos(a, b):
|
|
80
|
+
return float(a @ b / ((np.linalg.norm(a) * np.linalg.norm(b)) + 1e-12))
|
|
81
|
+
mu = rng.standard_normal(D); mu /= np.linalg.norm(mu)
|
|
82
|
+
|
|
83
|
+
# ACCUM-2: stationary stream -- harmonic converges (error falls with N), EMA plateaus
|
|
84
|
+
def stream(n, drift=0.0):
|
|
85
|
+
out = []; m = mu.copy()
|
|
86
|
+
for _ in range(n):
|
|
87
|
+
m = m + drift * rng.standard_normal(D) / np.sqrt(D)
|
|
88
|
+
out.append(m + 0.8 * rng.standard_normal(D) / np.sqrt(D))
|
|
89
|
+
return out, m
|
|
90
|
+
s50, _ = stream(50); s800, _ = stream(800)
|
|
91
|
+
e_h50 = 1 - cos(harmonic_accumulate(s50), mu)
|
|
92
|
+
e_h800 = 1 - cos(harmonic_accumulate(s800), mu)
|
|
93
|
+
e_ema800 = 1 - cos(robust_accumulate(s800, schedule="ema", alpha=0.2), mu)
|
|
94
|
+
assert e_h800 < e_h50 * 0.5, (e_h800, e_h50) # harmonic CONVERGES (error keeps falling with N)
|
|
95
|
+
assert e_h800 < e_ema800 * 0.5, (e_h800, e_ema800) # and beats the plateauing EMA on a stationary stream
|
|
96
|
+
|
|
97
|
+
# ACCUM-2 caveat: on a DRIFTING target the forgetful EMA tracks better than harmonic
|
|
98
|
+
sd, mfin = stream(400, drift=0.03)
|
|
99
|
+
e_hd = 1 - cos(harmonic_accumulate(sd), mfin)
|
|
100
|
+
e_emad = 1 - cos(robust_accumulate(sd, schedule="ema", alpha=0.2), mfin)
|
|
101
|
+
assert e_emad < e_hd, (e_emad, e_hd) # kept caveat: harmonic is for STATIONARY targets
|
|
102
|
+
|
|
103
|
+
# ACCUM-3: firefly clamping -- robust under outliers, no loss on clean
|
|
104
|
+
clean = [mu + 0.3 * rng.standard_normal(D) / np.sqrt(D) for _ in range(50)]
|
|
105
|
+
truth = np.mean(clean, 0)
|
|
106
|
+
fire = clean + [8.0 * rng.standard_normal(D) / np.sqrt(D) for _ in range(5)]
|
|
107
|
+
e_plain = 1 - cos(np.mean(fire, 0), truth)
|
|
108
|
+
e_clamp = 1 - cos(clamped_accumulate(fire), truth)
|
|
109
|
+
assert e_clamp < e_plain * 0.25, (e_clamp, e_plain) # clamping is far more robust to fireflies
|
|
110
|
+
assert 1 - cos(clamped_accumulate(clean), truth) < 1e-6 # and costs nothing on clean data
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
_selftest()
|
|
115
|
+
print("holographic_accumulate selftest passed")
|