specmem-hardwicksoftware 3.7.44 → 3.7.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -13
- package/container/Dockerfile +110 -70
- package/container/container-version.json +5 -0
- package/embedding-sandbox/warm-start.sh +100 -12
- package/package.json +1 -1
- package/scripts/global-postinstall.cjs +53 -14
- package/scripts/specmem-init.cjs +43 -11
- package/specmem/supervisord.conf +1 -1
- package/specmem/user-config.json +2 -2
package/README.md
CHANGED
|
@@ -1,18 +1,13 @@
|
|
|
1
|
+
## I am aware it currently only works for "root" users in it's full glory
|
|
2
|
+
## Sorry, this was developed on a baremetal VPS with XFCE Desktop
|
|
3
|
+
## Just sudo npm install -g specmem-hardwicksoftware && cd /yourprojectdir && specmem init
|
|
4
|
+
## Sorry guys again working out numerous kinks, look for assistance getting this user ready
|
|
5
|
+
## Hit up a new issue if you find any please, send me your logs on linkedin or smthn!
|
|
1
6
|
|
|
2
|
-
## COMPACTION PROXY FIXED - AGENT BEHAVIOR IMPROVEMENT SOON!
|
|
3
7
|
<!-- How to Install SVG -->
|
|
4
8
|
<img src="./svg-sections/readme-how-to-install.svg" alt="How to Install SpecMem on Linux" width="800">
|
|
5
9
|
|
|
6
|
-
<br/>
|
|
7
10
|
|
|
8
|
-
<!-- Token Compaction Pipeline SVG -->
|
|
9
|
-
<div align="center">
|
|
10
|
-
<picture>
|
|
11
|
-
<img alt="Token Compaction Pipeline — ~60% Input Token Reduction" src="./svg-sections/readme-token-compaction.svg" width="800">
|
|
12
|
-
</picture>
|
|
13
|
-
</div>
|
|
14
|
-
|
|
15
|
-
<br/>
|
|
16
11
|
|
|
17
12
|
<!-- Hero Banner -->
|
|
18
13
|
<picture>
|
|
@@ -133,6 +128,16 @@ Full technical deep-dive covering architecture, embeddings, database internals,
|
|
|
133
128
|
|
|
134
129
|
</div>
|
|
135
130
|
|
|
131
|
+
<br/>
|
|
132
|
+
|
|
133
|
+
<!-- Token Compaction Pipeline SVG -->
|
|
134
|
+
<div align="center">
|
|
135
|
+
<picture>
|
|
136
|
+
<img alt="Token Compaction Pipeline — ~60% Input Token Reduction" src="./svg-sections/readme-token-compaction.svg" width="800">
|
|
137
|
+
</picture>
|
|
138
|
+
</div>
|
|
139
|
+
|
|
140
|
+
<br/>
|
|
136
141
|
---
|
|
137
142
|
|
|
138
143
|
## ⚡ Quick Start
|
|
@@ -150,9 +155,9 @@ Full technical deep-dive covering architecture, embeddings, database internals,
|
|
|
150
155
|
## 🔓 Root Access (Optional)
|
|
151
156
|
|
|
152
157
|
<div align="center">
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
158
|
+
Hell nah, sorry, not a poweruser with control over their debian distro?
|
|
159
|
+
Looks like you aren't intelligent enough to run this technology
|
|
160
|
+
(IIRC THIS ACTUALLY PREVENTS CORPORTATE FROM INSTALLING BUT IRDC Plus in MY BOOK )
|
|
156
161
|
</div>
|
|
157
162
|
|
|
158
163
|
---
|
package/container/Dockerfile
CHANGED
|
@@ -1,30 +1,100 @@
|
|
|
1
1
|
# ============================================
|
|
2
|
-
# SPECMEM BRAIN CONTAINER
|
|
2
|
+
# SPECMEM BRAIN CONTAINER — MINIMAL BUILD
|
|
3
3
|
# ============================================
|
|
4
|
-
# Everything SpecMem needs,
|
|
4
|
+
# Everything SpecMem needs, nothing it doesn't:
|
|
5
5
|
# - PostgreSQL 16 + pgvector
|
|
6
6
|
# - Python embedding server (frankenstein-embeddings.py)
|
|
7
7
|
# - Translation server (hardwick-translate.py)
|
|
8
8
|
# - Mini-COT service (Pythia-410M)
|
|
9
9
|
# - Supervisord (process manager)
|
|
10
10
|
#
|
|
11
|
-
#
|
|
12
|
-
# Talks to host Node.js MCP server via unix sockets in /data/run/
|
|
13
|
-
#
|
|
14
|
-
# Build: podman build -t specmem-brain container/
|
|
15
|
-
# Run: podman run -d --name specmem-brain -v ./specmem:/data:Z specmem-brain
|
|
11
|
+
# Target: ~3GB final image
|
|
16
12
|
# ============================================
|
|
17
13
|
|
|
18
|
-
|
|
14
|
+
# ============================================
|
|
15
|
+
# Stage 1: Python deps builder
|
|
16
|
+
# ============================================
|
|
17
|
+
FROM python:3.11-slim AS python-builder
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
20
|
+
build-essential \
|
|
21
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
22
|
+
|
|
23
|
+
# Constraints file: block ALL nvidia/CUDA packages from ever being installed.
|
|
24
|
+
# pip constraints are absolute — no transitive dep can override them.
|
|
25
|
+
RUN cat > /tmp/constraints.txt <<'EOF'
|
|
26
|
+
nvidia-cuda-runtime-cu12==0.0.0.invalid
|
|
27
|
+
nvidia-cuda-cupti-cu12==0.0.0.invalid
|
|
28
|
+
nvidia-cuda-nvrtc-cu12==0.0.0.invalid
|
|
29
|
+
nvidia-cudnn-cu12==0.0.0.invalid
|
|
30
|
+
nvidia-cufft-cu12==0.0.0.invalid
|
|
31
|
+
nvidia-curand-cu12==0.0.0.invalid
|
|
32
|
+
nvidia-cusolver-cu12==0.0.0.invalid
|
|
33
|
+
nvidia-cusparse-cu12==0.0.0.invalid
|
|
34
|
+
nvidia-nccl-cu12==0.0.0.invalid
|
|
35
|
+
nvidia-nvjitlink-cu12==0.0.0.invalid
|
|
36
|
+
nvidia-nvtx-cu12==0.0.0.invalid
|
|
37
|
+
nvidia-cuda-runtime-cu11==0.0.0.invalid
|
|
38
|
+
nvidia-cuda-cupti-cu11==0.0.0.invalid
|
|
39
|
+
nvidia-cuda-nvrtc-cu11==0.0.0.invalid
|
|
40
|
+
nvidia-cudnn-cu11==0.0.0.invalid
|
|
41
|
+
nvidia-cufft-cu11==0.0.0.invalid
|
|
42
|
+
nvidia-curand-cu11==0.0.0.invalid
|
|
43
|
+
nvidia-cusolver-cu11==0.0.0.invalid
|
|
44
|
+
nvidia-cusparse-cu11==0.0.0.invalid
|
|
45
|
+
nvidia-nccl-cu11==0.0.0.invalid
|
|
46
|
+
nvidia-nvjitlink-cu11==0.0.0.invalid
|
|
47
|
+
nvidia-nvtx-cu11==0.0.0.invalid
|
|
48
|
+
triton==0.0.0.invalid
|
|
49
|
+
EOF
|
|
50
|
+
|
|
51
|
+
# Step 1: Install torch CPU-only using --index-url (REPLACES PyPI as primary).
|
|
52
|
+
# This is the ONLY way to guarantee pip doesn't pull CUDA torch from PyPI.
|
|
53
|
+
RUN pip install --no-cache-dir \
|
|
54
|
+
--index-url https://download.pytorch.org/whl/cpu \
|
|
55
|
+
torch
|
|
56
|
+
|
|
57
|
+
# Step 2: Install everything else from PyPI. torch is already satisfied
|
|
58
|
+
# from step 1 so pip won't re-download it. Constraints block nvidia.
|
|
59
|
+
RUN pip install --no-cache-dir \
|
|
60
|
+
-c /tmp/constraints.txt \
|
|
61
|
+
sentence-transformers \
|
|
62
|
+
onnxruntime \
|
|
63
|
+
psycopg2-binary \
|
|
64
|
+
numpy \
|
|
65
|
+
transformers \
|
|
66
|
+
argostranslate \
|
|
67
|
+
ctranslate2 \
|
|
68
|
+
sentencepiece \
|
|
69
|
+
emoji \
|
|
70
|
+
opencc-python-reimplemented
|
|
71
|
+
|
|
72
|
+
# Verify no nvidia packages made it through
|
|
73
|
+
RUN pip list | grep -i nvidia && echo "NVIDIA PACKAGES FOUND - BUILD SHOULD FAIL" && exit 1 || echo "Clean: no nvidia packages"
|
|
74
|
+
|
|
75
|
+
# Strip bloat from installed packages
|
|
76
|
+
RUN find /usr/local/lib/python3.11/site-packages -type d \
|
|
77
|
+
\( -name "test" -o -name "tests" -o -name "__pycache__" \) \
|
|
78
|
+
-exec rm -rf {} + 2>/dev/null; \
|
|
79
|
+
find /usr/local/lib/python3.11/site-packages -name "*.pyc" -delete 2>/dev/null; \
|
|
80
|
+
rm -rf /usr/local/lib/python3.11/site-packages/torch/testing 2>/dev/null; \
|
|
81
|
+
rm -rf /usr/local/lib/python3.11/site-packages/torch/utils/benchmark 2>/dev/null; \
|
|
82
|
+
rm -rf /usr/local/lib/python3.11/site-packages/torch/include 2>/dev/null; \
|
|
83
|
+
rm -rf /usr/local/lib/python3.11/site-packages/torch/share 2>/dev/null; \
|
|
84
|
+
rm -rf /usr/local/lib/python3.11/site-packages/caffe2 2>/dev/null; \
|
|
85
|
+
find /usr/local/lib/python3.11/site-packages -name "*.so" \
|
|
86
|
+
-exec strip --strip-unneeded {} \; 2>/dev/null; \
|
|
87
|
+
true
|
|
22
88
|
|
|
23
89
|
# ============================================
|
|
24
|
-
# Stage
|
|
90
|
+
# Stage 2: Final minimal image
|
|
25
91
|
# ============================================
|
|
92
|
+
FROM python:3.11-slim
|
|
93
|
+
|
|
94
|
+
ENV DEBIAN_FRONTEND=noninteractive
|
|
95
|
+
|
|
96
|
+
# PostgreSQL 16 + pgvector + supervisor — single layer, full cleanup
|
|
26
97
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
27
|
-
# PostgreSQL 16 + pgvector
|
|
28
98
|
gnupg2 lsb-release curl ca-certificates \
|
|
29
99
|
&& echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
|
|
30
100
|
> /etc/apt/sources.list.d/pgdg.list \
|
|
@@ -33,85 +103,55 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
|
33
103
|
&& apt-get update && apt-get install -y --no-install-recommends \
|
|
34
104
|
postgresql-16 \
|
|
35
105
|
postgresql-16-pgvector \
|
|
36
|
-
# Process manager
|
|
37
106
|
supervisor \
|
|
38
|
-
# Cleanup
|
|
39
107
|
&& apt-get purge -y gnupg2 lsb-release curl \
|
|
40
|
-
&& apt-get autoremove -y \
|
|
41
|
-
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
|
108
|
+
&& apt-get autoremove -y --purge \
|
|
109
|
+
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
|
|
110
|
+
/usr/share/doc /usr/share/man /usr/share/locale \
|
|
111
|
+
/var/log/* /var/cache/*
|
|
42
112
|
|
|
43
|
-
#
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
RUN pip install --no-cache-dir \
|
|
47
|
-
torch --index-url https://download.pytorch.org/whl/cpu \
|
|
48
|
-
&& pip install --no-cache-dir \
|
|
49
|
-
sentence-transformers[onnx] \
|
|
50
|
-
onnxruntime \
|
|
51
|
-
psycopg2-binary \
|
|
52
|
-
numpy \
|
|
53
|
-
transformers
|
|
54
|
-
|
|
55
|
-
# Argos Translate + deps (for hardwick-translate.py)
|
|
56
|
-
RUN pip install --no-cache-dir \
|
|
57
|
-
argostranslate \
|
|
58
|
-
ctranslate2 \
|
|
59
|
-
sentencepiece \
|
|
60
|
-
emoji \
|
|
61
|
-
opencc-python-reimplemented
|
|
113
|
+
# Copy Python packages from builder (no --prefix, just site-packages)
|
|
114
|
+
COPY --from=python-builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
|
115
|
+
COPY --from=python-builder /usr/local/bin /usr/local/bin
|
|
62
116
|
|
|
63
|
-
# ============================================
|
|
64
|
-
# Stage 3: Copy pre-optimized models from package
|
|
65
|
-
# ============================================
|
|
66
|
-
# Models are already in the npm package — no download needed!
|
|
67
|
-
# - all-MiniLM-L6-v2 (ONNX quantized) — embedding model
|
|
68
|
-
# - argos-translate models — translation models
|
|
69
117
|
WORKDIR /app
|
|
70
|
-
COPY embedding-sandbox/models/ /app/models/
|
|
71
118
|
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
#
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
COPY
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
COPY container/
|
|
119
|
+
# Create non-root user BEFORE copying files so COPY --chown works
|
|
120
|
+
# without creating a duplicate chown layer
|
|
121
|
+
RUN useradd -r -m -s /bin/bash specmem && \
|
|
122
|
+
mkdir -p /data && chown specmem:specmem /data
|
|
123
|
+
|
|
124
|
+
# Models
|
|
125
|
+
COPY --chown=specmem:specmem embedding-sandbox/models/ /app/models/
|
|
126
|
+
|
|
127
|
+
# Application files
|
|
128
|
+
COPY --chown=specmem:specmem embedding-sandbox/frankenstein-embeddings.py /app/frankenstein-embeddings.py
|
|
129
|
+
COPY --chown=specmem:specmem embedding-sandbox/hardwick-translate.py /app/hardwick-translate.py
|
|
130
|
+
COPY --chown=specmem:specmem embedding-sandbox/mini-cot-service.py /app/mini-cot-service.py
|
|
131
|
+
|
|
132
|
+
# Container config
|
|
133
|
+
COPY --chown=specmem:specmem container/supervisord.conf /etc/supervisor/conf.d/specmem.conf
|
|
134
|
+
COPY --chown=specmem:specmem container/entrypoint.sh /app/entrypoint.sh
|
|
135
|
+
COPY --chown=specmem:specmem container/healthcheck.sh /app/healthcheck.sh
|
|
136
|
+
COPY --chown=specmem:specmem container/sock_check.py /app/sock_check.py
|
|
87
137
|
RUN chmod +x /app/entrypoint.sh /app/healthcheck.sh
|
|
88
138
|
|
|
89
|
-
#
|
|
90
|
-
# Stage 5: Configure PostgreSQL for container
|
|
91
|
-
# ============================================
|
|
92
|
-
# PostgreSQL data and sockets live in /data/ (volume mount)
|
|
93
|
-
# No systemd - we manage postgres directly via pg_ctl
|
|
139
|
+
# PostgreSQL config
|
|
94
140
|
ENV PGDATA=/data/pgdata
|
|
95
141
|
ENV PGHOST=/data/run
|
|
96
142
|
ENV PGPORT=5432
|
|
97
143
|
ENV PGUSER=specmem
|
|
98
144
|
ENV PGDATABASE=specmem
|
|
99
145
|
|
|
100
|
-
#
|
|
146
|
+
# Model paths
|
|
101
147
|
ENV SPECMEM_MODEL_PATH=/app/models/all-MiniLM-L6-v2
|
|
102
148
|
ENV SENTENCE_TRANSFORMERS_HOME=/app/models
|
|
103
149
|
|
|
104
|
-
#
|
|
150
|
+
# CPU-only resource limits
|
|
105
151
|
ENV OMP_NUM_THREADS=1
|
|
106
152
|
ENV MKL_NUM_THREADS=1
|
|
107
153
|
ENV OPENBLAS_NUM_THREADS=1
|
|
108
154
|
|
|
109
|
-
# Create specmem user to run services (postgres won't run as root)
|
|
110
|
-
RUN useradd -r -m -s /bin/bash specmem && \
|
|
111
|
-
mkdir -p /data && chown -R specmem:specmem /data /app
|
|
112
|
-
|
|
113
|
-
# Volume mount point
|
|
114
155
|
VOLUME /data
|
|
115
|
-
|
|
116
156
|
USER specmem
|
|
117
157
|
ENTRYPOINT ["/app/entrypoint.sh"]
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
# 4. No container -> create & run + COLD START (slow, ~20-30s + full feed)
|
|
9
9
|
#
|
|
10
10
|
# VERSION SAFETY:
|
|
11
|
-
# - Every container gets specmem.
|
|
12
|
-
# - Before using ANY container, check
|
|
11
|
+
# - Every container gets specmem.container_version label (decoupled from npm version)
|
|
12
|
+
# - Before using ANY container, check container_version label
|
|
13
13
|
# - Missing/mismatched version = KILL & REBUILD (old code!)
|
|
14
14
|
#
|
|
15
15
|
# On idle: Container is PAUSED (not stopped) - stays in RAM, 0% CPU
|
|
@@ -22,9 +22,19 @@ set -e
|
|
|
22
22
|
# Get specmem installation directory
|
|
23
23
|
SPECMEM_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
|
24
24
|
|
|
25
|
-
# Get specmem version from package.json
|
|
25
|
+
# Get specmem version from package.json (informational only)
|
|
26
26
|
SPECMEM_VERSION=$(node -p "require('$SPECMEM_DIR/package.json').version" 2>/dev/null || echo "0.0.0")
|
|
27
27
|
|
|
28
|
+
# Get CONTAINER version from container-version.json (used for rebuild decisions)
|
|
29
|
+
CONTAINER_VERSION_FILE="$SPECMEM_DIR/container/container-version.json"
|
|
30
|
+
if [ -f "$CONTAINER_VERSION_FILE" ]; then
|
|
31
|
+
CONTAINER_VERSION=$(node -p "require('$CONTAINER_VERSION_FILE').version" 2>/dev/null || echo "1.0.0")
|
|
32
|
+
REGISTRY_IMAGE=$(node -p "require('$CONTAINER_VERSION_FILE').image" 2>/dev/null || echo "ghcr.io/hardwicksoftware/specmem-brain")
|
|
33
|
+
else
|
|
34
|
+
CONTAINER_VERSION="1.0.0"
|
|
35
|
+
REGISTRY_IMAGE="ghcr.io/hardwicksoftware/specmem-brain"
|
|
36
|
+
fi
|
|
37
|
+
|
|
28
38
|
# SOCKET PATH RESOLUTION (in priority order):
|
|
29
39
|
# 1. SPECMEM_EMBEDDING_SOCKET env var (per-project, set by hooks)
|
|
30
40
|
# 2. SPECMEM_SOCKET_DIR env var + embeddings.sock
|
|
@@ -93,9 +103,16 @@ CPU_LIMIT="${SPECMEM_EMBEDDING_CPU_LIMIT:-0.5}"
|
|
|
93
103
|
# Warm start feeder script
|
|
94
104
|
FEEDER_SCRIPT="$SPECMEM_DIR/embedding-sandbox/warm_start_feeder.py"
|
|
95
105
|
|
|
96
|
-
# Check container version label
|
|
106
|
+
# Check container version label (uses container_version for rebuild decisions)
|
|
97
107
|
get_container_version() {
|
|
98
|
-
|
|
108
|
+
# First try new label, fall back to legacy label
|
|
109
|
+
local ver
|
|
110
|
+
ver=$(docker inspect --format='{{index .Config.Labels "specmem.container_version"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
|
|
111
|
+
if [ -z "$ver" ] || [ "$ver" = "<no value>" ]; then
|
|
112
|
+
# Fall back to legacy specmem.version label for old containers
|
|
113
|
+
ver=$(docker inspect --format='{{index .Config.Labels "specmem.version"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
|
|
114
|
+
fi
|
|
115
|
+
echo "$ver"
|
|
99
116
|
}
|
|
100
117
|
|
|
101
118
|
# Check if version matches current specmem
|
|
@@ -105,18 +122,18 @@ version_matches() {
|
|
|
105
122
|
log "VERSION CHECK: No version label (old code!)"
|
|
106
123
|
return 1
|
|
107
124
|
fi
|
|
108
|
-
if [ "$container_version" != "$
|
|
109
|
-
log "VERSION CHECK: Mismatch - container=$container_version,
|
|
125
|
+
if [ "$container_version" != "$CONTAINER_VERSION" ]; then
|
|
126
|
+
log "VERSION CHECK: Mismatch - container=$container_version, expected=$CONTAINER_VERSION"
|
|
110
127
|
return 1
|
|
111
128
|
fi
|
|
112
|
-
log "VERSION CHECK: OK -
|
|
129
|
+
log "VERSION CHECK: OK - container_version=$container_version"
|
|
113
130
|
return 0
|
|
114
131
|
}
|
|
115
132
|
|
|
116
133
|
# Kill container due to version mismatch
|
|
117
134
|
kill_old_version() {
|
|
118
135
|
local old_version=$(get_container_version)
|
|
119
|
-
log "KILLING OLD VERSION: $old_version != $
|
|
136
|
+
log "KILLING OLD VERSION: $old_version != $CONTAINER_VERSION"
|
|
120
137
|
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
|
121
138
|
rm -f "$SOCKET_PATH"
|
|
122
139
|
}
|
|
@@ -175,6 +192,43 @@ get_container_state() {
|
|
|
175
192
|
docker inspect --format='{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true
|
|
176
193
|
}
|
|
177
194
|
|
|
195
|
+
# Background container auto-update check
|
|
196
|
+
# Compares ~/.specmem/container-state.json vs container-version.json
|
|
197
|
+
# If newer version available, pulls in background for next cold start
|
|
198
|
+
check_container_update() {
|
|
199
|
+
local STATE_FILE="${HOME}/.specmem/container-state.json"
|
|
200
|
+
if [ ! -f "$STATE_FILE" ]; then
|
|
201
|
+
return # No state file = first run, skip update check
|
|
202
|
+
fi
|
|
203
|
+
|
|
204
|
+
local CURRENT_CV
|
|
205
|
+
CURRENT_CV=$(node -p "try{JSON.parse(require('fs').readFileSync('$STATE_FILE','utf8')).currentVersion}catch(e){''}" 2>/dev/null || echo "")
|
|
206
|
+
|
|
207
|
+
if [ -n "$CURRENT_CV" ] && [ "$CURRENT_CV" != "$CONTAINER_VERSION" ]; then
|
|
208
|
+
log "UPDATE CHECK: Container update available: $CURRENT_CV -> $CONTAINER_VERSION"
|
|
209
|
+
local REGISTRY_FULL_IMAGE="${REGISTRY_IMAGE}:${CONTAINER_VERSION}"
|
|
210
|
+
# Background pull — don't block the user
|
|
211
|
+
(
|
|
212
|
+
if docker pull "$REGISTRY_FULL_IMAGE" 2>/dev/null; then
|
|
213
|
+
docker tag "$REGISTRY_FULL_IMAGE" "$IMAGE_NAME" 2>/dev/null
|
|
214
|
+
# Update state file
|
|
215
|
+
mkdir -p "${HOME}/.specmem"
|
|
216
|
+
cat > "$STATE_FILE" <<UPDATEJSON
|
|
217
|
+
{
|
|
218
|
+
"currentVersion": "$CONTAINER_VERSION",
|
|
219
|
+
"lastPulled": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
220
|
+
"imageId": "$(docker inspect --format='{{.Id}}' "$IMAGE_NAME" 2>/dev/null || echo "")",
|
|
221
|
+
"source": "registry-update"
|
|
222
|
+
}
|
|
223
|
+
UPDATEJSON
|
|
224
|
+
log "UPDATE CHECK: Pull complete. New image ready for next cold start."
|
|
225
|
+
else
|
|
226
|
+
log "UPDATE CHECK: Pull failed, will retry next time."
|
|
227
|
+
fi
|
|
228
|
+
) &
|
|
229
|
+
fi
|
|
230
|
+
}
|
|
231
|
+
|
|
178
232
|
# Feed overflow queue after warm start
|
|
179
233
|
feed_overflow() {
|
|
180
234
|
if [ -f "$FEEDER_SCRIPT" ]; then
|
|
@@ -208,9 +262,12 @@ main() {
|
|
|
208
262
|
ensure_socket_dir
|
|
209
263
|
|
|
210
264
|
log "=========================================="
|
|
211
|
-
log "WARM START v$SPECMEM_VERSION"
|
|
265
|
+
log "WARM START v$SPECMEM_VERSION (container_version=$CONTAINER_VERSION)"
|
|
212
266
|
log "=========================================="
|
|
213
267
|
|
|
268
|
+
# Check for container updates in background (non-blocking)
|
|
269
|
+
check_container_update
|
|
270
|
+
|
|
214
271
|
# Quick check - already running and responsive?
|
|
215
272
|
if socket_alive; then
|
|
216
273
|
# But check version first!
|
|
@@ -280,15 +337,45 @@ main() {
|
|
|
280
337
|
|
|
281
338
|
*)
|
|
282
339
|
# No container - cold start (only happens once per version)
|
|
283
|
-
log "Creating new container (cold start) with
|
|
340
|
+
log "Creating new container (cold start) with container_version=$CONTAINER_VERSION..."
|
|
284
341
|
rm -f "$SOCKET_PATH"
|
|
285
342
|
|
|
286
343
|
# Remove any old container with same name
|
|
287
344
|
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
|
288
345
|
|
|
346
|
+
# Pull-first: try registry image before falling back to local
|
|
347
|
+
REGISTRY_FULL_IMAGE="${REGISTRY_IMAGE}:${CONTAINER_VERSION}"
|
|
348
|
+
if ! docker image inspect "$IMAGE_NAME" &>/dev/null; then
|
|
349
|
+
log "No local image found, trying registry pull: $REGISTRY_FULL_IMAGE"
|
|
350
|
+
if docker pull "$REGISTRY_FULL_IMAGE" 2>/dev/null; then
|
|
351
|
+
docker tag "$REGISTRY_FULL_IMAGE" "$IMAGE_NAME"
|
|
352
|
+
log "Pulled and tagged registry image as $IMAGE_NAME"
|
|
353
|
+
|
|
354
|
+
# Store container state
|
|
355
|
+
SPECMEM_STATE_DIR="${HOME}/.specmem"
|
|
356
|
+
mkdir -p "$SPECMEM_STATE_DIR"
|
|
357
|
+
cat > "$SPECMEM_STATE_DIR/container-state.json" <<STATEJSON
|
|
358
|
+
{
|
|
359
|
+
"currentVersion": "$CONTAINER_VERSION",
|
|
360
|
+
"lastPulled": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
361
|
+
"imageId": "$(docker inspect --format='{{.Id}}' "$IMAGE_NAME" 2>/dev/null || echo "")",
|
|
362
|
+
"source": "registry"
|
|
363
|
+
}
|
|
364
|
+
STATEJSON
|
|
365
|
+
else
|
|
366
|
+
log "Registry pull failed, will use local image if available"
|
|
367
|
+
fi
|
|
368
|
+
fi
|
|
369
|
+
|
|
370
|
+
# If still no image, build locally (air-gapped fallback)
|
|
371
|
+
if ! docker image inspect "$IMAGE_NAME" &>/dev/null; then
|
|
372
|
+
log "Building image locally from $SPECMEM_DIR/container/"
|
|
373
|
+
docker build -t "$IMAGE_NAME" "$SPECMEM_DIR/container/"
|
|
374
|
+
fi
|
|
375
|
+
|
|
289
376
|
# Create and run with network disabled (air-gapped security)
|
|
290
377
|
# Mount socket directory for access (per-project or /tmp for machine-shared)
|
|
291
|
-
# CRITICAL: Add specmem.
|
|
378
|
+
# CRITICAL: Add specmem.container_version label!
|
|
292
379
|
|
|
293
380
|
# Determine mount strategy
|
|
294
381
|
if [ "$SOCKET_DIR" = "/tmp" ]; then
|
|
@@ -315,6 +402,7 @@ main() {
|
|
|
315
402
|
-e "SPECMEM_CPU_THREADS=1" \
|
|
316
403
|
-e "SPECMEM_EMBEDDING_MAX_WORKERS=4" \
|
|
317
404
|
-l "specmem.user=$USER_ID" \
|
|
405
|
+
-l "specmem.container_version=$CONTAINER_VERSION" \
|
|
318
406
|
-l "specmem.version=$SPECMEM_VERSION" \
|
|
319
407
|
-l "specmem.created=$(date +%s)" \
|
|
320
408
|
-l "specmem.socket=$SOCKET_PATH" \
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "specmem-hardwicksoftware",
|
|
3
|
-
"version": "3.7.
|
|
3
|
+
"version": "3.7.46",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Your Claude Code sessions don't have to start from scratch anymore — SpecMem gives your AI real memory. It won't forget your conversations, your code, or your architecture decisions between sessions. That's the whole point. Semantic code indexing that actually works: TypeScript, JavaScript, Python, Go, Rust, Java, Kotlin, C, C++, HTML and more. It doesn't just track functions — it gets classes, methods, fields, constants, enums, macros, imports, structs, the whole codebase graph. There's chat memory too, powered by pgvector embeddings. You've also got token compression, team coordination, multi-agent comms, and file watching built in. 74+ MCP tools. Runs on PostgreSQL + Docker. It's kind of a big deal. justcalljon.pro",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -76,20 +76,51 @@ if (currentPlatform === 'darwin') {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
if (!isGlobalInstall) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
console.log('\
|
|
84
|
-
console.log('\x1b[
|
|
85
|
-
console.log('\x1b[
|
|
86
|
-
console.log('\x1b[
|
|
87
|
-
console.log('\x1b[
|
|
88
|
-
console.log('\x1b[
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
79
|
+
// LOCAL INSTALL: Create wrapper scripts in ~/.local/bin/
|
|
80
|
+
const LOCAL_BIN = path.join(os.homedir(), '.local', 'bin');
|
|
81
|
+
const SPECMEM_PKG = path.dirname(__dirname);
|
|
82
|
+
|
|
83
|
+
console.log('\n\x1b[36m╔════════════════════════════════════════════════════════════╗\x1b[0m');
|
|
84
|
+
console.log('\x1b[36m║\x1b[0m \x1b[1mSpecMem: Local install detected\x1b[0m \x1b[36m║\x1b[0m');
|
|
85
|
+
console.log('\x1b[36m╠════════════════════════════════════════════════════════════╣\x1b[0m');
|
|
86
|
+
console.log('\x1b[36m║\x1b[0m Setting up CLI wrappers in ~/.local/bin/ \x1b[36m║\x1b[0m');
|
|
87
|
+
console.log('\x1b[36m║\x1b[0m Using container mode (no root needed) \x1b[36m║\x1b[0m');
|
|
88
|
+
console.log('\x1b[36m╚════════════════════════════════════════════════════════════╝\x1b[0m');
|
|
89
|
+
|
|
90
|
+
try {
|
|
91
|
+
fs.mkdirSync(LOCAL_BIN, { recursive: true });
|
|
92
|
+
|
|
93
|
+
// Create wrapper scripts for each CLI entry point
|
|
94
|
+
const wrappers = {
|
|
95
|
+
'specmem': 'bin/specmem-cli.cjs',
|
|
96
|
+
'specmem-init': 'scripts/specmem-init.cjs',
|
|
97
|
+
'specmem-console': 'bin/specmem-console.cjs',
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
for (const [cmd, relPath] of Object.entries(wrappers)) {
|
|
101
|
+
const targetScript = path.join(SPECMEM_PKG, relPath);
|
|
102
|
+
const wrapperPath = path.join(LOCAL_BIN, cmd);
|
|
103
|
+
const wrapperContent = `#!/bin/sh\nexec node "${targetScript}" "$@"\n`;
|
|
104
|
+
fs.writeFileSync(wrapperPath, wrapperContent, { mode: 0o755 });
|
|
105
|
+
console.log(` \x1b[32m✓\x1b[0m Created ${wrapperPath}`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Check if ~/.local/bin is in PATH
|
|
109
|
+
const userPath = process.env.PATH || '';
|
|
110
|
+
if (!userPath.includes(LOCAL_BIN)) {
|
|
111
|
+
console.log(`\n \x1b[33m⚠\x1b[0m Add ~/.local/bin to your PATH:`);
|
|
112
|
+
console.log(` \x1b[36mexport PATH="$HOME/.local/bin:$PATH"\x1b[0m`);
|
|
113
|
+
console.log(` Add this to your ~/.bashrc or ~/.zshrc\n`);
|
|
114
|
+
} else {
|
|
115
|
+
console.log(`\n \x1b[32m✓\x1b[0m ~/.local/bin is already in PATH`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
console.log(` \x1b[32m✓\x1b[0m Run \x1b[1mspecmem-init\x1b[0m to complete setup\n`);
|
|
119
|
+
} catch (e) {
|
|
120
|
+
console.log(` \x1b[31m✗\x1b[0m Failed to create wrappers: ${e.message}`);
|
|
121
|
+
console.log(` \x1b[33m⚠\x1b[0m You can still use: npx specmem-hardwicksoftware\n`);
|
|
122
|
+
}
|
|
123
|
+
process.exit(0);
|
|
93
124
|
}
|
|
94
125
|
|
|
95
126
|
// Colors for terminal output
|
|
@@ -2221,6 +2252,14 @@ ${c.red}${c.bright}╠═══════════════════
|
|
|
2221
2252
|
if (canInstallSystem) {
|
|
2222
2253
|
// Step 0: Install ALL core system deps FIRST
|
|
2223
2254
|
installCoreDeps();
|
|
2255
|
+
} else if (needsSudo.length > 0) {
|
|
2256
|
+
// Non-root: skip system deps, use container-only mode
|
|
2257
|
+
log.warn('No root access — using container-only mode (no system PG/Python needed)');
|
|
2258
|
+
log.info(`Container bundles everything. You only need Docker or Podman.`);
|
|
2259
|
+
if (!dockerAvailable) {
|
|
2260
|
+
log.warn('Docker not found! Install Docker (or rootless Podman) for container mode.');
|
|
2261
|
+
log.info(' See: https://docs.docker.com/engine/install/');
|
|
2262
|
+
}
|
|
2224
2263
|
}
|
|
2225
2264
|
|
|
2226
2265
|
// Step 0a: Claude Code (REQUIRED) - no sudo needed
|
package/scripts/specmem-init.cjs
CHANGED
|
@@ -2860,14 +2860,14 @@ const qoms = {
|
|
|
2860
2860
|
// DEAD → Clean immediately (broken garbage)
|
|
2861
2861
|
//
|
|
2862
2862
|
// VERSION SAFETY:
|
|
2863
|
-
// - Every container gets labeled with specmem.
|
|
2864
|
-
// - Before using ANY container, check
|
|
2863
|
+
// - Every container gets labeled with specmem.container_version on creation
|
|
2864
|
+
// - Before using ANY container, check container_version label (NOT npm version)
|
|
2865
2865
|
// - Missing label = old code = KILL IT
|
|
2866
2866
|
// - Version mismatch = old code = KILL IT
|
|
2867
2867
|
// - Only exact version match = safe to use
|
|
2868
2868
|
// ============================================================================
|
|
2869
2869
|
|
|
2870
|
-
// Get current specmem version from package.json
|
|
2870
|
+
// Get current specmem version from package.json (informational only)
|
|
2871
2871
|
function getSpecmemVersion() {
|
|
2872
2872
|
try {
|
|
2873
2873
|
const pkgPath = path.join(__dirname, '..', 'package.json');
|
|
@@ -2878,7 +2878,19 @@ function getSpecmemVersion() {
|
|
|
2878
2878
|
}
|
|
2879
2879
|
}
|
|
2880
2880
|
|
|
2881
|
+
// Get container version from container-version.json (used for rebuild decisions)
|
|
2882
|
+
function getContainerVersionInfo() {
|
|
2883
|
+
try {
|
|
2884
|
+
const cvPath = path.join(__dirname, '..', 'container', 'container-version.json');
|
|
2885
|
+
return JSON.parse(fs.readFileSync(cvPath, 'utf8'));
|
|
2886
|
+
} catch (e) {
|
|
2887
|
+
return { version: '1.0.0', image: 'ghcr.io/jonhardwick-spec/specmem-brain', minSpecmemVersion: '3.7.0' };
|
|
2888
|
+
}
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2881
2891
|
const SPECMEM_VERSION = getSpecmemVersion();
|
|
2892
|
+
const CONTAINER_VERSION_INFO = getContainerVersionInfo();
|
|
2893
|
+
const CONTAINER_VERSION = CONTAINER_VERSION_INFO.version;
|
|
2882
2894
|
|
|
2883
2895
|
/**
|
|
2884
2896
|
* Check if a container has the correct specmem version
|
|
@@ -2894,16 +2906,24 @@ function checkContainerVersion(containerId) {
|
|
|
2894
2906
|
};
|
|
2895
2907
|
|
|
2896
2908
|
try {
|
|
2897
|
-
//
|
|
2898
|
-
|
|
2899
|
-
`docker inspect --format='{{index .Config.Labels "specmem.
|
|
2909
|
+
// Try new container_version label first, fall back to legacy specmem.version
|
|
2910
|
+
let labelOutput = execSync(
|
|
2911
|
+
`docker inspect --format='{{index .Config.Labels "specmem.container_version"}}' ${containerId} 2>/dev/null`,
|
|
2900
2912
|
{ encoding: 'utf8' }
|
|
2901
2913
|
).trim();
|
|
2902
2914
|
|
|
2915
|
+
if (!labelOutput || labelOutput === '<no value>' || labelOutput === '') {
|
|
2916
|
+
// Fall back to legacy label for old containers
|
|
2917
|
+
labelOutput = execSync(
|
|
2918
|
+
`docker inspect --format='{{index .Config.Labels "specmem.version"}}' ${containerId} 2>/dev/null`,
|
|
2919
|
+
{ encoding: 'utf8' }
|
|
2920
|
+
).trim();
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2903
2923
|
if (labelOutput && labelOutput !== '<no value>' && labelOutput !== '') {
|
|
2904
2924
|
result.hasVersion = true;
|
|
2905
2925
|
result.version = labelOutput;
|
|
2906
|
-
result.matches = labelOutput ===
|
|
2926
|
+
result.matches = labelOutput === CONTAINER_VERSION;
|
|
2907
2927
|
result.needsRebuild = !result.matches;
|
|
2908
2928
|
}
|
|
2909
2929
|
} catch (e) {
|
|
@@ -2946,6 +2966,7 @@ async function cleanupDockerContainers() {
|
|
|
2946
2966
|
errors: [],
|
|
2947
2967
|
dockerAvailable: false,
|
|
2948
2968
|
currentVersion: SPECMEM_VERSION,
|
|
2969
|
+
containerVersion: CONTAINER_VERSION,
|
|
2949
2970
|
};
|
|
2950
2971
|
|
|
2951
2972
|
try {
|
|
@@ -3000,14 +3021,14 @@ async function cleanupDockerContainers() {
|
|
|
3000
3021
|
// Kill containers with version mismatch (OLD CODE!) - ONLY for THIS project
|
|
3001
3022
|
if (versionCheck.needsRebuild && status !== 'Dead') {
|
|
3002
3023
|
const oldVersion = versionCheck.version || 'MISSING';
|
|
3003
|
-
if (killOutdatedContainer(id, `version mismatch: ${oldVersion} != ${
|
|
3024
|
+
if (killOutdatedContainer(id, `container version mismatch: ${oldVersion} != ${CONTAINER_VERSION}`)) {
|
|
3004
3025
|
results.versionMismatch++;
|
|
3005
3026
|
results.cleaned.push({
|
|
3006
3027
|
id,
|
|
3007
3028
|
name,
|
|
3008
3029
|
type: 'version_mismatch',
|
|
3009
3030
|
oldVersion,
|
|
3010
|
-
newVersion:
|
|
3031
|
+
newVersion: CONTAINER_VERSION,
|
|
3011
3032
|
wasState: status?.includes('Paused') ? 'paused' : (status?.startsWith('Up') ? 'running' : 'exited'),
|
|
3012
3033
|
});
|
|
3013
3034
|
}
|
|
@@ -10066,9 +10087,19 @@ ${lastOutput}
|
|
|
10066
10087
|
|
|
10067
10088
|
// ==========================================================================
|
|
10068
10089
|
// STAGE 1: MODEL DOWNLOAD — auto-fetch any missing ML models
|
|
10090
|
+
// SKIP if container mode (models are in the brain container image)
|
|
10069
10091
|
// ==========================================================================
|
|
10070
10092
|
ui.setStage(1, 'MODEL DOWNLOAD');
|
|
10071
|
-
|
|
10093
|
+
// Check if container runtime is available - if so, models are in container, skip download
|
|
10094
|
+
const rtForModels = (() => {
|
|
10095
|
+
try { execSync('which podman 2>/dev/null', { stdio: 'pipe' }); return 'podman'; } catch { try { execSync('which docker 2>/dev/null', { stdio: 'pipe' }); return 'docker'; } catch { return null; } }
|
|
10096
|
+
})();
|
|
10097
|
+
if (rtForModels) {
|
|
10098
|
+
initLog('[MODELS] Container runtime detected — models are in container image, skipping host download');
|
|
10099
|
+
if (ui) ui.setSubStatus('✓ Models in container (skipping host download)');
|
|
10100
|
+
} else {
|
|
10101
|
+
await ensureModels(ui);
|
|
10102
|
+
}
|
|
10072
10103
|
|
|
10073
10104
|
// ==========================================================================
|
|
10074
10105
|
// CONTAINER FAST-PATH: If podman/docker is available, use container mode
|
|
@@ -10164,7 +10195,8 @@ ${lastOutput}
|
|
|
10164
10195
|
// Stage 3: Container engine — pull image if needed
|
|
10165
10196
|
ui.setStage(3, 'CONTAINER ENGINE');
|
|
10166
10197
|
ui.setStatus('Pulling specmem-brain image...');
|
|
10167
|
-
const
|
|
10198
|
+
const cvInfo = getContainerVersionInfo();
|
|
10199
|
+
const containerImage = process.env.SPECMEM_CONTAINER_IMAGE || `${cvInfo.image}:${cvInfo.version}`;
|
|
10168
10200
|
const rtCmd = (() => {
|
|
10169
10201
|
try { execSync('which podman 2>/dev/null', { stdio: 'pipe' }); return 'podman'; } catch { return 'docker'; }
|
|
10170
10202
|
})();
|
package/specmem/supervisord.conf
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
; ============================================
|
|
2
2
|
; SPECMEM BRAIN CONTAINER - DYNAMIC SUPERVISORD CONFIG
|
|
3
|
-
; Generated by specmem-init at 2026-03-
|
|
3
|
+
; Generated by specmem-init at 2026-03-10T03:09:08.421Z
|
|
4
4
|
; Thread counts from model-config.json resourcePool
|
|
5
5
|
; ============================================
|
|
6
6
|
|
package/specmem/user-config.json
CHANGED