vcti-array-display 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vcti_array_display-1.1.0/CHANGELOG.md +70 -0
- vcti_array_display-1.1.0/CONTRIBUTING.md +57 -0
- vcti_array_display-1.1.0/LICENSE +8 -0
- vcti_array_display-1.1.0/MANIFEST.in +12 -0
- vcti_array_display-1.1.0/PKG-INFO +222 -0
- vcti_array_display-1.1.0/README.md +199 -0
- vcti_array_display-1.1.0/SECURITY.md +28 -0
- vcti_array_display-1.1.0/pyproject.toml +44 -0
- vcti_array_display-1.1.0/setup.cfg +4 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/__init__.py +19 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/_exclusion.py +60 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/_flattening.py +46 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/_formatting.py +45 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/array_display.py +624 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/array_display.pyi +92 -0
- vcti_array_display-1.1.0/src/vcti/arraydisplay/py.typed +0 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/PKG-INFO +222 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/SOURCES.txt +20 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/dependency_links.txt +1 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/requires.txt +17 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/top_level.txt +1 -0
- vcti_array_display-1.1.0/src/vcti_array_display.egg-info/zip-safe +1 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `vcti-array-display` are documented here.
|
|
4
|
+
|
|
5
|
+
## [1.1.0] — 2026-03-29
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
- `column_list=[]` was treated as `None` (showed all columns instead of none).
|
|
9
|
+
- `set_view_columns()` now copies the input list defensively — external
|
|
10
|
+
mutation of the passed list no longer corrupts internal state.
|
|
11
|
+
- `assert` statements replaced with proper `TypeError`/`ValueError` raises —
|
|
12
|
+
safe under `python -O`.
|
|
13
|
+
- `pd.get_option("display.max_rows")` returning `0` no longer falls through
|
|
14
|
+
to default.
|
|
15
|
+
- Stale zero-copy claims removed from README and performance guide (pandas
|
|
16
|
+
copies structured arrays into DataFrames).
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
- **Read-only properties** — `view_columns`, `name_columns`, `field_components`
|
|
20
|
+
return defensive copies / `MappingProxyType` to prevent accidental mutation.
|
|
21
|
+
- **`copy()`** — shallow copy with shared array and independent configuration.
|
|
22
|
+
- **`to_slice()` conflict detection** — raises `ValueError` when incompatible
|
|
23
|
+
parameters are combined (e.g. `mask` with `head`/`tail`).
|
|
24
|
+
- **`ColumnExclusion`** exported from top-level `__init__.py` for typed
|
|
25
|
+
consumer code.
|
|
26
|
+
- **`__repr__`** now shows active features: flattened field count and enum
|
|
27
|
+
mapping count alongside rows/cols.
|
|
28
|
+
- **`extending.md`** — contributor guide for adding adapters, patterns, and
|
|
29
|
+
integrating with vcti-array-tree (including ecosystem diagram).
|
|
30
|
+
- **`examples/full_pipeline.py`** — complete end-to-end script demonstrating
|
|
31
|
+
all five pillars.
|
|
32
|
+
- **`array_display.pyi`** — type stub file for improved IDE support.
|
|
33
|
+
- **Sphinx `conf.py`** — enables doc builds with autodoc and MyST-Parser.
|
|
34
|
+
- **`MANIFEST.in`** — controls sdist contents, excludes tests/docs/.claude.
|
|
35
|
+
- **`tests/bench_array_display.py`** — performance regression benchmarks
|
|
36
|
+
verifying O(displayed rows) on 10M-row arrays (marked `@pytest.mark.benchmark`).
|
|
37
|
+
- **Python 3.14** added to CI test matrix.
|
|
38
|
+
- Parametrized exclusion pattern tests.
|
|
39
|
+
- Tests for empty column list, to_slice conflicts, read-only properties, copy().
|
|
40
|
+
- Troubleshooting entries: empty `field_components`, enum debugging,
|
|
41
|
+
`_column_aliases` tracking.
|
|
42
|
+
- Performance guide: O() complexity table and bounded-vs-full guidance.
|
|
43
|
+
- Source guide: `_column_aliases` lifecycle documentation with examples.
|
|
44
|
+
- Design guide: copy-vs-mutation guidance, strict slicing parameters.
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
- `import_pandas()` return type narrowed from `Any` to `types.ModuleType`.
|
|
48
|
+
- `math.prod()` replaces manual loop in `_flattening.py`.
|
|
49
|
+
- `to_dataframe()` docstring no longer claims zero-copy.
|
|
50
|
+
- `CONTRIBUTING.md` includes `mypy` and `typecheck` dependency group.
|
|
51
|
+
|
|
52
|
+
### Removed
|
|
53
|
+
- Unused `conftest.py` fixtures (tests use inline helpers).
|
|
54
|
+
|
|
55
|
+
## [1.0.0] — 2026-03-28
|
|
56
|
+
|
|
57
|
+
### Added
|
|
58
|
+
- Initial release.
|
|
59
|
+
- **Column filtering** — `exclude_columns` with str, regex, or callable rules.
|
|
60
|
+
Pre-built patterns: `FILLER_COLUMNS` (C++ alignment) and `LENGTH_COLUMNS`
|
|
61
|
+
(internal string lengths).
|
|
62
|
+
- **Dtype flattening** — `flatten_dtype` expands vector/matrix fields.
|
|
63
|
+
`component_names` and `set_component_names()` for custom suffixes.
|
|
64
|
+
`field_components` tracks the grouping for consumer use.
|
|
65
|
+
- **Enum mapping** — `add_name_columns()` with lazy resolution on bounded slices.
|
|
66
|
+
- **Array slicing** — `to_slice()` returns index arrays (head, tail, mask, indices).
|
|
67
|
+
- **Adapters** — `to_table()` (pure numpy text table), `to_dataframe()`
|
|
68
|
+
(pandas with `pd.Categorical`), `_repr_html_()` (Jupyter).
|
|
69
|
+
- Optional `pandas` dependency for DataFrame and Jupyter features.
|
|
70
|
+
- Structured array validation in `set_array()`.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Contributing to vcti-array-display
|
|
2
|
+
|
|
3
|
+
## Development Setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
python -m venv .venv
|
|
7
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
8
|
+
pip install -e ".[test,lint,typecheck]"
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Running Tests
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pytest
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Coverage is enforced at 95% via `pyproject.toml`.
|
|
18
|
+
|
|
19
|
+
## Type Checking
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
mypy src/
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Linting and Formatting
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
ruff check src/ tests/ # lint
|
|
29
|
+
ruff format --check src/ tests/ # format check
|
|
30
|
+
ruff check --fix src/ tests/ # auto-fix
|
|
31
|
+
ruff format src/ tests/ # apply formatting
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Project Layout
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
src/vcti/arraydisplay/ # Source code
|
|
38
|
+
tests/ # Tests (test_<module>.py)
|
|
39
|
+
docs/ # Documentation
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Architecture
|
|
43
|
+
|
|
44
|
+
The source code is organized around five pillars:
|
|
45
|
+
|
|
46
|
+
1. **Column filtering** — name, regex, and dtype-based exclusion
|
|
47
|
+
2. **Dtype flattening** — vector/matrix expansion with component tracking
|
|
48
|
+
3. **Enum mapping** — lazy integer-to-name resolution
|
|
49
|
+
4. **Array slicing** — lightweight index arrays for bounded access
|
|
50
|
+
5. **Adapters** — text table and pandas DataFrame rendering
|
|
51
|
+
|
|
52
|
+
See [docs/source-guide.md](docs/source-guide.md) for a detailed walkthrough.
|
|
53
|
+
|
|
54
|
+
## Guidelines
|
|
55
|
+
|
|
56
|
+
- Follow Google-style docstrings and PEP 585/604 type annotations.
|
|
57
|
+
- See [docs/design.md](docs/design.md) for design decisions and boundaries.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
Copyright (c) 2018-2026 Visual Collaboration Technologies Inc.
|
|
2
|
+
All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
This software is proprietary and confidential. Unauthorized copying,
|
|
5
|
+
distribution, or use of this software, via any medium, is strictly
|
|
6
|
+
prohibited. Access is granted only to authorized VCollab developers
|
|
7
|
+
and individuals explicitly authorized by Visual Collaboration
|
|
8
|
+
Technologies Inc.
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vcti-array-display
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Presentation layer for NumPy structured arrays — column filtering, dtype flattening, enum mapping, array slicing, and adapters
|
|
5
|
+
Author: Visual Collaboration Technologies Inc.
|
|
6
|
+
Requires-Python: <3.15,>=3.12
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Dist: vcti-nputils>=1.0.0
|
|
11
|
+
Provides-Extra: pandas
|
|
12
|
+
Requires-Dist: pandas>=2.0; extra == "pandas"
|
|
13
|
+
Provides-Extra: test
|
|
14
|
+
Requires-Dist: pytest; extra == "test"
|
|
15
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
16
|
+
Requires-Dist: pandas>=2.0; extra == "test"
|
|
17
|
+
Provides-Extra: lint
|
|
18
|
+
Requires-Dist: ruff; extra == "lint"
|
|
19
|
+
Provides-Extra: typecheck
|
|
20
|
+
Requires-Dist: mypy; extra == "typecheck"
|
|
21
|
+
Requires-Dist: pandas-stubs; extra == "typecheck"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Array Display
|
|
25
|
+
|
|
26
|
+
Presentation layer for NumPy structured arrays.
|
|
27
|
+
|
|
28
|
+
ArrayDisplay wraps a structured NumPy array reference (no copy) and provides
|
|
29
|
+
five capabilities for controlling how the data is presented:
|
|
30
|
+
|
|
31
|
+
1. **Column filtering** — hide columns by name, regex, or predicate
|
|
32
|
+
2. **Dtype flattening** — expand vector/matrix fields into scalar columns with custom component names
|
|
33
|
+
3. **Enum mapping** — lazily map integer ID columns to human-readable names
|
|
34
|
+
4. **Array slicing** — create lightweight index arrays for bounded access
|
|
35
|
+
5. **Adapters** — render as text table or pandas DataFrame on bounded slices
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install vcti-array-display>=1.1.0
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
With pandas support (optional):
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install vcti-array-display[pandas]>=1.1.0
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## 1. Column Filtering
|
|
52
|
+
|
|
53
|
+
Hide unwanted columns using any combination of exact names, regex patterns,
|
|
54
|
+
and callable predicates.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import re
|
|
58
|
+
import numpy as np
|
|
59
|
+
from vcti.arraydisplay import ArrayDisplay, FILLER_COLUMNS, LENGTH_COLUMNS, VOID_COLUMNS
|
|
60
|
+
|
|
61
|
+
dt = np.dtype([
|
|
62
|
+
('id', 'i4'), ('f0', 'V4'), ('label', 'U20'),
|
|
63
|
+
('label_len', 'i4'), ('value', 'f8'), ('f3', 'V8'),
|
|
64
|
+
])
|
|
65
|
+
arr = np.zeros(10, dtype=dt)
|
|
66
|
+
|
|
67
|
+
# Pre-built patterns for C++ interop noise
|
|
68
|
+
view = ArrayDisplay(arr, exclude_columns=[FILLER_COLUMNS, LENGTH_COLUMNS])
|
|
69
|
+
view.view_columns # ['id', 'label', 'value']
|
|
70
|
+
|
|
71
|
+
# Or exclude by dtype — catches all void padding regardless of name
|
|
72
|
+
view = ArrayDisplay(arr, exclude_columns=[VOID_COLUMNS, LENGTH_COLUMNS])
|
|
73
|
+
|
|
74
|
+
# Mix exact names, regex, and callables
|
|
75
|
+
view = ArrayDisplay(arr, exclude_columns=[
|
|
76
|
+
FILLER_COLUMNS, # regex: ^f\d+$
|
|
77
|
+
LENGTH_COLUMNS, # regex: _len$
|
|
78
|
+
"debug_flag", # exact name
|
|
79
|
+
re.compile(r"^tmp_"), # custom regex
|
|
80
|
+
lambda name, dtype: dtype.kind == 'V', # by dtype
|
|
81
|
+
])
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
No magic defaults — `ArrayDisplay(arr)` shows all columns. Pre-built patterns
|
|
85
|
+
`FILLER_COLUMNS`, `LENGTH_COLUMNS`, and `VOID_COLUMNS` are opt-in.
|
|
86
|
+
Callables receive `(name, dtype)` for filtering by type, shape, or both.
|
|
87
|
+
|
|
88
|
+
## 2. Dtype Flattening
|
|
89
|
+
|
|
90
|
+
Expand vector and matrix fields into individual scalar columns, with
|
|
91
|
+
optional user-defined component names.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
dt = np.dtype([('id', 'i4'), ('position', 'f8', (3,))])
|
|
95
|
+
arr = np.zeros(5, dtype=dt)
|
|
96
|
+
|
|
97
|
+
# Default numeric suffixes
|
|
98
|
+
view = ArrayDisplay(arr, flatten_dtype=True)
|
|
99
|
+
view.view_columns # ['id', 'position_0', 'position_1', 'position_2']
|
|
100
|
+
|
|
101
|
+
# Custom component names
|
|
102
|
+
view = ArrayDisplay(arr, flatten_dtype=True,
|
|
103
|
+
component_names={'position': ['x', 'y', 'z']})
|
|
104
|
+
view.view_columns # ['id', 'position_x', 'position_y', 'position_z']
|
|
105
|
+
|
|
106
|
+
# Component grouping is tracked for consumers (e.g., header spanning)
|
|
107
|
+
view.field_components
|
|
108
|
+
# {'position': ['position_x', 'position_y', 'position_z']}
|
|
109
|
+
|
|
110
|
+
# Rename after construction
|
|
111
|
+
view.set_component_names('position', ['lat', 'lon', 'alt'])
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
> **Note:** Default flattened names (e.g., `position_0`) are generated by
|
|
115
|
+
> `vcti-nputils` and may truncate long field names. Use `component_names`
|
|
116
|
+
> to ensure predictable, readable column names regardless of the defaults.
|
|
117
|
+
|
|
118
|
+
## 3. Enum Mapping
|
|
119
|
+
|
|
120
|
+
Map integer ID columns to human-readable names. The mapping is stored as
|
|
121
|
+
a recipe and resolved lazily — only when presenting a bounded slice.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
dt = np.dtype([('id', 'i4'), ('element_type', 'i4'), ('value', 'f8')])
|
|
125
|
+
arr = np.array([(1, 1, 10.5), (2, 2, 20.3), (3, 1, 15.0)], dtype=dt)
|
|
126
|
+
view = ArrayDisplay(arr)
|
|
127
|
+
|
|
128
|
+
view.add_name_columns({
|
|
129
|
+
'element_type_name': ('element_type', {1: 'QUAD', 2: 'HEX'}),
|
|
130
|
+
})
|
|
131
|
+
view.view_columns # ['id', 'element_type_name', 'value']
|
|
132
|
+
|
|
133
|
+
# The mapping is NOT materialized yet — it's a recipe:
|
|
134
|
+
view.name_columns
|
|
135
|
+
# {'element_type_name': ('element_type', {1: 'QUAD', 2: 'HEX'})}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## 4. Array Slicing
|
|
139
|
+
|
|
140
|
+
Create lightweight index arrays for bounded access. The index array is
|
|
141
|
+
tiny regardless of the underlying array size.
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
idx = view.to_slice(head=20) # first 20 rows
|
|
145
|
+
idx = view.to_slice(tail=10) # last 10 rows
|
|
146
|
+
idx = view.to_slice(head=10, tail=5) # first 10 + last 5
|
|
147
|
+
idx = view.to_slice(mask=arr['value'] > 50) # boolean filter
|
|
148
|
+
idx = view.to_slice(indices=np.array([0, 100, 999])) # explicit
|
|
149
|
+
|
|
150
|
+
view.array[idx] # sliced data
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## 5. Adapters
|
|
154
|
+
|
|
155
|
+
### Text table (pure numpy — no extra dependencies)
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
print(view.to_table(head=10, tail=5))
|
|
159
|
+
# id | element_type_name | value
|
|
160
|
+
# ---+-------------------+------
|
|
161
|
+
# 1 | QUAD | 10.5
|
|
162
|
+
# 2 | HEX | 20.3
|
|
163
|
+
# ...
|
|
164
|
+
# [1000 rows x 3 columns]
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### pandas DataFrame (optional dependency)
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
df = view.to_dataframe() # full array
|
|
171
|
+
df = view.to_dataframe(head=100) # first 100 rows
|
|
172
|
+
df = view.to_dataframe(resolve_names=False) # raw ID columns
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Enum names are materialized using `pd.Categorical` for memory efficiency.
|
|
176
|
+
|
|
177
|
+
### Jupyter notebooks
|
|
178
|
+
|
|
179
|
+
ArrayDisplay provides `_repr_html_()` — displays a bounded row window with
|
|
180
|
+
enum names resolved automatically.
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## Performance
|
|
185
|
+
|
|
186
|
+
Designed for large CAE arrays (millions of rows, GBs of data):
|
|
187
|
+
|
|
188
|
+
- **No array copy** — wraps a reference to the original numpy array
|
|
189
|
+
- **Lazy enum resolution** — resolved only on the displayed slice
|
|
190
|
+
- **Index arrays** — `to_slice()` returns kilobytes regardless of array size
|
|
191
|
+
- **Bounded presentation** — `to_table()` and `to_dataframe(head=N)` never touch the full array
|
|
192
|
+
- **`pd.Categorical`** — ~100x less memory than string columns for enum values
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## API Summary
|
|
197
|
+
|
|
198
|
+
| Method | Pillar | Description |
|
|
199
|
+
|--------|--------|-------------|
|
|
200
|
+
| `set_view_columns(...)` | Filtering | Configure visible columns |
|
|
201
|
+
| `include_view_columns(cols)` | Filtering | Add columns to the view |
|
|
202
|
+
| `exclude_view_columns(cols)` | Filtering | Remove columns from the view |
|
|
203
|
+
| `replace_view_columns(mapping)` | Filtering | Rename columns in the view |
|
|
204
|
+
| `set_array(arr, ...)` | Flattening | Set array with optional flattening |
|
|
205
|
+
| `set_component_names(field, names)` | Flattening | Rename flattened components |
|
|
206
|
+
| `add_name_columns(mappings)` | Enum mapping | Register lazy enum mappings |
|
|
207
|
+
| `to_slice(...)` | Slicing | Create index array for bounded access |
|
|
208
|
+
| `to_table(...)` | Adapter | Render text table (pure numpy) |
|
|
209
|
+
| `to_dataframe(...)` | Adapter | Convert to pandas DataFrame |
|
|
210
|
+
| `copy()` | Misc | Shallow copy with independent config |
|
|
211
|
+
| `append_columns(cols)` | Misc | Join additional structured columns |
|
|
212
|
+
|
|
213
|
+
## Examples
|
|
214
|
+
|
|
215
|
+
See [examples/full_pipeline.py](examples/full_pipeline.py) for a complete
|
|
216
|
+
end-to-end script demonstrating all five pillars.
|
|
217
|
+
|
|
218
|
+
## Dependencies
|
|
219
|
+
|
|
220
|
+
- [numpy](https://numpy.org/) (>=1.24) — required
|
|
221
|
+
- [vcti-nputils](https://pypi.org/project/vcti-nputils/) (>=1.0.0) — required
|
|
222
|
+
- [pandas](https://pandas.pydata.org/) (>=2.0) — optional, for `to_dataframe()` and Jupyter display
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Array Display
|
|
2
|
+
|
|
3
|
+
Presentation layer for NumPy structured arrays.
|
|
4
|
+
|
|
5
|
+
ArrayDisplay wraps a structured NumPy array reference (no copy) and provides
|
|
6
|
+
five capabilities for controlling how the data is presented:
|
|
7
|
+
|
|
8
|
+
1. **Column filtering** — hide columns by name, regex, or predicate
|
|
9
|
+
2. **Dtype flattening** — expand vector/matrix fields into scalar columns with custom component names
|
|
10
|
+
3. **Enum mapping** — lazily map integer ID columns to human-readable names
|
|
11
|
+
4. **Array slicing** — create lightweight index arrays for bounded access
|
|
12
|
+
5. **Adapters** — render as text table or pandas DataFrame on bounded slices
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install vcti-array-display>=1.1.0
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
With pandas support (optional):
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install vcti-array-display[pandas]>=1.1.0
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## 1. Column Filtering
|
|
29
|
+
|
|
30
|
+
Hide unwanted columns using any combination of exact names, regex patterns,
|
|
31
|
+
and callable predicates.
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
import re
|
|
35
|
+
import numpy as np
|
|
36
|
+
from vcti.arraydisplay import ArrayDisplay, FILLER_COLUMNS, LENGTH_COLUMNS, VOID_COLUMNS
|
|
37
|
+
|
|
38
|
+
dt = np.dtype([
|
|
39
|
+
('id', 'i4'), ('f0', 'V4'), ('label', 'U20'),
|
|
40
|
+
('label_len', 'i4'), ('value', 'f8'), ('f3', 'V8'),
|
|
41
|
+
])
|
|
42
|
+
arr = np.zeros(10, dtype=dt)
|
|
43
|
+
|
|
44
|
+
# Pre-built patterns for C++ interop noise
|
|
45
|
+
view = ArrayDisplay(arr, exclude_columns=[FILLER_COLUMNS, LENGTH_COLUMNS])
|
|
46
|
+
view.view_columns # ['id', 'label', 'value']
|
|
47
|
+
|
|
48
|
+
# Or exclude by dtype — catches all void padding regardless of name
|
|
49
|
+
view = ArrayDisplay(arr, exclude_columns=[VOID_COLUMNS, LENGTH_COLUMNS])
|
|
50
|
+
|
|
51
|
+
# Mix exact names, regex, and callables
|
|
52
|
+
view = ArrayDisplay(arr, exclude_columns=[
|
|
53
|
+
FILLER_COLUMNS, # regex: ^f\d+$
|
|
54
|
+
LENGTH_COLUMNS, # regex: _len$
|
|
55
|
+
"debug_flag", # exact name
|
|
56
|
+
re.compile(r"^tmp_"), # custom regex
|
|
57
|
+
lambda name, dtype: dtype.kind == 'V', # by dtype
|
|
58
|
+
])
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
No magic defaults — `ArrayDisplay(arr)` shows all columns. Pre-built patterns
|
|
62
|
+
`FILLER_COLUMNS`, `LENGTH_COLUMNS`, and `VOID_COLUMNS` are opt-in.
|
|
63
|
+
Callables receive `(name, dtype)` for filtering by type, shape, or both.
|
|
64
|
+
|
|
65
|
+
## 2. Dtype Flattening
|
|
66
|
+
|
|
67
|
+
Expand vector and matrix fields into individual scalar columns, with
|
|
68
|
+
optional user-defined component names.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
dt = np.dtype([('id', 'i4'), ('position', 'f8', (3,))])
|
|
72
|
+
arr = np.zeros(5, dtype=dt)
|
|
73
|
+
|
|
74
|
+
# Default numeric suffixes
|
|
75
|
+
view = ArrayDisplay(arr, flatten_dtype=True)
|
|
76
|
+
view.view_columns # ['id', 'position_0', 'position_1', 'position_2']
|
|
77
|
+
|
|
78
|
+
# Custom component names
|
|
79
|
+
view = ArrayDisplay(arr, flatten_dtype=True,
|
|
80
|
+
component_names={'position': ['x', 'y', 'z']})
|
|
81
|
+
view.view_columns # ['id', 'position_x', 'position_y', 'position_z']
|
|
82
|
+
|
|
83
|
+
# Component grouping is tracked for consumers (e.g., header spanning)
|
|
84
|
+
view.field_components
|
|
85
|
+
# {'position': ['position_x', 'position_y', 'position_z']}
|
|
86
|
+
|
|
87
|
+
# Rename after construction
|
|
88
|
+
view.set_component_names('position', ['lat', 'lon', 'alt'])
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
> **Note:** Default flattened names (e.g., `position_0`) are generated by
|
|
92
|
+
> `vcti-nputils` and may truncate long field names. Use `component_names`
|
|
93
|
+
> to ensure predictable, readable column names regardless of the defaults.
|
|
94
|
+
|
|
95
|
+
## 3. Enum Mapping
|
|
96
|
+
|
|
97
|
+
Map integer ID columns to human-readable names. The mapping is stored as
|
|
98
|
+
a recipe and resolved lazily — only when presenting a bounded slice.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
dt = np.dtype([('id', 'i4'), ('element_type', 'i4'), ('value', 'f8')])
|
|
102
|
+
arr = np.array([(1, 1, 10.5), (2, 2, 20.3), (3, 1, 15.0)], dtype=dt)
|
|
103
|
+
view = ArrayDisplay(arr)
|
|
104
|
+
|
|
105
|
+
view.add_name_columns({
|
|
106
|
+
'element_type_name': ('element_type', {1: 'QUAD', 2: 'HEX'}),
|
|
107
|
+
})
|
|
108
|
+
view.view_columns # ['id', 'element_type_name', 'value']
|
|
109
|
+
|
|
110
|
+
# The mapping is NOT materialized yet — it's a recipe:
|
|
111
|
+
view.name_columns
|
|
112
|
+
# {'element_type_name': ('element_type', {1: 'QUAD', 2: 'HEX'})}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## 4. Array Slicing
|
|
116
|
+
|
|
117
|
+
Create lightweight index arrays for bounded access. The index array is
|
|
118
|
+
tiny regardless of the underlying array size.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
idx = view.to_slice(head=20) # first 20 rows
|
|
122
|
+
idx = view.to_slice(tail=10) # last 10 rows
|
|
123
|
+
idx = view.to_slice(head=10, tail=5) # first 10 + last 5
|
|
124
|
+
idx = view.to_slice(mask=arr['value'] > 50) # boolean filter
|
|
125
|
+
idx = view.to_slice(indices=np.array([0, 100, 999])) # explicit
|
|
126
|
+
|
|
127
|
+
view.array[idx] # sliced data
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## 5. Adapters
|
|
131
|
+
|
|
132
|
+
### Text table (pure numpy — no extra dependencies)
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
print(view.to_table(head=10, tail=5))
|
|
136
|
+
# id | element_type_name | value
|
|
137
|
+
# ---+-------------------+------
|
|
138
|
+
# 1 | QUAD | 10.5
|
|
139
|
+
# 2 | HEX | 20.3
|
|
140
|
+
# ...
|
|
141
|
+
# [1000 rows x 3 columns]
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### pandas DataFrame (optional dependency)
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
df = view.to_dataframe() # full array
|
|
148
|
+
df = view.to_dataframe(head=100) # first 100 rows
|
|
149
|
+
df = view.to_dataframe(resolve_names=False) # raw ID columns
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Enum names are materialized using `pd.Categorical` for memory efficiency.
|
|
153
|
+
|
|
154
|
+
### Jupyter notebooks
|
|
155
|
+
|
|
156
|
+
ArrayDisplay provides `_repr_html_()` — displays a bounded row window with
|
|
157
|
+
enum names resolved automatically.
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## Performance
|
|
162
|
+
|
|
163
|
+
Designed for large CAE arrays (millions of rows, GBs of data):
|
|
164
|
+
|
|
165
|
+
- **No array copy** — wraps a reference to the original numpy array
|
|
166
|
+
- **Lazy enum resolution** — resolved only on the displayed slice
|
|
167
|
+
- **Index arrays** — `to_slice()` returns kilobytes regardless of array size
|
|
168
|
+
- **Bounded presentation** — `to_table()` and `to_dataframe(head=N)` never touch the full array
|
|
169
|
+
- **`pd.Categorical`** — ~100x less memory than string columns for enum values
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## API Summary
|
|
174
|
+
|
|
175
|
+
| Method | Pillar | Description |
|
|
176
|
+
|--------|--------|-------------|
|
|
177
|
+
| `set_view_columns(...)` | Filtering | Configure visible columns |
|
|
178
|
+
| `include_view_columns(cols)` | Filtering | Add columns to the view |
|
|
179
|
+
| `exclude_view_columns(cols)` | Filtering | Remove columns from the view |
|
|
180
|
+
| `replace_view_columns(mapping)` | Filtering | Rename columns in the view |
|
|
181
|
+
| `set_array(arr, ...)` | Flattening | Set array with optional flattening |
|
|
182
|
+
| `set_component_names(field, names)` | Flattening | Rename flattened components |
|
|
183
|
+
| `add_name_columns(mappings)` | Enum mapping | Register lazy enum mappings |
|
|
184
|
+
| `to_slice(...)` | Slicing | Create index array for bounded access |
|
|
185
|
+
| `to_table(...)` | Adapter | Render text table (pure numpy) |
|
|
186
|
+
| `to_dataframe(...)` | Adapter | Convert to pandas DataFrame |
|
|
187
|
+
| `copy()` | Misc | Shallow copy with independent config |
|
|
188
|
+
| `append_columns(cols)` | Misc | Join additional structured columns |
|
|
189
|
+
|
|
190
|
+
## Examples
|
|
191
|
+
|
|
192
|
+
See [examples/full_pipeline.py](examples/full_pipeline.py) for a complete
|
|
193
|
+
end-to-end script demonstrating all five pillars.
|
|
194
|
+
|
|
195
|
+
## Dependencies
|
|
196
|
+
|
|
197
|
+
- [numpy](https://numpy.org/) (>=1.24) — required
|
|
198
|
+
- [vcti-nputils](https://pypi.org/project/vcti-nputils/) (>=1.0.0) — required
|
|
199
|
+
- [pandas](https://pandas.pydata.org/) (>=2.0) — optional, for `to_dataframe()` and Jupyter display
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting a Vulnerability
|
|
4
|
+
|
|
5
|
+
If you discover a security vulnerability in any vcti package, please report
|
|
6
|
+
it responsibly using one of the following channels:
|
|
7
|
+
|
|
8
|
+
- **Public repositories:** Open an issue on the GitHub repository, or email
|
|
9
|
+
security@vcollab.com if the vulnerability should not be disclosed publicly
|
|
10
|
+
before a fix is available.
|
|
11
|
+
- **Private repositories:** Email security@vcollab.com directly.
|
|
12
|
+
|
|
13
|
+
Please include:
|
|
14
|
+
|
|
15
|
+
- Description of the vulnerability
|
|
16
|
+
- Steps to reproduce
|
|
17
|
+
- Affected versions
|
|
18
|
+
- Any potential impact assessment
|
|
19
|
+
|
|
20
|
+
## Scope
|
|
21
|
+
|
|
22
|
+
vcti-array-display is a pure data presentation library that operates on
|
|
23
|
+
in-memory NumPy arrays only. It does not handle network I/O, file I/O,
|
|
24
|
+
user authentication, or external service calls. Security concerns are
|
|
25
|
+
limited to:
|
|
26
|
+
|
|
27
|
+
- Processing untrusted structured array data (malformed dtypes)
|
|
28
|
+
- Memory safety when working with numpy array views
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vcti-array-display"
|
|
7
|
+
version = "1.1.0"
|
|
8
|
+
description = "Presentation layer for NumPy structured arrays — column filtering, dtype flattening, enum mapping, array slicing, and adapters"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{name = "Visual Collaboration Technologies Inc."}
|
|
12
|
+
]
|
|
13
|
+
requires-python = ">=3.12,<3.15"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"numpy>=1.24",
|
|
16
|
+
"vcti-nputils>=1.0.0",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
pandas = ["pandas>=2.0"]
|
|
21
|
+
test = ["pytest", "pytest-cov", "pandas>=2.0"]
|
|
22
|
+
lint = ["ruff"]
|
|
23
|
+
typecheck = ["mypy", "pandas-stubs"]
|
|
24
|
+
|
|
25
|
+
[tool.setuptools.packages.find]
|
|
26
|
+
where = ["src"]
|
|
27
|
+
include = ["vcti.arraydisplay", "vcti.arraydisplay.*"]
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.package-data]
|
|
30
|
+
"vcti.arraydisplay" = ["py.typed", "*.pyi"]
|
|
31
|
+
|
|
32
|
+
[tool.setuptools]
|
|
33
|
+
zip-safe = true
|
|
34
|
+
|
|
35
|
+
[tool.pytest.ini_options]
|
|
36
|
+
addopts = "--cov=vcti.arraydisplay --cov-report=term-missing --cov-fail-under=95"
|
|
37
|
+
markers = ["benchmark: performance regression benchmarks (may allocate large arrays)"]
|
|
38
|
+
|
|
39
|
+
[tool.ruff]
|
|
40
|
+
target-version = "py312"
|
|
41
|
+
line-length = 99
|
|
42
|
+
|
|
43
|
+
[tool.ruff.lint]
|
|
44
|
+
select = ["E", "F", "W", "I", "UP"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
|
|
2
|
+
# See LICENSE for details.
|
|
3
|
+
"""vcti.arraydisplay — Presentation layer for NumPy structured arrays."""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
|
|
7
|
+
from ._exclusion import FILLER_COLUMNS, LENGTH_COLUMNS, VOID_COLUMNS, ColumnExclusion
|
|
8
|
+
from .array_display import ArrayDisplay
|
|
9
|
+
|
|
10
|
+
__version__ = version("vcti-array-display")
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"__version__",
|
|
14
|
+
"ArrayDisplay",
|
|
15
|
+
"ColumnExclusion",
|
|
16
|
+
"FILLER_COLUMNS",
|
|
17
|
+
"LENGTH_COLUMNS",
|
|
18
|
+
"VOID_COLUMNS",
|
|
19
|
+
]
|