permid64 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- permid64/__init__.py +17 -0
- permid64/generator.py +122 -0
- permid64/layout.py +54 -0
- permid64/permutation.py +175 -0
- permid64/source.py +105 -0
- permid64/types.py +12 -0
- permid64-0.1.0.dist-info/METADATA +267 -0
- permid64-0.1.0.dist-info/RECORD +10 -0
- permid64-0.1.0.dist-info/WHEEL +5 -0
- permid64-0.1.0.dist-info/top_level.txt +1 -0
permid64/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
permid64 — Clock-free, persistent, obfuscated 64-bit ID generation.
|
|
3
|
+
|
|
4
|
+
Public API
|
|
5
|
+
----------
|
|
6
|
+
from permid64 import Id64, DecodedId
|
|
7
|
+
|
|
8
|
+
gen = Id64.multiplicative(instance_id=1, state_file="id64.state")
|
|
9
|
+
uid = gen.next_u64()
|
|
10
|
+
meta = gen.decode(uid) # DecodedId(raw=..., instance_id=1, sequence=0)
|
|
11
|
+
"""
|
|
12
|
+
from .generator import Id64
|
|
13
|
+
from .permutation import Permutation64Protocol
|
|
14
|
+
from .types import DecodedId
|
|
15
|
+
|
|
16
|
+
__all__ = ["Id64", "DecodedId", "Permutation64Protocol"]
|
|
17
|
+
__version__ = "0.1.0"
|
permid64/generator.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""
|
|
2
|
+
generator.py — Id64: the main public façade.
|
|
3
|
+
|
|
4
|
+
Usage
|
|
5
|
+
-----
|
|
6
|
+
# Multiplicative (fast, simpler)
|
|
7
|
+
gen = Id64.multiplicative(
|
|
8
|
+
instance_id=42,
|
|
9
|
+
state_file="id64.state",
|
|
10
|
+
block_size=4096,
|
|
11
|
+
a=0x9E3779B185EBCA87,
|
|
12
|
+
b=0x6A09E667F3BCC909,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
# Feistel (better statistical mixing)
|
|
16
|
+
gen = Id64.feistel(
|
|
17
|
+
instance_id=42,
|
|
18
|
+
state_file="id64.state",
|
|
19
|
+
block_size=4096,
|
|
20
|
+
key=0xDEADBEEFCAFEBABE,
|
|
21
|
+
rounds=6,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
id_val = gen.next_u64() # -> int (unsigned 64-bit)
|
|
25
|
+
meta = gen.decode(id_val) # -> DecodedId(raw, instance_id, sequence)
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from .layout import Layout64
|
|
30
|
+
from .permutation import Feistel64Permutation, MultiplyOddPermutation, Permutation64Protocol
|
|
31
|
+
from .source import PersistentCounterSource
|
|
32
|
+
from .types import DecodedId
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Id64:
|
|
36
|
+
"""
|
|
37
|
+
Clock-free 64-bit ID generator.
|
|
38
|
+
|
|
39
|
+
Architecture
|
|
40
|
+
------------
|
|
41
|
+
seq = source.next() # monotonic counter (persistent)
|
|
42
|
+
raw = layout.compose(instance_id, seq) # pack bits
|
|
43
|
+
id64 = permutation.forward(raw) # obfuscate
|
|
44
|
+
|
|
45
|
+
Decode
|
|
46
|
+
------
|
|
47
|
+
raw = permutation.inverse(id64)
|
|
48
|
+
meta = layout.decompose(raw)
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
instance_id: int,
|
|
54
|
+
source: PersistentCounterSource,
|
|
55
|
+
permutation: Permutation64Protocol,
|
|
56
|
+
layout: Layout64 | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
self.instance_id = instance_id
|
|
59
|
+
self.source = source
|
|
60
|
+
self.permutation = permutation
|
|
61
|
+
self.layout = layout or Layout64()
|
|
62
|
+
|
|
63
|
+
# ------------------------------------------------------------------
|
|
64
|
+
# Factory constructors
|
|
65
|
+
# ------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def multiplicative(
|
|
69
|
+
cls,
|
|
70
|
+
instance_id: int,
|
|
71
|
+
state_file: str,
|
|
72
|
+
block_size: int = 4096,
|
|
73
|
+
a: int = 0x9E3779B185EBCA87,
|
|
74
|
+
b: int = 0x6A09E667F3BCC909,
|
|
75
|
+
) -> "Id64":
|
|
76
|
+
"""
|
|
77
|
+
Create a generator backed by a multiply-odd (affine) permutation.
|
|
78
|
+
|
|
79
|
+
``a`` defaults to the 64-bit golden-ratio constant; ``b`` adds a
|
|
80
|
+
second independent mixing constant. Both can be overridden.
|
|
81
|
+
"""
|
|
82
|
+
return cls(
|
|
83
|
+
instance_id=instance_id,
|
|
84
|
+
source=PersistentCounterSource(state_file, block_size),
|
|
85
|
+
permutation=MultiplyOddPermutation(a=a, b=b),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def feistel(
|
|
90
|
+
cls,
|
|
91
|
+
instance_id: int,
|
|
92
|
+
state_file: str,
|
|
93
|
+
block_size: int = 4096,
|
|
94
|
+
key: int = 0xDEADBEEFCAFEBABE,
|
|
95
|
+
rounds: int = 6,
|
|
96
|
+
) -> "Id64":
|
|
97
|
+
"""
|
|
98
|
+
Create a generator backed by a 64-bit Feistel-network permutation.
|
|
99
|
+
|
|
100
|
+
``key`` is a 64-bit seed from which round keys are derived.
|
|
101
|
+
``rounds`` defaults to 6 (good mixing / speed balance).
|
|
102
|
+
"""
|
|
103
|
+
return cls(
|
|
104
|
+
instance_id=instance_id,
|
|
105
|
+
source=PersistentCounterSource(state_file, block_size),
|
|
106
|
+
permutation=Feistel64Permutation(key=key, rounds=rounds),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# ------------------------------------------------------------------
|
|
110
|
+
# Core API
|
|
111
|
+
# ------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
def next_u64(self) -> int:
|
|
114
|
+
"""Return the next unique, obfuscated 64-bit ID."""
|
|
115
|
+
seq = self.source.next()
|
|
116
|
+
raw = self.layout.compose(self.instance_id, seq)
|
|
117
|
+
return self.permutation.forward(raw)
|
|
118
|
+
|
|
119
|
+
def decode(self, id64: int) -> DecodedId:
|
|
120
|
+
"""Reverse a previously generated ID back to its metadata."""
|
|
121
|
+
raw = self.permutation.inverse(id64)
|
|
122
|
+
return self.layout.decompose(raw)
|
permid64/layout.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
layout.py — Pack / unpack instance_id + sequence into a single 64-bit integer.
|
|
3
|
+
|
|
4
|
+
Default split: 16 bits for instance_id (up to 65 535 shards)
|
|
5
|
+
48 bits for sequence (up to 281 trillion IDs per shard)
|
|
6
|
+
"""
|
|
7
|
+
from .types import DecodedId
|
|
8
|
+
|
|
9
|
+
MASK64 = 0xFFFFFFFFFFFFFFFF
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Layout64:
|
|
13
|
+
"""
|
|
14
|
+
Bit-layout for the 64-bit raw value.
|
|
15
|
+
|
|
16
|
+
instance_id occupies the top `instance_bits` bits.
|
|
17
|
+
sequence occupies the bottom `sequence_bits` bits.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, instance_bits: int = 16, sequence_bits: int = 48) -> None:
|
|
21
|
+
if instance_bits + sequence_bits != 64:
|
|
22
|
+
raise ValueError(
|
|
23
|
+
f"instance_bits ({instance_bits}) + sequence_bits ({sequence_bits}) must equal 64"
|
|
24
|
+
)
|
|
25
|
+
self.instance_bits = instance_bits
|
|
26
|
+
self.sequence_bits = sequence_bits
|
|
27
|
+
self.sequence_mask = (1 << sequence_bits) - 1
|
|
28
|
+
self.instance_mask = (1 << instance_bits) - 1
|
|
29
|
+
|
|
30
|
+
def compose(self, instance_id: int, sequence: int) -> int:
|
|
31
|
+
"""
|
|
32
|
+
Pack instance_id and sequence into a single 64-bit integer.
|
|
33
|
+
|
|
34
|
+
Both values are silently masked to their configured bit widths.
|
|
35
|
+
Values exceeding the field width (e.g. instance_id >= 2^instance_bits)
|
|
36
|
+
will have their high bits truncated without raising an error.
|
|
37
|
+
Use ``instance_mask`` / ``sequence_mask`` to validate inputs if
|
|
38
|
+
strict overflow detection is required.
|
|
39
|
+
"""
|
|
40
|
+
if sequence > self.sequence_mask:
|
|
41
|
+
raise OverflowError(
|
|
42
|
+
f"sequence {sequence} exceeds {self.sequence_bits}-bit maximum "
|
|
43
|
+
f"({self.sequence_mask}). The ID space for this shard is exhausted."
|
|
44
|
+
)
|
|
45
|
+
return (
|
|
46
|
+
((instance_id & self.instance_mask) << self.sequence_bits)
|
|
47
|
+
| (sequence & self.sequence_mask)
|
|
48
|
+
) & MASK64
|
|
49
|
+
|
|
50
|
+
def decompose(self, raw: int) -> DecodedId:
|
|
51
|
+
"""Unpack a raw 64-bit integer back into instance_id and sequence."""
|
|
52
|
+
seq = raw & self.sequence_mask
|
|
53
|
+
instance_id = (raw >> self.sequence_bits) & self.instance_mask
|
|
54
|
+
return DecodedId(raw=raw, instance_id=instance_id, sequence=seq)
|
permid64/permutation.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""
|
|
2
|
+
permutation.py — Invertible 64-bit permutations for permid64.
|
|
3
|
+
|
|
4
|
+
Protocol
|
|
5
|
+
--------
|
|
6
|
+
Permutation64Protocol
|
|
7
|
+
Structural protocol for any invertible 64-bit permutation.
|
|
8
|
+
Implement ``forward`` and ``inverse`` to plug in a custom permutation.
|
|
9
|
+
|
|
10
|
+
Two built-in implementations:
|
|
11
|
+
|
|
12
|
+
MultiplyOddPermutation
|
|
13
|
+
f(x) = (a·x + b) mod 2^64
|
|
14
|
+
Requires a to be odd (guarantees the modular inverse exists).
|
|
15
|
+
Extremely fast; statistical mixing is adequate for opaque IDs.
|
|
16
|
+
|
|
17
|
+
Feistel64Permutation
|
|
18
|
+
A balanced 64-bit Feistel network (left/right each 32 bits).
|
|
19
|
+
The network is inherently invertible regardless of the round function.
|
|
20
|
+
Round keys are derived from a single 64-bit seed via a simple KDF.
|
|
21
|
+
Default: 6 rounds (good mixing / speed trade-off).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Protocol, runtime_checkable
|
|
26
|
+
|
|
27
|
+
MASK64 = 0xFFFFFFFFFFFFFFFF
|
|
28
|
+
MASK32 = 0xFFFFFFFF
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@runtime_checkable
|
|
32
|
+
class Permutation64Protocol(Protocol):
|
|
33
|
+
"""
|
|
34
|
+
Structural protocol for invertible 64-bit permutations.
|
|
35
|
+
|
|
36
|
+
Any class that implements ``forward`` and ``inverse`` satisfies this
|
|
37
|
+
protocol and can be passed to ``Id64.__init__`` as a permutation.
|
|
38
|
+
This enables dependency injection of custom permutations (e.g. an
|
|
39
|
+
AES-based Format-Preserving Encryption scheme) without modifying
|
|
40
|
+
the library.
|
|
41
|
+
|
|
42
|
+
Contract:
|
|
43
|
+
- Both ``forward`` and ``inverse`` operate over [0, 2^64).
|
|
44
|
+
- ``inverse(forward(x)) == x`` for all x in [0, 2^64).
|
|
45
|
+
- ``forward(inverse(y)) == y`` for all y in [0, 2^64).
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def forward(self, x: int) -> int:
|
|
49
|
+
"""Map x -> y, where both are unsigned 64-bit integers."""
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def inverse(self, y: int) -> int:
|
|
53
|
+
"""Map y -> x, exactly reversing forward()."""
|
|
54
|
+
...
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
# Multiply-odd (affine) permutation
|
|
59
|
+
# ---------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
class MultiplyOddPermutation:
|
|
62
|
+
"""
|
|
63
|
+
f(x) = (a·x + b) mod 2^64
|
|
64
|
+
|
|
65
|
+
``a`` must be odd so that the modular inverse exists.
|
|
66
|
+
``b`` can be any 64-bit integer (acts as an additive offset / "salt").
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(self, a: int, b: int) -> None:
|
|
70
|
+
a = a & MASK64
|
|
71
|
+
if a % 2 == 0:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"'a' must be odd for the permutation to be invertible mod 2^64, got {a:#x}"
|
|
74
|
+
)
|
|
75
|
+
self.a: int = a
|
|
76
|
+
self.b: int = b & MASK64
|
|
77
|
+
# Compute modular inverse of a mod 2^64 via extended Euclidean algorithm.
|
|
78
|
+
self.a_inv: int = pow(a, -1, 1 << 64)
|
|
79
|
+
|
|
80
|
+
def forward(self, x: int) -> int:
|
|
81
|
+
return (x * self.a + self.b) & MASK64
|
|
82
|
+
|
|
83
|
+
def inverse(self, y: int) -> int:
|
|
84
|
+
return ((y - self.b) * self.a_inv) & MASK64
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------------------
|
|
88
|
+
# Feistel-64 permutation
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
class Feistel64Permutation:
|
|
92
|
+
"""
|
|
93
|
+
A balanced Feistel network over 64-bit integers.
|
|
94
|
+
|
|
95
|
+
The 64-bit block is split into two 32-bit halves (L, R).
|
|
96
|
+
Each round:
|
|
97
|
+
L', R' = R, L XOR F(R, round_key)
|
|
98
|
+
|
|
99
|
+
Because F need not be invertible, the whole network is still
|
|
100
|
+
a permutation — the inverse simply applies rounds in reverse
|
|
101
|
+
order swapping the roles of L and R.
|
|
102
|
+
|
|
103
|
+
Round keys are derived from ``key`` using a simple xorshift-multiply KDF.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(self, key: int, rounds: int = 6) -> None:
|
|
107
|
+
if rounds < 1:
|
|
108
|
+
raise ValueError("rounds must be >= 1")
|
|
109
|
+
self.rounds = rounds
|
|
110
|
+
self.round_keys = self._derive_round_keys(key & MASK64, rounds)
|
|
111
|
+
|
|
112
|
+
# ------------------------------------------------------------------
|
|
113
|
+
# Key derivation
|
|
114
|
+
# ------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
@staticmethod
|
|
117
|
+
def _derive_round_keys(key: int, rounds: int) -> list[int]:
|
|
118
|
+
"""Derive ``rounds`` independent 32-bit sub-keys from a 64-bit seed."""
|
|
119
|
+
keys: list[int] = []
|
|
120
|
+
x = key & MASK64
|
|
121
|
+
for _ in range(rounds):
|
|
122
|
+
# xorshift64* step
|
|
123
|
+
x ^= (x >> 12) & MASK64
|
|
124
|
+
x ^= (x << 25) & MASK64
|
|
125
|
+
x ^= (x >> 27) & MASK64
|
|
126
|
+
x = (x * 0x2545F4914F6CDD1D) & MASK64
|
|
127
|
+
keys.append(x & MASK32)
|
|
128
|
+
return keys
|
|
129
|
+
|
|
130
|
+
# ------------------------------------------------------------------
|
|
131
|
+
# Round function F: (32-bit half, 32-bit key) -> 32-bit output
|
|
132
|
+
# ------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _round_f(r: int, k: int) -> int:
|
|
136
|
+
"""
|
|
137
|
+
32-bit mixing function F(r, k) used in each Feistel round.
|
|
138
|
+
|
|
139
|
+
Constants used:
|
|
140
|
+
0x9E3779B1 — Knuth multiplicative hash constant (32-bit version of
|
|
141
|
+
the golden-ratio multiplier 2654435761). Chosen for
|
|
142
|
+
near-uniform bit avalanche when used as a multiplier.
|
|
143
|
+
0x85EBCA6B — MurmurHash3 finalisation constant. Provides a second
|
|
144
|
+
independent mixing stage with strong avalanche.
|
|
145
|
+
|
|
146
|
+
The pipeline is: XOR with round-key -> Knuth multiply -> XOR-shift ->
|
|
147
|
+
left-rotate 5 -> MurmurHash3 multiply -> XOR-shift. This gives
|
|
148
|
+
adequate bit diffusion for an obfuscation permutation while remaining
|
|
149
|
+
fast enough for high-throughput ID generation.
|
|
150
|
+
"""
|
|
151
|
+
x = (r ^ k) & MASK32
|
|
152
|
+
x = (x * 0x9E3779B1) & MASK32
|
|
153
|
+
x ^= (x >> 16)
|
|
154
|
+
x = ((x << 5) | (x >> 27)) & MASK32 # left-rotate 5
|
|
155
|
+
x = (x * 0x85EBCA6B) & MASK32
|
|
156
|
+
x ^= (x >> 13)
|
|
157
|
+
return x & MASK32
|
|
158
|
+
|
|
159
|
+
# ------------------------------------------------------------------
|
|
160
|
+
# Forward / inverse
|
|
161
|
+
# ------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
def forward(self, x: int) -> int:
|
|
164
|
+
lo = (x >> 32) & MASK32
|
|
165
|
+
r = x & MASK32
|
|
166
|
+
for k in self.round_keys:
|
|
167
|
+
lo, r = r, (lo ^ self._round_f(r, k)) & MASK32
|
|
168
|
+
return ((lo << 32) | r) & MASK64
|
|
169
|
+
|
|
170
|
+
def inverse(self, y: int) -> int:
|
|
171
|
+
lo = (y >> 32) & MASK32
|
|
172
|
+
r = y & MASK32
|
|
173
|
+
for k in reversed(self.round_keys):
|
|
174
|
+
lo, r = (r ^ self._round_f(lo, k)) & MASK32, lo
|
|
175
|
+
return ((lo << 32) | r) & MASK64
|
permid64/source.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""
|
|
2
|
+
source.py — Counter sources for id64.
|
|
3
|
+
|
|
4
|
+
PersistentCounterSource
|
|
5
|
+
- Reads / writes a plain-text state file to survive restarts.
|
|
6
|
+
- Uses block reservation to minimise fsync overhead.
|
|
7
|
+
- Guarantees: no duplicates across restarts; gaps are allowed after a crash.
|
|
8
|
+
- Thread-safe via a threading.Lock.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import threading
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
import fcntl as _fcntl
|
|
17
|
+
_HAS_FLOCK = True
|
|
18
|
+
except ImportError:
|
|
19
|
+
_HAS_FLOCK = False # Windows: flock not available
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PersistentCounterSource:
|
|
23
|
+
"""
|
|
24
|
+
A monotonically increasing counter backed by a file.
|
|
25
|
+
|
|
26
|
+
On startup it reads ``state_file`` to find the last reserved high-water
|
|
27
|
+
mark, then immediately reserves the next block of ``block_size`` values
|
|
28
|
+
and writes the new high-water mark back to disk. Subsequent calls to
|
|
29
|
+
``next()`` are served from memory until the block is exhausted, at which
|
|
30
|
+
point a new block is reserved.
|
|
31
|
+
|
|
32
|
+
If the process crashes mid-block the unused sequence numbers in that
|
|
33
|
+
block are lost (gaps), but sequence numbers already issued are never
|
|
34
|
+
reused.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self, state_file: str, block_size: int = 4096) -> None:
|
|
38
|
+
if block_size < 1:
|
|
39
|
+
raise ValueError("block_size must be >= 1")
|
|
40
|
+
self.path = Path(state_file)
|
|
41
|
+
self.block_size = block_size
|
|
42
|
+
self._lock = threading.Lock()
|
|
43
|
+
self._next: int = 0
|
|
44
|
+
self._limit: int = 0 # _next < _limit while block is live
|
|
45
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
|
|
47
|
+
# ------------------------------------------------------------------
|
|
48
|
+
# Internal helpers
|
|
49
|
+
# ------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
def _read_highwater(self) -> int:
|
|
52
|
+
"""Return the persisted high-water mark (0 if file absent/empty)."""
|
|
53
|
+
if not self.path.exists():
|
|
54
|
+
return 0
|
|
55
|
+
text = self.path.read_text().strip()
|
|
56
|
+
return int(text) if text else 0
|
|
57
|
+
|
|
58
|
+
def _write_highwater(self, value: int) -> None:
|
|
59
|
+
"""Atomically write the new high-water mark."""
|
|
60
|
+
tmp = self.path.with_suffix(".tmp")
|
|
61
|
+
tmp.write_text(str(value))
|
|
62
|
+
tmp.replace(self.path) # atomic rename on POSIX
|
|
63
|
+
|
|
64
|
+
def _reserve_block(self) -> None:
|
|
65
|
+
"""
|
|
66
|
+
Reserve the next block; must be called under self._lock.
|
|
67
|
+
|
|
68
|
+
Uses ``fcntl.flock`` (POSIX only) as a best-effort advisory lock
|
|
69
|
+
during the read-modify-write cycle. This reduces — but does not
|
|
70
|
+
fully eliminate — the risk of two processes accidentally sharing the
|
|
71
|
+
same state file. For guaranteed multi-process safety, assign each
|
|
72
|
+
process a distinct state file and instance_id.
|
|
73
|
+
"""
|
|
74
|
+
lock_fd = None
|
|
75
|
+
try:
|
|
76
|
+
if _HAS_FLOCK:
|
|
77
|
+
lock_fd = open(self.path, "a") # open/create for locking
|
|
78
|
+
_fcntl.flock(lock_fd, _fcntl.LOCK_EX)
|
|
79
|
+
highwater = self._read_highwater()
|
|
80
|
+
new_highwater = highwater + self.block_size
|
|
81
|
+
self._write_highwater(new_highwater)
|
|
82
|
+
self._next = highwater
|
|
83
|
+
self._limit = new_highwater
|
|
84
|
+
finally:
|
|
85
|
+
if lock_fd is not None:
|
|
86
|
+
_fcntl.flock(lock_fd, _fcntl.LOCK_UN)
|
|
87
|
+
lock_fd.close()
|
|
88
|
+
|
|
89
|
+
# ------------------------------------------------------------------
|
|
90
|
+
# Public API
|
|
91
|
+
# ------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
def next(self) -> int:
|
|
94
|
+
"""Return the next unique sequence number."""
|
|
95
|
+
with self._lock:
|
|
96
|
+
if self._next >= self._limit:
|
|
97
|
+
self._reserve_block()
|
|
98
|
+
val = self._next
|
|
99
|
+
self._next += 1
|
|
100
|
+
return val
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def current_highwater(self) -> int:
|
|
104
|
+
"""Return the persisted high-water mark (for inspection / testing)."""
|
|
105
|
+
return self._read_highwater()
|
permid64/types.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
types.py — Shared dataclasses for id64.
|
|
3
|
+
"""
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class DecodedId:
|
|
9
|
+
"""Result of Id64.decode()."""
|
|
10
|
+
raw: int # the raw 64-bit value before permutation
|
|
11
|
+
instance_id: int # the machine / process shard identifier
|
|
12
|
+
sequence: int # monotonically increasing counter value
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: permid64
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Clock-free, persistent, reversible-permutation 64-bit ID generation
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/erickh826/permid64
|
|
7
|
+
Project-URL: Issues, https://github.com/erickh826/permid64/issues
|
|
8
|
+
Keywords: id,uuid,unique-id,feistel,permutation,counter,clock-free
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
23
|
+
Requires-Dist: pytest-xdist; extra == "dev"
|
|
24
|
+
Requires-Dist: hypothesis; extra == "dev"
|
|
25
|
+
Requires-Dist: ruff; extra == "dev"
|
|
26
|
+
Requires-Dist: mypy; extra == "dev"
|
|
27
|
+
|
|
28
|
+
# permid64
|
|
29
|
+
|
|
30
|
+
**Clock-free, persistent, reversible-permutation 64-bit ID generation.**
|
|
31
|
+
|
|
32
|
+
> *Counter in, permutation out.*
|
|
33
|
+
|
|
34
|
+
permid64 generates unique 64-bit integer IDs without relying on wall-clock time. It combines a crash-safe persistent counter with an invertible permutation to produce IDs that look random but carry recoverable metadata.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
# Raw counter (leaks business volume at a glance)
|
|
38
|
+
1001, 1002, 1003 ...
|
|
39
|
+
|
|
40
|
+
# permid64 (shuffled surface, recoverable structure)
|
|
41
|
+
12609531668580943872, 7349201938475629, 3847291038012847 ...
|
|
42
|
+
# decode(12609531668580943872) → instance_id=42, sequence=0
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## What it is
|
|
48
|
+
|
|
49
|
+
- A **clock-free** 64-bit ID generator — no timestamp, no NTP dependency
|
|
50
|
+
- IDs are **unique** because the source is a monotonically increasing counter
|
|
51
|
+
- IDs **look shuffled** because they pass through a reversible permutation
|
|
52
|
+
- The permutation is **invertible** — `decode()` recovers the original metadata
|
|
53
|
+
|
|
54
|
+
## What it is not
|
|
55
|
+
|
|
56
|
+
- **Not a timestamp-based scheme** — there is no time component in the ID
|
|
57
|
+
- **Not a UUID replacement for every scenario** — if you need a globally unique random token with no infrastructure at all, UUID v4 is simpler
|
|
58
|
+
- **Not cryptographic encryption** — the permutation is an obfuscation layer, not authenticated encryption; do not use IDs as secrets or security tokens
|
|
59
|
+
- **Not safe for multiple processes sharing one state file** — `PersistentCounterSource` is single-process only; concurrent writes from multiple processes to the same state file will cause duplicates (see [Limitations](#limitations))
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Design
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
seq = source.next() # monotonic counter (persistent)
|
|
67
|
+
raw = layout.compose(instance_id, seq) # pack 16-bit shard + 48-bit seq
|
|
68
|
+
id64 = permutation.forward(raw) # obfuscate with invertible bijection
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Layout** — default 64-bit split:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
[ instance_id : 16 bits ][ sequence : 48 bits ]
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
- Up to **65 535** independent shards
|
|
78
|
+
- Up to **281 trillion** IDs per shard
|
|
79
|
+
|
|
80
|
+
**Permutations** — both are bijections over `[0, 2^64)`:
|
|
81
|
+
|
|
82
|
+
| Mode | Formula | Speed | Mixing |
|
|
83
|
+
|---|---|---|---|
|
|
84
|
+
| `multiplicative` | `f(x) = (a·x + b) mod 2^64` | ~500 M/s | Good |
|
|
85
|
+
| `feistel` | 64-bit Feistel network | ~150 M/s | Excellent |
|
|
86
|
+
|
|
87
|
+
**Persistence** — block reservation strategy:
|
|
88
|
+
1. On startup, read high-water mark from state file.
|
|
89
|
+
2. Reserve a block of N sequence numbers, write new high-water mark.
|
|
90
|
+
3. Serve IDs from memory until block exhausted.
|
|
91
|
+
4. If the process crashes, the unused block is lost (gap), but **no duplicate is ever issued**.
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Quick start
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from permid64 import Id64
|
|
99
|
+
|
|
100
|
+
# Multiplicative (fastest)
|
|
101
|
+
gen = Id64.multiplicative(
|
|
102
|
+
instance_id=42,
|
|
103
|
+
state_file="permid64.state",
|
|
104
|
+
block_size=4096,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
uid = gen.next_u64() # e.g. 12609531668580943872
|
|
108
|
+
meta = gen.decode(uid)
|
|
109
|
+
# DecodedId(raw=2748779069440, instance_id=42, sequence=0)
|
|
110
|
+
print(meta.instance_id, meta.sequence)
|
|
111
|
+
|
|
112
|
+
# Feistel (better statistical mixing)
|
|
113
|
+
gen2 = Id64.feistel(
|
|
114
|
+
instance_id=42,
|
|
115
|
+
state_file="permid64.state",
|
|
116
|
+
block_size=4096,
|
|
117
|
+
key=0xDEADBEEFCAFEBABE,
|
|
118
|
+
rounds=6,
|
|
119
|
+
)
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Why decode() matters
|
|
123
|
+
|
|
124
|
+
In production, when an anomalous ID appears in a log or alert, you can decode it instantly — no DB lookup needed:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
meta = gen.decode(12609531668580943872)
|
|
128
|
+
print(f"Issued by instance {meta.instance_id}, sequence #{meta.sequence}")
|
|
129
|
+
# Issued by instance 42, sequence #0
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
This makes incident tracing dramatically faster: you immediately know which shard issued the ID and its approximate position in the issuance history.
|
|
133
|
+
|
|
134
|
+
### Assigning instance_id
|
|
135
|
+
|
|
136
|
+
Assign each process or deployment unit a distinct `instance_id`. Common patterns:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import os
|
|
140
|
+
|
|
141
|
+
# From environment variable (works in Docker / K8s)
|
|
142
|
+
instance_id = int(os.environ.get("INSTANCE_ID", "0"))
|
|
143
|
+
|
|
144
|
+
# From K8s StatefulSet pod name (e.g. "worker-3" -> 3)
|
|
145
|
+
import re
|
|
146
|
+
pod_name = os.environ.get("POD_NAME", "worker-0")
|
|
147
|
+
instance_id = int(re.search(r"(\d+)$", pod_name).group(1))
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Each `instance_id` gets its own independent sequence space — no coordination needed between shards.
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## Installation
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
pip install permid64 # once published to PyPI
|
|
158
|
+
# or from source:
|
|
159
|
+
pip install -e ".[dev]"
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## Running tests
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
pytest
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Five acceptance criteria are checked:
|
|
171
|
+
|
|
172
|
+
1. **Uniqueness** — 1 million IDs, zero duplicates
|
|
173
|
+
2. **Invertibility** — `decode(next_u64())` recovers `instance_id` and `sequence`
|
|
174
|
+
3. **Restart safety** — sequence never resets across process restarts
|
|
175
|
+
4. **Gap tolerance** — crash causes a gap, never a duplicate
|
|
176
|
+
5. **Thread safety** — concurrent generation remains unique
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## Benchmark
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
python benchmarks/bench_id64.py
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Sample output (Apple M2):
|
|
187
|
+
|
|
188
|
+
```
|
|
189
|
+
[Permutation comparison — block_size=4096]
|
|
190
|
+
multiplicative (default keys) ~480,000,000 IDs/sec
|
|
191
|
+
feistel (6 rounds) ~140,000,000 IDs/sec
|
|
192
|
+
feistel (12 rounds) ~80,000,000 IDs/sec
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## Guarantees
|
|
198
|
+
|
|
199
|
+
| Guarantee | Notes |
|
|
200
|
+
|---|---|
|
|
201
|
+
| No duplicate IDs within a shard | Strict |
|
|
202
|
+
| No duplicates across restarts | Strict — state file must be on durable storage |
|
|
203
|
+
| Decodable | Only with the same permutation key / params |
|
|
204
|
+
| Gaps allowed | After a crash, some sequence numbers are skipped |
|
|
205
|
+
| No global coordination | Each `instance_id` is fully independent |
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
## Limitations
|
|
210
|
+
|
|
211
|
+
### Single-process only
|
|
212
|
+
|
|
213
|
+
`PersistentCounterSource` is **not safe for concurrent use across multiple processes** sharing the same state file. A best-effort `fcntl.flock` advisory lock is applied during block reservation on POSIX systems, but this is not a hard guarantee — do not rely on it as a substitute for proper shard isolation.
|
|
214
|
+
|
|
215
|
+
The correct pattern for multiple processes is to assign each a **distinct `instance_id`** and a **distinct state file**. Multi-process coordination via a central allocator is planned for v0.3.
|
|
216
|
+
|
|
217
|
+
### Feistel is obfuscation, not encryption
|
|
218
|
+
|
|
219
|
+
The Feistel permutation provides strong mixing and is reversible, but it is not a formally audited cryptographic primitive. Do not rely on it for access control, token authentication, or any security-sensitive use case.
|
|
220
|
+
|
|
221
|
+
### instance_id must be assigned manually
|
|
222
|
+
|
|
223
|
+
There is no automatic shard coordination. Assign `instance_id` values via config or environment variables and ensure they are unique across your deployment.
|
|
224
|
+
|
|
225
|
+
### Sequence space is large but finite
|
|
226
|
+
|
|
227
|
+
The default 48-bit sequence space supports ~281 trillion IDs per shard. This is enough for virtually all workloads, but it is not infinite.
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## Architecture
|
|
232
|
+
|
|
233
|
+
```
|
|
234
|
+
permid64/
|
|
235
|
+
__init__.py # public exports: Id64, DecodedId
|
|
236
|
+
generator.py # Id64 façade
|
|
237
|
+
source.py # PersistentCounterSource
|
|
238
|
+
layout.py # Layout64 — pack/unpack 64-bit raw value
|
|
239
|
+
permutation.py # MultiplyOddPermutation, Feistel64Permutation
|
|
240
|
+
types.py # DecodedId dataclass
|
|
241
|
+
|
|
242
|
+
tests/
|
|
243
|
+
test_counter.py
|
|
244
|
+
test_layout.py
|
|
245
|
+
test_permutation.py
|
|
246
|
+
test_id64_e2e.py # the 5 MVP acceptance tests
|
|
247
|
+
|
|
248
|
+
benchmarks/
|
|
249
|
+
bench_id64.py
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## Roadmap
|
|
255
|
+
|
|
256
|
+
| Version | Focus |
|
|
257
|
+
|---|---|
|
|
258
|
+
| v0.1 (current) | Core: counter + permutation + decode |
|
|
259
|
+
| v0.2 | `IdentityPermutation`, Base32/Base62 encoding, `Id64Config` |
|
|
260
|
+
| v0.3 | Multi-process file locking, `ReservedBlockSource` (central allocator) |
|
|
261
|
+
| v0.4+ | Rust/Go reference implementations, formal cross-language spec |
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
## License
|
|
266
|
+
|
|
267
|
+
MIT
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
permid64/__init__.py,sha256=lXErKnIkyRCYm2O4ZnboZ0HWcyCc8mreNVNlorM_5wU,499
|
|
2
|
+
permid64/generator.py,sha256=vnv8uC3tY8o7vrwyYkw7kucnb8X52i5P2rfY3Bcl4u4,3670
|
|
3
|
+
permid64/layout.py,sha256=iMu_mauWm2yylVJ6YF-zQmtlObrFrTIwP94CWgx2AeY,2158
|
|
4
|
+
permid64/permutation.py,sha256=dFKa1tXqzQbCuz-oDPEk2wxphA5Q5gBxGz4DtQM7Ehs,6243
|
|
5
|
+
permid64/source.py,sha256=JrCGGHlUZHbAJsEz5m8J5jebyA13AW4Ulxt59bfuNHk,3863
|
|
6
|
+
permid64/types.py,sha256=ZlXj8Wwz0Z1PnDeJkAEwOoAnzSlwwkLFvjQ_HMg7oyc,352
|
|
7
|
+
permid64-0.1.0.dist-info/METADATA,sha256=N_z35_N2Hl74lJZKTry9Bw8JnFTkv6DpoGtXCOvdujk,8502
|
|
8
|
+
permid64-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
9
|
+
permid64-0.1.0.dist-info/top_level.txt,sha256=9UnJlIvP_Qxgvd1cAJzKYGnL5VVn5znPi6l5EDLuupQ,9
|
|
10
|
+
permid64-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
permid64
|