cimo 0.0.3__tar.gz → 0.0.4__tar.gz

{cimo-0.0.3 → cimo-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cimo
-Version: 0.0.3
+Version: 0.0.4
 Summary: A C compiler written in pure Python: C kernels in, machine code out.
 Author: Fariz Rahman
 License: MIT

{cimo-0.0.3 → cimo-0.0.4}/cimo/__init__.py

@@ -1,5 +1,5 @@
 """cimo: a C compiler written in pure Python."""
 
-from .driver import compile, load, Module, CompileError, KernelError
+from .driver import compile, load, Module, CompileError, KernelError, add_cache_entry
 
-__version__ = "0.0.3"
+__version__ = "0.0.4"

{cimo-0.0.3 → cimo-0.0.4}/cimo/assembler.py

@@ -236,9 +236,9 @@ def _split(s):
                 q = False
         elif c == '"':
             q = True
-        elif c == '[':
+        elif c in '[{':
             depth += 1
-        elif c == ']':
+        elif c in ']}':
             depth -= 1
         elif c == ',' and depth == 0:
             out.append(s[start:i].strip())
@@ -329,6 +329,8 @@ def _bitmask(imm, width):
 
 def _logimm(value, width):
     """GNU as: the bits above a 32-bit operation's width must be all zeros or all ones."""
+    if not -(1 << 64) < value < 1 << 64:
+        return None
     value &= M64
     if width == 32:
         hi = value >> 32
@@ -339,6 +341,8 @@ def _logimm(value, width):
 
 def _wide(value, is32):
     """(imm16, shift) if value fits a single movz, else None (GNU aarch64_wide_constant_p)."""
+    if not -(1 << 64) < value < 1 << 64:
+        return None
     value &= M64
     if is32:
         if value >> 32 not in (0, 0xffffffff):
@@ -380,6 +384,12 @@ def _shiftop(rest):
 _ADDSUB = {'add': (0, 0), 'adds': (0, 1), 'sub': (1, 0), 'subs': (1, 1)}
 
 
+def _notzr(r):
+    """r isn't xzr/wzr: for a field in which 31 means sp, where zr would silently become sp."""
+    if not r[2] and r[1] == 31:
+        raise AssemblerError('the zero register is not allowed here (31 would be sp)')
+
+
 def _addsub(a, op, s, rd, rn, ops3, ops_rest):
     """ops3: the third operand string; ops_rest: [optional shift operand]."""
     if _isreg(ops3):
@@ -394,13 +404,19 @@ def _addsub(a, op, s, rd, rn, ops3, ops_rest):
                     raise AssemblerError('shift amount out of range')
                 return (0x0b000000 | _sf(rd) | op << 30 | s << 29 | _SHIFTOP[how] << 22
                         | rm[1] << 16 | n << 10 | rn[1] << 5 | rd[1])
-            if how not in _EXTOP or rm[0] != 'w' or rd[0] != 'x' or not 0 <= n <= 4:
+            if how not in _EXTOP or rm[0] != 'w' or rd[0] != 'x' or rn[0] != 'x' or not 0 <= n <= 4:
                 raise AssemblerError('bad extended register %r' % ops_rest[0])
+            _notzr(rn)  # field 31 is sp here, in Rn always and in Rd unless the flags are set
+            if not s:
+                _notzr(rd)
             return (0x0b200000 | _sf(rd) | op << 30 | s << 29 | rm[1] << 16 | _EXTOP[how] << 13
                     | n << 10 | rn[1] << 5 | rd[1])
         if rd[2] or rn[2]:
             # extended register form (needed when sp is involved): UXTX / UXTW
             _same(rd, rn)
+            _notzr(rn)
+            if not s:
+                _notzr(rd)
             opt = 3 if rd[0] == 'x' else 2
             if rm[0] != rd[0]:
                 raise AssemblerError('operand size mismatch')
@@ -492,7 +508,7 @@ def _i_logic(a, mn, ops):
                 | rd[1])
     if imm_op is None or len(ops) != 3:
         raise AssemblerError('%s takes registers' % mn)
-    rd = _reg(ops[0], sp=mn != 'ands')
+    rd = _reg(ops[0], sp=mn != 'ands', zr=mn == 'ands')
     _same(rd, rn)
     width = 64 if rd[0] == 'x' else 32
     enc = _logimm(_imm(ops[2], True), width)
@@ -643,20 +659,24 @@ def _i_mov(a, mn, ops):
         rd, rm = _reg(ops[0], sp=True), _reg(ops[1], sp=True)
         _same(rd, rm)
         if rd[2] or rm[2]:
+            _notzr(rd), _notzr(rm)
             return 0x11000000 | _sf(rd) | rm[1] << 5 | rd[1]  # add Rd, Rn, #0
         return 0x2a0003e0 | _sf(rd) | rm[1] << 16 | rd[1]  # orr Rd, zr, Rm
     rd = _reg(ops[0], sp=True)
     if not _isimm(ops[1]):
         raise AssemblerError('unsupported mov operand %r' % ops[1])
     v = _imm(ops[1])
+    if not -(1 << 64) < v < 1 << 64:
+        raise AssemblerError('immediate %s does not fit 64 bits' % ops[1])  # GNU as: (-2^64, 2^64), wrapped
     is32 = rd[0] == 'w'
     if not rd[2]:
         w = _wide(v, is32)
         if w is not None:
             return 0x52800000 | _sf(rd) | (w[1] // 16) << 21 | w[0] << 5 | rd[1]
-        w = _wide(~v & (0xffffffff if is32 else M64), is32)
+        w = _wide(~v & M64, is32)  # the whole value, so _wide sees bits above a w register's 32
         if w is not None:
             return 0x12800000 | _sf(rd) | (w[1] // 16) << 21 | w[0] << 5 | rd[1]
+    _notzr(rd)  # the orr below writes sp for register 31
     enc = _logimm(v, 32 if is32 else 64)
     if enc is None:
         raise AssemblerError('immediate %s cannot be moved by a single instruction' % ops[1])
@@ -761,6 +781,128 @@ def _i_simd(a, mn, ops):  # cnt Vd.8B, Vn.8B / addv Bd, Vn.8B (the popcount buil
     return (0x0e205800 if mn == 'cnt' else 0x0e31b800) | int(n.group(1)) << 5 | int(d.group(1))
 
 
+_VREG8H = re.compile(r'^v([0-9]|[12][0-9]|3[01])\.8h$')
+_LIST3 = re.compile(r'^\{\s*v([0-9]|[12][0-9]|3[01])\.(8b|16b)\s*,\s*v([0-9]|[12][0-9]|3[01])\.\2\s*,'
+                    r'\s*v([0-9]|[12][0-9]|3[01])\.\2\s*\}$')
+_VREG16B = re.compile(r'^v([0-9]|[12][0-9]|3[01])\.16b$')
+
+
+def _i_bytes(a, mn, ops):
+    """cimo (vectorize's loops over bytes): ushll Vd.8H, Vn.8B, #0 (uxtl); xtn Vd.8B, Vn.8H; ld3 {Vt.8B, Vt+1.8B, Vt+2.8B},
+    [Xn], consecutive registers modulo 32."""
+    if mn == 'ld3':
+        if len(ops) != 2:
+            raise AssemblerError('ld3 needs 2 operands')
+        m = _LIST3.match(ops[0].strip().lower())
+        t = [int(m.group(1)), int(m.group(3)), int(m.group(4))] if m else None
+        if t is None or t[1] != (t[0] + 1) % 32 or t[2] != (t[0] + 2) % 32:
+            raise AssemblerError('ld3 takes three consecutive .8b registers: %r' % ops[0])
+        b = ops[1].strip().lower()
+        if not (b.startswith('[') and b.endswith(']')):
+            raise AssemblerError('ld3 takes [Xn]')
+        n = _reg(b[1:-1].strip(), 'x', sp=True, zr=False)[1]
+        return (0x4c404000 if m.group(2) == '16b' else 0x0c404000) | n << 5 | t[0]
+    if mn in ('umull', 'umlal', 'umull2', 'umlal2'):  # Vd.8H, Vn.8B, Vm.8B (the 2 forms: .16B, the upper halves)
+        if len(ops) != 3:
+            raise AssemblerError('%s needs 3 operands' % mn)
+        src = _VREG16B if mn.endswith('2') else _VREG8B
+        d = _VREG8H.match(ops[0].strip().lower())
+        n, m = src.match(ops[1].strip().lower()), src.match(ops[2].strip().lower())
+        if not d or not n or not m:
+            raise AssemblerError('unsupported operands for %s' % mn)
+        return ((0x2e20c000 if mn.startswith('umull') else 0x2e208000) | (0x40000000 if mn.endswith('2') else 0)
+                | int(m.group(1)) << 16 | int(n.group(1)) << 5 | int(d.group(1)))
+    if mn == 'ushll2':
+        if len(ops) != 3 or _imm(ops[2], True) != 0:
+            raise AssemblerError('ushll2 takes Vd.8h, Vn.16b, #0')
+        d, n = _VREG8H.match(ops[0].strip().lower()), _VREG16B.match(ops[1].strip().lower())
+        if not d or not n:
+            raise AssemblerError('unsupported operands for ushll2')
+        return 0x6f08a400 | int(n.group(1)) << 5 | int(d.group(1))
+    if mn == 'ushll':
+        if len(ops) != 3 or _imm(ops[2], True) != 0:
+            raise AssemblerError('ushll takes Vd.8h, Vn.8b, #0')
+        d, n = _VREG8H.match(ops[0].strip().lower()), _VREG8B.match(ops[1].strip().lower())
+        if not d or not n:
+            raise AssemblerError('unsupported operands for ushll')
+        return 0x2f08a400 | int(n.group(1)) << 5 | int(d.group(1))
+    if mn == 'shrn':  # Vd.8B, Vn.8H, #s
+        if len(ops) != 3:
+            raise AssemblerError('shrn needs 3 operands')
+        d, n = _VREG8B.match(ops[0].strip().lower()), _VREG8H.match(ops[1].strip().lower())
+        s = _imm(ops[2], True)
+        if not d or not n or not 1 <= s <= 8:
+            raise AssemblerError('unsupported operands for shrn')
+        return 0x0f008400 | (16 - s) << 16 | int(n.group(1)) << 5 | int(d.group(1))
+    if len(ops) != 2:  # xtn
+        raise AssemblerError('xtn needs 2 operands')
+    d, n = _VREG8B.match(ops[0].strip().lower()), _VREG8H.match(ops[1].strip().lower())
+    if not d or not n:
+        raise AssemblerError('unsupported operands for xtn')
+    return 0x0e212800 | int(n.group(1)) << 5 | int(d.group(1))
+
+
+_VARR = re.compile(r'^v([0-9]|[12][0-9]|3[01])\.(8b|16b|4h|8h|2s|4s|2d)$')
+# conversions between vector arrangements: (destination arrangement, source arrangement) -> base encoding (Q in it)
+_VCVT = {
+    'sshll': {('8h', '8b'): 0x0f08a400, ('4s', '4h'): 0x0f10a400, ('2d', '2s'): 0x0f20a400},
+    'sshll2': {('8h', '16b'): 0x4f08a400, ('4s', '8h'): 0x4f10a400, ('2d', '4s'): 0x4f20a400},
+    'ushll': {('8h', '8b'): 0x2f08a400, ('4s', '4h'): 0x2f10a400, ('2d', '2s'): 0x2f20a400},
+    'ushll2': {('8h', '16b'): 0x6f08a400, ('4s', '8h'): 0x6f10a400, ('2d', '4s'): 0x6f20a400},
+    'xtn': {('8b', '8h'): 0x0e212800, ('4h', '4s'): 0x0e612800, ('2s', '2d'): 0x0ea12800},
+    'xtn2': {('16b', '8h'): 0x4e212800, ('8h', '4s'): 0x4e612800, ('4s', '2d'): 0x4ea12800},
+    'scvtf': {('4s', '4s'): 0x4e21d800, ('2d', '2d'): 0x4e61d800},
+    'ucvtf': {('4s', '4s'): 0x6e21d800, ('2d', '2d'): 0x6e61d800},
+    'fcvtzs': {('4s', '4s'): 0x4ea1b800, ('2d', '2d'): 0x4ee1b800},
+    'fcvtzu': {('4s', '4s'): 0x6ea1b800, ('2d', '2d'): 0x6ee1b800},
+    'fcvtl': {('2d', '2s'): 0x0e617800, ('4s', '4h'): 0x0e217800}, 'fcvtl2': {('2d', '4s'): 0x4e617800},
+    'fcvtn': {('2s', '2d'): 0x0e616800, ('4h', '4s'): 0x0e216800}, 'fcvtn2': {('4s', '2d'): 0x4e616800},
+}
+
+
+def _i_vcvt(a, mn, ops):
+    """sshll/ushll(2) Vd, Vn, #0 (sxtl/uxtl), xtn(2), scvtf/ucvtf/fcvtzs/fcvtzu on vectors, fcvtl(2), fcvtn(2)."""
+    if mn in ('sshll', 'sshll2', 'ushll', 'ushll2'):
+        if len(ops) != 3 or _imm(ops[2], True) != 0:
+            raise AssemblerError('%s takes Vd, Vn, #0' % mn)
+    elif len(ops) != 2:
+        raise AssemblerError('%s needs 2 operands' % mn)
+    d, n = _VARR.match(ops[0].strip().lower()), _VARR.match(ops[1].strip().lower())
+    base = _VCVT[mn].get((d.group(2), n.group(2))) if d and n else None
+    if base is None:
+        raise AssemblerError('unsupported operands for %s' % mn)
+    return base | int(n.group(1)) << 5 | int(d.group(1))
+
+
+def _i_tbl(a, mn, ops):
+    """tbl Vd.16b, {Vn.16b}, Vm.16b: one table register."""
+    if len(ops) != 3:
+        raise AssemblerError('tbl needs 3 operands')
+    t = ops[1].strip().lower()
+    d, m = _VREG16B.match(ops[0].strip().lower()), _VREG16B.match(ops[2].strip().lower())
+    n = _VREG16B.match(t[1:-1].strip()) if t.startswith('{') and t.endswith('}') else None
+    if not d or not n or not m:
+        raise AssemblerError('unsupported operands for tbl')
+    return 0x4e000000 | int(m.group(1)) << 16 | int(n.group(1)) << 5 | int(d.group(1))
+
+
+def _i_adc(a, mn, ops):
+    """adc Xd, Xn, Xm (the 64-bit form: __int128's carry)."""
+    if len(ops) != 3:
+        raise AssemblerError('adc needs 3 operands')
+    rd, rn, rm = (_reg(o, 'x') for o in ops)
+    return 0x9a000000 | rm[1] << 16 | rn[1] << 5 | rd[1]
+
+
+def _vector_or(scalar):
+    """The encoder for a mnemonic with a scalar and a vector form: vector operands go to _i_vcvt."""
+    def enc(a, mn, ops):
+        if ops and ops[0].strip().lower().startswith('v'):
+            return _i_vcvt(a, mn, ops)
+        return scalar(a, mn, ops)
+    return enc
+
+
 def _fpimm8(v):
     """abcdefgh such that v is (-1)^a * (16 + efgh) / 16 * 2^n, with n from bcd: 000 is 1, 011 is
     4, 100 is -3, 111 is 0; or None."""
@@ -801,10 +943,11 @@ def _i_fmov(a, mn, ops):
 def _i_fcvt(a, mn, ops):
     if len(ops) != 2:
         raise AssemblerError('fcvt needs 2 operands')
-    rd, rn = _reg(ops[0], 'sd'), _reg(ops[1], 'sd')
+    rd, rn = _reg(ops[0], 'hsd'), _reg(ops[1], 'hsd')
     if rd[0] == rn[0]:
         raise AssemblerError('fcvt needs different precisions')
-    return (0x1e22c000 if rd[0] == 'd' else 0x1e624000) | rn[1] << 5 | rd[1]
+    t = {'s': 0, 'd': 1, 'h': 3}  # the source's type at 22, the destination's at 15
+    return 0x1e224000 | t[rn[0]] << 22 | t[rd[0]] << 15 | rn[1] << 5 | rd[1]
 
 
 def _i_fcvtz(a, mn, ops):  # fcvtzs/fcvtzu Rd, Fn
@@ -930,13 +1073,16 @@ _NEON3 = {'add': (0x4e208400, 's'), 'sub': (0x6e208400, 's'), 'mul': (0x4e209c00
           'fadd': (0x4e20d400, 'f'), 'fsub': (0x4ea0d400, 'f'), 'fmul': (0x6e20dc00, 'f'), 'fdiv': (0x6e20fc00, 'f'),
           'fminnm': (0x4ea0c400, 'f'), 'fmaxnm': (0x4e20c400, 'f'), 'fmla': (0x4e20cc00, 'f'), 'fcmeq': (0x4e20e400, 'f'),
           'fcmge': (0x6e20e400, 'f'), 'fcmgt': (0x6ea0e400, 'f'), 'and': (0x4e201c00, ''), 'orr': (0x4ea01c00, ''),
-          'eor': (0x6e201c00, ''), 'bsl': (0x6e601c00, '')}
+          'eor': (0x6e201c00, ''), 'bsl': (0x6e601c00, ''), 'bit': (0x6ea01c00, ''), 'bif': (0x6ee01c00, ''), 'sshl': (0x4e204400, 's'), 'ushl': (0x6e204400, 's'),
+          'zip1': (0x4e003800, 's'), 'zip2': (0x4e007800, 's'), 'uzp1': (0x4e001800, 's'), 'uzp2': (0x4e005800, 's'),
+          'trn1': (0x4e002800, 's'), 'trn2': (0x4e006800, 's')}
 _NEON2 = {'neg': (0x6e20b800, 's'), 'fneg': (0x6ea0f800, 'f'), 'fabs': (0x4ea0f800, 'f'), 'fsqrt': (0x6ea1f800, 'f'),
           'mvn': (0x6e205800, '')}
 _NEONSH = {'shl': 0x4f005400, 'ushr': 0x6f000400, 'sshr': 0x4f000400}
 _LANES = {'16b': 0, '8h': 1, '4s': 2, '2d': 3}
 _NEONW = {'saddw': 0x0ea01000, 'saddw2': 0x4ea01000, 'uaddw': 0x2ea01000, 'uaddw2': 0x6ea01000}  # 2d, 2d, 2s or 4s
 _VREG = re.compile(r'v(\d+)\.(16b|8h|4s|2d|2s)$')
+_VNARROW = re.compile(r'v(\d+)\.(8b|4h|2s|16b|8h|4s)$')  # saddw's third operand
 _VELEM = re.compile(r'v(\d+)\.([bhsd])\[(\d+)\]$')
 
 
@@ -944,63 +1090,114 @@ def _isneon(ops):
     return bool(ops) and ops[0].strip().lower()[:1] == 'v' and _VREG.match(ops[0].strip().lower()) is not None
 
 
-def _v(s):
-    m = _VREG.match(s.strip().lower())
-    if m is None:
+def _v(s, lanes=None, pattern=_VREG):
+    """(register, arrangement) of a vector operand; lanes: the arrangements allowed."""
+    m = pattern.match(s.strip().lower())
+    if m is None or int(m[1]) > 31:
         raise AssemblerError('expected a vector register, got %r' % s)
+    if lanes is not None and m[2] not in lanes:
+        raise AssemblerError('arrangement .%s not allowed here: %r' % (m[2], s))
     return int(m[1]), m[2]
 
 
+_NO2D = frozenset(('mul', 'smin', 'smax', 'umin', 'umax'))
+
+
+def _elem(s):
+    """(register, element size letter, index) of Vn.T[i], the index in range for the element size."""
+    e = _VELEM.match(s.strip().lower())
+    if e is None or int(e[1]) > 31:
+        raise AssemblerError('expected a vector element, got %r' % s)
+    k, i = e[2], int(e[3])
+    if i >= {'b': 16, 'h': 8, 's': 4, 'd': 2}[k]:
+        raise AssemblerError('lane index out of range: %r' % s)
+    return int(e[1]), k, i
+
+
+def _imm5(k, i):
+    return {'b': 1, 'h': 2, 's': 4, 'd': 8}[k] | i << ({'b': 1, 'h': 2, 's': 3, 'd': 4}[k])
+
+
 def _neon(mn, ops):
-    """The word of a NEON instruction whose first operand is a vector register."""
-    d, lane = _v(ops[0])
-    size = _LANES.get(lane, 2)
-    if mn == 'mov':
-        n, _ = _v(ops[1])
-        return 0x4ea01c00 | n << 16 | n << 5 | d
-    if mn in _NEONW:
-        n, _ = _v(ops[1])
-        m, _ = _v(ops[2])
-        return _NEONW[mn] | m << 16 | n << 5 | d
+    """The word of a NEON instruction whose first operand is a vector register. Arrangements are checked as GNU as
+    does: one for all the operands, and one the instruction has (.2s only in saddw and uaddw)."""
+    if mn in _NEONW:  # saddw: .8h/.4s/.2d twice, then .8b/.4h/.2s; saddw2: then .16b/.8h/.4s
+        d, lane = _v(ops[0], ('8h', '4s', '2d'))
+        n, _ = _v(ops[1], (lane,))
+        size = {'8h': 0, '4s': 1, '2d': 2}[lane]
+        narrow = ('16b', '8h', '4s') if mn.endswith('2') else ('8b', '4h', '2s')
+        m, _ = _v(ops[2], (narrow[size],), _VNARROW)
+        return (_NEONW[mn] & ~0x00c00000) | size << 22 | m << 16 | n << 5 | d
+    if mn == 'ext':  # ext Vd.16b, Vn.16b, Vm.16b, #bytes
+        if len(ops) != 4:
+            raise AssemblerError('ext needs 4 operands')
+        d, _ = _v(ops[0], ('16b',))
+        n, _ = _v(ops[1], ('16b',))
+        m, _ = _v(ops[2], ('16b',))
+        k = _imm(ops[3], True)
+        if not 0 <= k <= 15:
+            raise AssemblerError('ext index out of range')
+        return 0x6e000000 | m << 16 | k << 11 | n << 5 | d
     if mn == 'dup':
+        d, lane = _v(ops[0], _LANES)
+        size = _LANES[lane]
         s = ops[1].strip().lower()
-        e = _VELEM.match(s)
-        if e:
-            imm5 = {'b': 1, 'h': 2, 's': 4, 'd': 8}[e[2]] | int(e[3]) << ({'b': 1, 'h': 2, 's': 3, 'd': 4}[e[2]])
-            return 0x4e000400 | imm5 << 16 | int(e[1]) << 5 | d
-        r = _reg(s, 'wx')
+        if _VELEM.match(s):
+            n, k, i = _elem(s)
+            if k != 'bhsd'[size]:
+                raise AssemblerError('element size does not match .%s' % lane)
+            return 0x4e000400 | _imm5(k, i) << 16 | n << 5 | d
+        r = _reg(s, 'x' if size == 3 else 'w')
         return 0x4e000c00 | (1 << size) << 16 | r[1] << 5 | d
+    d, lane = _v(ops[0], _LANES)
+    size = _LANES[lane]
+    same = (lane,)
+    if mn == 'mov':
+        n, _ = _v(ops[1], ('16b',) if lane == '16b' else same)
+        if lane != '16b':
+            raise AssemblerError('mov of vectors takes .16b')
+        return 0x4ea01c00 | n << 16 | n << 5 | d
     if mn in _NEONSH:
-        n, _ = _v(ops[1])
+        n, _ = _v(ops[1], same)
         amount = _imm(ops[2], True)
         bits = 8 << size
+        if not (0 <= amount < bits if mn == 'shl' else 1 <= amount <= bits):
+            raise AssemblerError('shift amount %d out of range for .%s' % (amount, lane))
         immhb = bits + amount if mn == 'shl' else 2 * bits - amount
         return _NEONSH[mn] | immhb << 16 | n << 5 | d
     if mn in _NEON2:
         op, kind = _NEON2[mn]
-        n, _ = _v(ops[1])
+        _lanes_ok(mn, kind, lane)
+        n, _ = _v(ops[1], same)
         w = op | n << 5 | d
         return w | (size << 22 if kind == 's' else (lane == '2d') << 22 if kind == 'f' else 0)
     if mn in _NEON3:
         op, kind = _NEON3[mn]
-        n, _ = _v(ops[1])
-        m, _ = _v(ops[2])
+        _lanes_ok(mn, kind, lane)
+        n, _ = _v(ops[1], same)
+        m, _ = _v(ops[2], same)
         w = op | m << 16 | n << 5 | d
         return w | (size << 22 if kind == 's' else (lane == '2d') << 22 if kind == 'f' else 0)
     raise AssemblerError('unsupported vector instruction %r' % mn)
 
 
+def _lanes_ok(mn, kind, lane):
+    if kind == 'f' and lane not in ('4s', '2d') or kind == '' and lane != '16b' or \
+            kind == 's' and lane == '2d' and mn in _NO2D:
+        raise AssemblerError('%s has no .%s form' % (mn, lane))
+
+
 def _i_vscalar(a, mn, ops):
     """umov Wd, Vn.T[i]; dup Sd, Vn.s[i] (and Dd)."""
-    e = _VELEM.match(ops[1].strip().lower())
-    if e is None:
-        raise AssemblerError('expected a vector element, got %r' % ops[1])
-    imm5 = {'b': 1, 'h': 2, 's': 4, 'd': 8}[e[2]] | int(e[3]) << ({'b': 1, 'h': 2, 's': 3, 'd': 4}[e[2]])
-    if mn == 'umov':
-        r = _reg(ops[0], 'wx')
-        return (0x4e003c00 if r[0] == 'x' else 0x0e003c00) | imm5 << 16 | int(e[1]) << 5 | r[1]
-    r = _reg(ops[0], 'sd')
-    return 0x5e000400 | imm5 << 16 | int(e[1]) << 5 | r[1]
+    n, k, i = _elem(ops[1])
+    imm5 = _imm5(k, i)
+    if mn == 'umov':  # an x register exactly for a .d element
+        r = _reg(ops[0], 'x' if k == 'd' else 'w')
+        return (0x4e003c00 if r[0] == 'x' else 0x0e003c00) | imm5 << 16 | n << 5 | r[1]
+    r = _reg(ops[0], {'s': 's', 'd': 'd'}.get(k, 'sd'))
+    if k not in 'sd':
+        raise AssemblerError('dup to a scalar register takes .s or .d')
+    return 0x5e000400 | imm5 << 16 | n << 5 | r[1]
 
 
 def _i_ldst(a, mn, ops):
@@ -1200,12 +1397,22 @@ def _i_adrp(a, mn, ops):
     return 0x90000000 | rd[1]
 
 
+_SYSREGS = {'fpcr': 0x4400, 'fpsr': 0x4420}  # op1=3, CRn=4, CRm=4, op2 0 or 1 (the floating-point environment)
+
+
 def _i_tls(a, mn, ops):
+    """mrs Xt, fpcr|fpsr and msr fpcr|fpsr, Xt; other system registers (tpidr_el0: thread-local storage) are not
+    supported"""
+    if len(ops) == 2:
+        sysreg, reg = (ops[1], ops[0]) if mn == 'mrs' else (ops[0], ops[1])
+        if sysreg.lower() in _SYSREGS:
+            rt = _reg(reg, 'x')
+            return (0xd53b0000 if mn == 'mrs' else 0xd51b0000) | _SYSREGS[sysreg.lower()] | rt[1]
     raise AssemblerError('thread-local storage is not supported')
 
 
 _INSNS = {
-    'add': _i_addsub, 'adds': _i_addsub, 'sub': _i_addsub, 'subs': _i_addsub,
+    'adc': _i_adc, 'add': _i_addsub, 'adds': _i_addsub, 'sub': _i_addsub, 'subs': _i_addsub,
     'cmp': _i_cmp, 'cmn': _i_cmp, 'neg': _i_neg,
     'and': _i_logic, 'orr': _i_logic, 'eor': _i_logic, 'ands': _i_logic,
     'bic': _i_logic, 'orn': _i_logic, 'eon': _i_logic, 'ubfx': _i_bitfield, 'sbfx': _i_bitfield,
@@ -1224,13 +1431,14 @@ _INSNS = {
     'frintm': _i_fneg, 'frintz': _i_fneg, 'frinta': _i_fneg, 'frintx': _i_fneg,
     'frinti': _i_fneg,
     'clz': _i_dp1, 'rbit': _i_dp1, 'rev16': _i_dp1, 'rev': _i_dp1, 'cnt': _i_simd,
-    'addv': _i_simd,
+    'addv': _i_simd, 'ushll': _i_vcvt, 'xtn': _i_vcvt, 'sshll': _i_vcvt, 'sshll2': _i_vcvt, 'xtn2': _i_vcvt,
+    'tbl': _i_tbl, 'fcvtl': _i_vcvt, 'fcvtl2': _i_vcvt, 'fcvtn': _i_vcvt, 'fcvtn2': _i_vcvt, 'ld3': _i_bytes, 'umlal': _i_bytes, 'shrn': _i_bytes, 'umull2': _i_bytes, 'umlal2': _i_bytes, 'ushll2': _i_vcvt,
     'fmadd': _i_fp4, 'fmsub': _i_fp4, 'fnmadd': _i_fp4, 'fnmsub': _i_fp4,
-    'fmov': _i_fmov, 'fcvt': _i_fcvt, 'fcvtzs': _i_fcvtz, 'fcvtzu': _i_fcvtz,
-    'scvtf': _i_cvtf, 'ucvtf': _i_cvtf, 'fcmp': _i_fcmp, 'fcmpe': _i_fcmp,
+    'fmov': _i_fmov, 'fcvt': _i_fcvt, 'fcvtzs': _vector_or(_i_fcvtz), 'fcvtzu': _vector_or(_i_fcvtz),
+    'scvtf': _vector_or(_i_cvtf), 'ucvtf': _vector_or(_i_cvtf), 'fcmp': _i_fcmp, 'fcmpe': _i_fcmp,
     'ldp': _i_pair, 'stp': _i_pair,
     'b': _i_b, 'bl': _i_b, 'blr': _i_blr, 'br': _i_blr, 'ret': _i_ret,
-    'brk': _i_brk, 'hint': _i_brk, 'nop': _i_nop, 'adrp': _i_adrp, 'mrs': _i_tls,
+    'brk': _i_brk, 'hint': _i_brk, 'nop': _i_nop, 'adrp': _i_adrp, 'mrs': _i_tls, 'msr': _i_tls,
 }
 for _k in _LDST:
     _INSNS[_k[0]] = _i_ldst
@@ -1387,6 +1595,15 @@ class _Asm:
         s.data += b
         s.size += len(b)
 
+    def fill(self, unit, n):
+        """n copies of unit: zeros in a nobits section only count (a `.zero` for a huge array in .bss)"""
+        if self.sec.nobits and not any(unit):
+            self.sec.size += len(unit) * max(n, 0)
+        elif len(unit) * n > 1 << 32:
+            raise AssemblerError('%d bytes of data: too large' % (len(unit) * n))
+        else:
+            self.emit(unit * n)
+
     def align(self, n):
         if n <= 0 or n & (n - 1):
             raise AssemblerError('alignment must be a power of 2')
@@ -1418,6 +1635,23 @@ class _Asm:
 
     # driver
 
+    far = frozenset()  # the lines whose short branch can't reach its target: see farbranch
+
+    def farbranch(self, f, mn, ops):
+        """b.cond, cbz or tbz to a target out of its reach (1 MB, 32 KB for tbz): the opposite test over a b"""
+        if f is _i_b and mn not in ('b', 'bl'):
+            c = _COND[mn[2:] if mn[1] == '.' else mn[1:]]
+            inv = 'b.' + next(k for k, v in _COND.items() if v == c ^ 1)
+        elif mn in ('cbz', 'cbnz', 'tbz', 'tbnz'):
+            inv = {'cbz': 'cbnz', 'cbnz': 'cbz', 'tbz': 'tbnz', 'tbnz': 'tbz'}[mn]
+        else:
+            raise AssemblerError('branch out of range')
+        skip = '.Lcimo_far%d' % self.lineno
+        self.sec.align = max(self.sec.align, 4)
+        self.emit(struct.pack('<I', f(self, inv, list(ops[:-1]) + [skip])))  # each fixup at its own offset
+        self.emit(struct.pack('<I', _i_b(self, 'b', [ops[-1]])))
+        self.label(skip)
+
     def run(self, text):
         text = _strip_comments(text)
         insns = _INSNS
@@ -1455,7 +1689,12 @@ class _Asm:
                     continue
                 ops = _split(rest)
                 f = insns.get(mn)
-                if _isneon(ops):
+                if f is _i_bytes or f is _i_tbl or mn in _VCVT:
+                    if mn in _VCVT and ops and ops[0].strip().lower().startswith('v'):
+                        f = _i_vcvt
+                elif mn in ('umull', 'umlal', 'umull2', 'umlal2') and ops and ops[0].strip().lower().startswith('v'):
+                    f = _i_bytes
+                elif _isneon(ops):
                     f = lambda a, mn, ops: _neon(mn, ops)
                 elif mn in ('umov', 'dup') and ops:
                     f = _i_vscalar
@@ -1463,6 +1702,9 @@ class _Asm:
                     raise AssemblerError('unsupported instruction %r' % parts[0])
                 if self.sec.nobits:
                     raise AssemblerError('instruction in nobits section %s' % self.sec.name)
+                if self.lineno in self.far:
+                    self.farbranch(f, mn, ops)
+                    continue
                 word = f(self, mn, ops)
                 if self.sec.size % 4:
                     raise AssemblerError('misaligned instruction')
@@ -1532,11 +1774,11 @@ class _Asm:
                 if size > 8 or size < 0:
                     raise AssemblerError('.fill size must be at most 8')
                 unit = struct.pack('<Q', val & M64)[:min(size, 4)].ljust(size, b'\0')
-                self.emit(unit * n)
+                self.fill(unit, n)
             else:
                 n = _int(args[0])
                 val = _imm(args[1]) if len(args) > 1 else 0
-                self.emit(bytes([val & 0xff]) * n)
+                self.fill(bytes([val & 0xff]), n)
         elif d == '.comm':
             args = _split(rest)
             if len(args) not in (2, 3):
@@ -1594,6 +1836,7 @@ class _Asm:
         """Resolve branches to non-global labels of the same section (as GNU as does) and turn
         everything else into relocations."""
         syms = self.obj.symbols
+        far = set()
         for sec, off, kind, name, add, lineno in self.fixups:
             s = syms.get(name)
             if s is None:
@@ -1604,7 +1847,10 @@ class _Asm:
                     raise AssemblerError('line %d: misaligned branch target %s' % (lineno, name))
                 bits = {'CONDBR19': 19, 'TSTBR14': 14}.get(kind, 26)  # tbz: 32 KB either way
                 if not -(1 << (bits + 1)) <= d < 1 << (bits + 1):
-                    raise AssemblerError('line %d: branch out of range: %s' % (lineno, name))
+                    if bits == 26 or lineno in self.far:
+                        raise AssemblerError('line %d: branch out of range: %s' % (lineno, name))
+                    far.add(lineno)
+                    continue
                 w = struct.unpack_from('<I', sec.data, off)[0]
                 if bits == 26:
                     w |= (d >> 2) & 0x3ffffff
@@ -1613,9 +1859,27 @@ class _Asm:
                 struct.pack_into('<I', sec.data, off, w)
             else:
                 sec.relocs.append(Reloc(off, kind, name, add))
+        if far:
+            raise _Far(far)
+
+
+class _Far(AssemblerError):
+    """Short branches out of reach, on these lines: assemble() runs again with them made long."""
+
+    def __init__(self, lines):
+        AssemblerError.__init__(self, 'branch out of range on line %d' % min(lines))
+        self.lines = lines
 
 
 def assemble(text, syntax='elf'):
     """Assemble arm64 GNU-syntax assembly text into an Object. syntax: 'elf' (QBE target
-    arm64) or 'macho' (arm64_apple)."""
-    return _Asm(syntax).run(text)
+    arm64) or 'macho' (arm64_apple). A b.cond, cbz or tbz that can't reach its target is assembled again as the
+    opposite test over a b (as LLVM's branch relaxation does)."""
+    far = set()
+    while True:
+        a = _Asm(syntax)
+        a.far = frozenset(far)
+        try:
+            return a.run(text)
+        except _Far as e:
+            far |= e.lines