12 files changed, 3413 insertions, 366 deletions

diff --git a/dynasm/dasm_arm.lua b/dynasm/dasm_arm.lua
index 21fb5022..164980a1 100644
--- a/dynasm/dasm_arm.lua
+++ b/dynasm/dasm_arm.lua
@@ -9,9 +9,9 @@
 local _info = {
   arch =        "arm",
   description = "DynASM ARM module",
-  version =     "1.3.0",
-  vernum =       10300,
-  release =     "2011-05-05",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2015-10-18",
   author =      "Mike Pall",
   license =     "MIT",
 }
diff --git a/dynasm/dasm_arm64.h b/dynasm/dasm_arm64.h
new file mode 100644
index 00000000..3455981f
--- /dev/null
+++ b/dynasm/dasm_arm64.h
@@ -0,0 +1,519 @@
+/*
+** DynASM ARM64 encoding engine.
+** Copyright (C) 2005-2020 Mike Pall. All rights reserved.
+** Released under the MIT license. See dynasm.lua for full copyright notice.
+*/
+
+#include <stddef.h>
+#include <stdarg.h>
+#include <string.h>
+#include <stdlib.h>
+
+#define DASM_ARCH               "arm64"
+
+#ifndef DASM_EXTERN
+#define DASM_EXTERN(a,b,c,d)    0
+#endif
+
+/* Action definitions. */
+enum {
+  DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT,
+  /* The following actions need a buffer position. */
+  DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
+  /* The following actions also have an argument. */
+  DASM_REL_PC, DASM_LABEL_PC,
+  DASM_IMM, DASM_IMM6, DASM_IMM12, DASM_IMM13W, DASM_IMM13X, DASM_IMML,
+  DASM__MAX
+};
+
+/* Maximum number of section buffer positions for a single dasm_put() call. */
+#define DASM_MAXSECPOS          25
+
+/* DynASM encoder status codes. Action list offset or number are or'ed in. */
+#define DASM_S_OK               0x00000000
+#define DASM_S_NOMEM            0x01000000
+#define DASM_S_PHASE            0x02000000
+#define DASM_S_MATCH_SEC        0x03000000
+#define DASM_S_RANGE_I          0x11000000
+#define DASM_S_RANGE_SEC        0x12000000
+#define DASM_S_RANGE_LG         0x13000000
+#define DASM_S_RANGE_PC         0x14000000
+#define DASM_S_RANGE_REL        0x15000000
+#define DASM_S_UNDEF_LG         0x21000000
+#define DASM_S_UNDEF_PC         0x22000000
+
+/* Macros to convert positions (8 bit section + 24 bit index). */
+#define DASM_POS2IDX(pos)       ((pos)&0x00ffffff)
+#define DASM_POS2BIAS(pos)      ((pos)&0xff000000)
+#define DASM_SEC2POS(sec)       ((sec)<<24)
+#define DASM_POS2SEC(pos)       ((pos)>>24)
+#define DASM_POS2PTR(D, pos)    (D->sections[DASM_POS2SEC(pos)].rbuf + (pos))
+
+/* Action list type. */
+typedef const unsigned int *dasm_ActList;
+
+/* Per-section structure. */
+typedef struct dasm_Section {
+  int *rbuf;            /* Biased buffer pointer (negative section bias). */
+  int *buf;             /* True buffer pointer. */
+  size_t bsize;         /* Buffer size in bytes. */
+  int pos;              /* Biased buffer position. */
+  int epos;             /* End of biased buffer position - max single put. */
+  int ofs;              /* Byte offset into section. */
+} dasm_Section;
+
+/* Core structure holding the DynASM encoding state. */
+struct dasm_State {
+  size_t psize;                 /* Allocated size of this structure. */
+  dasm_ActList actionlist;      /* Current actionlist pointer. */
+  int *lglabels;                /* Local/global chain/pos ptrs. */
+  size_t lgsize;
+  int *pclabels;                /* PC label chains/pos ptrs. */
+  size_t pcsize;
+  void **globals;               /* Array of globals (bias -10). */
+  dasm_Section *section;        /* Pointer to active section. */
+  size_t codesize;              /* Total size of all code sections. */
+  int maxsection;               /* 0 <= sectionidx < maxsection. */
+  int status;                   /* Status code. */
+  dasm_Section sections[1];     /* All sections. Alloc-extended. */
+};
+
+/* The size of the core structure depends on the max. number of sections. */
+#define DASM_PSZ(ms)    (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section))
+
+
+/* Initialize DynASM state. */
+void dasm_init(Dst_DECL, int maxsection)
+{
+  dasm_State *D;
+  size_t psz = 0;
+  int i;
+  Dst_REF = NULL;
+  DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
+  D = Dst_REF;
+  D->psize = psz;
+  D->lglabels = NULL;
+  D->lgsize = 0;
+  D->pclabels = NULL;
+  D->pcsize = 0;
+  D->globals = NULL;
+  D->maxsection = maxsection;
+  for (i = 0; i < maxsection; i++) {
+    D->sections[i].buf = NULL;  /* Need this for pass3. */
+    D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i);
+    D->sections[i].bsize = 0;
+    D->sections[i].epos = 0;  /* Wrong, but is recalculated after resize. */
+  }
+}
+
+/* Free DynASM state. */
+void dasm_free(Dst_DECL)
+{
+  dasm_State *D = Dst_REF;
+  int i;
+  for (i = 0; i < D->maxsection; i++)
+    if (D->sections[i].buf)
+      DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize);
+  if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize);
+  if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize);
+  DASM_M_FREE(Dst, D, D->psize);
+}
+
+/* Setup global label array. Must be called before dasm_setup(). */
+void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
+{
+  dasm_State *D = Dst_REF;
+  D->globals = gl - 10;  /* Negative bias to compensate for locals. */
+  DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
+}
+
+/* Grow PC label array. Can be called after dasm_setup(), too. */
+void dasm_growpc(Dst_DECL, unsigned int maxpc)
+{
+  dasm_State *D = Dst_REF;
+  size_t osz = D->pcsize;
+  DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int));
+  memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz);
+}
+
+/* Setup encoder. */
+void dasm_setup(Dst_DECL, const void *actionlist)
+{
+  dasm_State *D = Dst_REF;
+  int i;
+  D->actionlist = (dasm_ActList)actionlist;
+  D->status = DASM_S_OK;
+  D->section = &D->sections[0];
+  memset((void *)D->lglabels, 0, D->lgsize);
+  if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
+  for (i = 0; i < D->maxsection; i++) {
+    D->sections[i].pos = DASM_SEC2POS(i);
+    D->sections[i].ofs = 0;
+  }
+}
+
+
+#ifdef DASM_CHECKS
+#define CK(x, st) \
+  do { if (!(x)) { \
+    D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0)
+#define CKPL(kind, st) \
+  do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \
+    D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0)
+#else
+#define CK(x, st)       ((void)0)
+#define CKPL(kind, st)  ((void)0)
+#endif
+
+static int dasm_imm12(unsigned int n)
+{
+  if ((n >> 12) == 0)
+    return n;
+  else if ((n & 0xff000fff) == 0)
+    return (n >> 12) | 0x1000;
+  else
+    return -1;
+}
+
+static int dasm_ffs(unsigned long long x)
+{
+  int n = -1;
+  while (x) { x >>= 1; n++; }
+  return n;
+}
+
+static int dasm_imm13(int lo, int hi)
+{
+  int inv = 0, w = 64, s = 0xfff, xa, xb;
+  unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo;
+  unsigned long long m = 1ULL, a, b, c;
+  if (n & 1) { n = ~n; inv = 1; }
+  a = n & -n; b = (n+a)&-(n+a); c = (n+a-b)&-(n+a-b);
+  xa = dasm_ffs(a); xb = dasm_ffs(b);
+  if (c) {
+    w = dasm_ffs(c) - xa;
+    if (w == 32) m = 0x0000000100000001UL;
+    else if (w == 16) m = 0x0001000100010001UL;
+    else if (w == 8) m = 0x0101010101010101UL;
+    else if (w == 4) m = 0x1111111111111111UL;
+    else if (w == 2) m = 0x5555555555555555UL;
+    else return -1;
+    s = (-2*w & 0x3f) - 1;
+  } else if (!a) {
+    return -1;
+  } else if (xb == -1) {
+    xb = 64;
+  }
+  if ((b-a) * m != n) return -1;
+  if (inv) {
+    return ((w - xb) << 6) | (s+w+xa-xb);
+  } else {
+    return ((w - xa) << 6) | (s+xb-xa);
+  }
+  return -1;
+}
+
+/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */
+void dasm_put(Dst_DECL, int start, ...)
+{
+  va_list ap;
+  dasm_State *D = Dst_REF;
+  dasm_ActList p = D->actionlist + start;
+  dasm_Section *sec = D->section;
+  int pos = sec->pos, ofs = sec->ofs;
+  int *b;
+
+  if (pos >= sec->epos) {
+    DASM_M_GROW(Dst, int, sec->buf, sec->bsize,
+      sec->bsize + 2*DASM_MAXSECPOS*sizeof(int));
+    sec->rbuf = sec->buf - DASM_POS2BIAS(pos);
+    sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos);
+  }
+
+  b = sec->rbuf;
+  b[pos++] = start;
+
+  va_start(ap, start);
+  while (1) {
+    unsigned int ins = *p++;
+    unsigned int action = (ins >> 16);
+    if (action >= DASM__MAX) {
+      ofs += 4;
+    } else {
+      int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0;
+      switch (action) {
+      case DASM_STOP: goto stop;
+      case DASM_SECTION:
+        n = (ins & 255); CK(n < D->maxsection, RANGE_SEC);
+        D->section = &D->sections[n]; goto stop;
+      case DASM_ESC: p++; ofs += 4; break;
+      case DASM_REL_EXT: break;
+      case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break;
+      case DASM_REL_LG:
+        n = (ins & 2047) - 10; pl = D->lglabels + n;
+        /* Bkwd rel or global. */
+        if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; }
+        pl += 10; n = *pl;
+        if (n < 0) n = 0;  /* Start new chain for fwd rel if label exists. */
+        goto linkrel;
+      case DASM_REL_PC:
+        pl = D->pclabels + n; CKPL(pc, PC);
+      putrel:
+        n = *pl;
+        if (n < 0) {  /* Label exists. Get label pos and store it. */
+          b[pos] = -n;
+        } else {
+      linkrel:
+          b[pos] = n;  /* Else link to rel chain, anchored at label. */
+          *pl = pos;
+        }
+        pos++;
+        break;
+      case DASM_LABEL_LG:
+        pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel;
+      case DASM_LABEL_PC:
+        pl = D->pclabels + n; CKPL(pc, PC);
+      putlabel:
+        n = *pl;  /* n > 0: Collapse rel chain and replace with label pos. */
+        while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos;
+        }
+        *pl = -pos;  /* Label exists now. */
+        b[pos++] = ofs;  /* Store pass1 offset estimate. */
+        break;
+      case DASM_IMM:
+        CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I);
+        n >>= ((ins>>10)&31);
+#ifdef DASM_CHECKS
+        if ((ins & 0x8000))
+          CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I);
+        else
+          CK((n>>((ins>>5)&31)) == 0, RANGE_I);
+#endif
+        b[pos++] = n;
+        break;
+      case DASM_IMM6:
+        CK((n >> 6) == 0, RANGE_I);
+        b[pos++] = n;
+        break;
+      case DASM_IMM12:
+        CK(dasm_imm12((unsigned int)n) != -1, RANGE_I);
+        b[pos++] = n;
+        break;
+      case DASM_IMM13W:
+        CK(dasm_imm13(n, n) != -1, RANGE_I);
+        b[pos++] = n;
+        break;
+      case DASM_IMM13X: {
+        int m = va_arg(ap, int);
+        CK(dasm_imm13(n, m) != -1, RANGE_I);
+        b[pos++] = n;
+        b[pos++] = m;
+        break;
+        }
+      case DASM_IMML: {
+#ifdef DASM_CHECKS
+        int scale = (p[-2] >> 30);
+        CK((!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ||
+           (unsigned int)(n+256) < 512, RANGE_I);
+#endif
+        b[pos++] = n;
+        break;
+        }
+      }
+    }
+  }
+stop:
+  va_end(ap);
+  sec->pos = pos;
+  sec->ofs = ofs;
+}
+#undef CK
+
+/* Pass 2: Link sections, shrink aligns, fix label offsets. */
+int dasm_link(Dst_DECL, size_t *szp)
+{
+  dasm_State *D = Dst_REF;
+  int secnum;
+  int ofs = 0;
+
+#ifdef DASM_CHECKS
+  *szp = 0;
+  if (D->status != DASM_S_OK) return D->status;
+  {
+    int pc;
+    for (pc = 0; pc*sizeof(int) < D->pcsize; pc++)
+      if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc;
+  }
+#endif
+
+  { /* Handle globals not defined in this translation unit. */
+    int idx;
+    for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) {
+      int n = D->lglabels[idx];
+      /* Undefined label: Collapse rel chain and replace with marker (< 0). */
+      while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
+    }
+  }
+
+  /* Combine all code sections. No support for data sections (yet). */
+  for (secnum = 0; secnum < D->maxsection; secnum++) {
+    dasm_Section *sec = D->sections + secnum;
+    int *b = sec->rbuf;
+    int pos = DASM_SEC2POS(secnum);
+    int lastpos = sec->pos;
+
+    while (pos != lastpos) {
+      dasm_ActList p = D->actionlist + b[pos++];
+      while (1) {
+        unsigned int ins = *p++;
+        unsigned int action = (ins >> 16);
+        switch (action) {
+        case DASM_STOP: case DASM_SECTION: goto stop;
+        case DASM_ESC: p++; break;
+        case DASM_REL_EXT: break;
+        case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
+        case DASM_REL_LG: case DASM_REL_PC: pos++; break;
+        case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
+        case DASM_IMM: case DASM_IMM6: case DASM_IMM12: case DASM_IMM13W:
+        case DASM_IMML: pos++; break;
+        case DASM_IMM13X: pos += 2; break;
+        }
+      }
+      stop: (void)0;
+    }
+    ofs += sec->ofs;  /* Next section starts right after current section. */
+  }
+
+  D->codesize = ofs;  /* Total size of all code sections */
+  *szp = ofs;
+  return DASM_S_OK;
+}
+
+#ifdef DASM_CHECKS
+#define CK(x, st) \
+  do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0)
+#else
+#define CK(x, st)       ((void)0)
+#endif
+
+/* Pass 3: Encode sections. */
+int dasm_encode(Dst_DECL, void *buffer)
+{
+  dasm_State *D = Dst_REF;
+  char *base = (char *)buffer;
+  unsigned int *cp = (unsigned int *)buffer;
+  int secnum;
+
+  /* Encode all code sections. No support for data sections (yet). */
+  for (secnum = 0; secnum < D->maxsection; secnum++) {
+    dasm_Section *sec = D->sections + secnum;
+    int *b = sec->buf;
+    int *endb = sec->rbuf + sec->pos;
+
+    while (b != endb) {
+      dasm_ActList p = D->actionlist + *b++;
+      while (1) {
+        unsigned int ins = *p++;
+        unsigned int action = (ins >> 16);
+        int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0;
+        switch (action) {
+        case DASM_STOP: case DASM_SECTION: goto stop;
+        case DASM_ESC: *cp++ = *p++; break;
+        case DASM_REL_EXT:
+          n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins&2047), !(ins&2048));
+          goto patchrel;
+        case DASM_ALIGN:
+          ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000;
+          break;
+        case DASM_REL_LG:
+          CK(n >= 0, UNDEF_LG);
+          /* fallthrough */
+        case DASM_REL_PC:
+          CK(n >= 0, UNDEF_PC);
+          n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4;
+        patchrel:
+          if (!(ins & 0xf800)) {  /* B, BL */
+            CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL);
+            cp[-1] |= ((n >> 2) & 0x03ffffff);
+          } else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
+            CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL);
+            cp[-1] |= ((n << 3) & 0x00ffffe0);
+          } else if ((ins & 0x3000) == 0x2000) {  /* ADR */
+            CK(((n+0x00100000) >> 21) == 0, RANGE_REL);
+            cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29);
+          } else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
+            cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29);
+          } else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
+            CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL);
+            cp[-1] |= ((n << 3) & 0x0007ffe0);
+          }
+          break;
+        case DASM_LABEL_LG:
+          ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n);
+          break;
+        case DASM_LABEL_PC: break;
+        case DASM_IMM:
+          cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
+          break;
+        case DASM_IMM6:
+          cp[-1] |= ((n&31) << 19) | ((n&32) << 26);
+          break;
+        case DASM_IMM12:
+          cp[-1] |= (dasm_imm12((unsigned int)n) << 10);
+          break;
+        case DASM_IMM13W:
+          cp[-1] |= (dasm_imm13(n, n) << 10);
+          break;
+        case DASM_IMM13X:
+          cp[-1] |= (dasm_imm13(n, *b++) << 10);
+          break;
+        case DASM_IMML: {
+          int scale = (p[-2] >> 30);
+          cp[-1] |= (!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ?
+            ((n << (10-scale)) | 0x01000000) : ((n & 511) << 12);
+          break;
+          }
+        default: *cp++ = ins; break;
+        }
+      }
+      stop: (void)0;
+    }
+  }
+
+  if (base + D->codesize != (char *)cp)  /* Check for phase errors. */
+    return DASM_S_PHASE;
+  return DASM_S_OK;
+}
+#undef CK
+
+/* Get PC label offset. */
+int dasm_getpclabel(Dst_DECL, unsigned int pc)
+{
+  dasm_State *D = Dst_REF;
+  if (pc*sizeof(int) < D->pcsize) {
+    int pos = D->pclabels[pc];
+    if (pos < 0) return *DASM_POS2PTR(D, -pos);
+    if (pos > 0) return -1;  /* Undefined. */
+  }
+  return -2;  /* Unused or out of range. */
+}
+
+#ifdef DASM_CHECKS
+/* Optional sanity checker to call between isolated encoding steps. */
+int dasm_checkstep(Dst_DECL, int secmatch)
+{
+  dasm_State *D = Dst_REF;
+  if (D->status == DASM_S_OK) {
+    int i;
+    for (i = 1; i <= 9; i++) {
+      if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; }
+      D->lglabels[i] = 0;
+    }
+  }
+  if (D->status == DASM_S_OK && secmatch >= 0 &&
+      D->section != &D->sections[secmatch])
+    D->status = DASM_S_MATCH_SEC|(D->section-D->sections);
+  return D->status;
+}
+#endif
+
diff --git a/dynasm/dasm_arm64.lua b/dynasm/dasm_arm64.lua
new file mode 100644
index 00000000..d5640842
--- /dev/null
+++ b/dynasm/dasm_arm64.lua
@@ -0,0 +1,1166 @@
+------------------------------------------------------------------------------
+-- DynASM ARM64 module.
+--
+-- Copyright (C) 2005-2020 Mike Pall. All rights reserved.
+-- See dynasm.lua for full copyright notice.
+------------------------------------------------------------------------------
+
+-- Module information:
+local _info = {
+  arch =        "arm",
+  description = "DynASM ARM64 module",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2015-10-18",
+  author =      "Mike Pall",
+  license =     "MIT",
+}
+
+-- Exported glue functions for the arch-specific module.
+local _M = { _info = _info }
+
+-- Cache library functions.
+local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs
+local assert, setmetatable, rawget = assert, setmetatable, rawget
+local _s = string
+local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char
+local match, gmatch, gsub = _s.match, _s.gmatch, _s.gsub
+local concat, sort, insert = table.concat, table.sort, table.insert
+local bit = bit or require("bit")
+local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift
+local ror, tohex = bit.ror, bit.tohex
+
+-- Inherited tables and callbacks.
+local g_opt, g_arch
+local wline, werror, wfatal, wwarn
+
+-- Action name list.
+-- CHECK: Keep this in sync with the C code!
+local action_names = {
+  "STOP", "SECTION", "ESC", "REL_EXT",
+  "ALIGN", "REL_LG", "LABEL_LG",
+  "REL_PC", "LABEL_PC", "IMM", "IMM6", "IMM12", "IMM13W", "IMM13X", "IMML",
+}
+
+-- Maximum number of section buffer positions for dasm_put().
+-- CHECK: Keep this in sync with the C code!
+local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines.
+
+-- Action name -> action number.
+local map_action = {}
+for n,name in ipairs(action_names) do
+  map_action[name] = n-1
+end
+
+-- Action list buffer.
+local actlist = {}
+
+-- Argument list for next dasm_put(). Start with offset 0 into action list.
+local actargs = { 0 }
+
+-- Current number of section buffer positions for dasm_put().
+local secpos = 1
+
+------------------------------------------------------------------------------
+
+-- Dump action names and numbers.
+local function dumpactions(out)
+  out:write("DynASM encoding engine action codes:\n")
+  for n,name in ipairs(action_names) do
+    local num = map_action[name]
+    out:write(format("  %-10s %02X  %d\n", name, num, num))
+  end
+  out:write("\n")
+end
+
+-- Write action list buffer as a huge static C array.
+local function writeactions(out, name)
+  local nn = #actlist
+  if nn == 0 then nn = 1; actlist[0] = map_action.STOP end
+  out:write("static const unsigned int ", name, "[", nn, "] = {\n")
+  for i = 1,nn-1 do
+    assert(out:write("0x", tohex(actlist[i]), ",\n"))
+  end
+  assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n"))
+end
+
+------------------------------------------------------------------------------
+
+-- Add word to action list.
+local function wputxw(n)
+  assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range")
+  actlist[#actlist+1] = n
+end
+
+-- Add action to list with optional arg. Advance buffer pos, too.
+local function waction(action, val, a, num)
+  local w = assert(map_action[action], "bad action name `"..action.."'")
+  wputxw(w * 0x10000 + (val or 0))
+  if a then actargs[#actargs+1] = a end
+  if a or num then secpos = secpos + (num or 1) end
+end
+
+-- Flush action list (intervening C code or buffer pos overflow).
+local function wflush(term)
+  if #actlist == actargs[1] then return end -- Nothing to flush.
+  if not term then waction("STOP") end -- Terminate action list.
+  wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true)
+  actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put().
+  secpos = 1 -- The actionlist offset occupies a buffer position, too.
+end
+
+-- Put escaped word.
+local function wputw(n)
+  if n <= 0x000fffff then waction("ESC") end
+  wputxw(n)
+end
+
+-- Reserve position for word.
+local function wpos()
+  local pos = #actlist+1
+  actlist[pos] = ""
+  return pos
+end
+
+-- Store word to reserved position.
+local function wputpos(pos, n)
+  assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range")
+  if n <= 0x000fffff then
+    insert(actlist, pos+1, n)
+    n = map_action.ESC * 0x10000
+  end
+  actlist[pos] = n
+end
+
+------------------------------------------------------------------------------
+
+-- Global label name -> global label number. With auto assignment on 1st use.
+local next_global = 20
+local map_global = setmetatable({}, { __index = function(t, name)
+  if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end
+  local n = next_global
+  if n > 2047 then werror("too many global labels") end
+  next_global = n + 1
+  t[name] = n
+  return n
+end})
+
+-- Dump global labels.
+local function dumpglobals(out, lvl)
+  local t = {}
+  for name, n in pairs(map_global) do t[n] = name end
+  out:write("Global labels:\n")
+  for i=20,next_global-1 do
+    out:write(format("  %s\n", t[i]))
+  end
+  out:write("\n")
+end
+
+-- Write global label enum.
+local function writeglobals(out, prefix)
+  local t = {}
+  for name, n in pairs(map_global) do t[n] = name end
+  out:write("enum {\n")
+  for i=20,next_global-1 do
+    out:write("  ", prefix, t[i], ",\n")
+  end
+  out:write("  ", prefix, "_MAX\n};\n")
+end
+
+-- Write global label names.
+local function writeglobalnames(out, name)
+  local t = {}
+  for name, n in pairs(map_global) do t[n] = name end
+  out:write("static const char *const ", name, "[] = {\n")
+  for i=20,next_global-1 do
+    out:write("  \"", t[i], "\",\n")
+  end
+  out:write("  (const char *)0\n};\n")
+end
+
+------------------------------------------------------------------------------
+
+-- Extern label name -> extern label number. With auto assignment on 1st use.
+local next_extern = 0
+local map_extern_ = {}
+local map_extern = setmetatable({}, { __index = function(t, name)
+  -- No restrictions on the name for now.
+  local n = next_extern
+  if n > 2047 then werror("too many extern labels") end
+  next_extern = n + 1
+  t[name] = n
+  map_extern_[n] = name
+  return n
+end})
+
+-- Dump extern labels.
+local function dumpexterns(out, lvl)
+  out:write("Extern labels:\n")
+  for i=0,next_extern-1 do
+    out:write(format("  %s\n", map_extern_[i]))
+  end
+  out:write("\n")
+end
+
+-- Write extern label names.
+local function writeexternnames(out, name)
+  out:write("static const char *const ", name, "[] = {\n")
+  for i=0,next_extern-1 do
+    out:write("  \"", map_extern_[i], "\",\n")
+  end
+  out:write("  (const char *)0\n};\n")
+end
+
+------------------------------------------------------------------------------
+
+-- Arch-specific maps.
+
+-- Ext. register name -> int. name.
+local map_archdef = { xzr = "@x31", wzr = "@w31", lr = "x30", }
+
+-- Int. register name -> ext. name.
+local map_reg_rev = { ["@x31"] = "xzr", ["@w31"] = "wzr", x30 = "lr", }
+
+local map_type = {}             -- Type name -> { ctype, reg }
+local ctypenum = 0              -- Type number (for Dt... macros).
+
+-- Reverse defines for registers.
+function _M.revdef(s)
+  return map_reg_rev[s] or s
+end
+
+local map_shift = { lsl = 0, lsr = 1, asr = 2, }
+
+local map_extend = {
+  uxtb = 0, uxth = 1, uxtw = 2, uxtx = 3,
+  sxtb = 4, sxth = 5, sxtw = 6, sxtx = 7,
+}
+
+local map_cond = {
+  eq = 0, ne = 1, cs = 2, cc = 3, mi = 4, pl = 5, vs = 6, vc = 7,
+  hi = 8, ls = 9, ge = 10, lt = 11, gt = 12, le = 13, al = 14,
+  hs = 2, lo = 3,
+}
+
+------------------------------------------------------------------------------
+
+local parse_reg_type
+
+local function parse_reg(expr)
+  if not expr then werror("expected register name") end
+  local tname, ovreg = match(expr, "^([%w_]+):(@?%l%d+)$")
+  local tp = map_type[tname or expr]
+  if tp then
+    local reg = ovreg or tp.reg
+    if not reg then
+      werror("type `"..(tname or expr).."' needs a register override")
+    end
+    expr = reg
+  end
+  local ok31, rt, r = match(expr, "^(@?)([xwqdshb])([123]?[0-9])$")
+  if r then
+    r = tonumber(r)
+    if r <= 30 or (r == 31 and ok31 ~= "" or (rt ~= "w" and rt ~= "x")) then
+      if not parse_reg_type then
+        parse_reg_type = rt
+      elseif parse_reg_type ~= rt then
+        werror("register size mismatch")
+      end
+      return r, tp
+    end
+  end
+  werror("bad register name `"..expr.."'")
+end
+
+local function parse_reg_base(expr)
+  if expr == "sp" then return 0x3e0 end
+  local base, tp = parse_reg(expr)
+  if parse_reg_type ~= "x" then werror("bad register type") end
+  parse_reg_type = false
+  return shl(base, 5), tp
+end
+
+local parse_ctx = {}
+
+local loadenv = setfenv and function(s)
+  local code = loadstring(s, "")
+  if code then setfenv(code, parse_ctx) end
+  return code
+end or function(s)
+  return load(s, "", nil, parse_ctx)
+end
+
+-- Try to parse simple arithmetic, too, since some basic ops are aliases.
+local function parse_number(n)
+  local x = tonumber(n)
+  if x then return x end
+  local code = loadenv("return "..n)
+  if code then
+    local ok, y = pcall(code)
+    if ok then return y end
+  end
+  return nil
+end
+
+local function parse_imm(imm, bits, shift, scale, signed)
+  imm = match(imm, "^#(.*)$")
+  if not imm then werror("expected immediate operand") end
+  local n = parse_number(imm)
+  if n then
+    local m = sar(n, scale)
+    if shl(m, scale) == n then
+      if signed then
+        local s = sar(m, bits-1)
+        if s == 0 then return shl(m, shift)
+        elseif s == -1 then return shl(m + shl(1, bits), shift) end
+      else
+        if sar(m, bits) == 0 then return shl(m, shift) end
+      end
+    end
+    werror("out of range immediate `"..imm.."'")
+  else
+    waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm)
+    return 0
+  end
+end
+
+local function parse_imm12(imm)
+  imm = match(imm, "^#(.*)$")
+  if not imm then werror("expected immediate operand") end
+  local n = parse_number(imm)
+  if n then
+    if shr(n, 12) == 0 then
+      return shl(n, 10)
+    elseif band(n, 0xff000fff) == 0 then
+      return shr(n, 2) + 0x00400000
+    end
+    werror("out of range immediate `"..imm.."'")
+  else
+    waction("IMM12", 0, imm)
+    return 0
+  end
+end
+
+local function parse_imm13(imm)
+  imm = match(imm, "^#(.*)$")
+  if not imm then werror("expected immediate operand") end
+  local n = parse_number(imm)
+  local r64 = parse_reg_type == "x"
+  if n and n % 1 == 0 and n >= 0 and n <= 0xffffffff then
+    local inv = false
+    if band(n, 1) == 1 then n = bit.bnot(n); inv = true end
+    local t = {}
+    for i=1,32 do t[i] = band(n, 1); n = shr(n, 1) end
+    local b = table.concat(t)
+    b = b..(r64 and (inv and "1" or "0"):rep(32) or b)
+    local p0, p1, p0a, p1a = b:match("^(0+)(1+)(0*)(1*)")
+    if p0 then
+      local w = p1a == "" and (r64 and 64 or 32) or #p1+#p0a
+      if band(w, w-1) == 0 and b == b:sub(1, w):rep(64/w) then
+        local s = band(-2*w, 0x3f) - 1
+        if w == 64 then s = s + 0x1000 end
+        if inv then
+          return shl(w-#p1-#p0, 16) + shl(s+w-#p1, 10)
+        else
+          return shl(w-#p0, 16) + shl(s+#p1, 10)
+        end
+      end
+    end
+    werror("out of range immediate `"..imm.."'")
+  elseif r64 then
+    waction("IMM13X", 0, format("(unsigned int)(%s)", imm))
+    actargs[#actargs+1] = format("(unsigned int)((unsigned long long)(%s)>>32)", imm)
+    return 0
+  else
+    waction("IMM13W", 0, imm)
+    return 0
+  end
+end
+
+local function parse_imm6(imm)
+  imm = match(imm, "^#(.*)$")
+  if not imm then werror("expected immediate operand") end
+  local n = parse_number(imm)
+  if n then
+    if n >= 0 and n <= 63 then
+      return shl(band(n, 0x1f), 19) + (n >= 32 and 0x80000000 or 0)
+    end
+    werror("out of range immediate `"..imm.."'")
+  else
+    waction("IMM6", 0, imm)
+    return 0
+  end
+end
+
+local function parse_imm_load(imm, scale)
+  local n = parse_number(imm)
+  if n then
+    local m = sar(n, scale)
+    if shl(m, scale) == n and m >= 0 and m < 0x1000 then
+      return shl(m, 10) + 0x01000000 -- Scaled, unsigned 12 bit offset.
+    elseif n >= -256 and n < 256 then
+      return shl(band(n, 511), 12) -- Unscaled, signed 9 bit offset.
+    end
+    werror("out of range immediate `"..imm.."'")
+  else
+    waction("IMML", 0, imm)
+    return 0
+  end
+end
+
+local function parse_fpimm(imm)
+  imm = match(imm, "^#(.*)$")
+  if not imm then werror("expected immediate operand") end
+  local n = parse_number(imm)
+  if n then
+    local m, e = math.frexp(n)
+    local s, e2 = 0, band(e-2, 7)
+    if m < 0 then m = -m; s = 0x00100000 end
+    m = m*32-16
+    if m % 1 == 0 and m >= 0 and m <= 15 and sar(shl(e2, 29), 29)+2 == e then
+      return s + shl(e2, 17) + shl(m, 13)
+    end
+    werror("out of range immediate `"..imm.."'")
+  else
+    werror("NYI fpimm action")
+  end
+end
+
+local function parse_shift(expr)
+  local s, s2 = match(expr, "^(%S+)%s*(.*)$")
+  s = map_shift[s]
+  if not s then werror("expected shift operand") end
+  return parse_imm(s2, 6, 10, 0, false) + shl(s, 22)
+end
+
+local function parse_lslx16(expr)
+  local n = match(expr, "^lsl%s*#(%d+)$")
+  n = tonumber(n)
+  if not n then werror("expected shift operand") end
+  if band(n, parse_reg_type == "x" and 0xffffffcf or 0xffffffef) ~= 0 then
+    werror("bad shift amount")
+  end
+  return shl(n, 17)
+end
+
+local function parse_extend(expr)
+  local s, s2 = match(expr, "^(%S+)%s*(.*)$")
+  if s == "lsl" then
+    s = parse_reg_type == "x" and 3 or 2
+  else
+    s = map_extend[s]
+  end
+  if not s then werror("expected extend operand") end
+  return (s2 == "" and 0 or parse_imm(s2, 3, 10, 0, false)) + shl(s, 13)
+end
+
+local function parse_cond(expr, inv)
+  local c = map_cond[expr]
+  if not c then werror("expected condition operand") end
+  return shl(bit.bxor(c, inv), 12)
+end
+
+local function parse_load(params, nparams, n, op)
+  if params[n+2] then werror("too many operands") end
+  local pn, p2 = params[n], params[n+1]
+  local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$")
+  if not p1 then
+    if not p2 then
+      local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$")
+      if reg and tailr ~= "" then
+        local base, tp = parse_reg_base(reg)
+        if tp then
+          waction("IMML", 0, format(tp.ctypefmt, tailr))
+          return op + base
+        end
+      end
+    end
+    werror("expected address operand")
+  end
+  local scale = shr(op, 30)
+  if p2 then
+    if wb == "!" then werror("bad use of '!'") end
+    op = op + parse_reg_base(p1) + parse_imm(p2, 9, 12, 0, true) + 0x400
+  elseif wb == "!" then
+    local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$")
+    if not p1a then werror("bad use of '!'") end
+    op = op + parse_reg_base(p1a) + parse_imm(p2a, 9, 12, 0, true) + 0xc00
+  else
+    local p1a, p2a = match(p1, "^([^,%s]*)%s*(.*)$")
+    op = op + parse_reg_base(p1a)
+    if p2a ~= "" then
+      local imm = match(p2a, "^,%s*#(.*)$")
+      if imm then
+        op = op + parse_imm_load(imm, scale)
+      else
+        local p2b, p3b, p3s = match(p2a, "^,%s*([^,%s]*)%s*,?%s*(%S*)%s*(.*)$")
+        op = op + shl(parse_reg(p2b), 16) + 0x00200800
+        if parse_reg_type ~= "x" and parse_reg_type ~= "w" then
+          werror("bad index register type")
+        end
+        if p3b == "" then
+          if parse_reg_type ~= "x" then werror("bad index register type") end
+          op = op + 0x6000
+        else
+          if p3s == "" or p3s == "#0" then
+          elseif p3s == "#"..scale then
+            op = op + 0x1000
+          else
+            werror("bad scale")
+          end
+          if parse_reg_type == "x" then
+            if p3b == "lsl" and p3s ~= "" then op = op + 0x6000
+            elseif p3b == "sxtx" then op = op + 0xe000
+            else
+              werror("bad extend/shift specifier")
+            end
+          else
+            if p3b == "uxtw" then op = op + 0x4000
+            elseif p3b == "sxtw" then op = op + 0xc000
+            else
+              werror("bad extend/shift specifier")
+            end
+          end
+        end
+      end
+    else
+      if wb == "!" then werror("bad use of '!'") end
+      op = op + 0x01000000
+    end
+  end
+  return op
+end
+
+local function parse_load_pair(params, nparams, n, op)
+  if params[n+2] then werror("too many operands") end
+  local pn, p2 = params[n], params[n+1]
+  local scale = shr(op, 30) == 0 and 2 or 3
+  local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$")
+  if not p1 then
+    if not p2 then
+      local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$")
+      if reg and tailr ~= "" then
+        local base, tp = parse_reg_base(reg)
+        if tp then
+          waction("IMM", 32768+7*32+15+scale*1024, format(tp.ctypefmt, tailr))
+          return op + base + 0x01000000
+        end
+      end
+    end
+    werror("expected address operand")
+  end
+  if p2 then
+    if wb == "!" then werror("bad use of '!'") end
+    op = op + 0x00800000
+  else
+    local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$")
+    if p1a then p1, p2 = p1a, p2a else p2 = "#0" end
+    op = op + (wb == "!" and 0x01800000 or 0x01000000)
+  end
+  return op + parse_reg_base(p1) + parse_imm(p2, 7, 15, scale, true)
+end
+
+local function parse_label(label, def)
+  local prefix = sub(label, 1, 2)
+  -- =>label (pc label reference)
+  if prefix == "=>" then
+    return "PC", 0, sub(label, 3)
+  end
+  -- ->name (global label reference)
+  if prefix == "->" then
+    return "LG", map_global[sub(label, 3)]
+  end
+  if def then
+    -- [1-9] (local label definition)
+    if match(label, "^[1-9]$") then
+      return "LG", 10+tonumber(label)
+    end
+  else
+    -- [<>][1-9] (local label reference)
+    local dir, lnum = match(label, "^([<>])([1-9])$")
+    if dir then -- Fwd: 1-9, Bkwd: 11-19.
+      return "LG", lnum + (dir == ">" and 0 or 10)
+    end
+    -- extern label (extern label reference)
+    local extname = match(label, "^extern%s+(%S+)$")
+    if extname then
+      return "EXT", map_extern[extname]
+    end
+  end
+  werror("bad label `"..label.."'")
+end
+
+local function branch_type(op)
+  if band(op, 0x7c000000) == 0x14000000 then return 0 -- B, BL
+  elseif shr(op, 24) == 0x54 or band(op, 0x7e000000) == 0x34000000 or
+         band(op, 0x3b000000) == 0x18000000 then
+    return 0x800 -- B.cond, CBZ, CBNZ, LDR* literal
+  elseif band(op, 0x7e000000) == 0x36000000 then return 0x1000 -- TBZ, TBNZ
+  elseif band(op, 0x9f000000) == 0x10000000 then return 0x2000 -- ADR
+  elseif band(op, 0x9f000000) == band(0x90000000) then return 0x3000 -- ADRP
+  else
+    assert(false, "unknown branch type")
+  end
+end
+
+------------------------------------------------------------------------------
+
+local map_op, op_template
+
+local function op_alias(opname, f)
+  return function(params, nparams)
+    if not params then return "-> "..opname:sub(1, -3) end
+    f(params, nparams)
+    op_template(params, map_op[opname], nparams)
+  end
+end
+
+local function alias_bfx(p)
+  p[4] = "#("..p[3]:sub(2)..")+("..p[4]:sub(2)..")-1"
+end
+
+local function alias_bfiz(p)
+  parse_reg(p[1])
+  if parse_reg_type == "w" then
+    p[3] = "#-("..p[3]:sub(2)..")%32"
+    p[4] = "#("..p[4]:sub(2)..")-1"
+  else
+    p[3] = "#-("..p[3]:sub(2)..")%64"
+    p[4] = "#("..p[4]:sub(2)..")-1"
+  end
+end
+
+local alias_lslimm = op_alias("ubfm_4", function(p)
+  parse_reg(p[1])
+  local sh = p[3]:sub(2)
+  if parse_reg_type == "w" then
+    p[3] = "#-("..sh..")%32"
+    p[4] = "#31-("..sh..")"
+  else
+    p[3] = "#-("..sh..")%64"
+    p[4] = "#63-("..sh..")"
+  end
+end)
+
+-- Template strings for ARM instructions.
+map_op = {
+  -- Basic data processing instructions.
+  add_3  = "0b000000DNMg|11000000pDpNIg|8b206000pDpNMx",
+  add_4  = "0b000000DNMSg|0b200000DNMXg|8b200000pDpNMXx|8b200000pDpNxMwX",
+  adds_3 = "2b000000DNMg|31000000DpNIg|ab206000DpNMx",
+  adds_4 = "2b000000DNMSg|2b200000DNMXg|ab200000DpNMXx|ab200000DpNxMwX",
+  cmn_2  = "2b00001fNMg|3100001fpNIg|ab20601fpNMx",
+  cmn_3  = "2b00001fNMSg|2b20001fNMXg|ab20001fpNMXx|ab20001fpNxMwX",
+
+  sub_3  = "4b000000DNMg|51000000pDpNIg|cb206000pDpNMx",
+  sub_4  = "4b000000DNMSg|4b200000DNMXg|cb200000pDpNMXx|cb200000pDpNxMwX",
+  subs_3 = "6b000000DNMg|71000000DpNIg|eb206000DpNMx",
+  subs_4 = "6b000000DNMSg|6b200000DNMXg|eb200000DpNMXx|eb200000DpNxMwX",
+  cmp_2  = "6b00001fNMg|7100001fpNIg|eb20601fpNMx",
+  cmp_3  = "6b00001fNMSg|6b20001fNMXg|eb20001fpNMXx|eb20001fpNxMwX",
+
+  neg_2  = "4b0003e0DMg",
+  neg_3  = "4b0003e0DMSg",
+  negs_2 = "6b0003e0DMg",
+  negs_3 = "6b0003e0DMSg",
+
+  adc_3  = "1a000000DNMg",
+  adcs_3 = "3a000000DNMg",
+  sbc_3  = "5a000000DNMg",
+  sbcs_3 = "7a000000DNMg",
+  ngc_2  = "5a0003e0DMg",
+  ngcs_2 = "7a0003e0DMg",
+
+  and_3  = "0a000000DNMg|12000000pDNig",
+  and_4  = "0a000000DNMSg",
+  orr_3  = "2a000000DNMg|32000000pDNig",
+  orr_4  = "2a000000DNMSg",
+  eor_3  = "4a000000DNMg|52000000pDNig",
+  eor_4  = "4a000000DNMSg",
+  ands_3 = "6a000000DNMg|72000000DNig",
+  ands_4 = "6a000000DNMSg",
+  tst_2  = "6a00001fNMg|7200001fNig",
+  tst_3  = "6a00001fNMSg",
+
+  bic_3  = "0a200000DNMg",
+  bic_4  = "0a200000DNMSg",
+  orn_3  = "2a200000DNMg",
+  orn_4  = "2a200000DNMSg",
+  eon_3  = "4a200000DNMg",
+  eon_4  = "4a200000DNMSg",
+  bics_3 = "6a200000DNMg",
+  bics_4 = "6a200000DNMSg",
+
+  movn_2 = "12800000DWg",
+  movn_3 = "12800000DWRg",
+  movz_2 = "52800000DWg",
+  movz_3 = "52800000DWRg",
+  movk_2 = "72800000DWg",
+  movk_3 = "72800000DWRg",
+
+  -- TODO: this doesn't cover all valid immediates for mov reg, #imm.
+  mov_2  = "2a0003e0DMg|52800000DW|320003e0pDig|11000000pDpNg",
+  mov_3  = "2a0003e0DMSg",
+  mvn_2  = "2a2003e0DMg",
+  mvn_3  = "2a2003e0DMSg",
+
+  adr_2  = "10000000DBx",
+  adrp_2 = "90000000DBx",
+
+  csel_4  = "1a800000DNMCg",
+  csinc_4 = "1a800400DNMCg",
+  csinv_4 = "5a800000DNMCg",
+  csneg_4 = "5a800400DNMCg",
+  cset_2  = "1a9f07e0Dcg",
+  csetm_2 = "5a9f03e0Dcg",
+  cinc_3  = "1a800400DNmcg",
+  cinv_3  = "5a800000DNmcg",
+  cneg_3  = "5a800400DNmcg",
+
+  ccmn_4 = "3a400000NMVCg|3a400800N5VCg",
+  ccmp_4 = "7a400000NMVCg|7a400800N5VCg",
+
+  madd_4 = "1b000000DNMAg",
+  msub_4 = "1b008000DNMAg",
+  mul_3  = "1b007c00DNMg",
+  mneg_3 = "1b00fc00DNMg",
+
+  smaddl_4 = "9b200000DxNMwAx",
+  smsubl_4 = "9b208000DxNMwAx",
+  smull_3  = "9b207c00DxNMw",
+  smnegl_3 = "9b20fc00DxNMw",
+  smulh_3  = "9b407c00DNMx",
+  umaddl_4 = "9ba00000DxNMwAx",
+  umsubl_4 = "9ba08000DxNMwAx",
+  umull_3  = "9ba07c00DxNMw",
+  umnegl_3 = "9ba0fc00DxNMw",
+  umulh_3  = "9bc07c00DNMx",
+
+  udiv_3 = "1ac00800DNMg",
+  sdiv_3 = "1ac00c00DNMg",
+
+  -- Bit operations.
+  sbfm_4 = "13000000DN12w|93400000DN12x",
+  bfm_4  = "33000000DN12w|b3400000DN12x",
+  ubfm_4 = "53000000DN12w|d3400000DN12x",
+  extr_4 = "13800000DNM2w|93c00000DNM2x",
+
+  sxtb_2 = "13001c00DNw|93401c00DNx",
+  sxth_2 = "13003c00DNw|93403c00DNx",
+  sxtw_2 = "93407c00DxNw",
+  uxtb_2 = "53001c00DNw",
+  uxth_2 = "53003c00DNw",
+
+  sbfx_4  = op_alias("sbfm_4", alias_bfx),
+  bfxil_4 = op_alias("bfm_4", alias_bfx),
+  ubfx_4  = op_alias("ubfm_4", alias_bfx),
+  sbfiz_4 = op_alias("sbfm_4", alias_bfiz),
+  bfi_4   = op_alias("bfm_4", alias_bfiz),
+  ubfiz_4 = op_alias("ubfm_4", alias_bfiz),
+
+  lsl_3  = function(params, nparams)
+    if params and params[3]:byte() == 35 then
+      return alias_lslimm(params, nparams)
+    else
+      return op_template(params, "1ac02000DNMg", nparams)
+    end
+  end,
+  lsr_3  = "1ac02400DNMg|53007c00DN1w|d340fc00DN1x",
+  asr_3  = "1ac02800DNMg|13007c00DN1w|9340fc00DN1x",
+  ror_3  = "1ac02c00DNMg|13800000DNm2w|93c00000DNm2x",
+
+  clz_2   = "5ac01000DNg",
+  cls_2   = "5ac01400DNg",
+  rbit_2  = "5ac00000DNg",
+  rev_2   = "5ac00800DNw|dac00c00DNx",
+  rev16_2 = "5ac00400DNg",
+  rev32_2 = "dac00800DNx",
+
+  -- Loads and stores.
+  ["strb_*"]  = "38000000DwL",
+  ["ldrb_*"]  = "38400000DwL",
+  ["ldrsb_*"] = "38c00000DwL|38800000DxL",
+  ["strh_*"]  = "78000000DwL",
+  ["ldrh_*"]  = "78400000DwL",
+  ["ldrsh_*"] = "78c00000DwL|78800000DxL",
+  ["str_*"]   = "b8000000DwL|f8000000DxL|bc000000DsL|fc000000DdL",
+  ["ldr_*"]   = "18000000DwB|58000000DxB|1c000000DsB|5c000000DdB|b8400000DwL|f8400000DxL|bc400000DsL|fc400000DdL",
+  ["ldrsw_*"] = "98000000DxB|b8800000DxL",
+  -- NOTE: ldur etc. are handled by ldr et al.
+
+  ["stp_*"]   = "28000000DAwP|a8000000DAxP|2c000000DAsP|6c000000DAdP",
+  ["ldp_*"]   = "28400000DAwP|a8400000DAxP|2c400000DAsP|6c400000DAdP",
+  ["ldpsw_*"] = "68400000DAxP",
+
+  -- Branches.
+  b_1    = "14000000B",
+  bl_1   = "94000000B",
+  blr_1  = "d63f0000Nx",
+  br_1   = "d61f0000Nx",
+  ret_0  = "d65f03c0",
+  ret_1  = "d65f0000Nx",
+  -- b.cond is added below.
+  cbz_2  = "34000000DBg",
+  cbnz_2 = "35000000DBg",
+  tbz_3  = "36000000DTBw|36000000DTBx",
+  tbnz_3 = "37000000DTBw|37000000DTBx",
+
+  -- Miscellaneous instructions.
+  -- TODO: hlt, hvc, smc, svc, eret, dcps[123], drps, mrs, msr
+  -- TODO: sys, sysl, ic, dc, at, tlbi
+  -- TODO: hint, yield, wfe, wfi, sev, sevl
+  -- TODO: clrex, dsb, dmb, isb
+  nop_0  = "d503201f",
+  brk_0  = "d4200000",
+  brk_1  = "d4200000W",
+
+  -- Floating point instructions.
+  fmov_2  = "1e204000DNf|1e260000DwNs|1e270000DsNw|9e660000DxNd|9e670000DdNx|1e201000DFf",
+  fabs_2  = "1e20c000DNf",
+  fneg_2  = "1e214000DNf",
+  fsqrt_2 = "1e21c000DNf",
+
+  fcvt_2  = "1e22c000DdNs|1e624000DsNd",
+
+  -- TODO: half-precision and fixed-point conversions.
+  fcvtas_2 = "1e240000DwNs|9e240000DxNs|1e640000DwNd|9e640000DxNd",
+  fcvtau_2 = "1e250000DwNs|9e250000DxNs|1e650000DwNd|9e650000DxNd",
+  fcvtms_2 = "1e300000DwNs|9e300000DxNs|1e700000DwNd|9e700000DxNd",
+  fcvtmu_2 = "1e310000DwNs|9e310000DxNs|1e710000DwNd|9e710000DxNd",
+  fcvtns_2 = "1e200000DwNs|9e200000DxNs|1e600000DwNd|9e600000DxNd",
+  fcvtnu_2 = "1e210000DwNs|9e210000DxNs|1e610000DwNd|9e610000DxNd",
+  fcvtps_2 = "1e280000DwNs|9e280000DxNs|1e680000DwNd|9e680000DxNd",
+  fcvtpu_2 = "1e290000DwNs|9e290000DxNs|1e690000DwNd|9e690000DxNd",
+  fcvtzs_2 = "1e380000DwNs|9e380000DxNs|1e780000DwNd|9e780000DxNd",
+  fcvtzu_2 = "1e390000DwNs|9e390000DxNs|1e790000DwNd|9e790000DxNd",
+
+  scvtf_2  = "1e220000DsNw|9e220000DsNx|1e620000DdNw|9e620000DdNx",
+  ucvtf_2  = "1e230000DsNw|9e230000DsNx|1e630000DdNw|9e630000DdNx",
+
+  frintn_2 = "1e244000DNf",
+  frintp_2 = "1e24c000DNf",
+  frintm_2 = "1e254000DNf",
+  frintz_2 = "1e25c000DNf",
+  frinta_2 = "1e264000DNf",
+  frintx_2 = "1e274000DNf",
+  frinti_2 = "1e27c000DNf",
+
+  fadd_3   = "1e202800DNMf",
+  fsub_3   = "1e203800DNMf",
+  fmul_3   = "1e200800DNMf",
+  fnmul_3  = "1e208800DNMf",
+  fdiv_3   = "1e201800DNMf",
+
+  fmadd_4  = "1f000000DNMAf",
+  fmsub_4  = "1f008000DNMAf",
+  fnmadd_4 = "1f200000DNMAf",
+  fnmsub_4 = "1f208000DNMAf",
+
+  fmax_3   = "1e204800DNMf",
+  fmaxnm_3 = "1e206800DNMf",
+  fmin_3   = "1e205800DNMf",
+  fminnm_3 = "1e207800DNMf",
+
+  fcmp_2   = "1e202000NMf|1e202008NZf",
+  fcmpe_2  = "1e202010NMf|1e202018NZf",
+
+  fccmp_4  = "1e200400NMVCf",
+  fccmpe_4 = "1e200410NMVCf",
+
+  fcsel_4  = "1e200c00DNMCf",
+
+  -- TODO: crc32*, aes*, sha*, pmull
+  -- TODO: SIMD instructions.
+}
+
+for cond,c in pairs(map_cond) do
+  map_op["b"..cond.."_1"] = tohex(0x54000000+c).."B"
+end
+
+------------------------------------------------------------------------------
+
+-- Handle opcodes defined with template strings.
+local function parse_template(params, template, nparams, pos)
+  local op = tonumber(sub(template, 1, 8), 16)
+  local n = 1
+  local rtt = {}
+
+  parse_reg_type = false
+
+  -- Process each character.
+  for p in gmatch(sub(template, 9), ".") do
+    local q = params[n]
+    if p == "D" then
+      op = op + parse_reg(q); n = n + 1
+    elseif p == "N" then
+      op = op + shl(parse_reg(q), 5); n = n + 1
+    elseif p == "M" then
+      op = op + shl(parse_reg(q), 16); n = n + 1
+    elseif p == "A" then
+      op = op + shl(parse_reg(q), 10); n = n + 1
+    elseif p == "m" then
+      op = op + shl(parse_reg(params[n-1]), 16)
+
+    elseif p == "p" then
+      if q == "sp" then params[n] = "@x31" end
+    elseif p == "g" then
+      if parse_reg_type == "x" then
+        op = op + 0x80000000
+      elseif parse_reg_type ~= "w" then
+        werror("bad register type")
+      end
+      parse_reg_type = false
+    elseif p == "f" then
+      if parse_reg_type == "d" then
+        op = op + 0x00400000
+      elseif parse_reg_type ~= "s" then
+        werror("bad register type")
+      end
+      parse_reg_type = false
+    elseif p == "x" or p == "w" or p == "d" or p == "s" then
+      if parse_reg_type ~= p then
+        werror("register size mismatch")
+      end
+      parse_reg_type = false
+
+    elseif p == "L" then
+      op = parse_load(params, nparams, n, op)
+    elseif p == "P" then
+      op = parse_load_pair(params, nparams, n, op)
+
+    elseif p == "B" then
+      local mode, v, s = parse_label(q, false); n = n + 1
+      local m = branch_type(op)
+      waction("REL_"..mode, v+m, s, 1)
+
+    elseif p == "I" then
+      op = op + parse_imm12(q); n = n + 1
+    elseif p == "i" then
+      op = op + parse_imm13(q); n = n + 1
+    elseif p == "W" then
+      op = op + parse_imm(q, 16, 5, 0, false); n = n + 1
+    elseif p == "T" then
+      op = op + parse_imm6(q); n = n + 1
+    elseif p == "1" then
+      op = op + parse_imm(q, 6, 16, 0, false); n = n + 1
+    elseif p == "2" then
+      op = op + parse_imm(q, 6, 10, 0, false); n = n + 1
+    elseif p == "5" then
+      op = op + parse_imm(q, 5, 16, 0, false); n = n + 1
+    elseif p == "V" then
+      op = op + parse_imm(q, 4, 0, 0, false); n = n + 1
+    elseif p == "F" then
+      op = op + parse_fpimm(q); n = n + 1
+    elseif p == "Z" then
+      if q ~= "#0" and q ~= "#0.0" then werror("expected zero immediate") end
+      n = n + 1
+
+    elseif p == "S" then
+      op = op + parse_shift(q); n = n + 1
+    elseif p == "X" then
+      op = op + parse_extend(q); n = n + 1
+    elseif p == "R" then
+      op = op + parse_lslx16(q); n = n + 1
+    elseif p == "C" then
+      op = op + parse_cond(q, 0); n = n + 1
+    elseif p == "c" then
+      op = op + parse_cond(q, 1); n = n + 1
+
+    else
+      assert(false)
+    end
+  end
+  wputpos(pos, op)
+end
+
+function op_template(params, template, nparams)
+  if not params then return template:gsub("%x%x%x%x%x%x%x%x", "") end
+
+  -- Limit number of section buffer positions used by a single dasm_put().
+  -- A single opcode needs a maximum of 3 positions.
+  if secpos+3 > maxsecpos then wflush() end
+  local pos = wpos()
+  local lpos, apos, spos = #actlist, #actargs, secpos
+
+  local ok, err
+  for t in gmatch(template, "[^|]+") do
+    ok, err = pcall(parse_template, params, t, nparams, pos)
+    if ok then return end
+    secpos = spos
+    actlist[lpos+1] = nil
+    actlist[lpos+2] = nil
+    actlist[lpos+3] = nil
+    actargs[apos+1] = nil
+    actargs[apos+2] = nil
+    actargs[apos+3] = nil
+  end
+  error(err, 0)
+end
+
+map_op[".template__"] = op_template
+
+------------------------------------------------------------------------------
+
+-- Pseudo-opcode to mark the position where the action list is to be emitted.
+map_op[".actionlist_1"] = function(params)
+  if not params then return "cvar" end
+  local name = params[1] -- No syntax check. You get to keep the pieces.
+  wline(function(out) writeactions(out, name) end)
+end
+
+-- Pseudo-opcode to mark the position where the global enum is to be emitted.
+map_op[".globals_1"] = function(params)
+  if not params then return "prefix" end
+  local prefix = params[1] -- No syntax check. You get to keep the pieces.
+  wline(function(out) writeglobals(out, prefix) end)
+end
+
+-- Pseudo-opcode to mark the position where the global names are to be emitted.
+map_op[".globalnames_1"] = function(params)
+  if not params then return "cvar" end
+  local name = params[1] -- No syntax check. You get to keep the pieces.
+  wline(function(out) writeglobalnames(out, name) end)
+end
+
+-- Pseudo-opcode to mark the position where the extern names are to be emitted.
+map_op[".externnames_1"] = function(params)
+  if not params then return "cvar" end
+  local name = params[1] -- No syntax check. You get to keep the pieces.
+  wline(function(out) writeexternnames(out, name) end)
+end
+
+------------------------------------------------------------------------------
+
+-- Label pseudo-opcode (converted from trailing colon form).
+map_op[".label_1"] = function(params)
+  if not params then return "[1-9] | ->global | =>pcexpr" end
+  if secpos+1 > maxsecpos then wflush() end
+  local mode, n, s = parse_label(params[1], true)
+  if mode == "EXT" then werror("bad label definition") end
+  waction("LABEL_"..mode, n, s, 1)
+end
+
+------------------------------------------------------------------------------
+
+-- Pseudo-opcodes for data storage.
+map_op[".long_*"] = function(params)
+  if not params then return "imm..." end
+  for _,p in ipairs(params) do
+    local n = tonumber(p)
+    if not n then werror("bad immediate `"..p.."'") end
+    if n < 0 then n = n + 2^32 end
+    wputw(n)
+    if secpos+2 > maxsecpos then wflush() end
+  end
+end
+
+-- Alignment pseudo-opcode.
+map_op[".align_1"] = function(params)
+  if not params then return "numpow2" end
+  if secpos+1 > maxsecpos then wflush() end
+  local align = tonumber(params[1])
+  if align then
+    local x = align
+    -- Must be a power of 2 in the range (2 ... 256).
+    for i=1,8 do
+      x = x / 2
+      if x == 1 then
+        waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1.
+        return
+      end
+    end
+  end
+  werror("bad alignment")
+end
+
+------------------------------------------------------------------------------
+
+-- Pseudo-opcode for (primitive) type definitions (map to C types).
+map_op[".type_3"] = function(params, nparams)
+  if not params then
+    return nparams == 2 and "name, ctype" or "name, ctype, reg"
+  end
+  local name, ctype, reg = params[1], params[2], params[3]
+  if not match(name, "^[%a_][%w_]*$") then
+    werror("bad type name `"..name.."'")
+  end
+  local tp = map_type[name]
+  if tp then
+    werror("duplicate type `"..name.."'")
+  end
+  -- Add #type to defines. A bit unclean to put it in map_archdef.
+  map_archdef["#"..name] = "sizeof("..ctype..")"
+  -- Add new type and emit shortcut define.
+  local num = ctypenum + 1
+  map_type[name] = {
+    ctype = ctype,
+    ctypefmt = format("Dt%X(%%s)", num),
+    reg = reg,
+  }
+  wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype))
+  ctypenum = num
+end
+map_op[".type_2"] = map_op[".type_3"]
+
+-- Dump type definitions.
+local function dumptypes(out, lvl)
+  local t = {}
+  for name in pairs(map_type) do t[#t+1] = name end
+  sort(t)
+  out:write("Type definitions:\n")
+  for _,name in ipairs(t) do
+    local tp = map_type[name]
+    local reg = tp.reg or ""
+    out:write(format("  %-20s %-20s %s\n", name, tp.ctype, reg))
+  end
+  out:write("\n")
+end
+
+------------------------------------------------------------------------------
+
+-- Set the current section.
+function _M.section(num)
+  waction("SECTION", num)
+  wflush(true) -- SECTION is a terminal action.
+end
+
+------------------------------------------------------------------------------
+
+-- Dump architecture description.
+function _M.dumparch(out)
+  out:write(format("DynASM %s version %s, released %s\n\n",
+    _info.arch, _info.version, _info.release))
+  dumpactions(out)
+end
+
+-- Dump all user defined elements.
+function _M.dumpdef(out, lvl)
+  dumptypes(out, lvl)
+  dumpglobals(out, lvl)
+  dumpexterns(out, lvl)
+end
+
+------------------------------------------------------------------------------
+
+-- Pass callbacks from/to the DynASM core.
+function _M.passcb(wl, we, wf, ww)
+  wline, werror, wfatal, wwarn = wl, we, wf, ww
+  return wflush
+end
+
+-- Setup the arch-specific module.
+function _M.setup(arch, opt)
+  g_arch, g_opt = arch, opt
+end
+
+-- Merge the core maps and the arch-specific maps.
+function _M.mergemaps(map_coreop, map_def)
+  setmetatable(map_op, { __index = map_coreop })
+  setmetatable(map_def, { __index = map_archdef })
+  return map_op, map_def
+end
+
+return _M
+
+------------------------------------------------------------------------------
+
diff --git a/dynasm/dasm_mips.h b/dynasm/dasm_mips.h
index 1b309edd..143c3cbe 100644
--- a/dynasm/dasm_mips.h
+++ b/dynasm/dasm_mips.h
@@ -21,7 +21,7 @@ enum {
   /* The following actions need a buffer position. */
   DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
   /* The following actions also have an argument. */
-  DASM_REL_PC, DASM_LABEL_PC, DASM_IMM,
+  DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMS,
   DASM__MAX
 };
 
@@ -231,7 +231,7 @@ void dasm_put(Dst_DECL, int start, ...)
         *pl = -pos;  /* Label exists now. */
         b[pos++] = ofs;  /* Store pass1 offset estimate. */
         break;
-      case DASM_IMM:
+      case DASM_IMM: case DASM_IMMS:
 #ifdef DASM_CHECKS
         CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I);
 #endif
@@ -299,7 +299,7 @@ int dasm_link(Dst_DECL, size_t *szp)
         case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
         case DASM_REL_LG: case DASM_REL_PC: pos++; break;
         case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
-        case DASM_IMM: pos++; break;
+        case DASM_IMM: case DASM_IMMS: pos++; break;
         }
       }
       stop: (void)0;
@@ -355,19 +355,23 @@ int dasm_encode(Dst_DECL, void *buffer)
           CK(n >= 0, UNDEF_PC);
           n = *DASM_POS2PTR(D, n);
           if (ins & 2048)
-            n = n - (int)((char *)cp - base);
+            n = (n + (int)(size_t)base) & 0x0fffffff;
           else
-            n = (n + (int)base) & 0x0fffffff;
-        patchrel:
+            n = n - (int)((char *)cp - base);
+        patchrel: {
+          unsigned int e = 16 + ((ins >> 12) & 15);
           CK((n & 3) == 0 &&
-             ((n + ((ins & 2048) ? 0x00020000 : 0)) >>
-               ((ins & 2048) ? 18 : 28)) == 0, RANGE_REL);
-          cp[-1] |= ((n>>2) & ((ins & 2048) ? 0x0000ffff: 0x03ffffff));
+             ((n + ((ins & 2048) ? 0 : (1<<(e+1)))) >> (e+2)) == 0, RANGE_REL);
+          cp[-1] |= ((n>>2) & ((1<<e)-1));
+          }
           break;
         case DASM_LABEL_LG:
           ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n);
           break;
         case DASM_LABEL_PC: break;
+        case DASM_IMMS:
+          cp[-1] |= ((n>>3) & 4); n &= 0x1f;
+          /* fallthrough */
         case DASM_IMM:
           cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
           break;
diff --git a/dynasm/dasm_mips.lua b/dynasm/dasm_mips.lua
index 9ef280e3..3e41df52 100644
--- a/dynasm/dasm_mips.lua
+++ b/dynasm/dasm_mips.lua
@@ -1,17 +1,20 @@
 ------------------------------------------------------------------------------
--- DynASM MIPS module.
+-- DynASM MIPS32/MIPS64 module.
 --
 -- Copyright (C) 2005-2020 Mike Pall. All rights reserved.
 -- See dynasm.lua for full copyright notice.
 ------------------------------------------------------------------------------
 
+local mips64 = mips64
+local mipsr6 = _map_def.MIPSR6
+
 -- Module information:
 local _info = {
-  arch =        "mips",
-  description = "DynASM MIPS module",
-  version =     "1.3.0",
-  vernum =       10300,
-  release =     "2012-01-23",
+  arch =        mips64 and "mips64" or "mips",
+  description = "DynASM MIPS32/MIPS64 module",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2020-01-20",
   author =      "Mike Pall",
   license =     "MIT",
 }
@@ -27,7 +30,8 @@ local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char
 local match, gmatch = _s.match, _s.gmatch
 local concat, sort = table.concat, table.sort
 local bit = bit or require("bit")
-local band, shl, sar, tohex = bit.band, bit.lshift, bit.arshift, bit.tohex
+local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift
+local tohex = bit.tohex
 
 -- Inherited tables and callbacks.
 local g_opt, g_arch
@@ -38,7 +42,7 @@ local wline, werror, wfatal, wwarn
 local action_names = {
   "STOP", "SECTION", "ESC", "REL_EXT",
   "ALIGN", "REL_LG", "LABEL_LG",
-  "REL_PC", "LABEL_PC", "IMM",
+  "REL_PC", "LABEL_PC", "IMM", "IMMS",
 }
 
 -- Maximum number of section buffer positions for dasm_put().
@@ -235,7 +239,6 @@ local map_op = {
   bne_3 =       "14000000STB",
   blez_2 =      "18000000SB",
   bgtz_2 =      "1c000000SB",
-  addi_3 =      "20000000TSI",
   li_2 =        "24000000TI",
   addiu_3 =     "24000000TSI",
   slti_3 =      "28000000TSI",
@@ -245,70 +248,52 @@ local map_op = {
   ori_3 =       "34000000TSU",
   xori_3 =      "38000000TSU",
   lui_2 =       "3c000000TU",
-  beqzl_2 =     "50000000SB",
-  beql_3 =      "50000000STB",
-  bnezl_2 =     "54000000SB",
-  bnel_3 =      "54000000STB",
-  blezl_2 =     "58000000SB",
-  bgtzl_2 =     "5c000000SB",
+  daddiu_3 =    mips64 and "64000000TSI",
+  ldl_2 =       mips64 and "68000000TO",
+  ldr_2 =       mips64 and "6c000000TO",
   lb_2 =        "80000000TO",
   lh_2 =        "84000000TO",
-  lwl_2 =       "88000000TO",
   lw_2 =        "8c000000TO",
   lbu_2 =       "90000000TO",
   lhu_2 =       "94000000TO",
-  lwr_2 =       "98000000TO",
+  lwu_2 =       mips64 and "9c000000TO",
   sb_2 =        "a0000000TO",
   sh_2 =        "a4000000TO",
-  swl_2 =       "a8000000TO",
   sw_2 =        "ac000000TO",
-  swr_2 =       "b8000000TO",
-  cache_2 =     "bc000000NO",
-  ll_2 =        "c0000000TO",
   lwc1_2 =      "c4000000HO",
-  pref_2 =      "cc000000NO",
   ldc1_2 =      "d4000000HO",
-  sc_2 =        "e0000000TO",
+  ld_2 =        mips64 and "dc000000TO",
   swc1_2 =      "e4000000HO",
   sdc1_2 =      "f4000000HO",
+  sd_2 =        mips64 and "fc000000TO",
 
   -- Opcode SPECIAL.
   nop_0 =       "00000000",
   sll_3 =       "00000000DTA",
-  movf_2 =      "00000001DS",
-  movf_3 =      "00000001DSC",
-  movt_2 =      "00010001DS",
-  movt_3 =      "00010001DSC",
+  sextw_2 =     "00000000DT",
   srl_3 =       "00000002DTA",
   rotr_3 =      "00200002DTA",
   sra_3 =       "00000003DTA",
   sllv_3 =      "00000004DTS",
   srlv_3 =      "00000006DTS",
   rotrv_3 =     "00000046DTS",
+  drotrv_3 =    mips64 and "00000056DTS",
   srav_3 =      "00000007DTS",
-  jr_1 =        "00000008S",
   jalr_1 =      "0000f809S",
   jalr_2 =      "00000009DS",
-  movz_3 =      "0000000aDST",
-  movn_3 =      "0000000bDST",
   syscall_0 =   "0000000c",
   syscall_1 =   "0000000cY",
   break_0 =     "0000000d",
   break_1 =     "0000000dY",
   sync_0 =      "0000000f",
-  mfhi_1 =      "00000010D",
-  mthi_1 =      "00000011S",
-  mflo_1 =      "00000012D",
-  mtlo_1 =      "00000013S",
-  mult_2 =      "00000018ST",
-  multu_2 =     "00000019ST",
-  div_2 =       "0000001aST",
-  divu_2 =      "0000001bST",
+  dsllv_3 =     mips64 and "00000014DTS",
+  dsrlv_3 =     mips64 and "00000016DTS",
+  dsrav_3 =     mips64 and "00000017DTS",
   add_3 =       "00000020DST",
-  move_2 =      "00000021DS",
+  move_2 =      mips64 and "00000025DS" or "00000021DS",
   addu_3 =      "00000021DST",
   sub_3 =       "00000022DST",
-  negu_2 =      "00000023DT",
+  negu_2 =      mips64 and "0000002fDT" or "00000023DT",
   subu_3 =      "00000023DST",
   and_3 =       "00000024DST",
   or_3 =        "00000025DST",
@@ -317,6 +302,10 @@ local map_op = {
   nor_3 =       "00000027DST",
   slt_3 =       "0000002aDST",
   sltu_3 =      "0000002bDST",
+  dadd_3 =      mips64 and "0000002cDST",
+  daddu_3 =     mips64 and "0000002dDST",
+  dsub_3 =      mips64 and "0000002eDST",
+  dsubu_3 =     mips64 and "0000002fDST",
   tge_2 =       "00000030ST",
   tge_3 =       "00000030STZ",
   tgeu_2 =      "00000031ST",
@@ -329,40 +318,36 @@ local map_op = {
   teq_3 =       "00000034STZ",
   tne_2 =       "00000036ST",
   tne_3 =       "00000036STZ",
+  dsll_3 =      mips64 and "00000038DTa",
+  dsrl_3 =      mips64 and "0000003aDTa",
+  drotr_3 =     mips64 and "0020003aDTa",
+  dsra_3 =      mips64 and "0000003bDTa",
+  dsll32_3 =    mips64 and "0000003cDTA",
+  dsrl32_3 =    mips64 and "0000003eDTA",
+  drotr32_3 =   mips64 and "0020003eDTA",
+  dsra32_3 =    mips64 and "0000003fDTA",
 
   -- Opcode REGIMM.
   bltz_2 =      "04000000SB",
   bgez_2 =      "04010000SB",
   bltzl_2 =     "04020000SB",
   bgezl_2 =     "04030000SB",
-  tgei_2 =      "04080000SI",
-  tgeiu_2 =     "04090000SI",
-  tlti_2 =      "040a0000SI",
-  tltiu_2 =     "040b0000SI",
-  teqi_2 =      "040c0000SI",
-  tnei_2 =      "040e0000SI",
-  bltzal_2 =    "04100000SB",
   bal_1 =       "04110000B",
-  bgezal_2 =    "04110000SB",
-  bltzall_2 =   "04120000SB",
-  bgezall_2 =   "04130000SB",
   synci_1 =     "041f0000O",
 
-  -- Opcode SPECIAL2.
-  madd_2 =      "70000000ST",
-  maddu_2 =     "70000001ST",
-  mul_3 =       "70000002DST",
-  msub_2 =      "70000004ST",
-  msubu_2 =     "70000005ST",
-  clz_2 =       "70000020DS=",
-  clo_2 =       "70000021DS=",
-  sdbbp_0 =     "7000003f",
-  sdbbp_1 =     "7000003fY",
-
   -- Opcode SPECIAL3.
   ext_4 =       "7c000000TSAM", -- Note: last arg is msbd = size-1
+  dextm_4 =     mips64 and "7c000001TSAM", -- Args: pos    | size-1-32
+  dextu_4 =     mips64 and "7c000002TSAM", -- Args: pos-32 | size-1
+  dext_4 =      mips64 and "7c000003TSAM", -- Args: pos    | size-1
+  zextw_2 =     mips64 and "7c00f803TS",
   ins_4 =       "7c000004TSAM", -- Note: last arg is msb = pos+size-1
+  dinsm_4 =     mips64 and "7c000005TSAM", -- Args: pos    | pos+size-33
+  dinsu_4 =     mips64 and "7c000006TSAM", -- Args: pos-32 | pos+size-33
+  dins_4 =      mips64 and "7c000007TSAM", -- Args: pos    | pos+size-1
   wsbh_2 =      "7c0000a0DT",
+  dsbh_2 =      mips64 and "7c0000a4DT",
+  dshd_2 =      mips64 and "7c000164DT",
   seb_2 =       "7c000420DT",
   seh_2 =       "7c000620DT",
   rdhwr_2 =     "7c00003bTD",
@@ -370,8 +355,12 @@ local map_op = {
   -- Opcode COP0.
   mfc0_2 =      "40000000TD",
   mfc0_3 =      "40000000TDW",
+  dmfc0_2 =     mips64 and "40200000TD",
+  dmfc0_3 =     mips64 and "40200000TDW",
   mtc0_2 =      "40800000TD",
   mtc0_3 =      "40800000TDW",
+  dmtc0_2 =     mips64 and "40a00000TD",
+  dmtc0_3 =     mips64 and "40a00000TDW",
   rdpgpr_2 =    "41400000DT",
   di_0 =        "41606000",
   di_1 =        "41606000T",
@@ -388,21 +377,14 @@ local map_op = {
 
   -- Opcode COP1.
   mfc1_2 =      "44000000TG",
+  dmfc1_2 =     mips64 and "44200000TG",
   cfc1_2 =      "44400000TG",
   mfhc1_2 =     "44600000TG",
   mtc1_2 =      "44800000TG",
+  dmtc1_2 =     mips64 and "44a00000TG",
   ctc1_2 =      "44c00000TG",
   mthc1_2 =     "44e00000TG",
 
-  bc1f_1 =      "45000000B",
-  bc1f_2 =      "45000000CB",
-  bc1t_1 =      "45010000B",
-  bc1t_2 =      "45010000CB",
-  bc1fl_1 =     "45020000B",
-  bc1fl_2 =     "45020000CB",
-  bc1tl_1 =     "45030000B",
-  bc1tl_2 =     "45030000CB",
-
   ["add.s_3"] =         "46000000FGH",
   ["sub.s_3"] =         "46000001FGH",
   ["mul.s_3"] =         "46000002FGH",
@@ -419,51 +401,11 @@ local map_op = {
   ["trunc.w.s_2"] =     "4600000dFG",
   ["ceil.w.s_2"] =      "4600000eFG",
   ["floor.w.s_2"] =     "4600000fFG",
-  ["movf.s_2"] =        "46000011FG",
-  ["movf.s_3"] =        "46000011FGC",
-  ["movt.s_2"] =        "46010011FG",
-  ["movt.s_3"] =        "46010011FGC",
-  ["movz.s_3"] =        "46000012FGT",
-  ["movn.s_3"] =        "46000013FGT",
   ["recip.s_2"] =       "46000015FG",
   ["rsqrt.s_2"] =       "46000016FG",
   ["cvt.d.s_2"] =       "46000021FG",
   ["cvt.w.s_2"] =       "46000024FG",
   ["cvt.l.s_2"] =       "46000025FG",
-  ["cvt.ps.s_3"] =      "46000026FGH",
-  ["c.f.s_2"] =         "46000030GH",
-  ["c.f.s_3"] =         "46000030VGH",
-  ["c.un.s_2"] =        "46000031GH",
-  ["c.un.s_3"] =        "46000031VGH",
-  ["c.eq.s_2"] =        "46000032GH",
-  ["c.eq.s_3"] =        "46000032VGH",
-  ["c.ueq.s_2"] =       "46000033GH",
-  ["c.ueq.s_3"] =       "46000033VGH",
-  ["c.olt.s_2"] =       "46000034GH",
-  ["c.olt.s_3"] =       "46000034VGH",
-  ["c.ult.s_2"] =       "46000035GH",
-  ["c.ult.s_3"] =       "46000035VGH",
-  ["c.ole.s_2"] =       "46000036GH",
-  ["c.ole.s_3"] =       "46000036VGH",
-  ["c.ule.s_2"] =       "46000037GH",
-  ["c.ule.s_3"] =       "46000037VGH",
-  ["c.sf.s_2"] =        "46000038GH",
-  ["c.sf.s_3"] =        "46000038VGH",
-  ["c.ngle.s_2"] =      "46000039GH",
-  ["c.ngle.s_3"] =      "46000039VGH",
-  ["c.seq.s_2"] =       "4600003aGH",
-  ["c.seq.s_3"] =       "4600003aVGH",
-  ["c.ngl.s_2"] =       "4600003bGH",
-  ["c.ngl.s_3"] =       "4600003bVGH",
-  ["c.lt.s_2"] =        "4600003cGH",
-  ["c.lt.s_3"] =        "4600003cVGH",
-  ["c.nge.s_2"] =       "4600003dGH",
-  ["c.nge.s_3"] =       "4600003dVGH",
-  ["c.le.s_2"] =        "4600003eGH",
-  ["c.le.s_3"] =        "4600003eVGH",
-  ["c.ngt.s_2"] =       "4600003fGH",
-  ["c.ngt.s_3"] =       "4600003fVGH",
-
   ["add.d_3"] =         "46200000FGH",
   ["sub.d_3"] =         "46200001FGH",
   ["mul.d_3"] =         "46200002FGH",
@@ -480,130 +422,410 @@ local map_op = {
   ["trunc.w.d_2"] =     "4620000dFG",
   ["ceil.w.d_2"] =      "4620000eFG",
   ["floor.w.d_2"] =     "4620000fFG",
-  ["movf.d_2"] =        "46200011FG",
-  ["movf.d_3"] =        "46200011FGC",
-  ["movt.d_2"] =        "46210011FG",
-  ["movt.d_3"] =        "46210011FGC",
-  ["movz.d_3"] =        "46200012FGT",
-  ["movn.d_3"] =        "46200013FGT",
   ["recip.d_2"] =       "46200015FG",
   ["rsqrt.d_2"] =       "46200016FG",
   ["cvt.s.d_2"] =       "46200020FG",
   ["cvt.w.d_2"] =       "46200024FG",
   ["cvt.l.d_2"] =       "46200025FG",
-  ["c.f.d_2"] =         "46200030GH",
-  ["c.f.d_3"] =         "46200030VGH",
-  ["c.un.d_2"] =        "46200031GH",
-  ["c.un.d_3"] =        "46200031VGH",
-  ["c.eq.d_2"] =        "46200032GH",
-  ["c.eq.d_3"] =        "46200032VGH",
-  ["c.ueq.d_2"] =       "46200033GH",
-  ["c.ueq.d_3"] =       "46200033VGH",
-  ["c.olt.d_2"] =       "46200034GH",
-  ["c.olt.d_3"] =       "46200034VGH",
-  ["c.ult.d_2"] =       "46200035GH",
-  ["c.ult.d_3"] =       "46200035VGH",
-  ["c.ole.d_2"] =       "46200036GH",
-  ["c.ole.d_3"] =       "46200036VGH",
-  ["c.ule.d_2"] =       "46200037GH",
-  ["c.ule.d_3"] =       "46200037VGH",
-  ["c.sf.d_2"] =        "46200038GH",
-  ["c.sf.d_3"] =        "46200038VGH",
-  ["c.ngle.d_2"] =      "46200039GH",
-  ["c.ngle.d_3"] =      "46200039VGH",
-  ["c.seq.d_2"] =       "4620003aGH",
-  ["c.seq.d_3"] =       "4620003aVGH",
-  ["c.ngl.d_2"] =       "4620003bGH",
-  ["c.ngl.d_3"] =       "4620003bVGH",
-  ["c.lt.d_2"] =        "4620003cGH",
-  ["c.lt.d_3"] =        "4620003cVGH",
-  ["c.nge.d_2"] =       "4620003dGH",
-  ["c.nge.d_3"] =       "4620003dVGH",
-  ["c.le.d_2"] =        "4620003eGH",
-  ["c.le.d_3"] =        "4620003eVGH",
-  ["c.ngt.d_2"] =       "4620003fGH",
-  ["c.ngt.d_3"] =       "4620003fVGH",
-
-  ["add.ps_3"] =        "46c00000FGH",
-  ["sub.ps_3"] =        "46c00001FGH",
-  ["mul.ps_3"] =        "46c00002FGH",
-  ["abs.ps_2"] =        "46c00005FG",
-  ["mov.ps_2"] =        "46c00006FG",
-  ["neg.ps_2"] =        "46c00007FG",
-  ["movf.ps_2"] =       "46c00011FG",
-  ["movf.ps_3"] =       "46c00011FGC",
-  ["movt.ps_2"] =       "46c10011FG",
-  ["movt.ps_3"] =       "46c10011FGC",
-  ["movz.ps_3"] =       "46c00012FGT",
-  ["movn.ps_3"] =       "46c00013FGT",
-  ["cvt.s.pu_2"] =      "46c00020FG",
-  ["cvt.s.pl_2"] =      "46c00028FG",
-  ["pll.ps_3"] =        "46c0002cFGH",
-  ["plu.ps_3"] =        "46c0002dFGH",
-  ["pul.ps_3"] =        "46c0002eFGH",
-  ["puu.ps_3"] =        "46c0002fFGH",
-  ["c.f.ps_2"] =        "46c00030GH",
-  ["c.f.ps_3"] =        "46c00030VGH",
-  ["c.un.ps_2"] =       "46c00031GH",
-  ["c.un.ps_3"] =       "46c00031VGH",
-  ["c.eq.ps_2"] =       "46c00032GH",
-  ["c.eq.ps_3"] =       "46c00032VGH",
-  ["c.ueq.ps_2"] =      "46c00033GH",
-  ["c.ueq.ps_3"] =      "46c00033VGH",
-  ["c.olt.ps_2"] =      "46c00034GH",
-  ["c.olt.ps_3"] =      "46c00034VGH",
-  ["c.ult.ps_2"] =      "46c00035GH",
-  ["c.ult.ps_3"] =      "46c00035VGH",
-  ["c.ole.ps_2"] =      "46c00036GH",
-  ["c.ole.ps_3"] =      "46c00036VGH",
-  ["c.ule.ps_2"] =      "46c00037GH",
-  ["c.ule.ps_3"] =      "46c00037VGH",
-  ["c.sf.ps_2"] =       "46c00038GH",
-  ["c.sf.ps_3"] =       "46c00038VGH",
-  ["c.ngle.ps_2"] =     "46c00039GH",
-  ["c.ngle.ps_3"] =     "46c00039VGH",
-  ["c.seq.ps_2"] =      "46c0003aGH",
-  ["c.seq.ps_3"] =      "46c0003aVGH",
-  ["c.ngl.ps_2"] =      "46c0003bGH",
-  ["c.ngl.ps_3"] =      "46c0003bVGH",
-  ["c.lt.ps_2"] =       "46c0003cGH",
-  ["c.lt.ps_3"] =       "46c0003cVGH",
-  ["c.nge.ps_2"] =      "46c0003dGH",
-  ["c.nge.ps_3"] =      "46c0003dVGH",
-  ["c.le.ps_2"] =       "46c0003eGH",
-  ["c.le.ps_3"] =       "46c0003eVGH",
-  ["c.ngt.ps_2"] =      "46c0003fGH",
-  ["c.ngt.ps_3"] =      "46c0003fVGH",
-
   ["cvt.s.w_2"] =       "46800020FG",
   ["cvt.d.w_2"] =       "46800021FG",
-
   ["cvt.s.l_2"] =       "46a00020FG",
   ["cvt.d.l_2"] =       "46a00021FG",
-
-  -- Opcode COP1X.
-  lwxc1_2 =             "4c000000FX",
-  ldxc1_2 =             "4c000001FX",
-  luxc1_2 =             "4c000005FX",
-  swxc1_2 =             "4c000008FX",
-  sdxc1_2 =             "4c000009FX",
-  suxc1_2 =             "4c00000dFX",
-  prefx_2 =             "4c00000fMX",
-  ["alnv.ps_4"] =       "4c00001eFGHS",
-  ["madd.s_4"] =        "4c000020FRGH",
-  ["madd.d_4"] =        "4c000021FRGH",
-  ["madd.ps_4"] =       "4c000026FRGH",
-  ["msub.s_4"] =        "4c000028FRGH",
-  ["msub.d_4"] =        "4c000029FRGH",
-  ["msub.ps_4"] =       "4c00002eFRGH",
-  ["nmadd.s_4"] =       "4c000030FRGH",
-  ["nmadd.d_4"] =       "4c000031FRGH",
-  ["nmadd.ps_4"] =      "4c000036FRGH",
-  ["nmsub.s_4"] =       "4c000038FRGH",
-  ["nmsub.d_4"] =       "4c000039FRGH",
-  ["nmsub.ps_4"] =      "4c00003eFRGH",
 }
 
+if mipsr6 then -- Instructions added with MIPSR6.
+
+  for k,v in pairs({
+
+    -- Add immediate to upper bits.
+    aui_3 =     "3c000000TSI",
+    daui_3 =    mips64 and "74000000TSI",
+    dahi_2 =    mips64 and "04060000SI",
+    dati_2 =    mips64 and "041e0000SI",
+
+    -- TODO: addiupc, auipc, aluipc, lwpc, lwupc, ldpc.
+
+    -- Compact branches.
+    blezalc_2 = "18000000TB",   -- rt != 0.
+    bgezalc_2 = "18000000T=SB", -- rt != 0.
+    bgtzalc_2 = "1c000000TB",   -- rt != 0.
+    bltzalc_2 = "1c000000T=SB", -- rt != 0.
+
+    blezc_2 =   "58000000TB",   -- rt != 0.
+    bgezc_2 =   "58000000T=SB", -- rt != 0.
+    bgec_3 =    "58000000STB",  -- rs != rt.
+    blec_3 =    "58000000TSB",  -- rt != rs.
+
+    bgtzc_2 =   "5c000000TB",   -- rt != 0.
+    bltzc_2 =   "5c000000T=SB", -- rt != 0.
+    bltc_3 =    "5c000000STB",  -- rs != rt.
+    bgtc_3 =    "5c000000TSB",  -- rt != rs.
+
+    bgeuc_3 =   "18000000STB",  -- rs != rt.
+    bleuc_3 =   "18000000TSB",  -- rt != rs.
+    bltuc_3 =   "1c000000STB",  -- rs != rt.
+    bgtuc_3 =   "1c000000TSB",  -- rt != rs.
+
+    beqzalc_2 = "20000000TB",   -- rt != 0.
+    bnezalc_2 = "60000000TB",   -- rt != 0.
+    beqc_3 =    "20000000STB",  -- rs < rt.
+    bnec_3 =    "60000000STB",  -- rs < rt.
+    bovc_3 =    "20000000STB",  -- rs >= rt.
+    bnvc_3 =    "60000000STB",  -- rs >= rt.
+
+    beqzc_2 =   "d8000000SK",   -- rs != 0.
+    bnezc_2 =   "f8000000SK",   -- rs != 0.
+    jic_2 =     "d8000000TI",
+    jialc_2 =   "f8000000TI",
+    bc_1 =      "c8000000L",
+    balc_1 =    "e8000000L",
+
+    -- Opcode SPECIAL.
+    jr_1 =      "00000009S",
+    sdbbp_0 =   "0000000e",
+    sdbbp_1 =   "0000000eY",
+    lsa_4 =     "00000005DSTA",
+    dlsa_4 =    mips64 and "00000015DSTA",
+    seleqz_3 =  "00000035DST",
+    selnez_3 =  "00000037DST",
+    clz_2 =     "00000050DS",
+    clo_2 =     "00000051DS",
+    dclz_2 =    mips64 and "00000052DS",
+    dclo_2 =    mips64 and "00000053DS",
+    mul_3 =     "00000098DST",
+    muh_3 =     "000000d8DST",
+    mulu_3 =    "00000099DST",
+    muhu_3 =    "000000d9DST",
+    div_3 =     "0000009aDST",
+    mod_3 =     "000000daDST",
+    divu_3 =    "0000009bDST",
+    modu_3 =    "000000dbDST",
+    dmul_3 =    mips64 and "0000009cDST",
+    dmuh_3 =    mips64 and "000000dcDST",
+    dmulu_3 =   mips64 and "0000009dDST",
+    dmuhu_3 =   mips64 and "000000ddDST",
+    ddiv_3 =    mips64 and "0000009eDST",
+    dmod_3 =    mips64 and "000000deDST",
+    ddivu_3 =   mips64 and "0000009fDST",
+    dmodu_3 =   mips64 and "000000dfDST",
+
+    -- Opcode SPECIAL3.
+    align_4 =           "7c000220DSTA",
+    dalign_4 =          mips64 and "7c000224DSTA",
+    bitswap_2 =         "7c000020DT",
+    dbitswap_2 =        mips64 and "7c000024DT",
+
+    -- Opcode COP1.
+    bc1eqz_2 =  "45200000HB",
+    bc1nez_2 =  "45a00000HB",
+
+    ["sel.s_3"] =       "46000010FGH",
+    ["seleqz.s_3"] =    "46000014FGH",
+    ["selnez.s_3"] =    "46000017FGH",
+    ["maddf.s_3"] =     "46000018FGH",
+    ["msubf.s_3"] =     "46000019FGH",
+    ["rint.s_2"] =      "4600001aFG",
+    ["class.s_2"] =     "4600001bFG",
+    ["min.s_3"] =       "4600001cFGH",
+    ["mina.s_3"] =      "4600001dFGH",
+    ["max.s_3"] =       "4600001eFGH",
+    ["maxa.s_3"] =      "4600001fFGH",
+    ["cmp.af.s_3"] =    "46800000FGH",
+    ["cmp.un.s_3"] =    "46800001FGH",
+    ["cmp.or.s_3"] =    "46800011FGH",
+    ["cmp.eq.s_3"] =    "46800002FGH",
+    ["cmp.une.s_3"] =   "46800012FGH",
+    ["cmp.ueq.s_3"] =   "46800003FGH",
+    ["cmp.ne.s_3"] =    "46800013FGH",
+    ["cmp.lt.s_3"] =    "46800004FGH",
+    ["cmp.ult.s_3"] =   "46800005FGH",
+    ["cmp.le.s_3"] =    "46800006FGH",
+    ["cmp.ule.s_3"] =   "46800007FGH",
+    ["cmp.saf.s_3"] =   "46800008FGH",
+    ["cmp.sun.s_3"] =   "46800009FGH",
+    ["cmp.sor.s_3"] =   "46800019FGH",
+    ["cmp.seq.s_3"] =   "4680000aFGH",
+    ["cmp.sune.s_3"] =  "4680001aFGH",
+    ["cmp.sueq.s_3"] =  "4680000bFGH",
+    ["cmp.sne.s_3"] =   "4680001bFGH",
+    ["cmp.slt.s_3"] =   "4680000cFGH",
+    ["cmp.sult.s_3"] =  "4680000dFGH",
+    ["cmp.sle.s_3"] =   "4680000eFGH",
+    ["cmp.sule.s_3"] =  "4680000fFGH",
+
+    ["sel.d_3"] =       "46200010FGH",
+    ["seleqz.d_3"] =    "46200014FGH",
+    ["selnez.d_3"] =    "46200017FGH",
+    ["maddf.d_3"] =     "46200018FGH",
+    ["msubf.d_3"] =     "46200019FGH",
+    ["rint.d_2"] =      "4620001aFG",
+    ["class.d_2"] =     "4620001bFG",
+    ["min.d_3"] =       "4620001cFGH",
+    ["mina.d_3"] =      "4620001dFGH",
+    ["max.d_3"] =       "4620001eFGH",
+    ["maxa.d_3"] =      "4620001fFGH",
+    ["cmp.af.d_3"] =    "46a00000FGH",
+    ["cmp.un.d_3"] =    "46a00001FGH",
+    ["cmp.or.d_3"] =    "46a00011FGH",
+    ["cmp.eq.d_3"] =    "46a00002FGH",
+    ["cmp.une.d_3"] =   "46a00012FGH",
+    ["cmp.ueq.d_3"] =   "46a00003FGH",
+    ["cmp.ne.d_3"] =    "46a00013FGH",
+    ["cmp.lt.d_3"] =    "46a00004FGH",
+    ["cmp.ult.d_3"] =   "46a00005FGH",
+    ["cmp.le.d_3"] =    "46a00006FGH",
+    ["cmp.ule.d_3"] =   "46a00007FGH",
+    ["cmp.saf.d_3"] =   "46a00008FGH",
+    ["cmp.sun.d_3"] =   "46a00009FGH",
+    ["cmp.sor.d_3"] =   "46a00019FGH",
+    ["cmp.seq.d_3"] =   "46a0000aFGH",
+    ["cmp.sune.d_3"] =  "46a0001aFGH",
+    ["cmp.sueq.d_3"] =  "46a0000bFGH",
+    ["cmp.sne.d_3"] =   "46a0001bFGH",
+    ["cmp.slt.d_3"] =   "46a0000cFGH",
+    ["cmp.sult.d_3"] =  "46a0000dFGH",
+    ["cmp.sle.d_3"] =   "46a0000eFGH",
+    ["cmp.sule.d_3"] =  "46a0000fFGH",
+
+  }) do map_op[k] = v end
+
+else -- Instructions removed by MIPSR6.
+
+  for k,v in pairs({
+    -- Traps, don't use.
+    addi_3 =    "20000000TSI",
+    daddi_3 =   mips64 and "60000000TSI",
+
+    -- Branch on likely, don't use.
+    beqzl_2 =   "50000000SB",
+    beql_3 =    "50000000STB",
+    bnezl_2 =   "54000000SB",
+    bnel_3 =    "54000000STB",
+    blezl_2 =   "58000000SB",
+    bgtzl_2 =   "5c000000SB",
+
+    lwl_2 =     "88000000TO",
+    lwr_2 =     "98000000TO",
+    swl_2 =     "a8000000TO",
+    sdl_2 =     mips64 and "b0000000TO",
+    sdr_2 =     mips64 and "b1000000TO",
+    swr_2 =     "b8000000TO",
+    cache_2 =   "bc000000NO",
+    ll_2 =      "c0000000TO",
+    pref_2 =    "cc000000NO",
+    sc_2 =      "e0000000TO",
+    scd_2 =     mips64 and "f0000000TO",
+
+    -- Opcode SPECIAL.
+    movf_2 =    "00000001DS",
+    movf_3 =    "00000001DSC",
+    movt_2 =    "00010001DS",
+    movt_3 =    "00010001DSC",
+    jr_1 =      "00000008S",
+    movz_3 =    "0000000aDST",
+    movn_3 =    "0000000bDST",
+    mfhi_1 =    "00000010D",
+    mthi_1 =    "00000011S",
+    mflo_1 =    "00000012D",
+    mtlo_1 =    "00000013S",
+    mult_2 =    "00000018ST",
+    multu_2 =   "00000019ST",
+    div_3 =     "0000001aST",
+    divu_3 =    "0000001bST",
+    ddiv_3 =    mips64 and "0000001eST",
+    ddivu_3 =   mips64 and "0000001fST",
+    dmult_2 =   mips64 and "0000001cST",
+    dmultu_2 =  mips64 and "0000001dST",
+
+    -- Opcode REGIMM.
+    tgei_2 =    "04080000SI",
+    tgeiu_2 =   "04090000SI",
+    tlti_2 =    "040a0000SI",
+    tltiu_2 =   "040b0000SI",
+    teqi_2 =    "040c0000SI",
+    tnei_2 =    "040e0000SI",
+    bltzal_2 =  "04100000SB",
+    bgezal_2 =  "04110000SB",
+    bltzall_2 = "04120000SB",
+    bgezall_2 = "04130000SB",
+
+    -- Opcode SPECIAL2.
+    madd_2 =    "70000000ST",
+    maddu_2 =   "70000001ST",
+    mul_3 =     "70000002DST",
+    msub_2 =    "70000004ST",
+    msubu_2 =   "70000005ST",
+    clz_2 =     "70000020D=TS",
+    clo_2 =     "70000021D=TS",
+    dclz_2 =    mips64 and "70000024D=TS",
+    dclo_2 =    mips64 and "70000025D=TS",
+    sdbbp_0 =   "7000003f",
+    sdbbp_1 =   "7000003fY",
+
+    -- Opcode COP1.
+    bc1f_1 =    "45000000B",
+    bc1f_2 =    "45000000CB",
+    bc1t_1 =    "45010000B",
+    bc1t_2 =    "45010000CB",
+    bc1fl_1 =   "45020000B",
+    bc1fl_2 =   "45020000CB",
+    bc1tl_1 =   "45030000B",
+    bc1tl_2 =   "45030000CB",
+
+    ["movf.s_2"] =      "46000011FG",
+    ["movf.s_3"] =      "46000011FGC",
+    ["movt.s_2"] =      "46010011FG",
+    ["movt.s_3"] =      "46010011FGC",
+    ["movz.s_3"] =      "46000012FGT",
+    ["movn.s_3"] =      "46000013FGT",
+    ["cvt.ps.s_3"] =    "46000026FGH",
+    ["c.f.s_2"] =       "46000030GH",
+    ["c.f.s_3"] =       "46000030VGH",
+    ["c.un.s_2"] =      "46000031GH",
+    ["c.un.s_3"] =      "46000031VGH",
+    ["c.eq.s_2"] =      "46000032GH",
+    ["c.eq.s_3"] =      "46000032VGH",
+    ["c.ueq.s_2"] =     "46000033GH",
+    ["c.ueq.s_3"] =     "46000033VGH",
+    ["c.olt.s_2"] =     "46000034GH",
+    ["c.olt.s_3"] =     "46000034VGH",
+    ["c.ult.s_2"] =     "46000035GH",
+    ["c.ult.s_3"] =     "46000035VGH",
+    ["c.ole.s_2"] =     "46000036GH",
+    ["c.ole.s_3"] =     "46000036VGH",
+    ["c.ule.s_2"] =     "46000037GH",
+    ["c.ule.s_3"] =     "46000037VGH",
+    ["c.sf.s_2"] =      "46000038GH",
+    ["c.sf.s_3"] =      "46000038VGH",
+    ["c.ngle.s_2"] =    "46000039GH",
+    ["c.ngle.s_3"] =    "46000039VGH",
+    ["c.seq.s_2"] =     "4600003aGH",
+    ["c.seq.s_3"] =     "4600003aVGH",
+    ["c.ngl.s_2"] =     "4600003bGH",
+    ["c.ngl.s_3"] =     "4600003bVGH",
+    ["c.lt.s_2"] =      "4600003cGH",
+    ["c.lt.s_3"] =      "4600003cVGH",
+    ["c.nge.s_2"] =     "4600003dGH",
+    ["c.nge.s_3"] =     "4600003dVGH",
+    ["c.le.s_2"] =      "4600003eGH",
+    ["c.le.s_3"] =      "4600003eVGH",
+    ["c.ngt.s_2"] =     "4600003fGH",
+    ["c.ngt.s_3"] =     "4600003fVGH",
+    ["movf.d_2"] =      "46200011FG",
+    ["movf.d_3"] =      "46200011FGC",
+    ["movt.d_2"] =      "46210011FG",
+    ["movt.d_3"] =      "46210011FGC",
+    ["movz.d_3"] =      "46200012FGT",
+    ["movn.d_3"] =      "46200013FGT",
+    ["c.f.d_2"] =       "46200030GH",
+    ["c.f.d_3"] =       "46200030VGH",
+    ["c.un.d_2"] =      "46200031GH",
+    ["c.un.d_3"] =      "46200031VGH",
+    ["c.eq.d_2"] =      "46200032GH",
+    ["c.eq.d_3"] =      "46200032VGH",
+    ["c.ueq.d_2"] =     "46200033GH",
+    ["c.ueq.d_3"] =     "46200033VGH",
+    ["c.olt.d_2"] =     "46200034GH",
+    ["c.olt.d_3"] =     "46200034VGH",
+    ["c.ult.d_2"] =     "46200035GH",
+    ["c.ult.d_3"] =     "46200035VGH",
+    ["c.ole.d_2"] =     "46200036GH",
+    ["c.ole.d_3"] =     "46200036VGH",
+    ["c.ule.d_2"] =     "46200037GH",
+    ["c.ule.d_3"] =     "46200037VGH",
+    ["c.sf.d_2"] =      "46200038GH",
+    ["c.sf.d_3"] =      "46200038VGH",
+    ["c.ngle.d_2"] =    "46200039GH",
+    ["c.ngle.d_3"] =    "46200039VGH",
+    ["c.seq.d_2"] =     "4620003aGH",
+    ["c.seq.d_3"] =     "4620003aVGH",
+    ["c.ngl.d_2"] =     "4620003bGH",
+    ["c.ngl.d_3"] =     "4620003bVGH",
+    ["c.lt.d_2"] =      "4620003cGH",
+    ["c.lt.d_3"] =      "4620003cVGH",
+    ["c.nge.d_2"] =     "4620003dGH",
+    ["c.nge.d_3"] =     "4620003dVGH",
+    ["c.le.d_2"] =      "4620003eGH",
+    ["c.le.d_3"] =      "4620003eVGH",
+    ["c.ngt.d_2"] =     "4620003fGH",
+    ["c.ngt.d_3"] =     "4620003fVGH",
+    ["add.ps_3"] =      "46c00000FGH",
+    ["sub.ps_3"] =      "46c00001FGH",
+    ["mul.ps_3"] =      "46c00002FGH",
+    ["abs.ps_2"] =      "46c00005FG",
+    ["mov.ps_2"] =      "46c00006FG",
+    ["neg.ps_2"] =      "46c00007FG",
+    ["movf.ps_2"] =     "46c00011FG",
+    ["movf.ps_3"] =     "46c00011FGC",
+    ["movt.ps_2"] =     "46c10011FG",
+    ["movt.ps_3"] =     "46c10011FGC",
+    ["movz.ps_3"] =     "46c00012FGT",
+    ["movn.ps_3"] =     "46c00013FGT",
+    ["cvt.s.pu_2"] =    "46c00020FG",
+    ["cvt.s.pl_2"] =    "46c00028FG",
+    ["pll.ps_3"] =      "46c0002cFGH",
+    ["plu.ps_3"] =      "46c0002dFGH",
+    ["pul.ps_3"] =      "46c0002eFGH",
+    ["puu.ps_3"] =      "46c0002fFGH",
+    ["c.f.ps_2"] =      "46c00030GH",
+    ["c.f.ps_3"] =      "46c00030VGH",
+    ["c.un.ps_2"] =     "46c00031GH",
+    ["c.un.ps_3"] =     "46c00031VGH",
+    ["c.eq.ps_2"] =     "46c00032GH",
+    ["c.eq.ps_3"] =     "46c00032VGH",
+    ["c.ueq.ps_2"] =    "46c00033GH",
+    ["c.ueq.ps_3"] =    "46c00033VGH",
+    ["c.olt.ps_2"] =    "46c00034GH",
+    ["c.olt.ps_3"] =    "46c00034VGH",
+    ["c.ult.ps_2"] =    "46c00035GH",
+    ["c.ult.ps_3"] =    "46c00035VGH",
+    ["c.ole.ps_2"] =    "46c00036GH",
+    ["c.ole.ps_3"] =    "46c00036VGH",
+    ["c.ule.ps_2"] =    "46c00037GH",
+    ["c.ule.ps_3"] =    "46c00037VGH",
+    ["c.sf.ps_2"] =     "46c00038GH",
+    ["c.sf.ps_3"] =     "46c00038VGH",
+    ["c.ngle.ps_2"] =   "46c00039GH",
+    ["c.ngle.ps_3"] =   "46c00039VGH",
+    ["c.seq.ps_2"] =    "46c0003aGH",
+    ["c.seq.ps_3"] =    "46c0003aVGH",
+    ["c.ngl.ps_2"] =    "46c0003bGH",
+    ["c.ngl.ps_3"] =    "46c0003bVGH",
+    ["c.lt.ps_2"] =     "46c0003cGH",
+    ["c.lt.ps_3"] =     "46c0003cVGH",
+    ["c.nge.ps_2"] =    "46c0003dGH",
+    ["c.nge.ps_3"] =    "46c0003dVGH",
+    ["c.le.ps_2"] =     "46c0003eGH",
+    ["c.le.ps_3"] =     "46c0003eVGH",
+    ["c.ngt.ps_2"] =    "46c0003fGH",
+    ["c.ngt.ps_3"] =    "46c0003fVGH",
+
+    -- Opcode COP1X.
+    lwxc1_2 =   "4c000000FX",
+    ldxc1_2 =   "4c000001FX",
+    luxc1_2 =   "4c000005FX",
+    swxc1_2 =   "4c000008FX",
+    sdxc1_2 =   "4c000009FX",
+    suxc1_2 =   "4c00000dFX",
+    prefx_2 =   "4c00000fMX",
+    ["alnv.ps_4"] =     "4c00001eFGHS",
+    ["madd.s_4"] =      "4c000020FRGH",
+    ["madd.d_4"] =      "4c000021FRGH",
+    ["madd.ps_4"] =     "4c000026FRGH",
+    ["msub.s_4"] =      "4c000028FRGH",
+    ["msub.d_4"] =      "4c000029FRGH",
+    ["msub.ps_4"] =     "4c00002eFRGH",
+    ["nmadd.s_4"] =     "4c000030FRGH",
+    ["nmadd.d_4"] =     "4c000031FRGH",
+    ["nmadd.ps_4"] =    "4c000036FRGH",
+    ["nmsub.s_4"] =     "4c000038FRGH",
+    ["nmsub.d_4"] =     "4c000039FRGH",
+    ["nmsub.ps_4"] =    "4c00003eFRGH",
+
+  }) do map_op[k] = v end
+
+end
+
 ------------------------------------------------------------------------------
 
 local function parse_gpr(expr)
@@ -633,7 +855,7 @@ local function parse_fpr(expr)
   werror("bad register name `"..expr.."'")
 end
 
-local function parse_imm(imm, bits, shift, scale, signed)
+local function parse_imm(imm, bits, shift, scale, signed, action)
   local n = tonumber(imm)
   if n then
     local m = sar(n, scale)
@@ -651,7 +873,8 @@ local function parse_imm(imm, bits, shift, scale, signed)
          match(imm, "^([%w_]+):([rf][1-3]?[0-9])$") then
     werror("expected immediate operand, got register")
   else
-    waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm)
+    waction(action or "IMM",
+            (signed and 32768 or 0)+shl(scale, 10)+shl(bits, 5)+shift, imm)
     return 0
   end
 end
@@ -756,13 +979,18 @@ map_op[".template__"] = function(params, template, nparams)
       op = op + parse_disp(params[n]); n = n + 1
     elseif p == "X" then
       op = op + parse_index(params[n]); n = n + 1
-    elseif p == "B" or p == "J" then
+    elseif p == "B" or p == "J" or p == "K" or p == "L" then
       local mode, m, s = parse_label(params[n], false)
-      if p == "B" then m = m + 2048 end
+      if p == "J" then m = m + 0xa800
+      elseif p == "K" then m = m + 0x5000
+      elseif p == "L" then m = m + 0xa000 end
       waction("REL_"..mode, m, s, 1)
       n = n + 1
     elseif p == "A" then
       op = op + parse_imm(params[n], 5, 6, 0, false); n = n + 1
+    elseif p == "a" then
+      local m = parse_imm(params[n], 6, 6, 0, false, "IMMS"); n = n + 1
+      op = op + band(m, 0x7c0) + band(shr(m, 9), 4)
     elseif p == "M" then
       op = op + parse_imm(params[n], 5, 11, 0, false); n = n + 1
     elseif p == "N" then
@@ -778,7 +1006,7 @@ map_op[".template__"] = function(params, template, nparams)
     elseif p == "Z" then
       op = op + parse_imm(params[n], 10, 6, 0, false); n = n + 1
     elseif p == "=" then
-      op = op + shl(band(op, 0xf800), 5) -- Copy D to T for clz, clo.
+      n = n - 1 -- Re-use previous parameter for next template char.
     else
       assert(false)
     end
diff --git a/dynasm/dasm_mips64.lua b/dynasm/dasm_mips64.lua
new file mode 100644
index 00000000..0aae291b
--- /dev/null
+++ b/dynasm/dasm_mips64.lua
@@ -0,0 +1,12 @@
+------------------------------------------------------------------------------
+-- DynASM MIPS64 module.
+--
+-- Copyright (C) 2005-2020 Mike Pall. All rights reserved.
+-- See dynasm.lua for full copyright notice.
+------------------------------------------------------------------------------
+-- This module just sets 64 bit mode for the combined MIPS/MIPS64 module.
+-- All the interesting stuff is there.
+------------------------------------------------------------------------------
+
+mips64 = true -- Using a global is an ugly, but effective solution.
+return require("dasm_mips")
diff --git a/dynasm/dasm_ppc.h b/dynasm/dasm_ppc.h
index 699d5c31..6e7cc7ab 100644
--- a/dynasm/dasm_ppc.h
+++ b/dynasm/dasm_ppc.h
@@ -1,5 +1,5 @@
 /*
-** DynASM PPC encoding engine.
+** DynASM PPC/PPC64 encoding engine.
 ** Copyright (C) 2005-2020 Mike Pall. All rights reserved.
 ** Released under the MIT license. See dynasm.lua for full copyright notice.
 */
@@ -21,7 +21,7 @@ enum {
   /* The following actions need a buffer position. */
   DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
   /* The following actions also have an argument. */
-  DASM_REL_PC, DASM_LABEL_PC, DASM_IMM,
+  DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMSH,
   DASM__MAX
 };
 
@@ -244,6 +244,10 @@ void dasm_put(Dst_DECL, int start, ...)
 #endif
         b[pos++] = n;
         break;
+      case DASM_IMMSH:
+        CK((n >> 6) == 0, RANGE_I);
+        b[pos++] = n;
+        break;
       }
     }
   }
@@ -299,7 +303,7 @@ int dasm_link(Dst_DECL, size_t *szp)
         case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
         case DASM_REL_LG: case DASM_REL_PC: pos++; break;
         case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
-        case DASM_IMM: pos++; break;
+        case DASM_IMM: case DASM_IMMSH: pos++; break;
         }
       }
       stop: (void)0;
@@ -367,6 +371,9 @@ int dasm_encode(Dst_DECL, void *buffer)
         case DASM_IMM:
           cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
           break;
+        case DASM_IMMSH:
+          cp[-1] |= (ins & 1) ? ((n&31)<<11)|((n&32)>>4) : ((n&31)<<6)|(n&32);
+          break;
         default: *cp++ = ins; break;
         }
       }
diff --git a/dynasm/dasm_ppc.lua b/dynasm/dasm_ppc.lua
index 05981760..20634e13 100644
--- a/dynasm/dasm_ppc.lua
+++ b/dynasm/dasm_ppc.lua
@@ -1,17 +1,19 @@
 ------------------------------------------------------------------------------
--- DynASM PPC module.
+-- DynASM PPC/PPC64 module.
 --
 -- Copyright (C) 2005-2020 Mike Pall. All rights reserved.
 -- See dynasm.lua for full copyright notice.
+--
+-- Support for various extensions contributed by Caio Souza Oliveira.
 ------------------------------------------------------------------------------
 
 -- Module information:
 local _info = {
   arch =        "ppc",
   description = "DynASM PPC module",
-  version =     "1.3.0",
-  vernum =       10300,
-  release =     "2011-05-05",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2015-10-18",
   author =      "Mike Pall",
   license =     "MIT",
 }
@@ -39,7 +41,7 @@ local wline, werror, wfatal, wwarn
 local action_names = {
   "STOP", "SECTION", "ESC", "REL_EXT",
   "ALIGN", "REL_LG", "LABEL_LG",
-  "REL_PC", "LABEL_PC", "IMM",
+  "REL_PC", "LABEL_PC", "IMM", "IMMSH"
 }
 
 -- Maximum number of section buffer positions for dasm_put().
@@ -228,8 +230,18 @@ local map_cond = {
 
 ------------------------------------------------------------------------------
 
+local map_op, op_template
+
+local function op_alias(opname, f)
+  return function(params, nparams)
+    if not params then return "-> "..opname:sub(1, -3) end
+    f(params, nparams)
+    op_template(params, map_op[opname], nparams)
+  end
+end
+
 -- Template strings for PPC instructions.
-local map_op = {
+map_op = {
   tdi_3 =       "08000000ARI",
   twi_3 =       "0c000000ARI",
   mulli_3 =     "1c000000RRI",
@@ -297,6 +309,250 @@ local map_op = {
   std_2 =       "f8000000RD",
   stdu_2 =      "f8000001RD",
 
+  subi_3 =      op_alias("addi_3", function(p) p[3] = "-("..p[3]..")" end),
+  subis_3 =     op_alias("addis_3", function(p) p[3] = "-("..p[3]..")" end),
+  subic_3 =     op_alias("addic_3", function(p) p[3] = "-("..p[3]..")" end),
+  ["subic._3"] = op_alias("addic._3", function(p) p[3] = "-("..p[3]..")" end),
+
+  rotlwi_3 =    op_alias("rlwinm_5", function(p)
+    p[4] = "0"; p[5] = "31"
+  end),
+  rotrwi_3 =    op_alias("rlwinm_5", function(p)
+    p[3] = "32-("..p[3]..")"; p[4] = "0"; p[5] = "31"
+  end),
+  rotlw_3 =     op_alias("rlwnm_5", function(p)
+    p[4] = "0"; p[5] = "31"
+  end),
+  slwi_3 =      op_alias("rlwinm_5", function(p)
+    p[5] = "31-("..p[3]..")"; p[4] = "0"
+  end),
+  srwi_3 =      op_alias("rlwinm_5", function(p)
+    p[4] = p[3]; p[3] = "32-("..p[3]..")"; p[5] = "31"
+  end),
+  clrlwi_3 =    op_alias("rlwinm_5", function(p)
+    p[4] = p[3]; p[3] = "0"; p[5] = "31"
+  end),
+  clrrwi_3 =    op_alias("rlwinm_5", function(p)
+    p[5] = "31-("..p[3]..")"; p[3] = "0"; p[4] = "0"
+  end),
+
+  -- Primary opcode 4:
+  mulhhwu_3 =           "10000010RRR.",
+  machhwu_3 =           "10000018RRR.",
+  mulhhw_3 =            "10000050RRR.",
+  nmachhw_3 =           "1000005cRRR.",
+  machhwsu_3 =          "10000098RRR.",
+  machhws_3 =           "100000d8RRR.",
+  nmachhws_3 =          "100000dcRRR.",
+  mulchwu_3 =           "10000110RRR.",
+  macchwu_3 =           "10000118RRR.",
+  mulchw_3 =            "10000150RRR.",
+  macchw_3 =            "10000158RRR.",
+  nmacchw_3 =           "1000015cRRR.",
+  macchwsu_3 =          "10000198RRR.",
+  macchws_3 =           "100001d8RRR.",
+  nmacchws_3 =          "100001dcRRR.",
+  mullhw_3 =            "10000350RRR.",
+  maclhw_3 =            "10000358RRR.",
+  nmaclhw_3 =           "1000035cRRR.",
+  maclhwsu_3 =          "10000398RRR.",
+  maclhws_3 =           "100003d8RRR.",
+  nmaclhws_3 =          "100003dcRRR.",
+  machhwuo_3 =          "10000418RRR.",
+  nmachhwo_3 =          "1000045cRRR.",
+  machhwsuo_3 =         "10000498RRR.",
+  machhwso_3 =          "100004d8RRR.",
+  nmachhwso_3 =         "100004dcRRR.",
+  macchwuo_3 =          "10000518RRR.",
+  macchwo_3 =           "10000558RRR.",
+  nmacchwo_3 =          "1000055cRRR.",
+  macchwsuo_3 =         "10000598RRR.",
+  macchwso_3 =          "100005d8RRR.",
+  nmacchwso_3 =         "100005dcRRR.",
+  maclhwo_3 =           "10000758RRR.",
+  nmaclhwo_3 =          "1000075cRRR.",
+  maclhwsuo_3 =         "10000798RRR.",
+  maclhwso_3 =          "100007d8RRR.",
+  nmaclhwso_3 =         "100007dcRRR.",
+
+  vaddubm_3 =           "10000000VVV",
+  vmaxub_3 =            "10000002VVV",
+  vrlb_3 =              "10000004VVV",
+  vcmpequb_3 =          "10000006VVV",
+  vmuloub_3 =           "10000008VVV",
+  vaddfp_3 =            "1000000aVVV",
+  vmrghb_3 =            "1000000cVVV",
+  vpkuhum_3 =           "1000000eVVV",
+  vmhaddshs_4 =         "10000020VVVV",
+  vmhraddshs_4 =        "10000021VVVV",
+  vmladduhm_4 =         "10000022VVVV",
+  vmsumubm_4 =          "10000024VVVV",
+  vmsummbm_4 =          "10000025VVVV",
+  vmsumuhm_4 =          "10000026VVVV",
+  vmsumuhs_4 =          "10000027VVVV",
+  vmsumshm_4 =          "10000028VVVV",
+  vmsumshs_4 =          "10000029VVVV",
+  vsel_4 =              "1000002aVVVV",
+  vperm_4 =             "1000002bVVVV",
+  vsldoi_4 =            "1000002cVVVP",
+  vpermxor_4 =          "1000002dVVVV",
+  vmaddfp_4 =           "1000002eVVVV~",
+  vnmsubfp_4 =          "1000002fVVVV~",
+  vaddeuqm_4 =          "1000003cVVVV",
+  vaddecuq_4 =          "1000003dVVVV",
+  vsubeuqm_4 =          "1000003eVVVV",
+  vsubecuq_4 =          "1000003fVVVV",
+  vadduhm_3 =           "10000040VVV",
+  vmaxuh_3 =            "10000042VVV",
+  vrlh_3 =              "10000044VVV",
+  vcmpequh_3 =          "10000046VVV",
+  vmulouh_3 =           "10000048VVV",
+  vsubfp_3 =            "1000004aVVV",
+  vmrghh_3 =            "1000004cVVV",
+  vpkuwum_3 =           "1000004eVVV",
+  vadduwm_3 =           "10000080VVV",
+  vmaxuw_3 =            "10000082VVV",
+  vrlw_3 =              "10000084VVV",
+  vcmpequw_3 =          "10000086VVV",
+  vmulouw_3 =           "10000088VVV",
+  vmuluwm_3 =           "10000089VVV",
+  vmrghw_3 =            "1000008cVVV",
+  vpkuhus_3 =           "1000008eVVV",
+  vaddudm_3 =           "100000c0VVV",
+  vmaxud_3 =            "100000c2VVV",
+  vrld_3 =              "100000c4VVV",
+  vcmpeqfp_3 =          "100000c6VVV",
+  vcmpequd_3 =          "100000c7VVV",
+  vpkuwus_3 =           "100000ceVVV",
+  vadduqm_3 =           "10000100VVV",
+  vmaxsb_3 =            "10000102VVV",
+  vslb_3 =              "10000104VVV",
+  vmulosb_3 =           "10000108VVV",
+  vrefp_2 =             "1000010aV-V",
+  vmrglb_3 =            "1000010cVVV",
+  vpkshus_3 =           "1000010eVVV",
+  vaddcuq_3 =           "10000140VVV",
+  vmaxsh_3 =            "10000142VVV",
+  vslh_3 =              "10000144VVV",
+  vmulosh_3 =           "10000148VVV",
+  vrsqrtefp_2 =         "1000014aV-V",
+  vmrglh_3 =            "1000014cVVV",
+  vpkswus_3 =           "1000014eVVV",
+  vaddcuw_3 =           "10000180VVV",
+  vmaxsw_3 =            "10000182VVV",
+  vslw_3 =              "10000184VVV",
+  vmulosw_3 =           "10000188VVV",
+  vexptefp_2 =          "1000018aV-V",
+  vmrglw_3 =            "1000018cVVV",
+  vpkshss_3 =           "1000018eVVV",
+  vmaxsd_3 =            "100001c2VVV",
+  vsl_3 =               "100001c4VVV",
+  vcmpgefp_3 =          "100001c6VVV",
+  vlogefp_2 =           "100001caV-V",
+  vpkswss_3 =           "100001ceVVV",
+  vadduhs_3 =           "10000240VVV",
+  vminuh_3 =            "10000242VVV",
+  vsrh_3 =              "10000244VVV",
+  vcmpgtuh_3 =          "10000246VVV",
+  vmuleuh_3 =           "10000248VVV",
+  vrfiz_2 =             "1000024aV-V",
+  vsplth_3 =            "1000024cVV3",
+  vupkhsh_2 =           "1000024eV-V",
+  vminuw_3 =            "10000282VVV",
+  vminud_3 =            "100002c2VVV",
+  vcmpgtud_3 =          "100002c7VVV",
+  vrfim_2 =             "100002caV-V",
+  vcmpgtsb_3 =          "10000306VVV",
+  vcfux_3 =             "1000030aVVA~",
+  vaddshs_3 =           "10000340VVV",
+  vminsh_3 =            "10000342VVV",
+  vsrah_3 =             "10000344VVV",
+  vcmpgtsh_3 =          "10000346VVV",
+  vmulesh_3 =           "10000348VVV",
+  vcfsx_3 =             "1000034aVVA~",
+  vspltish_2 =          "1000034cVS",
+  vupkhpx_2 =           "1000034eV-V",
+  vaddsws_3 =           "10000380VVV",
+  vminsw_3 =            "10000382VVV",
+  vsraw_3 =             "10000384VVV",
+  vcmpgtsw_3 =          "10000386VVV",
+  vmulesw_3 =           "10000388VVV",
+  vctuxs_3 =            "1000038aVVA~",
+  vspltisw_2 =          "1000038cVS",
+  vminsd_3 =            "100003c2VVV",
+  vsrad_3 =             "100003c4VVV",
+  vcmpbfp_3 =           "100003c6VVV",
+  vcmpgtsd_3 =          "100003c7VVV",
+  vctsxs_3 =            "100003caVVA~",
+  vupklpx_2 =           "100003ceV-V",
+  vsububm_3 =           "10000400VVV",
+  ["bcdadd._4"] =       "10000401VVVy.",
+  vavgub_3 =            "10000402VVV",
+  vand_3 =              "10000404VVV",
+  ["vcmpequb._3"] =     "10000406VVV",
+  vmaxfp_3 =            "1000040aVVV",
+  vsubuhm_3 =           "10000440VVV",
+  ["bcdsub._4"] =       "10000441VVVy.",
+  vavguh_3 =            "10000442VVV",
+  vandc_3 =             "10000444VVV",
+  ["vcmpequh._3"] =     "10000446VVV",
+  vminfp_3 =            "1000044aVVV",
+  vpkudum_3 =           "1000044eVVV",
+  vsubuwm_3 =           "10000480VVV",
+  vavguw_3 =            "10000482VVV",
+  vor_3 =               "10000484VVV",
+  ["vcmpequw._3"] =     "10000486VVV",
+  vpmsumw_3 =           "10000488VVV",
+  ["vcmpeqfp._3"] =     "100004c6VVV",
+  ["vcmpequd._3"] =     "100004c7VVV",
+  vpkudus_3 =           "100004ceVVV",
+  vavgsb_3 =            "10000502VVV",
+  vavgsh_3 =            "10000542VVV",
+  vorc_3 =              "10000544VVV",
+  vbpermq_3 =           "1000054cVVV",
+  vpksdus_3 =           "1000054eVVV",
+  vavgsw_3 =            "10000582VVV",
+  vsld_3 =              "100005c4VVV",
+  ["vcmpgefp._3"] =     "100005c6VVV",
+  vpksdss_3 =           "100005ceVVV",
+  vsububs_3 =           "10000600VVV",
+  mfvscr_1 =            "10000604V--",
+  vsum4ubs_3 =          "10000608VVV",
+  vsubuhs_3 =           "10000640VVV",
+  mtvscr_1 =            "10000644--V",
+  ["vcmpgtuh._3"] =     "10000646VVV",
+  vsum4shs_3 =          "10000648VVV",
+  vupkhsw_2 =           "1000064eV-V",
+  vsubuws_3 =           "10000680VVV",
+  vshasigmaw_4 =        "10000682VVYp",
+  veqv_3 =              "10000684VVV",
+  vsum2sws_3 =          "10000688VVV",
+  vmrgow_3 =            "1000068cVVV",
+  vshasigmad_4 =        "100006c2VVYp",
+  vsrd_3 =              "100006c4VVV",
+  ["vcmpgtud._3"] =     "100006c7VVV",
+  vupklsw_2 =           "100006ceV-V",
+  vupkslw_2 =           "100006ceV-V",
+  vsubsbs_3 =           "10000700VVV",
+  vclzb_2 =             "10000702V-V",
+  vpopcntb_2 =          "10000703V-V",
+  ["vcmpgtsb._3"] =     "10000706VVV",
+  vsum4sbs_3 =          "10000708VVV",
+  vsubshs_3 =           "10000740VVV",
+  vclzh_2 =             "10000742V-V",
+  vpopcnth_2 =          "10000743V-V",
+  ["vcmpgtsh._3"] =     "10000746VVV",
+  vsubsws_3 =           "10000780VVV",
+  vclzw_2 =             "10000782V-V",
+  vpopcntw_2 =          "10000783V-V",
+  ["vcmpgtsw._3"] =     "10000786VVV",
+  vsumsws_3 =           "10000788VVV",
+  vmrgew_3 =            "1000078cVVV",
+  vclzd_2 =             "100007c2V-V",
+  vpopcntd_2 =          "100007c3V-V",
+  ["vcmpbfp._3"] =      "100007c6VVV",
+  ["vcmpgtsd._3"] =     "100007c7VVV",
+
   -- Primary opcode 19:
   mcrf_2 =      "4c000000XX",
   isync_0 =     "4c00012c",
@@ -316,6 +572,8 @@ local map_op = {
   bclrl_2 =     "4c000021AA",
   bcctr_2 =     "4c000420AA",
   bcctrl_2 =    "4c000421AA",
+  bctar_2 =     "4c000460AA",
+  bctarl_2 =    "4c000461AA",
   blr_0 =       "4e800020",
   blrl_0 =      "4e800021",
   bctr_0 =      "4e800420",
@@ -327,6 +585,7 @@ local map_op = {
   cmpd_3 =      "7c200000XRR",
   cmpd_2 =      "7c200000-RR",
   tw_3 =        "7c000008ARR",
+  lvsl_3 =      "7c00000cVRR",
   subfc_3 =     "7c000010RRR.",
   subc_3 =      "7c000010RRR~.",
   mulhdu_3 =    "7c000012RRR.",
@@ -351,50 +610,68 @@ local map_op = {
   cmplw_2 =     "7c000040-RR",
   cmpld_3 =     "7c200040XRR",
   cmpld_2 =     "7c200040-RR",
+  lvsr_3 =      "7c00004cVRR",
   subf_3 =      "7c000050RRR.",
   sub_3 =       "7c000050RRR~.",
+  lbarx_3 =     "7c000068RR0R",
   ldux_3 =      "7c00006aRR0R",
   dcbst_2 =     "7c00006c-RR",
   lwzux_3 =     "7c00006eRR0R",
   cntlzd_2 =    "7c000074RR~",
   andc_3 =      "7c000078RR~R.",
   td_3 =        "7c000088ARR",
+  lvewx_3 =     "7c00008eVRR",
   mulhd_3 =     "7c000092RRR.",
+  addg6s_3 =    "7c000094RRR",
   mulhw_3 =     "7c000096RRR.",
+  dlmzb_3 =     "7c00009cRR~R.",
   ldarx_3 =     "7c0000a8RR0R",
   dcbf_2 =      "7c0000ac-RR",
   lbzx_3 =      "7c0000aeRR0R",
+  lvx_3 =       "7c0000ceVRR",
   neg_2 =       "7c0000d0RR.",
+  lharx_3 =     "7c0000e8RR0R",
   lbzux_3 =     "7c0000eeRR0R",
   popcntb_2 =   "7c0000f4RR~",
   not_2 =       "7c0000f8RR~%.",
   nor_3 =       "7c0000f8RR~R.",
+  stvebx_3 =    "7c00010eVRR",
   subfe_3 =     "7c000110RRR.",
   sube_3 =      "7c000110RRR~.",
   adde_3 =      "7c000114RRR.",
   stdx_3 =      "7c00012aRR0R",
-  stwcx_3 =     "7c00012cRR0R.",
+  ["stwcx._3"] =        "7c00012dRR0R.",
   stwx_3 =      "7c00012eRR0R",
   prtyw_2 =     "7c000134RR~",
+  stvehx_3 =    "7c00014eVRR",
   stdux_3 =     "7c00016aRR0R",
+  ["stqcx._3"] =        "7c00016dR:R0R.",
   stwux_3 =     "7c00016eRR0R",
   prtyd_2 =     "7c000174RR~",
+  stvewx_3 =    "7c00018eVRR",
   subfze_2 =    "7c000190RR.",
   addze_2 =     "7c000194RR.",
-  stdcx_3 =     "7c0001acRR0R.",
+  ["stdcx._3"] =        "7c0001adRR0R.",
   stbx_3 =      "7c0001aeRR0R",
+  stvx_3 =      "7c0001ceVRR",
   subfme_2 =    "7c0001d0RR.",
   mulld_3 =     "7c0001d2RRR.",
   addme_2 =     "7c0001d4RR.",
   mullw_3 =     "7c0001d6RRR.",
   dcbtst_2 =    "7c0001ec-RR",
   stbux_3 =     "7c0001eeRR0R",
+  bpermd_3 =    "7c0001f8RR~R",
+  lvepxl_3 =    "7c00020eVRR",
   add_3 =       "7c000214RRR.",
+  lqarx_3 =     "7c000228R:R0R",
   dcbt_2 =      "7c00022c-RR",
   lhzx_3 =      "7c00022eRR0R",
+  cdtbcd_2 =    "7c000234RR~",
   eqv_3 =       "7c000238RR~R.",
+  lvepx_3 =     "7c00024eVRR",
   eciwx_3 =     "7c00026cRR0R",
   lhzux_3 =     "7c00026eRR0R",
+  cbcdtd_2 =    "7c000274RR~",
   xor_3 =       "7c000278RR~R.",
   mfspefscr_1 = "7c0082a6R",
   mfxer_1 =     "7c0102a6R",
@@ -404,8 +681,12 @@ local map_op = {
   lhax_3 =      "7c0002aeRR0R",
   mftb_1 =      "7c0c42e6R",
   mftbu_1 =     "7c0d42e6R",
+  lvxl_3 =      "7c0002ceVRR",
   lwaux_3 =     "7c0002eaRR0R",
   lhaux_3 =     "7c0002eeRR0R",
+  popcntw_2 =   "7c0002f4RR~",
+  divdeu_3 =    "7c000312RRR.",
+  divweu_3 =    "7c000316RRR.",
   sthx_3 =      "7c00032eRR0R",
   orc_3 =       "7c000338RR~R.",
   ecowx_3 =     "7c00036cRR0R",
@@ -420,10 +701,14 @@ local map_op = {
   mtctr_1 =     "7c0903a6R",
   dcbi_2 =      "7c0003ac-RR",
   nand_3 =      "7c0003b8RR~R.",
+  dsn_2 =       "7c0003c6-RR",
+  stvxl_3 =     "7c0003ceVRR",
   divd_3 =      "7c0003d2RRR.",
   divw_3 =      "7c0003d6RRR.",
+  popcntd_2 =   "7c0003f4RR~",
   cmpb_3 =      "7c0003f8RR~R.",
   mcrxr_1 =     "7c000400X",
+  lbdx_3 =      "7c000406RRR",
   subfco_3 =    "7c000410RRR.",
   subco_3 =     "7c000410RRR~.",
   addco_3 =     "7c000414RRR.",
@@ -433,16 +718,20 @@ local map_op = {
   lfsx_3 =      "7c00042eFR0R",
   srw_3 =       "7c000430RR~R.",
   srd_3 =       "7c000436RR~R.",
+  lhdx_3 =      "7c000446RRR",
   subfo_3 =     "7c000450RRR.",
   subo_3 =      "7c000450RRR~.",
   lfsux_3 =     "7c00046eFR0R",
+  lwdx_3 =      "7c000486RRR",
   lswi_3 =      "7c0004aaRR0A",
   sync_0 =      "7c0004ac",
   lwsync_0 =    "7c2004ac",
   ptesync_0 =   "7c4004ac",
   lfdx_3 =      "7c0004aeFR0R",
+  lddx_3 =      "7c0004c6RRR",
   nego_2 =      "7c0004d0RR.",
   lfdux_3 =     "7c0004eeFR0R",
+  stbdx_3 =     "7c000506RRR",
   subfeo_3 =    "7c000510RRR.",
   subeo_3 =     "7c000510RRR~.",
   addeo_3 =     "7c000514RRR.",
@@ -450,27 +739,42 @@ local map_op = {
   stswx_3 =     "7c00052aRR0R",
   stwbrx_3 =    "7c00052cRR0R",
   stfsx_3 =     "7c00052eFR0R",
+  sthdx_3 =     "7c000546RRR",
+  ["stbcx._3"] =        "7c00056dRRR",
   stfsux_3 =    "7c00056eFR0R",
+  stwdx_3 =     "7c000586RRR",
   subfzeo_2 =   "7c000590RR.",
   addzeo_2 =    "7c000594RR.",
   stswi_3 =     "7c0005aaRR0A",
+  ["sthcx._3"] =        "7c0005adRRR",
   stfdx_3 =     "7c0005aeFR0R",
+  stddx_3 =     "7c0005c6RRR",
   subfmeo_2 =   "7c0005d0RR.",
   mulldo_3 =    "7c0005d2RRR.",
   addmeo_2 =    "7c0005d4RR.",
   mullwo_3 =    "7c0005d6RRR.",
   dcba_2 =      "7c0005ec-RR",
   stfdux_3 =    "7c0005eeFR0R",
+  stvepxl_3 =   "7c00060eVRR",
   addo_3 =      "7c000614RRR.",
   lhbrx_3 =     "7c00062cRR0R",
+  lfdpx_3 =     "7c00062eF:RR",
   sraw_3 =      "7c000630RR~R.",
   srad_3 =      "7c000634RR~R.",
+  lfddx_3 =     "7c000646FRR",
+  stvepx_3 =    "7c00064eVRR",
   srawi_3 =     "7c000670RR~A.",
   sradi_3 =     "7c000674RR~H.",
   eieio_0 =     "7c0006ac",
   lfiwax_3 =    "7c0006aeFR0R",
+  divdeuo_3 =   "7c000712RRR.",
+  divweuo_3 =   "7c000716RRR.",
   sthbrx_3 =    "7c00072cRR0R",
+  stfdpx_3 =    "7c00072eF:RR",
   extsh_2 =     "7c000734RR~.",
+  stfddx_3 =    "7c000746FRR",
+  divdeo_3 =    "7c000752RRR.",
+  divweo_3 =    "7c000756RRR.",
   extsb_2 =     "7c000774RR~.",
   divduo_3 =    "7c000792RRR.",
   divwou_3 =    "7c000796RRR.",
@@ -481,6 +785,40 @@ local map_op = {
   divwo_3 =     "7c0007d6RRR.",
   dcbz_2 =      "7c0007ec-RR",
 
+  ["tbegin._1"] =       "7c00051d1",
+  ["tbegin._0"] =       "7c00051d",
+  ["tend._1"] =         "7c00055dY",
+  ["tend._0"] =         "7c00055d",
+  ["tendall._0"] =      "7e00055d",
+  tcheck_1 =            "7c00059cX",
+  ["tsr._1"] =          "7c0005dd1",
+  ["tsuspend._0"] =     "7c0005dd",
+  ["tresume._0"] =      "7c2005dd",
+  ["tabortwc._3"] =     "7c00061dARR",
+  ["tabortdc._3"] =     "7c00065dARR",
+  ["tabortwci._3"] =    "7c00069dARS",
+  ["tabortdci._3"] =    "7c0006ddARS",
+  ["tabort._1"] =       "7c00071d-R-",
+  ["treclaim._1"] =     "7c00075d-R",
+  ["trechkpt._0"] =     "7c0007dd",
+
+  lxsiwzx_3 =   "7c000018QRR",
+  lxsiwax_3 =   "7c000098QRR",
+  mfvsrd_2 =    "7c000066-Rq",
+  mfvsrwz_2 =   "7c0000e6-Rq",
+  stxsiwx_3 =   "7c000118QRR",
+  mtvsrd_2 =    "7c000166QR",
+  mtvsrwa_2 =   "7c0001a6QR",
+  lxvdsx_3 =    "7c000298QRR",
+  lxsspx_3 =    "7c000418QRR",
+  lxsdx_3 =     "7c000498QRR",
+  stxsspx_3 =   "7c000518QRR",
+  stxsdx_3 =    "7c000598QRR",
+  lxvw4x_3 =    "7c000618QRR",
+  lxvd2x_3 =    "7c000698QRR",
+  stxvw4x_3 =   "7c000718QRR",
+  stxvd2x_3 =   "7c000798QRR",
+
   -- Primary opcode 30:
   rldicl_4 =    "78000000RR~HM.",
   rldicr_4 =    "78000004RR~HM.",
@@ -489,6 +827,34 @@ local map_op = {
   rldcl_4 =     "78000010RR~RM.",
   rldcr_4 =     "78000012RR~RM.",
 
+  rotldi_3 =    op_alias("rldicl_4", function(p)
+    p[4] = "0"
+  end),
+  rotrdi_3 =    op_alias("rldicl_4", function(p)
+    p[3] = "64-("..p[3]..")"; p[4] = "0"
+  end),
+  rotld_3 =     op_alias("rldcl_4", function(p)
+    p[4] = "0"
+  end),
+  sldi_3 =      op_alias("rldicr_4", function(p)
+    p[4] = "63-("..p[3]..")"
+  end),
+  srdi_3 =      op_alias("rldicl_4", function(p)
+    p[4] = p[3]; p[3] = "64-("..p[3]..")"
+  end),
+  clrldi_3 =    op_alias("rldicl_4", function(p)
+    p[4] = p[3]; p[3] = "0"
+  end),
+  clrrdi_3 =    op_alias("rldicr_4", function(p)
+    p[4] = "63-("..p[3]..")"; p[3] = "0"
+  end),
+
+  -- Primary opcode 56:
+  lq_2 =        "e0000000R:D", -- NYI: displacement must be divisible by 8.
+
+  -- Primary opcode 57:
+  lfdp_2 =      "e4000000F:D", -- NYI: displacement must be divisible by 4.
+
   -- Primary opcode 59:
   fdivs_3 =     "ec000024FFF.",
   fsubs_3 =     "ec000028FFF.",
@@ -501,6 +867,200 @@ local map_op = {
   fmadds_4 =    "ec00003aFFFF~.",
   fnmsubs_4 =   "ec00003cFFFF~.",
   fnmadds_4 =   "ec00003eFFFF~.",
+  fcfids_2 =    "ec00069cF-F.",
+  fcfidus_2 =   "ec00079cF-F.",
+
+  dadd_3 =      "ec000004FFF.",
+  dqua_4 =      "ec000006FFFZ.",
+  dmul_3 =      "ec000044FFF.",
+  drrnd_4 =     "ec000046FFFZ.",
+  dscli_3 =     "ec000084FF6.",
+  dquai_4 =     "ec000086SF~FZ.",
+  dscri_3 =     "ec0000c4FF6.",
+  drintx_4 =    "ec0000c61F~FZ.",
+  dcmpo_3 =     "ec000104XFF",
+  dtstex_3 =    "ec000144XFF",
+  dtstdc_3 =    "ec000184XF6",
+  dtstdg_3 =    "ec0001c4XF6",
+  drintn_4 =    "ec0001c61F~FZ.",
+  dctdp_2 =     "ec000204F-F.",
+  dctfix_2 =    "ec000244F-F.",
+  ddedpd_3 =    "ec000284ZF~F.",
+  dxex_2 =      "ec0002c4F-F.",
+  dsub_3 =      "ec000404FFF.",
+  ddiv_3 =      "ec000444FFF.",
+  dcmpu_3 =     "ec000504XFF",
+  dtstsf_3 =    "ec000544XFF",
+  drsp_2 =      "ec000604F-F.",
+  dcffix_2 =    "ec000644F-F.",
+  denbcd_3 =    "ec000684YF~F.",
+  diex_3 =      "ec0006c4FFF.",
+
+  -- Primary opcode 60:
+  xsaddsp_3 =           "f0000000QQQ",
+  xsmaddasp_3 =         "f0000008QQQ",
+  xxsldwi_4 =           "f0000010QQQz",
+  xsrsqrtesp_2 =        "f0000028Q-Q",
+  xssqrtsp_2 =          "f000002cQ-Q",
+  xxsel_4 =             "f0000030QQQQ",
+  xssubsp_3 =           "f0000040QQQ",
+  xsmaddmsp_3 =         "f0000048QQQ",
+  xxpermdi_4 =          "f0000050QQQz",
+  xsresp_2 =            "f0000068Q-Q",
+  xsmulsp_3 =           "f0000080QQQ",
+  xsmsubasp_3 =         "f0000088QQQ",
+  xxmrghw_3 =           "f0000090QQQ",
+  xsdivsp_3 =           "f00000c0QQQ",
+  xsmsubmsp_3 =         "f00000c8QQQ",
+  xsadddp_3 =           "f0000100QQQ",
+  xsmaddadp_3 =         "f0000108QQQ",
+  xscmpudp_3 =          "f0000118XQQ",
+  xscvdpuxws_2 =        "f0000120Q-Q",
+  xsrdpi_2 =            "f0000124Q-Q",
+  xsrsqrtedp_2 =        "f0000128Q-Q",
+  xssqrtdp_2 =          "f000012cQ-Q",
+  xssubdp_3 =           "f0000140QQQ",
+  xsmaddmdp_3 =         "f0000148QQQ",
+  xscmpodp_3 =          "f0000158XQQ",
+  xscvdpsxws_2 =        "f0000160Q-Q",
+  xsrdpiz_2 =           "f0000164Q-Q",
+  xsredp_2 =            "f0000168Q-Q",
+  xsmuldp_3 =           "f0000180QQQ",
+  xsmsubadp_3 =         "f0000188QQQ",
+  xxmrglw_3 =           "f0000190QQQ",
+  xsrdpip_2 =           "f00001a4Q-Q",
+  xstsqrtdp_2 =         "f00001a8X-Q",
+  xsrdpic_2 =           "f00001acQ-Q",
+  xsdivdp_3 =           "f00001c0QQQ",
+  xsmsubmdp_3 =         "f00001c8QQQ",
+  xsrdpim_2 =           "f00001e4Q-Q",
+  xstdivdp_3 =          "f00001e8XQQ",
+  xvaddsp_3 =           "f0000200QQQ",
+  xvmaddasp_3 =         "f0000208QQQ",
+  xvcmpeqsp_3 =         "f0000218QQQ",
+  xvcvspuxws_2 =        "f0000220Q-Q",
+  xvrspi_2 =            "f0000224Q-Q",
+  xvrsqrtesp_2 =        "f0000228Q-Q",
+  xvsqrtsp_2 =          "f000022cQ-Q",
+  xvsubsp_3 =           "f0000240QQQ",
+  xvmaddmsp_3 =         "f0000248QQQ",
+  xvcmpgtsp_3 =         "f0000258QQQ",
+  xvcvspsxws_2 =        "f0000260Q-Q",
+  xvrspiz_2 =           "f0000264Q-Q",
+  xvresp_2 =            "f0000268Q-Q",
+  xvmulsp_3 =           "f0000280QQQ",
+  xvmsubasp_3 =         "f0000288QQQ",
+  xxspltw_3 =           "f0000290QQg~",
+  xvcmpgesp_3 =         "f0000298QQQ",
+  xvcvuxwsp_2 =         "f00002a0Q-Q",
+  xvrspip_2 =           "f00002a4Q-Q",
+  xvtsqrtsp_2 =         "f00002a8X-Q",
+  xvrspic_2 =           "f00002acQ-Q",
+  xvdivsp_3 =           "f00002c0QQQ",
+  xvmsubmsp_3 =         "f00002c8QQQ",
+  xvcvsxwsp_2 =         "f00002e0Q-Q",
+  xvrspim_2 =           "f00002e4Q-Q",
+  xvtdivsp_3 =          "f00002e8XQQ",
+  xvadddp_3 =           "f0000300QQQ",
+  xvmaddadp_3 =         "f0000308QQQ",
+  xvcmpeqdp_3 =         "f0000318QQQ",
+  xvcvdpuxws_2 =        "f0000320Q-Q",
+  xvrdpi_2 =            "f0000324Q-Q",
+  xvrsqrtedp_2 =        "f0000328Q-Q",
+  xvsqrtdp_2 =          "f000032cQ-Q",
+  xvsubdp_3 =           "f0000340QQQ",
+  xvmaddmdp_3 =         "f0000348QQQ",
+  xvcmpgtdp_3 =         "f0000358QQQ",
+  xvcvdpsxws_2 =        "f0000360Q-Q",
+  xvrdpiz_2 =           "f0000364Q-Q",
+  xvredp_2 =            "f0000368Q-Q",
+  xvmuldp_3 =           "f0000380QQQ",
+  xvmsubadp_3 =         "f0000388QQQ",
+  xvcmpgedp_3 =         "f0000398QQQ",
+  xvcvuxwdp_2 =         "f00003a0Q-Q",
+  xvrdpip_2 =           "f00003a4Q-Q",
+  xvtsqrtdp_2 =         "f00003a8X-Q",
+  xvrdpic_2 =           "f00003acQ-Q",
+  xvdivdp_3 =           "f00003c0QQQ",
+  xvmsubmdp_3 =         "f00003c8QQQ",
+  xvcvsxwdp_2 =         "f00003e0Q-Q",
+  xvrdpim_2 =           "f00003e4Q-Q",
+  xvtdivdp_3 =          "f00003e8XQQ",
+  xsnmaddasp_3 =        "f0000408QQQ",
+  xxland_3 =            "f0000410QQQ",
+  xscvdpsp_2 =          "f0000424Q-Q",
+  xscvdpspn_2 =         "f000042cQ-Q",
+  xsnmaddmsp_3 =        "f0000448QQQ",
+  xxlandc_3 =           "f0000450QQQ",
+  xsrsp_2 =             "f0000464Q-Q",
+  xsnmsubasp_3 =        "f0000488QQQ",
+  xxlor_3 =             "f0000490QQQ",
+  xscvuxdsp_2 =         "f00004a0Q-Q",
+  xsnmsubmsp_3 =        "f00004c8QQQ",
+  xxlxor_3 =            "f00004d0QQQ",
+  xscvsxdsp_2 =         "f00004e0Q-Q",
+  xsmaxdp_3 =           "f0000500QQQ",
+  xsnmaddadp_3 =        "f0000508QQQ",
+  xxlnor_3 =            "f0000510QQQ",
+  xscvdpuxds_2 =        "f0000520Q-Q",
+  xscvspdp_2 =          "f0000524Q-Q",
+  xscvspdpn_2 =         "f000052cQ-Q",
+  xsmindp_3 =           "f0000540QQQ",
+  xsnmaddmdp_3 =        "f0000548QQQ",
+  xxlorc_3 =            "f0000550QQQ",
+  xscvdpsxds_2 =        "f0000560Q-Q",
+  xsabsdp_2 =           "f0000564Q-Q",
+  xscpsgndp_3 =         "f0000580QQQ",
+  xsnmsubadp_3 =        "f0000588QQQ",
+  xxlnand_3 =           "f0000590QQQ",
+  xscvuxddp_2 =         "f00005a0Q-Q",
+  xsnabsdp_2 =          "f00005a4Q-Q",
+  xsnmsubmdp_3 =        "f00005c8QQQ",
+  xxleqv_3 =            "f00005d0QQQ",
+  xscvsxddp_2 =         "f00005e0Q-Q",
+  xsnegdp_2 =           "f00005e4Q-Q",
+  xvmaxsp_3 =           "f0000600QQQ",
+  xvnmaddasp_3 =        "f0000608QQQ",
+  ["xvcmpeqsp._3"] =    "f0000618QQQ",
+  xvcvspuxds_2 =        "f0000620Q-Q",
+  xvcvdpsp_2 =          "f0000624Q-Q",
+  xvminsp_3 =           "f0000640QQQ",
+  xvnmaddmsp_3 =        "f0000648QQQ",
+  ["xvcmpgtsp._3"] =    "f0000658QQQ",
+  xvcvspsxds_2 =        "f0000660Q-Q",
+  xvabssp_2 =           "f0000664Q-Q",
+  xvcpsgnsp_3 =         "f0000680QQQ",
+  xvnmsubasp_3 =        "f0000688QQQ",
+  ["xvcmpgesp._3"] =    "f0000698QQQ",
+  xvcvuxdsp_2 =         "f00006a0Q-Q",
+  xvnabssp_2 =          "f00006a4Q-Q",
+  xvnmsubmsp_3 =        "f00006c8QQQ",
+  xvcvsxdsp_2 =         "f00006e0Q-Q",
+  xvnegsp_2 =           "f00006e4Q-Q",
+  xvmaxdp_3 =           "f0000700QQQ",
+  xvnmaddadp_3 =        "f0000708QQQ",
+  ["xvcmpeqdp._3"] =    "f0000718QQQ",
+  xvcvdpuxds_2 =        "f0000720Q-Q",
+  xvcvspdp_2 =          "f0000724Q-Q",
+  xvmindp_3 =           "f0000740QQQ",
+  xvnmaddmdp_3 =        "f0000748QQQ",
+  ["xvcmpgtdp._3"] =    "f0000758QQQ",
+  xvcvdpsxds_2 =        "f0000760Q-Q",
+  xvabsdp_2 =           "f0000764Q-Q",
+  xvcpsgndp_3 =         "f0000780QQQ",
+  xvnmsubadp_3 =        "f0000788QQQ",
+  ["xvcmpgedp._3"] =    "f0000798QQQ",
+  xvcvuxddp_2 =         "f00007a0Q-Q",
+  xvnabsdp_2 =          "f00007a4Q-Q",
+  xvnmsubmdp_3 =        "f00007c8QQQ",
+  xvcvsxddp_2 =         "f00007e0Q-Q",
+  xvnegdp_2 =           "f00007e4Q-Q",
+
+  -- Primary opcode 61:
+  stfdp_2 =     "f4000000F:D", -- NYI: displacement must be divisible by 4.
+
+  -- Primary opcode 62:
+  stq_2 =       "f8000002R:D", -- NYI: displacement must be divisible by 8.
 
   -- Primary opcode 63:
   fdiv_3 =      "fc000024FFF.",
@@ -526,8 +1086,12 @@ local map_op = {
   frsp_2 =      "fc000018F-F.",
   fctiw_2 =     "fc00001cF-F.",
   fctiwz_2 =    "fc00001eF-F.",
+  ftdiv_2 =     "fc000100X-F.",
+  fctiwu_2 =    "fc00011cF-F.",
+  fctiwuz_2 =   "fc00011eF-F.",
   mtfsfi_2 =    "fc00010cAA", -- NYI: upshift.
   fnabs_2 =     "fc000110F-F.",
+  ftsqrt_2 =    "fc000140X-F.",
   fabs_2 =      "fc000210F-F.",
   frin_2 =      "fc000310F-F.",
   friz_2 =      "fc000350F-F.",
@@ -537,7 +1101,38 @@ local map_op = {
   -- NYI: mtfsf, mtfsb0, mtfsb1.
   fctid_2 =     "fc00065cF-F.",
   fctidz_2 =    "fc00065eF-F.",
+  fmrgow_3 =    "fc00068cFFF",
   fcfid_2 =     "fc00069cF-F.",
+  fctidu_2 =    "fc00075cF-F.",
+  fctiduz_2 =   "fc00075eF-F.",
+  fmrgew_3 =    "fc00078cFFF",
+  fcfidu_2 =    "fc00079cF-F.",
+
+  daddq_3 =     "fc000004F:F:F:.",
+  dquaq_4 =     "fc000006F:F:F:Z.",
+  dmulq_3 =     "fc000044F:F:F:.",
+  drrndq_4 =    "fc000046F:F:F:Z.",
+  dscliq_3 =    "fc000084F:F:6.",
+  dquaiq_4 =    "fc000086SF:~F:Z.",
+  dscriq_3 =    "fc0000c4F:F:6.",
+  drintxq_4 =   "fc0000c61F:~F:Z.",
+  dcmpoq_3 =    "fc000104XF:F:",
+  dtstexq_3 =   "fc000144XF:F:",
+  dtstdcq_3 =   "fc000184XF:6",
+  dtstdgq_3 =   "fc0001c4XF:6",
+  drintnq_4 =   "fc0001c61F:~F:Z.",
+  dctqpq_2 =    "fc000204F:-F:.",
+  dctfixq_2 =   "fc000244F:-F:.",
+  ddedpdq_3 =   "fc000284ZF:~F:.",
+  dxexq_2 =     "fc0002c4F:-F:.",
+  dsubq_3 =     "fc000404F:F:F:.",
+  ddivq_3 =     "fc000444F:F:F:.",
+  dcmpuq_3 =    "fc000504XF:F:",
+  dtstsfq_3 =   "fc000544XF:F:",
+  drdpq_2 =     "fc000604F:-F:.",
+  dcffixq_2 =   "fc000644F:-F:.",
+  denbcdq_3 =   "fc000684YF:~F:.",
+  diexq_3 =     "fc0006c4F:FF:.",
 
   -- Primary opcode 4, SPE APU extension:
   evaddw_3 =            "10000200RRR",
@@ -822,7 +1417,7 @@ local map_op = {
 do
   local t = {}
   for k,v in pairs(map_op) do
-    if sub(v, -1) == "." then
+    if type(v) == "string" and sub(v, -1) == "." then
       local v2 = sub(v, 1, 7)..char(byte(v, 8)+1)..sub(v, 9, -2)
       t[sub(k, 1, -3).."."..sub(k, -2)] = v2
     end
@@ -884,6 +1479,24 @@ local function parse_fpr(expr)
   werror("bad register name `"..expr.."'")
 end
 
+local function parse_vr(expr)
+  local r = match(expr, "^v([1-3]?[0-9])$")
+  if r then
+    r = tonumber(r)
+    if r <= 31 then return r end
+  end
+  werror("bad register name `"..expr.."'")
+end
+
+local function parse_vs(expr)
+  local r = match(expr, "^vs([1-6]?[0-9])$")
+  if r then
+    r = tonumber(r)
+    if r <= 63 then return r end
+  end
+  werror("bad register name `"..expr.."'")
+end
+
 local function parse_cr(expr)
   local r = match(expr, "^cr([0-7])$")
   if r then return tonumber(r) end
@@ -900,8 +1513,30 @@ local function parse_cond(expr)
   werror("bad condition bit name `"..expr.."'")
 end
 
+local parse_ctx = {}
+
+local loadenv = setfenv and function(s)
+  local code = loadstring(s, "")
+  if code then setfenv(code, parse_ctx) end
+  return code
+end or function(s)
+  return load(s, "", nil, parse_ctx)
+end
+
+-- Try to parse simple arithmetic, too, since some basic ops are aliases.
+local function parse_number(n)
+  local x = tonumber(n)
+  if x then return x end
+  local code = loadenv("return "..n)
+  if code then
+    local ok, y = pcall(code)
+    if ok then return y end
+  end
+  return nil
+end
+
 local function parse_imm(imm, bits, shift, scale, signed)
-  local n = tonumber(imm)
+  local n = parse_number(imm)
   if n then
     local m = sar(n, scale)
     if shl(m, scale) == n then
@@ -914,7 +1549,8 @@ local function parse_imm(imm, bits, shift, scale, signed)
       end
     end
     werror("out of range immediate `"..imm.."'")
-  elseif match(imm, "^r([1-3]?[0-9])$") or
+  elseif match(imm, "^[rfv]([1-3]?[0-9])$") or
+         match(imm, "^vs([1-6]?[0-9])$") or
          match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then
     werror("expected immediate operand, got register")
   else
@@ -924,11 +1560,11 @@ local function parse_imm(imm, bits, shift, scale, signed)
 end
 
 local function parse_shiftmask(imm, isshift)
-  local n = tonumber(imm)
+  local n = parse_number(imm)
   if n then
     if shr(n, 6) == 0 then
-      local lsb = band(imm, 31)
-      local msb = imm - lsb
+      local lsb = band(n, 31)
+      local msb = n - lsb
       return isshift and (shl(lsb, 11)+shr(msb, 4)) or (shl(lsb, 6)+msb)
     end
     werror("out of range immediate `"..imm.."'")
@@ -936,7 +1572,8 @@ local function parse_shiftmask(imm, isshift)
          match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then
     werror("expected immediate operand, got register")
   else
-    werror("NYI: parameterized 64 bit shift/mask")
+    waction("IMMSH", isshift and 1 or 0, imm)
+    return 0;
   end
 end
 
@@ -1011,7 +1648,7 @@ end
 ------------------------------------------------------------------------------
 
 -- Handle opcodes defined with template strings.
-map_op[".template__"] = function(params, template, nparams)
+op_template = function(params, template, nparams)
   if not params then return sub(template, 9) end
   local op = tonumber(sub(template, 1, 8), 16)
   local n, rs = 1, 26
@@ -1027,6 +1664,15 @@ map_op[".template__"] = function(params, template, nparams)
       rs = rs - 5; op = op + shl(parse_gpr(params[n]), rs); n = n + 1
     elseif p == "F" then
       rs = rs - 5; op = op + shl(parse_fpr(params[n]), rs); n = n + 1
+    elseif p == "V" then
+      rs = rs - 5; op = op + shl(parse_vr(params[n]), rs); n = n + 1
+    elseif p == "Q" then
+      local vs = parse_vs(params[n]); n = n + 1; rs = rs - 5
+      local sh = rs == 6 and 2 or 3 + band(shr(rs, 1), 3)
+      op = op + shl(band(vs, 31), rs) + shr(band(vs, 32), sh)
+    elseif p == "q" then
+      local vs = parse_vs(params[n]); n = n + 1
+      op = op + shl(band(vs, 31), 21) + shr(band(vs, 32), 5)
     elseif p == "A" then
       rs = rs - 5; op = op + parse_imm(params[n], 5, rs, 0, false); n = n + 1
     elseif p == "S" then
@@ -1047,6 +1693,26 @@ map_op[".template__"] = function(params, template, nparams)
       rs = rs - 5; op = op + shl(parse_cond(params[n]), rs); n = n + 1
     elseif p == "X" then
       rs = rs - 5; op = op + shl(parse_cr(params[n]), rs+2); n = n + 1
+    elseif p == "1" then
+      rs = rs - 5; op = op + parse_imm(params[n], 1, rs, 0, false); n = n + 1
+    elseif p == "g" then
+      rs = rs - 5; op = op + parse_imm(params[n], 2, rs, 0, false); n = n + 1
+    elseif p == "3" then
+      rs = rs - 5; op = op + parse_imm(params[n], 3, rs, 0, false); n = n + 1
+    elseif p == "P" then
+      rs = rs - 5; op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1
+    elseif p == "p" then
+      op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1
+    elseif p == "6" then
+      rs = rs - 6; op = op + parse_imm(params[n], 6, rs, 0, false); n = n + 1
+    elseif p == "Y" then
+      rs = rs - 5; op = op + parse_imm(params[n], 1, rs+4, 0, false); n = n + 1
+    elseif p == "y" then
+      rs = rs - 5; op = op + parse_imm(params[n], 1, rs+3, 0, false); n = n + 1
+    elseif p == "Z" then
+      rs = rs - 5; op = op + parse_imm(params[n], 2, rs+3, 0, false); n = n + 1
+    elseif p == "z" then
+      rs = rs - 5; op = op + parse_imm(params[n], 2, rs+2, 0, false); n = n + 1
     elseif p == "W" then
       op = op + parse_cr(params[n]); n = n + 1
     elseif p == "G" then
@@ -1071,6 +1737,8 @@ map_op[".template__"] = function(params, template, nparams)
       local lo = band(op, mm)
       local hi = band(op, shl(mm, 5))
       op = op - lo - hi + shl(lo, 5) + shr(hi, 5)
+    elseif p == ":" then
+      if band(shr(op, rs), 1) ~= 0 then werror("register pair expected") end
     elseif p == "-" then
       rs = rs - 5
     elseif p == "." then
@@ -1082,6 +1750,8 @@ map_op[".template__"] = function(params, template, nparams)
   wputpos(pos, op)
 end
 
+map_op[".template__"] = op_template
+
 ------------------------------------------------------------------------------
 
 -- Pseudo-opcode to mark the position where the action list is to be emitted.
diff --git a/dynasm/dasm_proto.h b/dynasm/dasm_proto.h
index a7278e85..ba038e87 100644
--- a/dynasm/dasm_proto.h
+++ b/dynasm/dasm_proto.h
@@ -10,8 +10,8 @@
 #include <stddef.h>
 #include <stdarg.h>
 
-#define DASM_IDENT      "DynASM 1.3.0"
-#define DASM_VERSION    10300   /* 1.3.0 */
+#define DASM_IDENT      "DynASM 1.4.0"
+#define DASM_VERSION    10400   /* 1.4.0 */
 
 #ifndef Dst_DECL
 #define Dst_DECL        dasm_State **Dst
diff --git a/dynasm/dasm_x86.h b/dynasm/dasm_x86.h
index 84b9d17f..edaddf54 100644
--- a/dynasm/dasm_x86.h
+++ b/dynasm/dasm_x86.h
@@ -170,7 +170,7 @@ void dasm_put(Dst_DECL, int start, ...)
   dasm_State *D = Dst_REF;
   dasm_ActList p = D->actionlist + start;
   dasm_Section *sec = D->section;
-  int pos = sec->pos, ofs = sec->ofs, mrm = 4;
+  int pos = sec->pos, ofs = sec->ofs, mrm = -1;
   int *b;
 
   if (pos >= sec->epos) {
@@ -193,7 +193,7 @@ void dasm_put(Dst_DECL, int start, ...)
       b[pos++] = n;
       switch (action) {
       case DASM_DISP:
-        if (n == 0) { if ((mrm&7) == 4) mrm = p[-2]; if ((mrm&7) != 5) break; }
+        if (n == 0) { if (mrm < 0) mrm = p[-2]; if ((mrm&7) != 5) break; }
         /* fallthrough */
       case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */
       case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */
@@ -204,11 +204,17 @@ void dasm_put(Dst_DECL, int start, ...)
       case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break;
       case DASM_SPACE: p++; ofs += n; break;
       case DASM_SETLABEL: b[pos-2] = -0x40000000; break;  /* Neg. label ofs. */
-      case DASM_VREG: CK((n&-8) == 0 && (n != 4 || (*p&1) == 0), RANGE_VREG);
-        if (*p++ == 1 && *p == DASM_DISP) mrm = n;
+      case DASM_VREG: CK((n&-16) == 0 && (n != 4 || (*p>>5) != 2), RANGE_VREG);
+        if (*p < 0x40 && p[1] == DASM_DISP) mrm = n;
+        if (*p < 0x20 && (n&7) == 4) ofs++;
+        switch ((*p++ >> 3) & 3) {
+        case 3: n |= b[pos-3]; /* fallthrough */
+        case 2: n |= b[pos-2]; /* fallthrough */
+        case 1: if (n <= 7) { b[pos-1] |= 0x10; ofs--; }
+        }
         continue;
       }
-      mrm = 4;
+      mrm = -1;
     } else {
       int *pl, n;
       switch (action) {
@@ -399,7 +405,27 @@ int dasm_encode(Dst_DECL, void *buffer)
         case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL;
           /* fallthrough */
         case DASM_IMM_W: dasmw(n); break;
-        case DASM_VREG: { int t = *p++; if (t >= 2) n<<=3; cp[-1] |= n; break; }
+        case DASM_VREG: {
+          int t = *p++;
+          unsigned char *ex = cp - (t&7);
+          if ((n & 8) && t < 0xa0) {
+            if (*ex & 0x80) ex[1] ^= 0x20 << (t>>6); else *ex ^= 1 << (t>>6);
+            n &= 7;
+          } else if (n & 0x10) {
+            if (*ex & 0x80) {
+              *ex = 0xc5; ex[1] = (ex[1] & 0x80) | ex[2]; ex += 2;
+            }
+            while (++ex < cp) ex[-1] = *ex;
+            if (mark) mark--;
+            cp--;
+            n &= 7;
+          }
+          if (t >= 0xc0) n <<= 4;
+          else if (t >= 0x40) n <<= 3;
+          else if (n == 4 && t < 0x20) { cp[-1] ^= n; *cp++ = 0x20; }
+          cp[-1] ^= n;
+          break;
+        }
         case DASM_REL_LG: p++; if (n >= 0) goto rel_pc;
           b++; n = (int)(ptrdiff_t)D->globals[-n];
           /* fallthrough */
diff --git a/dynasm/dasm_x86.lua b/dynasm/dasm_x86.lua
index 13aa68ff..c5c8c17b 100644
--- a/dynasm/dasm_x86.lua
+++ b/dynasm/dasm_x86.lua
@@ -11,9 +11,9 @@ local x64 = x64
 local _info = {
   arch =        x64 and "x64" or "x86",
   description = "DynASM x86/x64 module",
-  version =     "1.3.0",
-  vernum =       10300,
-  release =     "2011-05-05",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2015-10-18",
   author =      "Mike Pall",
   license =     "MIT",
 }
@@ -27,9 +27,9 @@ local assert, unpack, setmetatable = assert, unpack or table.unpack, setmetatabl
 local _s = string
 local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char
 local find, match, gmatch, gsub = _s.find, _s.match, _s.gmatch, _s.gsub
-local concat, sort = table.concat, table.sort
+local concat, sort, remove = table.concat, table.sort, table.remove
 local bit = bit or require("bit")
-local band, shl, shr = bit.band, bit.lshift, bit.rshift
+local band, bxor, shl, shr = bit.band, bit.bxor, bit.lshift, bit.rshift
 
 -- Inherited tables and callbacks.
 local g_opt, g_arch
@@ -41,7 +41,7 @@ local action_names = {
   -- int arg, 1 buffer pos:
   "DISP",  "IMM_S", "IMM_B", "IMM_W", "IMM_D",  "IMM_WB", "IMM_DB",
   -- action arg (1 byte), int arg, 1 buffer pos (reg/num):
-  "VREG", "SPACE", -- !x64: VREG support NYI.
+  "VREG", "SPACE",
   -- ptrdiff_t arg, 1 buffer pos (address): !x64
   "SETLABEL", "REL_A",
   -- action arg (1 byte) or int arg, 2 buffer pos (link, offset):
@@ -83,6 +83,21 @@ local actargs = { 0 }
 -- Current number of section buffer positions for dasm_put().
 local secpos = 1
 
+-- VREG kind encodings, pre-shifted by 5 bits.
+local map_vreg = {
+  ["modrm.rm.m"] = 0x00,
+  ["modrm.rm.r"] = 0x20,
+  ["opcode"] =     0x20,
+  ["sib.base"] =   0x20,
+  ["sib.index"] =  0x40,
+  ["modrm.reg"] =  0x80,
+  ["vex.v"] =      0xa0,
+  ["imm.hi"] =     0xc0,
+}
+
+-- Current number of VREG actions contributing to REX/VEX shrinkage.
+local vreg_shrink_count = 0
+
 ------------------------------------------------------------------------------
 
 -- Compute action numbers for action names.
@@ -134,6 +149,21 @@ local function waction(action, a, num)
   if a or num then secpos = secpos + (num or 1) end
 end
 
+-- Optionally add a VREG action.
+local function wvreg(kind, vreg, psz, sk, defer)
+  if not vreg then return end
+  waction("VREG", vreg)
+  local b = assert(map_vreg[kind], "bad vreg kind `"..vreg.."'")
+  if b < (sk or 0) then
+    vreg_shrink_count = vreg_shrink_count + 1
+  end
+  if not defer then
+    b = b + vreg_shrink_count * 8
+    vreg_shrink_count = 0
+  end
+  wputxb(b + (psz or 0))
+end
+
 -- Add call to embedded DynASM C code.
 local function wcall(func, args)
   wline(format("dasm_%s(Dst, %s);", func, concat(args, ", ")), true)
@@ -299,7 +329,7 @@ local function mkrmap(sz, cl, names)
     local iname = format("@%s%x%s", sz, i, needrex and "R" or "")
     if needrex then map_reg_needrex[iname] = true end
     local name
-    if sz == "o" then name = format("xmm%d", i)
+    if sz == "o" or sz == "y" then name = format("%s%d", cl, i)
     elseif sz == "f" then name = format("st%d", i)
     else name = format("r%d%s", i, sz == addrsize and "" or sz) end
     map_archdef[name] = iname
@@ -326,6 +356,7 @@ mkrmap("w", "Rw", {"ax", "cx", "dx", "bx", "sp", "bp", "si", "di"})
 mkrmap("b", "Rb", {"al", "cl", "dl", "bl", "ah", "ch", "dh", "bh"})
 map_reg_valid_index[map_archdef.esp] = false
 if x64 then map_reg_valid_index[map_archdef.rsp] = false end
+if x64 then map_reg_needrex[map_archdef.Rb] = true end
 map_archdef["Ra"] = "@"..addrsize
 
 -- FP registers (internally tword sized, but use "f" as operand size).
@@ -334,21 +365,24 @@ mkrmap("f", "Rf")
 -- SSE registers (oword sized, but qword and dword accessible).
 mkrmap("o", "xmm")
 
+-- AVX registers (yword sized, but oword, qword and dword accessible).
+mkrmap("y", "ymm")
+
 -- Operand size prefixes to codes.
 local map_opsize = {
-  byte = "b", word = "w", dword = "d", qword = "q", oword = "o", tword = "t",
-  aword = addrsize,
+  byte = "b", word = "w", dword = "d", qword = "q", oword = "o", yword = "y",
+  tword = "t", aword = addrsize,
 }
 
 -- Operand size code to number.
 local map_opsizenum = {
-  b = 1, w = 2, d = 4, q = 8, o = 16, t = 10,
+  b = 1, w = 2, d = 4, q = 8, o = 16, y = 32, t = 10,
 }
 
 -- Operand size code to name.
 local map_opsizename = {
-  b = "byte", w = "word", d = "dword", q = "qword", o = "oword", t = "tword",
-  f = "fpword",
+  b = "byte", w = "word", d = "dword", q = "qword", o = "oword", y = "yword",
+  t = "tword", f = "fpword",
 }
 
 -- Valid index register scale factors.
@@ -460,9 +494,45 @@ local function wputszarg(sz, n)
 end
 
 -- Put multi-byte opcode with operand-size dependent modifications.
-local function wputop(sz, op, rex)
+local function wputop(sz, op, rex, vex, vregr, vregxb)
+  local psz, sk = 0, nil
+  if vex then
+    local tail
+    if vex.m == 1 and band(rex, 11) == 0 then
+      if x64 and vregxb then
+        sk = map_vreg["modrm.reg"]
+      else
+        wputb(0xc5)
+      tail = shl(bxor(band(rex, 4), 4), 5)
+      psz = 3
+      end
+    end
+    if not tail then
+      wputb(0xc4)
+      wputb(shl(bxor(band(rex, 7), 7), 5) + vex.m)
+      tail = shl(band(rex, 8), 4)
+      psz = 4
+    end
+    local reg, vreg = 0, nil
+    if vex.v then
+      reg = vex.v.reg
+      if not reg then werror("bad vex operand") end
+      if reg < 0 then reg = 0; vreg = vex.v.vreg end
+    end
+    if sz == "y" or vex.l then tail = tail + 4 end
+    wputb(tail + shl(bxor(reg, 15), 3) + vex.p)
+    wvreg("vex.v", vreg)
+    rex = 0
+    if op >= 256 then werror("bad vex opcode") end
+  else
+    if rex ~= 0 then
+      if not x64 then werror("bad operand size") end
+    elseif (vregr or vregxb) and x64 then
+      rex = 0x10
+      sk = map_vreg["vex.v"]
+    end
+  end
   local r
-  if rex ~= 0 and not x64 then werror("bad operand size") end
   if sz == "w" then wputb(102) end
   -- Needs >32 bit numbers, but only for crc32 eax, word [ebx]
   if op >= 4294967296 then r = op%4294967296 wputb((op-r)/4294967296) op = r end
@@ -471,20 +541,20 @@ local function wputop(sz, op, rex)
     if rex ~= 0 then
       local opc3 = band(op, 0xffff00)
       if opc3 == 0x0f3a00 or opc3 == 0x0f3800 then
-        wputb(64 + band(rex, 15)); rex = 0
+        wputb(64 + band(rex, 15)); rex = 0; psz = 2
       end
     end
-    wputb(shr(op, 16)); op = band(op, 0xffff)
+    wputb(shr(op, 16)); op = band(op, 0xffff); psz = psz + 1
   end
   if op >= 256 then
     local b = shr(op, 8)
-    if b == 15 and rex ~= 0 then wputb(64 + band(rex, 15)); rex = 0 end
-    wputb(b)
-    op = band(op, 255)
+    if b == 15 and rex ~= 0 then wputb(64 + band(rex, 15)); rex = 0; psz = 2 end
+    wputb(b); op = band(op, 255); psz = psz + 1
   end
-  if rex ~= 0 then wputb(64 + band(rex, 15)) end
+  if rex ~= 0 then wputb(64 + band(rex, 15)); psz = 2 end
   if sz == "b" then op = op - 1 end
   wputb(op)
+  return psz, sk
 end
 
 -- Put ModRM or SIB formatted byte.
@@ -494,7 +564,7 @@ local function wputmodrm(m, s, rm, vs, vrm)
 end
 
 -- Put ModRM/SIB plus optional displacement.
-local function wputmrmsib(t, imark, s, vsreg)
+local function wputmrmsib(t, imark, s, vsreg, psz, sk)
   local vreg, vxreg
   local reg, xreg = t.reg, t.xreg
   if reg and reg < 0 then reg = 0; vreg = t.vreg end
@@ -504,8 +574,8 @@ local function wputmrmsib(t, imark, s, vsreg)
   -- Register mode.
   if sub(t.mode, 1, 1) == "r" then
     wputmodrm(3, s, reg)
-    if vsreg then waction("VREG", vsreg); wputxb(2) end
-    if vreg then waction("VREG", vreg); wputxb(0) end
+    wvreg("modrm.reg", vsreg, psz+1, sk, vreg)
+    wvreg("modrm.rm.r", vreg, psz+1, sk)
     return
   end
 
@@ -519,21 +589,22 @@ local function wputmrmsib(t, imark, s, vsreg)
       -- [xreg*xsc+disp] -> (0, s, esp) (xsc, xreg, ebp)
       wputmodrm(0, s, 4)
       if imark == "I" then waction("MARK") end
-      if vsreg then waction("VREG", vsreg); wputxb(2) end
+      wvreg("modrm.reg", vsreg, psz+1, sk, vxreg)
       wputmodrm(t.xsc, xreg, 5)
-      if vxreg then waction("VREG", vxreg); wputxb(3) end
+      wvreg("sib.index", vxreg, psz+2, sk)
     else
       -- Pure 32 bit displacement.
       if x64 and tdisp ~= "table" then
         wputmodrm(0, s, 4) -- [disp] -> (0, s, esp) (0, esp, ebp)
+        wvreg("modrm.reg", vsreg, psz+1, sk)
         if imark == "I" then waction("MARK") end
         wputmodrm(0, 4, 5)
       else
         riprel = x64
         wputmodrm(0, s, 5) -- [disp|rip-label] -> (0, s, ebp)
+        wvreg("modrm.reg", vsreg, psz+1, sk)
         if imark == "I" then waction("MARK") end
       end
-      if vsreg then waction("VREG", vsreg); wputxb(2) end
     end
     if riprel then -- Emit rip-relative displacement.
       if match("UWSiI", imark) then
@@ -561,16 +632,16 @@ local function wputmrmsib(t, imark, s, vsreg)
   if xreg or band(reg, 7) == 4 then
     wputmodrm(m or 2, s, 4) -- ModRM.
     if m == nil or imark == "I" then waction("MARK") end
-    if vsreg then waction("VREG", vsreg); wputxb(2) end
+    wvreg("modrm.reg", vsreg, psz+1, sk, vxreg or vreg)
     wputmodrm(t.xsc or 0, xreg or 4, reg) -- SIB.
-    if vxreg then waction("VREG", vxreg); wputxb(3) end
-    if vreg then waction("VREG", vreg); wputxb(1) end
+    wvreg("sib.index", vxreg, psz+2, sk, vreg)
+    wvreg("sib.base", vreg, psz+2, sk)
   else
     wputmodrm(m or 2, s, reg) -- ModRM.
     if (imark == "I" and (m == 1 or m == 2)) or
        (m == nil and (vsreg or vreg)) then waction("MARK") end
-    if vsreg then waction("VREG", vsreg); wputxb(2) end
-    if vreg then waction("VREG", vreg); wputxb(1) end
+    wvreg("modrm.reg", vsreg, psz+1, sk, vreg)
+    wvreg("modrm.rm.m", vreg, psz+1, sk)
   end
 
   -- Put displacement.
@@ -881,9 +952,16 @@ end
 --   "m"/"M"   generates ModRM/SIB from the 1st/2nd operand.
 --             The spare 3 bits are either filled with the last hex digit or
 --             the result from a previous "r"/"R". The opcode is restored.
+--   "u"       Use VEX encoding, vvvv unused.
+--   "v"/"V"   Use VEX encoding, vvvv from 1st/2nd operand (the operand is
+--             removed from the list used by future characters).
+--   "w"       Use VEX encoding, vvvv from 3rd operand.
+--   "L"       Force VEX.L
 --
 -- All of the following characters force a flush of the opcode:
 --   "o"/"O"   stores a pure 32 bit disp (offset) from the 1st/2nd operand.
+--   "s"       stores a 4 bit immediate from the last register operand,
+--             followed by 4 zero bits.
 --   "S"       stores a signed 8 bit immediate from the last operand.
 --   "U"       stores an unsigned 8 bit immediate from the last operand.
 --   "W"       stores an unsigned 16 bit immediate from the last operand.
@@ -1226,46 +1304,14 @@ local map_op = {
   movups_2 =    "rmo:0F10rM|mro:0F11Rm",
   orpd_2 =      "rmo:660F56rM",
   orps_2 =      "rmo:0F56rM",
-  packssdw_2 =  "rmo:660F6BrM",
-  packsswb_2 =  "rmo:660F63rM",
-  packuswb_2 =  "rmo:660F67rM",
-  paddb_2 =     "rmo:660FFCrM",
-  paddd_2 =     "rmo:660FFErM",
-  paddq_2 =     "rmo:660FD4rM",
-  paddsb_2 =    "rmo:660FECrM",
-  paddsw_2 =    "rmo:660FEDrM",
-  paddusb_2 =   "rmo:660FDCrM",
-  paddusw_2 =   "rmo:660FDDrM",
-  paddw_2 =     "rmo:660FFDrM",
-  pand_2 =      "rmo:660FDBrM",
-  pandn_2 =     "rmo:660FDFrM",
   pause_0 =     "F390",
-  pavgb_2 =     "rmo:660FE0rM",
-  pavgw_2 =     "rmo:660FE3rM",
-  pcmpeqb_2 =   "rmo:660F74rM",
-  pcmpeqd_2 =   "rmo:660F76rM",
-  pcmpeqw_2 =   "rmo:660F75rM",
-  pcmpgtb_2 =   "rmo:660F64rM",
-  pcmpgtd_2 =   "rmo:660F66rM",
-  pcmpgtw_2 =   "rmo:660F65rM",
   pextrw_3 =    "rri/do:660FC5rMU|xri/wo:660F3A15nRmU", -- Mem op: SSE4.1 only.
   pinsrw_3 =    "rri/od:660FC4rMU|rxi/ow:",
-  pmaddwd_2 =   "rmo:660FF5rM",
-  pmaxsw_2 =    "rmo:660FEErM",
-  pmaxub_2 =    "rmo:660FDErM",
-  pminsw_2 =    "rmo:660FEArM",
-  pminub_2 =    "rmo:660FDArM",
   pmovmskb_2 =  "rr/do:660FD7rM",
-  pmulhuw_2 =   "rmo:660FE4rM",
-  pmulhw_2 =    "rmo:660FE5rM",
-  pmullw_2 =    "rmo:660FD5rM",
-  pmuludq_2 =   "rmo:660FF4rM",
-  por_2 =       "rmo:660FEBrM",
   prefetchnta_1 = "xb:n0F180m",
   prefetcht0_1 = "xb:n0F181m",
   prefetcht1_1 = "xb:n0F182m",
   prefetcht2_1 = "xb:n0F183m",
-  psadbw_2 =    "rmo:660FF6rM",
   pshufd_3 =    "rmio:660F70rMU",
   pshufhw_3 =   "rmio:F30F70rMU",
   pshuflw_3 =   "rmio:F20F70rMU",
@@ -1279,23 +1325,6 @@ local map_op = {
   psrldq_2 =    "rio:660F733mU",
   psrlq_2 =     "rmo:660FD3rM|rio:660F732mU",
   psrlw_2 =     "rmo:660FD1rM|rio:660F712mU",
-  psubb_2 =     "rmo:660FF8rM",
-  psubd_2 =     "rmo:660FFArM",
-  psubq_2 =     "rmo:660FFBrM",
-  psubsb_2 =    "rmo:660FE8rM",
-  psubsw_2 =    "rmo:660FE9rM",
-  psubusb_2 =   "rmo:660FD8rM",
-  psubusw_2 =   "rmo:660FD9rM",
-  psubw_2 =     "rmo:660FF9rM",
-  punpckhbw_2 = "rmo:660F68rM",
-  punpckhdq_2 = "rmo:660F6ArM",
-  punpckhqdq_2 = "rmo:660F6DrM",
-  punpckhwd_2 = "rmo:660F69rM",
-  punpcklbw_2 = "rmo:660F60rM",
-  punpckldq_2 = "rmo:660F62rM",
-  punpcklqdq_2 = "rmo:660F6CrM",
-  punpcklwd_2 = "rmo:660F61rM",
-  pxor_2 =      "rmo:660FEFrM",
   rcpps_2 =     "rmo:0F53rM",
   rcpss_2 =     "rro:F30F53rM|rx/od:",
   rsqrtps_2 =   "rmo:0F52rM",
@@ -1413,6 +1442,327 @@ local map_op = {
   movntsd_2 =   "xr/qo:nF20F2BRm",
   movntss_2 =   "xr/do:F30F2BRm",
   -- popcnt is also in SSE4.2
+
+  -- AES-NI
+  aesdec_2 =    "rmo:660F38DErM",
+  aesdeclast_2 = "rmo:660F38DFrM",
+  aesenc_2 =    "rmo:660F38DCrM",
+  aesenclast_2 = "rmo:660F38DDrM",
+  aesimc_2 =    "rmo:660F38DBrM",
+  aeskeygenassist_3 = "rmio:660F3ADFrMU",
+  pclmulqdq_3 = "rmio:660F3A44rMU",
+
+   -- AVX FP ops
+  vaddsubpd_3 = "rrmoy:660FVD0rM",
+  vaddsubps_3 = "rrmoy:F20FVD0rM",
+  vandpd_3 =    "rrmoy:660FV54rM",
+  vandps_3 =    "rrmoy:0FV54rM",
+  vandnpd_3 =   "rrmoy:660FV55rM",
+  vandnps_3 =   "rrmoy:0FV55rM",
+  vblendpd_4 =  "rrmioy:660F3AV0DrMU",
+  vblendps_4 =  "rrmioy:660F3AV0CrMU",
+  vblendvpd_4 = "rrmroy:660F3AV4BrMs",
+  vblendvps_4 = "rrmroy:660F3AV4ArMs",
+  vbroadcastf128_2 = "rx/yo:660F38u1ArM",
+  vcmppd_4 =    "rrmioy:660FVC2rMU",
+  vcmpps_4 =    "rrmioy:0FVC2rMU",
+  vcmpsd_4 =    "rrrio:F20FVC2rMU|rrxi/ooq:",
+  vcmpss_4 =    "rrrio:F30FVC2rMU|rrxi/ood:",
+  vcomisd_2 =   "rro:660Fu2FrM|rx/oq:",
+  vcomiss_2 =   "rro:0Fu2FrM|rx/od:",
+  vcvtdq2pd_2 = "rro:F30FuE6rM|rx/oq:|rm/yo:",
+  vcvtdq2ps_2 = "rmoy:0Fu5BrM",
+  vcvtpd2dq_2 = "rmoy:F20FuE6rM",
+  vcvtpd2ps_2 = "rmoy:660Fu5ArM",
+  vcvtps2dq_2 = "rmoy:660Fu5BrM",
+  vcvtps2pd_2 = "rro:0Fu5ArM|rx/oq:|rm/yo:",
+  vcvtsd2si_2 = "rr/do:F20Fu2DrM|rx/dq:|rr/qo:|rxq:",
+  vcvtsd2ss_3 = "rrro:F20FV5ArM|rrx/ooq:",
+  vcvtsi2sd_3 = "rrm/ood:F20FV2ArM|rrm/ooq:F20FVX2ArM",
+  vcvtsi2ss_3 = "rrm/ood:F30FV2ArM|rrm/ooq:F30FVX2ArM",
+  vcvtss2sd_3 = "rrro:F30FV5ArM|rrx/ood:",
+  vcvtss2si_2 = "rr/do:F30Fu2DrM|rxd:|rr/qo:|rx/qd:",
+  vcvttpd2dq_2 = "rmo:660FuE6rM|rm/oy:660FuLE6rM",
+  vcvttps2dq_2 = "rmoy:F30Fu5BrM",
+  vcvttsd2si_2 = "rr/do:F20Fu2CrM|rx/dq:|rr/qo:|rxq:",
+  vcvttss2si_2 = "rr/do:F30Fu2CrM|rxd:|rr/qo:|rx/qd:",
+  vdppd_4 =     "rrmio:660F3AV41rMU",
+  vdpps_4 =     "rrmioy:660F3AV40rMU",
+  vextractf128_3 = "mri/oy:660F3AuL19RmU",
+  vextractps_3 = "mri/do:660F3Au17RmU",
+  vhaddpd_3 =   "rrmoy:660FV7CrM",
+  vhaddps_3 =   "rrmoy:F20FV7CrM",
+  vhsubpd_3 =   "rrmoy:660FV7DrM",
+  vhsubps_3 =   "rrmoy:F20FV7DrM",
+  vinsertf128_4 = "rrmi/yyo:660F3AV18rMU",
+  vinsertps_4 = "rrrio:660F3AV21rMU|rrxi/ood:",
+  vldmxcsr_1 =  "xd:0FuAE2m",
+  vmaskmovps_3 = "rrxoy:660F38V2CrM|xrroy:660F38V2ERm",
+  vmaskmovpd_3 = "rrxoy:660F38V2DrM|xrroy:660F38V2FRm",
+  vmovapd_2 =   "rmoy:660Fu28rM|mroy:660Fu29Rm",
+  vmovaps_2 =   "rmoy:0Fu28rM|mroy:0Fu29Rm",
+  vmovd_2 =     "rm/od:660Fu6ErM|rm/oq:660FuX6ErM|mr/do:660Fu7ERm|mr/qo:",
+  vmovq_2 =     "rro:F30Fu7ErM|rx/oq:|xr/qo:660FuD6Rm",
+  vmovddup_2 =  "rmy:F20Fu12rM|rro:|rx/oq:",
+  vmovhlps_3 =  "rrro:0FV12rM",
+  vmovhpd_2 =   "xr/qo:660Fu17Rm",
+  vmovhpd_3 =   "rrx/ooq:660FV16rM",
+  vmovhps_2 =   "xr/qo:0Fu17Rm",
+  vmovhps_3 =   "rrx/ooq:0FV16rM",
+  vmovlhps_3 =  "rrro:0FV16rM",
+  vmovlpd_2 =   "xr/qo:660Fu13Rm",
+  vmovlpd_3 =   "rrx/ooq:660FV12rM",
+  vmovlps_2 =   "xr/qo:0Fu13Rm",
+  vmovlps_3 =   "rrx/ooq:0FV12rM",
+  vmovmskpd_2 = "rr/do:660Fu50rM|rr/dy:660FuL50rM",
+  vmovmskps_2 = "rr/do:0Fu50rM|rr/dy:0FuL50rM",
+  vmovntpd_2 =  "xroy:660Fu2BRm",
+  vmovntps_2 =  "xroy:0Fu2BRm",
+  vmovsd_2 =    "rx/oq:F20Fu10rM|xr/qo:F20Fu11Rm",
+  vmovsd_3 =    "rrro:F20FV10rM",
+  vmovshdup_2 = "rmoy:F30Fu16rM",
+  vmovsldup_2 = "rmoy:F30Fu12rM",
+  vmovss_2 =    "rx/od:F30Fu10rM|xr/do:F30Fu11Rm",
+  vmovss_3 =    "rrro:F30FV10rM",
+  vmovupd_2 =   "rmoy:660Fu10rM|mroy:660Fu11Rm",
+  vmovups_2 =   "rmoy:0Fu10rM|mroy:0Fu11Rm",
+  vorpd_3 =     "rrmoy:660FV56rM",
+  vorps_3 =     "rrmoy:0FV56rM",
+  vpermilpd_3 = "rrmoy:660F38V0DrM|rmioy:660F3Au05rMU",
+  vpermilps_3 = "rrmoy:660F38V0CrM|rmioy:660F3Au04rMU",
+  vperm2f128_4 = "rrmiy:660F3AV06rMU",
+  vptestpd_2 =  "rmoy:660F38u0FrM",
+  vptestps_2 =  "rmoy:660F38u0ErM",
+  vrcpps_2 =    "rmoy:0Fu53rM",
+  vrcpss_3 =    "rrro:F30FV53rM|rrx/ood:",
+  vrsqrtps_2 =  "rmoy:0Fu52rM",
+  vrsqrtss_3 =  "rrro:F30FV52rM|rrx/ood:",
+  vroundpd_3 =  "rmioy:660F3Au09rMU",
+  vroundps_3 =  "rmioy:660F3Au08rMU",
+  vroundsd_4 =  "rrrio:660F3AV0BrMU|rrxi/ooq:",
+  vroundss_4 =  "rrrio:660F3AV0ArMU|rrxi/ood:",
+  vshufpd_4 =   "rrmioy:660FVC6rMU",
+  vshufps_4 =   "rrmioy:0FVC6rMU",
+  vsqrtps_2 =   "rmoy:0Fu51rM",
+  vsqrtss_2 =   "rro:F30Fu51rM|rx/od:",
+  vsqrtpd_2 =   "rmoy:660Fu51rM",
+  vsqrtsd_2 =   "rro:F20Fu51rM|rx/oq:",
+  vstmxcsr_1 =  "xd:0FuAE3m",
+  vucomisd_2 =  "rro:660Fu2ErM|rx/oq:",
+  vucomiss_2 =  "rro:0Fu2ErM|rx/od:",
+  vunpckhpd_3 = "rrmoy:660FV15rM",
+  vunpckhps_3 = "rrmoy:0FV15rM",
+  vunpcklpd_3 = "rrmoy:660FV14rM",
+  vunpcklps_3 = "rrmoy:0FV14rM",
+  vxorpd_3 =    "rrmoy:660FV57rM",
+  vxorps_3 =    "rrmoy:0FV57rM",
+  vzeroall_0 =  "0FuL77",
+  vzeroupper_0 = "0Fu77",
+
+  -- AVX2 FP ops
+  vbroadcastss_2 = "rx/od:660F38u18rM|rx/yd:|rro:|rr/yo:",
+  vbroadcastsd_2 = "rx/yq:660F38u19rM|rr/yo:",
+  -- *vgather* (!vsib)
+  vpermpd_3 =   "rmiy:660F3AuX01rMU",
+  vpermps_3 =   "rrmy:660F38V16rM",
+
+  -- AVX, AVX2 integer ops
+  -- In general, xmm requires AVX, ymm requires AVX2.
+  vaesdec_3 =  "rrmo:660F38VDErM",
+  vaesdeclast_3 = "rrmo:660F38VDFrM",
+  vaesenc_3 =  "rrmo:660F38VDCrM",
+  vaesenclast_3 = "rrmo:660F38VDDrM",
+  vaesimc_2 =  "rmo:660F38uDBrM",
+  vaeskeygenassist_3 = "rmio:660F3AuDFrMU",
+  vlddqu_2 =    "rxoy:F20FuF0rM",
+  vmaskmovdqu_2 = "rro:660FuF7rM",
+  vmovdqa_2 =   "rmoy:660Fu6FrM|mroy:660Fu7FRm",
+  vmovdqu_2 =   "rmoy:F30Fu6FrM|mroy:F30Fu7FRm",
+  vmovntdq_2 =  "xroy:660FuE7Rm",
+  vmovntdqa_2 = "rxoy:660F38u2ArM",
+  vmpsadbw_4 =  "rrmioy:660F3AV42rMU",
+  vpabsb_2 =    "rmoy:660F38u1CrM",
+  vpabsd_2 =    "rmoy:660F38u1ErM",
+  vpabsw_2 =    "rmoy:660F38u1DrM",
+  vpackusdw_3 = "rrmoy:660F38V2BrM",
+  vpalignr_4 =  "rrmioy:660F3AV0FrMU",
+  vpblendvb_4 = "rrmroy:660F3AV4CrMs",
+  vpblendw_4 =  "rrmioy:660F3AV0ErMU",
+  vpclmulqdq_4 = "rrmio:660F3AV44rMU",
+  vpcmpeqq_3 =  "rrmoy:660F38V29rM",
+  vpcmpestri_3 = "rmio:660F3Au61rMU",
+  vpcmpestrm_3 = "rmio:660F3Au60rMU",
+  vpcmpgtq_3 =  "rrmoy:660F38V37rM",
+  vpcmpistri_3 = "rmio:660F3Au63rMU",
+  vpcmpistrm_3 = "rmio:660F3Au62rMU",
+  vpextrb_3 =   "rri/do:660F3Au14nRmU|rri/qo:|xri/bo:",
+  vpextrw_3 =   "rri/do:660FuC5rMU|xri/wo:660F3Au15nRmU",
+  vpextrd_3 =   "mri/do:660F3Au16RmU",
+  vpextrq_3 =   "mri/qo:660F3Au16RmU",
+  vphaddw_3 =   "rrmoy:660F38V01rM",
+  vphaddd_3 =   "rrmoy:660F38V02rM",
+  vphaddsw_3 =  "rrmoy:660F38V03rM",
+  vphminposuw_2 = "rmo:660F38u41rM",
+  vphsubw_3 =   "rrmoy:660F38V05rM",
+  vphsubd_3 =   "rrmoy:660F38V06rM",
+  vphsubsw_3 =  "rrmoy:660F38V07rM",
+  vpinsrb_4 =   "rrri/ood:660F3AV20rMU|rrxi/oob:",
+  vpinsrw_4 =   "rrri/ood:660FVC4rMU|rrxi/oow:",
+  vpinsrd_4 =   "rrmi/ood:660F3AV22rMU",
+  vpinsrq_4 =   "rrmi/ooq:660F3AVX22rMU",
+  vpmaddubsw_3 = "rrmoy:660F38V04rM",
+  vpmaxsb_3 =   "rrmoy:660F38V3CrM",
+  vpmaxsd_3 =   "rrmoy:660F38V3DrM",
+  vpmaxuw_3 =   "rrmoy:660F38V3ErM",
+  vpmaxud_3 =   "rrmoy:660F38V3FrM",
+  vpminsb_3 =   "rrmoy:660F38V38rM",
+  vpminsd_3 =   "rrmoy:660F38V39rM",
+  vpminuw_3 =   "rrmoy:660F38V3ArM",
+  vpminud_3 =   "rrmoy:660F38V3BrM",
+  vpmovmskb_2 = "rr/do:660FuD7rM|rr/dy:660FuLD7rM",
+  vpmovsxbw_2 = "rroy:660F38u20rM|rx/oq:|rx/yo:",
+  vpmovsxbd_2 = "rroy:660F38u21rM|rx/od:|rx/yq:",
+  vpmovsxbq_2 = "rroy:660F38u22rM|rx/ow:|rx/yd:",
+  vpmovsxwd_2 = "rroy:660F38u23rM|rx/oq:|rx/yo:",
+  vpmovsxwq_2 = "rroy:660F38u24rM|rx/od:|rx/yq:",
+  vpmovsxdq_2 = "rroy:660F38u25rM|rx/oq:|rx/yo:",
+  vpmovzxbw_2 = "rroy:660F38u30rM|rx/oq:|rx/yo:",
+  vpmovzxbd_2 = "rroy:660F38u31rM|rx/od:|rx/yq:",
+  vpmovzxbq_2 = "rroy:660F38u32rM|rx/ow:|rx/yd:",
+  vpmovzxwd_2 = "rroy:660F38u33rM|rx/oq:|rx/yo:",
+  vpmovzxwq_2 = "rroy:660F38u34rM|rx/od:|rx/yq:",
+  vpmovzxdq_2 = "rroy:660F38u35rM|rx/oq:|rx/yo:",
+  vpmuldq_3 =   "rrmoy:660F38V28rM",
+  vpmulhrsw_3 = "rrmoy:660F38V0BrM",
+  vpmulld_3 =   "rrmoy:660F38V40rM",
+  vpshufb_3 =   "rrmoy:660F38V00rM",
+  vpshufd_3 =   "rmioy:660Fu70rMU",
+  vpshufhw_3 =  "rmioy:F30Fu70rMU",
+  vpshuflw_3 =  "rmioy:F20Fu70rMU",
+  vpsignb_3 =   "rrmoy:660F38V08rM",
+  vpsignw_3 =   "rrmoy:660F38V09rM",
+  vpsignd_3 =   "rrmoy:660F38V0ArM",
+  vpslldq_3 =   "rrioy:660Fv737mU",
+  vpsllw_3 =    "rrmoy:660FVF1rM|rrioy:660Fv716mU",
+  vpslld_3 =    "rrmoy:660FVF2rM|rrioy:660Fv726mU",
+  vpsllq_3 =    "rrmoy:660FVF3rM|rrioy:660Fv736mU",
+  vpsraw_3 =    "rrmoy:660FVE1rM|rrioy:660Fv714mU",
+  vpsrad_3 =    "rrmoy:660FVE2rM|rrioy:660Fv724mU",
+  vpsrldq_3 =   "rrioy:660Fv733mU",
+  vpsrlw_3 =    "rrmoy:660FVD1rM|rrioy:660Fv712mU",
+  vpsrld_3 =    "rrmoy:660FVD2rM|rrioy:660Fv722mU",
+  vpsrlq_3 =    "rrmoy:660FVD3rM|rrioy:660Fv732mU",
+  vptest_2 =    "rmoy:660F38u17rM",
+
+  -- AVX2 integer ops
+  vbroadcasti128_2 = "rx/yo:660F38u5ArM",
+  vinserti128_4 = "rrmi/yyo:660F3AV38rMU",
+  vextracti128_3 = "mri/oy:660F3AuL39RmU",
+  vpblendd_4 =  "rrmioy:660F3AV02rMU",
+  vpbroadcastb_2 = "rro:660F38u78rM|rx/ob:|rr/yo:|rx/yb:",
+  vpbroadcastw_2 = "rro:660F38u79rM|rx/ow:|rr/yo:|rx/yw:",
+  vpbroadcastd_2 = "rro:660F38u58rM|rx/od:|rr/yo:|rx/yd:",
+  vpbroadcastq_2 = "rro:660F38u59rM|rx/oq:|rr/yo:|rx/yq:",
+  vpermd_3 =    "rrmy:660F38V36rM",
+  vpermq_3 =    "rmiy:660F3AuX00rMU",
+  -- *vpgather* (!vsib)
+  vperm2i128_4 = "rrmiy:660F3AV46rMU",
+  vpmaskmovd_3 = "rrxoy:660F38V8CrM|xrroy:660F38V8ERm",
+  vpmaskmovq_3 = "rrxoy:660F38VX8CrM|xrroy:660F38VX8ERm",
+  vpsllvd_3 =   "rrmoy:660F38V47rM",
+  vpsllvq_3 =   "rrmoy:660F38VX47rM",
+  vpsravd_3 =   "rrmoy:660F38V46rM",
+  vpsrlvd_3 =   "rrmoy:660F38V45rM",
+  vpsrlvq_3 =   "rrmoy:660F38VX45rM",
+
+  -- Intel ADX
+  adcx_2 =      "rmqd:660F38F6rM",
+  adox_2 =      "rmqd:F30F38F6rM",
+
+  -- BMI1
+  andn_3 =      "rrmqd:0F38VF2rM",
+  bextr_3 =     "rmrqd:0F38wF7rM",
+  blsi_2 =      "rmqd:0F38vF33m",
+  blsmsk_2 =    "rmqd:0F38vF32m",
+  blsr_2 =      "rmqd:0F38vF31m",
+  tzcnt_2 =     "rmqdw:F30FBCrM",
+
+  -- BMI2
+  bzhi_3 =      "rmrqd:0F38wF5rM",
+  mulx_3 =      "rrmqd:F20F38VF6rM",
+  pdep_3 =      "rrmqd:F20F38VF5rM",
+  pext_3 =      "rrmqd:F30F38VF5rM",
+  rorx_3 =      "rmSqd:F20F3AuF0rMS",
+  sarx_3 =      "rmrqd:F30F38wF7rM",
+  shrx_3 =      "rmrqd:F20F38wF7rM",
+  shlx_3 =      "rmrqd:660F38wF7rM",
+
+  -- FMA3
+  vfmaddsub132pd_3 = "rrmoy:660F38VX96rM",
+  vfmaddsub132ps_3 = "rrmoy:660F38V96rM",
+  vfmaddsub213pd_3 = "rrmoy:660F38VXA6rM",
+  vfmaddsub213ps_3 = "rrmoy:660F38VA6rM",
+  vfmaddsub231pd_3 = "rrmoy:660F38VXB6rM",
+  vfmaddsub231ps_3 = "rrmoy:660F38VB6rM",
+
+  vfmsubadd132pd_3 = "rrmoy:660F38VX97rM",
+  vfmsubadd132ps_3 = "rrmoy:660F38V97rM",
+  vfmsubadd213pd_3 = "rrmoy:660F38VXA7rM",
+  vfmsubadd213ps_3 = "rrmoy:660F38VA7rM",
+  vfmsubadd231pd_3 = "rrmoy:660F38VXB7rM",
+  vfmsubadd231ps_3 = "rrmoy:660F38VB7rM",
+
+  vfmadd132pd_3 = "rrmoy:660F38VX98rM",
+  vfmadd132ps_3 = "rrmoy:660F38V98rM",
+  vfmadd132sd_3 = "rrro:660F38VX99rM|rrx/ooq:",
+  vfmadd132ss_3 = "rrro:660F38V99rM|rrx/ood:",
+  vfmadd213pd_3 = "rrmoy:660F38VXA8rM",
+  vfmadd213ps_3 = "rrmoy:660F38VA8rM",
+  vfmadd213sd_3 = "rrro:660F38VXA9rM|rrx/ooq:",
+  vfmadd213ss_3 = "rrro:660F38VA9rM|rrx/ood:",
+  vfmadd231pd_3 = "rrmoy:660F38VXB8rM",
+  vfmadd231ps_3 = "rrmoy:660F38VB8rM",
+  vfmadd231sd_3 = "rrro:660F38VXB9rM|rrx/ooq:",
+  vfmadd231ss_3 = "rrro:660F38VB9rM|rrx/ood:",
+
+  vfmsub132pd_3 = "rrmoy:660F38VX9ArM",
+  vfmsub132ps_3 = "rrmoy:660F38V9ArM",
+  vfmsub132sd_3 = "rrro:660F38VX9BrM|rrx/ooq:",
+  vfmsub132ss_3 = "rrro:660F38V9BrM|rrx/ood:",
+  vfmsub213pd_3 = "rrmoy:660F38VXAArM",
+  vfmsub213ps_3 = "rrmoy:660F38VAArM",
+  vfmsub213sd_3 = "rrro:660F38VXABrM|rrx/ooq:",
+  vfmsub213ss_3 = "rrro:660F38VABrM|rrx/ood:",
+  vfmsub231pd_3 = "rrmoy:660F38VXBArM",
+  vfmsub231ps_3 = "rrmoy:660F38VBArM",
+  vfmsub231sd_3 = "rrro:660F38VXBBrM|rrx/ooq:",
+  vfmsub231ss_3 = "rrro:660F38VBBrM|rrx/ood:",
+
+  vfnmadd132pd_3 = "rrmoy:660F38VX9CrM",
+  vfnmadd132ps_3 = "rrmoy:660F38V9CrM",
+  vfnmadd132sd_3 = "rrro:660F38VX9DrM|rrx/ooq:",
+  vfnmadd132ss_3 = "rrro:660F38V9DrM|rrx/ood:",
+  vfnmadd213pd_3 = "rrmoy:660F38VXACrM",
+  vfnmadd213ps_3 = "rrmoy:660F38VACrM",
+  vfnmadd213sd_3 = "rrro:660F38VXADrM|rrx/ooq:",
+  vfnmadd213ss_3 = "rrro:660F38VADrM|rrx/ood:",
+  vfnmadd231pd_3 = "rrmoy:660F38VXBCrM",
+  vfnmadd231ps_3 = "rrmoy:660F38VBCrM",
+  vfnmadd231sd_3 = "rrro:660F38VXBDrM|rrx/ooq:",
+  vfnmadd231ss_3 = "rrro:660F38VBDrM|rrx/ood:",
+
+  vfnmsub132pd_3 = "rrmoy:660F38VX9ErM",
+  vfnmsub132ps_3 = "rrmoy:660F38V9ErM",
+  vfnmsub132sd_3 = "rrro:660F38VX9FrM|rrx/ooq:",
+  vfnmsub132ss_3 = "rrro:660F38V9FrM|rrx/ood:",
+  vfnmsub213pd_3 = "rrmoy:660F38VXAErM",
+  vfnmsub213ps_3 = "rrmoy:660F38VAErM",
+  vfnmsub213sd_3 = "rrro:660F38VXAFrM|rrx/ooq:",
+  vfnmsub213ss_3 = "rrro:660F38VAFrM|rrx/ood:",
+  vfnmsub231pd_3 = "rrmoy:660F38VXBErM",
+  vfnmsub231ps_3 = "rrmoy:660F38VBErM",
+  vfnmsub231sd_3 = "rrro:660F38VXBFrM|rrx/ooq:",
+  vfnmsub231ss_3 = "rrro:660F38VBFrM|rrx/ood:",
 }
 
 ------------------------------------------------------------------------------
@@ -1463,28 +1813,58 @@ for cc,n in pairs{ b=0, e=1, be=2, u=3, nb=4, ne=5, nbe=6, nu=7 } do
   map_op["fcmov"..cc.."_2"] = format("Fff:%04XR", nc) -- P6+
 end
 
--- SSE FP arithmetic ops.
+-- SSE / AVX FP arithmetic ops.
 for name,n in pairs{ sqrt = 1, add = 8, mul = 9,
                      sub = 12, min = 13, div = 14, max = 15 } do
   map_op[name.."ps_2"] = format("rmo:0F5%XrM", n)
   map_op[name.."ss_2"] = format("rro:F30F5%XrM|rx/od:", n)
   map_op[name.."pd_2"] = format("rmo:660F5%XrM", n)
   map_op[name.."sd_2"] = format("rro:F20F5%XrM|rx/oq:", n)
+  if n ~= 1 then
+    map_op["v"..name.."ps_3"] = format("rrmoy:0FV5%XrM", n)
+    map_op["v"..name.."ss_3"] = format("rrro:F30FV5%XrM|rrx/ood:", n)
+    map_op["v"..name.."pd_3"] = format("rrmoy:660FV5%XrM", n)
+    map_op["v"..name.."sd_3"] = format("rrro:F20FV5%XrM|rrx/ooq:", n)
+  end
+end
+
+-- SSE2 / AVX / AVX2 integer arithmetic ops (66 0F leaf).
+for name,n in pairs{
+  paddb = 0xFC, paddw = 0xFD, paddd = 0xFE, paddq = 0xD4,
+  paddsb = 0xEC, paddsw = 0xED, packssdw = 0x6B,
+  packsswb = 0x63, packuswb = 0x67, paddusb = 0xDC,
+  paddusw = 0xDD, pand = 0xDB, pandn = 0xDF, pavgb = 0xE0,
+  pavgw = 0xE3, pcmpeqb = 0x74, pcmpeqd = 0x76,
+  pcmpeqw = 0x75, pcmpgtb = 0x64, pcmpgtd = 0x66,
+  pcmpgtw = 0x65, pmaddwd = 0xF5, pmaxsw = 0xEE,
+  pmaxub = 0xDE, pminsw = 0xEA, pminub = 0xDA,
+  pmulhuw = 0xE4, pmulhw = 0xE5, pmullw = 0xD5,
+  pmuludq = 0xF4, por = 0xEB, psadbw = 0xF6, psubb = 0xF8,
+  psubw = 0xF9, psubd = 0xFA, psubq = 0xFB, psubsb = 0xE8,
+  psubsw = 0xE9, psubusb = 0xD8, psubusw = 0xD9,
+  punpckhbw = 0x68, punpckhwd = 0x69, punpckhdq = 0x6A,
+  punpckhqdq = 0x6D, punpcklbw = 0x60, punpcklwd = 0x61,
+  punpckldq = 0x62, punpcklqdq = 0x6C, pxor = 0xEF
+} do
+  map_op[name.."_2"] = format("rmo:660F%02XrM", n)
+  map_op["v"..name.."_3"] = format("rrmoy:660FV%02XrM", n)
 end
 
 ------------------------------------------------------------------------------
 
+local map_vexarg = { u = false, v = 1, V = 2, w = 3 }
+
 -- Process pattern string.
 local function dopattern(pat, args, sz, op, needrex)
-  local digit, addin
+  local digit, addin, vex
   local opcode = 0
   local szov = sz
   local narg = 1
   local rex = 0
 
   -- Limit number of section buffer positions used by a single dasm_put().
-  -- A single opcode needs a maximum of 5 positions.
-  if secpos+5 > maxsecpos then wflush() end
+  -- A single opcode needs a maximum of 6 positions.
+  if secpos+6 > maxsecpos then wflush() end
 
   -- Process each character.
   for c in gmatch(pat.."|", ".") do
@@ -1498,6 +1878,8 @@ local function dopattern(pat, args, sz, op, needrex)
       szov = nil
     elseif c == "X" then        -- Force REX.W.
       rex = 8
+    elseif c == "L" then        -- Force VEX.L.
+      vex.l = true
     elseif c == "r" then        -- Merge 1st operand regno. into opcode.
       addin = args[1]; opcode = opcode + (addin.reg % 8)
       if narg < 2 then narg = 2 end
@@ -1521,21 +1903,42 @@ local function dopattern(pat, args, sz, op, needrex)
       if t.xreg and t.xreg > 7 then rex = rex + 2 end
       if s > 7 then rex = rex + 4 end
       if needrex then rex = rex + 16 end
-      wputop(szov, opcode, rex); opcode = nil
+      local psz, sk = wputop(szov, opcode, rex, vex, s < 0, t.vreg or t.vxreg)
+      opcode = nil
       local imark = sub(pat, -1) -- Force a mark (ugly).
       -- Put ModRM/SIB with regno/last digit as spare.
-      wputmrmsib(t, imark, s, addin and addin.vreg)
+      wputmrmsib(t, imark, s, addin and addin.vreg, psz, sk)
       addin = nil
+    elseif map_vexarg[c] ~= nil then -- Encode using VEX prefix
+      local b = band(opcode, 255); opcode = shr(opcode, 8)
+      local m = 1
+      if b == 0x38 then m = 2
+      elseif b == 0x3a then m = 3 end
+      if m ~= 1 then b = band(opcode, 255); opcode = shr(opcode, 8) end
+      if b ~= 0x0f then
+        werror("expected `0F', `0F38', or `0F3A' to precede `"..c..
+          "' in pattern `"..pat.."' for `"..op.."'")
+      end
+      local v = map_vexarg[c]
+      if v then v = remove(args, v) end
+      b = band(opcode, 255)
+      local p = 0
+      if b == 0x66 then p = 1
+      elseif b == 0xf3 then p = 2
+      elseif b == 0xf2 then p = 3 end
+      if p ~= 0 then opcode = shr(opcode, 8) end
+      if opcode ~= 0 then wputop(nil, opcode, 0); opcode = 0 end
+      vex = { m = m, p = p, v = v }
     else
       if opcode then -- Flush opcode.
         if szov == "q" and rex == 0 then rex = rex + 8 end
         if needrex then rex = rex + 16 end
         if addin and addin.reg == -1 then
-          wputop(szov, opcode - 7, rex)
-          waction("VREG", addin.vreg); wputxb(0)
+          local psz, sk = wputop(szov, opcode - 7, rex, vex, true)
+          wvreg("opcode", addin.vreg, psz, sk)
         else
           if addin and addin.reg > 7 then rex = rex + 1 end
-          wputop(szov, opcode, rex)
+          wputop(szov, opcode, rex, vex)
         end
         opcode = nil
       end
@@ -1572,6 +1975,14 @@ local function dopattern(pat, args, sz, op, needrex)
           else
             wputlabel("REL_", imm, 2)
           end
+        elseif c == "s" then
+          local reg = a.reg
+          if reg < 0 then
+            wputb(0)
+            wvreg("imm.hi", a.vreg)
+          else
+            wputb(shl(reg, 4))
+          end
         else
           werror("bad char `"..c.."' in pattern `"..pat.."' for `"..op.."'")
         end
@@ -1648,11 +2059,14 @@ map_op[".template__"] = function(params, template, nparams)
     if pat == "" then pat = lastpat else lastpat = pat end
     if matchtm(tm, args) then
       local prefix = sub(szm, 1, 1)
-      if prefix == "/" then -- Match both operand sizes.
-        if args[1].opsize == sub(szm, 2, 2) and
-           args[2].opsize == sub(szm, 3, 3) then
-          dopattern(pat, args, sz, params.op, needrex) -- Process pattern.
-          return
+      if prefix == "/" then -- Exactly match leading operand sizes.
+        for i = #szm,1,-1 do
+          if i == 1 then
+            dopattern(pat, args, sz, params.op, needrex) -- Process pattern.
+            return
+          elseif args[i-1].opsize ~= sub(szm, i, i) then
+            break
+          end
         end
       else -- Match common operand size.
         local szp = sz
@@ -1717,8 +2131,8 @@ if x64 then
         rex = a.reg > 7 and 9 or 8
       end
     end
-    wputop(sz, opcode, rex)
-    if vreg then waction("VREG", vreg); wputxb(0) end
+    local psz, sk = wputop(sz, opcode, rex, nil, vreg)
+    wvreg("opcode", vreg, psz, sk)
     waction("IMM_D", format("(unsigned int)(%s)", op64))
     waction("IMM_D", format("(unsigned int)((%s)>>32)", op64))
   end
diff --git a/dynasm/dynasm.lua b/dynasm/dynasm.lua
index 5fda425b..8e85af24 100644
--- a/dynasm/dynasm.lua
+++ b/dynasm/dynasm.lua
@@ -10,9 +10,9 @@
 local _info = {
   name =        "DynASM",
   description = "A dynamic assembler for code generation engines",
-  version =     "1.3.0",
-  vernum =       10300,
-  release =     "2011-05-05",
+  version =     "1.4.0",
+  vernum =       10400,
+  release =     "2015-10-18",
   author =      "Mike Pall",
   url =         "http://luajit.org/dynasm.html",
   license =     "MIT",
@@ -630,6 +630,7 @@ end
 -- Load architecture-specific module.
 local function loadarch(arch)
   if not match(arch, "^[%w_]+$") then return "bad arch name" end
+  _G._map_def = map_def
   local ok, m_arch = pcall(require, "dasm_"..arch)
   if not ok then return "cannot load module: "..m_arch end
   g_arch = m_arch