summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile22
-rw-r--r--README2
-rw-r--r--doc/contact.html8
-rw-r--r--doc/ext_buffer.html689
-rw-r--r--doc/ext_c_api.html8
-rw-r--r--doc/ext_ffi.html8
-rw-r--r--doc/ext_ffi_api.html18
-rw-r--r--doc/ext_ffi_semantics.html35
-rw-r--r--doc/ext_ffi_tutorial.html8
-rw-r--r--doc/ext_jit.html10
-rw-r--r--doc/ext_profiler.html359
-rw-r--r--doc/extensions.html112
-rw-r--r--doc/install.html150
-rw-r--r--doc/luajit.html14
-rw-r--r--doc/running.html18
-rw-r--r--dynasm/dasm_arm.h22
-rw-r--r--dynasm/dasm_arm.lua6
-rw-r--r--dynasm/dasm_arm64.h558
-rw-r--r--dynasm/dasm_arm64.lua1226
-rw-r--r--dynasm/dasm_mips.h52
-rw-r--r--dynasm/dasm_mips.lua684
-rw-r--r--dynasm/dasm_mips64.lua12
-rw-r--r--dynasm/dasm_ppc.h35
-rw-r--r--dynasm/dasm_ppc.lua702
-rw-r--r--dynasm/dasm_proto.h4
-rw-r--r--dynasm/dasm_x86.h79
-rw-r--r--dynasm/dasm_x86.lua648
-rw-r--r--dynasm/dynasm.lua7
-rw-r--r--etc/luajit.pc2
-rw-r--r--src/.gitignore2
-rw-r--r--src/Makefile109
-rw-r--r--src/Makefile.dep263
-rw-r--r--src/host/buildvm.c36
-rw-r--r--src/host/buildvm.h1
-rw-r--r--src/host/buildvm_asm.c63
-rw-r--r--src/host/buildvm_fold.c7
-rw-r--r--src/host/buildvm_lib.c63
-rw-r--r--src/host/buildvm_libbc.h81
-rw-r--r--src/host/buildvm_peobj.c43
-rw-r--r--src/host/genlibbc.lua225
-rw-r--r--src/jit/bc.lua17
-rw-r--r--src/jit/bcsave.lua166
-rw-r--r--src/jit/dis_arm.lua18
-rw-r--r--src/jit/dis_arm64.lua1216
-rw-r--r--src/jit/dis_arm64be.lua12
-rw-r--r--src/jit/dis_mips.lua372
-rw-r--r--src/jit/dis_mips64.lua17
-rw-r--r--src/jit/dis_mips64el.lua17
-rw-r--r--src/jit/dis_mips64r6.lua17
-rw-r--r--src/jit/dis_mips64r6el.lua17
-rw-r--r--src/jit/dis_mipsel.lua15
-rw-r--r--src/jit/dis_ppc.lua18
-rw-r--r--src/jit/dis_x64.lua15
-rw-r--r--src/jit/dis_x86.lua297
-rw-r--r--src/jit/dump.lua76
-rw-r--r--src/jit/p.lua311
-rw-r--r--src/jit/v.lua15
-rw-r--r--src/jit/zone.lua45
-rw-r--r--src/lauxlib.h34
-rw-r--r--src/lib_aux.c117
-rw-r--r--src/lib_base.c156
-rw-r--r--src/lib_bit.c135
-rw-r--r--src/lib_buffer.c360
-rw-r--r--src/lib_debug.c14
-rw-r--r--src/lib_ffi.c86
-rw-r--r--src/lib_io.c57
-rw-r--r--src/lib_jit.c236
-rw-r--r--src/lib_math.c92
-rw-r--r--src/lib_os.c39
-rw-r--r--src/lib_package.c73
-rw-r--r--src/lib_string.c427
-rw-r--r--src/lib_table.c187
-rw-r--r--src/lj.supp41
-rw-r--r--src/lj_alloc.c275
-rw-r--r--src/lj_alloc.h3
-rw-r--r--src/lj_api.c395
-rw-r--r--src/lj_arch.h440
-rw-r--r--src/lj_asm.c1016
-rw-r--r--src/lj_asm_arm.h669
-rw-r--r--src/lj_asm_arm64.h2070
-rw-r--r--src/lj_asm_mips.h1798
-rw-r--r--src/lj_asm_ppc.h923
-rw-r--r--src/lj_asm_x86.h1363
-rw-r--r--src/lj_assert.c28
-rw-r--r--src/lj_bc.h4
-rw-r--r--src/lj_bcdump.h6
-rw-r--r--src/lj_bcread.c158
-rw-r--r--src/lj_bcwrite.c248
-rw-r--r--src/lj_buf.c305
-rw-r--r--src/lj_buf.h198
-rw-r--r--src/lj_carith.c83
-rw-r--r--src/lj_carith.h10
-rw-r--r--src/lj_ccall.c397
-rw-r--r--src/lj_ccall.h49
-rw-r--r--src/lj_ccallback.c280
-rw-r--r--src/lj_cconv.c66
-rw-r--r--src/lj_cconv.h5
-rw-r--r--src/lj_cdata.c67
-rw-r--r--src/lj_cdata.h14
-rw-r--r--src/lj_clib.c47
-rw-r--r--src/lj_cparse.c178
-rw-r--r--src/lj_cparse.h2
-rw-r--r--src/lj_crecord.c392
-rw-r--r--src/lj_crecord.h12
-rw-r--r--src/lj_ctype.c30
-rw-r--r--src/lj_ctype.h27
-rw-r--r--src/lj_debug.c211
-rw-r--r--src/lj_debug.h8
-rw-r--r--src/lj_def.h56
-rw-r--r--src/lj_dispatch.c113
-rw-r--r--src/lj_dispatch.h49
-rw-r--r--src/lj_emit_arm.h71
-rw-r--r--src/lj_emit_arm64.h431
-rw-r--r--src/lj_emit_mips.h161
-rw-r--r--src/lj_emit_ppc.h34
-rw-r--r--src/lj_emit_x86.h200
-rw-r--r--src/lj_err.c714
-rw-r--r--src/lj_err.h19
-rw-r--r--src/lj_errmsg.h23
-rw-r--r--src/lj_ffrecord.c1013
-rw-r--r--src/lj_frame.h160
-rw-r--r--src/lj_func.c18
-rw-r--r--src/lj_gc.c193
-rw-r--r--src/lj_gc.h16
-rw-r--r--src/lj_gdbjit.c55
-rw-r--r--src/lj_ir.c175
-rw-r--r--src/lj_ir.h119
-rw-r--r--src/lj_ircall.h238
-rw-r--r--src/lj_iropt.h17
-rw-r--r--src/lj_jit.h240
-rw-r--r--src/lj_lex.c391
-rw-r--r--src/lj_lex.h23
-rw-r--r--src/lj_lib.c131
-rw-r--r--src/lj_lib.h40
-rw-r--r--src/lj_load.c6
-rw-r--r--src/lj_mcode.c78
-rw-r--r--src/lj_meta.c134
-rw-r--r--src/lj_meta.h1
-rw-r--r--src/lj_obj.c18
-rw-r--r--src/lj_obj.h317
-rw-r--r--src/lj_opt_fold.c626
-rw-r--r--src/lj_opt_loop.c45
-rw-r--r--src/lj_opt_mem.c192
-rw-r--r--src/lj_opt_narrow.c66
-rw-r--r--src/lj_opt_sink.c14
-rw-r--r--src/lj_opt_split.c198
-rw-r--r--src/lj_parse.c301
-rw-r--r--src/lj_prng.c259
-rw-r--r--src/lj_prng.h24
-rw-r--r--src/lj_profile.c371
-rw-r--r--src/lj_profile.h21
-rw-r--r--src/lj_record.c978
-rw-r--r--src/lj_record.h3
-rw-r--r--src/lj_serialize.c539
-rw-r--r--src/lj_serialize.h28
-rw-r--r--src/lj_snap.c271
-rw-r--r--src/lj_snap.h3
-rw-r--r--src/lj_state.c112
-rw-r--r--src/lj_state.h4
-rw-r--r--src/lj_str.c519
-rw-r--r--src/lj_str.h39
-rw-r--r--src/lj_strfmt.c606
-rw-r--r--src/lj_strfmt.h131
-rw-r--r--src/lj_strfmt_num.c592
-rw-r--r--src/lj_strscan.c80
-rw-r--r--src/lj_strscan.h3
-rw-r--r--src/lj_tab.c222
-rw-r--r--src/lj_tab.h34
-rw-r--r--src/lj_target.h9
-rw-r--r--src/lj_target_arm.h5
-rw-r--r--src/lj_target_arm64.h342
-rw-r--r--src/lj_target_mips.h195
-rw-r--r--src/lj_target_ppc.h2
-rw-r--r--src/lj_target_x86.h41
-rw-r--r--src/lj_trace.c291
-rw-r--r--src/lj_trace.h5
-rw-r--r--src/lj_traceerr.h4
-rw-r--r--src/lj_udata.c28
-rw-r--r--src/lj_udata.h3
-rw-r--r--src/lj_vm.h42
-rw-r--r--src/lj_vmevent.c1
-rw-r--r--src/lj_vmevent.h7
-rw-r--r--src/lj_vmmath.c110
-rw-r--r--src/ljamalg.c18
-rw-r--r--src/lua.h11
-rw-r--r--src/luaconf.h10
-rw-r--r--src/luajit.c133
-rw-r--r--src/luajit_rolling.h15
-rw-r--r--src/lualib.h1
-rw-r--r--src/msvcbuild.bat14
-rw-r--r--src/nxbuild.bat160
-rw-r--r--src/ps4build.bat36
-rw-r--r--src/ps5build.bat123
-rw-r--r--src/psvitabuild.bat2
-rw-r--r--src/vm_arm.dasc481
-rw-r--r--src/vm_arm64.dasc4201
-rw-r--r--src/vm_mips.dasc2696
-rw-r--r--src/vm_mips64.dasc5557
-rw-r--r--src/vm_ppc.dasc1700
-rw-r--r--src/vm_ppcspe.dasc3691
-rw-r--r--src/vm_x64.dasc4946
-rw-r--r--src/vm_x86.dasc1788
-rw-r--r--src/xb1build.bat101
-rw-r--r--src/xedkbuild.bat2
204 files changed, 47401 insertions, 13788 deletions
diff --git a/Makefile b/Makefile
index 792d7e56..ae194428 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@
14############################################################################## 14##############################################################################
15 15
16MAJVER= 2 16MAJVER= 2
17MINVER= 0 17MINVER= 1
18ABIVER= 5.1 18ABIVER= 5.1
19 19
20# LuaJIT uses rolling releases. The release version is based on the time of 20# LuaJIT uses rolling releases. The release version is based on the time of
@@ -95,8 +95,12 @@ FILE_SO= libluajit.so
95FILE_MAN= luajit.1 95FILE_MAN= luajit.1
96FILE_PC= luajit.pc 96FILE_PC= luajit.pc
97FILES_INC= lua.h lualib.h lauxlib.h luaconf.h lua.hpp luajit.h 97FILES_INC= lua.h lualib.h lauxlib.h luaconf.h lua.hpp luajit.h
98FILES_JITLIB= bc.lua v.lua dump.lua dis_x86.lua dis_x64.lua dis_arm.lua \ 98FILES_JITLIB= bc.lua bcsave.lua dump.lua p.lua v.lua zone.lua \
99 dis_ppc.lua dis_mips.lua dis_mipsel.lua bcsave.lua vmdef.lua 99 dis_x86.lua dis_x64.lua dis_arm.lua dis_arm64.lua \
100 dis_arm64be.lua dis_ppc.lua dis_mips.lua dis_mipsel.lua \
101 dis_mips64.lua dis_mips64el.lua \
102 dis_mips64r6.lua dis_mips64r6el.lua \
103 vmdef.lua
100 104
101ifeq (,$(findstring Windows,$(OS))) 105ifeq (,$(findstring Windows,$(OS)))
102 HOST_SYS:= $(shell uname -s) 106 HOST_SYS:= $(shell uname -s)
@@ -126,7 +130,7 @@ install: $(INSTALL_DEP)
126 $(MKDIR) $(INSTALL_DIRS) 130 $(MKDIR) $(INSTALL_DIRS)
127 cd src && $(INSTALL_X) $(FILE_T) $(INSTALL_T) 131 cd src && $(INSTALL_X) $(FILE_T) $(INSTALL_T)
128 cd src && test -f $(FILE_A) && $(INSTALL_F) $(FILE_A) $(INSTALL_STATIC) || : 132 cd src && test -f $(FILE_A) && $(INSTALL_F) $(FILE_A) $(INSTALL_STATIC) || :
129 $(RM) $(INSTALL_TSYM) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) 133 $(RM) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2)
130 cd src && test -f $(FILE_SO) && \ 134 cd src && test -f $(FILE_SO) && \
131 $(INSTALL_X) $(FILE_SO) $(INSTALL_DYN) && \ 135 $(INSTALL_X) $(FILE_SO) $(INSTALL_DYN) && \
132 ( $(LDCONFIG) $(INSTALL_LIB) || : ) && \ 136 ( $(LDCONFIG) $(INSTALL_LIB) || : ) && \
@@ -138,12 +142,18 @@ install: $(INSTALL_DEP)
138 $(RM) $(FILE_PC).tmp 142 $(RM) $(FILE_PC).tmp
139 cd src && $(INSTALL_F) $(FILES_INC) $(INSTALL_INC) 143 cd src && $(INSTALL_F) $(FILES_INC) $(INSTALL_INC)
140 cd src/jit && $(INSTALL_F) $(FILES_JITLIB) $(INSTALL_JITLIB) 144 cd src/jit && $(INSTALL_F) $(FILES_JITLIB) $(INSTALL_JITLIB)
141 $(SYMLINK) $(INSTALL_TNAME) $(INSTALL_TSYM)
142 @echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ====" 145 @echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ===="
146 @echo ""
147 @echo "Note: the development releases deliberately do NOT install a symlink for luajit"
148 @echo "You can do this now by running this command (with sudo):"
149 @echo ""
150 @echo " $(SYMLINK) $(INSTALL_TNAME) $(INSTALL_TSYM)"
151 @echo ""
152
143 153
144uninstall: 154uninstall:
145 @echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ====" 155 @echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ===="
146 $(UNINSTALL) $(INSTALL_TSYM) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC) 156 $(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC)
147 for file in $(FILES_JITLIB); do \ 157 for file in $(FILES_JITLIB); do \
148 $(UNINSTALL) $(INSTALL_JITLIB)/$$file; \ 158 $(UNINSTALL) $(INSTALL_JITLIB)/$$file; \
149 done 159 done
diff --git a/README b/README
index dfa7ca94..e4a69265 100644
--- a/README
+++ b/README
@@ -1,4 +1,4 @@
1README for LuaJIT 2.0 1README for LuaJIT 2.1
2--------------------- 2---------------------
3 3
4LuaJIT is a Just-In-Time (JIT) compiler for the Lua programming language. 4LuaJIT is a Just-In-Time (JIT) compiler for the Lua programming language.
diff --git a/doc/contact.html b/doc/contact.html
index c32bc9dc..cc4d8c72 100644
--- a/doc/contact.html
+++ b/doc/contact.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>Contact</title> 4<title>Contact</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -37,9 +37,13 @@
37<a href="ext_ffi_semantics.html">FFI Semantics</a> 37<a href="ext_ffi_semantics.html">FFI Semantics</a>
38</li></ul> 38</li></ul>
39</li><li> 39</li><li>
40<a href="ext_buffer.html">String Buffers</a>
41</li><li>
40<a href="ext_jit.html">jit.* Library</a> 42<a href="ext_jit.html">jit.* Library</a>
41</li><li> 43</li><li>
42<a href="ext_c_api.html">Lua/C API</a> 44<a href="ext_c_api.html">Lua/C API</a>
45</li><li>
46<a href="ext_profiler.html">Profiler</a>
43</li></ul> 47</li></ul>
44</li><li> 48</li><li>
45<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 49<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
diff --git a/doc/ext_buffer.html b/doc/ext_buffer.html
new file mode 100644
index 00000000..bfaa24cb
--- /dev/null
+++ b/doc/ext_buffer.html
@@ -0,0 +1,689 @@
1<!DOCTYPE html>
2<html>
3<head>
4<title>String Buffer Library</title>
5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
9<link rel="stylesheet" type="text/css" href="bluequad-print.css" media="print">
10<style type="text/css">
11.lib {
12 vertical-align: middle;
13 margin-left: 5px;
14 padding: 0 5px;
15 font-size: 60%;
16 border-radius: 5px;
17 background: #c5d5ff;
18 color: #000;
19}
20</style>
21</head>
22<body>
23<div id="site">
24<a href="https://luajit.org"><span>Lua<span id="logo">JIT</span></span></a>
25</div>
26<div id="head">
27<h1>String Buffer Library</h1>
28</div>
29<div id="nav">
30<ul><li>
31<a href="luajit.html">LuaJIT</a>
32<ul><li>
33<a href="https://luajit.org/download.html">Download <span class="ext">&raquo;</span></a>
34</li><li>
35<a href="install.html">Installation</a>
36</li><li>
37<a href="running.html">Running</a>
38</li></ul>
39</li><li>
40<a href="extensions.html">Extensions</a>
41<ul><li>
42<a href="ext_ffi.html">FFI Library</a>
43<ul><li>
44<a href="ext_ffi_tutorial.html">FFI Tutorial</a>
45</li><li>
46<a href="ext_ffi_api.html">ffi.* API</a>
47</li><li>
48<a href="ext_ffi_semantics.html">FFI Semantics</a>
49</li></ul>
50</li><li>
51<a class="current" href="ext_buffer.html">String Buffers</a>
52</li><li>
53<a href="ext_jit.html">jit.* Library</a>
54</li><li>
55<a href="ext_c_api.html">Lua/C API</a>
56</li><li>
57<a href="ext_profiler.html">Profiler</a>
58</li></ul>
59</li><li>
60<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
61</li><li>
62<a href="https://luajit.org/faq.html">FAQ <span class="ext">&raquo;</span></a>
63</li><li>
64<a href="https://luajit.org/list.html">Mailing List <span class="ext">&raquo;</span></a>
65</li></ul>
66</div>
67<div id="main">
68<p>
69The string buffer library allows <b>high-performance manipulation of
70string-like data</b>.
71</p>
72<p>
73Unlike Lua strings, which are constants, string buffers are
74<b>mutable</b> sequences of 8-bit (binary-transparent) characters. Data
75can be stored, formatted and encoded into a string buffer and later
76converted, extracted or decoded.
77</p>
78<p>
79The convenient string buffer API simplifies common string manipulation
80tasks, that would otherwise require creating many intermediate strings.
81String buffers improve performance by eliminating redundant memory
82copies, object creation, string interning and garbage collection
83overhead. In conjunction with the FFI library, they allow zero-copy
84operations.
85</p>
86<p>
87The string buffer library also includes a high-performance
88<a href="serialize">serializer</a> for Lua objects.
89</p>
90
91<h2 id="use">Using the String Buffer Library</h2>
92<p>
93The string buffer library is built into LuaJIT by default, but it's not
94loaded by default. Add this to the start of every Lua file that needs
95one of its functions:
96</p>
97<pre class="code">
98local buffer = require("string.buffer")
99</pre>
100<p>
101The convention for the syntax shown on this page is that <tt>buffer</tt>
102refers to the buffer library and <tt>buf</tt> refers to an individual
103buffer object.
104</p>
105<p>
106Please note the difference between a Lua function call, e.g.
107<tt>buffer.new()</tt> (with a dot) and a Lua method call, e.g.
108<tt>buf:reset()</tt> (with a colon).
109</p>
110
111<h3 id="buffer_object">Buffer Objects</h3>
112<p>
113A buffer object is a garbage-collected Lua object. After creation with
114<tt>buffer.new()</tt>, it can (and should) be reused for many operations.
115When the last reference to a buffer object is gone, it will eventually
116be freed by the garbage collector, along with the allocated buffer
117space.
118</p>
119<p>
120Buffers operate like a FIFO (first-in first-out) data structure. Data
121can be appended (written) to the end of the buffer and consumed (read)
122from the front of the buffer. These operations may be freely mixed.
123</p>
124<p>
125The buffer space that holds the characters is managed automatically
126&mdash; it grows as needed and already consumed space is recycled. Use
127<tt>buffer.new(size)</tt> and <tt>buf:free()</tt>, if you need more
128control.
129</p>
130<p>
131The maximum size of a single buffer is the same as the maximum size of a
132Lua string, which is slightly below two gigabytes. For huge data sizes,
133neither strings nor buffers are the right data structure &mdash; use the
134FFI library to directly map memory or files up to the virtual memory
135limit of your OS.
136</p>
137
138<h3 id="buffer_overview">Buffer Method Overview</h3>
139<ul>
140<li>
141The <tt>buf:put*()</tt>-like methods append (write) characters to the
142end of the buffer.
143</li>
144<li>
145The <tt>buf:get*()</tt>-like methods consume (read) characters from the
146front of the buffer.
147</li>
148<li>
149Other methods, like <tt>buf:tostring()</tt> only read the buffer
150contents, but don't change the buffer.
151</li>
152<li>
153The <tt>buf:set()</tt> method allows zero-copy consumption of a string
154or an FFI cdata object as a buffer.
155</li>
156<li>
157The FFI-specific methods allow zero-copy read/write-style operations or
158modifying the buffer contents in-place. Please check the
159<a href="#ffi_caveats">FFI caveats</a> below, too.
160</li>
161<li>
162Methods that don't need to return anything specific, return the buffer
163object itself as a convenience. This allows method chaining, e.g.:
164<tt>buf:reset():encode(obj)</tt> or <tt>buf:skip(len):get()</tt>
165</li>
166</ul>
167
168<h2 id="create">Buffer Creation and Management</h2>
169
170<h3 id="buffer_new"><tt>local buf = buffer.new([size [,options]])<br>
171local buf = buffer.new([options])</tt></h3>
172<p>
173Creates a new buffer object.
174</p>
175<p>
176The optional <tt>size</tt> argument ensures a minimum initial buffer
177size. This is strictly an optimization when the required buffer size is
178known beforehand. The buffer space will grow as needed, in any case.
179</p>
180<p>
181The optional table <tt>options</tt> sets various
182<a href="#serialize_options">serialization options</a>.
183</p>
184
185<h3 id="buffer_reset"><tt>buf = buf:reset()</tt></h3>
186<p>
187Reset (empty) the buffer. The allocated buffer space is not freed and
188may be reused.
189</p>
190
191<h3 id="buffer_free"><tt>buf = buf:free()</tt></h3>
192<p>
193The buffer space of the buffer object is freed. The object itself
194remains intact, empty and may be reused.
195</p>
196<p>
197Note: you normally don't need to use this method. The garbage collector
198automatically frees the buffer space, when the buffer object is
199collected. Use this method, if you need to free the associated memory
200immediately.
201</p>
202
203<h2 id="write">Buffer Writers</h2>
204
205<h3 id="buffer_put"><tt>buf = buf:put([str|num|obj] [,…])</tt></h3>
206<p>
207Appends a string <tt>str</tt>, a number <tt>num</tt> or any object
208<tt>obj</tt> with a <tt>__tostring</tt> metamethod to the buffer.
209Multiple arguments are appended in the given order.
210</p>
211<p>
212Appending a buffer to a buffer is possible and short-circuited
213internally. But it still involves a copy. Better combine the buffer
214writes to use a single buffer.
215</p>
216
217<h3 id="buffer_putf"><tt>buf = buf:putf(format, …)</tt></h3>
218<p>
219Appends the formatted arguments to the buffer. The <tt>format</tt>
220string supports the same options as <tt>string.format()</tt>.
221</p>
222
223<h3 id="buffer_putcdata"><tt>buf = buf:putcdata(cdata, len)</tt><span class="lib">FFI</span></h3>
224<p>
225Appends the given <tt>len</tt> number of bytes from the memory pointed
226to by the FFI <tt>cdata</tt> object to the buffer. The object needs to
227be convertible to a (constant) pointer.
228</p>
229
230<h3 id="buffer_set"><tt>buf = buf:set(str)<br>
231buf = buf:set(cdata, len)</tt><span class="lib">FFI</span></h3>
232<p>
233This method allows zero-copy consumption of a string or an FFI cdata
234object as a buffer. It stores a reference to the passed string
235<tt>str</tt> or the FFI <tt>cdata</tt> object in the buffer. Any buffer
236space originally allocated is freed. This is <i>not</i> an append
237operation, unlike the <tt>buf:put*()</tt> methods.
238</p>
239<p>
240After calling this method, the buffer behaves as if
241<tt>buf:free():put(str)</tt> or <tt>buf:free():put(cdata,&nbsp;len)</tt>
242had been called. However, the data is only referenced and not copied, as
243long as the buffer is only consumed.
244</p>
245<p>
246In case the buffer is written to later on, the referenced data is copied
247and the object reference is removed (copy-on-write semantics).
248</p>
249<p>
250The stored reference is an anchor for the garbage collector and keeps the
251originally passed string or FFI cdata object alive.
252</p>
253
254<h3 id="buffer_reserve"><tt>ptr, len = buf:reserve(size)</tt><span class="lib">FFI</span><br>
255<tt>buf = buf:commit(used)</tt><span class="lib">FFI</span></h3>
256<p>
257The <tt>reserve</tt> method reserves at least <tt>size</tt> bytes of
258write space in the buffer. It returns an <tt>uint8_t&nbsp;*</tt> FFI
259cdata pointer <tt>ptr</tt> that points to this space.
260</p>
261<p>
262The available length in bytes is returned in <tt>len</tt>. This is at
263least <tt>size</tt> bytes, but may be more to facilitate efficient
264buffer growth. You can either make use of the additional space or ignore
265<tt>len</tt> and only use <tt>size</tt> bytes.
266</p>
267<p>
268The <tt>commit</tt> method appends the <tt>used</tt> bytes of the
269previously returned write space to the buffer data.
270</p>
271<p>
272This pair of methods allows zero-copy use of C read-style APIs:
273</p>
274<pre class="code">
275local MIN_SIZE = 65536
276repeat
277 local ptr, len = buf:reserve(MIN_SIZE)
278 local n = C.read(fd, ptr, len)
279 if n == 0 then break end -- EOF.
280 if n &lt; 0 then error("read error") end
281 buf:commit(n)
282until false
283</pre>
284<p>
285The reserved write space is <i>not</i> initialized. At least the
286<tt>used</tt> bytes <b>must</b> be written to before calling the
287<tt>commit</tt> method. There's no need to call the <tt>commit</tt>
288method, if nothing is added to the buffer (e.g. on error).
289</p>
290
291<h2 id="read">Buffer Readers</h2>
292
293<h3 id="buffer_length"><tt>len = #buf</tt></h3>
294<p>
295Returns the current length of the buffer data in bytes.
296</p>
297
298<h3 id="buffer_concat"><tt>res = str|num|buf .. str|num|buf […]</tt></h3>
299<p>
300The Lua concatenation operator <tt>..</tt> also accepts buffers, just
301like strings or numbers. It always returns a string and not a buffer.
302</p>
303<p>
304Note that although this is supported for convenience, this thwarts one
305of the main reasons to use buffers, which is to avoid string
306allocations. Rewrite it with <tt>buf:put()</tt> and <tt>buf:get()</tt>.
307</p>
308<p>
309Mixing this with unrelated objects that have a <tt>__concat</tt>
310metamethod may not work, since these probably only expect strings.
311</p>
312
313<h3 id="buffer_skip"><tt>buf = buf:skip(len)</tt></h3>
314<p>
315Skips (consumes) <tt>len</tt> bytes from the buffer up to the current
316length of the buffer data.
317</p>
318
319<h3 id="buffer_get"><tt>str, … = buf:get([len|nil] [,…])</tt></h3>
320<p>
321Consumes the buffer data and returns one or more strings. If called
322without arguments, the whole buffer data is consumed. If called with a
323number, up to <tt>len</tt> bytes are consumed. A <tt>nil</tt> argument
324consumes the remaining buffer space (this only makes sense as the last
325argument). Multiple arguments consume the buffer data in the given
326order.
327</p>
328<p>
329Note: a zero length or no remaining buffer data returns an empty string
330and not <tt>nil</tt>.
331</p>
332
333<h3 id="buffer_tostring"><tt>str = buf:tostring()<br>
334str = tostring(buf)</tt></h3>
335<p>
336Creates a string from the buffer data, but doesn't consume it. The
337buffer remains unchanged.
338</p>
339<p>
340Buffer objects also define a <tt>__tostring</tt> metamethod. This means
341buffers can be passed to the global <tt>tostring()</tt> function and
342many other functions that accept this in place of strings. The important
343internal uses in functions like <tt>io.write()</tt> are short-circuited
344to avoid the creation of an intermediate string object.
345</p>
346
347<h3 id="buffer_ref"><tt>ptr, len = buf:ref()</tt><span class="lib">FFI</span></h3>
348<p>
349Returns an <tt>uint8_t&nbsp;*</tt> FFI cdata pointer <tt>ptr</tt> that
350points to the buffer data. The length of the buffer data in bytes is
351returned in <tt>len</tt>.
352</p>
353<p>
354The returned pointer can be directly passed to C functions that expect a
355buffer and a length. You can also do bytewise reads
356(<tt>local&nbsp;x&nbsp;=&nbsp;ptr[i]</tt>) or writes
357(<tt>ptr[i]&nbsp;=&nbsp;0x40</tt>) of the buffer data.
358</p>
359<p>
360In conjunction with the <tt>skip</tt> method, this allows zero-copy use
361of C write-style APIs:
362</p>
363<pre class="code">
364repeat
365 local ptr, len = buf:ref()
366 if len == 0 then break end
367 local n = C.write(fd, ptr, len)
368 if n &lt; 0 then error("write error") end
369 buf:skip(n)
370until n >= len
371</pre>
372<p>
373Unlike Lua strings, buffer data is <i>not</i> implicitly
374zero-terminated. It's not safe to pass <tt>ptr</tt> to C functions that
375expect zero-terminated strings. If you're not using <tt>len</tt>, then
376you're doing something wrong.
377</p>
378
379<h2 id="serialize">Serialization of Lua Objects</h2>
380<p>
381The following functions and methods allow <b>high-speed serialization</b>
382(encoding) of a Lua object into a string and decoding it back to a Lua
383object. This allows convenient storage and transport of <b>structured
384data</b>.
385</p>
386<p>
387The encoded data is in an <a href="#serialize_format">internal binary
388format</a>. The data can be stored in files, binary-transparent
389databases or transmitted to other LuaJIT instances across threads,
390processes or networks.
391</p>
392<p>
393Encoding speed can reach up to 1 Gigabyte/second on a modern desktop- or
394server-class system, even when serializing many small objects. Decoding
395speed is mostly constrained by object creation cost.
396</p>
397<p>
398The serializer handles most Lua types, common FFI number types and
399nested structures. Functions, thread objects, other FFI cdata and full
400userdata cannot be serialized (yet).
401</p>
402<p>
403The encoder serializes nested structures as trees. Multiple references
404to a single object will be stored separately and create distinct objects
405after decoding. Circular references cause an error.
406</p>
407
408<h3 id="serialize_methods">Serialization Functions and Methods</h3>
409
410<h3 id="buffer_encode"><tt>str = buffer.encode(obj)<br>
411buf = buf:encode(obj)</tt></h3>
412<p>
413Serializes (encodes) the Lua object <tt>obj</tt>. The stand-alone
414function returns a string <tt>str</tt>. The buffer method appends the
415encoding to the buffer.
416</p>
417<p>
418<tt>obj</tt> can be any of the supported Lua types &mdash; it doesn't
419need to be a Lua table.
420</p>
421<p>
422This function may throw an error when attempting to serialize
423unsupported object types, circular references or deeply nested tables.
424</p>
425
426<h3 id="buffer_decode"><tt>obj = buffer.decode(str)<br>
427obj = buf:decode()</tt></h3>
428<p>
429The stand-alone function deserializes (decodes) the string
430<tt>str</tt>, the buffer method deserializes one object from the
431buffer. Both return a Lua object <tt>obj</tt>.
432</p>
433<p>
434The returned object may be any of the supported Lua types &mdash;
435even <tt>nil</tt>.
436</p>
437<p>
438This function may throw an error when fed with malformed or incomplete
439encoded data. The stand-alone function throws when there's left-over
440data after decoding a single top-level object. The buffer method leaves
441any left-over data in the buffer.
442</p>
443<p>
444Attempting to deserialize an FFI type will throw an error, if the FFI
445library is not built-in or has not been loaded, yet.
446</p>
447
448<h3 id="serialize_options">Serialization Options</h3>
449<p>
450The <tt>options</tt> table passed to <tt>buffer.new()</tt> may contain
451the following members (all optional):
452</p>
453<ul>
454<li>
455<tt>dict</tt> is a Lua table holding a <b>dictionary of strings</b> that
456commonly occur as table keys of objects you are serializing. These keys
457are compactly encoded as indexes during serialization. A well-chosen
458dictionary saves space and improves serialization performance.
459</li>
460<li>
461<tt>metatable</tt> is a Lua table holding a <b>dictionary of metatables</b>
462for the table objects you are serializing.
463</li>
464</ul>
465<p>
466<tt>dict</tt> needs to be an array of strings and <tt>metatable</tt> needs
467to be an array of tables. Both starting at index 1 and without holes (no
468<tt>nil</tt> in between). The tables are anchored in the buffer object and
469internally modified into a two-way index (don't do this yourself, just pass
470a plain array). The tables must not be modified after they have been passed
471to <tt>buffer.new()</tt>.
472</p>
473<p>
474The <tt>dict</tt> and <tt>metatable</tt> tables used by the encoder and
475decoder must be the same. Put the most common entries at the front. Extend
476at the end to ensure backwards-compatibility &mdash; older encodings can
477then still be read. You may also set some indexes to <tt>false</tt> to
478explicitly drop backwards-compatibility. Old encodings that use these
479indexes will throw an error when decoded.
480</p>
481<p>
482Metatables that are not found in the <tt>metatable</tt> dictionary are
483ignored when encoding. Decoding returns a table with a <tt>nil</tt>
484metatable.
485</p>
486<p>
487Note: parsing and preparation of the options table is somewhat
488expensive. Create a buffer object only once and recycle it for multiple
489uses. Avoid mixing encoder and decoder buffers, since the
490<tt>buf:set()</tt> method frees the already allocated buffer space:
491</p>
492<pre class="code">
493local options = {
494 dict = { "commonly", "used", "string", "keys" },
495}
496local buf_enc = buffer.new(options)
497local buf_dec = buffer.new(options)
498
499local function encode(obj)
500 return buf_enc:reset():encode(obj):get()
501end
502
503local function decode(str)
504 return buf_dec:set(str):decode()
505end
506</pre>
507
508<h3 id="serialize_stream">Streaming Serialization</h3>
509<p>
510In some contexts, it's desirable to do piecewise serialization of large
511datasets, also known as <i>streaming</i>.
512</p>
513<p>
514This serialization format can be safely concatenated and supports streaming.
515Multiple encodings can simply be appended to a buffer and later decoded
516individually:
517</p>
518<pre class="code">
519local buf = buffer.new()
520buf:encode(obj1)
521buf:encode(obj2)
522local copy1 = buf:decode()
523local copy2 = buf:decode()
524</pre>
525<p>
526Here's how to iterate over a stream:
527</p>
528<pre class="code">
529while #buf ~= 0 do
530 local obj = buf:decode()
531 -- Do something with obj.
532end
533</pre>
534<p>
535Since the serialization format doesn't prepend a length to its encoding,
536network applications may need to transmit the length, too.
537</p>
538
539<h3 id="serialize_format">Serialization Format Specification</h3>
540<p>
541This serialization format is designed for <b>internal use</b> by LuaJIT
542applications. Serialized data is upwards-compatible and portable across
543all supported LuaJIT platforms.
544</p>
545<p>
546It's an <b>8-bit binary format</b> and not human-readable. It uses e.g.
547embedded zeroes and stores embedded Lua string objects unmodified, which
548are 8-bit-clean, too. Encoded data can be safely concatenated for
549streaming and later decoded one top-level object at a time.
550</p>
551<p>
552The encoding is reasonably compact, but tuned for maximum performance,
553not for minimum space usage. It compresses well with any of the common
554byte-oriented data compression algorithms.
555</p>
556<p>
557Although documented here for reference, this format is explicitly
558<b>not</b> intended to be a 'public standard' for structured data
559interchange across computer languages (like JSON or MessagePack). Please
560do not use it as such.
561</p>
562<p>
563The specification is given below as a context-free grammar with a
564top-level <tt>object</tt> as the starting point. Alternatives are
565separated by the <tt>|</tt> symbol and <tt>*</tt> indicates repeats.
566Grouping is implicit or indicated by <tt>{…}</tt>. Terminals are
567either plain hex numbers, encoded as bytes, or have a <tt>.format</tt>
568suffix.
569</p>
570<pre>
571object → nil | false | true
572 | null | lightud32 | lightud64
573 | int | num | tab | tab_mt
574 | int64 | uint64 | complex
575 | string
576
577nil → 0x00
578false → 0x01
579true → 0x02
580
581null → 0x03 // NULL lightuserdata
582lightud32 → 0x04 data.I // 32 bit lightuserdata
583lightud64 → 0x05 data.L // 64 bit lightuserdata
584
585int → 0x06 int.I // int32_t
586num → 0x07 double.L
587
588tab → 0x08 // Empty table
589 | 0x09 h.U h*{object object} // Key/value hash
590 | 0x0a a.U a*object // 0-based array
591 | 0x0b a.U a*object h.U h*{object object} // Mixed
592 | 0x0c a.U (a-1)*object // 1-based array
593 | 0x0d a.U (a-1)*object h.U h*{object object} // Mixed
594tab_mt → 0x0e (index-1).U tab // Metatable dict entry
595
596int64 → 0x10 int.L // FFI int64_t
597uint64 → 0x11 uint.L // FFI uint64_t
598complex → 0x12 re.L im.L // FFI complex
599
600string → (0x20+len).U len*char.B
601 | 0x0f (index-1).U // String dict entry
602
603.B = 8 bit
604.I = 32 bit little-endian
605.L = 64 bit little-endian
606.U = prefix-encoded 32 bit unsigned number n:
607 0x00..0xdf → n.B
608 0xe0..0x1fdf → (0xe0|(((n-0xe0)>>8)&0x1f)).B ((n-0xe0)&0xff).B
609 0x1fe0.. → 0xff n.I
610</pre>
611
612<h2 id="error">Error handling</h2>
613<p>
614Many of the buffer methods can throw an error. Out-of-memory or usage
615errors are best caught with an outer wrapper for larger parts of code.
616There's not much one can do after that, anyway.
617</p>
618<p>
619OTOH, you may want to catch some errors individually. Buffer methods need
620to receive the buffer object as the first argument. The Lua colon-syntax
621<tt>obj:method()</tt> does that implicitly. But to wrap a method with
622<tt>pcall()</tt>, the arguments need to be passed like this:
623</p>
624<pre class="code">
625local ok, err = pcall(buf.encode, buf, obj)
626if not ok then
627 -- Handle error in err.
628end
629</pre>
630
631<h2 id="ffi_caveats">FFI caveats</h2>
632<p>
633The string buffer library has been designed to work well together with
634the FFI library. But due to the low-level nature of the FFI library,
635some care needs to be taken:
636</p>
637<p>
638First, please remember that FFI pointers are zero-indexed. The space
639returned by <tt>buf:reserve()</tt> and <tt>buf:ref()</tt> starts at the
640returned pointer and ends before <tt>len</tt> bytes after that.
641</p>
642<p>
643I.e. the first valid index is <tt>ptr[0]</tt> and the last valid index
644is <tt>ptr[len-1]</tt>. If the returned length is zero, there's no valid
645index at all. The returned pointer may even be <tt>NULL</tt>.
646</p>
647<p>
648The space pointed to by the returned pointer is only valid as long as
649the buffer is not modified in any way (neither append, nor consume, nor
650reset, etc.). The pointer is also not a GC anchor for the buffer object
651itself.
652</p>
653<p>
654Buffer data is only guaranteed to be byte-aligned. Casting the returned
655pointer to a data type with higher alignment may cause unaligned
656accesses. It depends on the CPU architecture whether this is allowed or
657not (it's always OK on x86/x64 and mostly OK on other modern
658architectures).
659</p>
660<p>
661FFI pointers or references do not count as GC anchors for an underlying
662object. E.g. an <tt>array</tt> allocated with <tt>ffi.new()</tt> is
663anchored by <tt>buf:set(array,&nbsp;len)</tt>, but not by
664<tt>buf:set(array+offset,&nbsp;len)</tt>. The addition of the offset
665creates a new pointer, even when the offset is zero. In this case, you
666need to make sure there's still a reference to the original array as
667long as its contents are in use by the buffer.
668</p>
669<p>
670Even though each LuaJIT VM instance is single-threaded (but you can
671create multiple VMs), FFI data structures can be accessed concurrently.
672Be careful when reading/writing FFI cdata from/to buffers to avoid
673concurrent accesses or modifications. In particular, the memory
674referenced by <tt>buf:set(cdata,&nbsp;len)</tt> must not be modified
675while buffer readers are working on it. Shared, but read-only memory
676mappings of files are OK, but only if the file does not change.
677</p>
678<br class="flush">
679</div>
680<div id="foot">
681<hr class="hide">
682Copyright &copy; 2005-2023
683<span class="noprint">
684&middot;
685<a href="contact.html">Contact</a>
686</span>
687</div>
688</body>
689</html>
diff --git a/doc/ext_c_api.html b/doc/ext_c_api.html
index 43c82047..d5e6bb60 100644
--- a/doc/ext_c_api.html
+++ b/doc/ext_c_api.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>Lua/C API Extensions</title> 4<title>Lua/C API Extensions</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -37,9 +37,13 @@
37<a href="ext_ffi_semantics.html">FFI Semantics</a> 37<a href="ext_ffi_semantics.html">FFI Semantics</a>
38</li></ul> 38</li></ul>
39</li><li> 39</li><li>
40<a href="ext_buffer.html">String Buffers</a>
41</li><li>
40<a href="ext_jit.html">jit.* Library</a> 42<a href="ext_jit.html">jit.* Library</a>
41</li><li> 43</li><li>
42<a class="current" href="ext_c_api.html">Lua/C API</a> 44<a class="current" href="ext_c_api.html">Lua/C API</a>
45</li><li>
46<a href="ext_profiler.html">Profiler</a>
43</li></ul> 47</li></ul>
44</li><li> 48</li><li>
45<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 49<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
diff --git a/doc/ext_ffi.html b/doc/ext_ffi.html
index 683c0cd0..eaa176b6 100644
--- a/doc/ext_ffi.html
+++ b/doc/ext_ffi.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>FFI Library</title> 4<title>FFI Library</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -37,9 +37,13 @@
37<a href="ext_ffi_semantics.html">FFI Semantics</a> 37<a href="ext_ffi_semantics.html">FFI Semantics</a>
38</li></ul> 38</li></ul>
39</li><li> 39</li><li>
40<a href="ext_buffer.html">String Buffers</a>
41</li><li>
40<a href="ext_jit.html">jit.* Library</a> 42<a href="ext_jit.html">jit.* Library</a>
41</li><li> 43</li><li>
42<a href="ext_c_api.html">Lua/C API</a> 44<a href="ext_c_api.html">Lua/C API</a>
45</li><li>
46<a href="ext_profiler.html">Profiler</a>
43</li></ul> 47</li></ul>
44</li><li> 48</li><li>
45<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 49<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
diff --git a/doc/ext_ffi_api.html b/doc/ext_ffi_api.html
index d5f7032f..500a2143 100644
--- a/doc/ext_ffi_api.html
+++ b/doc/ext_ffi_api.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>ffi.* API Functions</title> 4<title>ffi.* API Functions</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -42,9 +42,13 @@ td.abiparam { font-weight: bold; width: 6em; }
42<a href="ext_ffi_semantics.html">FFI Semantics</a> 42<a href="ext_ffi_semantics.html">FFI Semantics</a>
43</li></ul> 43</li></ul>
44</li><li> 44</li><li>
45<a href="ext_buffer.html">String Buffers</a>
46</li><li>
45<a href="ext_jit.html">jit.* Library</a> 47<a href="ext_jit.html">jit.* Library</a>
46</li><li> 48</li><li>
47<a href="ext_c_api.html">Lua/C API</a> 49<a href="ext_c_api.html">Lua/C API</a>
50</li><li>
51<a href="ext_profiler.html">Profiler</a>
48</li></ul> 52</li></ul>
49</li><li> 53</li><li>
50<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 54<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -458,6 +462,12 @@ otherwise. The following parameters are currently defined:
458<td class="abiparam">eabi</td><td class="abidesc">EABI variant of the standard ABI</td></tr> 462<td class="abiparam">eabi</td><td class="abidesc">EABI variant of the standard ABI</td></tr>
459<tr class="odd"> 463<tr class="odd">
460<td class="abiparam">win</td><td class="abidesc">Windows variant of the standard ABI</td></tr> 464<td class="abiparam">win</td><td class="abidesc">Windows variant of the standard ABI</td></tr>
465<tr class="even">
466<td class="abiparam">pauth</td><td class="abidesc">Pointer authentication ABI</td></tr>
467<tr class="odd">
468<td class="abiparam">uwp</td><td class="abidesc">Universal Windows Platform</td></tr>
469<tr class="even">
470<td class="abiparam">gc64</td><td class="abidesc">64 bit GC references</td></tr>
461</table> 471</table>
462 472
463<h3 id="ffi_os"><tt>ffi.os</tt></h3> 473<h3 id="ffi_os"><tt>ffi.os</tt></h3>
@@ -534,8 +544,8 @@ corresponding ctype.
534The parser for Lua source code treats numeric literals with the 544The parser for Lua source code treats numeric literals with the
535suffixes <tt>LL</tt> or <tt>ULL</tt> as signed or unsigned 64&nbsp;bit 545suffixes <tt>LL</tt> or <tt>ULL</tt> as signed or unsigned 64&nbsp;bit
536integers. Case doesn't matter, but uppercase is recommended for 546integers. Case doesn't matter, but uppercase is recommended for
537readability. It handles both decimal (<tt>42LL</tt>) and hexadecimal 547readability. It handles decimal (<tt>42LL</tt>), hexadecimal
538(<tt>0x2aLL</tt>) literals. 548(<tt>0x2aLL</tt>) and binary (<tt>0b101010LL</tt>) literals.
539</p> 549</p>
540<p> 550<p>
541The imaginary part of complex numbers can be specified by suffixing 551The imaginary part of complex numbers can be specified by suffixing
diff --git a/doc/ext_ffi_semantics.html b/doc/ext_ffi_semantics.html
index 381a2010..5ba82a1e 100644
--- a/doc/ext_ffi_semantics.html
+++ b/doc/ext_ffi_semantics.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>FFI Semantics</title> 4<title>FFI Semantics</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -42,9 +42,13 @@ td.convop { font-style: italic; width: 40%; }
42<a class="current" href="ext_ffi_semantics.html">FFI Semantics</a> 42<a class="current" href="ext_ffi_semantics.html">FFI Semantics</a>
43</li></ul> 43</li></ul>
44</li><li> 44</li><li>
45<a href="ext_buffer.html">String Buffers</a>
46</li><li>
45<a href="ext_jit.html">jit.* Library</a> 47<a href="ext_jit.html">jit.* Library</a>
46</li><li> 48</li><li>
47<a href="ext_c_api.html">Lua/C API</a> 49<a href="ext_c_api.html">Lua/C API</a>
50</li><li>
51<a href="ext_profiler.html">Profiler</a>
48</li></ul> 52</li></ul>
49</li><li> 53</li><li>
50<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 54<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -175,6 +179,8 @@ a <tt>typedef</tt>, except re-declarations will be ignored):
175<tt>uint16_t</tt>, <tt>uint32_t</tt>, <tt>uint64_t</tt>, 179<tt>uint16_t</tt>, <tt>uint32_t</tt>, <tt>uint64_t</tt>,
176<tt>intptr_t</tt>, <tt>uintptr_t</tt>.</li> 180<tt>intptr_t</tt>, <tt>uintptr_t</tt>.</li>
177 181
182<li>From <tt>&lt;unistd.h&gt;</tt> (POSIX): <tt>ssize_t</tt>.</li>
183
178</ul> 184</ul>
179<p> 185<p>
180You're encouraged to use these types in preference to 186You're encouraged to use these types in preference to
@@ -722,6 +728,22 @@ You'll have to explicitly convert a 64&nbsp;bit integer to a Lua
722number (e.g. for regular floating-point calculations) with 728number (e.g. for regular floating-point calculations) with
723<tt>tonumber()</tt>. But note this may incur a precision loss.</li> 729<tt>tonumber()</tt>. But note this may incur a precision loss.</li>
724 730
731<li><b>64&nbsp;bit bitwise operations</b>: the rules for 64&nbsp;bit
732arithmetic operators apply analogously.<br>
733
734Unlike the other <tt>bit.*</tt> operations, <tt>bit.tobit()</tt>
735converts a cdata number via <tt>int64_t</tt> to <tt>int32_t</tt> and
736returns a Lua number.<br>
737
738For <tt>bit.band()</tt>, <tt>bit.bor()</tt> and <tt>bit.bxor()</tt>, the
739conversion to <tt>int64_t</tt> or <tt>uint64_t</tt> applies to
740<em>all</em> arguments, if <em>any</em> argument is a cdata number.<br>
741
742For all other operations, only the first argument is used to determine
743the output type. This implies that a cdata number as a shift count for
744shifts and rotates is accepted, but that alone does <em>not</em> cause
745a cdata number output.
746
725</ul> 747</ul>
726 748
727<h3 id="cdata_comp">Comparisons of cdata objects</h3> 749<h3 id="cdata_comp">Comparisons of cdata objects</h3>
@@ -1193,14 +1215,12 @@ The following operations are currently not compiled and may exhibit
1193suboptimal performance, especially when used in inner loops: 1215suboptimal performance, especially when used in inner loops:
1194</p> 1216</p>
1195<ul> 1217<ul>
1196<li>Bitfield accesses and initializations.</li>
1197<li>Vector operations.</li> 1218<li>Vector operations.</li>
1198<li>Table initializers.</li> 1219<li>Table initializers.</li>
1199<li>Initialization of nested <tt>struct</tt>/<tt>union</tt> types.</li> 1220<li>Initialization of nested <tt>struct</tt>/<tt>union</tt> types.</li>
1200<li>Allocations of variable-length arrays or structs.</li> 1221<li>Non-default initialization of VLA/VLS or large C&nbsp;types
1201<li>Allocations of C&nbsp;types with a size &gt; 128&nbsp;bytes or an 1222(&gt; 128&nbsp;bytes or &gt; 16 array elements).</li>
1202alignment &gt; 8&nbsp;bytes.</li> 1223<li>Bitfield initializations.</li>
1203<li>Conversions from lightuserdata to <tt>void&nbsp;*</tt>.</li>
1204<li>Pointer differences for element sizes that are not a power of 1224<li>Pointer differences for element sizes that are not a power of
1205two.</li> 1225two.</li>
1206<li>Calls to C&nbsp;functions with aggregates passed or returned by 1226<li>Calls to C&nbsp;functions with aggregates passed or returned by
@@ -1216,7 +1236,6 @@ value.</li>
1216Other missing features: 1236Other missing features:
1217</p> 1237</p>
1218<ul> 1238<ul>
1219<li>Bit operations for 64&nbsp;bit types.</li>
1220<li>Arithmetic for <tt>complex</tt> numbers.</li> 1239<li>Arithmetic for <tt>complex</tt> numbers.</li>
1221<li>Passing structs by value to vararg C&nbsp;functions.</li> 1240<li>Passing structs by value to vararg C&nbsp;functions.</li>
1222<li><a href="extensions.html#exceptions">C++ exception interoperability</a> 1241<li><a href="extensions.html#exceptions">C++ exception interoperability</a>
diff --git a/doc/ext_ffi_tutorial.html b/doc/ext_ffi_tutorial.html
index 03b6ec56..a5236f0b 100644
--- a/doc/ext_ffi_tutorial.html
+++ b/doc/ext_ffi_tutorial.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>FFI Tutorial</title> 4<title>FFI Tutorial</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -44,9 +44,13 @@ td.idiomlua b { font-weight: normal; color: #2142bf; }
44<a href="ext_ffi_semantics.html">FFI Semantics</a> 44<a href="ext_ffi_semantics.html">FFI Semantics</a>
45</li></ul> 45</li></ul>
46</li><li> 46</li><li>
47<a href="ext_buffer.html">String Buffers</a>
48</li><li>
47<a href="ext_jit.html">jit.* Library</a> 49<a href="ext_jit.html">jit.* Library</a>
48</li><li> 50</li><li>
49<a href="ext_c_api.html">Lua/C API</a> 51<a href="ext_c_api.html">Lua/C API</a>
52</li><li>
53<a href="ext_profiler.html">Profiler</a>
50</li></ul> 54</li></ul>
51</li><li> 55</li><li>
52<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 56<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
diff --git a/doc/ext_jit.html b/doc/ext_jit.html
index b1dbf36c..dd136d65 100644
--- a/doc/ext_jit.html
+++ b/doc/ext_jit.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>jit.* Library</title> 4<title>jit.* Library</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -37,9 +37,13 @@
37<a href="ext_ffi_semantics.html">FFI Semantics</a> 37<a href="ext_ffi_semantics.html">FFI Semantics</a>
38</li></ul> 38</li></ul>
39</li><li> 39</li><li>
40<a href="ext_buffer.html">String Buffers</a>
41</li><li>
40<a class="current" href="ext_jit.html">jit.* Library</a> 42<a class="current" href="ext_jit.html">jit.* Library</a>
41</li><li> 43</li><li>
42<a href="ext_c_api.html">Lua/C API</a> 44<a href="ext_c_api.html">Lua/C API</a>
45</li><li>
46<a href="ext_profiler.html">Profiler</a>
43</li></ul> 47</li></ul>
44</li><li> 48</li><li>
45<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 49<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -145,7 +149,7 @@ Contains the target OS name:
145<h3 id="jit_arch"><tt>jit.arch</tt></h3> 149<h3 id="jit_arch"><tt>jit.arch</tt></h3>
146<p> 150<p>
147Contains the target architecture name: 151Contains the target architecture name:
148"x86", "x64", "arm", "ppc", "ppcspe", or "mips". 152"x86", "x64", "arm", "arm64", "arm64be", "ppc", "mips", "mipsel", "mips64", "mips64el", "mips64r6", "mips64r6el".
149</p> 153</p>
150 154
151<h2 id="jit_opt"><tt>jit.opt.*</tt> &mdash; JIT compiler optimization control</h2> 155<h2 id="jit_opt"><tt>jit.opt.*</tt> &mdash; JIT compiler optimization control</h2>
diff --git a/doc/ext_profiler.html b/doc/ext_profiler.html
new file mode 100644
index 00000000..81b5d773
--- /dev/null
+++ b/doc/ext_profiler.html
@@ -0,0 +1,359 @@
1<!DOCTYPE html>
2<html>
3<head>
4<title>Profiler</title>
5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
9<link rel="stylesheet" type="text/css" href="bluequad-print.css" media="print">
10</head>
11<body>
12<div id="site">
13<a href="https://luajit.org"><span>Lua<span id="logo">JIT</span></span></a>
14</div>
15<div id="head">
16<h1>Profiler</h1>
17</div>
18<div id="nav">
19<ul><li>
20<a href="luajit.html">LuaJIT</a>
21<ul><li>
22<a href="https://luajit.org/download.html">Download <span class="ext">&raquo;</span></a>
23</li><li>
24<a href="install.html">Installation</a>
25</li><li>
26<a href="running.html">Running</a>
27</li></ul>
28</li><li>
29<a href="extensions.html">Extensions</a>
30<ul><li>
31<a href="ext_ffi.html">FFI Library</a>
32<ul><li>
33<a href="ext_ffi_tutorial.html">FFI Tutorial</a>
34</li><li>
35<a href="ext_ffi_api.html">ffi.* API</a>
36</li><li>
37<a href="ext_ffi_semantics.html">FFI Semantics</a>
38</li></ul>
39</li><li>
40<a href="ext_buffer.html">String Buffers</a>
41</li><li>
42<a href="ext_jit.html">jit.* Library</a>
43</li><li>
44<a href="ext_c_api.html">Lua/C API</a>
45</li><li>
46<a class="current" href="ext_profiler.html">Profiler</a>
47</li></ul>
48</li><li>
49<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
50</li><li>
51<a href="https://luajit.org/faq.html">FAQ <span class="ext">&raquo;</span></a>
52</li><li>
53<a href="https://luajit.org/list.html">Mailing List <span class="ext">&raquo;</span></a>
54</li></ul>
55</div>
56<div id="main">
57<p>
58LuaJIT has an integrated statistical profiler with very low overhead. It
59allows sampling the currently executing stack and other parameters in
60regular intervals.
61</p>
62<p>
63The integrated profiler can be accessed from three levels:
64</p>
65<ul>
66<li>The <a href="#hl_profiler">bundled high-level profiler</a>, invoked by the
67<a href="#j_p"><tt>-jp</tt></a> command line option.</li>
68<li>A <a href="#ll_lua_api">low-level Lua API</a> to control the profiler.</li>
69<li>A <a href="#ll_c_api">low-level C API</a> to control the profiler.</li>
70</ul>
71
72<h2 id="hl_profiler">High-Level Profiler</h2>
73<p>
74The bundled high-level profiler offers basic profiling functionality. It
75generates simple textual summaries or source code annotations. It can be
76accessed with the <a href="#j_p"><tt>-jp</tt></a> command line option
77or from Lua code by loading the underlying <tt>jit.p</tt> module.
78</p>
79<p>
80To cut to the chase &mdash; run this to get a CPU usage profile by
81function name:
82</p>
83<pre class="code">
84luajit -jp myapp.lua
85</pre>
86<p>
87It's <em>not</em> a stated goal of the bundled profiler to add every
88possible option or to cater for special profiling needs. The low-level
89profiler APIs are documented below. They may be used by third-party
90authors to implement advanced functionality, e.g. IDE integration or
91graphical profilers.
92</p>
93<p>
94Note: Sampling works for both interpreted and JIT-compiled code. The
95results for JIT-compiled code may sometimes be surprising. LuaJIT
96heavily optimizes and inlines Lua code &mdash; there's no simple
97one-to-one correspondence between source code lines and the sampled
98machine code.
99</p>
100
101<h3 id="j_p"><tt>-jp=[options[,output]]</tt></h3>
102<p>
103The <tt>-jp</tt> command line option starts the high-level profiler.
104When the application run by the command line terminates, the profiler
105stops and writes the results to <tt>stdout</tt> or to the specified
106<tt>output</tt> file.
107</p>
108<p>
109The <tt>options</tt> argument specifies how the profiling is to be
110performed:
111</p>
112<ul>
113<li><tt>f</tt> &mdash; Stack dump: function name, otherwise module:line.
114This is the default mode.</li>
115<li><tt>F</tt> &mdash; Stack dump: ditto, but dump module:name.</li>
116<li><tt>l</tt> &mdash; Stack dump: module:line.</li>
117<li><tt>&lt;number&gt;</tt> &mdash; stack dump depth (callee &larr;
118caller). Default: 1.</li>
119<li><tt>-&lt;number&gt;</tt> &mdash; Inverse stack dump depth (caller
120&rarr; callee).</li>
121<li><tt>s</tt> &mdash; Split stack dump after first stack level. Implies
122depth&nbsp;&ge;&nbsp;2 or depth&nbsp;&le;&nbsp;-2.</li>
123<li><tt>p</tt> &mdash; Show full path for module names.</li>
124<li><tt>v</tt> &mdash; Show VM states.</li>
125<li><tt>z</tt> &mdash; Show <a href="#jit_zone">zones</a>.</li>
126<li><tt>r</tt> &mdash; Show raw sample counts. Default: show percentages.</li>
127<li><tt>a</tt> &mdash; Annotate excerpts from source code files.</li>
128<li><tt>A</tt> &mdash; Annotate complete source code files.</li>
129<li><tt>G</tt> &mdash; Produce raw output suitable for graphical tools.</li>
130<li><tt>m&lt;number&gt;</tt> &mdash; Minimum sample percentage to be shown.
131Default: 3%.</li>
132<li><tt>i&lt;number&gt;</tt> &mdash; Sampling interval in milliseconds.
133Default: 10ms.<br>
134Note: The actual sampling precision is OS-dependent.</li>
135</ul>
136<p>
137The default output for <tt>-jp</tt> is a list of the most CPU consuming
138spots in the application. Increasing the stack dump depth with (say)
139<tt>-jp=2</tt> may help to point out the main callers or callees of
140hotspots. But sample aggregation is still flat per unique stack dump.
141</p>
142<p>
143To get a two-level view (split view) of callers/callees, use
144<tt>-jp=s</tt> or <tt>-jp=-s</tt>. The percentages shown for the second
145level are relative to the first level.
146</p>
147<p>
148To see how much time is spent in each line relative to a function, use
149<tt>-jp=fl</tt>.
150</p>
151<p>
152To see how much time is spent in different VM states or
153<a href="#jit_zone">zones</a>, use <tt>-jp=v</tt> or <tt>-jp=z</tt>.
154</p>
155<p>
156Combinations of <tt>v/z</tt> with <tt>f/F/l</tt> produce two-level
157views, e.g. <tt>-jp=vf</tt> or <tt>-jp=fv</tt>. This shows the time
158spent in a VM state or zone vs. hotspots. This can be used to answer
159questions like "Which time-consuming functions are only interpreted?" or
160"What's the garbage collector overhead for a specific function?".
161</p>
162<p>
163Multiple options can be combined &mdash; but not all combinations make
164sense, see above. E.g. <tt>-jp=3si4m1</tt> samples three stack levels
165deep in 4ms intervals and shows a split view of the CPU consuming
166functions and their callers with a 1% threshold.
167</p>
168<p>
169Source code annotations produced by <tt>-jp=a</tt> or <tt>-jp=A</tt> are
170always flat and at the line level. Obviously, the source code files need
171to be readable by the profiler script.
172</p>
173<p>
174The high-level profiler can also be started and stopped from Lua code with:
175</p>
176<pre class="code">
177require("jit.p").start(options, output)
178...
179require("jit.p").stop()
180</pre>
181
182<h3 id="jit_zone"><tt>jit.zone</tt> &mdash; Zones</h3>
183<p>
184Zones can be used to provide information about different parts of an
185application to the high-level profiler. E.g. a game could make use of an
186<tt>"AI"</tt> zone, a <tt>"PHYS"</tt> zone, etc. Zones are hierarchical,
187organized as a stack.
188</p>
189<p>
190The <tt>jit.zone</tt> module needs to be loaded explicitly:
191</p>
192<pre class="code">
193local zone = require("jit.zone")
194</pre>
195<ul>
196<li><tt>zone("name")</tt> pushes a named zone to the zone stack.</li>
197<li><tt>zone()</tt> pops the current zone from the zone stack and
198returns its name.</li>
199<li><tt>zone:get()</tt> returns the current zone name or <tt>nil</tt>.</li>
200<li><tt>zone:flush()</tt> flushes the zone stack.</li>
201</ul>
202<p>
203To show the time spent in each zone use <tt>-jp=z</tt>. To show the time
204spent relative to hotspots use e.g. <tt>-jp=zf</tt> or <tt>-jp=fz</tt>.
205</p>
206
207<h2 id="ll_lua_api">Low-level Lua API</h2>
208<p>
209The <tt>jit.profile</tt> module gives access to the low-level API of the
210profiler from Lua code. This module needs to be loaded explicitly:
211<pre class="code">
212local profile = require("jit.profile")
213</pre>
214<p>
215This module can be used to implement your own higher-level profiler.
216A typical profiling run starts the profiler, captures stack dumps in
217the profiler callback, adds them to a hash table to aggregate the number
218of samples, stops the profiler and then analyzes all captured
219stack dumps. Other parameters can be sampled in the profiler callback,
220too. But it's important not to spend too much time in the callback,
221since this may skew the statistics.
222</p>
223
224<h3 id="profile_start"><tt>profile.start(mode, cb)</tt>
225&mdash; Start profiler</h3>
226<p>
227This function starts the profiler. The <tt>mode</tt> argument is a
228string holding options:
229</p>
230<ul>
231<li><tt>f</tt> &mdash; Profile with precision down to the function level.</li>
232<li><tt>l</tt> &mdash; Profile with precision down to the line level.</li>
233<li><tt>i&lt;number&gt;</tt> &mdash; Sampling interval in milliseconds (default
23410ms).</br>
235Note: The actual sampling precision is OS-dependent.
236</li>
237</ul>
238<p>
239The <tt>cb</tt> argument is a callback function which is called with
240three arguments: <tt>(thread, samples, vmstate)</tt>. The callback is
241called on a separate coroutine, the <tt>thread</tt> argument is the
242state that holds the stack to sample for profiling. Note: do
243<em>not</em> modify the stack of that state or call functions on it.
244</p>
245<p>
246<tt>samples</tt> gives the number of accumulated samples since the last
247callback (usually 1).
248</p>
249<p>
250<tt>vmstate</tt> holds the VM state at the time the profiling timer
251triggered. This may or may not correspond to the state of the VM when
252the profiling callback is called. The state is either <tt>'N'</tt>
253native (compiled) code, <tt>'I'</tt> interpreted code, <tt>'C'</tt>
254C&nbsp;code, <tt>'G'</tt> the garbage collector, or <tt>'J'</tt> the JIT
255compiler.
256</p>
257
258<h3 id="profile_stop"><tt>profile.stop()</tt>
259&mdash; Stop profiler</h3>
260<p>
261This function stops the profiler.
262</p>
263
264<h3 id="profile_dump"><tt>dump = profile.dumpstack([thread,] fmt, depth)</tt>
265&mdash; Dump stack </h3>
266<p>
267This function allows taking stack dumps in an efficient manner. It
268returns a string with a stack dump for the <tt>thread</tt> (coroutine),
269formatted according to the <tt>fmt</tt> argument:
270</p>
271<ul>
272<li><tt>p</tt> &mdash; Preserve the full path for module names. Otherwise,
273only the file name is used.</li>
274<li><tt>f</tt> &mdash; Dump the function name if it can be derived. Otherwise,
275use module:line.</li>
276<li><tt>F</tt> &mdash; Ditto, but dump module:name.</li>
277<li><tt>l</tt> &mdash; Dump module:line.</li>
278<li><tt>Z</tt> &mdash; Zap the following characters for the last dumped
279frame.</li>
280<li>All other characters are added verbatim to the output string.</li>
281</ul>
282<p>
283The <tt>depth</tt> argument gives the number of frames to dump, starting
284at the topmost frame of the thread. A negative number dumps the frames in
285inverse order.
286</p>
287<p>
288The first example prints a list of the current module names and line
289numbers of up to 10 frames in separate lines. The second example prints
290semicolon-separated function names for all frames (up to 100) in inverse
291order:
292</p>
293<pre class="code">
294print(profile.dumpstack(thread, "l\n", 10))
295print(profile.dumpstack(thread, "lZ;", -100))
296</pre>
297
298<h2 id="ll_c_api">Low-level C API</h2>
299<p>
300The profiler can be controlled directly from C&nbsp;code, e.g. for
301use by IDEs. The declarations are in <tt>"luajit.h"</tt> (see
302<a href="ext_c_api.html">Lua/C API</a> extensions).
303</p>
304
305<h3 id="luaJIT_profile_start"><tt>luaJIT_profile_start(L, mode, cb, data)</tt>
306&mdash; Start profiler</h3>
307<p>
308This function starts the profiler. <a href="#profile_start">See
309above</a> for a description of the <tt>mode</tt> argument.
310</p>
311<p>
312The <tt>cb</tt> argument is a callback function with the following
313declaration:
314</p>
315<pre class="code">
316typedef void (*luaJIT_profile_callback)(void *data, lua_State *L,
317 int samples, int vmstate);
318</pre>
319<p>
320<tt>data</tt> is available for use by the callback. <tt>L</tt> is the
321state that holds the stack to sample for profiling. Note: do
322<em>not</em> modify this stack or call functions on this stack &mdash;
323use a separate coroutine for this purpose. <a href="#profile_start">See
324above</a> for a description of <tt>samples</tt> and <tt>vmstate</tt>.
325</p>
326
327<h3 id="luaJIT_profile_stop"><tt>luaJIT_profile_stop(L)</tt>
328&mdash; Stop profiler</h3>
329<p>
330This function stops the profiler.
331</p>
332
333<h3 id="luaJIT_profile_dumpstack"><tt>p = luaJIT_profile_dumpstack(L, fmt, depth, len)</tt>
334&mdash; Dump stack </h3>
335<p>
336This function allows taking stack dumps in an efficient manner.
337<a href="#profile_dump">See above</a> for a description of <tt>fmt</tt>
338and <tt>depth</tt>.
339</p>
340<p>
341This function returns a <tt>const&nbsp;char&nbsp;*</tt> pointing to a
342private string buffer of the profiler. The <tt>int&nbsp;*len</tt>
343argument returns the length of the output string. The buffer is
344overwritten on the next call and deallocated when the profiler stops.
345You either need to consume the content immediately or copy it for later
346use.
347</p>
348<br class="flush">
349</div>
350<div id="foot">
351<hr class="hide">
352Copyright &copy; 2005-2023
353<span class="noprint">
354&middot;
355<a href="contact.html">Contact</a>
356</span>
357</div>
358</body>
359</html>
diff --git a/doc/extensions.html b/doc/extensions.html
index f8b45c28..eb591d1e 100644
--- a/doc/extensions.html
+++ b/doc/extensions.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>Extensions</title> 4<title>Extensions</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -54,9 +54,13 @@ td.excinterop {
54<a href="ext_ffi_semantics.html">FFI Semantics</a> 54<a href="ext_ffi_semantics.html">FFI Semantics</a>
55</li></ul> 55</li></ul>
56</li><li> 56</li><li>
57<a href="ext_buffer.html">String Buffers</a>
58</li><li>
57<a href="ext_jit.html">jit.* Library</a> 59<a href="ext_jit.html">jit.* Library</a>
58</li><li> 60</li><li>
59<a href="ext_c_api.html">Lua/C API</a> 61<a href="ext_c_api.html">Lua/C API</a>
62</li><li>
63<a href="ext_profiler.html">Profiler</a>
60</li></ul> 64</li></ul>
61</li><li> 65</li><li>
62<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 66<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -106,6 +110,9 @@ bit.lshift bit.rshift bit.arshift bit.rol bit.ror bit.bswap
106This module is a LuaJIT built-in &mdash; you don't need to download or 110This module is a LuaJIT built-in &mdash; you don't need to download or
107install Lua BitOp. The Lua BitOp site has full documentation for all 111install Lua BitOp. The Lua BitOp site has full documentation for all
108<a href="https://bitop.luajit.org/api.html"><span class="ext">&raquo;</span>&nbsp;Lua BitOp API functions</a>. 112<a href="https://bitop.luajit.org/api.html"><span class="ext">&raquo;</span>&nbsp;Lua BitOp API functions</a>.
113The FFI adds support for
114<a href="ext_ffi_semantics.html#cdata_arith">64&nbsp;bit bitwise operations</a>,
115using the same API functions.
109</p> 116</p>
110<p> 117<p>
111Please make sure to <tt>require</tt> the module before using any of 118Please make sure to <tt>require</tt> the module before using any of
@@ -139,6 +146,11 @@ LuaJIT adds some
139<a href="ext_c_api.html">extra functions to the Lua/C API</a>. 146<a href="ext_c_api.html">extra functions to the Lua/C API</a>.
140</p> 147</p>
141 148
149<h3 id="profiler">Profiler</h3>
150<p>
151LuaJIT has an <a href="ext_profiler.html">integrated profiler</a>.
152</p>
153
142<h2 id="library">Enhanced Standard Library Functions</h2> 154<h2 id="library">Enhanced Standard Library Functions</h2>
143 155
144<h3 id="xpcall"><tt>xpcall(f, err [,args...])</tt> passes arguments</h3> 156<h3 id="xpcall"><tt>xpcall(f, err [,args...])</tt> passes arguments</h3>
@@ -166,7 +178,7 @@ in <tt>"-inf"</tt>.
166<h3 id="tonumber"><tt>tonumber()</tt> etc. use builtin string to number conversion</h3> 178<h3 id="tonumber"><tt>tonumber()</tt> etc. use builtin string to number conversion</h3>
167<p> 179<p>
168All string-to-number conversions consistently convert integer and 180All string-to-number conversions consistently convert integer and
169floating-point inputs in decimal and hexadecimal on all platforms. 181floating-point inputs in decimal, hexadecimal and binary on all platforms.
170<tt>strtod()</tt> is <em>not</em> used anymore, which avoids numerous 182<tt>strtod()</tt> is <em>not</em> used anymore, which avoids numerous
171problems with poor C library implementations. The builtin conversion 183problems with poor C library implementations. The builtin conversion
172function provides full precision according to the IEEE-754 standard, it 184function provides full precision according to the IEEE-754 standard, it
@@ -190,6 +202,36 @@ for dot releases (x.y.0 &rarr; x.y.1), but may change with major or
190minor releases (2.0 &rarr; 2.1) or between any beta release. Foreign 202minor releases (2.0 &rarr; 2.1) or between any beta release. Foreign
191bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded. 203bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded.
192</p> 204</p>
205<p>
206Note: <tt>LJ_GC64</tt> mode requires a different frame layout, which implies
207a different, incompatible bytecode format for all 64 bit ports. This may be
208rectified in the future.
209</p>
210
211<h3 id="table_new"><tt>table.new(narray, nhash)</tt> allocates a pre-sized table</h3>
212<p>
213An extra library function <tt>table.new()</tt> can be made available via
214<tt>require("table.new")</tt>. This creates a pre-sized table, just like
215the C API equivalent <tt>lua_createtable()</tt>. This is useful for big
216tables if the final table size is known and automatic table resizing is
217too expensive.
218</p>
219
220<h3 id="table_clear"><tt>table.clear(tab)</tt> clears a table</h3>
221<p>
222An extra library function <tt>table.clear()</tt> can be made available
223via <tt>require("table.clear")</tt>. This clears all keys and values
224from a table, but preserves the allocated array/hash sizes. This is
225useful when a table, which is linked from multiple places, needs to be
226cleared and/or when recycling a table for use by the same context. This
227avoids managing backlinks, saves an allocation and the overhead of
228incremental array/hash part growth.
229</p>
230<p>
231Please note, this function is meant for very specific situations. In most
232cases it's better to replace the (usually single) link with a new table
233and let the GC do its work.
234</p>
193 235
194<h3 id="math_random">Enhanced PRNG for <tt>math.random()</tt></h3> 236<h3 id="math_random">Enhanced PRNG for <tt>math.random()</tt></h3>
195<p> 237<p>
@@ -268,6 +310,26 @@ indexes for varargs.</li>
268<li><tt>debug.getupvalue()</tt> and <tt>debug.setupvalue()</tt> handle 310<li><tt>debug.getupvalue()</tt> and <tt>debug.setupvalue()</tt> handle
269C&nbsp;functions.</li> 311C&nbsp;functions.</li>
270<li><tt>debug.upvalueid()</tt> and <tt>debug.upvaluejoin()</tt>.</li> 312<li><tt>debug.upvalueid()</tt> and <tt>debug.upvaluejoin()</tt>.</li>
313<li>Lua/C API extensions:
314<tt>lua_version()</tt>
315<tt>lua_upvalueid()</tt>
316<tt>lua_upvaluejoin()</tt>
317<tt>lua_loadx()</tt>
318<tt>lua_copy()</tt>
319<tt>lua_tonumberx()</tt>
320<tt>lua_tointegerx()</tt>
321<tt>luaL_fileresult()</tt>
322<tt>luaL_execresult()</tt>
323<tt>luaL_loadfilex()</tt>
324<tt>luaL_loadbufferx()</tt>
325<tt>luaL_traceback()</tt>
326<tt>luaL_setfuncs()</tt>
327<tt>luaL_pushmodule()</tt>
328<tt>luaL_newlibtable()</tt>
329<tt>luaL_newlib()</tt>
330<tt>luaL_testudata()</tt>
331<tt>luaL_setmetatable()</tt>
332</li>
271<li>Command line option <tt>-E</tt>.</li> 333<li>Command line option <tt>-E</tt>.</li>
272<li>Command line checks <tt>__tostring</tt> for errors.</li> 334<li>Command line checks <tt>__tostring</tt> for errors.</li>
273</ul> 335</ul>
@@ -293,6 +355,8 @@ exit status.</li>
293<li><tt>debug.setmetatable()</tt> returns object.</li> 355<li><tt>debug.setmetatable()</tt> returns object.</li>
294<li><tt>debug.getuservalue()</tt> and <tt>debug.setuservalue()</tt>.</li> 356<li><tt>debug.getuservalue()</tt> and <tt>debug.setuservalue()</tt>.</li>
295<li>Remove <tt>math.mod()</tt>, <tt>string.gfind()</tt>.</li> 357<li>Remove <tt>math.mod()</tt>, <tt>string.gfind()</tt>.</li>
358<li><tt>package.searchers</tt>.</li>
359<li><tt>module()</tt> returns the module table.</li>
296</ul> 360</ul>
297<p> 361<p>
298Note: this provides only partial compatibility with Lua 5.2 at the 362Note: this provides only partial compatibility with Lua 5.2 at the
@@ -301,6 +365,21 @@ Lua&nbsp;5.1, which prevents implementing features that would otherwise
301break the Lua/C API and ABI (e.g. <tt>_ENV</tt>). 365break the Lua/C API and ABI (e.g. <tt>_ENV</tt>).
302</p> 366</p>
303 367
368<h2 id="lua53">Extensions from Lua 5.3</h2>
369<p>
370LuaJIT supports some extensions from Lua&nbsp;5.3:
371<ul>
372<li>Unicode escape <tt>'\u{XX...}'</tt> embeds the UTF-8 encoding in string literals.</li>
373<li>The argument table <tt>arg</tt> can be read (and modified) by <tt>LUA_INIT</tt> and <tt>-e</tt> chunks.</li>
374<li><tt>io.read()</tt> and <tt>file:read()</tt> accept formats with or without a leading <tt>*</tt>.</li>
375<li><tt>assert()</tt> accepts any type of error object.</li>
376<li><tt>table.move(a1, f, e, t [,a2])</tt>.</li>
377<li><tt>coroutine.isyieldable()</tt>.</li>
378<li>Lua/C API extensions:
379<tt>lua_isyieldable()</tt>
380</li>
381</ul>
382
304<h2 id="exceptions">C++ Exception Interoperability</h2> 383<h2 id="exceptions">C++ Exception Interoperability</h2>
305<p> 384<p>
306LuaJIT has built-in support for interoperating with C++&nbsp;exceptions. 385LuaJIT has built-in support for interoperating with C++&nbsp;exceptions.
@@ -314,26 +393,21 @@ the toolchain used to compile LuaJIT:
314<td class="excinterop">Interoperability</td> 393<td class="excinterop">Interoperability</td>
315</tr> 394</tr>
316<tr class="odd separate"> 395<tr class="odd separate">
317<td class="excplatform">POSIX/x64, DWARF2 unwinding</td> 396<td class="excplatform">External frame unwinding</td>
318<td class="exccompiler">GCC 4.3+</td> 397<td class="exccompiler">GCC, Clang, MSVC</td>
319<td class="excinterop"><b style="color: #00a000;">Full</b></td> 398<td class="excinterop"><b style="color: #00a000;">Full</b></td>
320</tr> 399</tr>
321<tr class="even"> 400<tr class="even">
322<td class="excplatform">Other platforms, DWARF2 unwinding</td> 401<td class="excplatform">Internal frame unwinding + DWARF2</td>
323<td class="exccompiler">GCC</td> 402<td class="exccompiler">GCC, Clang</td>
324<td class="excinterop"><b style="color: #c06000;">Limited</b></td> 403<td class="excinterop"><b style="color: #c06000;">Limited</b></td>
325</tr> 404</tr>
326<tr class="odd"> 405<tr class="odd">
327<td class="excplatform">Windows/x64</td> 406<td class="excplatform">Windows 64 bit</td>
328<td class="exccompiler">MSVC</td> 407<td class="exccompiler">non-MSVC</td>
329<td class="excinterop"><b style="color: #00a000;">Full</b></td> 408<td class="excinterop"><b style="color: #c06000;">Limited</b></td>
330</tr> 409</tr>
331<tr class="even"> 410<tr class="even">
332<td class="excplatform">Windows/x86</td>
333<td class="exccompiler">Any</td>
334<td class="excinterop"><b style="color: #a00000;">No</b></td>
335</tr>
336<tr class="odd">
337<td class="excplatform">Other platforms</td> 411<td class="excplatform">Other platforms</td>
338<td class="exccompiler">Other compilers</td> 412<td class="exccompiler">Other compilers</td>
339<td class="excinterop"><b style="color: #a00000;">No</b></td> 413<td class="excinterop"><b style="color: #a00000;">No</b></td>
@@ -384,14 +458,6 @@ C++ destructors.</li>
384<li>Lua errors <b>cannot</b> be caught on the C++ side.</li> 458<li>Lua errors <b>cannot</b> be caught on the C++ side.</li>
385<li>Throwing Lua errors across C++ frames will <b>not</b> call 459<li>Throwing Lua errors across C++ frames will <b>not</b> call
386C++ destructors.</li> 460C++ destructors.</li>
387<li>Additionally, on Windows/x86 with SEH-based C++&nbsp;exceptions:
388it's <b>not</b> safe to throw a Lua error across any frames containing
389a C++ function with any try/catch construct or using variables with
390(implicit) destructors. This also applies to any functions which may be
391inlined in such a function. It doesn't matter whether <tt>lua_error()</tt>
392is called inside or outside of a try/catch or whether any object actually
393needs to be destroyed: the SEH chain is corrupted and this will eventually
394lead to the termination of the process.</li>
395</ul> 461</ul>
396<br class="flush"> 462<br class="flush">
397</div> 463</div>
diff --git a/doc/install.html b/doc/install.html
index 21866315..be721031 100644
--- a/doc/install.html
+++ b/doc/install.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>Installation</title> 4<title>Installation</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -60,9 +60,13 @@ td.compatx {
60<a href="ext_ffi_semantics.html">FFI Semantics</a> 60<a href="ext_ffi_semantics.html">FFI Semantics</a>
61</li></ul> 61</li></ul>
62</li><li> 62</li><li>
63<a href="ext_buffer.html">String Buffers</a>
64</li><li>
63<a href="ext_jit.html">jit.* Library</a> 65<a href="ext_jit.html">jit.* Library</a>
64</li><li> 66</li><li>
65<a href="ext_c_api.html">Lua/C API</a> 67<a href="ext_c_api.html">Lua/C API</a>
68</li><li>
69<a href="ext_profiler.html">Profiler</a>
66</li></ul> 70</li></ul>
67</li><li> 71</li><li>
68<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 72<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -121,6 +125,13 @@ MSVC (Visual Studio).</li>
121Please read the instructions given in these files, before changing 125Please read the instructions given in these files, before changing
122any settings. 126any settings.
123</p> 127</p>
128<p>
129All LuaJIT 64 bit ports use 64 bit GC objects by default (<tt>LJ_GC64</tt>).
130For x64, you can select the old 32-on-64 bit mode by adding
131<tt>XCFLAGS=-DLUAJIT_DISABLE_GC64</tt> to the make command.
132Please check the note about the
133<a href="extensions.html#string_dump">bytecode format</a> differences, too.
134</p>
124 135
125<h2 id="posix">POSIX Systems (Linux, macOS, *BSD etc.)</h2> 136<h2 id="posix">POSIX Systems (Linux, macOS, *BSD etc.)</h2>
126<h3>Prerequisites</h3> 137<h3>Prerequisites</h3>
@@ -154,9 +165,12 @@ You can add an extra prefix to the search paths by appending the
154make PREFIX=/home/myself/lj2 165make PREFIX=/home/myself/lj2
155</pre> 166</pre>
156<p> 167<p>
157Please use the LuaJIT 2.1 branch to compile for 168Note for macOS: you <b>must</b> set the <tt>MACOSX_DEPLOYMENT_TARGET</tt>
158<b id="osx">macOS (OSX)</b>. 169environment variable to a value supported by your toolchain:
159</p> 170</p>
171<pre class="code">
172MACOSX_DEPLOYMENT_TARGET=XX.YY make
173</pre>
160<h3>Installing LuaJIT</h3> 174<h3>Installing LuaJIT</h3>
161<p> 175<p>
162The top-level Makefile installs LuaJIT by default under 176The top-level Makefile installs LuaJIT by default under
@@ -235,25 +249,36 @@ directory where <tt>luajit.exe</tt> is installed
235 249
236<h2 id="cross">Cross-compiling LuaJIT</h2> 250<h2 id="cross">Cross-compiling LuaJIT</h2>
237<p> 251<p>
252First, let's clear up some terminology:
253</p>
254<ul>
255<li>Host: This is your development system, usually based on a x64 or x86 CPU.</li>
256<li>Target: This is the target system you want LuaJIT to run on, e.g. Android/ARM.</li>
257<li>Toolchain: This comprises a C compiler, linker, assembler and a matching C library.</li>
258<li>Host (or system) toolchain: This is the toolchain used to build native binaries for your host system.</li>
259<li>Cross-compile toolchain: This is the toolchain used to build binaries for the target system. They can only be run on the target system.</li>
260</ul>
261<p>
238The GNU Makefile-based build system allows cross-compiling on any host 262The GNU Makefile-based build system allows cross-compiling on any host
239for any supported target, as long as both architectures have the same 263for any supported target:
240pointer size. If you want to cross-compile to any 32 bit target on an
241x64 OS, you need to install the multilib development package (e.g.
242<tt>libc6-dev-i386</tt> on Debian/Ubuntu) and build a 32 bit host part
243(<tt>HOST_CC="gcc -m32"</tt>).
244</p> 264</p>
265<ul>
266<li>Yes, you need a toolchain for both your host <em>and</em> your target!</li>
267<li>Both host and target architectures must have the same pointer size.</li>
268<li>E.g. if you want to cross-compile to a 32 bit target on a 64 bit host, you need to install the multilib development package (e.g. <tt>libc6-dev-i386</tt> on Debian/Ubuntu) and build a 32 bit host part (<tt>HOST_CC="gcc -m32"</tt>).</li>
269<li>64 bit targets always require compilation on a 64 bit host.</li>
270</ul>
245<p> 271<p>
246You need to specify <tt>TARGET_SYS</tt> whenever the host OS and the 272You need to specify <tt>TARGET_SYS</tt> whenever the host OS and the
247target OS differ, or you'll get assembler or linker errors. E.g. if 273target OS differ, or you'll get assembler or linker errors:
248you're compiling on a Windows or macOS host for embedded Linux or Android,
249you need to add <tt>TARGET_SYS=Linux</tt> to the examples below. For a
250minimal target OS, you may need to disable the built-in allocator in
251<tt>src/Makefile</tt> and use <tt>TARGET_SYS=Other</tt>. Don't forget to
252specify the same <tt>TARGET_SYS</tt> for the install step, too.
253</p> 274</p>
275<ul>
276<li>E.g. if you're compiling on a Windows or macOS host for embedded Linux or Android, you need to add <tt>TARGET_SYS=Linux</tt> to the examples below.</li>
277<li>For a minimal target OS, you may need to disable the built-in allocator in <tt>src/Makefile</tt> and use <tt>TARGET_SYS=Other</tt>.</li>
278<li>Don't forget to specify the same <tt>TARGET_SYS</tt> for the install step, too.</li>
279</ul>
254<p> 280<p>
255The examples below only show some popular targets &mdash; please check 281Here are some examples where host and target have the same CPU:
256the comments in <tt>src/Makefile</tt> for more details.
257</p> 282</p>
258<pre class="code"> 283<pre class="code">
259# Cross-compile to a 32 bit binary on a multilib x64 OS 284# Cross-compile to a 32 bit binary on a multilib x64 OS
@@ -271,34 +296,44 @@ use the canonical toolchain triplets for Linux.
271</p> 296</p>
272<p> 297<p>
273Since there's often no easy way to detect CPU features at runtime, it's 298Since there's often no easy way to detect CPU features at runtime, it's
274important to compile with the proper CPU or architecture settings. You 299important to compile with the proper CPU or architecture settings:
275can specify these when building the toolchain yourself. Or add 300</o>
276<tt>-mcpu=...</tt> or <tt>-march=...</tt> to <tt>TARGET_CFLAGS</tt>. For 301<ul>
277ARM it's important to have the correct <tt>-mfloat-abi=...</tt> setting, 302<li>The best way to get consistent results is to specify the correct settings when building the toolchain yourself.</li>
278too. Otherwise, LuaJIT may not run at the full performance of your target 303<li>For a pre-built, generic toolchain add <tt>-mcpu=...</tt> or <tt>-march=...</tt> and other necessary flags to <tt>TARGET_CFLAGS</tt>.</li>
279CPU. 304<li>For ARM it's important to have the correct <tt>-mfloat-abi=...</tt> setting, too. Otherwise LuaJIT may not run at the full performance of your target CPU.</li>
305<li>For MIPS it's important to select a supported ABI (o32 on MIPS32, n64 on MIPS64) and consistently compile your project either with hard-float or soft-float compiler settings.</li>
306</ul>
307<p>
308Here are some examples for targets with a different CPU than the host:
280</p> 309</p>
281<pre class="code"> 310<pre class="code">
282# ARM soft-float 311# ARM soft-float
283make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabi- \ 312make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabi- \
284 TARGET_CFLAGS="-mfloat-abi=soft" 313 TARGET_CFLAGS="-mfloat-abi=soft"
285 314
286# ARM soft-float ABI with VFP (example for Cortex-A8) 315# ARM soft-float ABI with VFP (example for Cortex-A9)
287make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabi- \ 316make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabi- \
288 TARGET_CFLAGS="-mcpu=cortex-a8 -mfloat-abi=softfp" 317 TARGET_CFLAGS="-mcpu=cortex-a9 -mfloat-abi=softfp"
289 318
290# ARM hard-float ABI with VFP (armhf, requires recent toolchain) 319# ARM hard-float ABI with VFP (armhf, most modern toolchains)
291make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabihf- 320make HOST_CC="gcc -m32" CROSS=arm-linux-gnueabihf-
292 321
322# ARM64
323make CROSS=aarch64-linux-gnu-
324
293# PPC 325# PPC
294make HOST_CC="gcc -m32" CROSS=powerpc-linux-gnu- 326make HOST_CC="gcc -m32" CROSS=powerpc-linux-gnu-
295# PPC/e500v2 (fast interpreter only)
296make HOST_CC="gcc -m32" CROSS=powerpc-e500v2-linux-gnuspe-
297 327
298# MIPS big-endian 328# MIPS32 big-endian
299make HOST_CC="gcc -m32" CROSS=mips-linux- 329make HOST_CC="gcc -m32" CROSS=mips-linux-gnu-
300# MIPS little-endian 330# MIPS32 little-endian
301make HOST_CC="gcc -m32" CROSS=mipsel-linux- 331make HOST_CC="gcc -m32" CROSS=mipsel-linux-gnu-
332
333# MIPS64 big-endian
334make CROSS=mips-linux- TARGET_CFLAGS="-mips64r2 -mabi=64"
335# MIPS64 little-endian
336make CROSS=mipsel-linux- TARGET_CFLAGS="-mips64r2 -mabi=64"
302</pre> 337</pre>
303<p> 338<p>
304You can cross-compile for <b id="android">Android</b> using the <a href="https://developer.android.com/ndk/"><span class="ext">&raquo;</span>&nbsp;Android NDK</a>. 339You can cross-compile for <b id="android">Android</b> using the <a href="https://developer.android.com/ndk/"><span class="ext">&raquo;</span>&nbsp;Android NDK</a>.
@@ -306,8 +341,17 @@ Please adapt the environment variables to match the install locations and the
306desired target platform. E.g. Android&nbsp;4.1 corresponds to ABI level&nbsp;16. 341desired target platform. E.g. Android&nbsp;4.1 corresponds to ABI level&nbsp;16.
307</p> 342</p>
308<pre class="code"> 343<pre class="code">
309# Android/ARM, armeabi-v7a (ARMv7 VFP), Android 4.1+ (JB) 344# Android/ARM64, aarch64, Android 5.0+ (L)
345NDKDIR=/opt/android/ndk
346NDKBIN=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin
347NDKCROSS=$NDKBIN/aarch64-linux-android-
348NDKCC=$NDKBIN/aarch64-linux-android21-clang
349make CROSS=$NDKCROSS \
350 STATIC_CC=$NDKCC DYNAMIC_CC="$NDKCC -fPIC" \
351 TARGET_LD=$NDKCC TARGET_AR="$NDKBIN/llvm-ar rcus" \
352 TARGET_STRIP=$NDKBIN/llvm-strip
310 353
354# Android/ARM, armeabi-v7a (ARMv7 VFP), Android 4.1+ (JB)
311NDKDIR=/opt/android/ndk 355NDKDIR=/opt/android/ndk
312NDKBIN=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin 356NDKBIN=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin
313NDKCROSS=$NDKBIN/arm-linux-androideabi- 357NDKCROSS=$NDKBIN/arm-linux-androideabi-
@@ -318,9 +362,23 @@ make HOST_CC="gcc -m32" CROSS=$NDKCROSS \
318 TARGET_STRIP=$NDKBIN/llvm-strip 362 TARGET_STRIP=$NDKBIN/llvm-strip
319</pre> 363</pre>
320<p> 364<p>
321Please use the LuaJIT 2.1 branch to compile for 365You can cross-compile for <b id="ios">iOS 3.0+</b> (iPhone/iPad) using the <a href="https://developer.apple.com/ios/"><span class="ext">&raquo;</span>&nbsp;iOS SDK</a>:
322<b id="ios">iOS</b> (iPhone/iPad). 366</p>
367<p style="font-size: 8pt;">
368Note: <b>the JIT compiler is disabled for iOS</b>, because regular iOS Apps
369are not allowed to generate code at runtime. You'll only get the performance
370of the LuaJIT interpreter on iOS. This is still faster than plain Lua, but
371much slower than the JIT compiler. Please complain to Apple, not me.
372Or use Android. :-p
323</p> 373</p>
374<pre class="code">
375# iOS/ARM64
376ISDKP=$(xcrun --sdk iphoneos --show-sdk-path)
377ICC=$(xcrun --sdk iphoneos --find clang)
378ISDKF="-arch arm64 -isysroot $ISDKP"
379make DEFAULT_CC=clang CROSS="$(dirname $ICC)/" \
380 TARGET_FLAGS="$ISDKF" TARGET_SYS=iOS
381</pre>
324 382
325<h3 id="consoles">Cross-compiling for consoles</h3> 383<h3 id="consoles">Cross-compiling for consoles</h3>
326<p> 384<p>
@@ -364,15 +422,35 @@ and run the build command given in the table:
364<td class="compatx"><tt>ps4build</tt></td> 422<td class="compatx"><tt>ps4build</tt></td>
365</tr> 423</tr>
366<tr class="even"> 424<tr class="even">
425<td class="compatname"><b id="ps5">PS5</b></td>
426<td class="compatbits">64</td>
427<td class="compatx"><tt>ps5build</tt></td>
428</tr>
429<tr class="odd">
367<td class="compatname"><b id="psvita">PS Vita</b></td> 430<td class="compatname"><b id="psvita">PS Vita</b></td>
368<td class="compatbits">32</td> 431<td class="compatbits">32</td>
369<td class="compatx"><tt>psvitabuild</tt></td> 432<td class="compatx"><tt>psvitabuild</tt></td>
370</tr> 433</tr>
371<tr class="odd"> 434<tr class="even">
372<td class="compatname"><b id="xbox360">Xbox 360</b></td> 435<td class="compatname"><b id="xbox360">Xbox 360</b></td>
373<td class="compatbits">32</td> 436<td class="compatbits">32</td>
374<td class="compatx"><tt>xedkbuild</tt></td> 437<td class="compatx"><tt>xedkbuild</tt></td>
375</tr> 438</tr>
439<tr class="odd">
440<td class="compatname"><b id="xboxone">Xbox One</b></td>
441<td class="compatbits">64</td>
442<td class="compatx"><tt>xb1build</tt></td>
443</tr>
444<tr class="even">
445<td class="compatname"><b id="nx32">Nintendo Switch NX32</b></td>
446<td class="compatbits">32</td>
447<td class="compatx"><tt>nxbuild</tt></td>
448</tr>
449<tr class="odd">
450<td class="compatname"><b id="nx64">Nintendo Switch NX64</b></td>
451<td class="compatbits">64</td>
452<td class="compatx"><tt>nxbuild</tt></td>
453</tr>
376</table> 454</table>
377<p> 455<p>
378Please check out the comments in the corresponding <tt>*.bat</tt> 456Please check out the comments in the corresponding <tt>*.bat</tt>
diff --git a/doc/luajit.html b/doc/luajit.html
index 7346acb3..030cf705 100644
--- a/doc/luajit.html
+++ b/doc/luajit.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>LuaJIT</title> 4<title>LuaJIT</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -98,9 +98,13 @@ table.fcompat td {
98<a href="ext_ffi_semantics.html">FFI Semantics</a> 98<a href="ext_ffi_semantics.html">FFI Semantics</a>
99</li></ul> 99</li></ul>
100</li><li> 100</li><li>
101<a href="ext_buffer.html">String Buffers</a>
102</li><li>
101<a href="ext_jit.html">jit.* Library</a> 103<a href="ext_jit.html">jit.* Library</a>
102</li><li> 104</li><li>
103<a href="ext_c_api.html">Lua/C API</a> 105<a href="ext_c_api.html">Lua/C API</a>
106</li><li>
107<a href="ext_profiler.html">Profiler</a>
104</li></ul> 108</li></ul>
105</li><li> 109</li><li>
106<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 110<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -132,13 +136,13 @@ LuaJIT is Copyright &copy; 2005-2023 Mike Pall, released under the
132<tr><td><span style="font-size:90%;">Embedded</span></td><td>Android</td><td>iOS</td></tr> 136<tr><td><span style="font-size:90%;">Embedded</span></td><td>Android</td><td>iOS</td></tr>
133</table> 137</table>
134<table class="feature os os3"> 138<table class="feature os os3">
135<tr><td>PS3</td><td>PS4</td><td>PS Vita</td><td>Xbox 360</td></tr> 139<tr><td>PS3</td><td>PS4<br>PS5</td><td>PS Vita</td><td>Xbox 360</td><td>Xbox One</td><td>Nintendo<br>Switch</td></tr>
136</table> 140</table>
137<table class="feature compiler"> 141<table class="feature compiler">
138<tr><td>GCC</td><td>CLANG<br>LLVM</td><td>MSVC</td></tr> 142<tr><td>GCC</td><td>Clang<br>LLVM</td><td>MSVC</td></tr>
139</table> 143</table>
140<table class="feature cpu"> 144<table class="feature cpu">
141<tr><td>x86</td><td>x64</td><td>ARM</td><td>PPC</td><td>e500</td><td>MIPS</td></tr> 145<tr><td>x86<br>x64</td><td>ARM<br>ARM64</td><td>PPC</td><td>MIPS32<br>MIPS64</td></tr>
142</table> 146</table>
143<table class="feature fcompat"> 147<table class="feature fcompat">
144<tr><td>Lua&nbsp;5.1<br>API+ABI</td><td>+&nbsp;JIT</td><td>+&nbsp;BitOp</td><td>+&nbsp;FFI</td><td>Drop-in<br>DLL/.so</td></tr> 148<tr><td>Lua&nbsp;5.1<br>API+ABI</td><td>+&nbsp;JIT</td><td>+&nbsp;BitOp</td><td>+&nbsp;FFI</td><td>Drop-in<br>DLL/.so</td></tr>
diff --git a/doc/running.html b/doc/running.html
index c7d9e9b6..3afc1b56 100644
--- a/doc/running.html
+++ b/doc/running.html
@@ -1,8 +1,8 @@
1<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1<!DOCTYPE html>
2<html> 2<html>
3<head> 3<head>
4<title>Running LuaJIT</title> 4<title>Running LuaJIT</title>
5<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> 5<meta charset="utf-8">
6<meta name="Copyright" content="Copyright (C) 2005-2023"> 6<meta name="Copyright" content="Copyright (C) 2005-2023">
7<meta name="Language" content="en"> 7<meta name="Language" content="en">
8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen"> 8<link rel="stylesheet" type="text/css" href="bluequad.css" media="screen">
@@ -59,9 +59,13 @@ td.param_default {
59<a href="ext_ffi_semantics.html">FFI Semantics</a> 59<a href="ext_ffi_semantics.html">FFI Semantics</a>
60</li></ul> 60</li></ul>
61</li><li> 61</li><li>
62<a href="ext_buffer.html">String Buffers</a>
63</li><li>
62<a href="ext_jit.html">jit.* Library</a> 64<a href="ext_jit.html">jit.* Library</a>
63</li><li> 65</li><li>
64<a href="ext_c_api.html">Lua/C API</a> 66<a href="ext_c_api.html">Lua/C API</a>
67</li><li>
68<a href="ext_profiler.html">Profiler</a>
65</li></ul> 69</li></ul>
66</li><li> 70</li><li>
67<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a> 71<a href="https://luajit.org/status.html">Status <span class="ext">&raquo;</span></a>
@@ -106,6 +110,7 @@ are accepted:
106<li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li> 110<li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li>
107<li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li> 111<li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li>
108<li><tt>-o os</tt> &mdash; Override OS for object files (default: native).</li> 112<li><tt>-o os</tt> &mdash; Override OS for object files (default: native).</li>
113<li><tt>-F name</tt> &mdash; Override filename (default: input filename).</li>
109<li><tt>-e chunk</tt> &mdash; Use chunk string as input.</li> 114<li><tt>-e chunk</tt> &mdash; Use chunk string as input.</li>
110<li><tt>-</tt> (a single minus sign) &mdash; Use stdin as input and/or stdout as output.</li> 115<li><tt>-</tt> (a single minus sign) &mdash; Use stdin as input and/or stdout as output.</li>
111</ul> 116</ul>
@@ -171,6 +176,7 @@ Here are the available LuaJIT control commands:
171<li id="j_flush"><tt>-jflush</tt> &mdash; Flushes the whole cache of compiled code.</li> 176<li id="j_flush"><tt>-jflush</tt> &mdash; Flushes the whole cache of compiled code.</li>
172<li id="j_v"><tt>-jv</tt> &mdash; Shows verbose information about the progress of the JIT compiler.</li> 177<li id="j_v"><tt>-jv</tt> &mdash; Shows verbose information about the progress of the JIT compiler.</li>
173<li id="j_dump"><tt>-jdump</tt> &mdash; Dumps the code and structures used in various compiler stages.</li> 178<li id="j_dump"><tt>-jdump</tt> &mdash; Dumps the code and structures used in various compiler stages.</li>
179<li id="j_p"><tt>-jp</tt> &mdash; Start the <a href="ext_profiler.html">integrated profiler</a>.</li>
174</ul> 180</ul>
175<p> 181<p>
176The <tt>-jv</tt> and <tt>-jdump</tt> commands are extension modules 182The <tt>-jv</tt> and <tt>-jdump</tt> commands are extension modules
@@ -215,6 +221,12 @@ mix the three forms, but note that setting an optimization level
215overrides all earlier flags. 221overrides all earlier flags.
216</p> 222</p>
217<p> 223<p>
224Note that <tt>-Ofma</tt> is not enabled by default at any level,
225because it affects floating-point result accuracy. Only enable this,
226if you fully understand the trade-offs of FMA for performance (higher),
227determinism (lower) and numerical accuracy (higher).
228</p>
229<p>
218Here are the available flags and at what optimization levels they 230Here are the available flags and at what optimization levels they
219are enabled: 231are enabled:
220</p> 232</p>
@@ -246,6 +258,8 @@ are enabled:
246<td class="flag_name">sink</td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_level">&bull;</td><td class="flag_desc">Allocation/Store Sinking</td></tr> 258<td class="flag_name">sink</td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_level">&bull;</td><td class="flag_desc">Allocation/Store Sinking</td></tr>
247<tr class="even"> 259<tr class="even">
248<td class="flag_name">fuse</td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_level">&bull;</td><td class="flag_desc">Fusion of operands into instructions</td></tr> 260<td class="flag_name">fuse</td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_level">&bull;</td><td class="flag_desc">Fusion of operands into instructions</td></tr>
261<tr class="odd">
262<td class="flag_name">fma </td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_level">&nbsp;</td><td class="flag_desc">Fused multiply-add</td></tr>
249</table> 263</table>
250<p> 264<p>
251Here are the parameters and their default settings: 265Here are the parameters and their default settings:
diff --git a/dynasm/dasm_arm.h b/dynasm/dasm_arm.h
index a93e831e..eaa94d9c 100644
--- a/dynasm/dasm_arm.h
+++ b/dynasm/dasm_arm.h
@@ -70,7 +70,7 @@ struct dasm_State {
70 size_t lgsize; 70 size_t lgsize;
71 int *pclabels; /* PC label chains/pos ptrs. */ 71 int *pclabels; /* PC label chains/pos ptrs. */
72 size_t pcsize; 72 size_t pcsize;
73 void **globals; /* Array of globals (bias -10). */ 73 void **globals; /* Array of globals. */
74 dasm_Section *section; /* Pointer to active section. */ 74 dasm_Section *section; /* Pointer to active section. */
75 size_t codesize; /* Total size of all code sections. */ 75 size_t codesize; /* Total size of all code sections. */
76 int maxsection; /* 0 <= sectionidx < maxsection. */ 76 int maxsection; /* 0 <= sectionidx < maxsection. */
@@ -87,7 +87,6 @@ void dasm_init(Dst_DECL, int maxsection)
87{ 87{
88 dasm_State *D; 88 dasm_State *D;
89 size_t psz = 0; 89 size_t psz = 0;
90 int i;
91 Dst_REF = NULL; 90 Dst_REF = NULL;
92 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); 91 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
93 D = Dst_REF; 92 D = Dst_REF;
@@ -98,12 +97,7 @@ void dasm_init(Dst_DECL, int maxsection)
98 D->pcsize = 0; 97 D->pcsize = 0;
99 D->globals = NULL; 98 D->globals = NULL;
100 D->maxsection = maxsection; 99 D->maxsection = maxsection;
101 for (i = 0; i < maxsection; i++) { 100 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
102 D->sections[i].buf = NULL; /* Need this for pass3. */
103 D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i);
104 D->sections[i].bsize = 0;
105 D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */
106 }
107} 101}
108 102
109/* Free DynASM state. */ 103/* Free DynASM state. */
@@ -123,7 +117,7 @@ void dasm_free(Dst_DECL)
123void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) 117void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
124{ 118{
125 dasm_State *D = Dst_REF; 119 dasm_State *D = Dst_REF;
126 D->globals = gl - 10; /* Negative bias to compensate for locals. */ 120 D->globals = gl;
127 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); 121 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
128} 122}
129 123
@@ -148,6 +142,7 @@ void dasm_setup(Dst_DECL, const void *actionlist)
148 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); 142 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
149 for (i = 0; i < D->maxsection; i++) { 143 for (i = 0; i < D->maxsection; i++) {
150 D->sections[i].pos = DASM_SEC2POS(i); 144 D->sections[i].pos = DASM_SEC2POS(i);
145 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
151 D->sections[i].ofs = 0; 146 D->sections[i].ofs = 0;
152 } 147 }
153} 148}
@@ -294,7 +289,7 @@ int dasm_link(Dst_DECL, size_t *szp)
294 289
295 { /* Handle globals not defined in this translation unit. */ 290 { /* Handle globals not defined in this translation unit. */
296 int idx; 291 int idx;
297 for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { 292 for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
298 int n = D->lglabels[idx]; 293 int n = D->lglabels[idx];
299 /* Undefined label: Collapse rel chain and replace with marker (< 0). */ 294 /* Undefined label: Collapse rel chain and replace with marker (< 0). */
300 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } 295 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
@@ -371,7 +366,10 @@ int dasm_encode(Dst_DECL, void *buffer)
371 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000; 366 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000;
372 break; 367 break;
373 case DASM_REL_LG: 368 case DASM_REL_LG:
374 CK(n >= 0, UNDEF_LG); 369 if (n < 0) {
370 n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp - 4);
371 goto patchrel;
372 }
375 /* fallthrough */ 373 /* fallthrough */
376 case DASM_REL_PC: 374 case DASM_REL_PC:
377 CK(n >= 0, UNDEF_PC); 375 CK(n >= 0, UNDEF_PC);
@@ -393,7 +391,7 @@ int dasm_encode(Dst_DECL, void *buffer)
393 } 391 }
394 break; 392 break;
395 case DASM_LABEL_LG: 393 case DASM_LABEL_LG:
396 ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); 394 ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n);
397 break; 395 break;
398 case DASM_LABEL_PC: break; 396 case DASM_LABEL_PC: break;
399 case DASM_IMM: 397 case DASM_IMM:
diff --git a/dynasm/dasm_arm.lua b/dynasm/dasm_arm.lua
index 960f1fe6..edb57536 100644
--- a/dynasm/dasm_arm.lua
+++ b/dynasm/dasm_arm.lua
@@ -9,9 +9,9 @@
9local _info = { 9local _info = {
10 arch = "arm", 10 arch = "arm",
11 description = "DynASM ARM module", 11 description = "DynASM ARM module",
12 version = "1.3.0", 12 version = "1.5.0",
13 vernum = 10300, 13 vernum = 10500,
14 release = "2011-05-05", 14 release = "2021-05-02",
15 author = "Mike Pall", 15 author = "Mike Pall",
16 license = "MIT", 16 license = "MIT",
17} 17}
diff --git a/dynasm/dasm_arm64.h b/dynasm/dasm_arm64.h
new file mode 100644
index 00000000..1c541e5d
--- /dev/null
+++ b/dynasm/dasm_arm64.h
@@ -0,0 +1,558 @@
1/*
2** DynASM ARM64 encoding engine.
3** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
4** Released under the MIT license. See dynasm.lua for full copyright notice.
5*/
6
7#include <stddef.h>
8#include <stdarg.h>
9#include <string.h>
10#include <stdlib.h>
11
12#define DASM_ARCH "arm64"
13
14#ifndef DASM_EXTERN
15#define DASM_EXTERN(a,b,c,d) 0
16#endif
17
18/* Action definitions. */
19enum {
20 DASM_STOP, DASM_SECTION, DASM_ESC, DASM_REL_EXT,
21 /* The following actions need a buffer position. */
22 DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
23 /* The following actions also have an argument. */
24 DASM_REL_PC, DASM_LABEL_PC, DASM_REL_A,
25 DASM_IMM, DASM_IMM6, DASM_IMM12, DASM_IMM13W, DASM_IMM13X, DASM_IMML,
26 DASM_IMMV, DASM_VREG,
27 DASM__MAX
28};
29
30/* Maximum number of section buffer positions for a single dasm_put() call. */
31#define DASM_MAXSECPOS 25
32
33/* DynASM encoder status codes. Action list offset or number are or'ed in. */
34#define DASM_S_OK 0x00000000
35#define DASM_S_NOMEM 0x01000000
36#define DASM_S_PHASE 0x02000000
37#define DASM_S_MATCH_SEC 0x03000000
38#define DASM_S_RANGE_I 0x11000000
39#define DASM_S_RANGE_SEC 0x12000000
40#define DASM_S_RANGE_LG 0x13000000
41#define DASM_S_RANGE_PC 0x14000000
42#define DASM_S_RANGE_REL 0x15000000
43#define DASM_S_RANGE_VREG 0x16000000
44#define DASM_S_UNDEF_LG 0x21000000
45#define DASM_S_UNDEF_PC 0x22000000
46
47/* Macros to convert positions (8 bit section + 24 bit index). */
48#define DASM_POS2IDX(pos) ((pos)&0x00ffffff)
49#define DASM_POS2BIAS(pos) ((pos)&0xff000000)
50#define DASM_SEC2POS(sec) ((sec)<<24)
51#define DASM_POS2SEC(pos) ((pos)>>24)
52#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos))
53
54/* Action list type. */
55typedef const unsigned int *dasm_ActList;
56
57/* Per-section structure. */
58typedef struct dasm_Section {
59 int *rbuf; /* Biased buffer pointer (negative section bias). */
60 int *buf; /* True buffer pointer. */
61 size_t bsize; /* Buffer size in bytes. */
62 int pos; /* Biased buffer position. */
63 int epos; /* End of biased buffer position - max single put. */
64 int ofs; /* Byte offset into section. */
65} dasm_Section;
66
67/* Core structure holding the DynASM encoding state. */
68struct dasm_State {
69 size_t psize; /* Allocated size of this structure. */
70 dasm_ActList actionlist; /* Current actionlist pointer. */
71 int *lglabels; /* Local/global chain/pos ptrs. */
72 size_t lgsize;
73 int *pclabels; /* PC label chains/pos ptrs. */
74 size_t pcsize;
75 void **globals; /* Array of globals. */
76 dasm_Section *section; /* Pointer to active section. */
77 size_t codesize; /* Total size of all code sections. */
78 int maxsection; /* 0 <= sectionidx < maxsection. */
79 int status; /* Status code. */
80 dasm_Section sections[1]; /* All sections. Alloc-extended. */
81};
82
83/* The size of the core structure depends on the max. number of sections. */
84#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section))
85
86
87/* Initialize DynASM state. */
88void dasm_init(Dst_DECL, int maxsection)
89{
90 dasm_State *D;
91 size_t psz = 0;
92 Dst_REF = NULL;
93 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
94 D = Dst_REF;
95 D->psize = psz;
96 D->lglabels = NULL;
97 D->lgsize = 0;
98 D->pclabels = NULL;
99 D->pcsize = 0;
100 D->globals = NULL;
101 D->maxsection = maxsection;
102 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
103}
104
105/* Free DynASM state. */
106void dasm_free(Dst_DECL)
107{
108 dasm_State *D = Dst_REF;
109 int i;
110 for (i = 0; i < D->maxsection; i++)
111 if (D->sections[i].buf)
112 DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize);
113 if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize);
114 if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize);
115 DASM_M_FREE(Dst, D, D->psize);
116}
117
118/* Setup global label array. Must be called before dasm_setup(). */
119void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
120{
121 dasm_State *D = Dst_REF;
122 D->globals = gl;
123 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
124}
125
126/* Grow PC label array. Can be called after dasm_setup(), too. */
127void dasm_growpc(Dst_DECL, unsigned int maxpc)
128{
129 dasm_State *D = Dst_REF;
130 size_t osz = D->pcsize;
131 DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int));
132 memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz);
133}
134
135/* Setup encoder. */
136void dasm_setup(Dst_DECL, const void *actionlist)
137{
138 dasm_State *D = Dst_REF;
139 int i;
140 D->actionlist = (dasm_ActList)actionlist;
141 D->status = DASM_S_OK;
142 D->section = &D->sections[0];
143 memset((void *)D->lglabels, 0, D->lgsize);
144 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
145 for (i = 0; i < D->maxsection; i++) {
146 D->sections[i].pos = DASM_SEC2POS(i);
147 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
148 D->sections[i].ofs = 0;
149 }
150}
151
152
153#ifdef DASM_CHECKS
154#define CK(x, st) \
155 do { if (!(x)) { \
156 D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0)
157#define CKPL(kind, st) \
158 do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \
159 D->status = DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0)
160#else
161#define CK(x, st) ((void)0)
162#define CKPL(kind, st) ((void)0)
163#endif
164
165static int dasm_imm12(unsigned int n)
166{
167 if ((n >> 12) == 0)
168 return n;
169 else if ((n & 0xff000fff) == 0)
170 return (n >> 12) | 0x1000;
171 else
172 return -1;
173}
174
175static int dasm_ffs(unsigned long long x)
176{
177 int n = -1;
178 while (x) { x >>= 1; n++; }
179 return n;
180}
181
182static int dasm_imm13(int lo, int hi)
183{
184 int inv = 0, w = 64, s = 0xfff, xa, xb;
185 unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo;
186 unsigned long long m = 1ULL, a, b, c;
187 if (n & 1) { n = ~n; inv = 1; }
188 a = n & (unsigned long long)-(long long)n;
189 b = (n+a)&(unsigned long long)-(long long)(n+a);
190 c = (n+a-b)&(unsigned long long)-(long long)(n+a-b);
191 xa = dasm_ffs(a); xb = dasm_ffs(b);
192 if (c) {
193 w = dasm_ffs(c) - xa;
194 if (w == 32) m = 0x0000000100000001UL;
195 else if (w == 16) m = 0x0001000100010001UL;
196 else if (w == 8) m = 0x0101010101010101UL;
197 else if (w == 4) m = 0x1111111111111111UL;
198 else if (w == 2) m = 0x5555555555555555UL;
199 else return -1;
200 s = (-2*w & 0x3f) - 1;
201 } else if (!a) {
202 return -1;
203 } else if (xb == -1) {
204 xb = 64;
205 }
206 if ((b-a) * m != n) return -1;
207 if (inv) {
208 return ((w - xb) << 6) | (s+w+xa-xb);
209 } else {
210 return ((w - xa) << 6) | (s+xb-xa);
211 }
212 return -1;
213}
214
215/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */
216void dasm_put(Dst_DECL, int start, ...)
217{
218 va_list ap;
219 dasm_State *D = Dst_REF;
220 dasm_ActList p = D->actionlist + start;
221 dasm_Section *sec = D->section;
222 int pos = sec->pos, ofs = sec->ofs;
223 int *b;
224
225 if (pos >= sec->epos) {
226 DASM_M_GROW(Dst, int, sec->buf, sec->bsize,
227 sec->bsize + 2*DASM_MAXSECPOS*sizeof(int));
228 sec->rbuf = sec->buf - DASM_POS2BIAS(pos);
229 sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos);
230 }
231
232 b = sec->rbuf;
233 b[pos++] = start;
234
235 va_start(ap, start);
236 while (1) {
237 unsigned int ins = *p++;
238 unsigned int action = (ins >> 16);
239 if (action >= DASM__MAX) {
240 ofs += 4;
241 } else {
242 int *pl, n = action >= DASM_REL_PC ? va_arg(ap, int) : 0;
243 switch (action) {
244 case DASM_STOP: goto stop;
245 case DASM_SECTION:
246 n = (ins & 255); CK(n < D->maxsection, RANGE_SEC);
247 D->section = &D->sections[n]; goto stop;
248 case DASM_ESC: p++; ofs += 4; break;
249 case DASM_REL_EXT: if ((ins & 0x8000)) ofs += 8; break;
250 case DASM_ALIGN: ofs += (ins & 255); b[pos++] = ofs; break;
251 case DASM_REL_LG:
252 n = (ins & 2047) - 10; pl = D->lglabels + n;
253 /* Bkwd rel or global. */
254 if (n >= 0) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; }
255 pl += 10; n = *pl;
256 if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */
257 goto linkrel;
258 case DASM_REL_PC:
259 pl = D->pclabels + n; CKPL(pc, PC);
260 putrel:
261 n = *pl;
262 if (n < 0) { /* Label exists. Get label pos and store it. */
263 b[pos] = -n;
264 } else {
265 linkrel:
266 b[pos] = n; /* Else link to rel chain, anchored at label. */
267 *pl = pos;
268 }
269 pos++;
270 if ((ins & 0x8000)) ofs += 8;
271 break;
272 case DASM_REL_A:
273 b[pos++] = n;
274 b[pos++] = va_arg(ap, int);
275 break;
276 case DASM_LABEL_LG:
277 pl = D->lglabels + (ins & 2047) - 10; CKPL(lg, LG); goto putlabel;
278 case DASM_LABEL_PC:
279 pl = D->pclabels + n; CKPL(pc, PC);
280 putlabel:
281 n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */
282 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos;
283 }
284 *pl = -pos; /* Label exists now. */
285 b[pos++] = ofs; /* Store pass1 offset estimate. */
286 break;
287 case DASM_IMM:
288 CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I);
289 n >>= ((ins>>10)&31);
290#ifdef DASM_CHECKS
291 if ((ins & 0x8000))
292 CK(((n + (1<<(((ins>>5)&31)-1)))>>((ins>>5)&31)) == 0, RANGE_I);
293 else
294 CK((n>>((ins>>5)&31)) == 0, RANGE_I);
295#endif
296 b[pos++] = n;
297 break;
298 case DASM_IMM6:
299 CK((n >> 6) == 0, RANGE_I);
300 b[pos++] = n;
301 break;
302 case DASM_IMM12:
303 CK(dasm_imm12((unsigned int)n) != -1, RANGE_I);
304 b[pos++] = n;
305 break;
306 case DASM_IMM13W:
307 CK(dasm_imm13(n, n) != -1, RANGE_I);
308 b[pos++] = n;
309 break;
310 case DASM_IMM13X: {
311 int m = va_arg(ap, int);
312 CK(dasm_imm13(n, m) != -1, RANGE_I);
313 b[pos++] = n;
314 b[pos++] = m;
315 break;
316 }
317 case DASM_IMML: {
318#ifdef DASM_CHECKS
319 int scale = (ins & 3);
320 CK((!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ||
321 (unsigned int)(n+256) < 512, RANGE_I);
322#endif
323 b[pos++] = n;
324 break;
325 }
326 case DASM_IMMV:
327 ofs += 4;
328 b[pos++] = n;
329 break;
330 case DASM_VREG:
331 CK(n < 32, RANGE_VREG);
332 b[pos++] = n;
333 break;
334 }
335 }
336 }
337stop:
338 va_end(ap);
339 sec->pos = pos;
340 sec->ofs = ofs;
341}
342#undef CK
343
344/* Pass 2: Link sections, shrink aligns, fix label offsets. */
345int dasm_link(Dst_DECL, size_t *szp)
346{
347 dasm_State *D = Dst_REF;
348 int secnum;
349 int ofs = 0;
350
351#ifdef DASM_CHECKS
352 *szp = 0;
353 if (D->status != DASM_S_OK) return D->status;
354 {
355 int pc;
356 for (pc = 0; pc*sizeof(int) < D->pcsize; pc++)
357 if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc;
358 }
359#endif
360
361 { /* Handle globals not defined in this translation unit. */
362 int idx;
363 for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
364 int n = D->lglabels[idx];
365 /* Undefined label: Collapse rel chain and replace with marker (< 0). */
366 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
367 }
368 }
369
370 /* Combine all code sections. No support for data sections (yet). */
371 for (secnum = 0; secnum < D->maxsection; secnum++) {
372 dasm_Section *sec = D->sections + secnum;
373 int *b = sec->rbuf;
374 int pos = DASM_SEC2POS(secnum);
375 int lastpos = sec->pos;
376
377 while (pos != lastpos) {
378 dasm_ActList p = D->actionlist + b[pos++];
379 while (1) {
380 unsigned int ins = *p++;
381 unsigned int action = (ins >> 16);
382 switch (action) {
383 case DASM_STOP: case DASM_SECTION: goto stop;
384 case DASM_ESC: p++; break;
385 case DASM_REL_EXT: break;
386 case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
387 case DASM_REL_LG: case DASM_REL_PC: pos++; break;
388 case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
389 case DASM_IMM: case DASM_IMM6: case DASM_IMM12: case DASM_IMM13W:
390 case DASM_IMML: case DASM_IMMV: case DASM_VREG: pos++; break;
391 case DASM_IMM13X: case DASM_REL_A: pos += 2; break;
392 }
393 }
394 stop: (void)0;
395 }
396 ofs += sec->ofs; /* Next section starts right after current section. */
397 }
398
399 D->codesize = ofs; /* Total size of all code sections */
400 *szp = ofs;
401 return DASM_S_OK;
402}
403
404#ifdef DASM_CHECKS
405#define CK(x, st) \
406 do { if (!(x)) return DASM_S_##st|(int)(p-D->actionlist-1); } while (0)
407#else
408#define CK(x, st) ((void)0)
409#endif
410
411/* Pass 3: Encode sections. */
412int dasm_encode(Dst_DECL, void *buffer)
413{
414 dasm_State *D = Dst_REF;
415 char *base = (char *)buffer;
416 unsigned int *cp = (unsigned int *)buffer;
417 int secnum;
418
419 /* Encode all code sections. No support for data sections (yet). */
420 for (secnum = 0; secnum < D->maxsection; secnum++) {
421 dasm_Section *sec = D->sections + secnum;
422 int *b = sec->buf;
423 int *endb = sec->rbuf + sec->pos;
424
425 while (b != endb) {
426 dasm_ActList p = D->actionlist + *b++;
427 while (1) {
428 unsigned int ins = *p++;
429 unsigned int action = (ins >> 16);
430 int n = (action >= DASM_ALIGN && action < DASM__MAX) ? *b++ : 0;
431 switch (action) {
432 case DASM_STOP: case DASM_SECTION: goto stop;
433 case DASM_ESC: *cp++ = *p++; break;
434 case DASM_REL_EXT:
435 n = DASM_EXTERN(Dst, (unsigned char *)cp, (ins&2047), !(ins&2048));
436 goto patchrel;
437 case DASM_ALIGN:
438 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xd503201f;
439 break;
440 case DASM_REL_LG:
441 if (n < 0) {
442 ptrdiff_t na = (ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp + 4;
443 n = (int)na;
444 CK((ptrdiff_t)n == na, RANGE_REL);
445 goto patchrel;
446 }
447 /* fallthrough */
448 case DASM_REL_PC:
449 CK(n >= 0, UNDEF_PC);
450 n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4;
451 patchrel:
452 if (!(ins & 0xf800)) { /* B, BL */
453 CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL);
454 cp[-1] |= ((n >> 2) & 0x03ffffff);
455 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */
456 CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL);
457 cp[-1] |= ((n << 3) & 0x00ffffe0);
458 } else if ((ins & 0x3000) == 0x2000) { /* ADR */
459 CK(((n+0x00100000) >> 21) == 0, RANGE_REL);
460 cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29);
461 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */
462 cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29);
463 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */
464 CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL);
465 cp[-1] |= ((n << 3) & 0x0007ffe0);
466 } else if ((ins & 0x8000)) { /* absolute */
467 cp[0] = (unsigned int)((ptrdiff_t)cp - 4 + n);
468 cp[1] = (unsigned int)(((ptrdiff_t)cp - 4 + n) >> 32);
469 cp += 2;
470 }
471 break;
472 case DASM_REL_A: {
473 ptrdiff_t na = (((ptrdiff_t)(*b++) << 32) | (unsigned int)n);
474 if ((ins & 0x3000) == 0x3000) { /* ADRP */
475 ins &= ~0x1000;
476 na = (na >> 12) - (((ptrdiff_t)cp - 4) >> 12);
477 } else {
478 na = na - (ptrdiff_t)cp + 4;
479 }
480 n = (int)na;
481 CK((ptrdiff_t)n == na, RANGE_REL);
482 goto patchrel;
483 }
484 case DASM_LABEL_LG:
485 ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n);
486 break;
487 case DASM_LABEL_PC: break;
488 case DASM_IMM:
489 cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
490 break;
491 case DASM_IMM6:
492 cp[-1] |= ((n&31) << 19) | ((n&32) << 26);
493 break;
494 case DASM_IMM12:
495 cp[-1] |= (dasm_imm12((unsigned int)n) << 10);
496 break;
497 case DASM_IMM13W:
498 cp[-1] |= (dasm_imm13(n, n) << 10);
499 break;
500 case DASM_IMM13X:
501 cp[-1] |= (dasm_imm13(n, *b++) << 10);
502 break;
503 case DASM_IMML: {
504 int scale = (ins & 3);
505 cp[-1] |= (!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ?
506 ((n << (10-scale)) | 0x01000000) : ((n & 511) << 12);
507 break;
508 }
509 case DASM_IMMV:
510 *cp++ = n;
511 break;
512 case DASM_VREG:
513 cp[-1] |= (n & 0x1f) << (ins & 0x1f);
514 break;
515 default: *cp++ = ins; break;
516 }
517 }
518 stop: (void)0;
519 }
520 }
521
522 if (base + D->codesize != (char *)cp) /* Check for phase errors. */
523 return DASM_S_PHASE;
524 return DASM_S_OK;
525}
526#undef CK
527
528/* Get PC label offset. */
529int dasm_getpclabel(Dst_DECL, unsigned int pc)
530{
531 dasm_State *D = Dst_REF;
532 if (pc*sizeof(int) < D->pcsize) {
533 int pos = D->pclabels[pc];
534 if (pos < 0) return *DASM_POS2PTR(D, -pos);
535 if (pos > 0) return -1; /* Undefined. */
536 }
537 return -2; /* Unused or out of range. */
538}
539
540#ifdef DASM_CHECKS
541/* Optional sanity checker to call between isolated encoding steps. */
542int dasm_checkstep(Dst_DECL, int secmatch)
543{
544 dasm_State *D = Dst_REF;
545 if (D->status == DASM_S_OK) {
546 int i;
547 for (i = 1; i <= 9; i++) {
548 if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_LG|i; break; }
549 D->lglabels[i] = 0;
550 }
551 }
552 if (D->status == DASM_S_OK && secmatch >= 0 &&
553 D->section != &D->sections[secmatch])
554 D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections);
555 return D->status;
556}
557#endif
558
diff --git a/dynasm/dasm_arm64.lua b/dynasm/dasm_arm64.lua
new file mode 100644
index 00000000..e69f8ef3
--- /dev/null
+++ b/dynasm/dasm_arm64.lua
@@ -0,0 +1,1226 @@
1------------------------------------------------------------------------------
2-- DynASM ARM64 module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- See dynasm.lua for full copyright notice.
6------------------------------------------------------------------------------
7
8-- Module information:
9local _info = {
10 arch = "arm",
11 description = "DynASM ARM64 module",
12 version = "1.5.0",
13 vernum = 10500,
14 release = "2021-05-02",
15 author = "Mike Pall",
16 license = "MIT",
17}
18
19-- Exported glue functions for the arch-specific module.
20local _M = { _info = _info }
21
22-- Cache library functions.
23local type, tonumber, pairs, ipairs = type, tonumber, pairs, ipairs
24local assert, setmetatable, rawget = assert, setmetatable, rawget
25local _s = string
26local format, byte, char = _s.format, _s.byte, _s.char
27local match, gmatch, gsub = _s.match, _s.gmatch, _s.gsub
28local concat, sort, insert = table.concat, table.sort, table.insert
29local bit = bit or require("bit")
30local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift
31local ror, tohex, tobit = bit.ror, bit.tohex, bit.tobit
32
33-- Inherited tables and callbacks.
34local g_opt, g_arch
35local wline, werror, wfatal, wwarn
36
37-- Action name list.
38-- CHECK: Keep this in sync with the C code!
39local action_names = {
40 "STOP", "SECTION", "ESC", "REL_EXT",
41 "ALIGN", "REL_LG", "LABEL_LG",
42 "REL_PC", "LABEL_PC", "REL_A",
43 "IMM", "IMM6", "IMM12", "IMM13W", "IMM13X", "IMML", "IMMV",
44 "VREG",
45}
46
47-- Maximum number of section buffer positions for dasm_put().
48-- CHECK: Keep this in sync with the C code!
49local maxsecpos = 25 -- Keep this low, to avoid excessively long C lines.
50
51-- Action name -> action number.
52local map_action = {}
53for n,name in ipairs(action_names) do
54 map_action[name] = n-1
55end
56
57-- Action list buffer.
58local actlist = {}
59
60-- Argument list for next dasm_put(). Start with offset 0 into action list.
61local actargs = { 0 }
62
63-- Current number of section buffer positions for dasm_put().
64local secpos = 1
65
66------------------------------------------------------------------------------
67
68-- Dump action names and numbers.
69local function dumpactions(out)
70 out:write("DynASM encoding engine action codes:\n")
71 for n,name in ipairs(action_names) do
72 local num = map_action[name]
73 out:write(format(" %-10s %02X %d\n", name, num, num))
74 end
75 out:write("\n")
76end
77
78-- Write action list buffer as a huge static C array.
79local function writeactions(out, name)
80 local nn = #actlist
81 if nn == 0 then nn = 1; actlist[0] = map_action.STOP end
82 out:write("static const unsigned int ", name, "[", nn, "] = {\n")
83 for i = 1,nn-1 do
84 assert(out:write("0x", tohex(actlist[i]), ",\n"))
85 end
86 assert(out:write("0x", tohex(actlist[nn]), "\n};\n\n"))
87end
88
89------------------------------------------------------------------------------
90
91-- Add word to action list.
92local function wputxw(n)
93 assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range")
94 actlist[#actlist+1] = n
95end
96
97-- Add action to list with optional arg. Advance buffer pos, too.
98local function waction(action, val, a, num)
99 local w = assert(map_action[action], "bad action name `"..action.."'")
100 wputxw(w * 0x10000 + (val or 0))
101 if a then actargs[#actargs+1] = a end
102 if a or num then secpos = secpos + (num or 1) end
103end
104
105-- Flush action list (intervening C code or buffer pos overflow).
106local function wflush(term)
107 if #actlist == actargs[1] then return end -- Nothing to flush.
108 if not term then waction("STOP") end -- Terminate action list.
109 wline(format("dasm_put(Dst, %s);", concat(actargs, ", ")), true)
110 actargs = { #actlist } -- Actionlist offset is 1st arg to next dasm_put().
111 secpos = 1 -- The actionlist offset occupies a buffer position, too.
112end
113
114-- Put escaped word.
115local function wputw(n)
116 if n <= 0x000fffff then waction("ESC") end
117 wputxw(n)
118end
119
120-- Reserve position for word.
121local function wpos()
122 local pos = #actlist+1
123 actlist[pos] = ""
124 return pos
125end
126
127-- Store word to reserved position.
128local function wputpos(pos, n)
129 assert(n >= 0 and n <= 0xffffffff and n % 1 == 0, "word out of range")
130 if n <= 0x000fffff then
131 insert(actlist, pos+1, n)
132 n = map_action.ESC * 0x10000
133 end
134 actlist[pos] = n
135end
136
137------------------------------------------------------------------------------
138
139-- Global label name -> global label number. With auto assignment on 1st use.
140local next_global = 20
141local map_global = setmetatable({}, { __index = function(t, name)
142 if not match(name, "^[%a_][%w_]*$") then werror("bad global label") end
143 local n = next_global
144 if n > 2047 then werror("too many global labels") end
145 next_global = n + 1
146 t[name] = n
147 return n
148end})
149
150-- Dump global labels.
151local function dumpglobals(out, lvl)
152 local t = {}
153 for name, n in pairs(map_global) do t[n] = name end
154 out:write("Global labels:\n")
155 for i=20,next_global-1 do
156 out:write(format(" %s\n", t[i]))
157 end
158 out:write("\n")
159end
160
161-- Write global label enum.
162local function writeglobals(out, prefix)
163 local t = {}
164 for name, n in pairs(map_global) do t[n] = name end
165 out:write("enum {\n")
166 for i=20,next_global-1 do
167 out:write(" ", prefix, t[i], ",\n")
168 end
169 out:write(" ", prefix, "_MAX\n};\n")
170end
171
172-- Write global label names.
173local function writeglobalnames(out, name)
174 local t = {}
175 for name, n in pairs(map_global) do t[n] = name end
176 out:write("static const char *const ", name, "[] = {\n")
177 for i=20,next_global-1 do
178 out:write(" \"", t[i], "\",\n")
179 end
180 out:write(" (const char *)0\n};\n")
181end
182
183------------------------------------------------------------------------------
184
185-- Extern label name -> extern label number. With auto assignment on 1st use.
186local next_extern = 0
187local map_extern_ = {}
188local map_extern = setmetatable({}, { __index = function(t, name)
189 -- No restrictions on the name for now.
190 local n = next_extern
191 if n > 2047 then werror("too many extern labels") end
192 next_extern = n + 1
193 t[name] = n
194 map_extern_[n] = name
195 return n
196end})
197
198-- Dump extern labels.
199local function dumpexterns(out, lvl)
200 out:write("Extern labels:\n")
201 for i=0,next_extern-1 do
202 out:write(format(" %s\n", map_extern_[i]))
203 end
204 out:write("\n")
205end
206
207-- Write extern label names.
208local function writeexternnames(out, name)
209 out:write("static const char *const ", name, "[] = {\n")
210 for i=0,next_extern-1 do
211 out:write(" \"", map_extern_[i], "\",\n")
212 end
213 out:write(" (const char *)0\n};\n")
214end
215
216------------------------------------------------------------------------------
217
218-- Arch-specific maps.
219
220-- Ext. register name -> int. name.
221local map_archdef = { xzr = "@x31", wzr = "@w31", lr = "x30", }
222
223-- Int. register name -> ext. name.
224local map_reg_rev = { ["@x31"] = "xzr", ["@w31"] = "wzr", x30 = "lr", }
225
226local map_type = {} -- Type name -> { ctype, reg }
227local ctypenum = 0 -- Type number (for Dt... macros).
228
229-- Reverse defines for registers.
230function _M.revdef(s)
231 return map_reg_rev[s] or s
232end
233
234local map_shift = { lsl = 0, lsr = 1, asr = 2, }
235
236local map_extend = {
237 uxtb = 0, uxth = 1, uxtw = 2, uxtx = 3,
238 sxtb = 4, sxth = 5, sxtw = 6, sxtx = 7,
239}
240
241local map_cond = {
242 eq = 0, ne = 1, cs = 2, cc = 3, mi = 4, pl = 5, vs = 6, vc = 7,
243 hi = 8, ls = 9, ge = 10, lt = 11, gt = 12, le = 13, al = 14,
244 hs = 2, lo = 3,
245}
246
247------------------------------------------------------------------------------
248
249local parse_reg_type
250
251local function parse_reg(expr, shift, no_vreg)
252 if not expr then werror("expected register name") end
253 local tname, ovreg = match(expr, "^([%w_]+):(@?%l%d+)$")
254 if not tname then
255 tname, ovreg = match(expr, "^([%w_]+):(R[xwqdshb]%b())$")
256 end
257 local tp = map_type[tname or expr]
258 if tp then
259 local reg = ovreg or tp.reg
260 if not reg then
261 werror("type `"..(tname or expr).."' needs a register override")
262 end
263 expr = reg
264 end
265 local ok31, rt, r = match(expr, "^(@?)([xwqdshb])([123]?[0-9])$")
266 if r then
267 r = tonumber(r)
268 if r <= 30 or (r == 31 and ok31 ~= "" or (rt ~= "w" and rt ~= "x")) then
269 if not parse_reg_type then
270 parse_reg_type = rt
271 elseif parse_reg_type ~= rt then
272 werror("register size mismatch")
273 end
274 return shl(r, shift), tp
275 end
276 end
277 local vrt, vreg = match(expr, "^R([xwqdshb])(%b())$")
278 if vreg then
279 if not parse_reg_type then
280 parse_reg_type = vrt
281 elseif parse_reg_type ~= vrt then
282 werror("register size mismatch")
283 end
284 if not no_vreg then waction("VREG", shift, vreg) end
285 return 0
286 end
287 werror("bad register name `"..expr.."'")
288end
289
290local function parse_reg_base(expr)
291 if expr == "sp" then return 0x3e0 end
292 local base, tp = parse_reg(expr, 5)
293 if parse_reg_type ~= "x" then werror("bad register type") end
294 parse_reg_type = false
295 return base, tp
296end
297
298local parse_ctx = {}
299
300local loadenv = setfenv and function(s)
301 local code = loadstring(s, "")
302 if code then setfenv(code, parse_ctx) end
303 return code
304end or function(s)
305 return load(s, "", nil, parse_ctx)
306end
307
308-- Try to parse simple arithmetic, too, since some basic ops are aliases.
309local function parse_number(n)
310 local x = tonumber(n)
311 if x then return x end
312 local code = loadenv("return "..n)
313 if code then
314 local ok, y = pcall(code)
315 if ok and type(y) == "number" then return y end
316 end
317 return nil
318end
319
320local function parse_imm(imm, bits, shift, scale, signed)
321 imm = match(imm, "^#(.*)$")
322 if not imm then werror("expected immediate operand") end
323 local n = parse_number(imm)
324 if n then
325 local m = sar(n, scale)
326 if shl(m, scale) == n then
327 if signed then
328 local s = sar(m, bits-1)
329 if s == 0 then return shl(m, shift)
330 elseif s == -1 then return shl(m + shl(1, bits), shift) end
331 else
332 if sar(m, bits) == 0 then return shl(m, shift) end
333 end
334 end
335 werror("out of range immediate `"..imm.."'")
336 else
337 waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm)
338 return 0
339 end
340end
341
342local function parse_imm12(imm)
343 imm = match(imm, "^#(.*)$")
344 if not imm then werror("expected immediate operand") end
345 local n = parse_number(imm)
346 if n then
347 if shr(n, 12) == 0 then
348 return shl(n, 10)
349 elseif band(n, 0xff000fff) == 0 then
350 return shr(n, 2) + 0x00400000
351 end
352 werror("out of range immediate `"..imm.."'")
353 else
354 waction("IMM12", 0, imm)
355 return 0
356 end
357end
358
359local function parse_imm13(imm)
360 imm = match(imm, "^#(.*)$")
361 if not imm then werror("expected immediate operand") end
362 local n = parse_number(imm)
363 local r64 = parse_reg_type == "x"
364 if n and n % 1 == 0 and n >= 0 and n <= 0xffffffff then
365 local inv = false
366 if band(n, 1) == 1 then n = bit.bnot(n); inv = true end
367 local t = {}
368 for i=1,32 do t[i] = band(n, 1); n = shr(n, 1) end
369 local b = table.concat(t)
370 b = b..(r64 and (inv and "1" or "0"):rep(32) or b)
371 local p0, p1, p0a, p1a = b:match("^(0+)(1+)(0*)(1*)")
372 if p0 then
373 local w = p1a == "" and (r64 and 64 or 32) or #p1+#p0a
374 if band(w, w-1) == 0 and b == b:sub(1, w):rep(64/w) then
375 local s = band(-2*w, 0x3f) - 1
376 if w == 64 then s = s + 0x1000 end
377 if inv then
378 return shl(w-#p1-#p0, 16) + shl(s+w-#p1, 10)
379 else
380 return shl(w-#p0, 16) + shl(s+#p1, 10)
381 end
382 end
383 end
384 werror("out of range immediate `"..imm.."'")
385 elseif r64 then
386 waction("IMM13X", 0, format("(unsigned int)(%s)", imm))
387 actargs[#actargs+1] = format("(unsigned int)((unsigned long long)(%s)>>32)", imm)
388 return 0
389 else
390 waction("IMM13W", 0, imm)
391 return 0
392 end
393end
394
395local function parse_imm6(imm)
396 imm = match(imm, "^#(.*)$")
397 if not imm then werror("expected immediate operand") end
398 local n = parse_number(imm)
399 if n then
400 if n >= 0 and n <= 63 then
401 return shl(band(n, 0x1f), 19) + (n >= 32 and 0x80000000 or 0)
402 end
403 werror("out of range immediate `"..imm.."'")
404 else
405 waction("IMM6", 0, imm)
406 return 0
407 end
408end
409
410local function parse_imm_load(imm, scale)
411 local n = parse_number(imm)
412 if n then
413 local m = sar(n, scale)
414 if shl(m, scale) == n and m >= 0 and m < 0x1000 then
415 return shl(m, 10) + 0x01000000 -- Scaled, unsigned 12 bit offset.
416 elseif n >= -256 and n < 256 then
417 return shl(band(n, 511), 12) -- Unscaled, signed 9 bit offset.
418 end
419 werror("out of range immediate `"..imm.."'")
420 else
421 waction("IMML", scale, imm)
422 return 0
423 end
424end
425
426local function parse_fpimm(imm)
427 imm = match(imm, "^#(.*)$")
428 if not imm then werror("expected immediate operand") end
429 local n = parse_number(imm)
430 if n then
431 local m, e = math.frexp(n)
432 local s, e2 = 0, band(e-2, 7)
433 if m < 0 then m = -m; s = 0x00100000 end
434 m = m*32-16
435 if m % 1 == 0 and m >= 0 and m <= 15 and sar(shl(e2, 29), 29)+2 == e then
436 return s + shl(e2, 17) + shl(m, 13)
437 end
438 werror("out of range immediate `"..imm.."'")
439 else
440 werror("NYI fpimm action")
441 end
442end
443
444local function parse_shift(expr)
445 local s, s2 = match(expr, "^(%S+)%s*(.*)$")
446 s = map_shift[s]
447 if not s then werror("expected shift operand") end
448 return parse_imm(s2, 6, 10, 0, false) + shl(s, 22)
449end
450
451local function parse_lslx16(expr)
452 local n = match(expr, "^lsl%s*#(%d+)$")
453 n = tonumber(n)
454 if not n then werror("expected shift operand") end
455 if band(n, parse_reg_type == "x" and 0xffffffcf or 0xffffffef) ~= 0 then
456 werror("bad shift amount")
457 end
458 return shl(n, 17)
459end
460
461local function parse_extend(expr)
462 local s, s2 = match(expr, "^(%S+)%s*(.*)$")
463 if s == "lsl" then
464 s = parse_reg_type == "x" and 3 or 2
465 else
466 s = map_extend[s]
467 end
468 if not s then werror("expected extend operand") end
469 return (s2 == "" and 0 or parse_imm(s2, 3, 10, 0, false)) + shl(s, 13)
470end
471
472local function parse_cond(expr, inv)
473 local c = map_cond[expr]
474 if not c then werror("expected condition operand") end
475 return shl(bit.bxor(c, inv), 12)
476end
477
478local function parse_load(params, nparams, n, op)
479 if params[n+2] then werror("too many operands") end
480 local scale = shr(op, 30)
481 local pn, p2 = params[n], params[n+1]
482 local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$")
483 if not p1 then
484 if not p2 then
485 local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$")
486 if reg and tailr ~= "" then
487 local base, tp = parse_reg_base(reg)
488 if tp then
489 waction("IMML", scale, format(tp.ctypefmt, tailr))
490 return op + base
491 end
492 end
493 end
494 werror("expected address operand")
495 end
496 if p2 then
497 if wb == "!" then werror("bad use of '!'") end
498 op = op + parse_reg_base(p1) + parse_imm(p2, 9, 12, 0, true) + 0x400
499 elseif wb == "!" then
500 local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$")
501 if not p1a then werror("bad use of '!'") end
502 op = op + parse_reg_base(p1a) + parse_imm(p2a, 9, 12, 0, true) + 0xc00
503 else
504 local p1a, p2a = match(p1, "^([^,%s]*)%s*(.*)$")
505 op = op + parse_reg_base(p1a)
506 if p2a ~= "" then
507 local imm = match(p2a, "^,%s*#(.*)$")
508 if imm then
509 op = op + parse_imm_load(imm, scale)
510 else
511 local p2b, p3b, p3s = match(p2a, "^,%s*([^,%s]*)%s*,?%s*(%S*)%s*(.*)$")
512 op = op + parse_reg(p2b, 16) + 0x00200800
513 if parse_reg_type ~= "x" and parse_reg_type ~= "w" then
514 werror("bad index register type")
515 end
516 if p3b == "" then
517 if parse_reg_type ~= "x" then werror("bad index register type") end
518 op = op + 0x6000
519 else
520 if p3s == "" or p3s == "#0" then
521 elseif p3s == "#"..scale then
522 op = op + 0x1000
523 else
524 werror("bad scale")
525 end
526 if parse_reg_type == "x" then
527 if p3b == "lsl" and p3s ~= "" then op = op + 0x6000
528 elseif p3b == "sxtx" then op = op + 0xe000
529 else
530 werror("bad extend/shift specifier")
531 end
532 else
533 if p3b == "uxtw" then op = op + 0x4000
534 elseif p3b == "sxtw" then op = op + 0xc000
535 else
536 werror("bad extend/shift specifier")
537 end
538 end
539 end
540 end
541 else
542 if wb == "!" then werror("bad use of '!'") end
543 op = op + 0x01000000
544 end
545 end
546 return op
547end
548
549local function parse_load_pair(params, nparams, n, op)
550 if params[n+2] then werror("too many operands") end
551 local pn, p2 = params[n], params[n+1]
552 local scale = shr(op, 30) == 0 and 2 or 3
553 local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$")
554 if not p1 then
555 if not p2 then
556 local reg, tailr = match(pn, "^([%w_:]+)%s*(.*)$")
557 if reg and tailr ~= "" then
558 local base, tp = parse_reg_base(reg)
559 if tp then
560 waction("IMM", 32768+7*32+15+scale*1024, format(tp.ctypefmt, tailr))
561 return op + base + 0x01000000
562 end
563 end
564 end
565 werror("expected address operand")
566 end
567 if p2 then
568 if wb == "!" then werror("bad use of '!'") end
569 op = op + 0x00800000
570 else
571 local p1a, p2a = match(p1, "^([^,%s]*)%s*,%s*(.*)$")
572 if p1a then p1, p2 = p1a, p2a else p2 = "#0" end
573 op = op + (wb == "!" and 0x01800000 or 0x01000000)
574 end
575 return op + parse_reg_base(p1) + parse_imm(p2, 7, 15, scale, true)
576end
577
578local function parse_label(label, def)
579 local prefix = label:sub(1, 2)
580 -- =>label (pc label reference)
581 if prefix == "=>" then
582 return "PC", 0, label:sub(3)
583 end
584 -- ->name (global label reference)
585 if prefix == "->" then
586 return "LG", map_global[label:sub(3)]
587 end
588 if def then
589 -- [1-9] (local label definition)
590 if match(label, "^[1-9]$") then
591 return "LG", 10+tonumber(label)
592 end
593 else
594 -- [<>][1-9] (local label reference)
595 local dir, lnum = match(label, "^([<>])([1-9])$")
596 if dir then -- Fwd: 1-9, Bkwd: 11-19.
597 return "LG", lnum + (dir == ">" and 0 or 10)
598 end
599 -- extern label (extern label reference)
600 local extname = match(label, "^extern%s+(%S+)$")
601 if extname then
602 return "EXT", map_extern[extname]
603 end
604 -- &expr (pointer)
605 if label:sub(1, 1) == "&" then
606 return "A", 0, format("(ptrdiff_t)(%s)", label:sub(2))
607 end
608 end
609end
610
611local function branch_type(op)
612 if band(op, 0x7c000000) == 0x14000000 then return 0 -- B, BL
613 elseif shr(op, 24) == 0x54 or band(op, 0x7e000000) == 0x34000000 or
614 band(op, 0x3b000000) == 0x18000000 then
615 return 0x800 -- B.cond, CBZ, CBNZ, LDR* literal
616 elseif band(op, 0x7e000000) == 0x36000000 then return 0x1000 -- TBZ, TBNZ
617 elseif band(op, 0x9f000000) == 0x10000000 then return 0x2000 -- ADR
618 elseif band(op, 0x9f000000) == band(0x90000000) then return 0x3000 -- ADRP
619 else
620 assert(false, "unknown branch type")
621 end
622end
623
624------------------------------------------------------------------------------
625
626local map_op, op_template
627
628local function op_alias(opname, f)
629 return function(params, nparams)
630 if not params then return "-> "..opname:sub(1, -3) end
631 f(params, nparams)
632 op_template(params, map_op[opname], nparams)
633 end
634end
635
636local function alias_bfx(p)
637 p[4] = "#("..p[3]:sub(2)..")+("..p[4]:sub(2)..")-1"
638end
639
640local function alias_bfiz(p)
641 parse_reg(p[1], 0, true)
642 if parse_reg_type == "w" then
643 p[3] = "#(32-("..p[3]:sub(2).."))%32"
644 p[4] = "#("..p[4]:sub(2)..")-1"
645 else
646 p[3] = "#(64-("..p[3]:sub(2).."))%64"
647 p[4] = "#("..p[4]:sub(2)..")-1"
648 end
649end
650
651local alias_lslimm = op_alias("ubfm_4", function(p)
652 parse_reg(p[1], 0, true)
653 local sh = p[3]:sub(2)
654 if parse_reg_type == "w" then
655 p[3] = "#(32-("..sh.."))%32"
656 p[4] = "#31-("..sh..")"
657 else
658 p[3] = "#(64-("..sh.."))%64"
659 p[4] = "#63-("..sh..")"
660 end
661end)
662
663-- Template strings for ARM instructions.
664map_op = {
665 -- Basic data processing instructions.
666 add_3 = "0b000000DNMg|11000000pDpNIg|8b206000pDpNMx",
667 add_4 = "0b000000DNMSg|0b200000DNMXg|8b200000pDpNMXx|8b200000pDpNxMwX",
668 adds_3 = "2b000000DNMg|31000000DpNIg|ab206000DpNMx",
669 adds_4 = "2b000000DNMSg|2b200000DNMXg|ab200000DpNMXx|ab200000DpNxMwX",
670 cmn_2 = "2b00001fNMg|3100001fpNIg|ab20601fpNMx",
671 cmn_3 = "2b00001fNMSg|2b20001fNMXg|ab20001fpNMXx|ab20001fpNxMwX",
672
673 sub_3 = "4b000000DNMg|51000000pDpNIg|cb206000pDpNMx",
674 sub_4 = "4b000000DNMSg|4b200000DNMXg|cb200000pDpNMXx|cb200000pDpNxMwX",
675 subs_3 = "6b000000DNMg|71000000DpNIg|eb206000DpNMx",
676 subs_4 = "6b000000DNMSg|6b200000DNMXg|eb200000DpNMXx|eb200000DpNxMwX",
677 cmp_2 = "6b00001fNMg|7100001fpNIg|eb20601fpNMx",
678 cmp_3 = "6b00001fNMSg|6b20001fNMXg|eb20001fpNMXx|eb20001fpNxMwX",
679
680 neg_2 = "4b0003e0DMg",
681 neg_3 = "4b0003e0DMSg",
682 negs_2 = "6b0003e0DMg",
683 negs_3 = "6b0003e0DMSg",
684
685 adc_3 = "1a000000DNMg",
686 adcs_3 = "3a000000DNMg",
687 sbc_3 = "5a000000DNMg",
688 sbcs_3 = "7a000000DNMg",
689 ngc_2 = "5a0003e0DMg",
690 ngcs_2 = "7a0003e0DMg",
691
692 and_3 = "0a000000DNMg|12000000pDNig",
693 and_4 = "0a000000DNMSg",
694 orr_3 = "2a000000DNMg|32000000pDNig",
695 orr_4 = "2a000000DNMSg",
696 eor_3 = "4a000000DNMg|52000000pDNig",
697 eor_4 = "4a000000DNMSg",
698 ands_3 = "6a000000DNMg|72000000DNig",
699 ands_4 = "6a000000DNMSg",
700 tst_2 = "6a00001fNMg|7200001fNig",
701 tst_3 = "6a00001fNMSg",
702
703 bic_3 = "0a200000DNMg",
704 bic_4 = "0a200000DNMSg",
705 orn_3 = "2a200000DNMg",
706 orn_4 = "2a200000DNMSg",
707 eon_3 = "4a200000DNMg",
708 eon_4 = "4a200000DNMSg",
709 bics_3 = "6a200000DNMg",
710 bics_4 = "6a200000DNMSg",
711
712 movn_2 = "12800000DWg",
713 movn_3 = "12800000DWRg",
714 movz_2 = "52800000DWg",
715 movz_3 = "52800000DWRg",
716 movk_2 = "72800000DWg",
717 movk_3 = "72800000DWRg",
718
719 -- TODO: this doesn't cover all valid immediates for mov reg, #imm.
720 mov_2 = "2a0003e0DMg|52800000DW|320003e0pDig|11000000pDpNg",
721 mov_3 = "2a0003e0DMSg",
722 mvn_2 = "2a2003e0DMg",
723 mvn_3 = "2a2003e0DMSg",
724
725 adr_2 = "10000000DBx",
726 adrp_2 = "90000000DBx",
727
728 csel_4 = "1a800000DNMCg",
729 csinc_4 = "1a800400DNMCg",
730 csinv_4 = "5a800000DNMCg",
731 csneg_4 = "5a800400DNMCg",
732 cset_2 = "1a9f07e0Dcg",
733 csetm_2 = "5a9f03e0Dcg",
734 cinc_3 = "1a800400DNmcg",
735 cinv_3 = "5a800000DNmcg",
736 cneg_3 = "5a800400DNmcg",
737
738 ccmn_4 = "3a400000NMVCg|3a400800N5VCg",
739 ccmp_4 = "7a400000NMVCg|7a400800N5VCg",
740
741 madd_4 = "1b000000DNMAg",
742 msub_4 = "1b008000DNMAg",
743 mul_3 = "1b007c00DNMg",
744 mneg_3 = "1b00fc00DNMg",
745
746 smaddl_4 = "9b200000DxNMwAx",
747 smsubl_4 = "9b208000DxNMwAx",
748 smull_3 = "9b207c00DxNMw",
749 smnegl_3 = "9b20fc00DxNMw",
750 smulh_3 = "9b407c00DNMx",
751 umaddl_4 = "9ba00000DxNMwAx",
752 umsubl_4 = "9ba08000DxNMwAx",
753 umull_3 = "9ba07c00DxNMw",
754 umnegl_3 = "9ba0fc00DxNMw",
755 umulh_3 = "9bc07c00DNMx",
756
757 udiv_3 = "1ac00800DNMg",
758 sdiv_3 = "1ac00c00DNMg",
759
760 -- Bit operations.
761 sbfm_4 = "13000000DN12w|93400000DN12x",
762 bfm_4 = "33000000DN12w|b3400000DN12x",
763 ubfm_4 = "53000000DN12w|d3400000DN12x",
764 extr_4 = "13800000DNM2w|93c00000DNM2x",
765
766 sxtb_2 = "13001c00DNw|93401c00DNx",
767 sxth_2 = "13003c00DNw|93403c00DNx",
768 sxtw_2 = "93407c00DxNw",
769 uxtb_2 = "53001c00DNw",
770 uxth_2 = "53003c00DNw",
771
772 sbfx_4 = op_alias("sbfm_4", alias_bfx),
773 bfxil_4 = op_alias("bfm_4", alias_bfx),
774 ubfx_4 = op_alias("ubfm_4", alias_bfx),
775 sbfiz_4 = op_alias("sbfm_4", alias_bfiz),
776 bfi_4 = op_alias("bfm_4", alias_bfiz),
777 ubfiz_4 = op_alias("ubfm_4", alias_bfiz),
778
779 lsl_3 = function(params, nparams)
780 if params and params[3]:byte() == 35 then
781 return alias_lslimm(params, nparams)
782 else
783 return op_template(params, "1ac02000DNMg", nparams)
784 end
785 end,
786 lsr_3 = "1ac02400DNMg|53007c00DN1w|d340fc00DN1x",
787 asr_3 = "1ac02800DNMg|13007c00DN1w|9340fc00DN1x",
788 ror_3 = "1ac02c00DNMg|13800000DNm2w|93c00000DNm2x",
789
790 clz_2 = "5ac01000DNg",
791 cls_2 = "5ac01400DNg",
792 rbit_2 = "5ac00000DNg",
793 rev_2 = "5ac00800DNw|dac00c00DNx",
794 rev16_2 = "5ac00400DNg",
795 rev32_2 = "dac00800DNx",
796
797 -- Loads and stores.
798 ["strb_*"] = "38000000DwL",
799 ["ldrb_*"] = "38400000DwL",
800 ["ldrsb_*"] = "38c00000DwL|38800000DxL",
801 ["strh_*"] = "78000000DwL",
802 ["ldrh_*"] = "78400000DwL",
803 ["ldrsh_*"] = "78c00000DwL|78800000DxL",
804 ["str_*"] = "b8000000DwL|f8000000DxL|bc000000DsL|fc000000DdL",
805 ["ldr_*"] = "18000000DwB|58000000DxB|1c000000DsB|5c000000DdB|b8400000DwL|f8400000DxL|bc400000DsL|fc400000DdL",
806 ["ldrsw_*"] = "98000000DxB|b8800000DxL",
807 -- NOTE: ldur etc. are handled by ldr et al.
808
809 ["stp_*"] = "28000000DAwP|a8000000DAxP|2c000000DAsP|6c000000DAdP",
810 ["ldp_*"] = "28400000DAwP|a8400000DAxP|2c400000DAsP|6c400000DAdP",
811 ["ldpsw_*"] = "68400000DAxP",
812
813 -- Branches.
814 b_1 = "14000000B",
815 bl_1 = "94000000B",
816 blr_1 = "d63f0000Nx",
817 br_1 = "d61f0000Nx",
818 ret_0 = "d65f03c0",
819 ret_1 = "d65f0000Nx",
820 -- b.cond is added below.
821 cbz_2 = "34000000DBg",
822 cbnz_2 = "35000000DBg",
823 tbz_3 = "36000000DTBw|36000000DTBx",
824 tbnz_3 = "37000000DTBw|37000000DTBx",
825
826 -- ARM64e: Pointer authentication codes (PAC).
827 blraaz_1 = "d63f081fNx",
828 braa_2 = "d71f0800NDx",
829 braaz_1 = "d61f081fNx",
830 pacibsp_0 = "d503237f",
831 retab_0 = "d65f0fff",
832
833 -- Miscellaneous instructions.
834 -- TODO: hlt, hvc, smc, svc, eret, dcps[123], drps, mrs, msr
835 -- TODO: sys, sysl, ic, dc, at, tlbi
836 -- TODO: hint, yield, wfe, wfi, sev, sevl
837 -- TODO: clrex, dsb, dmb, isb
838 nop_0 = "d503201f",
839 brk_0 = "d4200000",
840 brk_1 = "d4200000W",
841
842 -- Floating point instructions.
843 fmov_2 = "1e204000DNf|1e260000DwNs|1e270000DsNw|9e660000DxNd|9e670000DdNx|1e201000DFf",
844 fabs_2 = "1e20c000DNf",
845 fneg_2 = "1e214000DNf",
846 fsqrt_2 = "1e21c000DNf",
847
848 fcvt_2 = "1e22c000DdNs|1e624000DsNd",
849
850 -- TODO: half-precision and fixed-point conversions.
851 fcvtas_2 = "1e240000DwNs|9e240000DxNs|1e640000DwNd|9e640000DxNd",
852 fcvtau_2 = "1e250000DwNs|9e250000DxNs|1e650000DwNd|9e650000DxNd",
853 fcvtms_2 = "1e300000DwNs|9e300000DxNs|1e700000DwNd|9e700000DxNd",
854 fcvtmu_2 = "1e310000DwNs|9e310000DxNs|1e710000DwNd|9e710000DxNd",
855 fcvtns_2 = "1e200000DwNs|9e200000DxNs|1e600000DwNd|9e600000DxNd",
856 fcvtnu_2 = "1e210000DwNs|9e210000DxNs|1e610000DwNd|9e610000DxNd",
857 fcvtps_2 = "1e280000DwNs|9e280000DxNs|1e680000DwNd|9e680000DxNd",
858 fcvtpu_2 = "1e290000DwNs|9e290000DxNs|1e690000DwNd|9e690000DxNd",
859 fcvtzs_2 = "1e380000DwNs|9e380000DxNs|1e780000DwNd|9e780000DxNd",
860 fcvtzu_2 = "1e390000DwNs|9e390000DxNs|1e790000DwNd|9e790000DxNd",
861
862 scvtf_2 = "1e220000DsNw|9e220000DsNx|1e620000DdNw|9e620000DdNx",
863 ucvtf_2 = "1e230000DsNw|9e230000DsNx|1e630000DdNw|9e630000DdNx",
864
865 frintn_2 = "1e244000DNf",
866 frintp_2 = "1e24c000DNf",
867 frintm_2 = "1e254000DNf",
868 frintz_2 = "1e25c000DNf",
869 frinta_2 = "1e264000DNf",
870 frintx_2 = "1e274000DNf",
871 frinti_2 = "1e27c000DNf",
872
873 fadd_3 = "1e202800DNMf",
874 fsub_3 = "1e203800DNMf",
875 fmul_3 = "1e200800DNMf",
876 fnmul_3 = "1e208800DNMf",
877 fdiv_3 = "1e201800DNMf",
878
879 fmadd_4 = "1f000000DNMAf",
880 fmsub_4 = "1f008000DNMAf",
881 fnmadd_4 = "1f200000DNMAf",
882 fnmsub_4 = "1f208000DNMAf",
883
884 fmax_3 = "1e204800DNMf",
885 fmaxnm_3 = "1e206800DNMf",
886 fmin_3 = "1e205800DNMf",
887 fminnm_3 = "1e207800DNMf",
888
889 fcmp_2 = "1e202000NMf|1e202008NZf",
890 fcmpe_2 = "1e202010NMf|1e202018NZf",
891
892 fccmp_4 = "1e200400NMVCf",
893 fccmpe_4 = "1e200410NMVCf",
894
895 fcsel_4 = "1e200c00DNMCf",
896
897 -- TODO: crc32*, aes*, sha*, pmull
898 -- TODO: SIMD instructions.
899}
900
901for cond,c in pairs(map_cond) do
902 map_op["b"..cond.."_1"] = tohex(0x54000000+c).."B"
903end
904
905------------------------------------------------------------------------------
906
907-- Handle opcodes defined with template strings.
908local function parse_template(params, template, nparams, pos)
909 local op = tonumber(template:sub(1, 8), 16)
910 local n = 1
911 local rtt = {}
912
913 parse_reg_type = false
914
915 -- Process each character.
916 for p in gmatch(template:sub(9), ".") do
917 local q = params[n]
918 if p == "D" then
919 op = op + parse_reg(q, 0); n = n + 1
920 elseif p == "N" then
921 op = op + parse_reg(q, 5); n = n + 1
922 elseif p == "M" then
923 op = op + parse_reg(q, 16); n = n + 1
924 elseif p == "A" then
925 op = op + parse_reg(q, 10); n = n + 1
926 elseif p == "m" then
927 op = op + parse_reg(params[n-1], 16)
928
929 elseif p == "p" then
930 if q == "sp" then params[n] = "@x31" end
931 elseif p == "g" then
932 if parse_reg_type == "x" then
933 op = op + 0x80000000
934 elseif parse_reg_type ~= "w" then
935 werror("bad register type")
936 end
937 parse_reg_type = false
938 elseif p == "f" then
939 if parse_reg_type == "d" then
940 op = op + 0x00400000
941 elseif parse_reg_type ~= "s" then
942 werror("bad register type")
943 end
944 parse_reg_type = false
945 elseif p == "x" or p == "w" or p == "d" or p == "s" then
946 if parse_reg_type ~= p then
947 werror("register size mismatch")
948 end
949 parse_reg_type = false
950
951 elseif p == "L" then
952 op = parse_load(params, nparams, n, op)
953 elseif p == "P" then
954 op = parse_load_pair(params, nparams, n, op)
955
956 elseif p == "B" then
957 local mode, v, s = parse_label(q, false); n = n + 1
958 if not mode then werror("bad label `"..q.."'") end
959 local m = branch_type(op)
960 if mode == "A" then
961 waction("REL_"..mode, v+m, format("(unsigned int)(%s)", s))
962 actargs[#actargs+1] = format("(unsigned int)((%s)>>32)", s)
963 else
964 waction("REL_"..mode, v+m, s, 1)
965 end
966
967 elseif p == "I" then
968 op = op + parse_imm12(q); n = n + 1
969 elseif p == "i" then
970 op = op + parse_imm13(q); n = n + 1
971 elseif p == "W" then
972 op = op + parse_imm(q, 16, 5, 0, false); n = n + 1
973 elseif p == "T" then
974 op = op + parse_imm6(q); n = n + 1
975 elseif p == "1" then
976 op = op + parse_imm(q, 6, 16, 0, false); n = n + 1
977 elseif p == "2" then
978 op = op + parse_imm(q, 6, 10, 0, false); n = n + 1
979 elseif p == "5" then
980 op = op + parse_imm(q, 5, 16, 0, false); n = n + 1
981 elseif p == "V" then
982 op = op + parse_imm(q, 4, 0, 0, false); n = n + 1
983 elseif p == "F" then
984 op = op + parse_fpimm(q); n = n + 1
985 elseif p == "Z" then
986 if q ~= "#0" and q ~= "#0.0" then werror("expected zero immediate") end
987 n = n + 1
988
989 elseif p == "S" then
990 op = op + parse_shift(q); n = n + 1
991 elseif p == "X" then
992 op = op + parse_extend(q); n = n + 1
993 elseif p == "R" then
994 op = op + parse_lslx16(q); n = n + 1
995 elseif p == "C" then
996 op = op + parse_cond(q, 0); n = n + 1
997 elseif p == "c" then
998 op = op + parse_cond(q, 1); n = n + 1
999
1000 else
1001 assert(false)
1002 end
1003 end
1004 wputpos(pos, op)
1005end
1006
1007function op_template(params, template, nparams)
1008 if not params then return template:gsub("%x%x%x%x%x%x%x%x", "") end
1009
1010 -- Limit number of section buffer positions used by a single dasm_put().
1011 -- A single opcode needs a maximum of 4 positions.
1012 if secpos+4 > maxsecpos then wflush() end
1013 local pos = wpos()
1014 local lpos, apos, spos = #actlist, #actargs, secpos
1015
1016 local ok, err
1017 for t in gmatch(template, "[^|]+") do
1018 ok, err = pcall(parse_template, params, t, nparams, pos)
1019 if ok then return end
1020 secpos = spos
1021 actlist[lpos+1] = nil
1022 actlist[lpos+2] = nil
1023 actlist[lpos+3] = nil
1024 actlist[lpos+4] = nil
1025 actargs[apos+1] = nil
1026 actargs[apos+2] = nil
1027 actargs[apos+3] = nil
1028 actargs[apos+4] = nil
1029 end
1030 error(err, 0)
1031end
1032
1033map_op[".template__"] = op_template
1034
1035------------------------------------------------------------------------------
1036
1037-- Pseudo-opcode to mark the position where the action list is to be emitted.
1038map_op[".actionlist_1"] = function(params)
1039 if not params then return "cvar" end
1040 local name = params[1] -- No syntax check. You get to keep the pieces.
1041 wline(function(out) writeactions(out, name) end)
1042end
1043
1044-- Pseudo-opcode to mark the position where the global enum is to be emitted.
1045map_op[".globals_1"] = function(params)
1046 if not params then return "prefix" end
1047 local prefix = params[1] -- No syntax check. You get to keep the pieces.
1048 wline(function(out) writeglobals(out, prefix) end)
1049end
1050
1051-- Pseudo-opcode to mark the position where the global names are to be emitted.
1052map_op[".globalnames_1"] = function(params)
1053 if not params then return "cvar" end
1054 local name = params[1] -- No syntax check. You get to keep the pieces.
1055 wline(function(out) writeglobalnames(out, name) end)
1056end
1057
1058-- Pseudo-opcode to mark the position where the extern names are to be emitted.
1059map_op[".externnames_1"] = function(params)
1060 if not params then return "cvar" end
1061 local name = params[1] -- No syntax check. You get to keep the pieces.
1062 wline(function(out) writeexternnames(out, name) end)
1063end
1064
1065------------------------------------------------------------------------------
1066
1067-- Label pseudo-opcode (converted from trailing colon form).
1068map_op[".label_1"] = function(params)
1069 if not params then return "[1-9] | ->global | =>pcexpr" end
1070 if secpos+1 > maxsecpos then wflush() end
1071 local mode, n, s = parse_label(params[1], true)
1072 if not mode or mode == "EXT" then werror("bad label definition") end
1073 waction("LABEL_"..mode, n, s, 1)
1074end
1075
1076------------------------------------------------------------------------------
1077
1078-- Pseudo-opcodes for data storage.
1079local function op_data(params)
1080 if not params then return "imm..." end
1081 local sz = params.op == ".long" and 4 or 8
1082 for _,p in ipairs(params) do
1083 local imm = parse_number(p)
1084 if imm then
1085 local n = tobit(imm)
1086 if n == imm or (n < 0 and n + 2^32 == imm) then
1087 wputw(n < 0 and n + 2^32 or n)
1088 if sz == 8 then
1089 wputw(imm < 0 and 0xffffffff or 0)
1090 end
1091 elseif sz == 4 then
1092 werror("bad immediate `"..p.."'")
1093 else
1094 imm = nil
1095 end
1096 end
1097 if not imm then
1098 local mode, v, s = parse_label(p, false)
1099 if sz == 4 then
1100 if mode then werror("label does not fit into .long") end
1101 waction("IMMV", 0, p)
1102 elseif mode and mode ~= "A" then
1103 waction("REL_"..mode, v+0x8000, s, 1)
1104 else
1105 if mode == "A" then p = s end
1106 waction("IMMV", 0, format("(unsigned int)(%s)", p))
1107 waction("IMMV", 0, format("(unsigned int)((unsigned long long)(%s)>>32)", p))
1108 end
1109 end
1110 if secpos+2 > maxsecpos then wflush() end
1111 end
1112end
1113map_op[".long_*"] = op_data
1114map_op[".quad_*"] = op_data
1115map_op[".addr_*"] = op_data
1116
1117-- Alignment pseudo-opcode.
1118map_op[".align_1"] = function(params)
1119 if not params then return "numpow2" end
1120 if secpos+1 > maxsecpos then wflush() end
1121 local align = tonumber(params[1])
1122 if align then
1123 local x = align
1124 -- Must be a power of 2 in the range (2 ... 256).
1125 for i=1,8 do
1126 x = x / 2
1127 if x == 1 then
1128 waction("ALIGN", align-1, nil, 1) -- Action byte is 2**n-1.
1129 return
1130 end
1131 end
1132 end
1133 werror("bad alignment")
1134end
1135
1136------------------------------------------------------------------------------
1137
1138-- Pseudo-opcode for (primitive) type definitions (map to C types).
1139map_op[".type_3"] = function(params, nparams)
1140 if not params then
1141 return nparams == 2 and "name, ctype" or "name, ctype, reg"
1142 end
1143 local name, ctype, reg = params[1], params[2], params[3]
1144 if not match(name, "^[%a_][%w_]*$") then
1145 werror("bad type name `"..name.."'")
1146 end
1147 local tp = map_type[name]
1148 if tp then
1149 werror("duplicate type `"..name.."'")
1150 end
1151 -- Add #type to defines. A bit unclean to put it in map_archdef.
1152 map_archdef["#"..name] = "sizeof("..ctype..")"
1153 -- Add new type and emit shortcut define.
1154 local num = ctypenum + 1
1155 map_type[name] = {
1156 ctype = ctype,
1157 ctypefmt = format("Dt%X(%%s)", num),
1158 reg = reg,
1159 }
1160 wline(format("#define Dt%X(_V) (int)(ptrdiff_t)&(((%s *)0)_V)", num, ctype))
1161 ctypenum = num
1162end
1163map_op[".type_2"] = map_op[".type_3"]
1164
1165-- Dump type definitions.
1166local function dumptypes(out, lvl)
1167 local t = {}
1168 for name in pairs(map_type) do t[#t+1] = name end
1169 sort(t)
1170 out:write("Type definitions:\n")
1171 for _,name in ipairs(t) do
1172 local tp = map_type[name]
1173 local reg = tp.reg or ""
1174 out:write(format(" %-20s %-20s %s\n", name, tp.ctype, reg))
1175 end
1176 out:write("\n")
1177end
1178
1179------------------------------------------------------------------------------
1180
1181-- Set the current section.
1182function _M.section(num)
1183 waction("SECTION", num)
1184 wflush(true) -- SECTION is a terminal action.
1185end
1186
1187------------------------------------------------------------------------------
1188
1189-- Dump architecture description.
1190function _M.dumparch(out)
1191 out:write(format("DynASM %s version %s, released %s\n\n",
1192 _info.arch, _info.version, _info.release))
1193 dumpactions(out)
1194end
1195
1196-- Dump all user defined elements.
1197function _M.dumpdef(out, lvl)
1198 dumptypes(out, lvl)
1199 dumpglobals(out, lvl)
1200 dumpexterns(out, lvl)
1201end
1202
1203------------------------------------------------------------------------------
1204
1205-- Pass callbacks from/to the DynASM core.
1206function _M.passcb(wl, we, wf, ww)
1207 wline, werror, wfatal, wwarn = wl, we, wf, ww
1208 return wflush
1209end
1210
1211-- Setup the arch-specific module.
1212function _M.setup(arch, opt)
1213 g_arch, g_opt = arch, opt
1214end
1215
1216-- Merge the core maps and the arch-specific maps.
1217function _M.mergemaps(map_coreop, map_def)
1218 setmetatable(map_op, { __index = map_coreop })
1219 setmetatable(map_def, { __index = map_archdef })
1220 return map_op, map_def
1221end
1222
1223return _M
1224
1225------------------------------------------------------------------------------
1226
diff --git a/dynasm/dasm_mips.h b/dynasm/dasm_mips.h
index 7f3d6c35..7800e933 100644
--- a/dynasm/dasm_mips.h
+++ b/dynasm/dasm_mips.h
@@ -21,7 +21,7 @@ enum {
21 /* The following actions need a buffer position. */ 21 /* The following actions need a buffer position. */
22 DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, 22 DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
23 /* The following actions also have an argument. */ 23 /* The following actions also have an argument. */
24 DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, 24 DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMS,
25 DASM__MAX 25 DASM__MAX
26}; 26};
27 27
@@ -69,7 +69,7 @@ struct dasm_State {
69 size_t lgsize; 69 size_t lgsize;
70 int *pclabels; /* PC label chains/pos ptrs. */ 70 int *pclabels; /* PC label chains/pos ptrs. */
71 size_t pcsize; 71 size_t pcsize;
72 void **globals; /* Array of globals (bias -10). */ 72 void **globals; /* Array of globals. */
73 dasm_Section *section; /* Pointer to active section. */ 73 dasm_Section *section; /* Pointer to active section. */
74 size_t codesize; /* Total size of all code sections. */ 74 size_t codesize; /* Total size of all code sections. */
75 int maxsection; /* 0 <= sectionidx < maxsection. */ 75 int maxsection; /* 0 <= sectionidx < maxsection. */
@@ -86,7 +86,6 @@ void dasm_init(Dst_DECL, int maxsection)
86{ 86{
87 dasm_State *D; 87 dasm_State *D;
88 size_t psz = 0; 88 size_t psz = 0;
89 int i;
90 Dst_REF = NULL; 89 Dst_REF = NULL;
91 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); 90 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
92 D = Dst_REF; 91 D = Dst_REF;
@@ -97,12 +96,7 @@ void dasm_init(Dst_DECL, int maxsection)
97 D->pcsize = 0; 96 D->pcsize = 0;
98 D->globals = NULL; 97 D->globals = NULL;
99 D->maxsection = maxsection; 98 D->maxsection = maxsection;
100 for (i = 0; i < maxsection; i++) { 99 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
101 D->sections[i].buf = NULL; /* Need this for pass3. */
102 D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i);
103 D->sections[i].bsize = 0;
104 D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */
105 }
106} 100}
107 101
108/* Free DynASM state. */ 102/* Free DynASM state. */
@@ -122,7 +116,7 @@ void dasm_free(Dst_DECL)
122void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) 116void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
123{ 117{
124 dasm_State *D = Dst_REF; 118 dasm_State *D = Dst_REF;
125 D->globals = gl - 10; /* Negative bias to compensate for locals. */ 119 D->globals = gl;
126 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); 120 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
127} 121}
128 122
@@ -147,6 +141,7 @@ void dasm_setup(Dst_DECL, const void *actionlist)
147 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); 141 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
148 for (i = 0; i < D->maxsection; i++) { 142 for (i = 0; i < D->maxsection; i++) {
149 D->sections[i].pos = DASM_SEC2POS(i); 143 D->sections[i].pos = DASM_SEC2POS(i);
144 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
150 D->sections[i].ofs = 0; 145 D->sections[i].ofs = 0;
151 } 146 }
152} 147}
@@ -155,10 +150,10 @@ void dasm_setup(Dst_DECL, const void *actionlist)
155#ifdef DASM_CHECKS 150#ifdef DASM_CHECKS
156#define CK(x, st) \ 151#define CK(x, st) \
157 do { if (!(x)) { \ 152 do { if (!(x)) { \
158 D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) 153 D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0)
159#define CKPL(kind, st) \ 154#define CKPL(kind, st) \
160 do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ 155 do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \
161 D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) 156 D->status = DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0)
162#else 157#else
163#define CK(x, st) ((void)0) 158#define CK(x, st) ((void)0)
164#define CKPL(kind, st) ((void)0) 159#define CKPL(kind, st) ((void)0)
@@ -231,7 +226,7 @@ void dasm_put(Dst_DECL, int start, ...)
231 *pl = -pos; /* Label exists now. */ 226 *pl = -pos; /* Label exists now. */
232 b[pos++] = ofs; /* Store pass1 offset estimate. */ 227 b[pos++] = ofs; /* Store pass1 offset estimate. */
233 break; 228 break;
234 case DASM_IMM: 229 case DASM_IMM: case DASM_IMMS:
235#ifdef DASM_CHECKS 230#ifdef DASM_CHECKS
236 CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I); 231 CK((n & ((1<<((ins>>10)&31))-1)) == 0, RANGE_I);
237#endif 232#endif
@@ -273,7 +268,7 @@ int dasm_link(Dst_DECL, size_t *szp)
273 268
274 { /* Handle globals not defined in this translation unit. */ 269 { /* Handle globals not defined in this translation unit. */
275 int idx; 270 int idx;
276 for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { 271 for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
277 int n = D->lglabels[idx]; 272 int n = D->lglabels[idx];
278 /* Undefined label: Collapse rel chain and replace with marker (< 0). */ 273 /* Undefined label: Collapse rel chain and replace with marker (< 0). */
279 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } 274 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
@@ -299,7 +294,7 @@ int dasm_link(Dst_DECL, size_t *szp)
299 case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; 294 case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
300 case DASM_REL_LG: case DASM_REL_PC: pos++; break; 295 case DASM_REL_LG: case DASM_REL_PC: pos++; break;
301 case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; 296 case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
302 case DASM_IMM: pos++; break; 297 case DASM_IMM: case DASM_IMMS: pos++; break;
303 } 298 }
304 } 299 }
305 stop: (void)0; 300 stop: (void)0;
@@ -314,7 +309,7 @@ int dasm_link(Dst_DECL, size_t *szp)
314 309
315#ifdef DASM_CHECKS 310#ifdef DASM_CHECKS
316#define CK(x, st) \ 311#define CK(x, st) \
317 do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) 312 do { if (!(x)) return DASM_S_##st|(int)(p-D->actionlist-1); } while (0)
318#else 313#else
319#define CK(x, st) ((void)0) 314#define CK(x, st) ((void)0)
320#endif 315#endif
@@ -349,25 +344,32 @@ int dasm_encode(Dst_DECL, void *buffer)
349 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000; 344 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000;
350 break; 345 break;
351 case DASM_REL_LG: 346 case DASM_REL_LG:
352 CK(n >= 0, UNDEF_LG); 347 if (n < 0) {
348 n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp);
349 goto patchrel;
350 }
353 /* fallthrough */ 351 /* fallthrough */
354 case DASM_REL_PC: 352 case DASM_REL_PC:
355 CK(n >= 0, UNDEF_PC); 353 CK(n >= 0, UNDEF_PC);
356 n = *DASM_POS2PTR(D, n); 354 n = *DASM_POS2PTR(D, n);
357 if (ins & 2048) 355 if (ins & 2048)
358 n = n - (int)((char *)cp - base); 356 n = (n + (int)(size_t)base) & 0x0fffffff;
359 else 357 else
360 n = (n + (int)base) & 0x0fffffff; 358 n = n - (int)((char *)cp - base);
361 patchrel: 359 patchrel: {
360 unsigned int e = 16 + ((ins >> 12) & 15);
362 CK((n & 3) == 0 && 361 CK((n & 3) == 0 &&
363 ((n + ((ins & 2048) ? 0x00020000 : 0)) >> 362 ((n + ((ins & 2048) ? 0 : (1<<(e+1)))) >> (e+2)) == 0, RANGE_REL);
364 ((ins & 2048) ? 18 : 28)) == 0, RANGE_REL); 363 cp[-1] |= ((n>>2) & ((1<<e)-1));
365 cp[-1] |= ((n>>2) & ((ins & 2048) ? 0x0000ffff: 0x03ffffff)); 364 }
366 break; 365 break;
367 case DASM_LABEL_LG: 366 case DASM_LABEL_LG:
368 ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); 367 ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n);
369 break; 368 break;
370 case DASM_LABEL_PC: break; 369 case DASM_LABEL_PC: break;
370 case DASM_IMMS:
371 cp[-1] |= ((n>>3) & 4); n &= 0x1f;
372 /* fallthrough */
371 case DASM_IMM: 373 case DASM_IMM:
372 cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31); 374 cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
373 break; 375 break;
@@ -410,7 +412,7 @@ int dasm_checkstep(Dst_DECL, int secmatch)
410 } 412 }
411 if (D->status == DASM_S_OK && secmatch >= 0 && 413 if (D->status == DASM_S_OK && secmatch >= 0 &&
412 D->section != &D->sections[secmatch]) 414 D->section != &D->sections[secmatch])
413 D->status = DASM_S_MATCH_SEC|(D->section-D->sections); 415 D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections);
414 return D->status; 416 return D->status;
415} 417}
416#endif 418#endif
diff --git a/dynasm/dasm_mips.lua b/dynasm/dasm_mips.lua
index e2ff17f0..1c605b68 100644
--- a/dynasm/dasm_mips.lua
+++ b/dynasm/dasm_mips.lua
@@ -1,17 +1,20 @@
1------------------------------------------------------------------------------ 1------------------------------------------------------------------------------
2-- DynASM MIPS module. 2-- DynASM MIPS32/MIPS64 module.
3-- 3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. 4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- See dynasm.lua for full copyright notice. 5-- See dynasm.lua for full copyright notice.
6------------------------------------------------------------------------------ 6------------------------------------------------------------------------------
7 7
8local mips64 = mips64
9local mipsr6 = _map_def.MIPSR6
10
8-- Module information: 11-- Module information:
9local _info = { 12local _info = {
10 arch = "mips", 13 arch = mips64 and "mips64" or "mips",
11 description = "DynASM MIPS module", 14 description = "DynASM MIPS32/MIPS64 module",
12 version = "1.3.0", 15 version = "1.5.0",
13 vernum = 10300, 16 vernum = 10500,
14 release = "2012-01-23", 17 release = "2021-05-02",
15 author = "Mike Pall", 18 author = "Mike Pall",
16 license = "MIT", 19 license = "MIT",
17} 20}
@@ -27,7 +30,8 @@ local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char
27local match, gmatch = _s.match, _s.gmatch 30local match, gmatch = _s.match, _s.gmatch
28local concat, sort = table.concat, table.sort 31local concat, sort = table.concat, table.sort
29local bit = bit or require("bit") 32local bit = bit or require("bit")
30local band, shl, sar, tohex = bit.band, bit.lshift, bit.arshift, bit.tohex 33local band, shl, shr, sar = bit.band, bit.lshift, bit.rshift, bit.arshift
34local tohex = bit.tohex
31 35
32-- Inherited tables and callbacks. 36-- Inherited tables and callbacks.
33local g_opt, g_arch 37local g_opt, g_arch
@@ -38,7 +42,7 @@ local wline, werror, wfatal, wwarn
38local action_names = { 42local action_names = {
39 "STOP", "SECTION", "ESC", "REL_EXT", 43 "STOP", "SECTION", "ESC", "REL_EXT",
40 "ALIGN", "REL_LG", "LABEL_LG", 44 "ALIGN", "REL_LG", "LABEL_LG",
41 "REL_PC", "LABEL_PC", "IMM", 45 "REL_PC", "LABEL_PC", "IMM", "IMMS",
42} 46}
43 47
44-- Maximum number of section buffer positions for dasm_put(). 48-- Maximum number of section buffer positions for dasm_put().
@@ -235,7 +239,6 @@ local map_op = {
235 bne_3 = "14000000STB", 239 bne_3 = "14000000STB",
236 blez_2 = "18000000SB", 240 blez_2 = "18000000SB",
237 bgtz_2 = "1c000000SB", 241 bgtz_2 = "1c000000SB",
238 addi_3 = "20000000TSI",
239 li_2 = "24000000TI", 242 li_2 = "24000000TI",
240 addiu_3 = "24000000TSI", 243 addiu_3 = "24000000TSI",
241 slti_3 = "28000000TSI", 244 slti_3 = "28000000TSI",
@@ -245,70 +248,52 @@ local map_op = {
245 ori_3 = "34000000TSU", 248 ori_3 = "34000000TSU",
246 xori_3 = "38000000TSU", 249 xori_3 = "38000000TSU",
247 lui_2 = "3c000000TU", 250 lui_2 = "3c000000TU",
248 beqzl_2 = "50000000SB", 251 daddiu_3 = mips64 and "64000000TSI",
249 beql_3 = "50000000STB", 252 ldl_2 = mips64 and "68000000TO",
250 bnezl_2 = "54000000SB", 253 ldr_2 = mips64 and "6c000000TO",
251 bnel_3 = "54000000STB",
252 blezl_2 = "58000000SB",
253 bgtzl_2 = "5c000000SB",
254 lb_2 = "80000000TO", 254 lb_2 = "80000000TO",
255 lh_2 = "84000000TO", 255 lh_2 = "84000000TO",
256 lwl_2 = "88000000TO",
257 lw_2 = "8c000000TO", 256 lw_2 = "8c000000TO",
258 lbu_2 = "90000000TO", 257 lbu_2 = "90000000TO",
259 lhu_2 = "94000000TO", 258 lhu_2 = "94000000TO",
260 lwr_2 = "98000000TO", 259 lwu_2 = mips64 and "9c000000TO",
261 sb_2 = "a0000000TO", 260 sb_2 = "a0000000TO",
262 sh_2 = "a4000000TO", 261 sh_2 = "a4000000TO",
263 swl_2 = "a8000000TO",
264 sw_2 = "ac000000TO", 262 sw_2 = "ac000000TO",
265 swr_2 = "b8000000TO",
266 cache_2 = "bc000000NO",
267 ll_2 = "c0000000TO",
268 lwc1_2 = "c4000000HO", 263 lwc1_2 = "c4000000HO",
269 pref_2 = "cc000000NO",
270 ldc1_2 = "d4000000HO", 264 ldc1_2 = "d4000000HO",
271 sc_2 = "e0000000TO", 265 ld_2 = mips64 and "dc000000TO",
272 swc1_2 = "e4000000HO", 266 swc1_2 = "e4000000HO",
273 sdc1_2 = "f4000000HO", 267 sdc1_2 = "f4000000HO",
268 sd_2 = mips64 and "fc000000TO",
274 269
275 -- Opcode SPECIAL. 270 -- Opcode SPECIAL.
276 nop_0 = "00000000", 271 nop_0 = "00000000",
277 sll_3 = "00000000DTA", 272 sll_3 = "00000000DTA",
278 movf_2 = "00000001DS", 273 sextw_2 = "00000000DT",
279 movf_3 = "00000001DSC",
280 movt_2 = "00010001DS",
281 movt_3 = "00010001DSC",
282 srl_3 = "00000002DTA", 274 srl_3 = "00000002DTA",
283 rotr_3 = "00200002DTA", 275 rotr_3 = "00200002DTA",
284 sra_3 = "00000003DTA", 276 sra_3 = "00000003DTA",
285 sllv_3 = "00000004DTS", 277 sllv_3 = "00000004DTS",
286 srlv_3 = "00000006DTS", 278 srlv_3 = "00000006DTS",
287 rotrv_3 = "00000046DTS", 279 rotrv_3 = "00000046DTS",
280 drotrv_3 = mips64 and "00000056DTS",
288 srav_3 = "00000007DTS", 281 srav_3 = "00000007DTS",
289 jr_1 = "00000008S",
290 jalr_1 = "0000f809S", 282 jalr_1 = "0000f809S",
291 jalr_2 = "00000009DS", 283 jalr_2 = "00000009DS",
292 movz_3 = "0000000aDST",
293 movn_3 = "0000000bDST",
294 syscall_0 = "0000000c", 284 syscall_0 = "0000000c",
295 syscall_1 = "0000000cY", 285 syscall_1 = "0000000cY",
296 break_0 = "0000000d", 286 break_0 = "0000000d",
297 break_1 = "0000000dY", 287 break_1 = "0000000dY",
298 sync_0 = "0000000f", 288 sync_0 = "0000000f",
299 mfhi_1 = "00000010D", 289 dsllv_3 = mips64 and "00000014DTS",
300 mthi_1 = "00000011S", 290 dsrlv_3 = mips64 and "00000016DTS",
301 mflo_1 = "00000012D", 291 dsrav_3 = mips64 and "00000017DTS",
302 mtlo_1 = "00000013S",
303 mult_2 = "00000018ST",
304 multu_2 = "00000019ST",
305 div_2 = "0000001aST",
306 divu_2 = "0000001bST",
307 add_3 = "00000020DST", 292 add_3 = "00000020DST",
308 move_2 = "00000021DS", 293 move_2 = mips64 and "00000025DS" or "00000021DS",
309 addu_3 = "00000021DST", 294 addu_3 = "00000021DST",
310 sub_3 = "00000022DST", 295 sub_3 = "00000022DST",
311 negu_2 = "00000023DT", 296 negu_2 = mips64 and "0000002fDT" or "00000023DT",
312 subu_3 = "00000023DST", 297 subu_3 = "00000023DST",
313 and_3 = "00000024DST", 298 and_3 = "00000024DST",
314 or_3 = "00000025DST", 299 or_3 = "00000025DST",
@@ -317,6 +302,10 @@ local map_op = {
317 nor_3 = "00000027DST", 302 nor_3 = "00000027DST",
318 slt_3 = "0000002aDST", 303 slt_3 = "0000002aDST",
319 sltu_3 = "0000002bDST", 304 sltu_3 = "0000002bDST",
305 dadd_3 = mips64 and "0000002cDST",
306 daddu_3 = mips64 and "0000002dDST",
307 dsub_3 = mips64 and "0000002eDST",
308 dsubu_3 = mips64 and "0000002fDST",
320 tge_2 = "00000030ST", 309 tge_2 = "00000030ST",
321 tge_3 = "00000030STZ", 310 tge_3 = "00000030STZ",
322 tgeu_2 = "00000031ST", 311 tgeu_2 = "00000031ST",
@@ -329,40 +318,36 @@ local map_op = {
329 teq_3 = "00000034STZ", 318 teq_3 = "00000034STZ",
330 tne_2 = "00000036ST", 319 tne_2 = "00000036ST",
331 tne_3 = "00000036STZ", 320 tne_3 = "00000036STZ",
321 dsll_3 = mips64 and "00000038DTa",
322 dsrl_3 = mips64 and "0000003aDTa",
323 drotr_3 = mips64 and "0020003aDTa",
324 dsra_3 = mips64 and "0000003bDTa",
325 dsll32_3 = mips64 and "0000003cDTA",
326 dsrl32_3 = mips64 and "0000003eDTA",
327 drotr32_3 = mips64 and "0020003eDTA",
328 dsra32_3 = mips64 and "0000003fDTA",
332 329
333 -- Opcode REGIMM. 330 -- Opcode REGIMM.
334 bltz_2 = "04000000SB", 331 bltz_2 = "04000000SB",
335 bgez_2 = "04010000SB", 332 bgez_2 = "04010000SB",
336 bltzl_2 = "04020000SB", 333 bltzl_2 = "04020000SB",
337 bgezl_2 = "04030000SB", 334 bgezl_2 = "04030000SB",
338 tgei_2 = "04080000SI",
339 tgeiu_2 = "04090000SI",
340 tlti_2 = "040a0000SI",
341 tltiu_2 = "040b0000SI",
342 teqi_2 = "040c0000SI",
343 tnei_2 = "040e0000SI",
344 bltzal_2 = "04100000SB",
345 bal_1 = "04110000B", 335 bal_1 = "04110000B",
346 bgezal_2 = "04110000SB",
347 bltzall_2 = "04120000SB",
348 bgezall_2 = "04130000SB",
349 synci_1 = "041f0000O", 336 synci_1 = "041f0000O",
350 337
351 -- Opcode SPECIAL2.
352 madd_2 = "70000000ST",
353 maddu_2 = "70000001ST",
354 mul_3 = "70000002DST",
355 msub_2 = "70000004ST",
356 msubu_2 = "70000005ST",
357 clz_2 = "70000020DS=",
358 clo_2 = "70000021DS=",
359 sdbbp_0 = "7000003f",
360 sdbbp_1 = "7000003fY",
361
362 -- Opcode SPECIAL3. 338 -- Opcode SPECIAL3.
363 ext_4 = "7c000000TSAM", -- Note: last arg is msbd = size-1 339 ext_4 = "7c000000TSAM", -- Note: last arg is msbd = size-1
340 dextm_4 = mips64 and "7c000001TSAM", -- Args: pos | size-1-32
341 dextu_4 = mips64 and "7c000002TSAM", -- Args: pos-32 | size-1
342 dext_4 = mips64 and "7c000003TSAM", -- Args: pos | size-1
343 zextw_2 = mips64 and "7c00f803TS",
364 ins_4 = "7c000004TSAM", -- Note: last arg is msb = pos+size-1 344 ins_4 = "7c000004TSAM", -- Note: last arg is msb = pos+size-1
345 dinsm_4 = mips64 and "7c000005TSAM", -- Args: pos | pos+size-33
346 dinsu_4 = mips64 and "7c000006TSAM", -- Args: pos-32 | pos+size-33
347 dins_4 = mips64 and "7c000007TSAM", -- Args: pos | pos+size-1
365 wsbh_2 = "7c0000a0DT", 348 wsbh_2 = "7c0000a0DT",
349 dsbh_2 = mips64 and "7c0000a4DT",
350 dshd_2 = mips64 and "7c000164DT",
366 seb_2 = "7c000420DT", 351 seb_2 = "7c000420DT",
367 seh_2 = "7c000620DT", 352 seh_2 = "7c000620DT",
368 rdhwr_2 = "7c00003bTD", 353 rdhwr_2 = "7c00003bTD",
@@ -370,8 +355,12 @@ local map_op = {
370 -- Opcode COP0. 355 -- Opcode COP0.
371 mfc0_2 = "40000000TD", 356 mfc0_2 = "40000000TD",
372 mfc0_3 = "40000000TDW", 357 mfc0_3 = "40000000TDW",
358 dmfc0_2 = mips64 and "40200000TD",
359 dmfc0_3 = mips64 and "40200000TDW",
373 mtc0_2 = "40800000TD", 360 mtc0_2 = "40800000TD",
374 mtc0_3 = "40800000TDW", 361 mtc0_3 = "40800000TDW",
362 dmtc0_2 = mips64 and "40a00000TD",
363 dmtc0_3 = mips64 and "40a00000TDW",
375 rdpgpr_2 = "41400000DT", 364 rdpgpr_2 = "41400000DT",
376 di_0 = "41606000", 365 di_0 = "41606000",
377 di_1 = "41606000T", 366 di_1 = "41606000T",
@@ -388,21 +377,14 @@ local map_op = {
388 377
389 -- Opcode COP1. 378 -- Opcode COP1.
390 mfc1_2 = "44000000TG", 379 mfc1_2 = "44000000TG",
380 dmfc1_2 = mips64 and "44200000TG",
391 cfc1_2 = "44400000TG", 381 cfc1_2 = "44400000TG",
392 mfhc1_2 = "44600000TG", 382 mfhc1_2 = "44600000TG",
393 mtc1_2 = "44800000TG", 383 mtc1_2 = "44800000TG",
384 dmtc1_2 = mips64 and "44a00000TG",
394 ctc1_2 = "44c00000TG", 385 ctc1_2 = "44c00000TG",
395 mthc1_2 = "44e00000TG", 386 mthc1_2 = "44e00000TG",
396 387
397 bc1f_1 = "45000000B",
398 bc1f_2 = "45000000CB",
399 bc1t_1 = "45010000B",
400 bc1t_2 = "45010000CB",
401 bc1fl_1 = "45020000B",
402 bc1fl_2 = "45020000CB",
403 bc1tl_1 = "45030000B",
404 bc1tl_2 = "45030000CB",
405
406 ["add.s_3"] = "46000000FGH", 388 ["add.s_3"] = "46000000FGH",
407 ["sub.s_3"] = "46000001FGH", 389 ["sub.s_3"] = "46000001FGH",
408 ["mul.s_3"] = "46000002FGH", 390 ["mul.s_3"] = "46000002FGH",
@@ -419,51 +401,11 @@ local map_op = {
419 ["trunc.w.s_2"] = "4600000dFG", 401 ["trunc.w.s_2"] = "4600000dFG",
420 ["ceil.w.s_2"] = "4600000eFG", 402 ["ceil.w.s_2"] = "4600000eFG",
421 ["floor.w.s_2"] = "4600000fFG", 403 ["floor.w.s_2"] = "4600000fFG",
422 ["movf.s_2"] = "46000011FG",
423 ["movf.s_3"] = "46000011FGC",
424 ["movt.s_2"] = "46010011FG",
425 ["movt.s_3"] = "46010011FGC",
426 ["movz.s_3"] = "46000012FGT",
427 ["movn.s_3"] = "46000013FGT",
428 ["recip.s_2"] = "46000015FG", 404 ["recip.s_2"] = "46000015FG",
429 ["rsqrt.s_2"] = "46000016FG", 405 ["rsqrt.s_2"] = "46000016FG",
430 ["cvt.d.s_2"] = "46000021FG", 406 ["cvt.d.s_2"] = "46000021FG",
431 ["cvt.w.s_2"] = "46000024FG", 407 ["cvt.w.s_2"] = "46000024FG",
432 ["cvt.l.s_2"] = "46000025FG", 408 ["cvt.l.s_2"] = "46000025FG",
433 ["cvt.ps.s_3"] = "46000026FGH",
434 ["c.f.s_2"] = "46000030GH",
435 ["c.f.s_3"] = "46000030VGH",
436 ["c.un.s_2"] = "46000031GH",
437 ["c.un.s_3"] = "46000031VGH",
438 ["c.eq.s_2"] = "46000032GH",
439 ["c.eq.s_3"] = "46000032VGH",
440 ["c.ueq.s_2"] = "46000033GH",
441 ["c.ueq.s_3"] = "46000033VGH",
442 ["c.olt.s_2"] = "46000034GH",
443 ["c.olt.s_3"] = "46000034VGH",
444 ["c.ult.s_2"] = "46000035GH",
445 ["c.ult.s_3"] = "46000035VGH",
446 ["c.ole.s_2"] = "46000036GH",
447 ["c.ole.s_3"] = "46000036VGH",
448 ["c.ule.s_2"] = "46000037GH",
449 ["c.ule.s_3"] = "46000037VGH",
450 ["c.sf.s_2"] = "46000038GH",
451 ["c.sf.s_3"] = "46000038VGH",
452 ["c.ngle.s_2"] = "46000039GH",
453 ["c.ngle.s_3"] = "46000039VGH",
454 ["c.seq.s_2"] = "4600003aGH",
455 ["c.seq.s_3"] = "4600003aVGH",
456 ["c.ngl.s_2"] = "4600003bGH",
457 ["c.ngl.s_3"] = "4600003bVGH",
458 ["c.lt.s_2"] = "4600003cGH",
459 ["c.lt.s_3"] = "4600003cVGH",
460 ["c.nge.s_2"] = "4600003dGH",
461 ["c.nge.s_3"] = "4600003dVGH",
462 ["c.le.s_2"] = "4600003eGH",
463 ["c.le.s_3"] = "4600003eVGH",
464 ["c.ngt.s_2"] = "4600003fGH",
465 ["c.ngt.s_3"] = "4600003fVGH",
466
467 ["add.d_3"] = "46200000FGH", 409 ["add.d_3"] = "46200000FGH",
468 ["sub.d_3"] = "46200001FGH", 410 ["sub.d_3"] = "46200001FGH",
469 ["mul.d_3"] = "46200002FGH", 411 ["mul.d_3"] = "46200002FGH",
@@ -480,130 +422,410 @@ local map_op = {
480 ["trunc.w.d_2"] = "4620000dFG", 422 ["trunc.w.d_2"] = "4620000dFG",
481 ["ceil.w.d_2"] = "4620000eFG", 423 ["ceil.w.d_2"] = "4620000eFG",
482 ["floor.w.d_2"] = "4620000fFG", 424 ["floor.w.d_2"] = "4620000fFG",
483 ["movf.d_2"] = "46200011FG",
484 ["movf.d_3"] = "46200011FGC",
485 ["movt.d_2"] = "46210011FG",
486 ["movt.d_3"] = "46210011FGC",
487 ["movz.d_3"] = "46200012FGT",
488 ["movn.d_3"] = "46200013FGT",
489 ["recip.d_2"] = "46200015FG", 425 ["recip.d_2"] = "46200015FG",
490 ["rsqrt.d_2"] = "46200016FG", 426 ["rsqrt.d_2"] = "46200016FG",
491 ["cvt.s.d_2"] = "46200020FG", 427 ["cvt.s.d_2"] = "46200020FG",
492 ["cvt.w.d_2"] = "46200024FG", 428 ["cvt.w.d_2"] = "46200024FG",
493 ["cvt.l.d_2"] = "46200025FG", 429 ["cvt.l.d_2"] = "46200025FG",
494 ["c.f.d_2"] = "46200030GH",
495 ["c.f.d_3"] = "46200030VGH",
496 ["c.un.d_2"] = "46200031GH",
497 ["c.un.d_3"] = "46200031VGH",
498 ["c.eq.d_2"] = "46200032GH",
499 ["c.eq.d_3"] = "46200032VGH",
500 ["c.ueq.d_2"] = "46200033GH",
501 ["c.ueq.d_3"] = "46200033VGH",
502 ["c.olt.d_2"] = "46200034GH",
503 ["c.olt.d_3"] = "46200034VGH",
504 ["c.ult.d_2"] = "46200035GH",
505 ["c.ult.d_3"] = "46200035VGH",
506 ["c.ole.d_2"] = "46200036GH",
507 ["c.ole.d_3"] = "46200036VGH",
508 ["c.ule.d_2"] = "46200037GH",
509 ["c.ule.d_3"] = "46200037VGH",
510 ["c.sf.d_2"] = "46200038GH",
511 ["c.sf.d_3"] = "46200038VGH",
512 ["c.ngle.d_2"] = "46200039GH",
513 ["c.ngle.d_3"] = "46200039VGH",
514 ["c.seq.d_2"] = "4620003aGH",
515 ["c.seq.d_3"] = "4620003aVGH",
516 ["c.ngl.d_2"] = "4620003bGH",
517 ["c.ngl.d_3"] = "4620003bVGH",
518 ["c.lt.d_2"] = "4620003cGH",
519 ["c.lt.d_3"] = "4620003cVGH",
520 ["c.nge.d_2"] = "4620003dGH",
521 ["c.nge.d_3"] = "4620003dVGH",
522 ["c.le.d_2"] = "4620003eGH",
523 ["c.le.d_3"] = "4620003eVGH",
524 ["c.ngt.d_2"] = "4620003fGH",
525 ["c.ngt.d_3"] = "4620003fVGH",
526
527 ["add.ps_3"] = "46c00000FGH",
528 ["sub.ps_3"] = "46c00001FGH",
529 ["mul.ps_3"] = "46c00002FGH",
530 ["abs.ps_2"] = "46c00005FG",
531 ["mov.ps_2"] = "46c00006FG",
532 ["neg.ps_2"] = "46c00007FG",
533 ["movf.ps_2"] = "46c00011FG",
534 ["movf.ps_3"] = "46c00011FGC",
535 ["movt.ps_2"] = "46c10011FG",
536 ["movt.ps_3"] = "46c10011FGC",
537 ["movz.ps_3"] = "46c00012FGT",
538 ["movn.ps_3"] = "46c00013FGT",
539 ["cvt.s.pu_2"] = "46c00020FG",
540 ["cvt.s.pl_2"] = "46c00028FG",
541 ["pll.ps_3"] = "46c0002cFGH",
542 ["plu.ps_3"] = "46c0002dFGH",
543 ["pul.ps_3"] = "46c0002eFGH",
544 ["puu.ps_3"] = "46c0002fFGH",
545 ["c.f.ps_2"] = "46c00030GH",
546 ["c.f.ps_3"] = "46c00030VGH",
547 ["c.un.ps_2"] = "46c00031GH",
548 ["c.un.ps_3"] = "46c00031VGH",
549 ["c.eq.ps_2"] = "46c00032GH",
550 ["c.eq.ps_3"] = "46c00032VGH",
551 ["c.ueq.ps_2"] = "46c00033GH",
552 ["c.ueq.ps_3"] = "46c00033VGH",
553 ["c.olt.ps_2"] = "46c00034GH",
554 ["c.olt.ps_3"] = "46c00034VGH",
555 ["c.ult.ps_2"] = "46c00035GH",
556 ["c.ult.ps_3"] = "46c00035VGH",
557 ["c.ole.ps_2"] = "46c00036GH",
558 ["c.ole.ps_3"] = "46c00036VGH",
559 ["c.ule.ps_2"] = "46c00037GH",
560 ["c.ule.ps_3"] = "46c00037VGH",
561 ["c.sf.ps_2"] = "46c00038GH",
562 ["c.sf.ps_3"] = "46c00038VGH",
563 ["c.ngle.ps_2"] = "46c00039GH",
564 ["c.ngle.ps_3"] = "46c00039VGH",
565 ["c.seq.ps_2"] = "46c0003aGH",
566 ["c.seq.ps_3"] = "46c0003aVGH",
567 ["c.ngl.ps_2"] = "46c0003bGH",
568 ["c.ngl.ps_3"] = "46c0003bVGH",
569 ["c.lt.ps_2"] = "46c0003cGH",
570 ["c.lt.ps_3"] = "46c0003cVGH",
571 ["c.nge.ps_2"] = "46c0003dGH",
572 ["c.nge.ps_3"] = "46c0003dVGH",
573 ["c.le.ps_2"] = "46c0003eGH",
574 ["c.le.ps_3"] = "46c0003eVGH",
575 ["c.ngt.ps_2"] = "46c0003fGH",
576 ["c.ngt.ps_3"] = "46c0003fVGH",
577
578 ["cvt.s.w_2"] = "46800020FG", 430 ["cvt.s.w_2"] = "46800020FG",
579 ["cvt.d.w_2"] = "46800021FG", 431 ["cvt.d.w_2"] = "46800021FG",
580
581 ["cvt.s.l_2"] = "46a00020FG", 432 ["cvt.s.l_2"] = "46a00020FG",
582 ["cvt.d.l_2"] = "46a00021FG", 433 ["cvt.d.l_2"] = "46a00021FG",
583
584 -- Opcode COP1X.
585 lwxc1_2 = "4c000000FX",
586 ldxc1_2 = "4c000001FX",
587 luxc1_2 = "4c000005FX",
588 swxc1_2 = "4c000008FX",
589 sdxc1_2 = "4c000009FX",
590 suxc1_2 = "4c00000dFX",
591 prefx_2 = "4c00000fMX",
592 ["alnv.ps_4"] = "4c00001eFGHS",
593 ["madd.s_4"] = "4c000020FRGH",
594 ["madd.d_4"] = "4c000021FRGH",
595 ["madd.ps_4"] = "4c000026FRGH",
596 ["msub.s_4"] = "4c000028FRGH",
597 ["msub.d_4"] = "4c000029FRGH",
598 ["msub.ps_4"] = "4c00002eFRGH",
599 ["nmadd.s_4"] = "4c000030FRGH",
600 ["nmadd.d_4"] = "4c000031FRGH",
601 ["nmadd.ps_4"] = "4c000036FRGH",
602 ["nmsub.s_4"] = "4c000038FRGH",
603 ["nmsub.d_4"] = "4c000039FRGH",
604 ["nmsub.ps_4"] = "4c00003eFRGH",
605} 434}
606 435
436if mipsr6 then -- Instructions added with MIPSR6.
437
438 for k,v in pairs({
439
440 -- Add immediate to upper bits.
441 aui_3 = "3c000000TSI",
442 daui_3 = mips64 and "74000000TSI",
443 dahi_2 = mips64 and "04060000SI",
444 dati_2 = mips64 and "041e0000SI",
445
446 -- TODO: addiupc, auipc, aluipc, lwpc, lwupc, ldpc.
447
448 -- Compact branches.
449 blezalc_2 = "18000000TB", -- rt != 0.
450 bgezalc_2 = "18000000T=SB", -- rt != 0.
451 bgtzalc_2 = "1c000000TB", -- rt != 0.
452 bltzalc_2 = "1c000000T=SB", -- rt != 0.
453
454 blezc_2 = "58000000TB", -- rt != 0.
455 bgezc_2 = "58000000T=SB", -- rt != 0.
456 bgec_3 = "58000000STB", -- rs != rt.
457 blec_3 = "58000000TSB", -- rt != rs.
458
459 bgtzc_2 = "5c000000TB", -- rt != 0.
460 bltzc_2 = "5c000000T=SB", -- rt != 0.
461 bltc_3 = "5c000000STB", -- rs != rt.
462 bgtc_3 = "5c000000TSB", -- rt != rs.
463
464 bgeuc_3 = "18000000STB", -- rs != rt.
465 bleuc_3 = "18000000TSB", -- rt != rs.
466 bltuc_3 = "1c000000STB", -- rs != rt.
467 bgtuc_3 = "1c000000TSB", -- rt != rs.
468
469 beqzalc_2 = "20000000TB", -- rt != 0.
470 bnezalc_2 = "60000000TB", -- rt != 0.
471 beqc_3 = "20000000STB", -- rs < rt.
472 bnec_3 = "60000000STB", -- rs < rt.
473 bovc_3 = "20000000STB", -- rs >= rt.
474 bnvc_3 = "60000000STB", -- rs >= rt.
475
476 beqzc_2 = "d8000000SK", -- rs != 0.
477 bnezc_2 = "f8000000SK", -- rs != 0.
478 jic_2 = "d8000000TI",
479 jialc_2 = "f8000000TI",
480 bc_1 = "c8000000L",
481 balc_1 = "e8000000L",
482
483 -- Opcode SPECIAL.
484 jr_1 = "00000009S",
485 sdbbp_0 = "0000000e",
486 sdbbp_1 = "0000000eY",
487 lsa_4 = "00000005DSTA",
488 dlsa_4 = mips64 and "00000015DSTA",
489 seleqz_3 = "00000035DST",
490 selnez_3 = "00000037DST",
491 clz_2 = "00000050DS",
492 clo_2 = "00000051DS",
493 dclz_2 = mips64 and "00000052DS",
494 dclo_2 = mips64 and "00000053DS",
495 mul_3 = "00000098DST",
496 muh_3 = "000000d8DST",
497 mulu_3 = "00000099DST",
498 muhu_3 = "000000d9DST",
499 div_3 = "0000009aDST",
500 mod_3 = "000000daDST",
501 divu_3 = "0000009bDST",
502 modu_3 = "000000dbDST",
503 dmul_3 = mips64 and "0000009cDST",
504 dmuh_3 = mips64 and "000000dcDST",
505 dmulu_3 = mips64 and "0000009dDST",
506 dmuhu_3 = mips64 and "000000ddDST",
507 ddiv_3 = mips64 and "0000009eDST",
508 dmod_3 = mips64 and "000000deDST",
509 ddivu_3 = mips64 and "0000009fDST",
510 dmodu_3 = mips64 and "000000dfDST",
511
512 -- Opcode SPECIAL3.
513 align_4 = "7c000220DSTA",
514 dalign_4 = mips64 and "7c000224DSTA",
515 bitswap_2 = "7c000020DT",
516 dbitswap_2 = mips64 and "7c000024DT",
517
518 -- Opcode COP1.
519 bc1eqz_2 = "45200000HB",
520 bc1nez_2 = "45a00000HB",
521
522 ["sel.s_3"] = "46000010FGH",
523 ["seleqz.s_3"] = "46000014FGH",
524 ["selnez.s_3"] = "46000017FGH",
525 ["maddf.s_3"] = "46000018FGH",
526 ["msubf.s_3"] = "46000019FGH",
527 ["rint.s_2"] = "4600001aFG",
528 ["class.s_2"] = "4600001bFG",
529 ["min.s_3"] = "4600001cFGH",
530 ["mina.s_3"] = "4600001dFGH",
531 ["max.s_3"] = "4600001eFGH",
532 ["maxa.s_3"] = "4600001fFGH",
533 ["cmp.af.s_3"] = "46800000FGH",
534 ["cmp.un.s_3"] = "46800001FGH",
535 ["cmp.or.s_3"] = "46800011FGH",
536 ["cmp.eq.s_3"] = "46800002FGH",
537 ["cmp.une.s_3"] = "46800012FGH",
538 ["cmp.ueq.s_3"] = "46800003FGH",
539 ["cmp.ne.s_3"] = "46800013FGH",
540 ["cmp.lt.s_3"] = "46800004FGH",
541 ["cmp.ult.s_3"] = "46800005FGH",
542 ["cmp.le.s_3"] = "46800006FGH",
543 ["cmp.ule.s_3"] = "46800007FGH",
544 ["cmp.saf.s_3"] = "46800008FGH",
545 ["cmp.sun.s_3"] = "46800009FGH",
546 ["cmp.sor.s_3"] = "46800019FGH",
547 ["cmp.seq.s_3"] = "4680000aFGH",
548 ["cmp.sune.s_3"] = "4680001aFGH",
549 ["cmp.sueq.s_3"] = "4680000bFGH",
550 ["cmp.sne.s_3"] = "4680001bFGH",
551 ["cmp.slt.s_3"] = "4680000cFGH",
552 ["cmp.sult.s_3"] = "4680000dFGH",
553 ["cmp.sle.s_3"] = "4680000eFGH",
554 ["cmp.sule.s_3"] = "4680000fFGH",
555
556 ["sel.d_3"] = "46200010FGH",
557 ["seleqz.d_3"] = "46200014FGH",
558 ["selnez.d_3"] = "46200017FGH",
559 ["maddf.d_3"] = "46200018FGH",
560 ["msubf.d_3"] = "46200019FGH",
561 ["rint.d_2"] = "4620001aFG",
562 ["class.d_2"] = "4620001bFG",
563 ["min.d_3"] = "4620001cFGH",
564 ["mina.d_3"] = "4620001dFGH",
565 ["max.d_3"] = "4620001eFGH",
566 ["maxa.d_3"] = "4620001fFGH",
567 ["cmp.af.d_3"] = "46a00000FGH",
568 ["cmp.un.d_3"] = "46a00001FGH",
569 ["cmp.or.d_3"] = "46a00011FGH",
570 ["cmp.eq.d_3"] = "46a00002FGH",
571 ["cmp.une.d_3"] = "46a00012FGH",
572 ["cmp.ueq.d_3"] = "46a00003FGH",
573 ["cmp.ne.d_3"] = "46a00013FGH",
574 ["cmp.lt.d_3"] = "46a00004FGH",
575 ["cmp.ult.d_3"] = "46a00005FGH",
576 ["cmp.le.d_3"] = "46a00006FGH",
577 ["cmp.ule.d_3"] = "46a00007FGH",
578 ["cmp.saf.d_3"] = "46a00008FGH",
579 ["cmp.sun.d_3"] = "46a00009FGH",
580 ["cmp.sor.d_3"] = "46a00019FGH",
581 ["cmp.seq.d_3"] = "46a0000aFGH",
582 ["cmp.sune.d_3"] = "46a0001aFGH",
583 ["cmp.sueq.d_3"] = "46a0000bFGH",
584 ["cmp.sne.d_3"] = "46a0001bFGH",
585 ["cmp.slt.d_3"] = "46a0000cFGH",
586 ["cmp.sult.d_3"] = "46a0000dFGH",
587 ["cmp.sle.d_3"] = "46a0000eFGH",
588 ["cmp.sule.d_3"] = "46a0000fFGH",
589
590 }) do map_op[k] = v end
591
592else -- Instructions removed by MIPSR6.
593
594 for k,v in pairs({
595 -- Traps, don't use.
596 addi_3 = "20000000TSI",
597 daddi_3 = mips64 and "60000000TSI",
598
599 -- Branch on likely, don't use.
600 beqzl_2 = "50000000SB",
601 beql_3 = "50000000STB",
602 bnezl_2 = "54000000SB",
603 bnel_3 = "54000000STB",
604 blezl_2 = "58000000SB",
605 bgtzl_2 = "5c000000SB",
606
607 lwl_2 = "88000000TO",
608 lwr_2 = "98000000TO",
609 swl_2 = "a8000000TO",
610 sdl_2 = mips64 and "b0000000TO",
611 sdr_2 = mips64 and "b1000000TO",
612 swr_2 = "b8000000TO",
613 cache_2 = "bc000000NO",
614 ll_2 = "c0000000TO",
615 pref_2 = "cc000000NO",
616 sc_2 = "e0000000TO",
617 scd_2 = mips64 and "f0000000TO",
618
619 -- Opcode SPECIAL.
620 movf_2 = "00000001DS",
621 movf_3 = "00000001DSC",
622 movt_2 = "00010001DS",
623 movt_3 = "00010001DSC",
624 jr_1 = "00000008S",
625 movz_3 = "0000000aDST",
626 movn_3 = "0000000bDST",
627 mfhi_1 = "00000010D",
628 mthi_1 = "00000011S",
629 mflo_1 = "00000012D",
630 mtlo_1 = "00000013S",
631 mult_2 = "00000018ST",
632 multu_2 = "00000019ST",
633 div_3 = "0000001aST",
634 divu_3 = "0000001bST",
635 ddiv_3 = mips64 and "0000001eST",
636 ddivu_3 = mips64 and "0000001fST",
637 dmult_2 = mips64 and "0000001cST",
638 dmultu_2 = mips64 and "0000001dST",
639
640 -- Opcode REGIMM.
641 tgei_2 = "04080000SI",
642 tgeiu_2 = "04090000SI",
643 tlti_2 = "040a0000SI",
644 tltiu_2 = "040b0000SI",
645 teqi_2 = "040c0000SI",
646 tnei_2 = "040e0000SI",
647 bltzal_2 = "04100000SB",
648 bgezal_2 = "04110000SB",
649 bltzall_2 = "04120000SB",
650 bgezall_2 = "04130000SB",
651
652 -- Opcode SPECIAL2.
653 madd_2 = "70000000ST",
654 maddu_2 = "70000001ST",
655 mul_3 = "70000002DST",
656 msub_2 = "70000004ST",
657 msubu_2 = "70000005ST",
658 clz_2 = "70000020D=TS",
659 clo_2 = "70000021D=TS",
660 dclz_2 = mips64 and "70000024D=TS",
661 dclo_2 = mips64 and "70000025D=TS",
662 sdbbp_0 = "7000003f",
663 sdbbp_1 = "7000003fY",
664
665 -- Opcode COP1.
666 bc1f_1 = "45000000B",
667 bc1f_2 = "45000000CB",
668 bc1t_1 = "45010000B",
669 bc1t_2 = "45010000CB",
670 bc1fl_1 = "45020000B",
671 bc1fl_2 = "45020000CB",
672 bc1tl_1 = "45030000B",
673 bc1tl_2 = "45030000CB",
674
675 ["movf.s_2"] = "46000011FG",
676 ["movf.s_3"] = "46000011FGC",
677 ["movt.s_2"] = "46010011FG",
678 ["movt.s_3"] = "46010011FGC",
679 ["movz.s_3"] = "46000012FGT",
680 ["movn.s_3"] = "46000013FGT",
681 ["cvt.ps.s_3"] = "46000026FGH",
682 ["c.f.s_2"] = "46000030GH",
683 ["c.f.s_3"] = "46000030VGH",
684 ["c.un.s_2"] = "46000031GH",
685 ["c.un.s_3"] = "46000031VGH",
686 ["c.eq.s_2"] = "46000032GH",
687 ["c.eq.s_3"] = "46000032VGH",
688 ["c.ueq.s_2"] = "46000033GH",
689 ["c.ueq.s_3"] = "46000033VGH",
690 ["c.olt.s_2"] = "46000034GH",
691 ["c.olt.s_3"] = "46000034VGH",
692 ["c.ult.s_2"] = "46000035GH",
693 ["c.ult.s_3"] = "46000035VGH",
694 ["c.ole.s_2"] = "46000036GH",
695 ["c.ole.s_3"] = "46000036VGH",
696 ["c.ule.s_2"] = "46000037GH",
697 ["c.ule.s_3"] = "46000037VGH",
698 ["c.sf.s_2"] = "46000038GH",
699 ["c.sf.s_3"] = "46000038VGH",
700 ["c.ngle.s_2"] = "46000039GH",
701 ["c.ngle.s_3"] = "46000039VGH",
702 ["c.seq.s_2"] = "4600003aGH",
703 ["c.seq.s_3"] = "4600003aVGH",
704 ["c.ngl.s_2"] = "4600003bGH",
705 ["c.ngl.s_3"] = "4600003bVGH",
706 ["c.lt.s_2"] = "4600003cGH",
707 ["c.lt.s_3"] = "4600003cVGH",
708 ["c.nge.s_2"] = "4600003dGH",
709 ["c.nge.s_3"] = "4600003dVGH",
710 ["c.le.s_2"] = "4600003eGH",
711 ["c.le.s_3"] = "4600003eVGH",
712 ["c.ngt.s_2"] = "4600003fGH",
713 ["c.ngt.s_3"] = "4600003fVGH",
714 ["movf.d_2"] = "46200011FG",
715 ["movf.d_3"] = "46200011FGC",
716 ["movt.d_2"] = "46210011FG",
717 ["movt.d_3"] = "46210011FGC",
718 ["movz.d_3"] = "46200012FGT",
719 ["movn.d_3"] = "46200013FGT",
720 ["c.f.d_2"] = "46200030GH",
721 ["c.f.d_3"] = "46200030VGH",
722 ["c.un.d_2"] = "46200031GH",
723 ["c.un.d_3"] = "46200031VGH",
724 ["c.eq.d_2"] = "46200032GH",
725 ["c.eq.d_3"] = "46200032VGH",
726 ["c.ueq.d_2"] = "46200033GH",
727 ["c.ueq.d_3"] = "46200033VGH",
728 ["c.olt.d_2"] = "46200034GH",
729 ["c.olt.d_3"] = "46200034VGH",
730 ["c.ult.d_2"] = "46200035GH",
731 ["c.ult.d_3"] = "46200035VGH",
732 ["c.ole.d_2"] = "46200036GH",
733 ["c.ole.d_3"] = "46200036VGH",
734 ["c.ule.d_2"] = "46200037GH",
735 ["c.ule.d_3"] = "46200037VGH",
736 ["c.sf.d_2"] = "46200038GH",
737 ["c.sf.d_3"] = "46200038VGH",
738 ["c.ngle.d_2"] = "46200039GH",
739 ["c.ngle.d_3"] = "46200039VGH",
740 ["c.seq.d_2"] = "4620003aGH",
741 ["c.seq.d_3"] = "4620003aVGH",
742 ["c.ngl.d_2"] = "4620003bGH",
743 ["c.ngl.d_3"] = "4620003bVGH",
744 ["c.lt.d_2"] = "4620003cGH",
745 ["c.lt.d_3"] = "4620003cVGH",
746 ["c.nge.d_2"] = "4620003dGH",
747 ["c.nge.d_3"] = "4620003dVGH",
748 ["c.le.d_2"] = "4620003eGH",
749 ["c.le.d_3"] = "4620003eVGH",
750 ["c.ngt.d_2"] = "4620003fGH",
751 ["c.ngt.d_3"] = "4620003fVGH",
752 ["add.ps_3"] = "46c00000FGH",
753 ["sub.ps_3"] = "46c00001FGH",
754 ["mul.ps_3"] = "46c00002FGH",
755 ["abs.ps_2"] = "46c00005FG",
756 ["mov.ps_2"] = "46c00006FG",
757 ["neg.ps_2"] = "46c00007FG",
758 ["movf.ps_2"] = "46c00011FG",
759 ["movf.ps_3"] = "46c00011FGC",
760 ["movt.ps_2"] = "46c10011FG",
761 ["movt.ps_3"] = "46c10011FGC",
762 ["movz.ps_3"] = "46c00012FGT",
763 ["movn.ps_3"] = "46c00013FGT",
764 ["cvt.s.pu_2"] = "46c00020FG",
765 ["cvt.s.pl_2"] = "46c00028FG",
766 ["pll.ps_3"] = "46c0002cFGH",
767 ["plu.ps_3"] = "46c0002dFGH",
768 ["pul.ps_3"] = "46c0002eFGH",
769 ["puu.ps_3"] = "46c0002fFGH",
770 ["c.f.ps_2"] = "46c00030GH",
771 ["c.f.ps_3"] = "46c00030VGH",
772 ["c.un.ps_2"] = "46c00031GH",
773 ["c.un.ps_3"] = "46c00031VGH",
774 ["c.eq.ps_2"] = "46c00032GH",
775 ["c.eq.ps_3"] = "46c00032VGH",
776 ["c.ueq.ps_2"] = "46c00033GH",
777 ["c.ueq.ps_3"] = "46c00033VGH",
778 ["c.olt.ps_2"] = "46c00034GH",
779 ["c.olt.ps_3"] = "46c00034VGH",
780 ["c.ult.ps_2"] = "46c00035GH",
781 ["c.ult.ps_3"] = "46c00035VGH",
782 ["c.ole.ps_2"] = "46c00036GH",
783 ["c.ole.ps_3"] = "46c00036VGH",
784 ["c.ule.ps_2"] = "46c00037GH",
785 ["c.ule.ps_3"] = "46c00037VGH",
786 ["c.sf.ps_2"] = "46c00038GH",
787 ["c.sf.ps_3"] = "46c00038VGH",
788 ["c.ngle.ps_2"] = "46c00039GH",
789 ["c.ngle.ps_3"] = "46c00039VGH",
790 ["c.seq.ps_2"] = "46c0003aGH",
791 ["c.seq.ps_3"] = "46c0003aVGH",
792 ["c.ngl.ps_2"] = "46c0003bGH",
793 ["c.ngl.ps_3"] = "46c0003bVGH",
794 ["c.lt.ps_2"] = "46c0003cGH",
795 ["c.lt.ps_3"] = "46c0003cVGH",
796 ["c.nge.ps_2"] = "46c0003dGH",
797 ["c.nge.ps_3"] = "46c0003dVGH",
798 ["c.le.ps_2"] = "46c0003eGH",
799 ["c.le.ps_3"] = "46c0003eVGH",
800 ["c.ngt.ps_2"] = "46c0003fGH",
801 ["c.ngt.ps_3"] = "46c0003fVGH",
802
803 -- Opcode COP1X.
804 lwxc1_2 = "4c000000FX",
805 ldxc1_2 = "4c000001FX",
806 luxc1_2 = "4c000005FX",
807 swxc1_2 = "4c000008FX",
808 sdxc1_2 = "4c000009FX",
809 suxc1_2 = "4c00000dFX",
810 prefx_2 = "4c00000fMX",
811 ["alnv.ps_4"] = "4c00001eFGHS",
812 ["madd.s_4"] = "4c000020FRGH",
813 ["madd.d_4"] = "4c000021FRGH",
814 ["madd.ps_4"] = "4c000026FRGH",
815 ["msub.s_4"] = "4c000028FRGH",
816 ["msub.d_4"] = "4c000029FRGH",
817 ["msub.ps_4"] = "4c00002eFRGH",
818 ["nmadd.s_4"] = "4c000030FRGH",
819 ["nmadd.d_4"] = "4c000031FRGH",
820 ["nmadd.ps_4"] = "4c000036FRGH",
821 ["nmsub.s_4"] = "4c000038FRGH",
822 ["nmsub.d_4"] = "4c000039FRGH",
823 ["nmsub.ps_4"] = "4c00003eFRGH",
824
825 }) do map_op[k] = v end
826
827end
828
607------------------------------------------------------------------------------ 829------------------------------------------------------------------------------
608 830
609local function parse_gpr(expr) 831local function parse_gpr(expr)
@@ -633,7 +855,7 @@ local function parse_fpr(expr)
633 werror("bad register name `"..expr.."'") 855 werror("bad register name `"..expr.."'")
634end 856end
635 857
636local function parse_imm(imm, bits, shift, scale, signed) 858local function parse_imm(imm, bits, shift, scale, signed, action)
637 local n = tonumber(imm) 859 local n = tonumber(imm)
638 if n then 860 if n then
639 local m = sar(n, scale) 861 local m = sar(n, scale)
@@ -651,7 +873,8 @@ local function parse_imm(imm, bits, shift, scale, signed)
651 match(imm, "^([%w_]+):([rf][1-3]?[0-9])$") then 873 match(imm, "^([%w_]+):([rf][1-3]?[0-9])$") then
652 werror("expected immediate operand, got register") 874 werror("expected immediate operand, got register")
653 else 875 else
654 waction("IMM", (signed and 32768 or 0)+scale*1024+bits*32+shift, imm) 876 waction(action or "IMM",
877 (signed and 32768 or 0)+shl(scale, 10)+shl(bits, 5)+shift, imm)
655 return 0 878 return 0
656 end 879 end
657end 880end
@@ -756,13 +979,18 @@ map_op[".template__"] = function(params, template, nparams)
756 op = op + parse_disp(params[n]); n = n + 1 979 op = op + parse_disp(params[n]); n = n + 1
757 elseif p == "X" then 980 elseif p == "X" then
758 op = op + parse_index(params[n]); n = n + 1 981 op = op + parse_index(params[n]); n = n + 1
759 elseif p == "B" or p == "J" then 982 elseif p == "B" or p == "J" or p == "K" or p == "L" then
760 local mode, m, s = parse_label(params[n], false) 983 local mode, m, s = parse_label(params[n], false)
761 if p == "B" then m = m + 2048 end 984 if p == "J" then m = m + 0xa800
985 elseif p == "K" then m = m + 0x5000
986 elseif p == "L" then m = m + 0xa000 end
762 waction("REL_"..mode, m, s, 1) 987 waction("REL_"..mode, m, s, 1)
763 n = n + 1 988 n = n + 1
764 elseif p == "A" then 989 elseif p == "A" then
765 op = op + parse_imm(params[n], 5, 6, 0, false); n = n + 1 990 op = op + parse_imm(params[n], 5, 6, 0, false); n = n + 1
991 elseif p == "a" then
992 local m = parse_imm(params[n], 6, 6, 0, false, "IMMS"); n = n + 1
993 op = op + band(m, 0x7c0) + band(shr(m, 9), 4)
766 elseif p == "M" then 994 elseif p == "M" then
767 op = op + parse_imm(params[n], 5, 11, 0, false); n = n + 1 995 op = op + parse_imm(params[n], 5, 11, 0, false); n = n + 1
768 elseif p == "N" then 996 elseif p == "N" then
@@ -778,7 +1006,7 @@ map_op[".template__"] = function(params, template, nparams)
778 elseif p == "Z" then 1006 elseif p == "Z" then
779 op = op + parse_imm(params[n], 10, 6, 0, false); n = n + 1 1007 op = op + parse_imm(params[n], 10, 6, 0, false); n = n + 1
780 elseif p == "=" then 1008 elseif p == "=" then
781 op = op + shl(band(op, 0xf800), 5) -- Copy D to T for clz, clo. 1009 n = n - 1 -- Re-use previous parameter for next template char.
782 else 1010 else
783 assert(false) 1011 assert(false)
784 end 1012 end
diff --git a/dynasm/dasm_mips64.lua b/dynasm/dasm_mips64.lua
new file mode 100644
index 00000000..c97d666b
--- /dev/null
+++ b/dynasm/dasm_mips64.lua
@@ -0,0 +1,12 @@
1------------------------------------------------------------------------------
2-- DynASM MIPS64 module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- See dynasm.lua for full copyright notice.
6------------------------------------------------------------------------------
7-- This module just sets 64 bit mode for the combined MIPS/MIPS64 module.
8-- All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11mips64 = true -- Using a global is an ugly, but effective solution.
12return require("dasm_mips")
diff --git a/dynasm/dasm_ppc.h b/dynasm/dasm_ppc.h
index e2d6f1fc..4c7d7289 100644
--- a/dynasm/dasm_ppc.h
+++ b/dynasm/dasm_ppc.h
@@ -1,5 +1,5 @@
1/* 1/*
2** DynASM PPC encoding engine. 2** DynASM PPC/PPC64 encoding engine.
3** Copyright (C) 2005-2023 Mike Pall. All rights reserved. 3** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
4** Released under the MIT license. See dynasm.lua for full copyright notice. 4** Released under the MIT license. See dynasm.lua for full copyright notice.
5*/ 5*/
@@ -21,7 +21,7 @@ enum {
21 /* The following actions need a buffer position. */ 21 /* The following actions need a buffer position. */
22 DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG, 22 DASM_ALIGN, DASM_REL_LG, DASM_LABEL_LG,
23 /* The following actions also have an argument. */ 23 /* The following actions also have an argument. */
24 DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, 24 DASM_REL_PC, DASM_LABEL_PC, DASM_IMM, DASM_IMMSH,
25 DASM__MAX 25 DASM__MAX
26}; 26};
27 27
@@ -69,7 +69,7 @@ struct dasm_State {
69 size_t lgsize; 69 size_t lgsize;
70 int *pclabels; /* PC label chains/pos ptrs. */ 70 int *pclabels; /* PC label chains/pos ptrs. */
71 size_t pcsize; 71 size_t pcsize;
72 void **globals; /* Array of globals (bias -10). */ 72 void **globals; /* Array of globals. */
73 dasm_Section *section; /* Pointer to active section. */ 73 dasm_Section *section; /* Pointer to active section. */
74 size_t codesize; /* Total size of all code sections. */ 74 size_t codesize; /* Total size of all code sections. */
75 int maxsection; /* 0 <= sectionidx < maxsection. */ 75 int maxsection; /* 0 <= sectionidx < maxsection. */
@@ -86,7 +86,6 @@ void dasm_init(Dst_DECL, int maxsection)
86{ 86{
87 dasm_State *D; 87 dasm_State *D;
88 size_t psz = 0; 88 size_t psz = 0;
89 int i;
90 Dst_REF = NULL; 89 Dst_REF = NULL;
91 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); 90 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
92 D = Dst_REF; 91 D = Dst_REF;
@@ -97,12 +96,7 @@ void dasm_init(Dst_DECL, int maxsection)
97 D->pcsize = 0; 96 D->pcsize = 0;
98 D->globals = NULL; 97 D->globals = NULL;
99 D->maxsection = maxsection; 98 D->maxsection = maxsection;
100 for (i = 0; i < maxsection; i++) { 99 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
101 D->sections[i].buf = NULL; /* Need this for pass3. */
102 D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i);
103 D->sections[i].bsize = 0;
104 D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */
105 }
106} 100}
107 101
108/* Free DynASM state. */ 102/* Free DynASM state. */
@@ -122,7 +116,7 @@ void dasm_free(Dst_DECL)
122void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) 116void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
123{ 117{
124 dasm_State *D = Dst_REF; 118 dasm_State *D = Dst_REF;
125 D->globals = gl - 10; /* Negative bias to compensate for locals. */ 119 D->globals = gl;
126 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); 120 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
127} 121}
128 122
@@ -147,6 +141,7 @@ void dasm_setup(Dst_DECL, const void *actionlist)
147 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); 141 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
148 for (i = 0; i < D->maxsection; i++) { 142 for (i = 0; i < D->maxsection; i++) {
149 D->sections[i].pos = DASM_SEC2POS(i); 143 D->sections[i].pos = DASM_SEC2POS(i);
144 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
150 D->sections[i].ofs = 0; 145 D->sections[i].ofs = 0;
151 } 146 }
152} 147}
@@ -244,6 +239,10 @@ void dasm_put(Dst_DECL, int start, ...)
244#endif 239#endif
245 b[pos++] = n; 240 b[pos++] = n;
246 break; 241 break;
242 case DASM_IMMSH:
243 CK((n >> 6) == 0, RANGE_I);
244 b[pos++] = n;
245 break;
247 } 246 }
248 } 247 }
249 } 248 }
@@ -273,7 +272,7 @@ int dasm_link(Dst_DECL, size_t *szp)
273 272
274 { /* Handle globals not defined in this translation unit. */ 273 { /* Handle globals not defined in this translation unit. */
275 int idx; 274 int idx;
276 for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) { 275 for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
277 int n = D->lglabels[idx]; 276 int n = D->lglabels[idx];
278 /* Undefined label: Collapse rel chain and replace with marker (< 0). */ 277 /* Undefined label: Collapse rel chain and replace with marker (< 0). */
279 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } 278 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
@@ -299,7 +298,7 @@ int dasm_link(Dst_DECL, size_t *szp)
299 case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break; 298 case DASM_ALIGN: ofs -= (b[pos++] + ofs) & (ins & 255); break;
300 case DASM_REL_LG: case DASM_REL_PC: pos++; break; 299 case DASM_REL_LG: case DASM_REL_PC: pos++; break;
301 case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break; 300 case DASM_LABEL_LG: case DASM_LABEL_PC: b[pos++] += ofs; break;
302 case DASM_IMM: pos++; break; 301 case DASM_IMM: case DASM_IMMSH: pos++; break;
303 } 302 }
304 } 303 }
305 stop: (void)0; 304 stop: (void)0;
@@ -349,7 +348,10 @@ int dasm_encode(Dst_DECL, void *buffer)
349 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000; 348 ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0x60000000;
350 break; 349 break;
351 case DASM_REL_LG: 350 case DASM_REL_LG:
352 CK(n >= 0, UNDEF_LG); 351 if (n < 0) {
352 n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp);
353 goto patchrel;
354 }
353 /* fallthrough */ 355 /* fallthrough */
354 case DASM_REL_PC: 356 case DASM_REL_PC:
355 CK(n >= 0, UNDEF_PC); 357 CK(n >= 0, UNDEF_PC);
@@ -361,12 +363,15 @@ int dasm_encode(Dst_DECL, void *buffer)
361 cp[-1] |= ((n+4) & ((ins & 2048) ? 0x0000fffc: 0x03fffffc)); 363 cp[-1] |= ((n+4) & ((ins & 2048) ? 0x0000fffc: 0x03fffffc));
362 break; 364 break;
363 case DASM_LABEL_LG: 365 case DASM_LABEL_LG:
364 ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); 366 ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n);
365 break; 367 break;
366 case DASM_LABEL_PC: break; 368 case DASM_LABEL_PC: break;
367 case DASM_IMM: 369 case DASM_IMM:
368 cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31); 370 cp[-1] |= (n & ((1<<((ins>>5)&31))-1)) << (ins&31);
369 break; 371 break;
372 case DASM_IMMSH:
373 cp[-1] |= (ins & 1) ? ((n&31)<<11)|((n&32)>>4) : ((n&31)<<6)|(n&32);
374 break;
370 default: *cp++ = ins; break; 375 default: *cp++ = ins; break;
371 } 376 }
372 } 377 }
diff --git a/dynasm/dasm_ppc.lua b/dynasm/dasm_ppc.lua
index b4f5cea4..d66ae4a0 100644
--- a/dynasm/dasm_ppc.lua
+++ b/dynasm/dasm_ppc.lua
@@ -1,17 +1,19 @@
1------------------------------------------------------------------------------ 1------------------------------------------------------------------------------
2-- DynASM PPC module. 2-- DynASM PPC/PPC64 module.
3-- 3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. 4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- See dynasm.lua for full copyright notice. 5-- See dynasm.lua for full copyright notice.
6--
7-- Support for various extensions contributed by Caio Souza Oliveira.
6------------------------------------------------------------------------------ 8------------------------------------------------------------------------------
7 9
8-- Module information: 10-- Module information:
9local _info = { 11local _info = {
10 arch = "ppc", 12 arch = "ppc",
11 description = "DynASM PPC module", 13 description = "DynASM PPC module",
12 version = "1.3.0", 14 version = "1.5.0",
13 vernum = 10300, 15 vernum = 10500,
14 release = "2011-05-05", 16 release = "2021-05-02",
15 author = "Mike Pall", 17 author = "Mike Pall",
16 license = "MIT", 18 license = "MIT",
17} 19}
@@ -39,7 +41,7 @@ local wline, werror, wfatal, wwarn
39local action_names = { 41local action_names = {
40 "STOP", "SECTION", "ESC", "REL_EXT", 42 "STOP", "SECTION", "ESC", "REL_EXT",
41 "ALIGN", "REL_LG", "LABEL_LG", 43 "ALIGN", "REL_LG", "LABEL_LG",
42 "REL_PC", "LABEL_PC", "IMM", 44 "REL_PC", "LABEL_PC", "IMM", "IMMSH"
43} 45}
44 46
45-- Maximum number of section buffer positions for dasm_put(). 47-- Maximum number of section buffer positions for dasm_put().
@@ -228,8 +230,18 @@ local map_cond = {
228 230
229------------------------------------------------------------------------------ 231------------------------------------------------------------------------------
230 232
233local map_op, op_template
234
235local function op_alias(opname, f)
236 return function(params, nparams)
237 if not params then return "-> "..opname:sub(1, -3) end
238 f(params, nparams)
239 op_template(params, map_op[opname], nparams)
240 end
241end
242
231-- Template strings for PPC instructions. 243-- Template strings for PPC instructions.
232local map_op = { 244map_op = {
233 tdi_3 = "08000000ARI", 245 tdi_3 = "08000000ARI",
234 twi_3 = "0c000000ARI", 246 twi_3 = "0c000000ARI",
235 mulli_3 = "1c000000RRI", 247 mulli_3 = "1c000000RRI",
@@ -297,6 +309,250 @@ local map_op = {
297 std_2 = "f8000000RD", 309 std_2 = "f8000000RD",
298 stdu_2 = "f8000001RD", 310 stdu_2 = "f8000001RD",
299 311
312 subi_3 = op_alias("addi_3", function(p) p[3] = "-("..p[3]..")" end),
313 subis_3 = op_alias("addis_3", function(p) p[3] = "-("..p[3]..")" end),
314 subic_3 = op_alias("addic_3", function(p) p[3] = "-("..p[3]..")" end),
315 ["subic._3"] = op_alias("addic._3", function(p) p[3] = "-("..p[3]..")" end),
316
317 rotlwi_3 = op_alias("rlwinm_5", function(p)
318 p[4] = "0"; p[5] = "31"
319 end),
320 rotrwi_3 = op_alias("rlwinm_5", function(p)
321 p[3] = "32-("..p[3]..")"; p[4] = "0"; p[5] = "31"
322 end),
323 rotlw_3 = op_alias("rlwnm_5", function(p)
324 p[4] = "0"; p[5] = "31"
325 end),
326 slwi_3 = op_alias("rlwinm_5", function(p)
327 p[5] = "31-("..p[3]..")"; p[4] = "0"
328 end),
329 srwi_3 = op_alias("rlwinm_5", function(p)
330 p[4] = p[3]; p[3] = "32-("..p[3]..")"; p[5] = "31"
331 end),
332 clrlwi_3 = op_alias("rlwinm_5", function(p)
333 p[4] = p[3]; p[3] = "0"; p[5] = "31"
334 end),
335 clrrwi_3 = op_alias("rlwinm_5", function(p)
336 p[5] = "31-("..p[3]..")"; p[3] = "0"; p[4] = "0"
337 end),
338
339 -- Primary opcode 4:
340 mulhhwu_3 = "10000010RRR.",
341 machhwu_3 = "10000018RRR.",
342 mulhhw_3 = "10000050RRR.",
343 nmachhw_3 = "1000005cRRR.",
344 machhwsu_3 = "10000098RRR.",
345 machhws_3 = "100000d8RRR.",
346 nmachhws_3 = "100000dcRRR.",
347 mulchwu_3 = "10000110RRR.",
348 macchwu_3 = "10000118RRR.",
349 mulchw_3 = "10000150RRR.",
350 macchw_3 = "10000158RRR.",
351 nmacchw_3 = "1000015cRRR.",
352 macchwsu_3 = "10000198RRR.",
353 macchws_3 = "100001d8RRR.",
354 nmacchws_3 = "100001dcRRR.",
355 mullhw_3 = "10000350RRR.",
356 maclhw_3 = "10000358RRR.",
357 nmaclhw_3 = "1000035cRRR.",
358 maclhwsu_3 = "10000398RRR.",
359 maclhws_3 = "100003d8RRR.",
360 nmaclhws_3 = "100003dcRRR.",
361 machhwuo_3 = "10000418RRR.",
362 nmachhwo_3 = "1000045cRRR.",
363 machhwsuo_3 = "10000498RRR.",
364 machhwso_3 = "100004d8RRR.",
365 nmachhwso_3 = "100004dcRRR.",
366 macchwuo_3 = "10000518RRR.",
367 macchwo_3 = "10000558RRR.",
368 nmacchwo_3 = "1000055cRRR.",
369 macchwsuo_3 = "10000598RRR.",
370 macchwso_3 = "100005d8RRR.",
371 nmacchwso_3 = "100005dcRRR.",
372 maclhwo_3 = "10000758RRR.",
373 nmaclhwo_3 = "1000075cRRR.",
374 maclhwsuo_3 = "10000798RRR.",
375 maclhwso_3 = "100007d8RRR.",
376 nmaclhwso_3 = "100007dcRRR.",
377
378 vaddubm_3 = "10000000VVV",
379 vmaxub_3 = "10000002VVV",
380 vrlb_3 = "10000004VVV",
381 vcmpequb_3 = "10000006VVV",
382 vmuloub_3 = "10000008VVV",
383 vaddfp_3 = "1000000aVVV",
384 vmrghb_3 = "1000000cVVV",
385 vpkuhum_3 = "1000000eVVV",
386 vmhaddshs_4 = "10000020VVVV",
387 vmhraddshs_4 = "10000021VVVV",
388 vmladduhm_4 = "10000022VVVV",
389 vmsumubm_4 = "10000024VVVV",
390 vmsummbm_4 = "10000025VVVV",
391 vmsumuhm_4 = "10000026VVVV",
392 vmsumuhs_4 = "10000027VVVV",
393 vmsumshm_4 = "10000028VVVV",
394 vmsumshs_4 = "10000029VVVV",
395 vsel_4 = "1000002aVVVV",
396 vperm_4 = "1000002bVVVV",
397 vsldoi_4 = "1000002cVVVP",
398 vpermxor_4 = "1000002dVVVV",
399 vmaddfp_4 = "1000002eVVVV~",
400 vnmsubfp_4 = "1000002fVVVV~",
401 vaddeuqm_4 = "1000003cVVVV",
402 vaddecuq_4 = "1000003dVVVV",
403 vsubeuqm_4 = "1000003eVVVV",
404 vsubecuq_4 = "1000003fVVVV",
405 vadduhm_3 = "10000040VVV",
406 vmaxuh_3 = "10000042VVV",
407 vrlh_3 = "10000044VVV",
408 vcmpequh_3 = "10000046VVV",
409 vmulouh_3 = "10000048VVV",
410 vsubfp_3 = "1000004aVVV",
411 vmrghh_3 = "1000004cVVV",
412 vpkuwum_3 = "1000004eVVV",
413 vadduwm_3 = "10000080VVV",
414 vmaxuw_3 = "10000082VVV",
415 vrlw_3 = "10000084VVV",
416 vcmpequw_3 = "10000086VVV",
417 vmulouw_3 = "10000088VVV",
418 vmuluwm_3 = "10000089VVV",
419 vmrghw_3 = "1000008cVVV",
420 vpkuhus_3 = "1000008eVVV",
421 vaddudm_3 = "100000c0VVV",
422 vmaxud_3 = "100000c2VVV",
423 vrld_3 = "100000c4VVV",
424 vcmpeqfp_3 = "100000c6VVV",
425 vcmpequd_3 = "100000c7VVV",
426 vpkuwus_3 = "100000ceVVV",
427 vadduqm_3 = "10000100VVV",
428 vmaxsb_3 = "10000102VVV",
429 vslb_3 = "10000104VVV",
430 vmulosb_3 = "10000108VVV",
431 vrefp_2 = "1000010aV-V",
432 vmrglb_3 = "1000010cVVV",
433 vpkshus_3 = "1000010eVVV",
434 vaddcuq_3 = "10000140VVV",
435 vmaxsh_3 = "10000142VVV",
436 vslh_3 = "10000144VVV",
437 vmulosh_3 = "10000148VVV",
438 vrsqrtefp_2 = "1000014aV-V",
439 vmrglh_3 = "1000014cVVV",
440 vpkswus_3 = "1000014eVVV",
441 vaddcuw_3 = "10000180VVV",
442 vmaxsw_3 = "10000182VVV",
443 vslw_3 = "10000184VVV",
444 vmulosw_3 = "10000188VVV",
445 vexptefp_2 = "1000018aV-V",
446 vmrglw_3 = "1000018cVVV",
447 vpkshss_3 = "1000018eVVV",
448 vmaxsd_3 = "100001c2VVV",
449 vsl_3 = "100001c4VVV",
450 vcmpgefp_3 = "100001c6VVV",
451 vlogefp_2 = "100001caV-V",
452 vpkswss_3 = "100001ceVVV",
453 vadduhs_3 = "10000240VVV",
454 vminuh_3 = "10000242VVV",
455 vsrh_3 = "10000244VVV",
456 vcmpgtuh_3 = "10000246VVV",
457 vmuleuh_3 = "10000248VVV",
458 vrfiz_2 = "1000024aV-V",
459 vsplth_3 = "1000024cVV3",
460 vupkhsh_2 = "1000024eV-V",
461 vminuw_3 = "10000282VVV",
462 vminud_3 = "100002c2VVV",
463 vcmpgtud_3 = "100002c7VVV",
464 vrfim_2 = "100002caV-V",
465 vcmpgtsb_3 = "10000306VVV",
466 vcfux_3 = "1000030aVVA~",
467 vaddshs_3 = "10000340VVV",
468 vminsh_3 = "10000342VVV",
469 vsrah_3 = "10000344VVV",
470 vcmpgtsh_3 = "10000346VVV",
471 vmulesh_3 = "10000348VVV",
472 vcfsx_3 = "1000034aVVA~",
473 vspltish_2 = "1000034cVS",
474 vupkhpx_2 = "1000034eV-V",
475 vaddsws_3 = "10000380VVV",
476 vminsw_3 = "10000382VVV",
477 vsraw_3 = "10000384VVV",
478 vcmpgtsw_3 = "10000386VVV",
479 vmulesw_3 = "10000388VVV",
480 vctuxs_3 = "1000038aVVA~",
481 vspltisw_2 = "1000038cVS",
482 vminsd_3 = "100003c2VVV",
483 vsrad_3 = "100003c4VVV",
484 vcmpbfp_3 = "100003c6VVV",
485 vcmpgtsd_3 = "100003c7VVV",
486 vctsxs_3 = "100003caVVA~",
487 vupklpx_2 = "100003ceV-V",
488 vsububm_3 = "10000400VVV",
489 ["bcdadd._4"] = "10000401VVVy.",
490 vavgub_3 = "10000402VVV",
491 vand_3 = "10000404VVV",
492 ["vcmpequb._3"] = "10000406VVV",
493 vmaxfp_3 = "1000040aVVV",
494 vsubuhm_3 = "10000440VVV",
495 ["bcdsub._4"] = "10000441VVVy.",
496 vavguh_3 = "10000442VVV",
497 vandc_3 = "10000444VVV",
498 ["vcmpequh._3"] = "10000446VVV",
499 vminfp_3 = "1000044aVVV",
500 vpkudum_3 = "1000044eVVV",
501 vsubuwm_3 = "10000480VVV",
502 vavguw_3 = "10000482VVV",
503 vor_3 = "10000484VVV",
504 ["vcmpequw._3"] = "10000486VVV",
505 vpmsumw_3 = "10000488VVV",
506 ["vcmpeqfp._3"] = "100004c6VVV",
507 ["vcmpequd._3"] = "100004c7VVV",
508 vpkudus_3 = "100004ceVVV",
509 vavgsb_3 = "10000502VVV",
510 vavgsh_3 = "10000542VVV",
511 vorc_3 = "10000544VVV",
512 vbpermq_3 = "1000054cVVV",
513 vpksdus_3 = "1000054eVVV",
514 vavgsw_3 = "10000582VVV",
515 vsld_3 = "100005c4VVV",
516 ["vcmpgefp._3"] = "100005c6VVV",
517 vpksdss_3 = "100005ceVVV",
518 vsububs_3 = "10000600VVV",
519 mfvscr_1 = "10000604V--",
520 vsum4ubs_3 = "10000608VVV",
521 vsubuhs_3 = "10000640VVV",
522 mtvscr_1 = "10000644--V",
523 ["vcmpgtuh._3"] = "10000646VVV",
524 vsum4shs_3 = "10000648VVV",
525 vupkhsw_2 = "1000064eV-V",
526 vsubuws_3 = "10000680VVV",
527 vshasigmaw_4 = "10000682VVYp",
528 veqv_3 = "10000684VVV",
529 vsum2sws_3 = "10000688VVV",
530 vmrgow_3 = "1000068cVVV",
531 vshasigmad_4 = "100006c2VVYp",
532 vsrd_3 = "100006c4VVV",
533 ["vcmpgtud._3"] = "100006c7VVV",
534 vupklsw_2 = "100006ceV-V",
535 vupkslw_2 = "100006ceV-V",
536 vsubsbs_3 = "10000700VVV",
537 vclzb_2 = "10000702V-V",
538 vpopcntb_2 = "10000703V-V",
539 ["vcmpgtsb._3"] = "10000706VVV",
540 vsum4sbs_3 = "10000708VVV",
541 vsubshs_3 = "10000740VVV",
542 vclzh_2 = "10000742V-V",
543 vpopcnth_2 = "10000743V-V",
544 ["vcmpgtsh._3"] = "10000746VVV",
545 vsubsws_3 = "10000780VVV",
546 vclzw_2 = "10000782V-V",
547 vpopcntw_2 = "10000783V-V",
548 ["vcmpgtsw._3"] = "10000786VVV",
549 vsumsws_3 = "10000788VVV",
550 vmrgew_3 = "1000078cVVV",
551 vclzd_2 = "100007c2V-V",
552 vpopcntd_2 = "100007c3V-V",
553 ["vcmpbfp._3"] = "100007c6VVV",
554 ["vcmpgtsd._3"] = "100007c7VVV",
555
300 -- Primary opcode 19: 556 -- Primary opcode 19:
301 mcrf_2 = "4c000000XX", 557 mcrf_2 = "4c000000XX",
302 isync_0 = "4c00012c", 558 isync_0 = "4c00012c",
@@ -316,6 +572,8 @@ local map_op = {
316 bclrl_2 = "4c000021AA", 572 bclrl_2 = "4c000021AA",
317 bcctr_2 = "4c000420AA", 573 bcctr_2 = "4c000420AA",
318 bcctrl_2 = "4c000421AA", 574 bcctrl_2 = "4c000421AA",
575 bctar_2 = "4c000460AA",
576 bctarl_2 = "4c000461AA",
319 blr_0 = "4e800020", 577 blr_0 = "4e800020",
320 blrl_0 = "4e800021", 578 blrl_0 = "4e800021",
321 bctr_0 = "4e800420", 579 bctr_0 = "4e800420",
@@ -327,6 +585,7 @@ local map_op = {
327 cmpd_3 = "7c200000XRR", 585 cmpd_3 = "7c200000XRR",
328 cmpd_2 = "7c200000-RR", 586 cmpd_2 = "7c200000-RR",
329 tw_3 = "7c000008ARR", 587 tw_3 = "7c000008ARR",
588 lvsl_3 = "7c00000cVRR",
330 subfc_3 = "7c000010RRR.", 589 subfc_3 = "7c000010RRR.",
331 subc_3 = "7c000010RRR~.", 590 subc_3 = "7c000010RRR~.",
332 mulhdu_3 = "7c000012RRR.", 591 mulhdu_3 = "7c000012RRR.",
@@ -351,50 +610,68 @@ local map_op = {
351 cmplw_2 = "7c000040-RR", 610 cmplw_2 = "7c000040-RR",
352 cmpld_3 = "7c200040XRR", 611 cmpld_3 = "7c200040XRR",
353 cmpld_2 = "7c200040-RR", 612 cmpld_2 = "7c200040-RR",
613 lvsr_3 = "7c00004cVRR",
354 subf_3 = "7c000050RRR.", 614 subf_3 = "7c000050RRR.",
355 sub_3 = "7c000050RRR~.", 615 sub_3 = "7c000050RRR~.",
616 lbarx_3 = "7c000068RR0R",
356 ldux_3 = "7c00006aRR0R", 617 ldux_3 = "7c00006aRR0R",
357 dcbst_2 = "7c00006c-RR", 618 dcbst_2 = "7c00006c-RR",
358 lwzux_3 = "7c00006eRR0R", 619 lwzux_3 = "7c00006eRR0R",
359 cntlzd_2 = "7c000074RR~", 620 cntlzd_2 = "7c000074RR~",
360 andc_3 = "7c000078RR~R.", 621 andc_3 = "7c000078RR~R.",
361 td_3 = "7c000088ARR", 622 td_3 = "7c000088ARR",
623 lvewx_3 = "7c00008eVRR",
362 mulhd_3 = "7c000092RRR.", 624 mulhd_3 = "7c000092RRR.",
625 addg6s_3 = "7c000094RRR",
363 mulhw_3 = "7c000096RRR.", 626 mulhw_3 = "7c000096RRR.",
627 dlmzb_3 = "7c00009cRR~R.",
364 ldarx_3 = "7c0000a8RR0R", 628 ldarx_3 = "7c0000a8RR0R",
365 dcbf_2 = "7c0000ac-RR", 629 dcbf_2 = "7c0000ac-RR",
366 lbzx_3 = "7c0000aeRR0R", 630 lbzx_3 = "7c0000aeRR0R",
631 lvx_3 = "7c0000ceVRR",
367 neg_2 = "7c0000d0RR.", 632 neg_2 = "7c0000d0RR.",
633 lharx_3 = "7c0000e8RR0R",
368 lbzux_3 = "7c0000eeRR0R", 634 lbzux_3 = "7c0000eeRR0R",
369 popcntb_2 = "7c0000f4RR~", 635 popcntb_2 = "7c0000f4RR~",
370 not_2 = "7c0000f8RR~%.", 636 not_2 = "7c0000f8RR~%.",
371 nor_3 = "7c0000f8RR~R.", 637 nor_3 = "7c0000f8RR~R.",
638 stvebx_3 = "7c00010eVRR",
372 subfe_3 = "7c000110RRR.", 639 subfe_3 = "7c000110RRR.",
373 sube_3 = "7c000110RRR~.", 640 sube_3 = "7c000110RRR~.",
374 adde_3 = "7c000114RRR.", 641 adde_3 = "7c000114RRR.",
375 stdx_3 = "7c00012aRR0R", 642 stdx_3 = "7c00012aRR0R",
376 stwcx_3 = "7c00012cRR0R.", 643 ["stwcx._3"] = "7c00012dRR0R.",
377 stwx_3 = "7c00012eRR0R", 644 stwx_3 = "7c00012eRR0R",
378 prtyw_2 = "7c000134RR~", 645 prtyw_2 = "7c000134RR~",
646 stvehx_3 = "7c00014eVRR",
379 stdux_3 = "7c00016aRR0R", 647 stdux_3 = "7c00016aRR0R",
648 ["stqcx._3"] = "7c00016dR:R0R.",
380 stwux_3 = "7c00016eRR0R", 649 stwux_3 = "7c00016eRR0R",
381 prtyd_2 = "7c000174RR~", 650 prtyd_2 = "7c000174RR~",
651 stvewx_3 = "7c00018eVRR",
382 subfze_2 = "7c000190RR.", 652 subfze_2 = "7c000190RR.",
383 addze_2 = "7c000194RR.", 653 addze_2 = "7c000194RR.",
384 stdcx_3 = "7c0001acRR0R.", 654 ["stdcx._3"] = "7c0001adRR0R.",
385 stbx_3 = "7c0001aeRR0R", 655 stbx_3 = "7c0001aeRR0R",
656 stvx_3 = "7c0001ceVRR",
386 subfme_2 = "7c0001d0RR.", 657 subfme_2 = "7c0001d0RR.",
387 mulld_3 = "7c0001d2RRR.", 658 mulld_3 = "7c0001d2RRR.",
388 addme_2 = "7c0001d4RR.", 659 addme_2 = "7c0001d4RR.",
389 mullw_3 = "7c0001d6RRR.", 660 mullw_3 = "7c0001d6RRR.",
390 dcbtst_2 = "7c0001ec-RR", 661 dcbtst_2 = "7c0001ec-RR",
391 stbux_3 = "7c0001eeRR0R", 662 stbux_3 = "7c0001eeRR0R",
663 bpermd_3 = "7c0001f8RR~R",
664 lvepxl_3 = "7c00020eVRR",
392 add_3 = "7c000214RRR.", 665 add_3 = "7c000214RRR.",
666 lqarx_3 = "7c000228R:R0R",
393 dcbt_2 = "7c00022c-RR", 667 dcbt_2 = "7c00022c-RR",
394 lhzx_3 = "7c00022eRR0R", 668 lhzx_3 = "7c00022eRR0R",
669 cdtbcd_2 = "7c000234RR~",
395 eqv_3 = "7c000238RR~R.", 670 eqv_3 = "7c000238RR~R.",
671 lvepx_3 = "7c00024eVRR",
396 eciwx_3 = "7c00026cRR0R", 672 eciwx_3 = "7c00026cRR0R",
397 lhzux_3 = "7c00026eRR0R", 673 lhzux_3 = "7c00026eRR0R",
674 cbcdtd_2 = "7c000274RR~",
398 xor_3 = "7c000278RR~R.", 675 xor_3 = "7c000278RR~R.",
399 mfspefscr_1 = "7c0082a6R", 676 mfspefscr_1 = "7c0082a6R",
400 mfxer_1 = "7c0102a6R", 677 mfxer_1 = "7c0102a6R",
@@ -404,8 +681,12 @@ local map_op = {
404 lhax_3 = "7c0002aeRR0R", 681 lhax_3 = "7c0002aeRR0R",
405 mftb_1 = "7c0c42e6R", 682 mftb_1 = "7c0c42e6R",
406 mftbu_1 = "7c0d42e6R", 683 mftbu_1 = "7c0d42e6R",
684 lvxl_3 = "7c0002ceVRR",
407 lwaux_3 = "7c0002eaRR0R", 685 lwaux_3 = "7c0002eaRR0R",
408 lhaux_3 = "7c0002eeRR0R", 686 lhaux_3 = "7c0002eeRR0R",
687 popcntw_2 = "7c0002f4RR~",
688 divdeu_3 = "7c000312RRR.",
689 divweu_3 = "7c000316RRR.",
409 sthx_3 = "7c00032eRR0R", 690 sthx_3 = "7c00032eRR0R",
410 orc_3 = "7c000338RR~R.", 691 orc_3 = "7c000338RR~R.",
411 ecowx_3 = "7c00036cRR0R", 692 ecowx_3 = "7c00036cRR0R",
@@ -420,10 +701,14 @@ local map_op = {
420 mtctr_1 = "7c0903a6R", 701 mtctr_1 = "7c0903a6R",
421 dcbi_2 = "7c0003ac-RR", 702 dcbi_2 = "7c0003ac-RR",
422 nand_3 = "7c0003b8RR~R.", 703 nand_3 = "7c0003b8RR~R.",
704 dsn_2 = "7c0003c6-RR",
705 stvxl_3 = "7c0003ceVRR",
423 divd_3 = "7c0003d2RRR.", 706 divd_3 = "7c0003d2RRR.",
424 divw_3 = "7c0003d6RRR.", 707 divw_3 = "7c0003d6RRR.",
708 popcntd_2 = "7c0003f4RR~",
425 cmpb_3 = "7c0003f8RR~R.", 709 cmpb_3 = "7c0003f8RR~R.",
426 mcrxr_1 = "7c000400X", 710 mcrxr_1 = "7c000400X",
711 lbdx_3 = "7c000406RRR",
427 subfco_3 = "7c000410RRR.", 712 subfco_3 = "7c000410RRR.",
428 subco_3 = "7c000410RRR~.", 713 subco_3 = "7c000410RRR~.",
429 addco_3 = "7c000414RRR.", 714 addco_3 = "7c000414RRR.",
@@ -433,16 +718,20 @@ local map_op = {
433 lfsx_3 = "7c00042eFR0R", 718 lfsx_3 = "7c00042eFR0R",
434 srw_3 = "7c000430RR~R.", 719 srw_3 = "7c000430RR~R.",
435 srd_3 = "7c000436RR~R.", 720 srd_3 = "7c000436RR~R.",
721 lhdx_3 = "7c000446RRR",
436 subfo_3 = "7c000450RRR.", 722 subfo_3 = "7c000450RRR.",
437 subo_3 = "7c000450RRR~.", 723 subo_3 = "7c000450RRR~.",
438 lfsux_3 = "7c00046eFR0R", 724 lfsux_3 = "7c00046eFR0R",
725 lwdx_3 = "7c000486RRR",
439 lswi_3 = "7c0004aaRR0A", 726 lswi_3 = "7c0004aaRR0A",
440 sync_0 = "7c0004ac", 727 sync_0 = "7c0004ac",
441 lwsync_0 = "7c2004ac", 728 lwsync_0 = "7c2004ac",
442 ptesync_0 = "7c4004ac", 729 ptesync_0 = "7c4004ac",
443 lfdx_3 = "7c0004aeFR0R", 730 lfdx_3 = "7c0004aeFR0R",
731 lddx_3 = "7c0004c6RRR",
444 nego_2 = "7c0004d0RR.", 732 nego_2 = "7c0004d0RR.",
445 lfdux_3 = "7c0004eeFR0R", 733 lfdux_3 = "7c0004eeFR0R",
734 stbdx_3 = "7c000506RRR",
446 subfeo_3 = "7c000510RRR.", 735 subfeo_3 = "7c000510RRR.",
447 subeo_3 = "7c000510RRR~.", 736 subeo_3 = "7c000510RRR~.",
448 addeo_3 = "7c000514RRR.", 737 addeo_3 = "7c000514RRR.",
@@ -450,27 +739,42 @@ local map_op = {
450 stswx_3 = "7c00052aRR0R", 739 stswx_3 = "7c00052aRR0R",
451 stwbrx_3 = "7c00052cRR0R", 740 stwbrx_3 = "7c00052cRR0R",
452 stfsx_3 = "7c00052eFR0R", 741 stfsx_3 = "7c00052eFR0R",
742 sthdx_3 = "7c000546RRR",
743 ["stbcx._3"] = "7c00056dRRR",
453 stfsux_3 = "7c00056eFR0R", 744 stfsux_3 = "7c00056eFR0R",
745 stwdx_3 = "7c000586RRR",
454 subfzeo_2 = "7c000590RR.", 746 subfzeo_2 = "7c000590RR.",
455 addzeo_2 = "7c000594RR.", 747 addzeo_2 = "7c000594RR.",
456 stswi_3 = "7c0005aaRR0A", 748 stswi_3 = "7c0005aaRR0A",
749 ["sthcx._3"] = "7c0005adRRR",
457 stfdx_3 = "7c0005aeFR0R", 750 stfdx_3 = "7c0005aeFR0R",
751 stddx_3 = "7c0005c6RRR",
458 subfmeo_2 = "7c0005d0RR.", 752 subfmeo_2 = "7c0005d0RR.",
459 mulldo_3 = "7c0005d2RRR.", 753 mulldo_3 = "7c0005d2RRR.",
460 addmeo_2 = "7c0005d4RR.", 754 addmeo_2 = "7c0005d4RR.",
461 mullwo_3 = "7c0005d6RRR.", 755 mullwo_3 = "7c0005d6RRR.",
462 dcba_2 = "7c0005ec-RR", 756 dcba_2 = "7c0005ec-RR",
463 stfdux_3 = "7c0005eeFR0R", 757 stfdux_3 = "7c0005eeFR0R",
758 stvepxl_3 = "7c00060eVRR",
464 addo_3 = "7c000614RRR.", 759 addo_3 = "7c000614RRR.",
465 lhbrx_3 = "7c00062cRR0R", 760 lhbrx_3 = "7c00062cRR0R",
761 lfdpx_3 = "7c00062eF:RR",
466 sraw_3 = "7c000630RR~R.", 762 sraw_3 = "7c000630RR~R.",
467 srad_3 = "7c000634RR~R.", 763 srad_3 = "7c000634RR~R.",
764 lfddx_3 = "7c000646FRR",
765 stvepx_3 = "7c00064eVRR",
468 srawi_3 = "7c000670RR~A.", 766 srawi_3 = "7c000670RR~A.",
469 sradi_3 = "7c000674RR~H.", 767 sradi_3 = "7c000674RR~H.",
470 eieio_0 = "7c0006ac", 768 eieio_0 = "7c0006ac",
471 lfiwax_3 = "7c0006aeFR0R", 769 lfiwax_3 = "7c0006aeFR0R",
770 divdeuo_3 = "7c000712RRR.",
771 divweuo_3 = "7c000716RRR.",
472 sthbrx_3 = "7c00072cRR0R", 772 sthbrx_3 = "7c00072cRR0R",
773 stfdpx_3 = "7c00072eF:RR",
473 extsh_2 = "7c000734RR~.", 774 extsh_2 = "7c000734RR~.",
775 stfddx_3 = "7c000746FRR",
776 divdeo_3 = "7c000752RRR.",
777 divweo_3 = "7c000756RRR.",
474 extsb_2 = "7c000774RR~.", 778 extsb_2 = "7c000774RR~.",
475 divduo_3 = "7c000792RRR.", 779 divduo_3 = "7c000792RRR.",
476 divwou_3 = "7c000796RRR.", 780 divwou_3 = "7c000796RRR.",
@@ -481,6 +785,40 @@ local map_op = {
481 divwo_3 = "7c0007d6RRR.", 785 divwo_3 = "7c0007d6RRR.",
482 dcbz_2 = "7c0007ec-RR", 786 dcbz_2 = "7c0007ec-RR",
483 787
788 ["tbegin._1"] = "7c00051d1",
789 ["tbegin._0"] = "7c00051d",
790 ["tend._1"] = "7c00055dY",
791 ["tend._0"] = "7c00055d",
792 ["tendall._0"] = "7e00055d",
793 tcheck_1 = "7c00059cX",
794 ["tsr._1"] = "7c0005dd1",
795 ["tsuspend._0"] = "7c0005dd",
796 ["tresume._0"] = "7c2005dd",
797 ["tabortwc._3"] = "7c00061dARR",
798 ["tabortdc._3"] = "7c00065dARR",
799 ["tabortwci._3"] = "7c00069dARS",
800 ["tabortdci._3"] = "7c0006ddARS",
801 ["tabort._1"] = "7c00071d-R-",
802 ["treclaim._1"] = "7c00075d-R",
803 ["trechkpt._0"] = "7c0007dd",
804
805 lxsiwzx_3 = "7c000018QRR",
806 lxsiwax_3 = "7c000098QRR",
807 mfvsrd_2 = "7c000066-Rq",
808 mfvsrwz_2 = "7c0000e6-Rq",
809 stxsiwx_3 = "7c000118QRR",
810 mtvsrd_2 = "7c000166QR",
811 mtvsrwa_2 = "7c0001a6QR",
812 lxvdsx_3 = "7c000298QRR",
813 lxsspx_3 = "7c000418QRR",
814 lxsdx_3 = "7c000498QRR",
815 stxsspx_3 = "7c000518QRR",
816 stxsdx_3 = "7c000598QRR",
817 lxvw4x_3 = "7c000618QRR",
818 lxvd2x_3 = "7c000698QRR",
819 stxvw4x_3 = "7c000718QRR",
820 stxvd2x_3 = "7c000798QRR",
821
484 -- Primary opcode 30: 822 -- Primary opcode 30:
485 rldicl_4 = "78000000RR~HM.", 823 rldicl_4 = "78000000RR~HM.",
486 rldicr_4 = "78000004RR~HM.", 824 rldicr_4 = "78000004RR~HM.",
@@ -489,6 +827,34 @@ local map_op = {
489 rldcl_4 = "78000010RR~RM.", 827 rldcl_4 = "78000010RR~RM.",
490 rldcr_4 = "78000012RR~RM.", 828 rldcr_4 = "78000012RR~RM.",
491 829
830 rotldi_3 = op_alias("rldicl_4", function(p)
831 p[4] = "0"
832 end),
833 rotrdi_3 = op_alias("rldicl_4", function(p)
834 p[3] = "64-("..p[3]..")"; p[4] = "0"
835 end),
836 rotld_3 = op_alias("rldcl_4", function(p)
837 p[4] = "0"
838 end),
839 sldi_3 = op_alias("rldicr_4", function(p)
840 p[4] = "63-("..p[3]..")"
841 end),
842 srdi_3 = op_alias("rldicl_4", function(p)
843 p[4] = p[3]; p[3] = "64-("..p[3]..")"
844 end),
845 clrldi_3 = op_alias("rldicl_4", function(p)
846 p[4] = p[3]; p[3] = "0"
847 end),
848 clrrdi_3 = op_alias("rldicr_4", function(p)
849 p[4] = "63-("..p[3]..")"; p[3] = "0"
850 end),
851
852 -- Primary opcode 56:
853 lq_2 = "e0000000R:D", -- NYI: displacement must be divisible by 8.
854
855 -- Primary opcode 57:
856 lfdp_2 = "e4000000F:D", -- NYI: displacement must be divisible by 4.
857
492 -- Primary opcode 59: 858 -- Primary opcode 59:
493 fdivs_3 = "ec000024FFF.", 859 fdivs_3 = "ec000024FFF.",
494 fsubs_3 = "ec000028FFF.", 860 fsubs_3 = "ec000028FFF.",
@@ -501,6 +867,200 @@ local map_op = {
501 fmadds_4 = "ec00003aFFFF~.", 867 fmadds_4 = "ec00003aFFFF~.",
502 fnmsubs_4 = "ec00003cFFFF~.", 868 fnmsubs_4 = "ec00003cFFFF~.",
503 fnmadds_4 = "ec00003eFFFF~.", 869 fnmadds_4 = "ec00003eFFFF~.",
870 fcfids_2 = "ec00069cF-F.",
871 fcfidus_2 = "ec00079cF-F.",
872
873 dadd_3 = "ec000004FFF.",
874 dqua_4 = "ec000006FFFZ.",
875 dmul_3 = "ec000044FFF.",
876 drrnd_4 = "ec000046FFFZ.",
877 dscli_3 = "ec000084FF6.",
878 dquai_4 = "ec000086SF~FZ.",
879 dscri_3 = "ec0000c4FF6.",
880 drintx_4 = "ec0000c61F~FZ.",
881 dcmpo_3 = "ec000104XFF",
882 dtstex_3 = "ec000144XFF",
883 dtstdc_3 = "ec000184XF6",
884 dtstdg_3 = "ec0001c4XF6",
885 drintn_4 = "ec0001c61F~FZ.",
886 dctdp_2 = "ec000204F-F.",
887 dctfix_2 = "ec000244F-F.",
888 ddedpd_3 = "ec000284ZF~F.",
889 dxex_2 = "ec0002c4F-F.",
890 dsub_3 = "ec000404FFF.",
891 ddiv_3 = "ec000444FFF.",
892 dcmpu_3 = "ec000504XFF",
893 dtstsf_3 = "ec000544XFF",
894 drsp_2 = "ec000604F-F.",
895 dcffix_2 = "ec000644F-F.",
896 denbcd_3 = "ec000684YF~F.",
897 diex_3 = "ec0006c4FFF.",
898
899 -- Primary opcode 60:
900 xsaddsp_3 = "f0000000QQQ",
901 xsmaddasp_3 = "f0000008QQQ",
902 xxsldwi_4 = "f0000010QQQz",
903 xsrsqrtesp_2 = "f0000028Q-Q",
904 xssqrtsp_2 = "f000002cQ-Q",
905 xxsel_4 = "f0000030QQQQ",
906 xssubsp_3 = "f0000040QQQ",
907 xsmaddmsp_3 = "f0000048QQQ",
908 xxpermdi_4 = "f0000050QQQz",
909 xsresp_2 = "f0000068Q-Q",
910 xsmulsp_3 = "f0000080QQQ",
911 xsmsubasp_3 = "f0000088QQQ",
912 xxmrghw_3 = "f0000090QQQ",
913 xsdivsp_3 = "f00000c0QQQ",
914 xsmsubmsp_3 = "f00000c8QQQ",
915 xsadddp_3 = "f0000100QQQ",
916 xsmaddadp_3 = "f0000108QQQ",
917 xscmpudp_3 = "f0000118XQQ",
918 xscvdpuxws_2 = "f0000120Q-Q",
919 xsrdpi_2 = "f0000124Q-Q",
920 xsrsqrtedp_2 = "f0000128Q-Q",
921 xssqrtdp_2 = "f000012cQ-Q",
922 xssubdp_3 = "f0000140QQQ",
923 xsmaddmdp_3 = "f0000148QQQ",
924 xscmpodp_3 = "f0000158XQQ",
925 xscvdpsxws_2 = "f0000160Q-Q",
926 xsrdpiz_2 = "f0000164Q-Q",
927 xsredp_2 = "f0000168Q-Q",
928 xsmuldp_3 = "f0000180QQQ",
929 xsmsubadp_3 = "f0000188QQQ",
930 xxmrglw_3 = "f0000190QQQ",
931 xsrdpip_2 = "f00001a4Q-Q",
932 xstsqrtdp_2 = "f00001a8X-Q",
933 xsrdpic_2 = "f00001acQ-Q",
934 xsdivdp_3 = "f00001c0QQQ",
935 xsmsubmdp_3 = "f00001c8QQQ",
936 xsrdpim_2 = "f00001e4Q-Q",
937 xstdivdp_3 = "f00001e8XQQ",
938 xvaddsp_3 = "f0000200QQQ",
939 xvmaddasp_3 = "f0000208QQQ",
940 xvcmpeqsp_3 = "f0000218QQQ",
941 xvcvspuxws_2 = "f0000220Q-Q",
942 xvrspi_2 = "f0000224Q-Q",
943 xvrsqrtesp_2 = "f0000228Q-Q",
944 xvsqrtsp_2 = "f000022cQ-Q",
945 xvsubsp_3 = "f0000240QQQ",
946 xvmaddmsp_3 = "f0000248QQQ",
947 xvcmpgtsp_3 = "f0000258QQQ",
948 xvcvspsxws_2 = "f0000260Q-Q",
949 xvrspiz_2 = "f0000264Q-Q",
950 xvresp_2 = "f0000268Q-Q",
951 xvmulsp_3 = "f0000280QQQ",
952 xvmsubasp_3 = "f0000288QQQ",
953 xxspltw_3 = "f0000290QQg~",
954 xvcmpgesp_3 = "f0000298QQQ",
955 xvcvuxwsp_2 = "f00002a0Q-Q",
956 xvrspip_2 = "f00002a4Q-Q",
957 xvtsqrtsp_2 = "f00002a8X-Q",
958 xvrspic_2 = "f00002acQ-Q",
959 xvdivsp_3 = "f00002c0QQQ",
960 xvmsubmsp_3 = "f00002c8QQQ",
961 xvcvsxwsp_2 = "f00002e0Q-Q",
962 xvrspim_2 = "f00002e4Q-Q",
963 xvtdivsp_3 = "f00002e8XQQ",
964 xvadddp_3 = "f0000300QQQ",
965 xvmaddadp_3 = "f0000308QQQ",
966 xvcmpeqdp_3 = "f0000318QQQ",
967 xvcvdpuxws_2 = "f0000320Q-Q",
968 xvrdpi_2 = "f0000324Q-Q",
969 xvrsqrtedp_2 = "f0000328Q-Q",
970 xvsqrtdp_2 = "f000032cQ-Q",
971 xvsubdp_3 = "f0000340QQQ",
972 xvmaddmdp_3 = "f0000348QQQ",
973 xvcmpgtdp_3 = "f0000358QQQ",
974 xvcvdpsxws_2 = "f0000360Q-Q",
975 xvrdpiz_2 = "f0000364Q-Q",
976 xvredp_2 = "f0000368Q-Q",
977 xvmuldp_3 = "f0000380QQQ",
978 xvmsubadp_3 = "f0000388QQQ",
979 xvcmpgedp_3 = "f0000398QQQ",
980 xvcvuxwdp_2 = "f00003a0Q-Q",
981 xvrdpip_2 = "f00003a4Q-Q",
982 xvtsqrtdp_2 = "f00003a8X-Q",
983 xvrdpic_2 = "f00003acQ-Q",
984 xvdivdp_3 = "f00003c0QQQ",
985 xvmsubmdp_3 = "f00003c8QQQ",
986 xvcvsxwdp_2 = "f00003e0Q-Q",
987 xvrdpim_2 = "f00003e4Q-Q",
988 xvtdivdp_3 = "f00003e8XQQ",
989 xsnmaddasp_3 = "f0000408QQQ",
990 xxland_3 = "f0000410QQQ",
991 xscvdpsp_2 = "f0000424Q-Q",
992 xscvdpspn_2 = "f000042cQ-Q",
993 xsnmaddmsp_3 = "f0000448QQQ",
994 xxlandc_3 = "f0000450QQQ",
995 xsrsp_2 = "f0000464Q-Q",
996 xsnmsubasp_3 = "f0000488QQQ",
997 xxlor_3 = "f0000490QQQ",
998 xscvuxdsp_2 = "f00004a0Q-Q",
999 xsnmsubmsp_3 = "f00004c8QQQ",
1000 xxlxor_3 = "f00004d0QQQ",
1001 xscvsxdsp_2 = "f00004e0Q-Q",
1002 xsmaxdp_3 = "f0000500QQQ",
1003 xsnmaddadp_3 = "f0000508QQQ",
1004 xxlnor_3 = "f0000510QQQ",
1005 xscvdpuxds_2 = "f0000520Q-Q",
1006 xscvspdp_2 = "f0000524Q-Q",
1007 xscvspdpn_2 = "f000052cQ-Q",
1008 xsmindp_3 = "f0000540QQQ",
1009 xsnmaddmdp_3 = "f0000548QQQ",
1010 xxlorc_3 = "f0000550QQQ",
1011 xscvdpsxds_2 = "f0000560Q-Q",
1012 xsabsdp_2 = "f0000564Q-Q",
1013 xscpsgndp_3 = "f0000580QQQ",
1014 xsnmsubadp_3 = "f0000588QQQ",
1015 xxlnand_3 = "f0000590QQQ",
1016 xscvuxddp_2 = "f00005a0Q-Q",
1017 xsnabsdp_2 = "f00005a4Q-Q",
1018 xsnmsubmdp_3 = "f00005c8QQQ",
1019 xxleqv_3 = "f00005d0QQQ",
1020 xscvsxddp_2 = "f00005e0Q-Q",
1021 xsnegdp_2 = "f00005e4Q-Q",
1022 xvmaxsp_3 = "f0000600QQQ",
1023 xvnmaddasp_3 = "f0000608QQQ",
1024 ["xvcmpeqsp._3"] = "f0000618QQQ",
1025 xvcvspuxds_2 = "f0000620Q-Q",
1026 xvcvdpsp_2 = "f0000624Q-Q",
1027 xvminsp_3 = "f0000640QQQ",
1028 xvnmaddmsp_3 = "f0000648QQQ",
1029 ["xvcmpgtsp._3"] = "f0000658QQQ",
1030 xvcvspsxds_2 = "f0000660Q-Q",
1031 xvabssp_2 = "f0000664Q-Q",
1032 xvcpsgnsp_3 = "f0000680QQQ",
1033 xvnmsubasp_3 = "f0000688QQQ",
1034 ["xvcmpgesp._3"] = "f0000698QQQ",
1035 xvcvuxdsp_2 = "f00006a0Q-Q",
1036 xvnabssp_2 = "f00006a4Q-Q",
1037 xvnmsubmsp_3 = "f00006c8QQQ",
1038 xvcvsxdsp_2 = "f00006e0Q-Q",
1039 xvnegsp_2 = "f00006e4Q-Q",
1040 xvmaxdp_3 = "f0000700QQQ",
1041 xvnmaddadp_3 = "f0000708QQQ",
1042 ["xvcmpeqdp._3"] = "f0000718QQQ",
1043 xvcvdpuxds_2 = "f0000720Q-Q",
1044 xvcvspdp_2 = "f0000724Q-Q",
1045 xvmindp_3 = "f0000740QQQ",
1046 xvnmaddmdp_3 = "f0000748QQQ",
1047 ["xvcmpgtdp._3"] = "f0000758QQQ",
1048 xvcvdpsxds_2 = "f0000760Q-Q",
1049 xvabsdp_2 = "f0000764Q-Q",
1050 xvcpsgndp_3 = "f0000780QQQ",
1051 xvnmsubadp_3 = "f0000788QQQ",
1052 ["xvcmpgedp._3"] = "f0000798QQQ",
1053 xvcvuxddp_2 = "f00007a0Q-Q",
1054 xvnabsdp_2 = "f00007a4Q-Q",
1055 xvnmsubmdp_3 = "f00007c8QQQ",
1056 xvcvsxddp_2 = "f00007e0Q-Q",
1057 xvnegdp_2 = "f00007e4Q-Q",
1058
1059 -- Primary opcode 61:
1060 stfdp_2 = "f4000000F:D", -- NYI: displacement must be divisible by 4.
1061
1062 -- Primary opcode 62:
1063 stq_2 = "f8000002R:D", -- NYI: displacement must be divisible by 8.
504 1064
505 -- Primary opcode 63: 1065 -- Primary opcode 63:
506 fdiv_3 = "fc000024FFF.", 1066 fdiv_3 = "fc000024FFF.",
@@ -526,8 +1086,12 @@ local map_op = {
526 frsp_2 = "fc000018F-F.", 1086 frsp_2 = "fc000018F-F.",
527 fctiw_2 = "fc00001cF-F.", 1087 fctiw_2 = "fc00001cF-F.",
528 fctiwz_2 = "fc00001eF-F.", 1088 fctiwz_2 = "fc00001eF-F.",
1089 ftdiv_2 = "fc000100X-F.",
1090 fctiwu_2 = "fc00011cF-F.",
1091 fctiwuz_2 = "fc00011eF-F.",
529 mtfsfi_2 = "fc00010cAA", -- NYI: upshift. 1092 mtfsfi_2 = "fc00010cAA", -- NYI: upshift.
530 fnabs_2 = "fc000110F-F.", 1093 fnabs_2 = "fc000110F-F.",
1094 ftsqrt_2 = "fc000140X-F.",
531 fabs_2 = "fc000210F-F.", 1095 fabs_2 = "fc000210F-F.",
532 frin_2 = "fc000310F-F.", 1096 frin_2 = "fc000310F-F.",
533 friz_2 = "fc000350F-F.", 1097 friz_2 = "fc000350F-F.",
@@ -537,7 +1101,38 @@ local map_op = {
537 -- NYI: mtfsf, mtfsb0, mtfsb1. 1101 -- NYI: mtfsf, mtfsb0, mtfsb1.
538 fctid_2 = "fc00065cF-F.", 1102 fctid_2 = "fc00065cF-F.",
539 fctidz_2 = "fc00065eF-F.", 1103 fctidz_2 = "fc00065eF-F.",
1104 fmrgow_3 = "fc00068cFFF",
540 fcfid_2 = "fc00069cF-F.", 1105 fcfid_2 = "fc00069cF-F.",
1106 fctidu_2 = "fc00075cF-F.",
1107 fctiduz_2 = "fc00075eF-F.",
1108 fmrgew_3 = "fc00078cFFF",
1109 fcfidu_2 = "fc00079cF-F.",
1110
1111 daddq_3 = "fc000004F:F:F:.",
1112 dquaq_4 = "fc000006F:F:F:Z.",
1113 dmulq_3 = "fc000044F:F:F:.",
1114 drrndq_4 = "fc000046F:F:F:Z.",
1115 dscliq_3 = "fc000084F:F:6.",
1116 dquaiq_4 = "fc000086SF:~F:Z.",
1117 dscriq_3 = "fc0000c4F:F:6.",
1118 drintxq_4 = "fc0000c61F:~F:Z.",
1119 dcmpoq_3 = "fc000104XF:F:",
1120 dtstexq_3 = "fc000144XF:F:",
1121 dtstdcq_3 = "fc000184XF:6",
1122 dtstdgq_3 = "fc0001c4XF:6",
1123 drintnq_4 = "fc0001c61F:~F:Z.",
1124 dctqpq_2 = "fc000204F:-F:.",
1125 dctfixq_2 = "fc000244F:-F:.",
1126 ddedpdq_3 = "fc000284ZF:~F:.",
1127 dxexq_2 = "fc0002c4F:-F:.",
1128 dsubq_3 = "fc000404F:F:F:.",
1129 ddivq_3 = "fc000444F:F:F:.",
1130 dcmpuq_3 = "fc000504XF:F:",
1131 dtstsfq_3 = "fc000544XF:F:",
1132 drdpq_2 = "fc000604F:-F:.",
1133 dcffixq_2 = "fc000644F:-F:.",
1134 denbcdq_3 = "fc000684YF:~F:.",
1135 diexq_3 = "fc0006c4F:FF:.",
541 1136
542 -- Primary opcode 4, SPE APU extension: 1137 -- Primary opcode 4, SPE APU extension:
543 evaddw_3 = "10000200RRR", 1138 evaddw_3 = "10000200RRR",
@@ -822,7 +1417,7 @@ local map_op = {
822do 1417do
823 local t = {} 1418 local t = {}
824 for k,v in pairs(map_op) do 1419 for k,v in pairs(map_op) do
825 if sub(v, -1) == "." then 1420 if type(v) == "string" and sub(v, -1) == "." then
826 local v2 = sub(v, 1, 7)..char(byte(v, 8)+1)..sub(v, 9, -2) 1421 local v2 = sub(v, 1, 7)..char(byte(v, 8)+1)..sub(v, 9, -2)
827 t[sub(k, 1, -3).."."..sub(k, -2)] = v2 1422 t[sub(k, 1, -3).."."..sub(k, -2)] = v2
828 end 1423 end
@@ -884,6 +1479,24 @@ local function parse_fpr(expr)
884 werror("bad register name `"..expr.."'") 1479 werror("bad register name `"..expr.."'")
885end 1480end
886 1481
1482local function parse_vr(expr)
1483 local r = match(expr, "^v([1-3]?[0-9])$")
1484 if r then
1485 r = tonumber(r)
1486 if r <= 31 then return r end
1487 end
1488 werror("bad register name `"..expr.."'")
1489end
1490
1491local function parse_vs(expr)
1492 local r = match(expr, "^vs([1-6]?[0-9])$")
1493 if r then
1494 r = tonumber(r)
1495 if r <= 63 then return r end
1496 end
1497 werror("bad register name `"..expr.."'")
1498end
1499
887local function parse_cr(expr) 1500local function parse_cr(expr)
888 local r = match(expr, "^cr([0-7])$") 1501 local r = match(expr, "^cr([0-7])$")
889 if r then return tonumber(r) end 1502 if r then return tonumber(r) end
@@ -900,8 +1513,30 @@ local function parse_cond(expr)
900 werror("bad condition bit name `"..expr.."'") 1513 werror("bad condition bit name `"..expr.."'")
901end 1514end
902 1515
1516local parse_ctx = {}
1517
1518local loadenv = setfenv and function(s)
1519 local code = loadstring(s, "")
1520 if code then setfenv(code, parse_ctx) end
1521 return code
1522end or function(s)
1523 return load(s, "", nil, parse_ctx)
1524end
1525
1526-- Try to parse simple arithmetic, too, since some basic ops are aliases.
1527local function parse_number(n)
1528 local x = tonumber(n)
1529 if x then return x end
1530 local code = loadenv("return "..n)
1531 if code then
1532 local ok, y = pcall(code)
1533 if ok then return y end
1534 end
1535 return nil
1536end
1537
903local function parse_imm(imm, bits, shift, scale, signed) 1538local function parse_imm(imm, bits, shift, scale, signed)
904 local n = tonumber(imm) 1539 local n = parse_number(imm)
905 if n then 1540 if n then
906 local m = sar(n, scale) 1541 local m = sar(n, scale)
907 if shl(m, scale) == n then 1542 if shl(m, scale) == n then
@@ -914,7 +1549,8 @@ local function parse_imm(imm, bits, shift, scale, signed)
914 end 1549 end
915 end 1550 end
916 werror("out of range immediate `"..imm.."'") 1551 werror("out of range immediate `"..imm.."'")
917 elseif match(imm, "^r([1-3]?[0-9])$") or 1552 elseif match(imm, "^[rfv]([1-3]?[0-9])$") or
1553 match(imm, "^vs([1-6]?[0-9])$") or
918 match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then 1554 match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then
919 werror("expected immediate operand, got register") 1555 werror("expected immediate operand, got register")
920 else 1556 else
@@ -924,11 +1560,11 @@ local function parse_imm(imm, bits, shift, scale, signed)
924end 1560end
925 1561
926local function parse_shiftmask(imm, isshift) 1562local function parse_shiftmask(imm, isshift)
927 local n = tonumber(imm) 1563 local n = parse_number(imm)
928 if n then 1564 if n then
929 if shr(n, 6) == 0 then 1565 if shr(n, 6) == 0 then
930 local lsb = band(imm, 31) 1566 local lsb = band(n, 31)
931 local msb = imm - lsb 1567 local msb = n - lsb
932 return isshift and (shl(lsb, 11)+shr(msb, 4)) or (shl(lsb, 6)+msb) 1568 return isshift and (shl(lsb, 11)+shr(msb, 4)) or (shl(lsb, 6)+msb)
933 end 1569 end
934 werror("out of range immediate `"..imm.."'") 1570 werror("out of range immediate `"..imm.."'")
@@ -936,7 +1572,8 @@ local function parse_shiftmask(imm, isshift)
936 match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then 1572 match(imm, "^([%w_]+):(r[1-3]?[0-9])$") then
937 werror("expected immediate operand, got register") 1573 werror("expected immediate operand, got register")
938 else 1574 else
939 werror("NYI: parameterized 64 bit shift/mask") 1575 waction("IMMSH", isshift and 1 or 0, imm)
1576 return 0;
940 end 1577 end
941end 1578end
942 1579
@@ -1011,7 +1648,7 @@ end
1011------------------------------------------------------------------------------ 1648------------------------------------------------------------------------------
1012 1649
1013-- Handle opcodes defined with template strings. 1650-- Handle opcodes defined with template strings.
1014map_op[".template__"] = function(params, template, nparams) 1651op_template = function(params, template, nparams)
1015 if not params then return sub(template, 9) end 1652 if not params then return sub(template, 9) end
1016 local op = tonumber(sub(template, 1, 8), 16) 1653 local op = tonumber(sub(template, 1, 8), 16)
1017 local n, rs = 1, 26 1654 local n, rs = 1, 26
@@ -1027,6 +1664,15 @@ map_op[".template__"] = function(params, template, nparams)
1027 rs = rs - 5; op = op + shl(parse_gpr(params[n]), rs); n = n + 1 1664 rs = rs - 5; op = op + shl(parse_gpr(params[n]), rs); n = n + 1
1028 elseif p == "F" then 1665 elseif p == "F" then
1029 rs = rs - 5; op = op + shl(parse_fpr(params[n]), rs); n = n + 1 1666 rs = rs - 5; op = op + shl(parse_fpr(params[n]), rs); n = n + 1
1667 elseif p == "V" then
1668 rs = rs - 5; op = op + shl(parse_vr(params[n]), rs); n = n + 1
1669 elseif p == "Q" then
1670 local vs = parse_vs(params[n]); n = n + 1; rs = rs - 5
1671 local sh = rs == 6 and 2 or 3 + band(shr(rs, 1), 3)
1672 op = op + shl(band(vs, 31), rs) + shr(band(vs, 32), sh)
1673 elseif p == "q" then
1674 local vs = parse_vs(params[n]); n = n + 1
1675 op = op + shl(band(vs, 31), 21) + shr(band(vs, 32), 5)
1030 elseif p == "A" then 1676 elseif p == "A" then
1031 rs = rs - 5; op = op + parse_imm(params[n], 5, rs, 0, false); n = n + 1 1677 rs = rs - 5; op = op + parse_imm(params[n], 5, rs, 0, false); n = n + 1
1032 elseif p == "S" then 1678 elseif p == "S" then
@@ -1047,6 +1693,26 @@ map_op[".template__"] = function(params, template, nparams)
1047 rs = rs - 5; op = op + shl(parse_cond(params[n]), rs); n = n + 1 1693 rs = rs - 5; op = op + shl(parse_cond(params[n]), rs); n = n + 1
1048 elseif p == "X" then 1694 elseif p == "X" then
1049 rs = rs - 5; op = op + shl(parse_cr(params[n]), rs+2); n = n + 1 1695 rs = rs - 5; op = op + shl(parse_cr(params[n]), rs+2); n = n + 1
1696 elseif p == "1" then
1697 rs = rs - 5; op = op + parse_imm(params[n], 1, rs, 0, false); n = n + 1
1698 elseif p == "g" then
1699 rs = rs - 5; op = op + parse_imm(params[n], 2, rs, 0, false); n = n + 1
1700 elseif p == "3" then
1701 rs = rs - 5; op = op + parse_imm(params[n], 3, rs, 0, false); n = n + 1
1702 elseif p == "P" then
1703 rs = rs - 5; op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1
1704 elseif p == "p" then
1705 op = op + parse_imm(params[n], 4, rs, 0, false); n = n + 1
1706 elseif p == "6" then
1707 rs = rs - 6; op = op + parse_imm(params[n], 6, rs, 0, false); n = n + 1
1708 elseif p == "Y" then
1709 rs = rs - 5; op = op + parse_imm(params[n], 1, rs+4, 0, false); n = n + 1
1710 elseif p == "y" then
1711 rs = rs - 5; op = op + parse_imm(params[n], 1, rs+3, 0, false); n = n + 1
1712 elseif p == "Z" then
1713 rs = rs - 5; op = op + parse_imm(params[n], 2, rs+3, 0, false); n = n + 1
1714 elseif p == "z" then
1715 rs = rs - 5; op = op + parse_imm(params[n], 2, rs+2, 0, false); n = n + 1
1050 elseif p == "W" then 1716 elseif p == "W" then
1051 op = op + parse_cr(params[n]); n = n + 1 1717 op = op + parse_cr(params[n]); n = n + 1
1052 elseif p == "G" then 1718 elseif p == "G" then
@@ -1071,6 +1737,8 @@ map_op[".template__"] = function(params, template, nparams)
1071 local lo = band(op, mm) 1737 local lo = band(op, mm)
1072 local hi = band(op, shl(mm, 5)) 1738 local hi = band(op, shl(mm, 5))
1073 op = op - lo - hi + shl(lo, 5) + shr(hi, 5) 1739 op = op - lo - hi + shl(lo, 5) + shr(hi, 5)
1740 elseif p == ":" then
1741 if band(shr(op, rs), 1) ~= 0 then werror("register pair expected") end
1074 elseif p == "-" then 1742 elseif p == "-" then
1075 rs = rs - 5 1743 rs = rs - 5
1076 elseif p == "." then 1744 elseif p == "." then
@@ -1082,6 +1750,8 @@ map_op[".template__"] = function(params, template, nparams)
1082 wputpos(pos, op) 1750 wputpos(pos, op)
1083end 1751end
1084 1752
1753map_op[".template__"] = op_template
1754
1085------------------------------------------------------------------------------ 1755------------------------------------------------------------------------------
1086 1756
1087-- Pseudo-opcode to mark the position where the action list is to be emitted. 1757-- Pseudo-opcode to mark the position where the action list is to be emitted.
diff --git a/dynasm/dasm_proto.h b/dynasm/dasm_proto.h
index 73558c69..3f50f502 100644
--- a/dynasm/dasm_proto.h
+++ b/dynasm/dasm_proto.h
@@ -10,8 +10,8 @@
10#include <stddef.h> 10#include <stddef.h>
11#include <stdarg.h> 11#include <stdarg.h>
12 12
13#define DASM_IDENT "DynASM 1.3.0" 13#define DASM_IDENT "DynASM 1.5.0"
14#define DASM_VERSION 10300 /* 1.3.0 */ 14#define DASM_VERSION 10500 /* 1.5.0 */
15 15
16#ifndef Dst_DECL 16#ifndef Dst_DECL
17#define Dst_DECL dasm_State **Dst 17#define Dst_DECL dasm_State **Dst
diff --git a/dynasm/dasm_x86.h b/dynasm/dasm_x86.h
index 7b031c72..aded9990 100644
--- a/dynasm/dasm_x86.h
+++ b/dynasm/dasm_x86.h
@@ -68,7 +68,7 @@ struct dasm_State {
68 size_t lgsize; 68 size_t lgsize;
69 int *pclabels; /* PC label chains/pos ptrs. */ 69 int *pclabels; /* PC label chains/pos ptrs. */
70 size_t pcsize; 70 size_t pcsize;
71 void **globals; /* Array of globals (bias -10). */ 71 void **globals; /* Array of globals. */
72 dasm_Section *section; /* Pointer to active section. */ 72 dasm_Section *section; /* Pointer to active section. */
73 size_t codesize; /* Total size of all code sections. */ 73 size_t codesize; /* Total size of all code sections. */
74 int maxsection; /* 0 <= sectionidx < maxsection. */ 74 int maxsection; /* 0 <= sectionidx < maxsection. */
@@ -85,7 +85,6 @@ void dasm_init(Dst_DECL, int maxsection)
85{ 85{
86 dasm_State *D; 86 dasm_State *D;
87 size_t psz = 0; 87 size_t psz = 0;
88 int i;
89 Dst_REF = NULL; 88 Dst_REF = NULL;
90 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); 89 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
91 D = Dst_REF; 90 D = Dst_REF;
@@ -96,12 +95,7 @@ void dasm_init(Dst_DECL, int maxsection)
96 D->pcsize = 0; 95 D->pcsize = 0;
97 D->globals = NULL; 96 D->globals = NULL;
98 D->maxsection = maxsection; 97 D->maxsection = maxsection;
99 for (i = 0; i < maxsection; i++) { 98 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
100 D->sections[i].buf = NULL; /* Need this for pass3. */
101 D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i);
102 D->sections[i].bsize = 0;
103 D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */
104 }
105} 99}
106 100
107/* Free DynASM state. */ 101/* Free DynASM state. */
@@ -121,7 +115,7 @@ void dasm_free(Dst_DECL)
121void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) 115void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
122{ 116{
123 dasm_State *D = Dst_REF; 117 dasm_State *D = Dst_REF;
124 D->globals = gl - 10; /* Negative bias to compensate for locals. */ 118 D->globals = gl;
125 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); 119 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
126} 120}
127 121
@@ -146,6 +140,7 @@ void dasm_setup(Dst_DECL, const void *actionlist)
146 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); 140 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
147 for (i = 0; i < D->maxsection; i++) { 141 for (i = 0; i < D->maxsection; i++) {
148 D->sections[i].pos = DASM_SEC2POS(i); 142 D->sections[i].pos = DASM_SEC2POS(i);
143 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
149 D->sections[i].ofs = 0; 144 D->sections[i].ofs = 0;
150 } 145 }
151} 146}
@@ -170,7 +165,7 @@ void dasm_put(Dst_DECL, int start, ...)
170 dasm_State *D = Dst_REF; 165 dasm_State *D = Dst_REF;
171 dasm_ActList p = D->actionlist + start; 166 dasm_ActList p = D->actionlist + start;
172 dasm_Section *sec = D->section; 167 dasm_Section *sec = D->section;
173 int pos = sec->pos, ofs = sec->ofs, mrm = 4; 168 int pos = sec->pos, ofs = sec->ofs, mrm = -1;
174 int *b; 169 int *b;
175 170
176 if (pos >= sec->epos) { 171 if (pos >= sec->epos) {
@@ -193,7 +188,7 @@ void dasm_put(Dst_DECL, int start, ...)
193 b[pos++] = n; 188 b[pos++] = n;
194 switch (action) { 189 switch (action) {
195 case DASM_DISP: 190 case DASM_DISP:
196 if (n == 0) { if ((mrm&7) == 4) mrm = p[-2]; if ((mrm&7) != 5) break; } 191 if (n == 0) { if (mrm < 0) mrm = p[-2]; if ((mrm&7) != 5) break; }
197 /* fallthrough */ 192 /* fallthrough */
198 case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */ 193 case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */
199 case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */ 194 case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */
@@ -204,11 +199,17 @@ void dasm_put(Dst_DECL, int start, ...)
204 case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break; 199 case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break;
205 case DASM_SPACE: p++; ofs += n; break; 200 case DASM_SPACE: p++; ofs += n; break;
206 case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */ 201 case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */
207 case DASM_VREG: CK((n&-8) == 0 && (n != 4 || (*p&1) == 0), RANGE_VREG); 202 case DASM_VREG: CK((n&-16) == 0 && (n != 4 || (*p>>5) != 2), RANGE_VREG);
208 if (*p++ == 1 && *p == DASM_DISP) mrm = n; 203 if (*p < 0x40 && p[1] == DASM_DISP) mrm = n;
204 if (*p < 0x20 && (n&7) == 4) ofs++;
205 switch ((*p++ >> 3) & 3) {
206 case 3: n |= b[pos-3]; /* fallthrough */
207 case 2: n |= b[pos-2]; /* fallthrough */
208 case 1: if (n <= 7) { b[pos-1] |= 0x10; ofs--; }
209 }
209 continue; 210 continue;
210 } 211 }
211 mrm = 4; 212 mrm = -1;
212 } else { 213 } else {
213 int *pl, n; 214 int *pl, n;
214 switch (action) { 215 switch (action) {
@@ -233,8 +234,11 @@ void dasm_put(Dst_DECL, int start, ...)
233 } 234 }
234 pos++; 235 pos++;
235 ofs += 4; /* Maximum offset needed. */ 236 ofs += 4; /* Maximum offset needed. */
236 if (action == DASM_REL_LG || action == DASM_REL_PC) 237 if (action == DASM_REL_LG || action == DASM_REL_PC) {
237 b[pos++] = ofs; /* Store pass1 offset estimate. */ 238 b[pos++] = ofs; /* Store pass1 offset estimate. */
239 } else if (sizeof(ptrdiff_t) == 8) {
240 ofs += 4;
241 }
238 break; 242 break;
239 case DASM_LABEL_LG: pl = D->lglabels + *p++; CKPL(lg, LG); goto putlabel; 243 case DASM_LABEL_LG: pl = D->lglabels + *p++; CKPL(lg, LG); goto putlabel;
240 case DASM_LABEL_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC); 244 case DASM_LABEL_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
@@ -359,10 +363,22 @@ int dasm_link(Dst_DECL, size_t *szp)
359 do { *((unsigned short *)cp) = (unsigned short)(x); cp+=2; } while (0) 363 do { *((unsigned short *)cp) = (unsigned short)(x); cp+=2; } while (0)
360#define dasmd(x) \ 364#define dasmd(x) \
361 do { *((unsigned int *)cp) = (unsigned int)(x); cp+=4; } while (0) 365 do { *((unsigned int *)cp) = (unsigned int)(x); cp+=4; } while (0)
366#define dasmq(x) \
367 do { *((unsigned long long *)cp) = (unsigned long long)(x); cp+=8; } while (0)
362#else 368#else
363#define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0) 369#define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0)
364#define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0) 370#define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0)
371#define dasmq(x) do { dasmd(x); dasmd((x)>>32); } while (0)
365#endif 372#endif
373static unsigned char *dasma_(unsigned char *cp, ptrdiff_t x)
374{
375 if (sizeof(ptrdiff_t) == 8)
376 dasmq((unsigned long long)x);
377 else
378 dasmd((unsigned int)x);
379 return cp;
380}
381#define dasma(x) (cp = dasma_(cp, (x)))
366 382
367/* Pass 3: Encode sections. */ 383/* Pass 3: Encode sections. */
368int dasm_encode(Dst_DECL, void *buffer) 384int dasm_encode(Dst_DECL, void *buffer)
@@ -402,9 +418,29 @@ int dasm_encode(Dst_DECL, void *buffer)
402 case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL; 418 case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL;
403 /* fallthrough */ 419 /* fallthrough */
404 case DASM_IMM_W: dasmw(n); break; 420 case DASM_IMM_W: dasmw(n); break;
405 case DASM_VREG: { int t = *p++; if (t >= 2) n<<=3; cp[-1] |= n; break; } 421 case DASM_VREG: {
422 int t = *p++;
423 unsigned char *ex = cp - (t&7);
424 if ((n & 8) && t < 0xa0) {
425 if (*ex & 0x80) ex[1] ^= 0x20 << (t>>6); else *ex ^= 1 << (t>>6);
426 n &= 7;
427 } else if (n & 0x10) {
428 if (*ex & 0x80) {
429 *ex = 0xc5; ex[1] = (ex[1] & 0x80) | ex[2]; ex += 2;
430 }
431 while (++ex < cp) ex[-1] = *ex;
432 if (mark) mark--;
433 cp--;
434 n &= 7;
435 }
436 if (t >= 0xc0) n <<= 4;
437 else if (t >= 0x40) n <<= 3;
438 else if (n == 4 && t < 0x20) { cp[-1] ^= n; *cp++ = 0x20; }
439 cp[-1] ^= n;
440 break;
441 }
406 case DASM_REL_LG: p++; if (n >= 0) goto rel_pc; 442 case DASM_REL_LG: p++; if (n >= 0) goto rel_pc;
407 b++; n = (int)(ptrdiff_t)D->globals[-n]; 443 b++; n = (int)(ptrdiff_t)D->globals[-n-10];
408 /* fallthrough */ 444 /* fallthrough */
409 case DASM_REL_A: rel_a: 445 case DASM_REL_A: rel_a:
410 n -= (unsigned int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */ 446 n -= (unsigned int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */
@@ -417,17 +453,18 @@ int dasm_encode(Dst_DECL, void *buffer)
417 goto wb; 453 goto wb;
418 } 454 }
419 case DASM_IMM_LG: 455 case DASM_IMM_LG:
420 p++; if (n < 0) { n = (int)(ptrdiff_t)D->globals[-n]; goto wd; } 456 p++;
457 if (n < 0) { dasma((ptrdiff_t)D->globals[-n-10]); break; }
421 /* fallthrough */ 458 /* fallthrough */
422 case DASM_IMM_PC: { 459 case DASM_IMM_PC: {
423 int *pb = DASM_POS2PTR(D, n); 460 int *pb = DASM_POS2PTR(D, n);
424 n = *pb < 0 ? pb[1] : (*pb + (int)(ptrdiff_t)base); 461 dasma(*pb < 0 ? (ptrdiff_t)pb[1] : (*pb + (ptrdiff_t)base));
425 goto wd; 462 break;
426 } 463 }
427 case DASM_LABEL_LG: { 464 case DASM_LABEL_LG: {
428 int idx = *p++; 465 int idx = *p++;
429 if (idx >= 10) 466 if (idx >= 10)
430 D->globals[idx] = (void *)(base + (*p == DASM_SETLABEL ? *b : n)); 467 D->globals[idx-10] = (void *)(base + (*p == DASM_SETLABEL ? *b : n));
431 break; 468 break;
432 } 469 }
433 case DASM_LABEL_PC: case DASM_SETLABEL: break; 470 case DASM_LABEL_PC: case DASM_SETLABEL: break;
diff --git a/dynasm/dasm_x86.lua b/dynasm/dasm_x86.lua
index 8a4c93a2..787163c0 100644
--- a/dynasm/dasm_x86.lua
+++ b/dynasm/dasm_x86.lua
@@ -11,9 +11,9 @@ local x64 = x64
11local _info = { 11local _info = {
12 arch = x64 and "x64" or "x86", 12 arch = x64 and "x64" or "x86",
13 description = "DynASM x86/x64 module", 13 description = "DynASM x86/x64 module",
14 version = "1.3.0", 14 version = "1.5.0",
15 vernum = 10300, 15 vernum = 10500,
16 release = "2011-05-05", 16 release = "2021-05-02",
17 author = "Mike Pall", 17 author = "Mike Pall",
18 license = "MIT", 18 license = "MIT",
19} 19}
@@ -27,9 +27,9 @@ local assert, unpack, setmetatable = assert, unpack or table.unpack, setmetatabl
27local _s = string 27local _s = string
28local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char 28local sub, format, byte, char = _s.sub, _s.format, _s.byte, _s.char
29local find, match, gmatch, gsub = _s.find, _s.match, _s.gmatch, _s.gsub 29local find, match, gmatch, gsub = _s.find, _s.match, _s.gmatch, _s.gsub
30local concat, sort = table.concat, table.sort 30local concat, sort, remove = table.concat, table.sort, table.remove
31local bit = bit or require("bit") 31local bit = bit or require("bit")
32local band, shl, shr = bit.band, bit.lshift, bit.rshift 32local band, bxor, shl, shr = bit.band, bit.bxor, bit.lshift, bit.rshift
33 33
34-- Inherited tables and callbacks. 34-- Inherited tables and callbacks.
35local g_opt, g_arch 35local g_opt, g_arch
@@ -41,7 +41,7 @@ local action_names = {
41 -- int arg, 1 buffer pos: 41 -- int arg, 1 buffer pos:
42 "DISP", "IMM_S", "IMM_B", "IMM_W", "IMM_D", "IMM_WB", "IMM_DB", 42 "DISP", "IMM_S", "IMM_B", "IMM_W", "IMM_D", "IMM_WB", "IMM_DB",
43 -- action arg (1 byte), int arg, 1 buffer pos (reg/num): 43 -- action arg (1 byte), int arg, 1 buffer pos (reg/num):
44 "VREG", "SPACE", -- !x64: VREG support NYI. 44 "VREG", "SPACE",
45 -- ptrdiff_t arg, 1 buffer pos (address): !x64 45 -- ptrdiff_t arg, 1 buffer pos (address): !x64
46 "SETLABEL", "REL_A", 46 "SETLABEL", "REL_A",
47 -- action arg (1 byte) or int arg, 2 buffer pos (link, offset): 47 -- action arg (1 byte) or int arg, 2 buffer pos (link, offset):
@@ -83,6 +83,21 @@ local actargs = { 0 }
83-- Current number of section buffer positions for dasm_put(). 83-- Current number of section buffer positions for dasm_put().
84local secpos = 1 84local secpos = 1
85 85
86-- VREG kind encodings, pre-shifted by 5 bits.
87local map_vreg = {
88 ["modrm.rm.m"] = 0x00,
89 ["modrm.rm.r"] = 0x20,
90 ["opcode"] = 0x20,
91 ["sib.base"] = 0x20,
92 ["sib.index"] = 0x40,
93 ["modrm.reg"] = 0x80,
94 ["vex.v"] = 0xa0,
95 ["imm.hi"] = 0xc0,
96}
97
98-- Current number of VREG actions contributing to REX/VEX shrinkage.
99local vreg_shrink_count = 0
100
86------------------------------------------------------------------------------ 101------------------------------------------------------------------------------
87 102
88-- Compute action numbers for action names. 103-- Compute action numbers for action names.
@@ -134,6 +149,21 @@ local function waction(action, a, num)
134 if a or num then secpos = secpos + (num or 1) end 149 if a or num then secpos = secpos + (num or 1) end
135end 150end
136 151
152-- Optionally add a VREG action.
153local function wvreg(kind, vreg, psz, sk, defer)
154 if not vreg then return end
155 waction("VREG", vreg)
156 local b = assert(map_vreg[kind], "bad vreg kind `"..vreg.."'")
157 if b < (sk or 0) then
158 vreg_shrink_count = vreg_shrink_count + 1
159 end
160 if not defer then
161 b = b + vreg_shrink_count * 8
162 vreg_shrink_count = 0
163 end
164 wputxb(b + (psz or 0))
165end
166
137-- Add call to embedded DynASM C code. 167-- Add call to embedded DynASM C code.
138local function wcall(func, args) 168local function wcall(func, args)
139 wline(format("dasm_%s(Dst, %s);", func, concat(args, ", ")), true) 169 wline(format("dasm_%s(Dst, %s);", func, concat(args, ", ")), true)
@@ -299,7 +329,7 @@ local function mkrmap(sz, cl, names)
299 local iname = format("@%s%x%s", sz, i, needrex and "R" or "") 329 local iname = format("@%s%x%s", sz, i, needrex and "R" or "")
300 if needrex then map_reg_needrex[iname] = true end 330 if needrex then map_reg_needrex[iname] = true end
301 local name 331 local name
302 if sz == "o" then name = format("xmm%d", i) 332 if sz == "o" or sz == "y" then name = format("%s%d", cl, i)
303 elseif sz == "f" then name = format("st%d", i) 333 elseif sz == "f" then name = format("st%d", i)
304 else name = format("r%d%s", i, sz == addrsize and "" or sz) end 334 else name = format("r%d%s", i, sz == addrsize and "" or sz) end
305 map_archdef[name] = iname 335 map_archdef[name] = iname
@@ -326,6 +356,7 @@ mkrmap("w", "Rw", {"ax", "cx", "dx", "bx", "sp", "bp", "si", "di"})
326mkrmap("b", "Rb", {"al", "cl", "dl", "bl", "ah", "ch", "dh", "bh"}) 356mkrmap("b", "Rb", {"al", "cl", "dl", "bl", "ah", "ch", "dh", "bh"})
327map_reg_valid_index[map_archdef.esp] = false 357map_reg_valid_index[map_archdef.esp] = false
328if x64 then map_reg_valid_index[map_archdef.rsp] = false end 358if x64 then map_reg_valid_index[map_archdef.rsp] = false end
359if x64 then map_reg_needrex[map_archdef.Rb] = true end
329map_archdef["Ra"] = "@"..addrsize 360map_archdef["Ra"] = "@"..addrsize
330 361
331-- FP registers (internally tword sized, but use "f" as operand size). 362-- FP registers (internally tword sized, but use "f" as operand size).
@@ -334,21 +365,24 @@ mkrmap("f", "Rf")
334-- SSE registers (oword sized, but qword and dword accessible). 365-- SSE registers (oword sized, but qword and dword accessible).
335mkrmap("o", "xmm") 366mkrmap("o", "xmm")
336 367
368-- AVX registers (yword sized, but oword, qword and dword accessible).
369mkrmap("y", "ymm")
370
337-- Operand size prefixes to codes. 371-- Operand size prefixes to codes.
338local map_opsize = { 372local map_opsize = {
339 byte = "b", word = "w", dword = "d", qword = "q", oword = "o", tword = "t", 373 byte = "b", word = "w", dword = "d", qword = "q", oword = "o", yword = "y",
340 aword = addrsize, 374 tword = "t", aword = addrsize,
341} 375}
342 376
343-- Operand size code to number. 377-- Operand size code to number.
344local map_opsizenum = { 378local map_opsizenum = {
345 b = 1, w = 2, d = 4, q = 8, o = 16, t = 10, 379 b = 1, w = 2, d = 4, q = 8, o = 16, y = 32, t = 10,
346} 380}
347 381
348-- Operand size code to name. 382-- Operand size code to name.
349local map_opsizename = { 383local map_opsizename = {
350 b = "byte", w = "word", d = "dword", q = "qword", o = "oword", t = "tword", 384 b = "byte", w = "word", d = "dword", q = "qword", o = "oword", y = "yword",
351 f = "fpword", 385 t = "tword", f = "fpword",
352} 386}
353 387
354-- Valid index register scale factors. 388-- Valid index register scale factors.
@@ -450,6 +484,22 @@ local function wputdarg(n)
450 end 484 end
451end 485end
452 486
487-- Put signed or unsigned qword or arg.
488local function wputqarg(n)
489 local tn = type(n)
490 if tn == "number" then -- This is only used for numbers from -2^31..2^32-1.
491 wputb(band(n, 255))
492 wputb(band(shr(n, 8), 255))
493 wputb(band(shr(n, 16), 255))
494 wputb(shr(n, 24))
495 local sign = n < 0 and 255 or 0
496 wputb(sign); wputb(sign); wputb(sign); wputb(sign)
497 else
498 waction("IMM_D", format("(unsigned int)(%s)", n))
499 waction("IMM_D", format("(unsigned int)((unsigned long long)(%s)>>32)", n))
500 end
501end
502
453-- Put operand-size dependent number or arg (defaults to dword). 503-- Put operand-size dependent number or arg (defaults to dword).
454local function wputszarg(sz, n) 504local function wputszarg(sz, n)
455 if not sz or sz == "d" or sz == "q" then wputdarg(n) 505 if not sz or sz == "d" or sz == "q" then wputdarg(n)
@@ -460,9 +510,45 @@ local function wputszarg(sz, n)
460end 510end
461 511
462-- Put multi-byte opcode with operand-size dependent modifications. 512-- Put multi-byte opcode with operand-size dependent modifications.
463local function wputop(sz, op, rex) 513local function wputop(sz, op, rex, vex, vregr, vregxb)
514 local psz, sk = 0, nil
515 if vex then
516 local tail
517 if vex.m == 1 and band(rex, 11) == 0 then
518 if x64 and vregxb then
519 sk = map_vreg["modrm.reg"]
520 else
521 wputb(0xc5)
522 tail = shl(bxor(band(rex, 4), 4), 5)
523 psz = 3
524 end
525 end
526 if not tail then
527 wputb(0xc4)
528 wputb(shl(bxor(band(rex, 7), 7), 5) + vex.m)
529 tail = shl(band(rex, 8), 4)
530 psz = 4
531 end
532 local reg, vreg = 0, nil
533 if vex.v then
534 reg = vex.v.reg
535 if not reg then werror("bad vex operand") end
536 if reg < 0 then reg = 0; vreg = vex.v.vreg end
537 end
538 if sz == "y" or vex.l then tail = tail + 4 end
539 wputb(tail + shl(bxor(reg, 15), 3) + vex.p)
540 wvreg("vex.v", vreg)
541 rex = 0
542 if op >= 256 then werror("bad vex opcode") end
543 else
544 if rex ~= 0 then
545 if not x64 then werror("bad operand size") end
546 elseif (vregr or vregxb) and x64 then
547 rex = 0x10
548 sk = map_vreg["vex.v"]
549 end
550 end
464 local r 551 local r
465 if rex ~= 0 and not x64 then werror("bad operand size") end
466 if sz == "w" then wputb(102) end 552 if sz == "w" then wputb(102) end
467 -- Needs >32 bit numbers, but only for crc32 eax, word [ebx] 553 -- Needs >32 bit numbers, but only for crc32 eax, word [ebx]
468 if op >= 4294967296 then r = op%4294967296 wputb((op-r)/4294967296) op = r end 554 if op >= 4294967296 then r = op%4294967296 wputb((op-r)/4294967296) op = r end
@@ -471,20 +557,20 @@ local function wputop(sz, op, rex)
471 if rex ~= 0 then 557 if rex ~= 0 then
472 local opc3 = band(op, 0xffff00) 558 local opc3 = band(op, 0xffff00)
473 if opc3 == 0x0f3a00 or opc3 == 0x0f3800 then 559 if opc3 == 0x0f3a00 or opc3 == 0x0f3800 then
474 wputb(64 + band(rex, 15)); rex = 0 560 wputb(64 + band(rex, 15)); rex = 0; psz = 2
475 end 561 end
476 end 562 end
477 wputb(shr(op, 16)); op = band(op, 0xffff) 563 wputb(shr(op, 16)); op = band(op, 0xffff); psz = psz + 1
478 end 564 end
479 if op >= 256 then 565 if op >= 256 then
480 local b = shr(op, 8) 566 local b = shr(op, 8)
481 if b == 15 and rex ~= 0 then wputb(64 + band(rex, 15)); rex = 0 end 567 if b == 15 and rex ~= 0 then wputb(64 + band(rex, 15)); rex = 0; psz = 2 end
482 wputb(b) 568 wputb(b); op = band(op, 255); psz = psz + 1
483 op = band(op, 255)
484 end 569 end
485 if rex ~= 0 then wputb(64 + band(rex, 15)) end 570 if rex ~= 0 then wputb(64 + band(rex, 15)); psz = 2 end
486 if sz == "b" then op = op - 1 end 571 if sz == "b" then op = op - 1 end
487 wputb(op) 572 wputb(op)
573 return psz, sk
488end 574end
489 575
490-- Put ModRM or SIB formatted byte. 576-- Put ModRM or SIB formatted byte.
@@ -494,7 +580,7 @@ local function wputmodrm(m, s, rm, vs, vrm)
494end 580end
495 581
496-- Put ModRM/SIB plus optional displacement. 582-- Put ModRM/SIB plus optional displacement.
497local function wputmrmsib(t, imark, s, vsreg) 583local function wputmrmsib(t, imark, s, vsreg, psz, sk)
498 local vreg, vxreg 584 local vreg, vxreg
499 local reg, xreg = t.reg, t.xreg 585 local reg, xreg = t.reg, t.xreg
500 if reg and reg < 0 then reg = 0; vreg = t.vreg end 586 if reg and reg < 0 then reg = 0; vreg = t.vreg end
@@ -504,8 +590,8 @@ local function wputmrmsib(t, imark, s, vsreg)
504 -- Register mode. 590 -- Register mode.
505 if sub(t.mode, 1, 1) == "r" then 591 if sub(t.mode, 1, 1) == "r" then
506 wputmodrm(3, s, reg) 592 wputmodrm(3, s, reg)
507 if vsreg then waction("VREG", vsreg); wputxb(2) end 593 wvreg("modrm.reg", vsreg, psz+1, sk, vreg)
508 if vreg then waction("VREG", vreg); wputxb(0) end 594 wvreg("modrm.rm.r", vreg, psz+1, sk)
509 return 595 return
510 end 596 end
511 597
@@ -519,21 +605,22 @@ local function wputmrmsib(t, imark, s, vsreg)
519 -- [xreg*xsc+disp] -> (0, s, esp) (xsc, xreg, ebp) 605 -- [xreg*xsc+disp] -> (0, s, esp) (xsc, xreg, ebp)
520 wputmodrm(0, s, 4) 606 wputmodrm(0, s, 4)
521 if imark == "I" then waction("MARK") end 607 if imark == "I" then waction("MARK") end
522 if vsreg then waction("VREG", vsreg); wputxb(2) end 608 wvreg("modrm.reg", vsreg, psz+1, sk, vxreg)
523 wputmodrm(t.xsc, xreg, 5) 609 wputmodrm(t.xsc, xreg, 5)
524 if vxreg then waction("VREG", vxreg); wputxb(3) end 610 wvreg("sib.index", vxreg, psz+2, sk)
525 else 611 else
526 -- Pure 32 bit displacement. 612 -- Pure 32 bit displacement.
527 if x64 and tdisp ~= "table" then 613 if x64 and tdisp ~= "table" then
528 wputmodrm(0, s, 4) -- [disp] -> (0, s, esp) (0, esp, ebp) 614 wputmodrm(0, s, 4) -- [disp] -> (0, s, esp) (0, esp, ebp)
615 wvreg("modrm.reg", vsreg, psz+1, sk)
529 if imark == "I" then waction("MARK") end 616 if imark == "I" then waction("MARK") end
530 wputmodrm(0, 4, 5) 617 wputmodrm(0, 4, 5)
531 else 618 else
532 riprel = x64 619 riprel = x64
533 wputmodrm(0, s, 5) -- [disp|rip-label] -> (0, s, ebp) 620 wputmodrm(0, s, 5) -- [disp|rip-label] -> (0, s, ebp)
621 wvreg("modrm.reg", vsreg, psz+1, sk)
534 if imark == "I" then waction("MARK") end 622 if imark == "I" then waction("MARK") end
535 end 623 end
536 if vsreg then waction("VREG", vsreg); wputxb(2) end
537 end 624 end
538 if riprel then -- Emit rip-relative displacement. 625 if riprel then -- Emit rip-relative displacement.
539 if match("UWSiI", imark) then 626 if match("UWSiI", imark) then
@@ -561,16 +648,16 @@ local function wputmrmsib(t, imark, s, vsreg)
561 if xreg or band(reg, 7) == 4 then 648 if xreg or band(reg, 7) == 4 then
562 wputmodrm(m or 2, s, 4) -- ModRM. 649 wputmodrm(m or 2, s, 4) -- ModRM.
563 if m == nil or imark == "I" then waction("MARK") end 650 if m == nil or imark == "I" then waction("MARK") end
564 if vsreg then waction("VREG", vsreg); wputxb(2) end 651 wvreg("modrm.reg", vsreg, psz+1, sk, vxreg or vreg)
565 wputmodrm(t.xsc or 0, xreg or 4, reg) -- SIB. 652 wputmodrm(t.xsc or 0, xreg or 4, reg) -- SIB.
566 if vxreg then waction("VREG", vxreg); wputxb(3) end 653 wvreg("sib.index", vxreg, psz+2, sk, vreg)
567 if vreg then waction("VREG", vreg); wputxb(1) end 654 wvreg("sib.base", vreg, psz+2, sk)
568 else 655 else
569 wputmodrm(m or 2, s, reg) -- ModRM. 656 wputmodrm(m or 2, s, reg) -- ModRM.
570 if (imark == "I" and (m == 1 or m == 2)) or 657 if (imark == "I" and (m == 1 or m == 2)) or
571 (m == nil and (vsreg or vreg)) then waction("MARK") end 658 (m == nil and (vsreg or vreg)) then waction("MARK") end
572 if vsreg then waction("VREG", vsreg); wputxb(2) end 659 wvreg("modrm.reg", vsreg, psz+1, sk, vreg)
573 if vreg then waction("VREG", vreg); wputxb(1) end 660 wvreg("modrm.rm.m", vreg, psz+1, sk)
574 end 661 end
575 662
576 -- Put displacement. 663 -- Put displacement.
@@ -592,10 +679,16 @@ local function opmodestr(op, args)
592end 679end
593 680
594-- Convert number to valid integer or nil. 681-- Convert number to valid integer or nil.
595local function toint(expr) 682local function toint(expr, isqword)
596 local n = tonumber(expr) 683 local n = tonumber(expr)
597 if n then 684 if n then
598 if n % 1 ~= 0 or n < -2147483648 or n > 4294967295 then 685 if n % 1 ~= 0 then
686 werror("not an integer number `"..expr.."'")
687 elseif isqword then
688 if n < -2147483648 or n > 2147483647 then
689 n = nil -- Handle it as an expression to avoid precision loss.
690 end
691 elseif n < -2147483648 or n > 4294967295 then
599 werror("bad integer number `"..expr.."'") 692 werror("bad integer number `"..expr.."'")
600 end 693 end
601 return n 694 return n
@@ -678,7 +771,7 @@ local function rtexpr(expr)
678end 771end
679 772
680-- Parse operand and return { mode, opsize, reg, xreg, xsc, disp, imm }. 773-- Parse operand and return { mode, opsize, reg, xreg, xsc, disp, imm }.
681local function parseoperand(param) 774local function parseoperand(param, isqword)
682 local t = {} 775 local t = {}
683 776
684 local expr = param 777 local expr = param
@@ -766,7 +859,7 @@ local function parseoperand(param)
766 t.disp = dispexpr(tailx) 859 t.disp = dispexpr(tailx)
767 else 860 else
768 -- imm or opsize*imm 861 -- imm or opsize*imm
769 local imm = toint(expr) 862 local imm = toint(expr, isqword)
770 if not imm and sub(expr, 1, 1) == "*" and t.opsize then 863 if not imm and sub(expr, 1, 1) == "*" and t.opsize then
771 imm = toint(sub(expr, 2)) 864 imm = toint(sub(expr, 2))
772 if imm then 865 if imm then
@@ -881,9 +974,16 @@ end
881-- "m"/"M" generates ModRM/SIB from the 1st/2nd operand. 974-- "m"/"M" generates ModRM/SIB from the 1st/2nd operand.
882-- The spare 3 bits are either filled with the last hex digit or 975-- The spare 3 bits are either filled with the last hex digit or
883-- the result from a previous "r"/"R". The opcode is restored. 976-- the result from a previous "r"/"R". The opcode is restored.
977-- "u" Use VEX encoding, vvvv unused.
978-- "v"/"V" Use VEX encoding, vvvv from 1st/2nd operand (the operand is
979-- removed from the list used by future characters).
980-- "w" Use VEX encoding, vvvv from 3rd operand.
981-- "L" Force VEX.L
884-- 982--
885-- All of the following characters force a flush of the opcode: 983-- All of the following characters force a flush of the opcode:
886-- "o"/"O" stores a pure 32 bit disp (offset) from the 1st/2nd operand. 984-- "o"/"O" stores a pure 32 bit disp (offset) from the 1st/2nd operand.
985-- "s" stores a 4 bit immediate from the last register operand,
986-- followed by 4 zero bits.
887-- "S" stores a signed 8 bit immediate from the last operand. 987-- "S" stores a signed 8 bit immediate from the last operand.
888-- "U" stores an unsigned 8 bit immediate from the last operand. 988-- "U" stores an unsigned 8 bit immediate from the last operand.
889-- "W" stores an unsigned 16 bit immediate from the last operand. 989-- "W" stores an unsigned 16 bit immediate from the last operand.
@@ -1226,46 +1326,14 @@ local map_op = {
1226 movups_2 = "rmo:0F10rM|mro:0F11Rm", 1326 movups_2 = "rmo:0F10rM|mro:0F11Rm",
1227 orpd_2 = "rmo:660F56rM", 1327 orpd_2 = "rmo:660F56rM",
1228 orps_2 = "rmo:0F56rM", 1328 orps_2 = "rmo:0F56rM",
1229 packssdw_2 = "rmo:660F6BrM",
1230 packsswb_2 = "rmo:660F63rM",
1231 packuswb_2 = "rmo:660F67rM",
1232 paddb_2 = "rmo:660FFCrM",
1233 paddd_2 = "rmo:660FFErM",
1234 paddq_2 = "rmo:660FD4rM",
1235 paddsb_2 = "rmo:660FECrM",
1236 paddsw_2 = "rmo:660FEDrM",
1237 paddusb_2 = "rmo:660FDCrM",
1238 paddusw_2 = "rmo:660FDDrM",
1239 paddw_2 = "rmo:660FFDrM",
1240 pand_2 = "rmo:660FDBrM",
1241 pandn_2 = "rmo:660FDFrM",
1242 pause_0 = "F390", 1329 pause_0 = "F390",
1243 pavgb_2 = "rmo:660FE0rM",
1244 pavgw_2 = "rmo:660FE3rM",
1245 pcmpeqb_2 = "rmo:660F74rM",
1246 pcmpeqd_2 = "rmo:660F76rM",
1247 pcmpeqw_2 = "rmo:660F75rM",
1248 pcmpgtb_2 = "rmo:660F64rM",
1249 pcmpgtd_2 = "rmo:660F66rM",
1250 pcmpgtw_2 = "rmo:660F65rM",
1251 pextrw_3 = "rri/do:660FC5rMU|xri/wo:660F3A15nRmU", -- Mem op: SSE4.1 only. 1330 pextrw_3 = "rri/do:660FC5rMU|xri/wo:660F3A15nRmU", -- Mem op: SSE4.1 only.
1252 pinsrw_3 = "rri/od:660FC4rMU|rxi/ow:", 1331 pinsrw_3 = "rri/od:660FC4rMU|rxi/ow:",
1253 pmaddwd_2 = "rmo:660FF5rM",
1254 pmaxsw_2 = "rmo:660FEErM",
1255 pmaxub_2 = "rmo:660FDErM",
1256 pminsw_2 = "rmo:660FEArM",
1257 pminub_2 = "rmo:660FDArM",
1258 pmovmskb_2 = "rr/do:660FD7rM", 1332 pmovmskb_2 = "rr/do:660FD7rM",
1259 pmulhuw_2 = "rmo:660FE4rM",
1260 pmulhw_2 = "rmo:660FE5rM",
1261 pmullw_2 = "rmo:660FD5rM",
1262 pmuludq_2 = "rmo:660FF4rM",
1263 por_2 = "rmo:660FEBrM",
1264 prefetchnta_1 = "xb:n0F180m", 1333 prefetchnta_1 = "xb:n0F180m",
1265 prefetcht0_1 = "xb:n0F181m", 1334 prefetcht0_1 = "xb:n0F181m",
1266 prefetcht1_1 = "xb:n0F182m", 1335 prefetcht1_1 = "xb:n0F182m",
1267 prefetcht2_1 = "xb:n0F183m", 1336 prefetcht2_1 = "xb:n0F183m",
1268 psadbw_2 = "rmo:660FF6rM",
1269 pshufd_3 = "rmio:660F70rMU", 1337 pshufd_3 = "rmio:660F70rMU",
1270 pshufhw_3 = "rmio:F30F70rMU", 1338 pshufhw_3 = "rmio:F30F70rMU",
1271 pshuflw_3 = "rmio:F20F70rMU", 1339 pshuflw_3 = "rmio:F20F70rMU",
@@ -1279,23 +1347,6 @@ local map_op = {
1279 psrldq_2 = "rio:660F733mU", 1347 psrldq_2 = "rio:660F733mU",
1280 psrlq_2 = "rmo:660FD3rM|rio:660F732mU", 1348 psrlq_2 = "rmo:660FD3rM|rio:660F732mU",
1281 psrlw_2 = "rmo:660FD1rM|rio:660F712mU", 1349 psrlw_2 = "rmo:660FD1rM|rio:660F712mU",
1282 psubb_2 = "rmo:660FF8rM",
1283 psubd_2 = "rmo:660FFArM",
1284 psubq_2 = "rmo:660FFBrM",
1285 psubsb_2 = "rmo:660FE8rM",
1286 psubsw_2 = "rmo:660FE9rM",
1287 psubusb_2 = "rmo:660FD8rM",
1288 psubusw_2 = "rmo:660FD9rM",
1289 psubw_2 = "rmo:660FF9rM",
1290 punpckhbw_2 = "rmo:660F68rM",
1291 punpckhdq_2 = "rmo:660F6ArM",
1292 punpckhqdq_2 = "rmo:660F6DrM",
1293 punpckhwd_2 = "rmo:660F69rM",
1294 punpcklbw_2 = "rmo:660F60rM",
1295 punpckldq_2 = "rmo:660F62rM",
1296 punpcklqdq_2 = "rmo:660F6CrM",
1297 punpcklwd_2 = "rmo:660F61rM",
1298 pxor_2 = "rmo:660FEFrM",
1299 rcpps_2 = "rmo:0F53rM", 1350 rcpps_2 = "rmo:0F53rM",
1300 rcpss_2 = "rro:F30F53rM|rx/od:", 1351 rcpss_2 = "rro:F30F53rM|rx/od:",
1301 rsqrtps_2 = "rmo:0F52rM", 1352 rsqrtps_2 = "rmo:0F52rM",
@@ -1413,6 +1464,327 @@ local map_op = {
1413 movntsd_2 = "xr/qo:nF20F2BRm", 1464 movntsd_2 = "xr/qo:nF20F2BRm",
1414 movntss_2 = "xr/do:F30F2BRm", 1465 movntss_2 = "xr/do:F30F2BRm",
1415 -- popcnt is also in SSE4.2 1466 -- popcnt is also in SSE4.2
1467
1468 -- AES-NI
1469 aesdec_2 = "rmo:660F38DErM",
1470 aesdeclast_2 = "rmo:660F38DFrM",
1471 aesenc_2 = "rmo:660F38DCrM",
1472 aesenclast_2 = "rmo:660F38DDrM",
1473 aesimc_2 = "rmo:660F38DBrM",
1474 aeskeygenassist_3 = "rmio:660F3ADFrMU",
1475 pclmulqdq_3 = "rmio:660F3A44rMU",
1476
1477 -- AVX FP ops
1478 vaddsubpd_3 = "rrmoy:660FVD0rM",
1479 vaddsubps_3 = "rrmoy:F20FVD0rM",
1480 vandpd_3 = "rrmoy:660FV54rM",
1481 vandps_3 = "rrmoy:0FV54rM",
1482 vandnpd_3 = "rrmoy:660FV55rM",
1483 vandnps_3 = "rrmoy:0FV55rM",
1484 vblendpd_4 = "rrmioy:660F3AV0DrMU",
1485 vblendps_4 = "rrmioy:660F3AV0CrMU",
1486 vblendvpd_4 = "rrmroy:660F3AV4BrMs",
1487 vblendvps_4 = "rrmroy:660F3AV4ArMs",
1488 vbroadcastf128_2 = "rx/yo:660F38u1ArM",
1489 vcmppd_4 = "rrmioy:660FVC2rMU",
1490 vcmpps_4 = "rrmioy:0FVC2rMU",
1491 vcmpsd_4 = "rrrio:F20FVC2rMU|rrxi/ooq:",
1492 vcmpss_4 = "rrrio:F30FVC2rMU|rrxi/ood:",
1493 vcomisd_2 = "rro:660Fu2FrM|rx/oq:",
1494 vcomiss_2 = "rro:0Fu2FrM|rx/od:",
1495 vcvtdq2pd_2 = "rro:F30FuE6rM|rx/oq:|rm/yo:",
1496 vcvtdq2ps_2 = "rmoy:0Fu5BrM",
1497 vcvtpd2dq_2 = "rmoy:F20FuE6rM",
1498 vcvtpd2ps_2 = "rmoy:660Fu5ArM",
1499 vcvtps2dq_2 = "rmoy:660Fu5BrM",
1500 vcvtps2pd_2 = "rro:0Fu5ArM|rx/oq:|rm/yo:",
1501 vcvtsd2si_2 = "rr/do:F20Fu2DrM|rx/dq:|rr/qo:|rxq:",
1502 vcvtsd2ss_3 = "rrro:F20FV5ArM|rrx/ooq:",
1503 vcvtsi2sd_3 = "rrm/ood:F20FV2ArM|rrm/ooq:F20FVX2ArM",
1504 vcvtsi2ss_3 = "rrm/ood:F30FV2ArM|rrm/ooq:F30FVX2ArM",
1505 vcvtss2sd_3 = "rrro:F30FV5ArM|rrx/ood:",
1506 vcvtss2si_2 = "rr/do:F30Fu2DrM|rxd:|rr/qo:|rx/qd:",
1507 vcvttpd2dq_2 = "rmo:660FuE6rM|rm/oy:660FuLE6rM",
1508 vcvttps2dq_2 = "rmoy:F30Fu5BrM",
1509 vcvttsd2si_2 = "rr/do:F20Fu2CrM|rx/dq:|rr/qo:|rxq:",
1510 vcvttss2si_2 = "rr/do:F30Fu2CrM|rxd:|rr/qo:|rx/qd:",
1511 vdppd_4 = "rrmio:660F3AV41rMU",
1512 vdpps_4 = "rrmioy:660F3AV40rMU",
1513 vextractf128_3 = "mri/oy:660F3AuL19RmU",
1514 vextractps_3 = "mri/do:660F3Au17RmU",
1515 vhaddpd_3 = "rrmoy:660FV7CrM",
1516 vhaddps_3 = "rrmoy:F20FV7CrM",
1517 vhsubpd_3 = "rrmoy:660FV7DrM",
1518 vhsubps_3 = "rrmoy:F20FV7DrM",
1519 vinsertf128_4 = "rrmi/yyo:660F3AV18rMU",
1520 vinsertps_4 = "rrrio:660F3AV21rMU|rrxi/ood:",
1521 vldmxcsr_1 = "xd:0FuAE2m",
1522 vmaskmovps_3 = "rrxoy:660F38V2CrM|xrroy:660F38V2ERm",
1523 vmaskmovpd_3 = "rrxoy:660F38V2DrM|xrroy:660F38V2FRm",
1524 vmovapd_2 = "rmoy:660Fu28rM|mroy:660Fu29Rm",
1525 vmovaps_2 = "rmoy:0Fu28rM|mroy:0Fu29Rm",
1526 vmovd_2 = "rm/od:660Fu6ErM|rm/oq:660FuX6ErM|mr/do:660Fu7ERm|mr/qo:",
1527 vmovq_2 = "rro:F30Fu7ErM|rx/oq:|xr/qo:660FuD6Rm",
1528 vmovddup_2 = "rmy:F20Fu12rM|rro:|rx/oq:",
1529 vmovhlps_3 = "rrro:0FV12rM",
1530 vmovhpd_2 = "xr/qo:660Fu17Rm",
1531 vmovhpd_3 = "rrx/ooq:660FV16rM",
1532 vmovhps_2 = "xr/qo:0Fu17Rm",
1533 vmovhps_3 = "rrx/ooq:0FV16rM",
1534 vmovlhps_3 = "rrro:0FV16rM",
1535 vmovlpd_2 = "xr/qo:660Fu13Rm",
1536 vmovlpd_3 = "rrx/ooq:660FV12rM",
1537 vmovlps_2 = "xr/qo:0Fu13Rm",
1538 vmovlps_3 = "rrx/ooq:0FV12rM",
1539 vmovmskpd_2 = "rr/do:660Fu50rM|rr/dy:660FuL50rM",
1540 vmovmskps_2 = "rr/do:0Fu50rM|rr/dy:0FuL50rM",
1541 vmovntpd_2 = "xroy:660Fu2BRm",
1542 vmovntps_2 = "xroy:0Fu2BRm",
1543 vmovsd_2 = "rx/oq:F20Fu10rM|xr/qo:F20Fu11Rm",
1544 vmovsd_3 = "rrro:F20FV10rM",
1545 vmovshdup_2 = "rmoy:F30Fu16rM",
1546 vmovsldup_2 = "rmoy:F30Fu12rM",
1547 vmovss_2 = "rx/od:F30Fu10rM|xr/do:F30Fu11Rm",
1548 vmovss_3 = "rrro:F30FV10rM",
1549 vmovupd_2 = "rmoy:660Fu10rM|mroy:660Fu11Rm",
1550 vmovups_2 = "rmoy:0Fu10rM|mroy:0Fu11Rm",
1551 vorpd_3 = "rrmoy:660FV56rM",
1552 vorps_3 = "rrmoy:0FV56rM",
1553 vpermilpd_3 = "rrmoy:660F38V0DrM|rmioy:660F3Au05rMU",
1554 vpermilps_3 = "rrmoy:660F38V0CrM|rmioy:660F3Au04rMU",
1555 vperm2f128_4 = "rrmiy:660F3AV06rMU",
1556 vptestpd_2 = "rmoy:660F38u0FrM",
1557 vptestps_2 = "rmoy:660F38u0ErM",
1558 vrcpps_2 = "rmoy:0Fu53rM",
1559 vrcpss_3 = "rrro:F30FV53rM|rrx/ood:",
1560 vrsqrtps_2 = "rmoy:0Fu52rM",
1561 vrsqrtss_3 = "rrro:F30FV52rM|rrx/ood:",
1562 vroundpd_3 = "rmioy:660F3Au09rMU",
1563 vroundps_3 = "rmioy:660F3Au08rMU",
1564 vroundsd_4 = "rrrio:660F3AV0BrMU|rrxi/ooq:",
1565 vroundss_4 = "rrrio:660F3AV0ArMU|rrxi/ood:",
1566 vshufpd_4 = "rrmioy:660FVC6rMU",
1567 vshufps_4 = "rrmioy:0FVC6rMU",
1568 vsqrtps_2 = "rmoy:0Fu51rM",
1569 vsqrtss_2 = "rro:F30Fu51rM|rx/od:",
1570 vsqrtpd_2 = "rmoy:660Fu51rM",
1571 vsqrtsd_2 = "rro:F20Fu51rM|rx/oq:",
1572 vstmxcsr_1 = "xd:0FuAE3m",
1573 vucomisd_2 = "rro:660Fu2ErM|rx/oq:",
1574 vucomiss_2 = "rro:0Fu2ErM|rx/od:",
1575 vunpckhpd_3 = "rrmoy:660FV15rM",
1576 vunpckhps_3 = "rrmoy:0FV15rM",
1577 vunpcklpd_3 = "rrmoy:660FV14rM",
1578 vunpcklps_3 = "rrmoy:0FV14rM",
1579 vxorpd_3 = "rrmoy:660FV57rM",
1580 vxorps_3 = "rrmoy:0FV57rM",
1581 vzeroall_0 = "0FuL77",
1582 vzeroupper_0 = "0Fu77",
1583
1584 -- AVX2 FP ops
1585 vbroadcastss_2 = "rx/od:660F38u18rM|rx/yd:|rro:|rr/yo:",
1586 vbroadcastsd_2 = "rx/yq:660F38u19rM|rr/yo:",
1587 -- *vgather* (!vsib)
1588 vpermpd_3 = "rmiy:660F3AuX01rMU",
1589 vpermps_3 = "rrmy:660F38V16rM",
1590
1591 -- AVX, AVX2 integer ops
1592 -- In general, xmm requires AVX, ymm requires AVX2.
1593 vaesdec_3 = "rrmo:660F38VDErM",
1594 vaesdeclast_3 = "rrmo:660F38VDFrM",
1595 vaesenc_3 = "rrmo:660F38VDCrM",
1596 vaesenclast_3 = "rrmo:660F38VDDrM",
1597 vaesimc_2 = "rmo:660F38uDBrM",
1598 vaeskeygenassist_3 = "rmio:660F3AuDFrMU",
1599 vlddqu_2 = "rxoy:F20FuF0rM",
1600 vmaskmovdqu_2 = "rro:660FuF7rM",
1601 vmovdqa_2 = "rmoy:660Fu6FrM|mroy:660Fu7FRm",
1602 vmovdqu_2 = "rmoy:F30Fu6FrM|mroy:F30Fu7FRm",
1603 vmovntdq_2 = "xroy:660FuE7Rm",
1604 vmovntdqa_2 = "rxoy:660F38u2ArM",
1605 vmpsadbw_4 = "rrmioy:660F3AV42rMU",
1606 vpabsb_2 = "rmoy:660F38u1CrM",
1607 vpabsd_2 = "rmoy:660F38u1ErM",
1608 vpabsw_2 = "rmoy:660F38u1DrM",
1609 vpackusdw_3 = "rrmoy:660F38V2BrM",
1610 vpalignr_4 = "rrmioy:660F3AV0FrMU",
1611 vpblendvb_4 = "rrmroy:660F3AV4CrMs",
1612 vpblendw_4 = "rrmioy:660F3AV0ErMU",
1613 vpclmulqdq_4 = "rrmio:660F3AV44rMU",
1614 vpcmpeqq_3 = "rrmoy:660F38V29rM",
1615 vpcmpestri_3 = "rmio:660F3Au61rMU",
1616 vpcmpestrm_3 = "rmio:660F3Au60rMU",
1617 vpcmpgtq_3 = "rrmoy:660F38V37rM",
1618 vpcmpistri_3 = "rmio:660F3Au63rMU",
1619 vpcmpistrm_3 = "rmio:660F3Au62rMU",
1620 vpextrb_3 = "rri/do:660F3Au14nRmU|rri/qo:|xri/bo:",
1621 vpextrw_3 = "rri/do:660FuC5rMU|xri/wo:660F3Au15nRmU",
1622 vpextrd_3 = "mri/do:660F3Au16RmU",
1623 vpextrq_3 = "mri/qo:660F3Au16RmU",
1624 vphaddw_3 = "rrmoy:660F38V01rM",
1625 vphaddd_3 = "rrmoy:660F38V02rM",
1626 vphaddsw_3 = "rrmoy:660F38V03rM",
1627 vphminposuw_2 = "rmo:660F38u41rM",
1628 vphsubw_3 = "rrmoy:660F38V05rM",
1629 vphsubd_3 = "rrmoy:660F38V06rM",
1630 vphsubsw_3 = "rrmoy:660F38V07rM",
1631 vpinsrb_4 = "rrri/ood:660F3AV20rMU|rrxi/oob:",
1632 vpinsrw_4 = "rrri/ood:660FVC4rMU|rrxi/oow:",
1633 vpinsrd_4 = "rrmi/ood:660F3AV22rMU",
1634 vpinsrq_4 = "rrmi/ooq:660F3AVX22rMU",
1635 vpmaddubsw_3 = "rrmoy:660F38V04rM",
1636 vpmaxsb_3 = "rrmoy:660F38V3CrM",
1637 vpmaxsd_3 = "rrmoy:660F38V3DrM",
1638 vpmaxuw_3 = "rrmoy:660F38V3ErM",
1639 vpmaxud_3 = "rrmoy:660F38V3FrM",
1640 vpminsb_3 = "rrmoy:660F38V38rM",
1641 vpminsd_3 = "rrmoy:660F38V39rM",
1642 vpminuw_3 = "rrmoy:660F38V3ArM",
1643 vpminud_3 = "rrmoy:660F38V3BrM",
1644 vpmovmskb_2 = "rr/do:660FuD7rM|rr/dy:660FuLD7rM",
1645 vpmovsxbw_2 = "rroy:660F38u20rM|rx/oq:|rx/yo:",
1646 vpmovsxbd_2 = "rroy:660F38u21rM|rx/od:|rx/yq:",
1647 vpmovsxbq_2 = "rroy:660F38u22rM|rx/ow:|rx/yd:",
1648 vpmovsxwd_2 = "rroy:660F38u23rM|rx/oq:|rx/yo:",
1649 vpmovsxwq_2 = "rroy:660F38u24rM|rx/od:|rx/yq:",
1650 vpmovsxdq_2 = "rroy:660F38u25rM|rx/oq:|rx/yo:",
1651 vpmovzxbw_2 = "rroy:660F38u30rM|rx/oq:|rx/yo:",
1652 vpmovzxbd_2 = "rroy:660F38u31rM|rx/od:|rx/yq:",
1653 vpmovzxbq_2 = "rroy:660F38u32rM|rx/ow:|rx/yd:",
1654 vpmovzxwd_2 = "rroy:660F38u33rM|rx/oq:|rx/yo:",
1655 vpmovzxwq_2 = "rroy:660F38u34rM|rx/od:|rx/yq:",
1656 vpmovzxdq_2 = "rroy:660F38u35rM|rx/oq:|rx/yo:",
1657 vpmuldq_3 = "rrmoy:660F38V28rM",
1658 vpmulhrsw_3 = "rrmoy:660F38V0BrM",
1659 vpmulld_3 = "rrmoy:660F38V40rM",
1660 vpshufb_3 = "rrmoy:660F38V00rM",
1661 vpshufd_3 = "rmioy:660Fu70rMU",
1662 vpshufhw_3 = "rmioy:F30Fu70rMU",
1663 vpshuflw_3 = "rmioy:F20Fu70rMU",
1664 vpsignb_3 = "rrmoy:660F38V08rM",
1665 vpsignw_3 = "rrmoy:660F38V09rM",
1666 vpsignd_3 = "rrmoy:660F38V0ArM",
1667 vpslldq_3 = "rrioy:660Fv737mU",
1668 vpsllw_3 = "rrmoy:660FVF1rM|rrioy:660Fv716mU",
1669 vpslld_3 = "rrmoy:660FVF2rM|rrioy:660Fv726mU",
1670 vpsllq_3 = "rrmoy:660FVF3rM|rrioy:660Fv736mU",
1671 vpsraw_3 = "rrmoy:660FVE1rM|rrioy:660Fv714mU",
1672 vpsrad_3 = "rrmoy:660FVE2rM|rrioy:660Fv724mU",
1673 vpsrldq_3 = "rrioy:660Fv733mU",
1674 vpsrlw_3 = "rrmoy:660FVD1rM|rrioy:660Fv712mU",
1675 vpsrld_3 = "rrmoy:660FVD2rM|rrioy:660Fv722mU",
1676 vpsrlq_3 = "rrmoy:660FVD3rM|rrioy:660Fv732mU",
1677 vptest_2 = "rmoy:660F38u17rM",
1678
1679 -- AVX2 integer ops
1680 vbroadcasti128_2 = "rx/yo:660F38u5ArM",
1681 vinserti128_4 = "rrmi/yyo:660F3AV38rMU",
1682 vextracti128_3 = "mri/oy:660F3AuL39RmU",
1683 vpblendd_4 = "rrmioy:660F3AV02rMU",
1684 vpbroadcastb_2 = "rro:660F38u78rM|rx/ob:|rr/yo:|rx/yb:",
1685 vpbroadcastw_2 = "rro:660F38u79rM|rx/ow:|rr/yo:|rx/yw:",
1686 vpbroadcastd_2 = "rro:660F38u58rM|rx/od:|rr/yo:|rx/yd:",
1687 vpbroadcastq_2 = "rro:660F38u59rM|rx/oq:|rr/yo:|rx/yq:",
1688 vpermd_3 = "rrmy:660F38V36rM",
1689 vpermq_3 = "rmiy:660F3AuX00rMU",
1690 -- *vpgather* (!vsib)
1691 vperm2i128_4 = "rrmiy:660F3AV46rMU",
1692 vpmaskmovd_3 = "rrxoy:660F38V8CrM|xrroy:660F38V8ERm",
1693 vpmaskmovq_3 = "rrxoy:660F38VX8CrM|xrroy:660F38VX8ERm",
1694 vpsllvd_3 = "rrmoy:660F38V47rM",
1695 vpsllvq_3 = "rrmoy:660F38VX47rM",
1696 vpsravd_3 = "rrmoy:660F38V46rM",
1697 vpsrlvd_3 = "rrmoy:660F38V45rM",
1698 vpsrlvq_3 = "rrmoy:660F38VX45rM",
1699
1700 -- Intel ADX
1701 adcx_2 = "rmqd:660F38F6rM",
1702 adox_2 = "rmqd:F30F38F6rM",
1703
1704 -- BMI1
1705 andn_3 = "rrmqd:0F38VF2rM",
1706 bextr_3 = "rmrqd:0F38wF7rM",
1707 blsi_2 = "rmqd:0F38vF33m",
1708 blsmsk_2 = "rmqd:0F38vF32m",
1709 blsr_2 = "rmqd:0F38vF31m",
1710 tzcnt_2 = "rmqdw:F30FBCrM",
1711
1712 -- BMI2
1713 bzhi_3 = "rmrqd:0F38wF5rM",
1714 mulx_3 = "rrmqd:F20F38VF6rM",
1715 pdep_3 = "rrmqd:F20F38VF5rM",
1716 pext_3 = "rrmqd:F30F38VF5rM",
1717 rorx_3 = "rmSqd:F20F3AuF0rMS",
1718 sarx_3 = "rmrqd:F30F38wF7rM",
1719 shrx_3 = "rmrqd:F20F38wF7rM",
1720 shlx_3 = "rmrqd:660F38wF7rM",
1721
1722 -- FMA3
1723 vfmaddsub132pd_3 = "rrmoy:660F38VX96rM",
1724 vfmaddsub132ps_3 = "rrmoy:660F38V96rM",
1725 vfmaddsub213pd_3 = "rrmoy:660F38VXA6rM",
1726 vfmaddsub213ps_3 = "rrmoy:660F38VA6rM",
1727 vfmaddsub231pd_3 = "rrmoy:660F38VXB6rM",
1728 vfmaddsub231ps_3 = "rrmoy:660F38VB6rM",
1729
1730 vfmsubadd132pd_3 = "rrmoy:660F38VX97rM",
1731 vfmsubadd132ps_3 = "rrmoy:660F38V97rM",
1732 vfmsubadd213pd_3 = "rrmoy:660F38VXA7rM",
1733 vfmsubadd213ps_3 = "rrmoy:660F38VA7rM",
1734 vfmsubadd231pd_3 = "rrmoy:660F38VXB7rM",
1735 vfmsubadd231ps_3 = "rrmoy:660F38VB7rM",
1736
1737 vfmadd132pd_3 = "rrmoy:660F38VX98rM",
1738 vfmadd132ps_3 = "rrmoy:660F38V98rM",
1739 vfmadd132sd_3 = "rrro:660F38VX99rM|rrx/ooq:",
1740 vfmadd132ss_3 = "rrro:660F38V99rM|rrx/ood:",
1741 vfmadd213pd_3 = "rrmoy:660F38VXA8rM",
1742 vfmadd213ps_3 = "rrmoy:660F38VA8rM",
1743 vfmadd213sd_3 = "rrro:660F38VXA9rM|rrx/ooq:",
1744 vfmadd213ss_3 = "rrro:660F38VA9rM|rrx/ood:",
1745 vfmadd231pd_3 = "rrmoy:660F38VXB8rM",
1746 vfmadd231ps_3 = "rrmoy:660F38VB8rM",
1747 vfmadd231sd_3 = "rrro:660F38VXB9rM|rrx/ooq:",
1748 vfmadd231ss_3 = "rrro:660F38VB9rM|rrx/ood:",
1749
1750 vfmsub132pd_3 = "rrmoy:660F38VX9ArM",
1751 vfmsub132ps_3 = "rrmoy:660F38V9ArM",
1752 vfmsub132sd_3 = "rrro:660F38VX9BrM|rrx/ooq:",
1753 vfmsub132ss_3 = "rrro:660F38V9BrM|rrx/ood:",
1754 vfmsub213pd_3 = "rrmoy:660F38VXAArM",
1755 vfmsub213ps_3 = "rrmoy:660F38VAArM",
1756 vfmsub213sd_3 = "rrro:660F38VXABrM|rrx/ooq:",
1757 vfmsub213ss_3 = "rrro:660F38VABrM|rrx/ood:",
1758 vfmsub231pd_3 = "rrmoy:660F38VXBArM",
1759 vfmsub231ps_3 = "rrmoy:660F38VBArM",
1760 vfmsub231sd_3 = "rrro:660F38VXBBrM|rrx/ooq:",
1761 vfmsub231ss_3 = "rrro:660F38VBBrM|rrx/ood:",
1762
1763 vfnmadd132pd_3 = "rrmoy:660F38VX9CrM",
1764 vfnmadd132ps_3 = "rrmoy:660F38V9CrM",
1765 vfnmadd132sd_3 = "rrro:660F38VX9DrM|rrx/ooq:",
1766 vfnmadd132ss_3 = "rrro:660F38V9DrM|rrx/ood:",
1767 vfnmadd213pd_3 = "rrmoy:660F38VXACrM",
1768 vfnmadd213ps_3 = "rrmoy:660F38VACrM",
1769 vfnmadd213sd_3 = "rrro:660F38VXADrM|rrx/ooq:",
1770 vfnmadd213ss_3 = "rrro:660F38VADrM|rrx/ood:",
1771 vfnmadd231pd_3 = "rrmoy:660F38VXBCrM",
1772 vfnmadd231ps_3 = "rrmoy:660F38VBCrM",
1773 vfnmadd231sd_3 = "rrro:660F38VXBDrM|rrx/ooq:",
1774 vfnmadd231ss_3 = "rrro:660F38VBDrM|rrx/ood:",
1775
1776 vfnmsub132pd_3 = "rrmoy:660F38VX9ErM",
1777 vfnmsub132ps_3 = "rrmoy:660F38V9ErM",
1778 vfnmsub132sd_3 = "rrro:660F38VX9FrM|rrx/ooq:",
1779 vfnmsub132ss_3 = "rrro:660F38V9FrM|rrx/ood:",
1780 vfnmsub213pd_3 = "rrmoy:660F38VXAErM",
1781 vfnmsub213ps_3 = "rrmoy:660F38VAErM",
1782 vfnmsub213sd_3 = "rrro:660F38VXAFrM|rrx/ooq:",
1783 vfnmsub213ss_3 = "rrro:660F38VAFrM|rrx/ood:",
1784 vfnmsub231pd_3 = "rrmoy:660F38VXBErM",
1785 vfnmsub231ps_3 = "rrmoy:660F38VBErM",
1786 vfnmsub231sd_3 = "rrro:660F38VXBFrM|rrx/ooq:",
1787 vfnmsub231ss_3 = "rrro:660F38VBFrM|rrx/ood:",
1416} 1788}
1417 1789
1418------------------------------------------------------------------------------ 1790------------------------------------------------------------------------------
@@ -1463,28 +1835,58 @@ for cc,n in pairs{ b=0, e=1, be=2, u=3, nb=4, ne=5, nbe=6, nu=7 } do
1463 map_op["fcmov"..cc.."_2"] = format("Fff:%04XR", nc) -- P6+ 1835 map_op["fcmov"..cc.."_2"] = format("Fff:%04XR", nc) -- P6+
1464end 1836end
1465 1837
1466-- SSE FP arithmetic ops. 1838-- SSE / AVX FP arithmetic ops.
1467for name,n in pairs{ sqrt = 1, add = 8, mul = 9, 1839for name,n in pairs{ sqrt = 1, add = 8, mul = 9,
1468 sub = 12, min = 13, div = 14, max = 15 } do 1840 sub = 12, min = 13, div = 14, max = 15 } do
1469 map_op[name.."ps_2"] = format("rmo:0F5%XrM", n) 1841 map_op[name.."ps_2"] = format("rmo:0F5%XrM", n)
1470 map_op[name.."ss_2"] = format("rro:F30F5%XrM|rx/od:", n) 1842 map_op[name.."ss_2"] = format("rro:F30F5%XrM|rx/od:", n)
1471 map_op[name.."pd_2"] = format("rmo:660F5%XrM", n) 1843 map_op[name.."pd_2"] = format("rmo:660F5%XrM", n)
1472 map_op[name.."sd_2"] = format("rro:F20F5%XrM|rx/oq:", n) 1844 map_op[name.."sd_2"] = format("rro:F20F5%XrM|rx/oq:", n)
1845 if n ~= 1 then
1846 map_op["v"..name.."ps_3"] = format("rrmoy:0FV5%XrM", n)
1847 map_op["v"..name.."ss_3"] = format("rrro:F30FV5%XrM|rrx/ood:", n)
1848 map_op["v"..name.."pd_3"] = format("rrmoy:660FV5%XrM", n)
1849 map_op["v"..name.."sd_3"] = format("rrro:F20FV5%XrM|rrx/ooq:", n)
1850 end
1851end
1852
1853-- SSE2 / AVX / AVX2 integer arithmetic ops (66 0F leaf).
1854for name,n in pairs{
1855 paddb = 0xFC, paddw = 0xFD, paddd = 0xFE, paddq = 0xD4,
1856 paddsb = 0xEC, paddsw = 0xED, packssdw = 0x6B,
1857 packsswb = 0x63, packuswb = 0x67, paddusb = 0xDC,
1858 paddusw = 0xDD, pand = 0xDB, pandn = 0xDF, pavgb = 0xE0,
1859 pavgw = 0xE3, pcmpeqb = 0x74, pcmpeqd = 0x76,
1860 pcmpeqw = 0x75, pcmpgtb = 0x64, pcmpgtd = 0x66,
1861 pcmpgtw = 0x65, pmaddwd = 0xF5, pmaxsw = 0xEE,
1862 pmaxub = 0xDE, pminsw = 0xEA, pminub = 0xDA,
1863 pmulhuw = 0xE4, pmulhw = 0xE5, pmullw = 0xD5,
1864 pmuludq = 0xF4, por = 0xEB, psadbw = 0xF6, psubb = 0xF8,
1865 psubw = 0xF9, psubd = 0xFA, psubq = 0xFB, psubsb = 0xE8,
1866 psubsw = 0xE9, psubusb = 0xD8, psubusw = 0xD9,
1867 punpckhbw = 0x68, punpckhwd = 0x69, punpckhdq = 0x6A,
1868 punpckhqdq = 0x6D, punpcklbw = 0x60, punpcklwd = 0x61,
1869 punpckldq = 0x62, punpcklqdq = 0x6C, pxor = 0xEF
1870} do
1871 map_op[name.."_2"] = format("rmo:660F%02XrM", n)
1872 map_op["v"..name.."_3"] = format("rrmoy:660FV%02XrM", n)
1473end 1873end
1474 1874
1475------------------------------------------------------------------------------ 1875------------------------------------------------------------------------------
1476 1876
1877local map_vexarg = { u = false, v = 1, V = 2, w = 3 }
1878
1477-- Process pattern string. 1879-- Process pattern string.
1478local function dopattern(pat, args, sz, op, needrex) 1880local function dopattern(pat, args, sz, op, needrex)
1479 local digit, addin 1881 local digit, addin, vex
1480 local opcode = 0 1882 local opcode = 0
1481 local szov = sz 1883 local szov = sz
1482 local narg = 1 1884 local narg = 1
1483 local rex = 0 1885 local rex = 0
1484 1886
1485 -- Limit number of section buffer positions used by a single dasm_put(). 1887 -- Limit number of section buffer positions used by a single dasm_put().
1486 -- A single opcode needs a maximum of 5 positions. 1888 -- A single opcode needs a maximum of 6 positions.
1487 if secpos+5 > maxsecpos then wflush() end 1889 if secpos+6 > maxsecpos then wflush() end
1488 1890
1489 -- Process each character. 1891 -- Process each character.
1490 for c in gmatch(pat.."|", ".") do 1892 for c in gmatch(pat.."|", ".") do
@@ -1498,6 +1900,8 @@ local function dopattern(pat, args, sz, op, needrex)
1498 szov = nil 1900 szov = nil
1499 elseif c == "X" then -- Force REX.W. 1901 elseif c == "X" then -- Force REX.W.
1500 rex = 8 1902 rex = 8
1903 elseif c == "L" then -- Force VEX.L.
1904 vex.l = true
1501 elseif c == "r" then -- Merge 1st operand regno. into opcode. 1905 elseif c == "r" then -- Merge 1st operand regno. into opcode.
1502 addin = args[1]; opcode = opcode + (addin.reg % 8) 1906 addin = args[1]; opcode = opcode + (addin.reg % 8)
1503 if narg < 2 then narg = 2 end 1907 if narg < 2 then narg = 2 end
@@ -1521,21 +1925,42 @@ local function dopattern(pat, args, sz, op, needrex)
1521 if t.xreg and t.xreg > 7 then rex = rex + 2 end 1925 if t.xreg and t.xreg > 7 then rex = rex + 2 end
1522 if s > 7 then rex = rex + 4 end 1926 if s > 7 then rex = rex + 4 end
1523 if needrex then rex = rex + 16 end 1927 if needrex then rex = rex + 16 end
1524 wputop(szov, opcode, rex); opcode = nil 1928 local psz, sk = wputop(szov, opcode, rex, vex, s < 0, t.vreg or t.vxreg)
1929 opcode = nil
1525 local imark = sub(pat, -1) -- Force a mark (ugly). 1930 local imark = sub(pat, -1) -- Force a mark (ugly).
1526 -- Put ModRM/SIB with regno/last digit as spare. 1931 -- Put ModRM/SIB with regno/last digit as spare.
1527 wputmrmsib(t, imark, s, addin and addin.vreg) 1932 wputmrmsib(t, imark, s, addin and addin.vreg, psz, sk)
1528 addin = nil 1933 addin = nil
1934 elseif map_vexarg[c] ~= nil then -- Encode using VEX prefix
1935 local b = band(opcode, 255); opcode = shr(opcode, 8)
1936 local m = 1
1937 if b == 0x38 then m = 2
1938 elseif b == 0x3a then m = 3 end
1939 if m ~= 1 then b = band(opcode, 255); opcode = shr(opcode, 8) end
1940 if b ~= 0x0f then
1941 werror("expected `0F', `0F38', or `0F3A' to precede `"..c..
1942 "' in pattern `"..pat.."' for `"..op.."'")
1943 end
1944 local v = map_vexarg[c]
1945 if v then v = remove(args, v) end
1946 b = band(opcode, 255)
1947 local p = 0
1948 if b == 0x66 then p = 1
1949 elseif b == 0xf3 then p = 2
1950 elseif b == 0xf2 then p = 3 end
1951 if p ~= 0 then opcode = shr(opcode, 8) end
1952 if opcode ~= 0 then wputop(nil, opcode, 0); opcode = 0 end
1953 vex = { m = m, p = p, v = v }
1529 else 1954 else
1530 if opcode then -- Flush opcode. 1955 if opcode then -- Flush opcode.
1531 if szov == "q" and rex == 0 then rex = rex + 8 end 1956 if szov == "q" and rex == 0 then rex = rex + 8 end
1532 if needrex then rex = rex + 16 end 1957 if needrex then rex = rex + 16 end
1533 if addin and addin.reg == -1 then 1958 if addin and addin.reg == -1 then
1534 wputop(szov, opcode - 7, rex) 1959 local psz, sk = wputop(szov, opcode - 7, rex, vex, true)
1535 waction("VREG", addin.vreg); wputxb(0) 1960 wvreg("opcode", addin.vreg, psz, sk)
1536 else 1961 else
1537 if addin and addin.reg > 7 then rex = rex + 1 end 1962 if addin and addin.reg > 7 then rex = rex + 1 end
1538 wputop(szov, opcode, rex) 1963 wputop(szov, opcode, rex, vex)
1539 end 1964 end
1540 opcode = nil 1965 opcode = nil
1541 end 1966 end
@@ -1549,7 +1974,7 @@ local function dopattern(pat, args, sz, op, needrex)
1549 local a = args[narg] 1974 local a = args[narg]
1550 narg = narg + 1 1975 narg = narg + 1
1551 local mode, imm = a.mode, a.imm 1976 local mode, imm = a.mode, a.imm
1552 if mode == "iJ" and not match("iIJ", c) then 1977 if mode == "iJ" and not match(x64 and "J" or "iIJ", c) then
1553 werror("bad operand size for label") 1978 werror("bad operand size for label")
1554 end 1979 end
1555 if c == "S" then 1980 if c == "S" then
@@ -1572,6 +1997,14 @@ local function dopattern(pat, args, sz, op, needrex)
1572 else 1997 else
1573 wputlabel("REL_", imm, 2) 1998 wputlabel("REL_", imm, 2)
1574 end 1999 end
2000 elseif c == "s" then
2001 local reg = a.reg
2002 if reg < 0 then
2003 wputb(0)
2004 wvreg("imm.hi", a.vreg)
2005 else
2006 wputb(shl(reg, 4))
2007 end
1575 else 2008 else
1576 werror("bad char `"..c.."' in pattern `"..pat.."' for `"..op.."'") 2009 werror("bad char `"..c.."' in pattern `"..pat.."' for `"..op.."'")
1577 end 2010 end
@@ -1648,11 +2081,14 @@ map_op[".template__"] = function(params, template, nparams)
1648 if pat == "" then pat = lastpat else lastpat = pat end 2081 if pat == "" then pat = lastpat else lastpat = pat end
1649 if matchtm(tm, args) then 2082 if matchtm(tm, args) then
1650 local prefix = sub(szm, 1, 1) 2083 local prefix = sub(szm, 1, 1)
1651 if prefix == "/" then -- Match both operand sizes. 2084 if prefix == "/" then -- Exactly match leading operand sizes.
1652 if args[1].opsize == sub(szm, 2, 2) and 2085 for i = #szm,1,-1 do
1653 args[2].opsize == sub(szm, 3, 3) then 2086 if i == 1 then
1654 dopattern(pat, args, sz, params.op, needrex) -- Process pattern. 2087 dopattern(pat, args, sz, params.op, needrex) -- Process pattern.
1655 return 2088 return
2089 elseif args[i-1].opsize ~= sub(szm, i, i) then
2090 break
2091 end
1656 end 2092 end
1657 else -- Match common operand size. 2093 else -- Match common operand size.
1658 local szp = sz 2094 local szp = sz
@@ -1717,8 +2153,8 @@ if x64 then
1717 rex = a.reg > 7 and 9 or 8 2153 rex = a.reg > 7 and 9 or 8
1718 end 2154 end
1719 end 2155 end
1720 wputop(sz, opcode, rex) 2156 local psz, sk = wputop(sz, opcode, rex, nil, vreg)
1721 if vreg then waction("VREG", vreg); wputxb(0) end 2157 wvreg("opcode", vreg, psz, sk)
1722 waction("IMM_D", format("(unsigned int)(%s)", op64)) 2158 waction("IMM_D", format("(unsigned int)(%s)", op64))
1723 waction("IMM_D", format("(unsigned int)((%s)>>32)", op64)) 2159 waction("IMM_D", format("(unsigned int)((%s)>>32)", op64))
1724 end 2160 end
@@ -1730,14 +2166,16 @@ end
1730local function op_data(params) 2166local function op_data(params)
1731 if not params then return "imm..." end 2167 if not params then return "imm..." end
1732 local sz = sub(params.op, 2, 2) 2168 local sz = sub(params.op, 2, 2)
1733 if sz == "a" then sz = addrsize end 2169 if sz == "l" then sz = "d" elseif sz == "a" then sz = addrsize end
1734 for _,p in ipairs(params) do 2170 for _,p in ipairs(params) do
1735 local a = parseoperand(p) 2171 local a = parseoperand(p, sz == "q")
1736 if sub(a.mode, 1, 1) ~= "i" or (a.opsize and a.opsize ~= sz) then 2172 if sub(a.mode, 1, 1) ~= "i" or (a.opsize and a.opsize ~= sz) then
1737 werror("bad mode or size in `"..p.."'") 2173 werror("bad mode or size in `"..p.."'")
1738 end 2174 end
1739 if a.mode == "iJ" then 2175 if a.mode == "iJ" then
1740 wputlabel("IMM_", a.imm, 1) 2176 wputlabel("IMM_", a.imm, 1)
2177 elseif sz == "q" then
2178 wputqarg(a.imm)
1741 else 2179 else
1742 wputszarg(sz, a.imm) 2180 wputszarg(sz, a.imm)
1743 end 2181 end
@@ -1749,7 +2187,11 @@ map_op[".byte_*"] = op_data
1749map_op[".sbyte_*"] = op_data 2187map_op[".sbyte_*"] = op_data
1750map_op[".word_*"] = op_data 2188map_op[".word_*"] = op_data
1751map_op[".dword_*"] = op_data 2189map_op[".dword_*"] = op_data
2190map_op[".qword_*"] = op_data
1752map_op[".aword_*"] = op_data 2191map_op[".aword_*"] = op_data
2192map_op[".long_*"] = op_data
2193map_op[".quad_*"] = op_data
2194map_op[".addr_*"] = op_data
1753 2195
1754------------------------------------------------------------------------------ 2196------------------------------------------------------------------------------
1755 2197
diff --git a/dynasm/dynasm.lua b/dynasm/dynasm.lua
index 068efe2a..5be75f7f 100644
--- a/dynasm/dynasm.lua
+++ b/dynasm/dynasm.lua
@@ -10,9 +10,9 @@
10local _info = { 10local _info = {
11 name = "DynASM", 11 name = "DynASM",
12 description = "A dynamic assembler for code generation engines", 12 description = "A dynamic assembler for code generation engines",
13 version = "1.3.0", 13 version = "1.5.0",
14 vernum = 10300, 14 vernum = 10500,
15 release = "2011-05-05", 15 release = "2021-05-02",
16 author = "Mike Pall", 16 author = "Mike Pall",
17 url = "https://luajit.org/dynasm.html", 17 url = "https://luajit.org/dynasm.html",
18 license = "MIT", 18 license = "MIT",
@@ -630,6 +630,7 @@ end
630-- Load architecture-specific module. 630-- Load architecture-specific module.
631local function loadarch(arch) 631local function loadarch(arch)
632 if not match(arch, "^[%w_]+$") then return "bad arch name" end 632 if not match(arch, "^[%w_]+$") then return "bad arch name" end
633 _G._map_def = map_def
633 local ok, m_arch = pcall(require, "dasm_"..arch) 634 local ok, m_arch = pcall(require, "dasm_"..arch)
634 if not ok then return "cannot load module: "..m_arch end 635 if not ok then return "cannot load module: "..m_arch end
635 g_arch = m_arch 636 g_arch = m_arch
diff --git a/etc/luajit.pc b/etc/luajit.pc
index 68f8d5cc..96433008 100644
--- a/etc/luajit.pc
+++ b/etc/luajit.pc
@@ -1,6 +1,6 @@
1# Package information for LuaJIT to be used by pkg-config. 1# Package information for LuaJIT to be used by pkg-config.
2majver=2 2majver=2
3minver=0 3minver=1
4relver=ROLLING 4relver=ROLLING
5version=${majver}.${minver}.${relver} 5version=${majver}.${minver}.${relver}
6abiver=5.1 6abiver=5.1
diff --git a/src/.gitignore b/src/.gitignore
index 19f2a00e..736a729a 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -6,4 +6,4 @@ lj_ffdef.h
6lj_libdef.h 6lj_libdef.h
7lj_recdef.h 7lj_recdef.h
8lj_folddef.h 8lj_folddef.h
9lj_vm.s 9lj_vm.[sS]
diff --git a/src/Makefile b/src/Makefile
index e6954a95..3a6a4329 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -11,7 +11,7 @@
11############################################################################## 11##############################################################################
12 12
13MAJVER= 2 13MAJVER= 2
14MINVER= 0 14MINVER= 1
15ABIVER= 5.1 15ABIVER= 5.1
16NODOTABIVER= 51 16NODOTABIVER= 51
17 17
@@ -43,17 +43,14 @@ CCOPT= -O2 -fomit-frame-pointer
43# 43#
44# Target-specific compiler options: 44# Target-specific compiler options:
45# 45#
46# x86 only: it's recommended to compile at least for i686. Better yet,
47# compile for an architecture that has SSE2, too (-msse -msse2).
48#
49# x86/x64 only: For GCC 4.2 or higher and if you don't intend to distribute 46# x86/x64 only: For GCC 4.2 or higher and if you don't intend to distribute
50# the binaries to a different machine you could also use: -march=native 47# the binaries to a different machine you could also use: -march=native
51# 48#
52CCOPT_x86= -march=i686 49CCOPT_x86= -march=i686 -msse -msse2 -mfpmath=sse
53CCOPT_x64= 50CCOPT_x64=
54CCOPT_arm= 51CCOPT_arm=
52CCOPT_arm64=
55CCOPT_ppc= 53CCOPT_ppc=
56CCOPT_ppcspe=
57CCOPT_mips= 54CCOPT_mips=
58# 55#
59CCDEBUG= 56CCDEBUG=
@@ -112,6 +109,9 @@ XCFLAGS=
112#XCFLAGS+= -DLUAJIT_NUMMODE=1 109#XCFLAGS+= -DLUAJIT_NUMMODE=1
113#XCFLAGS+= -DLUAJIT_NUMMODE=2 110#XCFLAGS+= -DLUAJIT_NUMMODE=2
114# 111#
112# Disable LJ_GC64 mode for x64.
113#XCFLAGS+= -DLUAJIT_DISABLE_GC64
114#
115############################################################################## 115##############################################################################
116 116
117############################################################################## 117##############################################################################
@@ -123,15 +123,14 @@ XCFLAGS=
123# 123#
124# Use the system provided memory allocator (realloc) instead of the 124# Use the system provided memory allocator (realloc) instead of the
125# bundled memory allocator. This is slower, but sometimes helpful for 125# bundled memory allocator. This is slower, but sometimes helpful for
126# debugging. This option cannot be enabled on x64, since realloc usually 126# debugging. This option cannot be enabled on x64 without GC64, since
127# doesn't return addresses in the right address range. 127# realloc usually doesn't return addresses in the right address range.
128# OTOH this option is mandatory for Valgrind's memcheck tool on x64 and 128# OTOH this option is mandatory for Valgrind's memcheck tool on x64 and
129# the only way to get useful results from it for all other architectures. 129# the only way to get useful results from it for all other architectures.
130#XCFLAGS+= -DLUAJIT_USE_SYSMALLOC 130#XCFLAGS+= -DLUAJIT_USE_SYSMALLOC
131# 131#
132# This define is required to run LuaJIT under Valgrind. The Valgrind 132# This define is required to run LuaJIT under Valgrind. The Valgrind
133# header files must be installed. You should enable debug information, too. 133# header files must be installed. You should enable debug information, too.
134# Use --suppressions=lj.supp to avoid some false positives.
135#XCFLAGS+= -DLUAJIT_USE_VALGRIND 134#XCFLAGS+= -DLUAJIT_USE_VALGRIND
136# 135#
137# This is the client for the GDB JIT API. GDB 7.0 or higher is required 136# This is the client for the GDB JIT API. GDB 7.0 or higher is required
@@ -188,7 +187,8 @@ endif
188# make HOST_CC="gcc -m32" CROSS=i586-mingw32msvc- TARGET_SYS=Windows 187# make HOST_CC="gcc -m32" CROSS=i586-mingw32msvc- TARGET_SYS=Windows
189# make HOST_CC="gcc -m32" CROSS=powerpc-linux-gnu- 188# make HOST_CC="gcc -m32" CROSS=powerpc-linux-gnu-
190 189
191CCOPTIONS= $(CCDEBUG) $(CCOPT) $(CCWARN) $(XCFLAGS) $(CFLAGS) 190ASOPTIONS= $(CCOPT) $(CCWARN) $(XCFLAGS) $(CFLAGS)
191CCOPTIONS= $(CCDEBUG) $(ASOPTIONS)
192LDOPTIONS= $(CCDEBUG) $(LDFLAGS) 192LDOPTIONS= $(CCDEBUG) $(LDFLAGS)
193 193
194HOST_CC= $(CC) 194HOST_CC= $(CC)
@@ -228,6 +228,7 @@ TARGET_XLDFLAGS=
228TARGET_XLIBS= -lm 228TARGET_XLIBS= -lm
229TARGET_TCFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS) 229TARGET_TCFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS)
230TARGET_ACFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS) 230TARGET_ACFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS)
231TARGET_ASFLAGS= $(ASOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS)
231TARGET_ALDFLAGS= $(LDOPTIONS) $(TARGET_XLDFLAGS) $(TARGET_FLAGS) $(TARGET_LDFLAGS) 232TARGET_ALDFLAGS= $(LDOPTIONS) $(TARGET_XLDFLAGS) $(TARGET_FLAGS) $(TARGET_LDFLAGS)
232TARGET_ASHLDFLAGS= $(LDOPTIONS) $(TARGET_XSHLDFLAGS) $(TARGET_FLAGS) $(TARGET_SHLDFLAGS) 233TARGET_ASHLDFLAGS= $(LDOPTIONS) $(TARGET_XSHLDFLAGS) $(TARGET_FLAGS) $(TARGET_SHLDFLAGS)
233TARGET_ALIBS= $(TARGET_XLIBS) $(LIBS) $(TARGET_LIBS) 234TARGET_ALIBS= $(TARGET_XLIBS) $(LIBS) $(TARGET_LIBS)
@@ -242,17 +243,29 @@ else
242ifneq (,$(findstring LJ_TARGET_ARM ,$(TARGET_TESTARCH))) 243ifneq (,$(findstring LJ_TARGET_ARM ,$(TARGET_TESTARCH)))
243 TARGET_LJARCH= arm 244 TARGET_LJARCH= arm
244else 245else
246ifneq (,$(findstring LJ_TARGET_ARM64 ,$(TARGET_TESTARCH)))
247 ifneq (,$(findstring __AARCH64EB__ ,$(TARGET_TESTARCH)))
248 TARGET_ARCH= -D__AARCH64EB__=1
249 endif
250 TARGET_LJARCH= arm64
251else
245ifneq (,$(findstring LJ_TARGET_PPC ,$(TARGET_TESTARCH))) 252ifneq (,$(findstring LJ_TARGET_PPC ,$(TARGET_TESTARCH)))
253 ifneq (,$(findstring LJ_LE 1,$(TARGET_TESTARCH)))
254 TARGET_ARCH= -DLJ_ARCH_ENDIAN=LUAJIT_LE
255 else
256 TARGET_ARCH= -DLJ_ARCH_ENDIAN=LUAJIT_BE
257 endif
246 TARGET_LJARCH= ppc 258 TARGET_LJARCH= ppc
247else 259else
248ifneq (,$(findstring LJ_TARGET_PPCSPE ,$(TARGET_TESTARCH)))
249 TARGET_LJARCH= ppcspe
250else
251ifneq (,$(findstring LJ_TARGET_MIPS ,$(TARGET_TESTARCH))) 260ifneq (,$(findstring LJ_TARGET_MIPS ,$(TARGET_TESTARCH)))
252 ifneq (,$(findstring MIPSEL ,$(TARGET_TESTARCH))) 261 ifneq (,$(findstring MIPSEL ,$(TARGET_TESTARCH)))
253 TARGET_ARCH= -D__MIPSEL__=1 262 TARGET_ARCH= -D__MIPSEL__=1
254 endif 263 endif
255 TARGET_LJARCH= mips 264 ifneq (,$(findstring LJ_TARGET_MIPS64 ,$(TARGET_TESTARCH)))
265 TARGET_LJARCH= mips64
266 else
267 TARGET_LJARCH= mips
268 endif
256else 269else
257 $(error Unsupported target architecture) 270 $(error Unsupported target architecture)
258endif 271endif
@@ -266,6 +279,7 @@ ifneq (,$(findstring LJ_TARGET_PS3 1,$(TARGET_TESTARCH)))
266 TARGET_SYS= PS3 279 TARGET_SYS= PS3
267 TARGET_ARCH+= -D__CELLOS_LV2__ 280 TARGET_ARCH+= -D__CELLOS_LV2__
268 TARGET_XCFLAGS+= -DLUAJIT_USE_SYSMALLOC 281 TARGET_XCFLAGS+= -DLUAJIT_USE_SYSMALLOC
282 TARGET_XLIBS+= -lpthread
269endif 283endif
270 284
271TARGET_XCFLAGS+= $(CCOPT_$(TARGET_LJARCH)) 285TARGET_XCFLAGS+= $(CCOPT_$(TARGET_LJARCH))
@@ -305,20 +319,27 @@ ifeq (Darwin,$(TARGET_SYS))
305 $(error missing: export MACOSX_DEPLOYMENT_TARGET=XX.YY) 319 $(error missing: export MACOSX_DEPLOYMENT_TARGET=XX.YY)
306 endif 320 endif
307 TARGET_STRIP+= -x 321 TARGET_STRIP+= -x
322 TARGET_XCFLAGS+= -DLUAJIT_UNWIND_EXTERNAL
308 TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC 323 TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC
309 TARGET_DYNXLDOPTS= 324 TARGET_DYNXLDOPTS=
310 TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255 325 TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255
311 ifeq (x64,$(TARGET_LJARCH))
312 TARGET_XLDFLAGS+= -pagezero_size 10000 -image_base 100000000
313 TARGET_XSHLDFLAGS+= -image_base 7fff04c4a000
314 endif
315else 326else
316ifeq (iOS,$(TARGET_SYS)) 327ifeq (iOS,$(TARGET_SYS))
317 TARGET_STRIP+= -x 328 TARGET_STRIP+= -x
318 TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC 329 TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC
319 TARGET_DYNXLDOPTS= 330 TARGET_DYNXLDOPTS=
320 TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255 331 TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255
332 ifeq (arm64,$(TARGET_LJARCH))
333 TARGET_XCFLAGS+= -fno-omit-frame-pointer
334 endif
321else 335else
336 ifeq (,$(findstring LJ_NO_UNWIND 1,$(TARGET_TESTARCH)))
337 # Find out whether the target toolchain always generates unwind tables.
338 TARGET_TESTUNWIND=$(shell exec 2>/dev/null; echo 'extern void b(void);int a(void){b();return 0;}' | $(TARGET_CC) -c -x c - -o tmpunwind.o && { grep -qa -e eh_frame -e __unwind_info tmpunwind.o || grep -qU -e eh_frame -e __unwind_info tmpunwind.o; } && echo E; rm -f tmpunwind.o)
339 ifneq (,$(findstring E,$(TARGET_TESTUNWIND)))
340 TARGET_XCFLAGS+= -DLUAJIT_UNWIND_EXTERNAL
341 endif
342 endif
322 ifneq (SunOS,$(TARGET_SYS)) 343 ifneq (SunOS,$(TARGET_SYS))
323 ifneq (PS3,$(TARGET_SYS)) 344 ifneq (PS3,$(TARGET_SYS))
324 TARGET_XLDFLAGS+= -Wl,-E 345 TARGET_XLDFLAGS+= -Wl,-E
@@ -345,7 +366,7 @@ ifneq ($(HOST_SYS),$(TARGET_SYS))
345 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OSX 366 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OSX
346 else 367 else
347 ifeq (iOS,$(TARGET_SYS)) 368 ifeq (iOS,$(TARGET_SYS))
348 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OSX 369 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OSX -DTARGET_OS_IPHONE=1
349 else 370 else
350 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OTHER 371 HOST_XCFLAGS+= -DLUAJIT_OS=LUAJIT_OS_OTHER
351 endif 372 endif
@@ -379,6 +400,11 @@ DASM_XFLAGS=
379DASM_AFLAGS= 400DASM_AFLAGS=
380DASM_ARCH= $(TARGET_LJARCH) 401DASM_ARCH= $(TARGET_LJARCH)
381 402
403ifneq (,$(findstring LJ_LE 1,$(TARGET_TESTARCH)))
404 DASM_AFLAGS+= -D ENDIAN_LE
405else
406 DASM_AFLAGS+= -D ENDIAN_BE
407endif
382ifneq (,$(findstring LJ_ARCH_BITS 64,$(TARGET_TESTARCH))) 408ifneq (,$(findstring LJ_ARCH_BITS 64,$(TARGET_TESTARCH)))
383 DASM_AFLAGS+= -D P64 409 DASM_AFLAGS+= -D P64
384endif 410endif
@@ -407,23 +433,27 @@ ifneq (,$(findstring LJ_NO_UNWIND 1,$(TARGET_TESTARCH)))
407 DASM_AFLAGS+= -D NO_UNWIND 433 DASM_AFLAGS+= -D NO_UNWIND
408 TARGET_ARCH+= -DLUAJIT_NO_UNWIND 434 TARGET_ARCH+= -DLUAJIT_NO_UNWIND
409endif 435endif
436ifneq (,$(findstring LJ_ABI_PAUTH 1,$(TARGET_TESTARCH)))
437 DASM_AFLAGS+= -D PAUTH
438 TARGET_ARCH+= -DLJ_ABI_PAUTH=1
439endif
410DASM_AFLAGS+= -D VER=$(subst LJ_ARCH_VERSION_,,$(filter LJ_ARCH_VERSION_%,$(subst LJ_ARCH_VERSION ,LJ_ARCH_VERSION_,$(TARGET_TESTARCH)))) 440DASM_AFLAGS+= -D VER=$(subst LJ_ARCH_VERSION_,,$(filter LJ_ARCH_VERSION_%,$(subst LJ_ARCH_VERSION ,LJ_ARCH_VERSION_,$(TARGET_TESTARCH))))
411ifeq (Windows,$(TARGET_SYS)) 441ifeq (Windows,$(TARGET_SYS))
412 DASM_AFLAGS+= -D WIN 442 DASM_AFLAGS+= -D WIN
413endif 443endif
414ifeq (x86,$(TARGET_LJARCH))
415 ifneq (,$(findstring __SSE2__ 1,$(TARGET_TESTARCH)))
416 DASM_AFLAGS+= -D SSE
417 endif
418else
419ifeq (x64,$(TARGET_LJARCH)) 444ifeq (x64,$(TARGET_LJARCH))
420 DASM_ARCH= x86 445 ifeq (,$(findstring LJ_FR2 1,$(TARGET_TESTARCH)))
446 DASM_ARCH= x86
447 endif
421else 448else
422ifeq (arm,$(TARGET_LJARCH)) 449ifeq (arm,$(TARGET_LJARCH))
423 ifeq (iOS,$(TARGET_SYS)) 450 ifeq (iOS,$(TARGET_SYS))
424 DASM_AFLAGS+= -D IOS 451 DASM_AFLAGS+= -D IOS
425 endif 452 endif
426else 453else
454ifneq (,$(findstring LJ_TARGET_MIPSR6 ,$(TARGET_TESTARCH)))
455 DASM_AFLAGS+= -D MIPSR6
456endif
427ifeq (ppc,$(TARGET_LJARCH)) 457ifeq (ppc,$(TARGET_LJARCH))
428 ifneq (,$(findstring LJ_ARCH_SQRT 1,$(TARGET_TESTARCH))) 458 ifneq (,$(findstring LJ_ARCH_SQRT 1,$(TARGET_TESTARCH)))
429 DASM_AFLAGS+= -D SQRT 459 DASM_AFLAGS+= -D SQRT
@@ -431,7 +461,7 @@ ifeq (ppc,$(TARGET_LJARCH))
431 ifneq (,$(findstring LJ_ARCH_ROUND 1,$(TARGET_TESTARCH))) 461 ifneq (,$(findstring LJ_ARCH_ROUND 1,$(TARGET_TESTARCH)))
432 DASM_AFLAGS+= -D ROUND 462 DASM_AFLAGS+= -D ROUND
433 endif 463 endif
434 ifneq (,$(findstring LJ_ARCH_PPC64 1,$(TARGET_TESTARCH))) 464 ifneq (,$(findstring LJ_ARCH_PPC32ON64 1,$(TARGET_TESTARCH)))
435 DASM_AFLAGS+= -D GPR64 465 DASM_AFLAGS+= -D GPR64
436 endif 466 endif
437 ifeq (PS3,$(TARGET_SYS)) 467 ifeq (PS3,$(TARGET_SYS))
@@ -440,7 +470,6 @@ ifeq (ppc,$(TARGET_LJARCH))
440endif 470endif
441endif 471endif
442endif 472endif
443endif
444 473
445DASM_FLAGS= $(DASM_XFLAGS) $(DASM_AFLAGS) 474DASM_FLAGS= $(DASM_XFLAGS) $(DASM_AFLAGS)
446DASM_DASC= vm_$(DASM_ARCH).dasc 475DASM_DASC= vm_$(DASM_ARCH).dasc
@@ -457,19 +486,22 @@ BUILDVM_X= $(BUILDVM_T)
457HOST_O= $(MINILUA_O) $(BUILDVM_O) 486HOST_O= $(MINILUA_O) $(BUILDVM_O)
458HOST_T= $(MINILUA_T) $(BUILDVM_T) 487HOST_T= $(MINILUA_T) $(BUILDVM_T)
459 488
460LJVM_S= lj_vm.s 489LJVM_S= lj_vm.S
461LJVM_O= lj_vm.o 490LJVM_O= lj_vm.o
462LJVM_BOUT= $(LJVM_S) 491LJVM_BOUT= $(LJVM_S)
463LJVM_MODE= elfasm 492LJVM_MODE= elfasm
464 493
465LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \ 494LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \
466 lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o 495 lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o \
496 lib_buffer.o
467LJLIB_C= $(LJLIB_O:.o=.c) 497LJLIB_C= $(LJLIB_O:.o=.c)
468 498
469LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o \ 499LJCORE_O= lj_assert.o lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \
470 lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \ 500 lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \
471 lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o lj_strscan.o \ 501 lj_prng.o lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o \
472 lj_api.o lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \ 502 lj_strscan.o lj_strfmt.o lj_strfmt_num.o lj_serialize.o \
503 lj_api.o lj_profile.o \
504 lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \
473 lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \ 505 lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \
474 lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \ 506 lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \
475 lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \ 507 lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \
@@ -584,12 +616,15 @@ E= @echo
584default all: $(TARGET_T) 616default all: $(TARGET_T)
585 617
586amalg: 618amalg:
587 @grep "^[+|]" ljamalg.c
588 $(MAKE) all "LJCORE_O=ljamalg.o" 619 $(MAKE) all "LJCORE_O=ljamalg.o"
589 620
590clean: 621clean:
591 $(HOST_RM) $(ALL_RM) 622 $(HOST_RM) $(ALL_RM)
592 623
624libbc:
625 ./$(LUAJIT_T) host/genlibbc.lua -o host/buildvm_libbc.h $(LJLIB_C)
626 $(MAKE) all
627
593depend: 628depend:
594 @for file in $(ALL_HDRGEN); do \ 629 @for file in $(ALL_HDRGEN); do \
595 test -f $$file || touch $$file; \ 630 test -f $$file || touch $$file; \
@@ -604,7 +639,7 @@ depend:
604 test -s $$file || $(HOST_RM) $$file; \ 639 test -s $$file || $(HOST_RM) $$file; \
605 done 640 done
606 641
607.PHONY: default all amalg clean depend 642.PHONY: default all amalg clean libbc depend
608 643
609############################################################################## 644##############################################################################
610# Rules for generated files. 645# Rules for generated files.
@@ -666,10 +701,10 @@ lj_folddef.h: $(BUILDVM_T) lj_opt_fold.c
666 $(Q)$(TARGET_DYNCC) $(TARGET_ACFLAGS) -c -o $(@:.o=_dyn.o) $< 701 $(Q)$(TARGET_DYNCC) $(TARGET_ACFLAGS) -c -o $(@:.o=_dyn.o) $<
667 $(Q)$(TARGET_CC) $(TARGET_ACFLAGS) -c -o $@ $< 702 $(Q)$(TARGET_CC) $(TARGET_ACFLAGS) -c -o $@ $<
668 703
669%.o: %.s 704%.o: %.S
670 $(E) "ASM $@" 705 $(E) "ASM $@"
671 $(Q)$(TARGET_DYNCC) $(TARGET_ACFLAGS) -c -o $(@:.o=_dyn.o) $< 706 $(Q)$(TARGET_DYNCC) $(TARGET_ASFLAGS) -c -o $(@:.o=_dyn.o) $<
672 $(Q)$(TARGET_CC) $(TARGET_ACFLAGS) -c -o $@ $< 707 $(Q)$(TARGET_CC) $(TARGET_ASFLAGS) -c -o $@ $<
673 708
674$(LUAJIT_O): 709$(LUAJIT_O):
675 $(E) "CC $@" 710 $(E) "CC $@"
diff --git a/src/Makefile.dep b/src/Makefile.dep
index 9e14d617..400ef8b0 100644
--- a/src/Makefile.dep
+++ b/src/Makefile.dep
@@ -1,66 +1,79 @@
1lib_aux.o: lib_aux.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \ 1lib_aux.o: lib_aux.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \
2 lj_arch.h lj_err.h lj_errmsg.h lj_state.h lj_trace.h lj_jit.h lj_ir.h \ 2 lj_arch.h lj_err.h lj_errmsg.h lj_state.h lj_trace.h lj_jit.h lj_ir.h \
3 lj_dispatch.h lj_bc.h lj_traceerr.h lj_lib.h lj_alloc.h 3 lj_dispatch.h lj_bc.h lj_traceerr.h lj_lib.h lj_vmevent.h
4lib_base.o: lib_base.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 4lib_base.o: lib_base.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
5 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h \ 5 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_buf.h \
6 lj_tab.h lj_meta.h lj_state.h lj_ctype.h lj_cconv.h lj_bc.h lj_ff.h \ 6 lj_str.h lj_tab.h lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h \
7 lj_ffdef.h lj_dispatch.h lj_jit.h lj_ir.h lj_char.h lj_strscan.h \ 7 lj_cconv.h lj_ff.h lj_ffdef.h lj_dispatch.h lj_jit.h lj_ir.h lj_char.h \
8 lj_lib.h lj_libdef.h 8 lj_strscan.h lj_strfmt.h lj_lib.h lj_libdef.h
9lib_bit.o: lib_bit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ 9lib_bit.o: lib_bit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
10 lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_lib.h lj_libdef.h 10 lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_strscan.h \
11 lj_strfmt.h lj_ctype.h lj_cdata.h lj_cconv.h lj_carith.h lj_ff.h \
12 lj_ffdef.h lj_lib.h lj_libdef.h
13lib_buffer.o: lib_buffer.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
14 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \
15 lj_tab.h lj_udata.h lj_meta.h lj_ctype.h lj_cdata.h lj_cconv.h \
16 lj_strfmt.h lj_serialize.h lj_lib.h lj_libdef.h
11lib_debug.o: lib_debug.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 17lib_debug.o: lib_debug.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
12 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_lib.h \ 18 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_lib.h \
13 lj_libdef.h 19 lj_libdef.h
14lib_ffi.o: lib_ffi.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ 20lib_ffi.o: lib_ffi.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
15 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h \ 21 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h \
16 lj_ctype.h lj_cparse.h lj_cdata.h lj_cconv.h lj_carith.h lj_ccall.h \ 22 lj_ctype.h lj_cparse.h lj_cdata.h lj_cconv.h lj_carith.h lj_ccall.h \
17 lj_ccallback.h lj_clib.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h 23 lj_ccallback.h lj_clib.h lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h \
24 lj_libdef.h
18lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h 25lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h
19lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ 26lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
20 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_ff.h \ 27 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_state.h \
21 lj_ffdef.h lj_lib.h lj_libdef.h 28 lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
22lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h \ 29lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
23 lj_obj.h lj_def.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h \ 30 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h \
24 lj_bc.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_target.h \ 31 lj_state.h lj_bc.h lj_ctype.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
25 lj_target_*.h lj_dispatch.h lj_vm.h lj_vmevent.h lj_lib.h luajit.h \ 32 lj_target.h lj_target_*.h lj_trace.h lj_dispatch.h lj_traceerr.h \
26 lj_libdef.h 33 lj_vm.h lj_vmevent.h lj_lib.h luajit.h lj_libdef.h
27lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 34lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
28 lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_libdef.h 35 lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_prng.h lj_libdef.h
29lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ 36lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
30 lj_arch.h lj_err.h lj_errmsg.h lj_lib.h lj_libdef.h 37 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_lib.h \
38 lj_libdef.h
31lib_package.o: lib_package.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 39lib_package.o: lib_package.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
32 lj_def.h lj_arch.h lj_err.h lj_errmsg.h lj_lib.h 40 lj_def.h lj_arch.h lj_err.h lj_errmsg.h lj_lib.h
33lib_string.o: lib_string.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 41lib_string.o: lib_string.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
34 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h \ 42 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \
35 lj_meta.h lj_state.h lj_ff.h lj_ffdef.h lj_bcdump.h lj_lex.h lj_char.h \ 43 lj_tab.h lj_meta.h lj_state.h lj_ff.h lj_ffdef.h lj_bcdump.h lj_lex.h \
36 lj_lib.h lj_libdef.h 44 lj_char.h lj_strfmt.h lj_lib.h lj_libdef.h
37lib_table.o: lib_table.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ 45lib_table.o: lib_table.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
38 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_lib.h \ 46 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \
39 lj_libdef.h 47 lj_tab.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
40lj_alloc.o: lj_alloc.c lj_def.h lua.h luaconf.h lj_arch.h lj_alloc.h 48lj_alloc.o: lj_alloc.c lj_def.h lua.h luaconf.h lj_arch.h lj_alloc.h \
49 lj_prng.h
41lj_api.o: lj_api.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 50lj_api.o: lj_api.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
42 lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_func.h lj_udata.h \ 51 lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_func.h lj_udata.h \
43 lj_meta.h lj_state.h lj_bc.h lj_frame.h lj_trace.h lj_jit.h lj_ir.h \ 52 lj_meta.h lj_state.h lj_bc.h lj_frame.h lj_trace.h lj_jit.h lj_ir.h \
44 lj_dispatch.h lj_traceerr.h lj_vm.h lj_strscan.h 53 lj_dispatch.h lj_traceerr.h lj_vm.h lj_strscan.h lj_strfmt.h
45lj_asm.o: lj_asm.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 54lj_asm.o: lj_asm.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
46 lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h lj_ir.h lj_jit.h \ 55 lj_buf.h lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h lj_ir.h \
47 lj_ircall.h lj_iropt.h lj_mcode.h lj_trace.h lj_dispatch.h lj_traceerr.h \ 56 lj_jit.h lj_ircall.h lj_iropt.h lj_mcode.h lj_trace.h lj_dispatch.h \
48 lj_snap.h lj_asm.h lj_vm.h lj_target.h lj_target_*.h lj_emit_*.h \ 57 lj_traceerr.h lj_snap.h lj_asm.h lj_vm.h lj_target.h lj_target_*.h \
49 lj_asm_*.h 58 lj_emit_*.h lj_asm_*.h
59lj_assert.o: lj_assert.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
50lj_bc.o: lj_bc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_bc.h \ 60lj_bc.o: lj_bc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_bc.h \
51 lj_bcdef.h 61 lj_bcdef.h
52lj_bcread.o: lj_bcread.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 62lj_bcread.o: lj_bcread.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
53 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_bc.h lj_ctype.h \ 63 lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_bc.h \
54 lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h 64 lj_ctype.h lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h \
65 lj_strfmt.h
55lj_bcwrite.o: lj_bcwrite.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 66lj_bcwrite.o: lj_bcwrite.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
56 lj_gc.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h lj_ir.h \ 67 lj_gc.h lj_buf.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h \
57 lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h 68 lj_ir.h lj_strfmt.h lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h
69lj_buf.o: lj_buf.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
70 lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_strfmt.h
58lj_carith.o: lj_carith.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 71lj_carith.o: lj_carith.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
59 lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_meta.h lj_ctype.h lj_cconv.h \ 72 lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_meta.h lj_ir.h lj_ctype.h \
60 lj_cdata.h lj_carith.h 73 lj_cconv.h lj_cdata.h lj_carith.h lj_strscan.h
61lj_ccall.o: lj_ccall.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 74lj_ccall.o: lj_ccall.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
62 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cconv.h \ 75 lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h \
63 lj_cdata.h lj_ccall.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \ 76 lj_ccall.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
64 lj_traceerr.h 77 lj_traceerr.h
65lj_ccallback.o: lj_ccallback.c lj_obj.h lua.h luaconf.h lj_def.h \ 78lj_ccallback.o: lj_ccallback.c lj_obj.h lua.h luaconf.h lj_def.h \
66 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_state.h lj_frame.h \ 79 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_state.h lj_frame.h \
@@ -68,110 +81,127 @@ lj_ccallback.o: lj_ccallback.c lj_obj.h lua.h luaconf.h lj_def.h \
68 lj_target_*.h lj_mcode.h lj_jit.h lj_ir.h lj_trace.h lj_dispatch.h \ 81 lj_target_*.h lj_mcode.h lj_jit.h lj_ir.h lj_trace.h lj_dispatch.h \
69 lj_traceerr.h lj_vm.h 82 lj_traceerr.h lj_vm.h
70lj_cconv.o: lj_cconv.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 83lj_cconv.o: lj_cconv.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
71 lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_gc.h lj_cdata.h lj_cconv.h \ 84 lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_ctype.h \
72 lj_ccallback.h 85 lj_cdata.h lj_cconv.h lj_ccallback.h
73lj_cdata.o: lj_cdata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 86lj_cdata.o: lj_cdata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
74 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cconv.h \ 87 lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h
75 lj_cdata.h
76lj_char.o: lj_char.c lj_char.h lj_def.h lua.h luaconf.h 88lj_char.o: lj_char.c lj_char.h lj_def.h lua.h luaconf.h
77lj_clib.o: lj_clib.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 89lj_clib.o: lj_clib.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
78 lj_err.h lj_errmsg.h lj_tab.h lj_str.h lj_udata.h lj_ctype.h lj_cconv.h \ 90 lj_err.h lj_errmsg.h lj_tab.h lj_str.h lj_udata.h lj_ctype.h lj_cconv.h \
79 lj_cdata.h lj_clib.h 91 lj_cdata.h lj_clib.h lj_strfmt.h
80lj_cparse.o: lj_cparse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 92lj_cparse.o: lj_cparse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
81 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_ctype.h lj_cparse.h lj_frame.h \ 93 lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_ctype.h lj_cparse.h \
82 lj_bc.h lj_vm.h lj_char.h lj_strscan.h 94 lj_frame.h lj_bc.h lj_vm.h lj_char.h lj_strscan.h lj_strfmt.h
83lj_crecord.o: lj_crecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 95lj_crecord.o: lj_crecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
84 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h \ 96 lj_err.h lj_errmsg.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h lj_gc.h \
85 lj_gc.h lj_cdata.h lj_cparse.h lj_cconv.h lj_clib.h lj_ccall.h lj_ff.h \ 97 lj_cdata.h lj_cparse.h lj_cconv.h lj_carith.h lj_clib.h lj_ccall.h \
86 lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \ 98 lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
87 lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_snap.h \ 99 lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_snap.h \
88 lj_crecord.h 100 lj_crecord.h lj_strfmt.h
89lj_ctype.o: lj_ctype.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 101lj_ctype.o: lj_ctype.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
90 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_ccallback.h 102 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_strfmt.h lj_ctype.h \
103 lj_ccallback.h lj_buf.h
91lj_debug.o: lj_debug.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 104lj_debug.o: lj_debug.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
92 lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_state.h lj_frame.h \ 105 lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_gc.h lj_str.h lj_tab.h \
93 lj_bc.h lj_vm.h lj_jit.h lj_ir.h 106 lj_state.h lj_frame.h lj_bc.h lj_strfmt.h lj_jit.h lj_ir.h
94lj_dispatch.o: lj_dispatch.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 107lj_dispatch.o: lj_dispatch.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
95 lj_err.h lj_errmsg.h lj_func.h lj_str.h lj_tab.h lj_meta.h lj_debug.h \ 108 lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_func.h lj_tab.h \
96 lj_state.h lj_frame.h lj_bc.h lj_ff.h lj_ffdef.h lj_jit.h lj_ir.h \ 109 lj_meta.h lj_debug.h lj_state.h lj_frame.h lj_bc.h lj_ff.h lj_ffdef.h \
97 lj_ccallback.h lj_ctype.h lj_gc.h lj_trace.h lj_dispatch.h lj_traceerr.h \ 110 lj_strfmt.h lj_jit.h lj_ir.h lj_ccallback.h lj_ctype.h lj_trace.h \
98 lj_vm.h luajit.h 111 lj_dispatch.h lj_traceerr.h lj_profile.h lj_vm.h luajit.h
99lj_err.o: lj_err.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_err.h \ 112lj_err.o: lj_err.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_err.h \
100 lj_errmsg.h lj_debug.h lj_str.h lj_func.h lj_state.h lj_frame.h lj_bc.h \ 113 lj_errmsg.h lj_debug.h lj_str.h lj_func.h lj_state.h lj_frame.h lj_bc.h \
101 lj_ff.h lj_ffdef.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h \ 114 lj_ff.h lj_ffdef.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h \
102 lj_traceerr.h lj_vm.h 115 lj_traceerr.h lj_vm.h lj_strfmt.h
103lj_ffrecord.o: lj_ffrecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 116lj_ffrecord.o: lj_ffrecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
104 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ff.h \ 117 lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_frame.h \
105 lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \ 118 lj_bc.h lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
106 lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_crecord.h \ 119 lj_trace.h lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h \
107 lj_vm.h lj_strscan.h lj_recdef.h 120 lj_crecord.h lj_vm.h lj_strscan.h lj_strfmt.h lj_serialize.h lj_recdef.h
108lj_func.o: lj_func.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 121lj_func.o: lj_func.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
109 lj_func.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \ 122 lj_func.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
110 lj_traceerr.h lj_vm.h 123 lj_traceerr.h lj_vm.h
111lj_gc.o: lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 124lj_gc.o: lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
112 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_udata.h lj_meta.h \ 125 lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h lj_udata.h \
113 lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h lj_jit.h \ 126 lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h \
114 lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h 127 lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_vmevent.h
115lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 128lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
116 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_jit.h \ 129 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_buf.h \
117 lj_ir.h lj_dispatch.h 130 lj_str.h lj_strfmt.h lj_jit.h lj_ir.h lj_dispatch.h
118lj_ir.o: lj_ir.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 131lj_ir.o: lj_ir.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
119 lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \ 132 lj_buf.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
120 lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h lj_carith.h \ 133 lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h \
121 lj_vm.h lj_strscan.h lj_lib.h 134 lj_carith.h lj_vm.h lj_strscan.h lj_serialize.h lj_strfmt.h lj_prng.h
122lj_lex.o: lj_lex.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 135lj_lex.o: lj_lex.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
123 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cdata.h lualib.h \ 136 lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_ctype.h lj_cdata.h \
124 lj_state.h lj_lex.h lj_parse.h lj_char.h lj_strscan.h 137 lualib.h lj_state.h lj_lex.h lj_parse.h lj_char.h lj_strscan.h \
138 lj_strfmt.h
125lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \ 139lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \
126 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_bc.h \ 140 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_bc.h \
127 lj_dispatch.h lj_jit.h lj_ir.h lj_vm.h lj_strscan.h lj_lib.h 141 lj_dispatch.h lj_jit.h lj_ir.h lj_ctype.h lj_vm.h lj_strscan.h \
142 lj_strfmt.h lj_lex.h lj_bcdump.h lj_lib.h
128lj_load.o: lj_load.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \ 143lj_load.o: lj_load.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \
129 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_func.h lj_frame.h \ 144 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_func.h \
130 lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h 145 lj_frame.h lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h
131lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 146lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
132 lj_gc.h lj_err.h lj_errmsg.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h \ 147 lj_gc.h lj_err.h lj_errmsg.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h \
133 lj_dispatch.h lj_bc.h lj_traceerr.h lj_vm.h 148 lj_dispatch.h lj_bc.h lj_traceerr.h lj_prng.h lj_vm.h
134lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 149lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
135 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \ 150 lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_meta.h lj_frame.h \
136 lj_vm.h lj_strscan.h 151 lj_bc.h lj_vm.h lj_strscan.h lj_strfmt.h lj_lib.h
137lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h 152lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
138lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 153lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
139 lj_ir.h lj_jit.h lj_iropt.h 154 lj_ir.h lj_jit.h lj_iropt.h
140lj_opt_fold.o: lj_opt_fold.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 155lj_opt_fold.o: lj_opt_fold.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
141 lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \ 156 lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h \
142 lj_bc.h lj_traceerr.h lj_ctype.h lj_gc.h lj_carith.h lj_vm.h \ 157 lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h \
143 lj_strscan.h lj_folddef.h 158 lj_carith.h lj_vm.h lj_strscan.h lj_strfmt.h lj_folddef.h
144lj_opt_loop.o: lj_opt_loop.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 159lj_opt_loop.o: lj_opt_loop.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
145 lj_err.h lj_errmsg.h lj_str.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h \ 160 lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h lj_jit.h \
146 lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h lj_vm.h 161 lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h \
162 lj_vm.h
147lj_opt_mem.o: lj_opt_mem.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 163lj_opt_mem.o: lj_opt_mem.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
148 lj_tab.h lj_ir.h lj_jit.h lj_iropt.h 164 lj_tab.h lj_ir.h lj_jit.h lj_iropt.h lj_ircall.h lj_dispatch.h lj_bc.h
149lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \ 165lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
150 lj_arch.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \ 166 lj_arch.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \
151 lj_traceerr.h lj_vm.h lj_strscan.h 167 lj_traceerr.h lj_vm.h lj_strscan.h
152lj_opt_sink.o: lj_opt_sink.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 168lj_opt_sink.o: lj_opt_sink.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
153 lj_ir.h lj_jit.h lj_iropt.h lj_target.h lj_target_*.h 169 lj_ir.h lj_jit.h lj_iropt.h lj_target.h lj_target_*.h
154lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \ 170lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \
155 lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_ir.h lj_jit.h lj_ircall.h \ 171 lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h \
156 lj_iropt.h lj_vm.h 172 lj_jit.h lj_ircall.h lj_iropt.h lj_dispatch.h lj_bc.h lj_vm.h
157lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 173lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
158 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_func.h \ 174 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_str.h lj_tab.h \
159 lj_state.h lj_bc.h lj_ctype.h lj_lex.h lj_parse.h lj_vm.h lj_vmevent.h 175 lj_func.h lj_state.h lj_bc.h lj_ctype.h lj_strfmt.h lj_lex.h lj_parse.h \
176 lj_vm.h lj_vmevent.h
177lj_prng.o: lj_prng.c lj_def.h lua.h luaconf.h lj_arch.h lj_prng.h
178lj_profile.o: lj_profile.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
179 lj_buf.h lj_gc.h lj_str.h lj_frame.h lj_bc.h lj_debug.h lj_dispatch.h \
180 lj_jit.h lj_ir.h lj_trace.h lj_traceerr.h lj_profile.h luajit.h
160lj_record.o: lj_record.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 181lj_record.o: lj_record.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
161 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \ 182 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
162 lj_ctype.h lj_gc.h lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h \ 183 lj_ctype.h lj_gc.h lj_ff.h lj_ffdef.h lj_debug.h lj_ir.h lj_jit.h \
163 lj_iropt.h lj_trace.h lj_dispatch.h lj_traceerr.h lj_record.h \ 184 lj_ircall.h lj_iropt.h lj_trace.h lj_dispatch.h lj_traceerr.h \
164 lj_ffrecord.h lj_snap.h lj_vm.h 185 lj_record.h lj_ffrecord.h lj_snap.h lj_vm.h lj_prng.h
186lj_serialize.o: lj_serialize.c lj_obj.h lua.h luaconf.h lj_def.h \
187 lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_tab.h \
188 lj_udata.h lj_ctype.h lj_cdata.h lj_ir.h lj_serialize.h
165lj_snap.o: lj_snap.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 189lj_snap.o: lj_snap.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
166 lj_tab.h lj_state.h lj_frame.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h \ 190 lj_tab.h lj_state.h lj_frame.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h \
167 lj_trace.h lj_dispatch.h lj_traceerr.h lj_snap.h lj_target.h \ 191 lj_trace.h lj_dispatch.h lj_traceerr.h lj_snap.h lj_target.h \
168 lj_target_*.h lj_ctype.h lj_cdata.h 192 lj_target_*.h lj_ctype.h lj_cdata.h
169lj_state.o: lj_state.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 193lj_state.o: lj_state.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
170 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_meta.h \ 194 lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h \
171 lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_trace.h lj_jit.h lj_ir.h \ 195 lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_trace.h lj_jit.h \
172 lj_dispatch.h lj_traceerr.h lj_vm.h lj_lex.h lj_alloc.h 196 lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_prng.h lj_lex.h \
197 lj_alloc.h luajit.h
173lj_str.o: lj_str.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 198lj_str.o: lj_str.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
174 lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_char.h 199 lj_err.h lj_errmsg.h lj_str.h lj_char.h lj_prng.h
200lj_strfmt.o: lj_strfmt.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
201 lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_meta.h lj_state.h \
202 lj_char.h lj_strfmt.h lj_ctype.h lj_lib.h
203lj_strfmt_num.o: lj_strfmt_num.c lj_obj.h lua.h luaconf.h lj_def.h \
204 lj_arch.h lj_buf.h lj_gc.h lj_str.h lj_strfmt.h
175lj_strscan.o: lj_strscan.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 205lj_strscan.o: lj_strscan.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
176 lj_char.h lj_strscan.h 206 lj_char.h lj_strscan.h
177lj_tab.o: lj_tab.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ 207lj_tab.o: lj_tab.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
@@ -180,35 +210,37 @@ lj_trace.o: lj_trace.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
180 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_frame.h lj_bc.h \ 210 lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_frame.h lj_bc.h \
181 lj_state.h lj_ir.h lj_jit.h lj_iropt.h lj_mcode.h lj_trace.h \ 211 lj_state.h lj_ir.h lj_jit.h lj_iropt.h lj_mcode.h lj_trace.h \
182 lj_dispatch.h lj_traceerr.h lj_snap.h lj_gdbjit.h lj_record.h lj_asm.h \ 212 lj_dispatch.h lj_traceerr.h lj_snap.h lj_gdbjit.h lj_record.h lj_asm.h \
183 lj_vm.h lj_vmevent.h lj_target.h lj_target_*.h 213 lj_vm.h lj_vmevent.h lj_target.h lj_target_*.h lj_prng.h
184lj_udata.o: lj_udata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 214lj_udata.o: lj_udata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
185 lj_gc.h lj_udata.h 215 lj_gc.h lj_err.h lj_errmsg.h lj_udata.h
186lj_vmevent.o: lj_vmevent.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 216lj_vmevent.o: lj_vmevent.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
187 lj_str.h lj_tab.h lj_state.h lj_dispatch.h lj_bc.h lj_jit.h lj_ir.h \ 217 lj_str.h lj_tab.h lj_state.h lj_dispatch.h lj_bc.h lj_jit.h lj_ir.h \
188 lj_vm.h lj_vmevent.h 218 lj_vm.h lj_vmevent.h
189lj_vmmath.o: lj_vmmath.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ 219lj_vmmath.o: lj_vmmath.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
190 lj_ir.h lj_vm.h 220 lj_ir.h lj_vm.h
191ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \ 221ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_assert.c lj_obj.h \
192 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h \ 222 lj_def.h lj_arch.h lj_gc.c lj_gc.h lj_err.h lj_errmsg.h lj_buf.h \
193 lj_udata.h lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h \ 223 lj_str.h lj_tab.h lj_func.h lj_udata.h lj_meta.h lj_state.h lj_frame.h \
194 lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_err.c \ 224 lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h \
195 lj_debug.h lj_ff.h lj_ffdef.h lj_char.c lj_char.h lj_bc.c lj_bcdef.h \ 225 lj_traceerr.h lj_vm.h lj_vmevent.h lj_err.c lj_debug.h lj_ff.h \
196 lj_obj.c lj_str.c lj_tab.c lj_func.c lj_udata.c lj_meta.c lj_strscan.h \ 226 lj_ffdef.h lj_strfmt.h lj_char.c lj_char.h lj_bc.c lj_bcdef.h lj_obj.c \
197 lj_debug.c lj_state.c lj_lex.h lj_alloc.h lj_dispatch.c lj_ccallback.h \ 227 lj_buf.c lj_str.c lj_prng.h lj_tab.c lj_func.c lj_udata.c lj_meta.c \
198 luajit.h lj_vmevent.c lj_vmevent.h lj_vmmath.c lj_strscan.c lj_api.c \ 228 lj_strscan.h lj_lib.h lj_debug.c lj_prng.c lj_state.c lj_lex.h \
199 lj_lex.c lualib.h lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h \ 229 lj_alloc.h luajit.h lj_dispatch.c lj_ccallback.h lj_profile.h \
200 lj_bcwrite.c lj_load.c lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c \ 230 lj_vmevent.c lj_vmmath.c lj_strscan.c lj_strfmt.c lj_strfmt_num.c \
201 lj_ccall.c lj_ccall.h lj_ccallback.c lj_target.h lj_target_*.h \ 231 lj_serialize.c lj_serialize.h lj_api.c lj_profile.c lj_lex.c lualib.h \
202 lj_mcode.h lj_carith.c lj_carith.h lj_clib.c lj_clib.h lj_cparse.c \ 232 lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h lj_bcwrite.c lj_load.c \
203 lj_cparse.h lj_lib.c lj_lib.h lj_ir.c lj_ircall.h lj_iropt.h \ 233 lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c lj_ccall.c lj_ccall.h \
204 lj_opt_mem.c lj_opt_fold.c lj_folddef.h lj_opt_narrow.c lj_opt_dce.c \ 234 lj_ccallback.c lj_target.h lj_target_*.h lj_mcode.h lj_carith.c \
205 lj_opt_loop.c lj_snap.h lj_opt_split.c lj_opt_sink.c lj_mcode.c \ 235 lj_carith.h lj_clib.c lj_clib.h lj_cparse.c lj_cparse.h lj_lib.c lj_ir.c \
206 lj_snap.c lj_record.c lj_record.h lj_ffrecord.h lj_crecord.c \ 236 lj_ircall.h lj_iropt.h lj_opt_mem.c lj_opt_fold.c lj_folddef.h \
207 lj_crecord.h lj_ffrecord.c lj_recdef.h lj_asm.c lj_asm.h lj_emit_*.h \ 237 lj_opt_narrow.c lj_opt_dce.c lj_opt_loop.c lj_snap.h lj_opt_split.c \
208 lj_asm_*.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c lib_aux.c \ 238 lj_opt_sink.c lj_mcode.c lj_snap.c lj_record.c lj_record.h lj_ffrecord.h \
209 lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c lib_io.c \ 239 lj_crecord.c lj_crecord.h lj_ffrecord.c lj_recdef.h lj_asm.c lj_asm.h \
210 lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c lib_ffi.c \ 240 lj_emit_*.h lj_asm_*.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c \
211 lib_init.c 241 lib_aux.c lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c \
242 lib_io.c lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c \
243 lib_ffi.c lib_buffer.c lib_init.c
212luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h 244luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h
213host/buildvm.o: host/buildvm.c host/buildvm.h lj_def.h lua.h luaconf.h \ 245host/buildvm.o: host/buildvm.c host/buildvm.h lj_def.h lua.h luaconf.h \
214 lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_gc.h lj_obj.h lj_bc.h lj_ir.h \ 246 lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_gc.h lj_obj.h lj_bc.h lj_ir.h \
@@ -220,7 +252,8 @@ host/buildvm_asm.o: host/buildvm_asm.c host/buildvm.h lj_def.h lua.h luaconf.h \
220host/buildvm_fold.o: host/buildvm_fold.c host/buildvm.h lj_def.h lua.h \ 252host/buildvm_fold.o: host/buildvm_fold.c host/buildvm.h lj_def.h lua.h \
221 luaconf.h lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_ir.h lj_obj.h 253 luaconf.h lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_ir.h lj_obj.h
222host/buildvm_lib.o: host/buildvm_lib.c host/buildvm.h lj_def.h lua.h luaconf.h \ 254host/buildvm_lib.o: host/buildvm_lib.c host/buildvm.h lj_def.h lua.h luaconf.h \
223 lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_lib.h lj_obj.h 255 lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_bc.h lj_lib.h lj_obj.h \
256 host/buildvm_libbc.h
224host/buildvm_peobj.o: host/buildvm_peobj.c host/buildvm.h lj_def.h lua.h \ 257host/buildvm_peobj.o: host/buildvm_peobj.c host/buildvm.h lj_def.h lua.h \
225 luaconf.h lj_arch.h lj_bc.h lj_def.h lj_arch.h 258 luaconf.h lj_arch.h lj_bc.h lj_def.h lj_arch.h
226host/minilua.o: host/minilua.c 259host/minilua.o: host/minilua.c
diff --git a/src/host/buildvm.c b/src/host/buildvm.c
index 39c2bc24..ec99e501 100644
--- a/src/host/buildvm.c
+++ b/src/host/buildvm.c
@@ -18,8 +18,10 @@
18#include "lj_obj.h" 18#include "lj_obj.h"
19#include "lj_gc.h" 19#include "lj_gc.h"
20#include "lj_bc.h" 20#include "lj_bc.h"
21#if LJ_HASJIT
21#include "lj_ir.h" 22#include "lj_ir.h"
22#include "lj_ircall.h" 23#include "lj_ircall.h"
24#endif
23#include "lj_frame.h" 25#include "lj_frame.h"
24#include "lj_dispatch.h" 26#include "lj_dispatch.h"
25#if LJ_HASFFI 27#if LJ_HASFFI
@@ -59,10 +61,10 @@ static int collect_reloc(BuildCtx *ctx, uint8_t *addr, int idx, int type);
59#include "../dynasm/dasm_x86.h" 61#include "../dynasm/dasm_x86.h"
60#elif LJ_TARGET_ARM 62#elif LJ_TARGET_ARM
61#include "../dynasm/dasm_arm.h" 63#include "../dynasm/dasm_arm.h"
64#elif LJ_TARGET_ARM64
65#include "../dynasm/dasm_arm64.h"
62#elif LJ_TARGET_PPC 66#elif LJ_TARGET_PPC
63#include "../dynasm/dasm_ppc.h" 67#include "../dynasm/dasm_ppc.h"
64#elif LJ_TARGET_PPCSPE
65#include "../dynasm/dasm_ppc.h"
66#elif LJ_TARGET_MIPS 68#elif LJ_TARGET_MIPS
67#include "../dynasm/dasm_mips.h" 69#include "../dynasm/dasm_mips.h"
68#else 70#else
@@ -110,11 +112,11 @@ static const char *sym_decorate(BuildCtx *ctx,
110 if (p) { 112 if (p) {
111#if LJ_TARGET_X86ORX64 113#if LJ_TARGET_X86ORX64
112 if (!LJ_64 && (ctx->mode == BUILD_coffasm || ctx->mode == BUILD_peobj)) 114 if (!LJ_64 && (ctx->mode == BUILD_coffasm || ctx->mode == BUILD_peobj))
113 name[0] = '@'; 115 name[0] = name[1] == 'R' ? '_' : '@'; /* Just for _RtlUnwind@16. */
114 else 116 else
115 *p = '\0'; 117 *p = '\0';
116#elif (LJ_TARGET_PPC || LJ_TARGET_PPCSPE) && !LJ_TARGET_CONSOLE 118#elif LJ_TARGET_PPC && !LJ_TARGET_CONSOLE
117 /* Keep @plt. */ 119 /* Keep @plt etc. */
118#else 120#else
119 *p = '\0'; 121 *p = '\0';
120#endif 122#endif
@@ -179,6 +181,7 @@ static int build_code(BuildCtx *ctx)
179 ctx->nreloc = 0; 181 ctx->nreloc = 0;
180 182
181 ctx->globnames = globnames; 183 ctx->globnames = globnames;
184 ctx->extnames = extnames;
182 ctx->relocsym = (const char **)malloc(NRELOCSYM*sizeof(const char *)); 185 ctx->relocsym = (const char **)malloc(NRELOCSYM*sizeof(const char *));
183 ctx->nrelocsym = 0; 186 ctx->nrelocsym = 0;
184 for (i = 0; i < (int)NRELOCSYM; i++) relocmap[i] = -1; 187 for (i = 0; i < (int)NRELOCSYM; i++) relocmap[i] = -1;
@@ -249,6 +252,7 @@ BCDEF(BCNAME)
249 NULL 252 NULL
250}; 253};
251 254
255#if LJ_HASJIT
252const char *const ir_names[] = { 256const char *const ir_names[] = {
253#define IRNAME(name, m, m1, m2) #name, 257#define IRNAME(name, m, m1, m2) #name,
254IRDEF(IRNAME) 258IRDEF(IRNAME)
@@ -289,7 +293,9 @@ static const char *const trace_errors[] = {
289#include "lj_traceerr.h" 293#include "lj_traceerr.h"
290 NULL 294 NULL
291}; 295};
296#endif
292 297
298#if LJ_HASJIT
293static const char *lower(char *buf, const char *s) 299static const char *lower(char *buf, const char *s)
294{ 300{
295 char *p = buf; 301 char *p = buf;
@@ -300,6 +306,7 @@ static const char *lower(char *buf, const char *s)
300 *p = '\0'; 306 *p = '\0';
301 return buf; 307 return buf;
302} 308}
309#endif
303 310
304/* Emit C source code for bytecode-related definitions. */ 311/* Emit C source code for bytecode-related definitions. */
305static void emit_bcdef(BuildCtx *ctx) 312static void emit_bcdef(BuildCtx *ctx)
@@ -317,24 +324,27 @@ static void emit_bcdef(BuildCtx *ctx)
317/* Emit VM definitions as Lua code for debug modules. */ 324/* Emit VM definitions as Lua code for debug modules. */
318static void emit_vmdef(BuildCtx *ctx) 325static void emit_vmdef(BuildCtx *ctx)
319{ 326{
327#if LJ_HASJIT
320 char buf[80]; 328 char buf[80];
329#endif
321 int i; 330 int i;
322 fprintf(ctx->fp, "-- This is a generated file. DO NOT EDIT!\n\n"); 331 fprintf(ctx->fp, "-- This is a generated file. DO NOT EDIT!\n\n");
323 fprintf(ctx->fp, "assert(require(\"jit\").version == \"%s\", \"LuaJIT core/library version mismatch\")\n\n", LUAJIT_VERSION); 332 fprintf(ctx->fp, "assert(require(\"jit\").version == \"%s\", \"LuaJIT core/library version mismatch\")\n\n", LUAJIT_VERSION);
324 fprintf(ctx->fp, "module(...)\n\n"); 333 fprintf(ctx->fp, "return {\n\n");
325 334
326 fprintf(ctx->fp, "bcnames = \""); 335 fprintf(ctx->fp, "bcnames = \"");
327 for (i = 0; bc_names[i]; i++) fprintf(ctx->fp, "%-6s", bc_names[i]); 336 for (i = 0; bc_names[i]; i++) fprintf(ctx->fp, "%-6s", bc_names[i]);
328 fprintf(ctx->fp, "\"\n\n"); 337 fprintf(ctx->fp, "\",\n\n");
329 338
339#if LJ_HASJIT
330 fprintf(ctx->fp, "irnames = \""); 340 fprintf(ctx->fp, "irnames = \"");
331 for (i = 0; ir_names[i]; i++) fprintf(ctx->fp, "%-6s", ir_names[i]); 341 for (i = 0; ir_names[i]; i++) fprintf(ctx->fp, "%-6s", ir_names[i]);
332 fprintf(ctx->fp, "\"\n\n"); 342 fprintf(ctx->fp, "\",\n\n");
333 343
334 fprintf(ctx->fp, "irfpm = { [0]="); 344 fprintf(ctx->fp, "irfpm = { [0]=");
335 for (i = 0; irfpm_names[i]; i++) 345 for (i = 0; irfpm_names[i]; i++)
336 fprintf(ctx->fp, "\"%s\", ", lower(buf, irfpm_names[i])); 346 fprintf(ctx->fp, "\"%s\", ", lower(buf, irfpm_names[i]));
337 fprintf(ctx->fp, "}\n\n"); 347 fprintf(ctx->fp, "},\n\n");
338 348
339 fprintf(ctx->fp, "irfield = { [0]="); 349 fprintf(ctx->fp, "irfield = { [0]=");
340 for (i = 0; irfield_names[i]; i++) { 350 for (i = 0; irfield_names[i]; i++) {
@@ -344,17 +354,18 @@ static void emit_vmdef(BuildCtx *ctx)
344 if (p) *p = '.'; 354 if (p) *p = '.';
345 fprintf(ctx->fp, "\"%s\", ", buf); 355 fprintf(ctx->fp, "\"%s\", ", buf);
346 } 356 }
347 fprintf(ctx->fp, "}\n\n"); 357 fprintf(ctx->fp, "},\n\n");
348 358
349 fprintf(ctx->fp, "ircall = {\n[0]="); 359 fprintf(ctx->fp, "ircall = {\n[0]=");
350 for (i = 0; ircall_names[i]; i++) 360 for (i = 0; ircall_names[i]; i++)
351 fprintf(ctx->fp, "\"%s\",\n", ircall_names[i]); 361 fprintf(ctx->fp, "\"%s\",\n", ircall_names[i]);
352 fprintf(ctx->fp, "}\n\n"); 362 fprintf(ctx->fp, "},\n\n");
353 363
354 fprintf(ctx->fp, "traceerr = {\n[0]="); 364 fprintf(ctx->fp, "traceerr = {\n[0]=");
355 for (i = 0; trace_errors[i]; i++) 365 for (i = 0; trace_errors[i]; i++)
356 fprintf(ctx->fp, "\"%s\",\n", trace_errors[i]); 366 fprintf(ctx->fp, "\"%s\",\n", trace_errors[i]);
357 fprintf(ctx->fp, "}\n\n"); 367 fprintf(ctx->fp, "},\n\n");
368#endif
358} 369}
359 370
360/* -- Argument parsing ---------------------------------------------------- */ 371/* -- Argument parsing ---------------------------------------------------- */
@@ -491,6 +502,7 @@ int main(int argc, char **argv)
491 case BUILD_vmdef: 502 case BUILD_vmdef:
492 emit_vmdef(ctx); 503 emit_vmdef(ctx);
493 emit_lib(ctx); 504 emit_lib(ctx);
505 fprintf(ctx->fp, "}\n\n");
494 break; 506 break;
495 case BUILD_ffdef: 507 case BUILD_ffdef:
496 case BUILD_libdef: 508 case BUILD_libdef:
diff --git a/src/host/buildvm.h b/src/host/buildvm.h
index ded45d94..add8ee15 100644
--- a/src/host/buildvm.h
+++ b/src/host/buildvm.h
@@ -82,6 +82,7 @@ typedef struct BuildCtx {
82 const char *beginsym; 82 const char *beginsym;
83 /* Strings generated by DynASM. */ 83 /* Strings generated by DynASM. */
84 const char *const *globnames; 84 const char *const *globnames;
85 const char *const *extnames;
85 const char *dasm_ident; 86 const char *dasm_ident;
86 const char *dasm_arch; 87 const char *dasm_arch;
87 /* Relocations. */ 88 /* Relocations. */
diff --git a/src/host/buildvm_asm.c b/src/host/buildvm_asm.c
index 458ce733..e7c5de70 100644
--- a/src/host/buildvm_asm.c
+++ b/src/host/buildvm_asm.c
@@ -51,8 +51,8 @@ static const char *const jccnames[] = {
51 "js", "jns", "jpe", "jpo", "jl", "jge", "jle", "jg" 51 "js", "jns", "jpe", "jpo", "jl", "jge", "jle", "jg"
52}; 52};
53 53
54/* Emit relocation for the incredibly stupid OSX assembler. */ 54/* Emit x86/x64 text relocations. */
55static void emit_asm_reloc_mach(BuildCtx *ctx, uint8_t *cp, int n, 55static void emit_asm_reloc_text(BuildCtx *ctx, uint8_t *cp, int n,
56 const char *sym) 56 const char *sym)
57{ 57{
58 const char *opname = NULL; 58 const char *opname = NULL;
@@ -71,6 +71,20 @@ err:
71 exit(1); 71 exit(1);
72 } 72 }
73 emit_asm_bytes(ctx, cp, n); 73 emit_asm_bytes(ctx, cp, n);
74 if (strncmp(sym+(*sym == '_'), LABEL_PREFIX, sizeof(LABEL_PREFIX)-1)) {
75 /* Various fixups for external symbols outside of our binary. */
76 if (ctx->mode == BUILD_elfasm) {
77 if (LJ_32)
78 fprintf(ctx->fp, "#if __PIC__\n\t%s lj_wrap_%s\n#else\n", opname, sym);
79 fprintf(ctx->fp, "\t%s %s@PLT\n", opname, sym);
80 if (LJ_32)
81 fprintf(ctx->fp, "#endif\n");
82 return;
83 } else if (LJ_32 && ctx->mode == BUILD_machasm) {
84 fprintf(ctx->fp, "\t%s L%s$stub\n", opname, sym);
85 return;
86 }
87 }
74 fprintf(ctx->fp, "\t%s %s\n", opname, sym); 88 fprintf(ctx->fp, "\t%s %s\n", opname, sym);
75} 89}
76#else 90#else
@@ -79,10 +93,14 @@ static void emit_asm_words(BuildCtx *ctx, uint8_t *p, int n)
79{ 93{
80 int i; 94 int i;
81 for (i = 0; i < n; i += 4) { 95 for (i = 0; i < n; i += 4) {
96 uint32_t ins = *(uint32_t *)(p+i);
97#if LJ_TARGET_ARM64 && LJ_BE
98 ins = lj_bswap(ins); /* ARM64 instructions are always little-endian. */
99#endif
82 if ((i & 15) == 0) 100 if ((i & 15) == 0)
83 fprintf(ctx->fp, "\t.long 0x%08x", *(uint32_t *)(p+i)); 101 fprintf(ctx->fp, "\t.long 0x%08x", ins);
84 else 102 else
85 fprintf(ctx->fp, ",0x%08x", *(uint32_t *)(p+i)); 103 fprintf(ctx->fp, ",0x%08x", ins);
86 if ((i & 15) == 12) putc('\n', ctx->fp); 104 if ((i & 15) == 12) putc('\n', ctx->fp);
87 } 105 }
88 if ((n & 15) != 0) putc('\n', ctx->fp); 106 if ((n & 15) != 0) putc('\n', ctx->fp);
@@ -107,7 +125,16 @@ static void emit_asm_wordreloc(BuildCtx *ctx, uint8_t *p, int n,
107 ins, sym); 125 ins, sym);
108 exit(1); 126 exit(1);
109 } 127 }
110#elif LJ_TARGET_PPC || LJ_TARGET_PPCSPE 128#elif LJ_TARGET_ARM64
129 if ((ins >> 26) == 0x25u) {
130 fprintf(ctx->fp, "\tbl %s\n", sym);
131 } else {
132 fprintf(stderr,
133 "Error: unsupported opcode %08x for %s symbol relocation.\n",
134 ins, sym);
135 exit(1);
136 }
137#elif LJ_TARGET_PPC
111#if LJ_TARGET_PS3 138#if LJ_TARGET_PS3
112#define TOCPREFIX "." 139#define TOCPREFIX "."
113#else 140#else
@@ -216,6 +243,12 @@ void emit_asm(BuildCtx *ctx)
216 243
217 fprintf(ctx->fp, "\t.file \"buildvm_%s.dasc\"\n", ctx->dasm_arch); 244 fprintf(ctx->fp, "\t.file \"buildvm_%s.dasc\"\n", ctx->dasm_arch);
218 fprintf(ctx->fp, "\t.text\n"); 245 fprintf(ctx->fp, "\t.text\n");
246#if LJ_TARGET_MIPS32 && !LJ_ABI_SOFTFP
247 fprintf(ctx->fp, "\t.module fp=32\n");
248#endif
249#if LJ_TARGET_MIPS
250 fprintf(ctx->fp, "\t.set nomips16\n\t.abicalls\n\t.set noreorder\n\t.set nomacro\n");
251#endif
219 emit_asm_align(ctx, 4); 252 emit_asm_align(ctx, 4);
220 253
221#if LJ_TARGET_PS3 254#if LJ_TARGET_PS3
@@ -228,13 +261,19 @@ void emit_asm(BuildCtx *ctx)
228 261
229#if LJ_TARGET_ARM && defined(__GNUC__) && !LJ_NO_UNWIND 262#if LJ_TARGET_ARM && defined(__GNUC__) && !LJ_NO_UNWIND
230 /* This should really be moved into buildvm_arm.dasc. */ 263 /* This should really be moved into buildvm_arm.dasc. */
264#if LJ_ARCH_HASFPU
265 fprintf(ctx->fp,
266 ".fnstart\n"
267 ".save {r5, r6, r7, r8, r9, r10, r11, lr}\n"
268 ".vsave {d8-d15}\n"
269 ".save {r4}\n"
270 ".pad #28\n");
271#else
231 fprintf(ctx->fp, 272 fprintf(ctx->fp,
232 ".fnstart\n" 273 ".fnstart\n"
233 ".save {r4, r5, r6, r7, r8, r9, r10, r11, lr}\n" 274 ".save {r4, r5, r6, r7, r8, r9, r10, r11, lr}\n"
234 ".pad #28\n"); 275 ".pad #28\n");
235#endif 276#endif
236#if LJ_TARGET_MIPS
237 fprintf(ctx->fp, ".set nomips16\n.abicalls\n.set noreorder\n.set nomacro\n");
238#endif 277#endif
239 278
240 for (i = rel = 0; i < ctx->nsym; i++) { 279 for (i = rel = 0; i < ctx->nsym; i++) {
@@ -255,8 +294,9 @@ void emit_asm(BuildCtx *ctx)
255 BuildReloc *r = &ctx->reloc[rel]; 294 BuildReloc *r = &ctx->reloc[rel];
256 int n = r->ofs - ofs; 295 int n = r->ofs - ofs;
257#if LJ_TARGET_X86ORX64 296#if LJ_TARGET_X86ORX64
258 if (ctx->mode == BUILD_machasm && r->type != 0) { 297 if (r->type != 0 &&
259 emit_asm_reloc_mach(ctx, ctx->code+ofs, n, ctx->relocsym[r->sym]); 298 (ctx->mode == BUILD_elfasm || ctx->mode == BUILD_machasm)) {
299 emit_asm_reloc_text(ctx, ctx->code+ofs, n, ctx->relocsym[r->sym]);
260 } else { 300 } else {
261 emit_asm_bytes(ctx, ctx->code+ofs, n); 301 emit_asm_bytes(ctx, ctx->code+ofs, n);
262 emit_asm_reloc(ctx, r->type, ctx->relocsym[r->sym]); 302 emit_asm_reloc(ctx, r->type, ctx->relocsym[r->sym]);
@@ -290,10 +330,7 @@ void emit_asm(BuildCtx *ctx)
290#if !(LJ_TARGET_PS3 || LJ_TARGET_PSVITA) 330#if !(LJ_TARGET_PS3 || LJ_TARGET_PSVITA)
291 fprintf(ctx->fp, "\t.section .note.GNU-stack,\"\"," ELFASM_PX "progbits\n"); 331 fprintf(ctx->fp, "\t.section .note.GNU-stack,\"\"," ELFASM_PX "progbits\n");
292#endif 332#endif
293#if LJ_TARGET_PPCSPE 333#if LJ_TARGET_PPC && !LJ_TARGET_PS3 && !LJ_ABI_SOFTFP
294 /* Soft-float ABI + SPE. */
295 fprintf(ctx->fp, "\t.gnu_attribute 4, 2\n\t.gnu_attribute 8, 3\n");
296#elif LJ_TARGET_PPC && !LJ_TARGET_PS3
297 /* Hard-float ABI. */ 334 /* Hard-float ABI. */
298 fprintf(ctx->fp, "\t.gnu_attribute 4, 1\n"); 335 fprintf(ctx->fp, "\t.gnu_attribute 4, 1\n");
299#endif 336#endif
diff --git a/src/host/buildvm_fold.c b/src/host/buildvm_fold.c
index 388a8146..cc392e93 100644
--- a/src/host/buildvm_fold.c
+++ b/src/host/buildvm_fold.c
@@ -5,6 +5,7 @@
5 5
6#include "buildvm.h" 6#include "buildvm.h"
7#include "lj_obj.h" 7#include "lj_obj.h"
8#if LJ_HASJIT
8#include "lj_ir.h" 9#include "lj_ir.h"
9 10
10/* Context for the folding hash table generator. */ 11/* Context for the folding hash table generator. */
@@ -226,4 +227,10 @@ void emit_fold(BuildCtx *ctx)
226 227
227 makehash(ctx); 228 makehash(ctx);
228} 229}
230#else
231void emit_fold(BuildCtx *ctx)
232{
233 UNUSED(ctx);
234}
235#endif
229 236
diff --git a/src/host/buildvm_lib.c b/src/host/buildvm_lib.c
index 36797e4c..99d12a27 100644
--- a/src/host/buildvm_lib.c
+++ b/src/host/buildvm_lib.c
@@ -5,7 +5,9 @@
5 5
6#include "buildvm.h" 6#include "buildvm.h"
7#include "lj_obj.h" 7#include "lj_obj.h"
8#include "lj_bc.h"
8#include "lj_lib.h" 9#include "lj_lib.h"
10#include "buildvm_libbc.h"
9 11
10/* Context for library definitions. */ 12/* Context for library definitions. */
11static uint8_t obuf[8192]; 13static uint8_t obuf[8192];
@@ -151,6 +153,62 @@ static void libdef_func(BuildCtx *ctx, char *p, int arg)
151 regfunc = REGFUNC_OK; 153 regfunc = REGFUNC_OK;
152} 154}
153 155
156static uint8_t *libdef_uleb128(uint8_t *p, uint32_t *vv)
157{
158 uint32_t v = *p++;
159 if (v >= 0x80) {
160 int sh = 0; v &= 0x7f;
161 do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
162 }
163 *vv = v;
164 return p;
165}
166
167static void libdef_fixupbc(uint8_t *p)
168{
169 uint32_t i, sizebc;
170 p += 4;
171 p = libdef_uleb128(p, &sizebc);
172 p = libdef_uleb128(p, &sizebc);
173 p = libdef_uleb128(p, &sizebc);
174 for (i = 0; i < sizebc; i++, p += 4) {
175 uint8_t op = p[libbc_endian ? 3 : 0];
176 uint8_t ra = p[libbc_endian ? 2 : 1];
177 uint8_t rc = p[libbc_endian ? 1 : 2];
178 uint8_t rb = p[libbc_endian ? 0 : 3];
179 if (!LJ_DUALNUM && op == BC_ISTYPE && rc == ~LJ_TNUMX+1) {
180 op = BC_ISNUM; rc++;
181 }
182 p[LJ_ENDIAN_SELECT(0, 3)] = op;
183 p[LJ_ENDIAN_SELECT(1, 2)] = ra;
184 p[LJ_ENDIAN_SELECT(2, 1)] = rc;
185 p[LJ_ENDIAN_SELECT(3, 0)] = rb;
186 }
187}
188
189static void libdef_lua(BuildCtx *ctx, char *p, int arg)
190{
191 UNUSED(arg);
192 if (ctx->mode == BUILD_libdef) {
193 int i;
194 for (i = 0; libbc_map[i].name != NULL; i++) {
195 if (!strcmp(libbc_map[i].name, p)) {
196 int ofs = libbc_map[i].ofs;
197 int len = libbc_map[i+1].ofs - ofs;
198 obuf[2]++; /* Bump hash table size. */
199 *optr++ = LIBINIT_LUA;
200 libdef_name(p, 0);
201 memcpy(optr, libbc_code + ofs, len);
202 libdef_fixupbc(optr);
203 optr += len;
204 return;
205 }
206 }
207 fprintf(stderr, "Error: missing libbc definition for %s\n", p);
208 exit(1);
209 }
210}
211
154static uint32_t find_rec(char *name) 212static uint32_t find_rec(char *name)
155{ 213{
156 char *p = (char *)obuf; 214 char *p = (char *)obuf;
@@ -277,6 +335,7 @@ static const LibDefHandler libdef_handlers[] = {
277 { "CF(", ")", libdef_func, LIBINIT_CF }, 335 { "CF(", ")", libdef_func, LIBINIT_CF },
278 { "ASM(", ")", libdef_func, LIBINIT_ASM }, 336 { "ASM(", ")", libdef_func, LIBINIT_ASM },
279 { "ASM_(", ")", libdef_func, LIBINIT_ASM_ }, 337 { "ASM_(", ")", libdef_func, LIBINIT_ASM_ },
338 { "LUA(", ")", libdef_lua, 0 },
280 { "REC(", ")", libdef_rec, 0 }, 339 { "REC(", ")", libdef_rec, 0 },
281 { "PUSH(", ")", libdef_push, 0 }, 340 { "PUSH(", ")", libdef_push, 0 },
282 { "SET(", ")", libdef_set, 0 }, 341 { "SET(", ")", libdef_set, 0 },
@@ -333,6 +392,8 @@ void emit_lib(BuildCtx *ctx)
333 ok = LJ_HASJIT; 392 ok = LJ_HASJIT;
334 else if (!strcmp(buf, "#if LJ_HASFFI")) 393 else if (!strcmp(buf, "#if LJ_HASFFI"))
335 ok = LJ_HASFFI; 394 ok = LJ_HASFFI;
395 else if (!strcmp(buf, "#if LJ_HASBUFFER"))
396 ok = LJ_HASBUFFER;
336 if (!ok) { 397 if (!ok) {
337 int lvl = 1; 398 int lvl = 1;
338 while (fgets(buf, sizeof(buf), fp) != NULL) { 399 while (fgets(buf, sizeof(buf), fp) != NULL) {
@@ -380,7 +441,7 @@ void emit_lib(BuildCtx *ctx)
380 "#ifndef FF_NUM_ASMFUNC\n#define FF_NUM_ASMFUNC %d\n#endif\n\n", 441 "#ifndef FF_NUM_ASMFUNC\n#define FF_NUM_ASMFUNC %d\n#endif\n\n",
381 ffasmfunc); 442 ffasmfunc);
382 } else if (ctx->mode == BUILD_vmdef) { 443 } else if (ctx->mode == BUILD_vmdef) {
383 fprintf(ctx->fp, "}\n\n"); 444 fprintf(ctx->fp, "},\n\n");
384 } else if (ctx->mode == BUILD_bcdef) { 445 } else if (ctx->mode == BUILD_bcdef) {
385 int i; 446 int i;
386 fprintf(ctx->fp, "\n};\n\n"); 447 fprintf(ctx->fp, "\n};\n\n");
diff --git a/src/host/buildvm_libbc.h b/src/host/buildvm_libbc.h
new file mode 100644
index 00000000..276463b2
--- /dev/null
+++ b/src/host/buildvm_libbc.h
@@ -0,0 +1,81 @@
1/* This is a generated file. DO NOT EDIT! */
2
3static const int libbc_endian = 0;
4
5static const uint8_t libbc_code[] = {
6#if LJ_FR2
7/* math.deg */ 0,1,2,0,0,1,2,BC_MULVN,1,0,0,BC_RET1,1,2,0,241,135,158,166,3,
8220,203,178,130,4,
9/* math.rad */ 0,1,2,0,0,1,2,BC_MULVN,1,0,0,BC_RET1,1,2,0,243,244,148,165,20,
10198,190,199,252,3,
11/* string.len */ 0,1,2,0,0,0,3,BC_ISTYPE,0,5,0,BC_LEN,1,0,0,BC_RET1,1,2,0,
12/* table.foreachi */ 0,2,10,0,0,0,15,BC_ISTYPE,0,12,0,BC_ISTYPE,1,9,0,
13BC_KSHORT,2,1,0,BC_LEN,3,0,0,BC_KSHORT,4,1,0,BC_FORI,2,8,128,BC_MOV,6,1,0,
14BC_MOV,8,5,0,BC_TGETR,9,5,0,BC_CALL,6,3,2,BC_ISEQP,6,0,0,BC_JMP,7,1,128,
15BC_RET1,6,2,0,BC_FORL,2,248,127,BC_RET0,0,1,0,
16/* table.foreach */ 0,2,11,0,0,1,16,BC_ISTYPE,0,12,0,BC_ISTYPE,1,9,0,BC_KPRI,
172,0,0,BC_MOV,3,0,0,BC_KNUM,4,0,0,BC_JMP,5,7,128,BC_MOV,7,1,0,BC_MOV,9,5,0,
18BC_MOV,10,6,0,BC_CALL,7,3,2,BC_ISEQP,7,0,0,BC_JMP,8,1,128,BC_RET1,7,2,0,
19BC_ITERN,5,3,3,BC_ITERL,5,247,127,BC_RET0,0,1,0,1,255,255,249,255,15,
20/* table.getn */ 0,1,2,0,0,0,3,BC_ISTYPE,0,12,0,BC_LEN,1,0,0,BC_RET1,1,2,0,
21/* table.remove */ 0,2,10,0,0,2,30,BC_ISTYPE,0,12,0,BC_LEN,2,0,0,BC_ISNEP,1,0,
220,BC_JMP,3,7,128,BC_ISEQN,2,0,0,BC_JMP,3,23,128,BC_TGETR,3,2,0,BC_KPRI,4,0,0,
23BC_TSETR,4,2,0,BC_RET1,3,2,0,BC_JMP,3,18,128,BC_ISTYPE,1,14,0,BC_KSHORT,3,1,0,
24BC_ISGT,3,1,0,BC_JMP,3,14,128,BC_ISGT,1,2,0,BC_JMP,3,12,128,BC_TGETR,3,1,0,
25BC_ADDVN,4,1,1,BC_MOV,5,2,0,BC_KSHORT,6,1,0,BC_FORI,4,4,128,BC_SUBVN,8,1,7,
26BC_TGETR,9,7,0,BC_TSETR,9,8,0,BC_FORL,4,252,127,BC_KPRI,4,0,0,BC_TSETR,4,2,0,
27BC_RET1,3,2,0,BC_RET0,0,1,0,0,2,
28/* table.move */ 0,5,12,0,0,0,35,BC_ISTYPE,0,12,0,BC_ISTYPE,1,14,0,BC_ISTYPE,
292,14,0,BC_ISTYPE,3,14,0,BC_ISNEP,4,0,0,BC_JMP,5,1,128,BC_MOV,4,0,0,BC_ISTYPE,
304,12,0,BC_ISGT,1,2,0,BC_JMP,5,24,128,BC_SUBVV,5,1,3,BC_ISLT,2,3,0,BC_JMP,6,4,
31128,BC_ISLE,3,1,0,BC_JMP,6,2,128,BC_ISEQV,4,0,0,BC_JMP,6,9,128,BC_MOV,6,1,0,
32BC_MOV,7,2,0,BC_KSHORT,8,1,0,BC_FORI,6,4,128,BC_ADDVV,10,5,9,BC_TGETR,11,9,0,
33BC_TSETR,11,10,4,BC_FORL,6,252,127,BC_JMP,6,8,128,BC_MOV,6,2,0,BC_MOV,7,1,0,
34BC_KSHORT,8,255,255,BC_FORI,6,4,128,BC_ADDVV,10,5,9,BC_TGETR,11,9,0,BC_TSETR,
3511,10,4,BC_FORL,6,252,127,BC_RET1,4,2,0,
36#else
37/* math.deg */ 0,1,2,0,0,1,2,BC_MULVN,1,0,0,BC_RET1,1,2,0,241,135,158,166,3,
38220,203,178,130,4,
39/* math.rad */ 0,1,2,0,0,1,2,BC_MULVN,1,0,0,BC_RET1,1,2,0,243,244,148,165,20,
40198,190,199,252,3,
41/* string.len */ 0,1,2,0,0,0,3,BC_ISTYPE,0,5,0,BC_LEN,1,0,0,BC_RET1,1,2,0,
42/* table.foreachi */ 0,2,9,0,0,0,15,BC_ISTYPE,0,12,0,BC_ISTYPE,1,9,0,
43BC_KSHORT,2,1,0,BC_LEN,3,0,0,BC_KSHORT,4,1,0,BC_FORI,2,8,128,BC_MOV,6,1,0,
44BC_MOV,7,5,0,BC_TGETR,8,5,0,BC_CALL,6,3,2,BC_ISEQP,6,0,0,BC_JMP,7,1,128,
45BC_RET1,6,2,0,BC_FORL,2,248,127,BC_RET0,0,1,0,
46/* table.foreach */ 0,2,10,0,0,1,16,BC_ISTYPE,0,12,0,BC_ISTYPE,1,9,0,BC_KPRI,
472,0,0,BC_MOV,3,0,0,BC_KNUM,4,0,0,BC_JMP,5,7,128,BC_MOV,7,1,0,BC_MOV,8,5,0,
48BC_MOV,9,6,0,BC_CALL,7,3,2,BC_ISEQP,7,0,0,BC_JMP,8,1,128,BC_RET1,7,2,0,
49BC_ITERN,5,3,3,BC_ITERL,5,247,127,BC_RET0,0,1,0,1,255,255,249,255,15,
50/* table.getn */ 0,1,2,0,0,0,3,BC_ISTYPE,0,12,0,BC_LEN,1,0,0,BC_RET1,1,2,0,
51/* table.remove */ 0,2,10,0,0,2,30,BC_ISTYPE,0,12,0,BC_LEN,2,0,0,BC_ISNEP,1,0,
520,BC_JMP,3,7,128,BC_ISEQN,2,0,0,BC_JMP,3,23,128,BC_TGETR,3,2,0,BC_KPRI,4,0,0,
53BC_TSETR,4,2,0,BC_RET1,3,2,0,BC_JMP,3,18,128,BC_ISTYPE,1,14,0,BC_KSHORT,3,1,0,
54BC_ISGT,3,1,0,BC_JMP,3,14,128,BC_ISGT,1,2,0,BC_JMP,3,12,128,BC_TGETR,3,1,0,
55BC_ADDVN,4,1,1,BC_MOV,5,2,0,BC_KSHORT,6,1,0,BC_FORI,4,4,128,BC_SUBVN,8,1,7,
56BC_TGETR,9,7,0,BC_TSETR,9,8,0,BC_FORL,4,252,127,BC_KPRI,4,0,0,BC_TSETR,4,2,0,
57BC_RET1,3,2,0,BC_RET0,0,1,0,0,2,
58/* table.move */ 0,5,12,0,0,0,35,BC_ISTYPE,0,12,0,BC_ISTYPE,1,14,0,BC_ISTYPE,
592,14,0,BC_ISTYPE,3,14,0,BC_ISNEP,4,0,0,BC_JMP,5,1,128,BC_MOV,4,0,0,BC_ISTYPE,
604,12,0,BC_ISGT,1,2,0,BC_JMP,5,24,128,BC_SUBVV,5,1,3,BC_ISLT,2,3,0,BC_JMP,6,4,
61128,BC_ISLE,3,1,0,BC_JMP,6,2,128,BC_ISEQV,4,0,0,BC_JMP,6,9,128,BC_MOV,6,1,0,
62BC_MOV,7,2,0,BC_KSHORT,8,1,0,BC_FORI,6,4,128,BC_ADDVV,10,5,9,BC_TGETR,11,9,0,
63BC_TSETR,11,10,4,BC_FORL,6,252,127,BC_JMP,6,8,128,BC_MOV,6,2,0,BC_MOV,7,1,0,
64BC_KSHORT,8,255,255,BC_FORI,6,4,128,BC_ADDVV,10,5,9,BC_TGETR,11,9,0,BC_TSETR,
6511,10,4,BC_FORL,6,252,127,BC_RET1,4,2,0,
66#endif
670
68};
69
70static const struct { const char *name; int ofs; } libbc_map[] = {
71{"math_deg",0},
72{"math_rad",25},
73{"string_len",50},
74{"table_foreachi",69},
75{"table_foreach",136},
76{"table_getn",213},
77{"table_remove",232},
78{"table_move",361},
79{NULL,508}
80};
81
diff --git a/src/host/buildvm_peobj.c b/src/host/buildvm_peobj.c
index 4fbc11b9..5bca6df8 100644
--- a/src/host/buildvm_peobj.c
+++ b/src/host/buildvm_peobj.c
@@ -9,7 +9,7 @@
9#include "buildvm.h" 9#include "buildvm.h"
10#include "lj_bc.h" 10#include "lj_bc.h"
11 11
12#if LJ_TARGET_X86ORX64 || LJ_TARGET_PPC 12#if LJ_TARGET_X86ORX64
13 13
14/* Context for PE object emitter. */ 14/* Context for PE object emitter. */
15static char *strtab; 15static char *strtab;
@@ -93,12 +93,6 @@ typedef struct PEsymaux {
93#define PEOBJ_RELOC_ADDR32NB 0x03 93#define PEOBJ_RELOC_ADDR32NB 0x03
94#define PEOBJ_RELOC_OFS 0 94#define PEOBJ_RELOC_OFS 0
95#define PEOBJ_TEXT_FLAGS 0x60500020 /* 60=r+x, 50=align16, 20=code. */ 95#define PEOBJ_TEXT_FLAGS 0x60500020 /* 60=r+x, 50=align16, 20=code. */
96#elif LJ_TARGET_PPC
97#define PEOBJ_ARCH_TARGET 0x01f2
98#define PEOBJ_RELOC_REL32 0x06
99#define PEOBJ_RELOC_DIR32 0x02
100#define PEOBJ_RELOC_OFS (-4)
101#define PEOBJ_TEXT_FLAGS 0x60400020 /* 60=r+x, 40=align8, 20=code. */
102#endif 96#endif
103 97
104/* Section numbers (0-based). */ 98/* Section numbers (0-based). */
@@ -109,6 +103,8 @@ enum {
109#if LJ_TARGET_X64 103#if LJ_TARGET_X64
110 PEOBJ_SECT_PDATA, 104 PEOBJ_SECT_PDATA,
111 PEOBJ_SECT_XDATA, 105 PEOBJ_SECT_XDATA,
106#elif LJ_TARGET_X86
107 PEOBJ_SECT_SXDATA,
112#endif 108#endif
113 PEOBJ_SECT_RDATA_Z, 109 PEOBJ_SECT_RDATA_Z,
114 PEOBJ_NSECTIONS 110 PEOBJ_NSECTIONS
@@ -208,6 +204,13 @@ void emit_peobj(BuildCtx *ctx)
208 sofs += (pesect[PEOBJ_SECT_XDATA].nreloc = 1) * PEOBJ_RELOC_SIZE; 204 sofs += (pesect[PEOBJ_SECT_XDATA].nreloc = 1) * PEOBJ_RELOC_SIZE;
209 /* Flags: 40 = read, 30 = align4, 40 = initialized data. */ 205 /* Flags: 40 = read, 30 = align4, 40 = initialized data. */
210 pesect[PEOBJ_SECT_XDATA].flags = 0x40300040; 206 pesect[PEOBJ_SECT_XDATA].flags = 0x40300040;
207#elif LJ_TARGET_X86
208 memcpy(pesect[PEOBJ_SECT_SXDATA].name, ".sxdata", sizeof(".sxdata")-1);
209 pesect[PEOBJ_SECT_SXDATA].ofs = sofs;
210 sofs += (pesect[PEOBJ_SECT_SXDATA].size = 4);
211 pesect[PEOBJ_SECT_SXDATA].relocofs = sofs;
212 /* Flags: 40 = read, 30 = align4, 02 = lnk_info, 40 = initialized data. */
213 pesect[PEOBJ_SECT_SXDATA].flags = 0x40300240;
211#endif 214#endif
212 215
213 memcpy(pesect[PEOBJ_SECT_RDATA_Z].name, ".rdata$Z", sizeof(".rdata$Z")-1); 216 memcpy(pesect[PEOBJ_SECT_RDATA_Z].name, ".rdata$Z", sizeof(".rdata$Z")-1);
@@ -232,7 +235,7 @@ void emit_peobj(BuildCtx *ctx)
232 nrsym = ctx->nrelocsym; 235 nrsym = ctx->nrelocsym;
233 pehdr.nsyms = 1+PEOBJ_NSECTIONS*2 + 1+ctx->nsym + nrsym; 236 pehdr.nsyms = 1+PEOBJ_NSECTIONS*2 + 1+ctx->nsym + nrsym;
234#if LJ_TARGET_X64 237#if LJ_TARGET_X64
235 pehdr.nsyms += 1; /* Symbol for lj_err_unwind_win64. */ 238 pehdr.nsyms += 1; /* Symbol for lj_err_unwind_win. */
236#endif 239#endif
237 240
238 /* Write PE object header and all sections. */ 241 /* Write PE object header and all sections. */
@@ -242,15 +245,8 @@ void emit_peobj(BuildCtx *ctx)
242 /* Write .text section. */ 245 /* Write .text section. */
243 host_endian.u = 1; 246 host_endian.u = 1;
244 if (host_endian.b != LJ_ENDIAN_SELECT(1, 0)) { 247 if (host_endian.b != LJ_ENDIAN_SELECT(1, 0)) {
245#if LJ_TARGET_PPC
246 uint32_t *p = (uint32_t *)ctx->code;
247 int n = (int)(ctx->codesz >> 2);
248 for (i = 0; i < n; i++, p++)
249 *p = lj_bswap(*p); /* Byteswap .text section. */
250#else
251 fprintf(stderr, "Error: different byte order for host and target\n"); 248 fprintf(stderr, "Error: different byte order for host and target\n");
252 exit(1); 249 exit(1);
253#endif
254 } 250 }
255 owrite(ctx, ctx->code, ctx->codesz); 251 owrite(ctx, ctx->code, ctx->codesz);
256 for (i = 0; i < ctx->nreloc; i++) { 252 for (i = 0; i < ctx->nreloc; i++) {
@@ -312,6 +308,19 @@ void emit_peobj(BuildCtx *ctx)
312 reloc.type = PEOBJ_RELOC_ADDR32NB; 308 reloc.type = PEOBJ_RELOC_ADDR32NB;
313 owrite(ctx, &reloc, PEOBJ_RELOC_SIZE); 309 owrite(ctx, &reloc, PEOBJ_RELOC_SIZE);
314 } 310 }
311#elif LJ_TARGET_X86
312 /* Write .sxdata section. */
313 for (i = 0; i < nrsym; i++) {
314 if (!strcmp(ctx->relocsym[i], "_lj_err_unwind_win")) {
315 uint32_t symidx = 1+2+i;
316 owrite(ctx, &symidx, 4);
317 break;
318 }
319 }
320 if (i == nrsym) {
321 fprintf(stderr, "Error: extern lj_err_unwind_win not used\n");
322 exit(1);
323 }
315#endif 324#endif
316 325
317 /* Write .rdata$Z section. */ 326 /* Write .rdata$Z section. */
@@ -333,8 +342,10 @@ void emit_peobj(BuildCtx *ctx)
333#if LJ_TARGET_X64 342#if LJ_TARGET_X64
334 emit_peobj_sym_sect(ctx, pesect, PEOBJ_SECT_PDATA); 343 emit_peobj_sym_sect(ctx, pesect, PEOBJ_SECT_PDATA);
335 emit_peobj_sym_sect(ctx, pesect, PEOBJ_SECT_XDATA); 344 emit_peobj_sym_sect(ctx, pesect, PEOBJ_SECT_XDATA);
336 emit_peobj_sym(ctx, "lj_err_unwind_win64", 0, 345 emit_peobj_sym(ctx, "lj_err_unwind_win", 0,
337 PEOBJ_SECT_UNDEF, PEOBJ_TYPE_FUNC, PEOBJ_SCL_EXTERN); 346 PEOBJ_SECT_UNDEF, PEOBJ_TYPE_FUNC, PEOBJ_SCL_EXTERN);
347#elif LJ_TARGET_X86
348 emit_peobj_sym_sect(ctx, pesect, PEOBJ_SECT_SXDATA);
338#endif 349#endif
339 350
340 emit_peobj_sym(ctx, ctx->beginsym, 0, 351 emit_peobj_sym(ctx, ctx->beginsym, 0,
diff --git a/src/host/genlibbc.lua b/src/host/genlibbc.lua
new file mode 100644
index 00000000..3621c3f5
--- /dev/null
+++ b/src/host/genlibbc.lua
@@ -0,0 +1,225 @@
1----------------------------------------------------------------------------
2-- Lua script to dump the bytecode of the library functions written in Lua.
3-- The resulting 'buildvm_libbc.h' is used for the build process of LuaJIT.
4----------------------------------------------------------------------------
5-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
6-- Released under the MIT license. See Copyright Notice in luajit.h
7----------------------------------------------------------------------------
8
9local ffi = require("ffi")
10local bit = require("bit")
11local vmdef = require("jit.vmdef")
12local bcnames = vmdef.bcnames
13
14local format = string.format
15
16local isbe = (string.byte(string.dump(function() end), 5) % 2 == 1)
17
18local function usage(arg)
19 io.stderr:write("Usage: ", arg and arg[0] or "genlibbc",
20 " [-o buildvm_libbc.h] lib_*.c\n")
21 os.exit(1)
22end
23
24local function parse_arg(arg)
25 local outfile = "-"
26 if not (arg and arg[1]) then
27 usage(arg)
28 end
29 if arg[1] == "-o" then
30 outfile = arg[2]
31 if not outfile then usage(arg) end
32 table.remove(arg, 1)
33 table.remove(arg, 1)
34 end
35 return outfile
36end
37
38local function read_files(names)
39 local src = ""
40 for _,name in ipairs(names) do
41 local fp = assert(io.open(name))
42 src = src .. fp:read("*a")
43 fp:close()
44 end
45 return src
46end
47
48local function transform_lua(code)
49 local fixup = {}
50 local n = -30000
51 code = string.gsub(code, "CHECK_(%w*)%((.-)%)", function(tp, var)
52 n = n + 1
53 fixup[n] = { "CHECK", tp }
54 return format("%s=%d", var, n)
55 end)
56 code = string.gsub(code, "PAIRS%((.-)%)", function(var)
57 fixup.PAIRS = true
58 return format("nil, %s, 0x4dp80", var)
59 end)
60 return "return "..code, fixup
61end
62
63local function read_uleb128(p)
64 local v = p[0]; p = p + 1
65 if v >= 128 then
66 local sh = 7; v = v - 128
67 repeat
68 local r = p[0]
69 v = v + bit.lshift(bit.band(r, 127), sh)
70 sh = sh + 7
71 p = p + 1
72 until r < 128
73 end
74 return p, v
75end
76
77-- ORDER LJ_T
78local name2itype = {
79 str = 5, func = 9, tab = 12, int = 14, num = 15
80}
81
82local BC, BCN = {}, {}
83for i=0,#bcnames/6-1 do
84 local name = bcnames:sub(i*6+1, i*6+6):gsub(" ", "")
85 BC[name] = i
86 BCN[i] = name
87end
88local xop, xra = isbe and 3 or 0, isbe and 2 or 1
89local xrc, xrb = isbe and 1 or 2, isbe and 0 or 3
90
91local function fixup_dump(dump, fixup)
92 local buf = ffi.new("uint8_t[?]", #dump+1, dump)
93 local p = buf+5
94 local n, sizebc
95 p, n = read_uleb128(p)
96 local start = p
97 p = p + 4
98 p = read_uleb128(p)
99 p = read_uleb128(p)
100 p, sizebc = read_uleb128(p)
101 local startbc = tonumber(p - start)
102 local rawtab = {}
103 for i=0,sizebc-1 do
104 local op = p[xop]
105 if op == BC.KSHORT then
106 local rd = p[xrc] + 256*p[xrb]
107 rd = bit.arshift(bit.lshift(rd, 16), 16)
108 local f = fixup[rd]
109 if f then
110 if f[1] == "CHECK" then
111 local tp = f[2]
112 if tp == "tab" then rawtab[p[xra]] = true end
113 p[xop] = tp == "num" and BC.ISNUM or BC.ISTYPE
114 p[xrb] = 0
115 p[xrc] = name2itype[tp]
116 else
117 error("unhandled fixup type: "..f[1])
118 end
119 end
120 elseif op == BC.TGETV then
121 if rawtab[p[xrb]] then
122 p[xop] = BC.TGETR
123 end
124 elseif op == BC.TSETV then
125 if rawtab[p[xrb]] then
126 p[xop] = BC.TSETR
127 end
128 elseif op == BC.ITERC then
129 if fixup.PAIRS then
130 p[xop] = BC.ITERN
131 end
132 end
133 p = p + 4
134 end
135 local ndump = ffi.string(start, n)
136 -- Fixup hi-part of 0x4dp80 to LJ_KEYINDEX.
137 ndump = ndump:gsub("\x80\x80\xcd\xaa\x04", "\xff\xff\xf9\xff\x0f")
138 return { dump = ndump, startbc = startbc, sizebc = sizebc }
139end
140
141local function find_defs(src)
142 local defs = {}
143 for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do
144 local env = {}
145 local tcode, fixup = transform_lua(code)
146 local func = assert(load(tcode, "", nil, env))()
147 defs[name] = fixup_dump(string.dump(func, true), fixup)
148 defs[#defs+1] = name
149 end
150 return defs
151end
152
153local function gen_header(defs)
154 local t = {}
155 local function w(x) t[#t+1] = x end
156 w("/* This is a generated file. DO NOT EDIT! */\n\n")
157 w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n")
158 local s, sb = "", ""
159 for i,name in ipairs(defs) do
160 local d = defs[name]
161 s = s .. d.dump
162 sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1)
163 .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc)
164 .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4)
165 end
166 w("static const uint8_t libbc_code[] = {\n")
167 local n = 0
168 for i=1,#s do
169 local x = string.byte(s, i)
170 local xb = string.byte(sb, i)
171 if xb == 255 then
172 local name = BCN[x]
173 local m = #name + 4
174 if n + m > 78 then n = 0; w("\n") end
175 n = n + m
176 w("BC_"); w(name)
177 else
178 local m = x < 10 and 2 or (x < 100 and 3 or 4)
179 if xb == 0 then
180 if n + m > 78 then n = 0; w("\n") end
181 else
182 local name = defs[xb]:gsub("_", ".")
183 if n ~= 0 then w("\n") end
184 w("/* "); w(name); w(" */ ")
185 n = #name + 7
186 end
187 n = n + m
188 w(x)
189 end
190 w(",")
191 end
192 w("\n0\n};\n\n")
193 w("static const struct { const char *name; int ofs; } libbc_map[] = {\n")
194 local m = 0
195 for _,name in ipairs(defs) do
196 w('{"'); w(name); w('",'); w(m) w('},\n')
197 m = m + #defs[name].dump
198 end
199 w("{NULL,"); w(m); w("}\n};\n\n")
200 return table.concat(t)
201end
202
203local function write_file(name, data)
204 if name == "-" then
205 assert(io.write(data))
206 assert(io.flush())
207 else
208 local fp = io.open(name)
209 if fp then
210 local old = fp:read("*a")
211 fp:close()
212 if data == old then return end
213 end
214 fp = assert(io.open(name, "w"))
215 assert(fp:write(data))
216 assert(fp:close())
217 end
218end
219
220local outfile = parse_arg(arg)
221local src = read_files(arg)
222local defs = find_defs(src)
223local hdr = gen_header(defs)
224write_file(outfile, hdr)
225
diff --git a/src/jit/bc.lua b/src/jit/bc.lua
index 4b384012..f1a63b9c 100644
--- a/src/jit/bc.lua
+++ b/src/jit/bc.lua
@@ -178,13 +178,12 @@ local function bcliston(outfile)
178end 178end
179 179
180-- Public module functions. 180-- Public module functions.
181module(...) 181return {
182 182 line = bcline,
183line = bcline 183 dump = bcdump,
184dump = bcdump 184 targets = bctargets,
185targets = bctargets 185 on = bcliston,
186 186 off = bclistoff,
187on = bcliston 187 start = bcliston -- For -j command line option.
188off = bclistoff 188}
189start = bcliston -- For -j command line option.
190 189
diff --git a/src/jit/bcsave.lua b/src/jit/bcsave.lua
index 55fae993..74699f3d 100644
--- a/src/jit/bcsave.lua
+++ b/src/jit/bcsave.lua
@@ -11,12 +11,16 @@
11------------------------------------------------------------------------------ 11------------------------------------------------------------------------------
12 12
13local jit = require("jit") 13local jit = require("jit")
14assert(jit.version_num == 20099, "LuaJIT core/library version mismatch") 14assert(jit.version_num == 20199, "LuaJIT core/library version mismatch")
15local bit = require("bit") 15local bit = require("bit")
16 16
17-- Symbol name prefix for LuaJIT bytecode. 17-- Symbol name prefix for LuaJIT bytecode.
18local LJBC_PREFIX = "luaJIT_BC_" 18local LJBC_PREFIX = "luaJIT_BC_"
19 19
20local type, assert = type, assert
21local format = string.format
22local tremove, tconcat = table.remove, table.concat
23
20------------------------------------------------------------------------------ 24------------------------------------------------------------------------------
21 25
22local function usage() 26local function usage()
@@ -29,6 +33,7 @@ Save LuaJIT bytecode: luajit -b[options] input output
29 -t type Set output file type (default: auto-detect from output name). 33 -t type Set output file type (default: auto-detect from output name).
30 -a arch Override architecture for object files (default: native). 34 -a arch Override architecture for object files (default: native).
31 -o os Override OS for object files (default: native). 35 -o os Override OS for object files (default: native).
36 -F name Override filename (default: input filename).
32 -e chunk Use chunk string as input. 37 -e chunk Use chunk string as input.
33 -- Stop handling options. 38 -- Stop handling options.
34 - Use stdin as input and/or stdout as output. 39 - Use stdin as input and/or stdout as output.
@@ -45,10 +50,22 @@ local function check(ok, ...)
45 os.exit(1) 50 os.exit(1)
46end 51end
47 52
48local function readfile(input) 53local function readfile(ctx, input)
49 if type(input) == "function" then return input end 54 if type(input) == "function" then return input end
50 if input == "-" then input = nil end 55 if ctx.filename then
51 return check(loadfile(input)) 56 local data
57 if input == "-" then
58 data = io.stdin:read("*a")
59 else
60 local fp = assert(io.open(input, "rb"))
61 data = assert(fp:read("*a"))
62 assert(fp:close())
63 end
64 return check(load(data, ctx.filename))
65 else
66 if input == "-" then input = nil end
67 return check(loadfile(input))
68 end
52end 69end
53 70
54local function savefile(name, mode) 71local function savefile(name, mode)
@@ -56,6 +73,11 @@ local function savefile(name, mode)
56 return check(io.open(name, mode)) 73 return check(io.open(name, mode))
57end 74end
58 75
76local function set_stdout_binary(ffi)
77 ffi.cdef[[int _setmode(int fd, int mode);]]
78 ffi.C._setmode(1, 0x8000)
79end
80
59------------------------------------------------------------------------------ 81------------------------------------------------------------------------------
60 82
61local map_type = { 83local map_type = {
@@ -63,8 +85,18 @@ local map_type = {
63} 85}
64 86
65local map_arch = { 87local map_arch = {
66 x86 = true, x64 = true, arm = true, ppc = true, ppcspe = true, 88 x86 = { e = "le", b = 32, m = 3, p = 0x14c, },
67 mips = true, mipsel = true, 89 x64 = { e = "le", b = 64, m = 62, p = 0x8664, },
90 arm = { e = "le", b = 32, m = 40, p = 0x1c0, },
91 arm64 = { e = "le", b = 64, m = 183, p = 0xaa64, },
92 arm64be = { e = "be", b = 64, m = 183, },
93 ppc = { e = "be", b = 32, m = 20, },
94 mips = { e = "be", b = 32, m = 8, f = 0x50001006, },
95 mipsel = { e = "le", b = 32, m = 8, f = 0x50001006, },
96 mips64 = { e = "be", b = 64, m = 8, f = 0x80000007, },
97 mips64el = { e = "le", b = 64, m = 8, f = 0x80000007, },
98 mips64r6 = { e = "be", b = 64, m = 8, f = 0xa0000407, },
99 mips64r6el = { e = "le", b = 64, m = 8, f = 0xa0000407, },
68} 100}
69 101
70local map_os = { 102local map_os = {
@@ -73,33 +105,33 @@ local map_os = {
73} 105}
74 106
75local function checkarg(str, map, err) 107local function checkarg(str, map, err)
76 str = string.lower(str) 108 str = str:lower()
77 local s = check(map[str], "unknown ", err) 109 local s = check(map[str], "unknown ", err)
78 return s == true and str or s 110 return type(s) == "string" and s or str
79end 111end
80 112
81local function detecttype(str) 113local function detecttype(str)
82 local ext = string.match(string.lower(str), "%.(%a+)$") 114 local ext = str:lower():match("%.(%a+)$")
83 return map_type[ext] or "raw" 115 return map_type[ext] or "raw"
84end 116end
85 117
86local function checkmodname(str) 118local function checkmodname(str)
87 check(string.match(str, "^[%w_.%-]+$"), "bad module name") 119 check(str:match("^[%w_.%-]+$"), "bad module name")
88 return string.gsub(str, "[%.%-]", "_") 120 return str:gsub("[%.%-]", "_")
89end 121end
90 122
91local function detectmodname(str) 123local function detectmodname(str)
92 if type(str) == "string" then 124 if type(str) == "string" then
93 local tail = string.match(str, "[^/\\]+$") 125 local tail = str:match("[^/\\]+$")
94 if tail then str = tail end 126 if tail then str = tail end
95 local head = string.match(str, "^(.*)%.[^.]*$") 127 local head = str:match("^(.*)%.[^.]*$")
96 if head then str = head end 128 if head then str = head end
97 str = string.match(str, "^[%w_.%-]+") 129 str = str:match("^[%w_.%-]+")
98 else 130 else
99 str = nil 131 str = nil
100 end 132 end
101 check(str, "cannot derive module name, use -n name") 133 check(str, "cannot derive module name, use -n name")
102 return string.gsub(str, "[%.%-]", "_") 134 return str:gsub("[%.%-]", "_")
103end 135end
104 136
105------------------------------------------------------------------------------ 137------------------------------------------------------------------------------
@@ -111,6 +143,11 @@ local function bcsave_tail(fp, output, s)
111end 143end
112 144
113local function bcsave_raw(output, s) 145local function bcsave_raw(output, s)
146 if output == "-" and jit.os == "Windows" then
147 local ok, ffi = pcall(require, "ffi")
148 check(ok, "FFI library required to write binary file to stdout")
149 set_stdout_binary(ffi)
150 end
114 local fp = savefile(output, "wb") 151 local fp = savefile(output, "wb")
115 bcsave_tail(fp, output, s) 152 bcsave_tail(fp, output, s)
116end 153end
@@ -118,19 +155,19 @@ end
118local function bcsave_c(ctx, output, s) 155local function bcsave_c(ctx, output, s)
119 local fp = savefile(output, "w") 156 local fp = savefile(output, "w")
120 if ctx.type == "c" then 157 if ctx.type == "c" then
121 fp:write(string.format([[ 158 fp:write(format([[
122#ifdef __cplusplus 159#ifdef __cplusplus
123extern "C" 160extern "C"
124#endif 161#endif
125#ifdef _WIN32 162#ifdef _WIN32
126__declspec(dllexport) 163__declspec(dllexport)
127#endif 164#endif
128const char %s%s[] = { 165const unsigned char %s%s[] = {
129]], LJBC_PREFIX, ctx.modname)) 166]], LJBC_PREFIX, ctx.modname))
130 else 167 else
131 fp:write(string.format([[ 168 fp:write(format([[
132#define %s%s_SIZE %d 169#define %s%s_SIZE %d
133static const char %s%s[] = { 170static const unsigned char %s%s[] = {
134]], LJBC_PREFIX, ctx.modname, #s, LJBC_PREFIX, ctx.modname)) 171]], LJBC_PREFIX, ctx.modname, #s, LJBC_PREFIX, ctx.modname))
135 end 172 end
136 local t, n, m = {}, 0, 0 173 local t, n, m = {}, 0, 0
@@ -138,13 +175,13 @@ static const char %s%s[] = {
138 local b = tostring(string.byte(s, i)) 175 local b = tostring(string.byte(s, i))
139 m = m + #b + 1 176 m = m + #b + 1
140 if m > 78 then 177 if m > 78 then
141 fp:write(table.concat(t, ",", 1, n), ",\n") 178 fp:write(tconcat(t, ",", 1, n), ",\n")
142 n, m = 0, #b + 1 179 n, m = 0, #b + 1
143 end 180 end
144 n = n + 1 181 n = n + 1
145 t[n] = b 182 t[n] = b
146 end 183 end
147 bcsave_tail(fp, output, table.concat(t, ",", 1, n).."\n};\n") 184 bcsave_tail(fp, output, tconcat(t, ",", 1, n).."\n};\n")
148end 185end
149 186
150local function bcsave_elfobj(ctx, output, s, ffi) 187local function bcsave_elfobj(ctx, output, s, ffi)
@@ -199,12 +236,8 @@ typedef struct {
199} ELF64obj; 236} ELF64obj;
200]] 237]]
201 local symname = LJBC_PREFIX..ctx.modname 238 local symname = LJBC_PREFIX..ctx.modname
202 local is64, isbe = false, false 239 local ai = assert(map_arch[ctx.arch])
203 if ctx.arch == "x64" then 240 local is64, isbe = ai.b == 64, ai.e == "be"
204 is64 = true
205 elseif ctx.arch == "ppc" or ctx.arch == "ppcspe" or ctx.arch == "mips" then
206 isbe = true
207 end
208 241
209 -- Handle different host/target endianess. 242 -- Handle different host/target endianess.
210 local function f32(x) return x end 243 local function f32(x) return x end
@@ -237,10 +270,8 @@ typedef struct {
237 hdr.eendian = isbe and 2 or 1 270 hdr.eendian = isbe and 2 or 1
238 hdr.eversion = 1 271 hdr.eversion = 1
239 hdr.type = f16(1) 272 hdr.type = f16(1)
240 hdr.machine = f16(({ x86=3, x64=62, arm=40, ppc=20, ppcspe=20, mips=8, mipsel=8 })[ctx.arch]) 273 hdr.machine = f16(ai.m)
241 if ctx.arch == "mips" or ctx.arch == "mipsel" then 274 hdr.flags = f32(ai.f or 0)
242 hdr.flags = f32(0x50001006)
243 end
244 hdr.version = f32(1) 275 hdr.version = f32(1)
245 hdr.shofs = fofs(ffi.offsetof(o, "sect")) 276 hdr.shofs = fofs(ffi.offsetof(o, "sect"))
246 hdr.ehsize = f16(ffi.sizeof(hdr)) 277 hdr.ehsize = f16(ffi.sizeof(hdr))
@@ -336,12 +367,8 @@ typedef struct {
336} PEobj; 367} PEobj;
337]] 368]]
338 local symname = LJBC_PREFIX..ctx.modname 369 local symname = LJBC_PREFIX..ctx.modname
339 local is64 = false 370 local ai = assert(map_arch[ctx.arch])
340 if ctx.arch == "x86" then 371 local is64 = ai.b == 64
341 symname = "_"..symname
342 elseif ctx.arch == "x64" then
343 is64 = true
344 end
345 local symexport = " /EXPORT:"..symname..",DATA " 372 local symexport = " /EXPORT:"..symname..",DATA "
346 373
347 -- The file format is always little-endian. Swap if the host is big-endian. 374 -- The file format is always little-endian. Swap if the host is big-endian.
@@ -355,7 +382,7 @@ typedef struct {
355 -- Create PE object and fill in header. 382 -- Create PE object and fill in header.
356 local o = ffi.new("PEobj") 383 local o = ffi.new("PEobj")
357 local hdr = o.hdr 384 local hdr = o.hdr
358 hdr.arch = f16(({ x86=0x14c, x64=0x8664, arm=0x1c0, ppc=0x1f2, mips=0x366, mipsel=0x366 })[ctx.arch]) 385 hdr.arch = f16(assert(ai.p))
359 hdr.nsects = f16(2) 386 hdr.nsects = f16(2)
360 hdr.symtabofs = f32(ffi.offsetof(o, "sym0")) 387 hdr.symtabofs = f32(ffi.offsetof(o, "sym0"))
361 hdr.nsyms = f32(6) 388 hdr.nsyms = f32(6)
@@ -442,18 +469,18 @@ typedef struct {
442 uint32_t value; 469 uint32_t value;
443} mach_nlist; 470} mach_nlist;
444typedef struct { 471typedef struct {
445 uint32_t strx; 472 int32_t strx;
446 uint8_t type, sect; 473 uint8_t type, sect;
447 uint16_t desc; 474 uint16_t desc;
448 uint64_t value; 475 uint64_t value;
449} mach_nlist_64; 476} mach_nlist_64;
450typedef struct 477typedef struct
451{ 478{
452 uint32_t magic, nfat_arch; 479 int32_t magic, nfat_arch;
453} mach_fat_header; 480} mach_fat_header;
454typedef struct 481typedef struct
455{ 482{
456 uint32_t cputype, cpusubtype, offset, size, align; 483 int32_t cputype, cpusubtype, offset, size, align;
457} mach_fat_arch; 484} mach_fat_arch;
458typedef struct { 485typedef struct {
459 struct { 486 struct {
@@ -477,16 +504,28 @@ typedef struct {
477} mach_obj_64; 504} mach_obj_64;
478typedef struct { 505typedef struct {
479 mach_fat_header fat; 506 mach_fat_header fat;
480 mach_fat_arch fat_arch[4]; 507 mach_fat_arch fat_arch[2];
481 struct { 508 struct {
482 mach_header hdr; 509 mach_header hdr;
483 mach_segment_command seg; 510 mach_segment_command seg;
484 mach_section sec; 511 mach_section sec;
485 mach_symtab_command sym; 512 mach_symtab_command sym;
486 } arch[4]; 513 } arch[2];
487 mach_nlist sym_entry; 514 mach_nlist sym_entry;
488 uint8_t space[4096]; 515 uint8_t space[4096];
489} mach_fat_obj; 516} mach_fat_obj;
517typedef struct {
518 mach_fat_header fat;
519 mach_fat_arch fat_arch[2];
520 struct {
521 mach_header_64 hdr;
522 mach_segment_command_64 seg;
523 mach_section_64 sec;
524 mach_symtab_command sym;
525 } arch[2];
526 mach_nlist_64 sym_entry;
527 uint8_t space[4096];
528} mach_fat_obj_64;
490]] 529]]
491 local symname = '_'..LJBC_PREFIX..ctx.modname 530 local symname = '_'..LJBC_PREFIX..ctx.modname
492 local isfat, is64, align, mobj = false, false, 4, "mach_obj" 531 local isfat, is64, align, mobj = false, false, 4, "mach_obj"
@@ -494,6 +533,8 @@ typedef struct {
494 is64, align, mobj = true, 8, "mach_obj_64" 533 is64, align, mobj = true, 8, "mach_obj_64"
495 elseif ctx.arch == "arm" then 534 elseif ctx.arch == "arm" then
496 isfat, mobj = true, "mach_fat_obj" 535 isfat, mobj = true, "mach_fat_obj"
536 elseif ctx.arch == "arm64" then
537 is64, align, isfat, mobj = true, 8, true, "mach_fat_obj_64"
497 else 538 else
498 check(ctx.arch == "x86", "unsupported architecture for OSX") 539 check(ctx.arch == "x86", "unsupported architecture for OSX")
499 end 540 end
@@ -503,8 +544,8 @@ typedef struct {
503 -- Create Mach-O object and fill in header. 544 -- Create Mach-O object and fill in header.
504 local o = ffi.new(mobj) 545 local o = ffi.new(mobj)
505 local mach_size = aligned(ffi.offsetof(o, "space")+#symname+2, align) 546 local mach_size = aligned(ffi.offsetof(o, "space")+#symname+2, align)
506 local cputype = ({ x86={7}, x64={0x01000007}, arm={7,12,12,12} })[ctx.arch] 547 local cputype = ({ x86={7}, x64={0x01000007}, arm={7,12}, arm64={0x01000007,0x0100000c} })[ctx.arch]
507 local cpusubtype = ({ x86={3}, x64={3}, arm={3,6,9,11} })[ctx.arch] 548 local cpusubtype = ({ x86={3}, x64={3}, arm={3,9}, arm64={3,0} })[ctx.arch]
508 if isfat then 549 if isfat then
509 o.fat.magic = be32(0xcafebabe) 550 o.fat.magic = be32(0xcafebabe)
510 o.fat.nfat_arch = be32(#cpusubtype) 551 o.fat.nfat_arch = be32(#cpusubtype)
@@ -562,6 +603,9 @@ end
562local function bcsave_obj(ctx, output, s) 603local function bcsave_obj(ctx, output, s)
563 local ok, ffi = pcall(require, "ffi") 604 local ok, ffi = pcall(require, "ffi")
564 check(ok, "FFI library required to write this file type") 605 check(ok, "FFI library required to write this file type")
606 if output == "-" and jit.os == "Windows" then
607 set_stdout_binary(ffi)
608 end
565 if ctx.os == "windows" then 609 if ctx.os == "windows" then
566 return bcsave_peobj(ctx, output, s, ffi) 610 return bcsave_peobj(ctx, output, s, ffi)
567 elseif ctx.os == "osx" then 611 elseif ctx.os == "osx" then
@@ -573,13 +617,13 @@ end
573 617
574------------------------------------------------------------------------------ 618------------------------------------------------------------------------------
575 619
576local function bclist(input, output) 620local function bclist(ctx, input, output)
577 local f = readfile(input) 621 local f = readfile(ctx, input)
578 require("jit.bc").dump(f, savefile(output, "w"), true) 622 require("jit.bc").dump(f, savefile(output, "w"), true)
579end 623end
580 624
581local function bcsave(ctx, input, output) 625local function bcsave(ctx, input, output)
582 local f = readfile(input) 626 local f = readfile(ctx, input)
583 local s = string.dump(f, ctx.strip) 627 local s = string.dump(f, ctx.strip)
584 local t = ctx.type 628 local t = ctx.type
585 if not t then 629 if not t then
@@ -603,16 +647,16 @@ local function docmd(...)
603 local n = 1 647 local n = 1
604 local list = false 648 local list = false
605 local ctx = { 649 local ctx = {
606 strip = true, arch = jit.arch, os = string.lower(jit.os), 650 strip = true, arch = jit.arch, os = jit.os:lower(),
607 type = false, modname = false, 651 type = false, modname = false,
608 } 652 }
609 while n <= #arg do 653 while n <= #arg do
610 local a = arg[n] 654 local a = arg[n]
611 if type(a) == "string" and string.sub(a, 1, 1) == "-" and a ~= "-" then 655 if type(a) == "string" and a:sub(1, 1) == "-" and a ~= "-" then
612 table.remove(arg, n) 656 tremove(arg, n)
613 if a == "--" then break end 657 if a == "--" then break end
614 for m=2,#a do 658 for m=2,#a do
615 local opt = string.sub(a, m, m) 659 local opt = a:sub(m, m)
616 if opt == "l" then 660 if opt == "l" then
617 list = true 661 list = true
618 elseif opt == "s" then 662 elseif opt == "s" then
@@ -625,13 +669,15 @@ local function docmd(...)
625 if n ~= 1 then usage() end 669 if n ~= 1 then usage() end
626 arg[1] = check(loadstring(arg[1])) 670 arg[1] = check(loadstring(arg[1]))
627 elseif opt == "n" then 671 elseif opt == "n" then
628 ctx.modname = checkmodname(table.remove(arg, n)) 672 ctx.modname = checkmodname(tremove(arg, n))
629 elseif opt == "t" then 673 elseif opt == "t" then
630 ctx.type = checkarg(table.remove(arg, n), map_type, "file type") 674 ctx.type = checkarg(tremove(arg, n), map_type, "file type")
631 elseif opt == "a" then 675 elseif opt == "a" then
632 ctx.arch = checkarg(table.remove(arg, n), map_arch, "architecture") 676 ctx.arch = checkarg(tremove(arg, n), map_arch, "architecture")
633 elseif opt == "o" then 677 elseif opt == "o" then
634 ctx.os = checkarg(table.remove(arg, n), map_os, "OS name") 678 ctx.os = checkarg(tremove(arg, n), map_os, "OS name")
679 elseif opt == "F" then
680 ctx.filename = "@"..tremove(arg, n)
635 else 681 else
636 usage() 682 usage()
637 end 683 end
@@ -643,7 +689,7 @@ local function docmd(...)
643 end 689 end
644 if list then 690 if list then
645 if #arg == 0 or #arg > 2 then usage() end 691 if #arg == 0 or #arg > 2 then usage() end
646 bclist(arg[1], arg[2] or "-") 692 bclist(ctx, arg[1], arg[2] or "-")
647 else 693 else
648 if #arg ~= 2 then usage() end 694 if #arg ~= 2 then usage() end
649 bcsave(ctx, arg[1], arg[2]) 695 bcsave(ctx, arg[1], arg[2])
@@ -653,7 +699,7 @@ end
653------------------------------------------------------------------------------ 699------------------------------------------------------------------------------
654 700
655-- Public module functions. 701-- Public module functions.
656module(...) 702return {
657 703 start = docmd -- Process -b command line option.
658start = docmd -- Process -b command line option. 704}
659 705
diff --git a/src/jit/dis_arm.lua b/src/jit/dis_arm.lua
index 4db85306..a7546a45 100644
--- a/src/jit/dis_arm.lua
+++ b/src/jit/dis_arm.lua
@@ -658,7 +658,7 @@ local function disass_block(ctx, ofs, len)
658end 658end
659 659
660-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len). 660-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
661local function create_(code, addr, out) 661local function create(code, addr, out)
662 local ctx = {} 662 local ctx = {}
663 ctx.code = code 663 ctx.code = code
664 ctx.addr = addr or 0 664 ctx.addr = addr or 0
@@ -670,20 +670,20 @@ local function create_(code, addr, out)
670end 670end
671 671
672-- Simple API: disassemble code (a string) at address and output via out. 672-- Simple API: disassemble code (a string) at address and output via out.
673local function disass_(code, addr, out) 673local function disass(code, addr, out)
674 create_(code, addr, out):disass() 674 create(code, addr, out):disass()
675end 675end
676 676
677-- Return register name for RID. 677-- Return register name for RID.
678local function regname_(r) 678local function regname(r)
679 if r < 16 then return map_gpr[r] end 679 if r < 16 then return map_gpr[r] end
680 return "d"..(r-16) 680 return "d"..(r-16)
681end 681end
682 682
683-- Public module functions. 683-- Public module functions.
684module(...) 684return {
685 685 create = create,
686create = create_ 686 disass = disass,
687disass = disass_ 687 regname = regname
688regname = regname_ 688}
689 689
diff --git a/src/jit/dis_arm64.lua b/src/jit/dis_arm64.lua
new file mode 100644
index 00000000..b10e2fb1
--- /dev/null
+++ b/src/jit/dis_arm64.lua
@@ -0,0 +1,1216 @@
1----------------------------------------------------------------------------
2-- LuaJIT ARM64 disassembler module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6--
7-- Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com.
8-- Sponsored by Cisco Systems, Inc.
9----------------------------------------------------------------------------
10-- This is a helper module used by the LuaJIT machine code dumper module.
11--
12-- It disassembles most user-mode AArch64 instructions.
13-- NYI: Advanced SIMD and VFP instructions.
14------------------------------------------------------------------------------
15
16local type = type
17local sub, byte, format = string.sub, string.byte, string.format
18local match, gmatch, gsub = string.match, string.gmatch, string.gsub
19local concat = table.concat
20local bit = require("bit")
21local band, bor, bxor, tohex = bit.band, bit.bor, bit.bxor, bit.tohex
22local lshift, rshift, arshift = bit.lshift, bit.rshift, bit.arshift
23local ror = bit.ror
24
25------------------------------------------------------------------------------
26-- Opcode maps
27------------------------------------------------------------------------------
28
29local map_adr = { -- PC-relative addressing.
30 shift = 31, mask = 1,
31 [0] = "adrDBx", "adrpDBx"
32}
33
34local map_addsubi = { -- Add/subtract immediate.
35 shift = 29, mask = 3,
36 [0] = "add|movDNIg", "adds|cmnD0NIg", "subDNIg", "subs|cmpD0NIg",
37}
38
39local map_logi = { -- Logical immediate.
40 shift = 31, mask = 1,
41 [0] = {
42 shift = 22, mask = 1,
43 [0] = {
44 shift = 29, mask = 3,
45 [0] = "andDNig", "orr|movDN0ig", "eorDNig", "ands|tstD0Nig"
46 },
47 false -- unallocated
48 },
49 {
50 shift = 29, mask = 3,
51 [0] = "andDNig", "orr|movDN0ig", "eorDNig", "ands|tstD0Nig"
52 }
53}
54
55local map_movwi = { -- Move wide immediate.
56 shift = 31, mask = 1,
57 [0] = {
58 shift = 22, mask = 1,
59 [0] = {
60 shift = 29, mask = 3,
61 [0] = "movnDWRg", false, "movz|movDYRg", "movkDWRg"
62 }, false -- unallocated
63 },
64 {
65 shift = 29, mask = 3,
66 [0] = "movnDWRg", false, "movz|movDYRg", "movkDWRg"
67 },
68}
69
70local map_bitf = { -- Bitfield.
71 shift = 31, mask = 1,
72 [0] = {
73 shift = 22, mask = 1,
74 [0] = {
75 shift = 29, mask = 3,
76 [0] = "sbfm|sbfiz|sbfx|asr|sxtw|sxth|sxtbDN12w",
77 "bfm|bfi|bfxilDN13w",
78 "ubfm|ubfiz|ubfx|lsr|lsl|uxth|uxtbDN12w"
79 }
80 },
81 {
82 shift = 22, mask = 1,
83 {
84 shift = 29, mask = 3,
85 [0] = "sbfm|sbfiz|sbfx|asr|sxtw|sxth|sxtbDN12x",
86 "bfm|bfi|bfxilDN13x",
87 "ubfm|ubfiz|ubfx|lsr|lsl|uxth|uxtbDN12x"
88 }
89 }
90}
91
92local map_datai = { -- Data processing - immediate.
93 shift = 23, mask = 7,
94 [0] = map_adr, map_adr, map_addsubi, false,
95 map_logi, map_movwi, map_bitf,
96 {
97 shift = 15, mask = 0x1c0c1,
98 [0] = "extr|rorDNM4w", [0x10080] = "extr|rorDNM4x",
99 [0x10081] = "extr|rorDNM4x"
100 }
101}
102
103local map_logsr = { -- Logical, shifted register.
104 shift = 31, mask = 1,
105 [0] = {
106 shift = 15, mask = 1,
107 [0] = {
108 shift = 29, mask = 3,
109 [0] = {
110 shift = 21, mask = 7,
111 [0] = "andDNMSg", "bicDNMSg", "andDNMSg", "bicDNMSg",
112 "andDNMSg", "bicDNMSg", "andDNMg", "bicDNMg"
113 },
114 {
115 shift = 21, mask = 7,
116 [0] ="orr|movDN0MSg", "orn|mvnDN0MSg", "orr|movDN0MSg", "orn|mvnDN0MSg",
117 "orr|movDN0MSg", "orn|mvnDN0MSg", "orr|movDN0Mg", "orn|mvnDN0Mg"
118 },
119 {
120 shift = 21, mask = 7,
121 [0] = "eorDNMSg", "eonDNMSg", "eorDNMSg", "eonDNMSg",
122 "eorDNMSg", "eonDNMSg", "eorDNMg", "eonDNMg"
123 },
124 {
125 shift = 21, mask = 7,
126 [0] = "ands|tstD0NMSg", "bicsDNMSg", "ands|tstD0NMSg", "bicsDNMSg",
127 "ands|tstD0NMSg", "bicsDNMSg", "ands|tstD0NMg", "bicsDNMg"
128 }
129 },
130 false -- unallocated
131 },
132 {
133 shift = 29, mask = 3,
134 [0] = {
135 shift = 21, mask = 7,
136 [0] = "andDNMSg", "bicDNMSg", "andDNMSg", "bicDNMSg",
137 "andDNMSg", "bicDNMSg", "andDNMg", "bicDNMg"
138 },
139 {
140 shift = 21, mask = 7,
141 [0] = "orr|movDN0MSg", "orn|mvnDN0MSg", "orr|movDN0MSg", "orn|mvnDN0MSg",
142 "orr|movDN0MSg", "orn|mvnDN0MSg", "orr|movDN0Mg", "orn|mvnDN0Mg"
143 },
144 {
145 shift = 21, mask = 7,
146 [0] = "eorDNMSg", "eonDNMSg", "eorDNMSg", "eonDNMSg",
147 "eorDNMSg", "eonDNMSg", "eorDNMg", "eonDNMg"
148 },
149 {
150 shift = 21, mask = 7,
151 [0] = "ands|tstD0NMSg", "bicsDNMSg", "ands|tstD0NMSg", "bicsDNMSg",
152 "ands|tstD0NMSg", "bicsDNMSg", "ands|tstD0NMg", "bicsDNMg"
153 }
154 }
155}
156
157local map_assh = {
158 shift = 31, mask = 1,
159 [0] = {
160 shift = 15, mask = 1,
161 [0] = {
162 shift = 29, mask = 3,
163 [0] = {
164 shift = 22, mask = 3,
165 [0] = "addDNMSg", "addDNMSg", "addDNMSg", "addDNMg"
166 },
167 {
168 shift = 22, mask = 3,
169 [0] = "adds|cmnD0NMSg", "adds|cmnD0NMSg",
170 "adds|cmnD0NMSg", "adds|cmnD0NMg"
171 },
172 {
173 shift = 22, mask = 3,
174 [0] = "sub|negDN0MSg", "sub|negDN0MSg", "sub|negDN0MSg", "sub|negDN0Mg"
175 },
176 {
177 shift = 22, mask = 3,
178 [0] = "subs|cmp|negsD0N0MzSg", "subs|cmp|negsD0N0MzSg",
179 "subs|cmp|negsD0N0MzSg", "subs|cmp|negsD0N0Mzg"
180 },
181 },
182 false -- unallocated
183 },
184 {
185 shift = 29, mask = 3,
186 [0] = {
187 shift = 22, mask = 3,
188 [0] = "addDNMSg", "addDNMSg", "addDNMSg", "addDNMg"
189 },
190 {
191 shift = 22, mask = 3,
192 [0] = "adds|cmnD0NMSg", "adds|cmnD0NMSg", "adds|cmnD0NMSg",
193 "adds|cmnD0NMg"
194 },
195 {
196 shift = 22, mask = 3,
197 [0] = "sub|negDN0MSg", "sub|negDN0MSg", "sub|negDN0MSg", "sub|negDN0Mg"
198 },
199 {
200 shift = 22, mask = 3,
201 [0] = "subs|cmp|negsD0N0MzSg", "subs|cmp|negsD0N0MzSg",
202 "subs|cmp|negsD0N0MzSg", "subs|cmp|negsD0N0Mzg"
203 }
204 }
205}
206
207local map_addsubsh = { -- Add/subtract, shifted register.
208 shift = 22, mask = 3,
209 [0] = map_assh, map_assh, map_assh
210}
211
212local map_addsubex = { -- Add/subtract, extended register.
213 shift = 22, mask = 3,
214 [0] = {
215 shift = 29, mask = 3,
216 [0] = "addDNMXg", "adds|cmnD0NMXg", "subDNMXg", "subs|cmpD0NMzXg",
217 }
218}
219
220local map_addsubc = { -- Add/subtract, with carry.
221 shift = 10, mask = 63,
222 [0] = {
223 shift = 29, mask = 3,
224 [0] = "adcDNMg", "adcsDNMg", "sbc|ngcDN0Mg", "sbcs|ngcsDN0Mg",
225 }
226}
227
228local map_ccomp = {
229 shift = 4, mask = 1,
230 [0] = {
231 shift = 10, mask = 3,
232 [0] = { -- Conditional compare register.
233 shift = 29, mask = 3,
234 "ccmnNMVCg", false, "ccmpNMVCg",
235 },
236 [2] = { -- Conditional compare immediate.
237 shift = 29, mask = 3,
238 "ccmnN5VCg", false, "ccmpN5VCg",
239 }
240 }
241}
242
243local map_csel = { -- Conditional select.
244 shift = 11, mask = 1,
245 [0] = {
246 shift = 10, mask = 1,
247 [0] = {
248 shift = 29, mask = 3,
249 [0] = "cselDNMzCg", false, "csinv|cinv|csetmDNMcg", false,
250 },
251 {
252 shift = 29, mask = 3,
253 [0] = "csinc|cinc|csetDNMcg", false, "csneg|cnegDNMcg", false,
254 }
255 }
256}
257
258local map_data1s = { -- Data processing, 1 source.
259 shift = 29, mask = 1,
260 [0] = {
261 shift = 31, mask = 1,
262 [0] = {
263 shift = 10, mask = 0x7ff,
264 [0] = "rbitDNg", "rev16DNg", "revDNw", false, "clzDNg", "clsDNg"
265 },
266 {
267 shift = 10, mask = 0x7ff,
268 [0] = "rbitDNg", "rev16DNg", "rev32DNx", "revDNx", "clzDNg", "clsDNg"
269 }
270 }
271}
272
273local map_data2s = { -- Data processing, 2 sources.
274 shift = 29, mask = 1,
275 [0] = {
276 shift = 10, mask = 63,
277 false, "udivDNMg", "sdivDNMg", false, false, false, false, "lslDNMg",
278 "lsrDNMg", "asrDNMg", "rorDNMg"
279 }
280}
281
282local map_data3s = { -- Data processing, 3 sources.
283 shift = 29, mask = 7,
284 [0] = {
285 shift = 21, mask = 7,
286 [0] = {
287 shift = 15, mask = 1,
288 [0] = "madd|mulDNMA0g", "msub|mnegDNMA0g"
289 }
290 }, false, false, false,
291 {
292 shift = 15, mask = 1,
293 [0] = {
294 shift = 21, mask = 7,
295 [0] = "madd|mulDNMA0g", "smaddl|smullDxNMwA0x", "smulhDNMx", false,
296 false, "umaddl|umullDxNMwA0x", "umulhDNMx"
297 },
298 {
299 shift = 21, mask = 7,
300 [0] = "msub|mnegDNMA0g", "smsubl|smneglDxNMwA0x", false, false,
301 false, "umsubl|umneglDxNMwA0x"
302 }
303 }
304}
305
306local map_datar = { -- Data processing, register.
307 shift = 28, mask = 1,
308 [0] = {
309 shift = 24, mask = 1,
310 [0] = map_logsr,
311 {
312 shift = 21, mask = 1,
313 [0] = map_addsubsh, map_addsubex
314 }
315 },
316 {
317 shift = 21, mask = 15,
318 [0] = map_addsubc, false, map_ccomp, false, map_csel, false,
319 {
320 shift = 30, mask = 1,
321 [0] = map_data2s, map_data1s
322 },
323 false, map_data3s, map_data3s, map_data3s, map_data3s, map_data3s,
324 map_data3s, map_data3s, map_data3s
325 }
326}
327
328local map_lrl = { -- Load register, literal.
329 shift = 26, mask = 1,
330 [0] = {
331 shift = 30, mask = 3,
332 [0] = "ldrDwB", "ldrDxB", "ldrswDxB"
333 },
334 {
335 shift = 30, mask = 3,
336 [0] = "ldrDsB", "ldrDdB"
337 }
338}
339
340local map_lsriind = { -- Load/store register, immediate pre/post-indexed.
341 shift = 30, mask = 3,
342 [0] = {
343 shift = 26, mask = 1,
344 [0] = {
345 shift = 22, mask = 3,
346 [0] = "strbDwzL", "ldrbDwzL", "ldrsbDxzL", "ldrsbDwzL"
347 }
348 },
349 {
350 shift = 26, mask = 1,
351 [0] = {
352 shift = 22, mask = 3,
353 [0] = "strhDwzL", "ldrhDwzL", "ldrshDxzL", "ldrshDwzL"
354 }
355 },
356 {
357 shift = 26, mask = 1,
358 [0] = {
359 shift = 22, mask = 3,
360 [0] = "strDwzL", "ldrDwzL", "ldrswDxzL"
361 },
362 {
363 shift = 22, mask = 3,
364 [0] = "strDszL", "ldrDszL"
365 }
366 },
367 {
368 shift = 26, mask = 1,
369 [0] = {
370 shift = 22, mask = 3,
371 [0] = "strDxzL", "ldrDxzL"
372 },
373 {
374 shift = 22, mask = 3,
375 [0] = "strDdzL", "ldrDdzL"
376 }
377 }
378}
379
380local map_lsriro = {
381 shift = 21, mask = 1,
382 [0] = { -- Load/store register immediate.
383 shift = 10, mask = 3,
384 [0] = { -- Unscaled immediate.
385 shift = 26, mask = 1,
386 [0] = {
387 shift = 30, mask = 3,
388 [0] = {
389 shift = 22, mask = 3,
390 [0] = "sturbDwK", "ldurbDwK"
391 },
392 {
393 shift = 22, mask = 3,
394 [0] = "sturhDwK", "ldurhDwK"
395 },
396 {
397 shift = 22, mask = 3,
398 [0] = "sturDwK", "ldurDwK"
399 },
400 {
401 shift = 22, mask = 3,
402 [0] = "sturDxK", "ldurDxK"
403 }
404 }
405 }, map_lsriind, false, map_lsriind
406 },
407 { -- Load/store register, register offset.
408 shift = 10, mask = 3,
409 [2] = {
410 shift = 26, mask = 1,
411 [0] = {
412 shift = 30, mask = 3,
413 [0] = {
414 shift = 22, mask = 3,
415 [0] = "strbDwO", "ldrbDwO", "ldrsbDxO", "ldrsbDwO"
416 },
417 {
418 shift = 22, mask = 3,
419 [0] = "strhDwO", "ldrhDwO", "ldrshDxO", "ldrshDwO"
420 },
421 {
422 shift = 22, mask = 3,
423 [0] = "strDwO", "ldrDwO", "ldrswDxO"
424 },
425 {
426 shift = 22, mask = 3,
427 [0] = "strDxO", "ldrDxO"
428 }
429 },
430 {
431 shift = 30, mask = 3,
432 [2] = {
433 shift = 22, mask = 3,
434 [0] = "strDsO", "ldrDsO"
435 },
436 [3] = {
437 shift = 22, mask = 3,
438 [0] = "strDdO", "ldrDdO"
439 }
440 }
441 }
442 }
443}
444
445local map_lsp = { -- Load/store register pair, offset.
446 shift = 22, mask = 1,
447 [0] = {
448 shift = 30, mask = 3,
449 [0] = {
450 shift = 26, mask = 1,
451 [0] = "stpDzAzwP", "stpDzAzsP",
452 },
453 {
454 shift = 26, mask = 1,
455 "stpDzAzdP"
456 },
457 {
458 shift = 26, mask = 1,
459 [0] = "stpDzAzxP"
460 }
461 },
462 {
463 shift = 30, mask = 3,
464 [0] = {
465 shift = 26, mask = 1,
466 [0] = "ldpDzAzwP", "ldpDzAzsP",
467 },
468 {
469 shift = 26, mask = 1,
470 [0] = "ldpswDAxP", "ldpDzAzdP"
471 },
472 {
473 shift = 26, mask = 1,
474 [0] = "ldpDzAzxP"
475 }
476 }
477}
478
479local map_ls = { -- Loads and stores.
480 shift = 24, mask = 0x31,
481 [0x10] = map_lrl, [0x30] = map_lsriro,
482 [0x20] = {
483 shift = 23, mask = 3,
484 map_lsp, map_lsp, map_lsp
485 },
486 [0x21] = {
487 shift = 23, mask = 3,
488 map_lsp, map_lsp, map_lsp
489 },
490 [0x31] = {
491 shift = 26, mask = 1,
492 [0] = {
493 shift = 30, mask = 3,
494 [0] = {
495 shift = 22, mask = 3,
496 [0] = "strbDwzU", "ldrbDwzU"
497 },
498 {
499 shift = 22, mask = 3,
500 [0] = "strhDwzU", "ldrhDwzU"
501 },
502 {
503 shift = 22, mask = 3,
504 [0] = "strDwzU", "ldrDwzU"
505 },
506 {
507 shift = 22, mask = 3,
508 [0] = "strDxzU", "ldrDxzU"
509 }
510 },
511 {
512 shift = 30, mask = 3,
513 [2] = {
514 shift = 22, mask = 3,
515 [0] = "strDszU", "ldrDszU"
516 },
517 [3] = {
518 shift = 22, mask = 3,
519 [0] = "strDdzU", "ldrDdzU"
520 }
521 }
522 },
523}
524
525local map_datafp = { -- Data processing, SIMD and FP.
526 shift = 28, mask = 7,
527 { -- 001
528 shift = 24, mask = 1,
529 [0] = {
530 shift = 21, mask = 1,
531 {
532 shift = 10, mask = 3,
533 [0] = {
534 shift = 12, mask = 1,
535 [0] = {
536 shift = 13, mask = 1,
537 [0] = {
538 shift = 14, mask = 1,
539 [0] = {
540 shift = 15, mask = 1,
541 [0] = { -- FP/int conversion.
542 shift = 31, mask = 1,
543 [0] = {
544 shift = 16, mask = 0xff,
545 [0x20] = "fcvtnsDwNs", [0x21] = "fcvtnuDwNs",
546 [0x22] = "scvtfDsNw", [0x23] = "ucvtfDsNw",
547 [0x24] = "fcvtasDwNs", [0x25] = "fcvtauDwNs",
548 [0x26] = "fmovDwNs", [0x27] = "fmovDsNw",
549 [0x28] = "fcvtpsDwNs", [0x29] = "fcvtpuDwNs",
550 [0x30] = "fcvtmsDwNs", [0x31] = "fcvtmuDwNs",
551 [0x38] = "fcvtzsDwNs", [0x39] = "fcvtzuDwNs",
552 [0x60] = "fcvtnsDwNd", [0x61] = "fcvtnuDwNd",
553 [0x62] = "scvtfDdNw", [0x63] = "ucvtfDdNw",
554 [0x64] = "fcvtasDwNd", [0x65] = "fcvtauDwNd",
555 [0x68] = "fcvtpsDwNd", [0x69] = "fcvtpuDwNd",
556 [0x70] = "fcvtmsDwNd", [0x71] = "fcvtmuDwNd",
557 [0x78] = "fcvtzsDwNd", [0x79] = "fcvtzuDwNd"
558 },
559 {
560 shift = 16, mask = 0xff,
561 [0x20] = "fcvtnsDxNs", [0x21] = "fcvtnuDxNs",
562 [0x22] = "scvtfDsNx", [0x23] = "ucvtfDsNx",
563 [0x24] = "fcvtasDxNs", [0x25] = "fcvtauDxNs",
564 [0x28] = "fcvtpsDxNs", [0x29] = "fcvtpuDxNs",
565 [0x30] = "fcvtmsDxNs", [0x31] = "fcvtmuDxNs",
566 [0x38] = "fcvtzsDxNs", [0x39] = "fcvtzuDxNs",
567 [0x60] = "fcvtnsDxNd", [0x61] = "fcvtnuDxNd",
568 [0x62] = "scvtfDdNx", [0x63] = "ucvtfDdNx",
569 [0x64] = "fcvtasDxNd", [0x65] = "fcvtauDxNd",
570 [0x66] = "fmovDxNd", [0x67] = "fmovDdNx",
571 [0x68] = "fcvtpsDxNd", [0x69] = "fcvtpuDxNd",
572 [0x70] = "fcvtmsDxNd", [0x71] = "fcvtmuDxNd",
573 [0x78] = "fcvtzsDxNd", [0x79] = "fcvtzuDxNd"
574 }
575 }
576 },
577 { -- FP data-processing, 1 source.
578 shift = 31, mask = 1,
579 [0] = {
580 shift = 22, mask = 3,
581 [0] = {
582 shift = 15, mask = 63,
583 [0] = "fmovDNf", "fabsDNf", "fnegDNf",
584 "fsqrtDNf", false, "fcvtDdNs", false, false,
585 "frintnDNf", "frintpDNf", "frintmDNf", "frintzDNf",
586 "frintaDNf", false, "frintxDNf", "frintiDNf",
587 },
588 {
589 shift = 15, mask = 63,
590 [0] = "fmovDNf", "fabsDNf", "fnegDNf",
591 "fsqrtDNf", "fcvtDsNd", false, false, false,
592 "frintnDNf", "frintpDNf", "frintmDNf", "frintzDNf",
593 "frintaDNf", false, "frintxDNf", "frintiDNf",
594 }
595 }
596 }
597 },
598 { -- FP compare.
599 shift = 31, mask = 1,
600 [0] = {
601 shift = 14, mask = 3,
602 [0] = {
603 shift = 23, mask = 1,
604 [0] = {
605 shift = 0, mask = 31,
606 [0] = "fcmpNMf", [8] = "fcmpNZf",
607 [16] = "fcmpeNMf", [24] = "fcmpeNZf",
608 }
609 }
610 }
611 }
612 },
613 { -- FP immediate.
614 shift = 31, mask = 1,
615 [0] = {
616 shift = 5, mask = 31,
617 [0] = {
618 shift = 23, mask = 1,
619 [0] = "fmovDFf"
620 }
621 }
622 }
623 },
624 { -- FP conditional compare.
625 shift = 31, mask = 1,
626 [0] = {
627 shift = 23, mask = 1,
628 [0] = {
629 shift = 4, mask = 1,
630 [0] = "fccmpNMVCf", "fccmpeNMVCf"
631 }
632 }
633 },
634 { -- FP data-processing, 2 sources.
635 shift = 31, mask = 1,
636 [0] = {
637 shift = 23, mask = 1,
638 [0] = {
639 shift = 12, mask = 15,
640 [0] = "fmulDNMf", "fdivDNMf", "faddDNMf", "fsubDNMf",
641 "fmaxDNMf", "fminDNMf", "fmaxnmDNMf", "fminnmDNMf",
642 "fnmulDNMf"
643 }
644 }
645 },
646 { -- FP conditional select.
647 shift = 31, mask = 1,
648 [0] = {
649 shift = 23, mask = 1,
650 [0] = "fcselDNMCf"
651 }
652 }
653 }
654 },
655 { -- FP data-processing, 3 sources.
656 shift = 31, mask = 1,
657 [0] = {
658 shift = 15, mask = 1,
659 [0] = {
660 shift = 21, mask = 5,
661 [0] = "fmaddDNMAf", "fnmaddDNMAf"
662 },
663 {
664 shift = 21, mask = 5,
665 [0] = "fmsubDNMAf", "fnmsubDNMAf"
666 }
667 }
668 }
669 }
670}
671
672local map_br = { -- Branches, exception generating and system instructions.
673 shift = 29, mask = 7,
674 [0] = "bB",
675 { -- Compare & branch, immediate.
676 shift = 24, mask = 3,
677 [0] = "cbzDBg", "cbnzDBg", "tbzDTBw", "tbnzDTBw"
678 },
679 { -- Conditional branch, immediate.
680 shift = 24, mask = 3,
681 [0] = {
682 shift = 4, mask = 1,
683 [0] = {
684 shift = 0, mask = 15,
685 [0] = "beqB", "bneB", "bhsB", "bloB", "bmiB", "bplB", "bvsB", "bvcB",
686 "bhiB", "blsB", "bgeB", "bltB", "bgtB", "bleB", "balB"
687 }
688 }
689 }, false, "blB",
690 { -- Compare & branch, immediate.
691 shift = 24, mask = 3,
692 [0] = "cbzDBg", "cbnzDBg", "tbzDTBx", "tbnzDTBx"
693 },
694 {
695 shift = 24, mask = 3,
696 [0] = { -- Exception generation.
697 shift = 0, mask = 0xe0001f,
698 [0x200000] = "brkW"
699 },
700 { -- System instructions.
701 shift = 0, mask = 0x3fffff,
702 [0x03201f] = "nop"
703 },
704 { -- Unconditional branch, register.
705 shift = 0, mask = 0xfffc1f,
706 [0x1f0000] = "brNx", [0x3f0000] = "blrNx",
707 [0x5f0000] = "retNx"
708 },
709 }
710}
711
712local map_init = {
713 shift = 25, mask = 15,
714 [0] = false, false, false, false, map_ls, map_datar, map_ls, map_datafp,
715 map_datai, map_datai, map_br, map_br, map_ls, map_datar, map_ls, map_datafp
716}
717
718------------------------------------------------------------------------------
719
720local map_regs = { x = {}, w = {}, d = {}, s = {} }
721
722for i=0,30 do
723 map_regs.x[i] = "x"..i
724 map_regs.w[i] = "w"..i
725 map_regs.d[i] = "d"..i
726 map_regs.s[i] = "s"..i
727end
728map_regs.x[31] = "sp"
729map_regs.w[31] = "wsp"
730map_regs.d[31] = "d31"
731map_regs.s[31] = "s31"
732
733local map_cond = {
734 [0] = "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
735 "hi", "ls", "ge", "lt", "gt", "le", "al",
736}
737
738local map_shift = { [0] = "lsl", "lsr", "asr", }
739
740local map_extend = {
741 [0] = "uxtb", "uxth", "uxtw", "uxtx", "sxtb", "sxth", "sxtw", "sxtx",
742}
743
744------------------------------------------------------------------------------
745
746-- Output a nicely formatted line with an opcode and operands.
747local function putop(ctx, text, operands)
748 local pos = ctx.pos
749 local extra = ""
750 if ctx.rel then
751 local sym = ctx.symtab[ctx.rel]
752 if sym then
753 extra = "\t->"..sym
754 end
755 end
756 if ctx.hexdump > 0 then
757 ctx.out(format("%08x %s %-5s %s%s\n",
758 ctx.addr+pos, tohex(ctx.op), text, concat(operands, ", "), extra))
759 else
760 ctx.out(format("%08x %-5s %s%s\n",
761 ctx.addr+pos, text, concat(operands, ", "), extra))
762 end
763 ctx.pos = pos + 4
764end
765
766-- Fallback for unknown opcodes.
767local function unknown(ctx)
768 return putop(ctx, ".long", { "0x"..tohex(ctx.op) })
769end
770
771local function match_reg(p, pat, regnum)
772 return map_regs[match(pat, p.."%w-([xwds])")][regnum]
773end
774
775local function fmt_hex32(x)
776 if x < 0 then
777 return tohex(x)
778 else
779 return format("%x", x)
780 end
781end
782
783local imm13_rep = { 0x55555555, 0x11111111, 0x01010101, 0x00010001, 0x00000001 }
784
785local function decode_imm13(op)
786 local imms = band(rshift(op, 10), 63)
787 local immr = band(rshift(op, 16), 63)
788 if band(op, 0x00400000) == 0 then
789 local len = 5
790 if imms >= 56 then
791 if imms >= 60 then len = 1 else len = 2 end
792 elseif imms >= 48 then len = 3 elseif imms >= 32 then len = 4 end
793 local l = lshift(1, len)-1
794 local s = band(imms, l)
795 local r = band(immr, l)
796 local imm = ror(rshift(-1, 31-s), r)
797 if len ~= 5 then imm = band(imm, lshift(1, l)-1) + rshift(imm, 31-l) end
798 imm = imm * imm13_rep[len]
799 local ix = fmt_hex32(imm)
800 if rshift(op, 31) ~= 0 then
801 return ix..tohex(imm)
802 else
803 return ix
804 end
805 else
806 local lo, hi = -1, 0
807 if imms < 32 then lo = rshift(-1, 31-imms) else hi = rshift(-1, 63-imms) end
808 if immr ~= 0 then
809 lo, hi = ror(lo, immr), ror(hi, immr)
810 local x = immr == 32 and 0 or band(bxor(lo, hi), lshift(-1, 32-immr))
811 lo, hi = bxor(lo, x), bxor(hi, x)
812 if immr >= 32 then lo, hi = hi, lo end
813 end
814 if hi ~= 0 then
815 return fmt_hex32(hi)..tohex(lo)
816 else
817 return fmt_hex32(lo)
818 end
819 end
820end
821
822local function parse_immpc(op, name)
823 if name == "b" or name == "bl" then
824 return arshift(lshift(op, 6), 4)
825 elseif name == "adr" or name == "adrp" then
826 local immlo = band(rshift(op, 29), 3)
827 local immhi = lshift(arshift(lshift(op, 8), 13), 2)
828 return bor(immhi, immlo)
829 elseif name == "tbz" or name == "tbnz" then
830 return lshift(arshift(lshift(op, 13), 18), 2)
831 else
832 return lshift(arshift(lshift(op, 8), 13), 2)
833 end
834end
835
836local function parse_fpimm8(op)
837 local sign = band(op, 0x100000) == 0 and 1 or -1
838 local exp = bxor(rshift(arshift(lshift(op, 12), 5), 24), 0x80) - 131
839 local frac = 16+band(rshift(op, 13), 15)
840 return sign * frac * 2^exp
841end
842
843local function prefer_bfx(sf, uns, imms, immr)
844 if imms < immr or imms == 31 or imms == 63 then
845 return false
846 end
847 if immr == 0 then
848 if sf == 0 and (imms == 7 or imms == 15) then
849 return false
850 end
851 if sf ~= 0 and uns == 0 and (imms == 7 or imms == 15 or imms == 31) then
852 return false
853 end
854 end
855 return true
856end
857
858-- Disassemble a single instruction.
859local function disass_ins(ctx)
860 local pos = ctx.pos
861 local b0, b1, b2, b3 = byte(ctx.code, pos+1, pos+4)
862 local op = bor(lshift(b3, 24), lshift(b2, 16), lshift(b1, 8), b0)
863 local operands = {}
864 local suffix = ""
865 local last, name, pat
866 local map_reg
867 ctx.op = op
868 ctx.rel = nil
869 last = nil
870 local opat
871 opat = map_init[band(rshift(op, 25), 15)]
872 while type(opat) ~= "string" do
873 if not opat then return unknown(ctx) end
874 opat = opat[band(rshift(op, opat.shift), opat.mask)] or opat._
875 end
876 name, pat = match(opat, "^([a-z0-9]*)(.*)")
877 local altname, pat2 = match(pat, "|([a-z0-9_.|]*)(.*)")
878 if altname then pat = pat2 end
879 if sub(pat, 1, 1) == "." then
880 local s2, p2 = match(pat, "^([a-z0-9.]*)(.*)")
881 suffix = suffix..s2
882 pat = p2
883 end
884
885 local rt = match(pat, "[gf]")
886 if rt then
887 if rt == "g" then
888 map_reg = band(op, 0x80000000) ~= 0 and map_regs.x or map_regs.w
889 else
890 map_reg = band(op, 0x400000) ~= 0 and map_regs.d or map_regs.s
891 end
892 end
893
894 local second0, immr
895
896 for p in gmatch(pat, ".") do
897 local x = nil
898 if p == "D" then
899 local regnum = band(op, 31)
900 x = rt and map_reg[regnum] or match_reg(p, pat, regnum)
901 elseif p == "N" then
902 local regnum = band(rshift(op, 5), 31)
903 x = rt and map_reg[regnum] or match_reg(p, pat, regnum)
904 elseif p == "M" then
905 local regnum = band(rshift(op, 16), 31)
906 x = rt and map_reg[regnum] or match_reg(p, pat, regnum)
907 elseif p == "A" then
908 local regnum = band(rshift(op, 10), 31)
909 x = rt and map_reg[regnum] or match_reg(p, pat, regnum)
910 elseif p == "B" then
911 local addr = ctx.addr + pos + parse_immpc(op, name)
912 ctx.rel = addr
913 x = "0x"..tohex(addr)
914 elseif p == "T" then
915 x = bor(band(rshift(op, 26), 32), band(rshift(op, 19), 31))
916 elseif p == "V" then
917 x = band(op, 15)
918 elseif p == "C" then
919 x = map_cond[band(rshift(op, 12), 15)]
920 elseif p == "c" then
921 local rn = band(rshift(op, 5), 31)
922 local rm = band(rshift(op, 16), 31)
923 local cond = band(rshift(op, 12), 15)
924 local invc = bxor(cond, 1)
925 x = map_cond[cond]
926 if altname and cond ~= 14 and cond ~= 15 then
927 local a1, a2 = match(altname, "([^|]*)|(.*)")
928 if rn == rm then
929 local n = #operands
930 operands[n] = nil
931 x = map_cond[invc]
932 if rn ~= 31 then
933 if a1 then name = a1 else name = altname end
934 else
935 operands[n-1] = nil
936 name = a2
937 end
938 end
939 end
940 elseif p == "W" then
941 x = band(rshift(op, 5), 0xffff)
942 elseif p == "Y" then
943 x = band(rshift(op, 5), 0xffff)
944 local hw = band(rshift(op, 21), 3)
945 if altname and (hw == 0 or x ~= 0) then
946 name = altname
947 end
948 elseif p == "L" then
949 local rn = map_regs.x[band(rshift(op, 5), 31)]
950 local imm9 = arshift(lshift(op, 11), 23)
951 if band(op, 0x800) ~= 0 then
952 x = "["..rn..", #"..imm9.."]!"
953 else
954 x = "["..rn.."], #"..imm9
955 end
956 elseif p == "U" then
957 local rn = map_regs.x[band(rshift(op, 5), 31)]
958 local sz = band(rshift(op, 30), 3)
959 local imm12 = lshift(arshift(lshift(op, 10), 20), sz)
960 if imm12 ~= 0 then
961 x = "["..rn..", #"..imm12.."]"
962 else
963 x = "["..rn.."]"
964 end
965 elseif p == "K" then
966 local rn = map_regs.x[band(rshift(op, 5), 31)]
967 local imm9 = arshift(lshift(op, 11), 23)
968 if imm9 ~= 0 then
969 x = "["..rn..", #"..imm9.."]"
970 else
971 x = "["..rn.."]"
972 end
973 elseif p == "O" then
974 local rn, rm = map_regs.x[band(rshift(op, 5), 31)]
975 local m = band(rshift(op, 13), 1)
976 if m == 0 then
977 rm = map_regs.w[band(rshift(op, 16), 31)]
978 else
979 rm = map_regs.x[band(rshift(op, 16), 31)]
980 end
981 x = "["..rn..", "..rm
982 local opt = band(rshift(op, 13), 7)
983 local s = band(rshift(op, 12), 1)
984 local sz = band(rshift(op, 30), 3)
985 -- extension to be applied
986 if opt == 3 then
987 if s == 0 then x = x.."]"
988 else x = x..", lsl #"..sz.."]" end
989 elseif opt == 2 or opt == 6 or opt == 7 then
990 if s == 0 then x = x..", "..map_extend[opt].."]"
991 else x = x..", "..map_extend[opt].." #"..sz.."]" end
992 else
993 x = x.."]"
994 end
995 elseif p == "P" then
996 local opcv, sh = rshift(op, 26), 2
997 if opcv >= 0x2a then sh = 4 elseif opcv >= 0x1b then sh = 3 end
998 local imm7 = lshift(arshift(lshift(op, 10), 25), sh)
999 local rn = map_regs.x[band(rshift(op, 5), 31)]
1000 local ind = band(rshift(op, 23), 3)
1001 if ind == 1 then
1002 x = "["..rn.."], #"..imm7
1003 elseif ind == 2 then
1004 if imm7 == 0 then
1005 x = "["..rn.."]"
1006 else
1007 x = "["..rn..", #"..imm7.."]"
1008 end
1009 elseif ind == 3 then
1010 x = "["..rn..", #"..imm7.."]!"
1011 end
1012 elseif p == "I" then
1013 local shf = band(rshift(op, 22), 3)
1014 local imm12 = band(rshift(op, 10), 0x0fff)
1015 local rn, rd = band(rshift(op, 5), 31), band(op, 31)
1016 if altname == "mov" and shf == 0 and imm12 == 0 and (rn == 31 or rd == 31) then
1017 name = altname
1018 x = nil
1019 elseif shf == 0 then
1020 x = imm12
1021 elseif shf == 1 then
1022 x = imm12..", lsl #12"
1023 end
1024 elseif p == "i" then
1025 x = "#0x"..decode_imm13(op)
1026 elseif p == "1" then
1027 immr = band(rshift(op, 16), 63)
1028 x = immr
1029 elseif p == "2" then
1030 x = band(rshift(op, 10), 63)
1031 if altname then
1032 local a1, a2, a3, a4, a5, a6 =
1033 match(altname, "([^|]*)|([^|]*)|([^|]*)|([^|]*)|([^|]*)|(.*)")
1034 local sf = band(rshift(op, 26), 32)
1035 local uns = band(rshift(op, 30), 1)
1036 if prefer_bfx(sf, uns, x, immr) then
1037 name = a2
1038 x = x - immr + 1
1039 elseif immr == 0 and x == 7 then
1040 local n = #operands
1041 operands[n] = nil
1042 if sf ~= 0 then
1043 operands[n-1] = gsub(operands[n-1], "x", "w")
1044 end
1045 last = operands[n-1]
1046 name = a6
1047 x = nil
1048 elseif immr == 0 and x == 15 then
1049 local n = #operands
1050 operands[n] = nil
1051 if sf ~= 0 then
1052 operands[n-1] = gsub(operands[n-1], "x", "w")
1053 end
1054 last = operands[n-1]
1055 name = a5
1056 x = nil
1057 elseif x == 31 or x == 63 then
1058 if x == 31 and immr == 0 and name == "sbfm" then
1059 name = a4
1060 local n = #operands
1061 operands[n] = nil
1062 if sf ~= 0 then
1063 operands[n-1] = gsub(operands[n-1], "x", "w")
1064 end
1065 last = operands[n-1]
1066 else
1067 name = a3
1068 end
1069 x = nil
1070 elseif band(x, 31) ~= 31 and immr == x+1 and name == "ubfm" then
1071 name = a4
1072 last = "#"..(sf+32 - immr)
1073 operands[#operands] = last
1074 x = nil
1075 elseif x < immr then
1076 name = a1
1077 last = "#"..(sf+32 - immr)
1078 operands[#operands] = last
1079 x = x + 1
1080 end
1081 end
1082 elseif p == "3" then
1083 x = band(rshift(op, 10), 63)
1084 if altname then
1085 local a1, a2 = match(altname, "([^|]*)|(.*)")
1086 if x < immr then
1087 name = a1
1088 local sf = band(rshift(op, 26), 32)
1089 last = "#"..(sf+32 - immr)
1090 operands[#operands] = last
1091 x = x + 1
1092 else
1093 name = a2
1094 x = x - immr + 1
1095 end
1096 end
1097 elseif p == "4" then
1098 x = band(rshift(op, 10), 63)
1099 local rn = band(rshift(op, 5), 31)
1100 local rm = band(rshift(op, 16), 31)
1101 if altname and rn == rm then
1102 local n = #operands
1103 operands[n] = nil
1104 last = operands[n-1]
1105 name = altname
1106 end
1107 elseif p == "5" then
1108 x = band(rshift(op, 16), 31)
1109 elseif p == "S" then
1110 x = band(rshift(op, 10), 63)
1111 if x == 0 then x = nil
1112 else x = map_shift[band(rshift(op, 22), 3)].." #"..x end
1113 elseif p == "X" then
1114 local opt = band(rshift(op, 13), 7)
1115 -- Width specifier <R>.
1116 if opt ~= 3 and opt ~= 7 then
1117 last = map_regs.w[band(rshift(op, 16), 31)]
1118 operands[#operands] = last
1119 end
1120 x = band(rshift(op, 10), 7)
1121 -- Extension.
1122 if opt == 2 + band(rshift(op, 31), 1) and
1123 band(rshift(op, second0 and 5 or 0), 31) == 31 then
1124 if x == 0 then x = nil
1125 else x = "lsl #"..x end
1126 else
1127 if x == 0 then x = map_extend[band(rshift(op, 13), 7)]
1128 else x = map_extend[band(rshift(op, 13), 7)].." #"..x end
1129 end
1130 elseif p == "R" then
1131 x = band(rshift(op,21), 3)
1132 if x == 0 then x = nil
1133 else x = "lsl #"..x*16 end
1134 elseif p == "z" then
1135 local n = #operands
1136 if operands[n] == "sp" then operands[n] = "xzr"
1137 elseif operands[n] == "wsp" then operands[n] = "wzr"
1138 end
1139 elseif p == "Z" then
1140 x = 0
1141 elseif p == "F" then
1142 x = parse_fpimm8(op)
1143 elseif p == "g" or p == "f" or p == "x" or p == "w" or
1144 p == "d" or p == "s" then
1145 -- These are handled in D/N/M/A.
1146 elseif p == "0" then
1147 if last == "sp" or last == "wsp" then
1148 local n = #operands
1149 operands[n] = nil
1150 last = operands[n-1]
1151 if altname then
1152 local a1, a2 = match(altname, "([^|]*)|(.*)")
1153 if not a1 then
1154 name = altname
1155 elseif second0 then
1156 name, altname = a2, a1
1157 else
1158 name, altname = a1, a2
1159 end
1160 end
1161 end
1162 second0 = true
1163 else
1164 assert(false)
1165 end
1166 if x then
1167 last = x
1168 if type(x) == "number" then x = "#"..x end
1169 operands[#operands+1] = x
1170 end
1171 end
1172
1173 return putop(ctx, name..suffix, operands)
1174end
1175
1176------------------------------------------------------------------------------
1177
1178-- Disassemble a block of code.
1179local function disass_block(ctx, ofs, len)
1180 if not ofs then ofs = 0 end
1181 local stop = len and ofs+len or #ctx.code
1182 ctx.pos = ofs
1183 ctx.rel = nil
1184 while ctx.pos < stop do disass_ins(ctx) end
1185end
1186
1187-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
1188local function create(code, addr, out)
1189 local ctx = {}
1190 ctx.code = code
1191 ctx.addr = addr or 0
1192 ctx.out = out or io.write
1193 ctx.symtab = {}
1194 ctx.disass = disass_block
1195 ctx.hexdump = 8
1196 return ctx
1197end
1198
1199-- Simple API: disassemble code (a string) at address and output via out.
1200local function disass(code, addr, out)
1201 create(code, addr, out):disass()
1202end
1203
1204-- Return register name for RID.
1205local function regname(r)
1206 if r < 32 then return map_regs.x[r] end
1207 return map_regs.d[r-32]
1208end
1209
1210-- Public module functions.
1211return {
1212 create = create,
1213 disass = disass,
1214 regname = regname
1215}
1216
diff --git a/src/jit/dis_arm64be.lua b/src/jit/dis_arm64be.lua
new file mode 100644
index 00000000..f7a56352
--- /dev/null
+++ b/src/jit/dis_arm64be.lua
@@ -0,0 +1,12 @@
1----------------------------------------------------------------------------
2-- LuaJIT ARM64BE disassembler wrapper module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7-- ARM64 instructions are always little-endian. So just forward to the
8-- common ARM64 disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11return require((string.match(..., ".*%.") or "").."dis_arm64")
12
diff --git a/src/jit/dis_mips.lua b/src/jit/dis_mips.lua
index ebfed56a..b0e99df4 100644
--- a/src/jit/dis_mips.lua
+++ b/src/jit/dis_mips.lua
@@ -19,13 +19,34 @@ local band, bor, tohex = bit.band, bit.bor, bit.tohex
19local lshift, rshift, arshift = bit.lshift, bit.rshift, bit.arshift 19local lshift, rshift, arshift = bit.lshift, bit.rshift, bit.arshift
20 20
21------------------------------------------------------------------------------ 21------------------------------------------------------------------------------
22-- Primary and extended opcode maps 22-- Extended opcode maps common to all MIPS releases
23------------------------------------------------------------------------------ 23------------------------------------------------------------------------------
24 24
25local map_movci = { shift = 16, mask = 1, [0] = "movfDSC", "movtDSC", }
26local map_srl = { shift = 21, mask = 1, [0] = "srlDTA", "rotrDTA", } 25local map_srl = { shift = 21, mask = 1, [0] = "srlDTA", "rotrDTA", }
27local map_srlv = { shift = 6, mask = 1, [0] = "srlvDTS", "rotrvDTS", } 26local map_srlv = { shift = 6, mask = 1, [0] = "srlvDTS", "rotrvDTS", }
28 27
28local map_cop0 = {
29 shift = 25, mask = 1,
30 [0] = {
31 shift = 21, mask = 15,
32 [0] = "mfc0TDW", [4] = "mtc0TDW",
33 [10] = "rdpgprDT",
34 [11] = { shift = 5, mask = 1, [0] = "diT0", "eiT0", },
35 [14] = "wrpgprDT",
36 }, {
37 shift = 0, mask = 63,
38 [1] = "tlbr", [2] = "tlbwi", [6] = "tlbwr", [8] = "tlbp",
39 [24] = "eret", [31] = "deret",
40 [32] = "wait",
41 },
42}
43
44------------------------------------------------------------------------------
45-- Primary and extended opcode maps for MIPS R1-R5
46------------------------------------------------------------------------------
47
48local map_movci = { shift = 16, mask = 1, [0] = "movfDSC", "movtDSC", }
49
29local map_special = { 50local map_special = {
30 shift = 0, mask = 63, 51 shift = 0, mask = 63,
31 [0] = { shift = 0, mask = -1, [0] = "nop", _ = "sllDTA" }, 52 [0] = { shift = 0, mask = -1, [0] = "nop", _ = "sllDTA" },
@@ -34,15 +55,17 @@ local map_special = {
34 "jrS", "jalrD1S", "movzDST", "movnDST", 55 "jrS", "jalrD1S", "movzDST", "movnDST",
35 "syscallY", "breakY", false, "sync", 56 "syscallY", "breakY", false, "sync",
36 "mfhiD", "mthiS", "mfloD", "mtloS", 57 "mfhiD", "mthiS", "mfloD", "mtloS",
37 false, false, false, false, 58 "dsllvDST", false, "dsrlvDST", "dsravDST",
38 "multST", "multuST", "divST", "divuST", 59 "multST", "multuST", "divST", "divuST",
39 false, false, false, false, 60 "dmultST", "dmultuST", "ddivST", "ddivuST",
40 "addDST", "addu|moveDST0", "subDST", "subu|neguDS0T", 61 "addDST", "addu|moveDST0", "subDST", "subu|neguDS0T",
41 "andDST", "orDST", "xorDST", "nor|notDST0", 62 "andDST", "or|moveDST0", "xorDST", "nor|notDST0",
42 false, false, "sltDST", "sltuDST", 63 false, false, "sltDST", "sltuDST",
43 false, false, false, false, 64 "daddDST", "dadduDST", "dsubDST", "dsubuDST",
44 "tgeSTZ", "tgeuSTZ", "tltSTZ", "tltuSTZ", 65 "tgeSTZ", "tgeuSTZ", "tltSTZ", "tltuSTZ",
45 "teqSTZ", false, "tneSTZ", 66 "teqSTZ", false, "tneSTZ", false,
67 "dsllDTA", false, "dsrlDTA", "dsraDTA",
68 "dsll32DTA", false, "dsrl32DTA", "dsra32DTA",
46} 69}
47 70
48local map_special2 = { 71local map_special2 = {
@@ -60,11 +83,17 @@ local map_bshfl = {
60 [24] = "sehDT", 83 [24] = "sehDT",
61} 84}
62 85
86local map_dbshfl = {
87 shift = 6, mask = 31,
88 [2] = "dsbhDT",
89 [5] = "dshdDT",
90}
91
63local map_special3 = { 92local map_special3 = {
64 shift = 0, mask = 63, 93 shift = 0, mask = 63,
65 [0] = "extTSAK", [4] = "insTSAL", 94 [0] = "extTSAK", [1] = "dextmTSAP", [3] = "dextTSAK",
66 [32] = map_bshfl, 95 [4] = "insTSAL", [6] = "dinsuTSEQ", [7] = "dinsTSAL",
67 [59] = "rdhwrTD", 96 [32] = map_bshfl, [36] = map_dbshfl, [59] = "rdhwrTD",
68} 97}
69 98
70local map_regimm = { 99local map_regimm = {
@@ -79,22 +108,6 @@ local map_regimm = {
79 false, false, false, "synciSO", 108 false, false, false, "synciSO",
80} 109}
81 110
82local map_cop0 = {
83 shift = 25, mask = 1,
84 [0] = {
85 shift = 21, mask = 15,
86 [0] = "mfc0TDW", [4] = "mtc0TDW",
87 [10] = "rdpgprDT",
88 [11] = { shift = 5, mask = 1, [0] = "diT0", "eiT0", },
89 [14] = "wrpgprDT",
90 }, {
91 shift = 0, mask = 63,
92 [1] = "tlbr", [2] = "tlbwi", [6] = "tlbwr", [8] = "tlbp",
93 [24] = "eret", [31] = "deret",
94 [32] = "wait",
95 },
96}
97
98local map_cop1s = { 111local map_cop1s = {
99 shift = 0, mask = 63, 112 shift = 0, mask = 63,
100 [0] = "add.sFGH", "sub.sFGH", "mul.sFGH", "div.sFGH", 113 [0] = "add.sFGH", "sub.sFGH", "mul.sFGH", "div.sFGH",
@@ -178,8 +191,8 @@ local map_cop1bc = {
178 191
179local map_cop1 = { 192local map_cop1 = {
180 shift = 21, mask = 31, 193 shift = 21, mask = 31,
181 [0] = "mfc1TG", false, "cfc1TG", "mfhc1TG", 194 [0] = "mfc1TG", "dmfc1TG", "cfc1TG", "mfhc1TG",
182 "mtc1TG", false, "ctc1TG", "mthc1TG", 195 "mtc1TG", "dmtc1TG", "ctc1TG", "mthc1TG",
183 map_cop1bc, false, false, false, 196 map_cop1bc, false, false, false,
184 false, false, false, false, 197 false, false, false, false,
185 map_cop1s, map_cop1d, false, false, 198 map_cop1s, map_cop1d, false, false,
@@ -213,16 +226,218 @@ local map_pri = {
213 "andiTSU", "ori|liTS0U", "xoriTSU", "luiTU", 226 "andiTSU", "ori|liTS0U", "xoriTSU", "luiTU",
214 map_cop0, map_cop1, false, map_cop1x, 227 map_cop0, map_cop1, false, map_cop1x,
215 "beql|beqzlST0B", "bnel|bnezlST0B", "blezlSB", "bgtzlSB", 228 "beql|beqzlST0B", "bnel|bnezlST0B", "blezlSB", "bgtzlSB",
216 false, false, false, false, 229 "daddiTSI", "daddiuTSI", false, false,
217 map_special2, false, false, map_special3, 230 map_special2, "jalxJ", false, map_special3,
218 "lbTSO", "lhTSO", "lwlTSO", "lwTSO", 231 "lbTSO", "lhTSO", "lwlTSO", "lwTSO",
219 "lbuTSO", "lhuTSO", "lwrTSO", false, 232 "lbuTSO", "lhuTSO", "lwrTSO", false,
220 "sbTSO", "shTSO", "swlTSO", "swTSO", 233 "sbTSO", "shTSO", "swlTSO", "swTSO",
221 false, false, "swrTSO", "cacheNSO", 234 false, false, "swrTSO", "cacheNSO",
222 "llTSO", "lwc1HSO", "lwc2TSO", "prefNSO", 235 "llTSO", "lwc1HSO", "lwc2TSO", "prefNSO",
223 false, "ldc1HSO", "ldc2TSO", false, 236 false, "ldc1HSO", "ldc2TSO", "ldTSO",
224 "scTSO", "swc1HSO", "swc2TSO", false, 237 "scTSO", "swc1HSO", "swc2TSO", false,
225 false, "sdc1HSO", "sdc2TSO", false, 238 false, "sdc1HSO", "sdc2TSO", "sdTSO",
239}
240
241------------------------------------------------------------------------------
242-- Primary and extended opcode maps for MIPS R6
243------------------------------------------------------------------------------
244
245local map_mul_r6 = { shift = 6, mask = 3, [2] = "mulDST", [3] = "muhDST" }
246local map_mulu_r6 = { shift = 6, mask = 3, [2] = "muluDST", [3] = "muhuDST" }
247local map_div_r6 = { shift = 6, mask = 3, [2] = "divDST", [3] = "modDST" }
248local map_divu_r6 = { shift = 6, mask = 3, [2] = "divuDST", [3] = "moduDST" }
249local map_dmul_r6 = { shift = 6, mask = 3, [2] = "dmulDST", [3] = "dmuhDST" }
250local map_dmulu_r6 = { shift = 6, mask = 3, [2] = "dmuluDST", [3] = "dmuhuDST" }
251local map_ddiv_r6 = { shift = 6, mask = 3, [2] = "ddivDST", [3] = "dmodDST" }
252local map_ddivu_r6 = { shift = 6, mask = 3, [2] = "ddivuDST", [3] = "dmoduDST" }
253
254local map_special_r6 = {
255 shift = 0, mask = 63,
256 [0] = { shift = 0, mask = -1, [0] = "nop", _ = "sllDTA" },
257 false, map_srl, "sraDTA",
258 "sllvDTS", false, map_srlv, "sravDTS",
259 "jrS", "jalrD1S", false, false,
260 "syscallY", "breakY", false, "sync",
261 "clzDS", "cloDS", "dclzDS", "dcloDS",
262 "dsllvDST", "dlsaDSTA", "dsrlvDST", "dsravDST",
263 map_mul_r6, map_mulu_r6, map_div_r6, map_divu_r6,
264 map_dmul_r6, map_dmulu_r6, map_ddiv_r6, map_ddivu_r6,
265 "addDST", "addu|moveDST0", "subDST", "subu|neguDS0T",
266 "andDST", "or|moveDST0", "xorDST", "nor|notDST0",
267 false, false, "sltDST", "sltuDST",
268 "daddDST", "dadduDST", "dsubDST", "dsubuDST",
269 "tgeSTZ", "tgeuSTZ", "tltSTZ", "tltuSTZ",
270 "teqSTZ", "seleqzDST", "tneSTZ", "selnezDST",
271 "dsllDTA", false, "dsrlDTA", "dsraDTA",
272 "dsll32DTA", false, "dsrl32DTA", "dsra32DTA",
273}
274
275local map_bshfl_r6 = {
276 shift = 9, mask = 3,
277 [1] = "alignDSTa",
278 _ = {
279 shift = 6, mask = 31,
280 [0] = "bitswapDT",
281 [2] = "wsbhDT",
282 [16] = "sebDT",
283 [24] = "sehDT",
284 }
285}
286
287local map_dbshfl_r6 = {
288 shift = 9, mask = 3,
289 [1] = "dalignDSTa",
290 _ = {
291 shift = 6, mask = 31,
292 [0] = "dbitswapDT",
293 [2] = "dsbhDT",
294 [5] = "dshdDT",
295 }
296}
297
298local map_special3_r6 = {
299 shift = 0, mask = 63,
300 [0] = "extTSAK", [1] = "dextmTSAP", [3] = "dextTSAK",
301 [4] = "insTSAL", [6] = "dinsuTSEQ", [7] = "dinsTSAL",
302 [32] = map_bshfl_r6, [36] = map_dbshfl_r6, [59] = "rdhwrTD",
303}
304
305local map_regimm_r6 = {
306 shift = 16, mask = 31,
307 [0] = "bltzSB", [1] = "bgezSB",
308 [6] = "dahiSI", [30] = "datiSI",
309 [23] = "sigrieI", [31] = "synciSO",
310}
311
312local map_pcrel_r6 = {
313 shift = 19, mask = 3,
314 [0] = "addiupcS2", "lwpcS2", "lwupcS2", {
315 shift = 18, mask = 1,
316 [0] = "ldpcS3", { shift = 16, mask = 3, [2] = "auipcSI", [3] = "aluipcSI" }
317 }
318}
319
320local map_cop1s_r6 = {
321 shift = 0, mask = 63,
322 [0] = "add.sFGH", "sub.sFGH", "mul.sFGH", "div.sFGH",
323 "sqrt.sFG", "abs.sFG", "mov.sFG", "neg.sFG",
324 "round.l.sFG", "trunc.l.sFG", "ceil.l.sFG", "floor.l.sFG",
325 "round.w.sFG", "trunc.w.sFG", "ceil.w.sFG", "floor.w.sFG",
326 "sel.sFGH", false, false, false,
327 "seleqz.sFGH", "recip.sFG", "rsqrt.sFG", "selnez.sFGH",
328 "maddf.sFGH", "msubf.sFGH", "rint.sFG", "class.sFG",
329 "min.sFGH", "mina.sFGH", "max.sFGH", "maxa.sFGH",
330 false, "cvt.d.sFG", false, false,
331 "cvt.w.sFG", "cvt.l.sFG",
332}
333
334local map_cop1d_r6 = {
335 shift = 0, mask = 63,
336 [0] = "add.dFGH", "sub.dFGH", "mul.dFGH", "div.dFGH",
337 "sqrt.dFG", "abs.dFG", "mov.dFG", "neg.dFG",
338 "round.l.dFG", "trunc.l.dFG", "ceil.l.dFG", "floor.l.dFG",
339 "round.w.dFG", "trunc.w.dFG", "ceil.w.dFG", "floor.w.dFG",
340 "sel.dFGH", false, false, false,
341 "seleqz.dFGH", "recip.dFG", "rsqrt.dFG", "selnez.dFGH",
342 "maddf.dFGH", "msubf.dFGH", "rint.dFG", "class.dFG",
343 "min.dFGH", "mina.dFGH", "max.dFGH", "maxa.dFGH",
344 "cvt.s.dFG", false, false, false,
345 "cvt.w.dFG", "cvt.l.dFG",
346}
347
348local map_cop1w_r6 = {
349 shift = 0, mask = 63,
350 [0] = "cmp.af.sFGH", "cmp.un.sFGH", "cmp.eq.sFGH", "cmp.ueq.sFGH",
351 "cmp.lt.sFGH", "cmp.ult.sFGH", "cmp.le.sFGH", "cmp.ule.sFGH",
352 "cmp.saf.sFGH", "cmp.sun.sFGH", "cmp.seq.sFGH", "cmp.sueq.sFGH",
353 "cmp.slt.sFGH", "cmp.sult.sFGH", "cmp.sle.sFGH", "cmp.sule.sFGH",
354 false, "cmp.or.sFGH", "cmp.une.sFGH", "cmp.ne.sFGH",
355 false, false, false, false,
356 false, "cmp.sor.sFGH", "cmp.sune.sFGH", "cmp.sne.sFGH",
357 false, false, false, false,
358 "cvt.s.wFG", "cvt.d.wFG",
359}
360
361local map_cop1l_r6 = {
362 shift = 0, mask = 63,
363 [0] = "cmp.af.dFGH", "cmp.un.dFGH", "cmp.eq.dFGH", "cmp.ueq.dFGH",
364 "cmp.lt.dFGH", "cmp.ult.dFGH", "cmp.le.dFGH", "cmp.ule.dFGH",
365 "cmp.saf.dFGH", "cmp.sun.dFGH", "cmp.seq.dFGH", "cmp.sueq.dFGH",
366 "cmp.slt.dFGH", "cmp.sult.dFGH", "cmp.sle.dFGH", "cmp.sule.dFGH",
367 false, "cmp.or.dFGH", "cmp.une.dFGH", "cmp.ne.dFGH",
368 false, false, false, false,
369 false, "cmp.sor.dFGH", "cmp.sune.dFGH", "cmp.sne.dFGH",
370 false, false, false, false,
371 "cvt.s.lFG", "cvt.d.lFG",
372}
373
374local map_cop1_r6 = {
375 shift = 21, mask = 31,
376 [0] = "mfc1TG", "dmfc1TG", "cfc1TG", "mfhc1TG",
377 "mtc1TG", "dmtc1TG", "ctc1TG", "mthc1TG",
378 false, "bc1eqzHB", false, false,
379 false, "bc1nezHB", false, false,
380 map_cop1s_r6, map_cop1d_r6, false, false,
381 map_cop1w_r6, map_cop1l_r6,
382}
383
384local function maprs_popTS(rs, rt)
385 if rt == 0 then return 0 elseif rs == 0 then return 1
386 elseif rs == rt then return 2 else return 3 end
387end
388
389local map_pop06_r6 = {
390 maprs = maprs_popTS, [0] = "blezSB", "blezalcTB", "bgezalcTB", "bgeucSTB"
391}
392local map_pop07_r6 = {
393 maprs = maprs_popTS, [0] = "bgtzSB", "bgtzalcTB", "bltzalcTB", "bltucSTB"
394}
395local map_pop26_r6 = {
396 maprs = maprs_popTS, "blezcTB", "bgezcTB", "bgecSTB"
397}
398local map_pop27_r6 = {
399 maprs = maprs_popTS, "bgtzcTB", "bltzcTB", "bltcSTB"
400}
401
402local function maprs_popS(rs, rt)
403 if rs == 0 then return 0 else return 1 end
404end
405
406local map_pop66_r6 = {
407 maprs = maprs_popS, [0] = "jicTI", "beqzcSb"
408}
409local map_pop76_r6 = {
410 maprs = maprs_popS, [0] = "jialcTI", "bnezcSb"
411}
412
413local function maprs_popST(rs, rt)
414 if rs >= rt then return 0 elseif rs == 0 then return 1 else return 2 end
415end
416
417local map_pop10_r6 = {
418 maprs = maprs_popST, [0] = "bovcSTB", "beqzalcTB", "beqcSTB"
419}
420local map_pop30_r6 = {
421 maprs = maprs_popST, [0] = "bnvcSTB", "bnezalcTB", "bnecSTB"
422}
423
424local map_pri_r6 = {
425 [0] = map_special_r6, map_regimm_r6, "jJ", "jalJ",
426 "beq|beqz|bST00B", "bne|bnezST0B", map_pop06_r6, map_pop07_r6,
427 map_pop10_r6, "addiu|liTS0I", "sltiTSI", "sltiuTSI",
428 "andiTSU", "ori|liTS0U", "xoriTSU", "aui|luiTS0U",
429 map_cop0, map_cop1_r6, false, false,
430 false, false, map_pop26_r6, map_pop27_r6,
431 map_pop30_r6, "daddiuTSI", false, false,
432 false, "dauiTSI", false, map_special3_r6,
433 "lbTSO", "lhTSO", false, "lwTSO",
434 "lbuTSO", "lhuTSO", false, false,
435 "sbTSO", "shTSO", false, "swTSO",
436 false, false, false, false,
437 false, "lwc1HSO", "bc#", false,
438 false, "ldc1HSO", map_pop66_r6, "ldTSO",
439 false, "swc1HSO", "balc#", map_pcrel_r6,
440 false, "sdc1HSO", map_pop76_r6, "sdTSO",
226} 441}
227 442
228------------------------------------------------------------------------------ 443------------------------------------------------------------------------------
@@ -279,10 +494,14 @@ local function disass_ins(ctx)
279 ctx.op = op 494 ctx.op = op
280 ctx.rel = nil 495 ctx.rel = nil
281 496
282 local opat = map_pri[rshift(op, 26)] 497 local opat = ctx.map_pri[rshift(op, 26)]
283 while type(opat) ~= "string" do 498 while type(opat) ~= "string" do
284 if not opat then return unknown(ctx) end 499 if not opat then return unknown(ctx) end
285 opat = opat[band(rshift(op, opat.shift), opat.mask)] or opat._ 500 if opat.maprs then
501 opat = opat[opat.maprs(band(rshift(op,21),31), band(rshift(op,16),31))]
502 else
503 opat = opat[band(rshift(op, opat.shift), opat.mask)] or opat._
504 end
286 end 505 end
287 local name, pat = match(opat, "^([a-z0-9_.]*)(.*)") 506 local name, pat = match(opat, "^([a-z0-9_.]*)(.*)")
288 local altname, pat2 = match(pat, "|([a-z0-9_.|]*)(.*)") 507 local altname, pat2 = match(pat, "|([a-z0-9_.|]*)(.*)")
@@ -306,6 +525,10 @@ local function disass_ins(ctx)
306 x = "f"..band(rshift(op, 21), 31) 525 x = "f"..band(rshift(op, 21), 31)
307 elseif p == "A" then 526 elseif p == "A" then
308 x = band(rshift(op, 6), 31) 527 x = band(rshift(op, 6), 31)
528 elseif p == "a" then
529 x = band(rshift(op, 6), 7)
530 elseif p == "E" then
531 x = band(rshift(op, 6), 31) + 32
309 elseif p == "M" then 532 elseif p == "M" then
310 x = band(rshift(op, 11), 31) 533 x = band(rshift(op, 11), 31)
311 elseif p == "N" then 534 elseif p == "N" then
@@ -315,10 +538,18 @@ local function disass_ins(ctx)
315 if x == 0 then x = nil end 538 if x == 0 then x = nil end
316 elseif p == "K" then 539 elseif p == "K" then
317 x = band(rshift(op, 11), 31) + 1 540 x = band(rshift(op, 11), 31) + 1
541 elseif p == "P" then
542 x = band(rshift(op, 11), 31) + 33
318 elseif p == "L" then 543 elseif p == "L" then
319 x = band(rshift(op, 11), 31) - last + 1 544 x = band(rshift(op, 11), 31) - last + 1
545 elseif p == "Q" then
546 x = band(rshift(op, 11), 31) - last + 33
320 elseif p == "I" then 547 elseif p == "I" then
321 x = arshift(lshift(op, 16), 16) 548 x = arshift(lshift(op, 16), 16)
549 elseif p == "2" then
550 x = arshift(lshift(op, 13), 11)
551 elseif p == "3" then
552 x = arshift(lshift(op, 14), 11)
322 elseif p == "U" then 553 elseif p == "U" then
323 x = band(op, 0xffff) 554 x = band(op, 0xffff)
324 elseif p == "O" then 555 elseif p == "O" then
@@ -328,13 +559,22 @@ local function disass_ins(ctx)
328 local index = map_gpr[band(rshift(op, 16), 31)] 559 local index = map_gpr[band(rshift(op, 16), 31)]
329 operands[#operands] = format("%s(%s)", index, last) 560 operands[#operands] = format("%s(%s)", index, last)
330 elseif p == "B" then 561 elseif p == "B" then
331 x = ctx.addr + ctx.pos + arshift(lshift(op, 16), 16)*4 + 4 562 x = ctx.addr + ctx.pos + arshift(lshift(op, 16), 14) + 4
563 ctx.rel = x
564 x = format("0x%08x", x)
565 elseif p == "b" then
566 x = ctx.addr + ctx.pos + arshift(lshift(op, 11), 9) + 4
332 ctx.rel = x 567 ctx.rel = x
333 x = "0x"..tohex(x) 568 x = format("0x%08x", x)
569 elseif p == "#" then
570 x = ctx.addr + ctx.pos + arshift(lshift(op, 6), 4) + 4
571 ctx.rel = x
572 x = format("0x%08x", x)
334 elseif p == "J" then 573 elseif p == "J" then
335 x = band(ctx.addr + ctx.pos, 0xf0000000) + band(op, 0x03ffffff)*4 574 local a = ctx.addr + ctx.pos
575 x = a - band(a, 0x0fffffff) + band(op, 0x03ffffff)*4
336 ctx.rel = x 576 ctx.rel = x
337 x = "0x"..tohex(x) 577 x = format("0x%08x", x)
338 elseif p == "V" then 578 elseif p == "V" then
339 x = band(rshift(op, 8), 7) 579 x = band(rshift(op, 8), 7)
340 if x == 0 then x = nil end 580 if x == 0 then x = nil end
@@ -384,7 +624,7 @@ local function disass_block(ctx, ofs, len)
384end 624end
385 625
386-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len). 626-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
387local function create_(code, addr, out) 627local function create(code, addr, out)
388 local ctx = {} 628 local ctx = {}
389 ctx.code = code 629 ctx.code = code
390 ctx.addr = addr or 0 630 ctx.addr = addr or 0
@@ -393,36 +633,62 @@ local function create_(code, addr, out)
393 ctx.disass = disass_block 633 ctx.disass = disass_block
394 ctx.hexdump = 8 634 ctx.hexdump = 8
395 ctx.get = get_be 635 ctx.get = get_be
636 ctx.map_pri = map_pri
637 return ctx
638end
639
640local function create_el(code, addr, out)
641 local ctx = create(code, addr, out)
642 ctx.get = get_le
643 return ctx
644end
645
646local function create_r6(code, addr, out)
647 local ctx = create(code, addr, out)
648 ctx.map_pri = map_pri_r6
396 return ctx 649 return ctx
397end 650end
398 651
399local function create_el_(code, addr, out) 652local function create_r6_el(code, addr, out)
400 local ctx = create_(code, addr, out) 653 local ctx = create(code, addr, out)
401 ctx.get = get_le 654 ctx.get = get_le
655 ctx.map_pri = map_pri_r6
402 return ctx 656 return ctx
403end 657end
404 658
405-- Simple API: disassemble code (a string) at address and output via out. 659-- Simple API: disassemble code (a string) at address and output via out.
406local function disass_(code, addr, out) 660local function disass(code, addr, out)
407 create_(code, addr, out):disass() 661 create(code, addr, out):disass()
662end
663
664local function disass_el(code, addr, out)
665 create_el(code, addr, out):disass()
408end 666end
409 667
410local function disass_el_(code, addr, out) 668local function disass_r6(code, addr, out)
411 create_el_(code, addr, out):disass() 669 create_r6(code, addr, out):disass()
670end
671
672local function disass_r6_el(code, addr, out)
673 create_r6_el(code, addr, out):disass()
412end 674end
413 675
414-- Return register name for RID. 676-- Return register name for RID.
415local function regname_(r) 677local function regname(r)
416 if r < 32 then return map_gpr[r] end 678 if r < 32 then return map_gpr[r] end
417 return "f"..(r-32) 679 return "f"..(r-32)
418end 680end
419 681
420-- Public module functions. 682-- Public module functions.
421module(...) 683return {
422 684 create = create,
423create = create_ 685 create_el = create_el,
424create_el = create_el_ 686 create_r6 = create_r6,
425disass = disass_ 687 create_r6_el = create_r6_el,
426disass_el = disass_el_ 688 disass = disass,
427regname = regname_ 689 disass_el = disass_el,
690 disass_r6 = disass_r6,
691 disass_r6_el = disass_r6_el,
692 regname = regname
693}
428 694
diff --git a/src/jit/dis_mips64.lua b/src/jit/dis_mips64.lua
new file mode 100644
index 00000000..5f3a4dab
--- /dev/null
+++ b/src/jit/dis_mips64.lua
@@ -0,0 +1,17 @@
1----------------------------------------------------------------------------
2-- LuaJIT MIPS64 disassembler wrapper module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7-- This module just exports the big-endian functions from the
8-- MIPS disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
12return {
13 create = dis_mips.create,
14 disass = dis_mips.disass,
15 regname = dis_mips.regname
16}
17
diff --git a/src/jit/dis_mips64el.lua b/src/jit/dis_mips64el.lua
new file mode 100644
index 00000000..ea513649
--- /dev/null
+++ b/src/jit/dis_mips64el.lua
@@ -0,0 +1,17 @@
1----------------------------------------------------------------------------
2-- LuaJIT MIPS64EL disassembler wrapper module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7-- This module just exports the little-endian functions from the
8-- MIPS disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
12return {
13 create = dis_mips.create_el,
14 disass = dis_mips.disass_el,
15 regname = dis_mips.regname
16}
17
diff --git a/src/jit/dis_mips64r6.lua b/src/jit/dis_mips64r6.lua
new file mode 100644
index 00000000..1d948411
--- /dev/null
+++ b/src/jit/dis_mips64r6.lua
@@ -0,0 +1,17 @@
1----------------------------------------------------------------------------
2-- LuaJIT MIPS64R6 disassembler wrapper module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7-- This module just exports the r6 big-endian functions from the
8-- MIPS disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
12return {
13 create = dis_mips.create_r6,
14 disass = dis_mips.disass_r6,
15 regname = dis_mips.regname
16}
17
diff --git a/src/jit/dis_mips64r6el.lua b/src/jit/dis_mips64r6el.lua
new file mode 100644
index 00000000..26592e17
--- /dev/null
+++ b/src/jit/dis_mips64r6el.lua
@@ -0,0 +1,17 @@
1----------------------------------------------------------------------------
2-- LuaJIT MIPS64R6EL disassembler wrapper module.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7-- This module just exports the r6 little-endian functions from the
8-- MIPS disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------
10
11local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
12return {
13 create = dis_mips.create_r6_el,
14 disass = dis_mips.disass_r6_el,
15 regname = dis_mips.regname
16}
17
diff --git a/src/jit/dis_mipsel.lua b/src/jit/dis_mipsel.lua
index 6c14800e..6906a779 100644
--- a/src/jit/dis_mipsel.lua
+++ b/src/jit/dis_mipsel.lua
@@ -8,13 +8,10 @@
8-- MIPS disassembler module. All the interesting stuff is there. 8-- MIPS disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------ 9------------------------------------------------------------------------------
10 10
11local require = require 11local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
12 12return {
13module(...) 13 create = dis_mips.create_el,
14 14 disass = dis_mips.disass_el,
15local dis_mips = require(_PACKAGE.."dis_mips") 15 regname = dis_mips.regname
16 16}
17create = dis_mips.create_el
18disass = dis_mips.disass_el
19regname = dis_mips.regname
20 17
diff --git a/src/jit/dis_ppc.lua b/src/jit/dis_ppc.lua
index 26a6b343..95c3da84 100644
--- a/src/jit/dis_ppc.lua
+++ b/src/jit/dis_ppc.lua
@@ -560,7 +560,7 @@ local function disass_block(ctx, ofs, len)
560end 560end
561 561
562-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len). 562-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
563local function create_(code, addr, out) 563local function create(code, addr, out)
564 local ctx = {} 564 local ctx = {}
565 ctx.code = code 565 ctx.code = code
566 ctx.addr = addr or 0 566 ctx.addr = addr or 0
@@ -572,20 +572,20 @@ local function create_(code, addr, out)
572end 572end
573 573
574-- Simple API: disassemble code (a string) at address and output via out. 574-- Simple API: disassemble code (a string) at address and output via out.
575local function disass_(code, addr, out) 575local function disass(code, addr, out)
576 create_(code, addr, out):disass() 576 create(code, addr, out):disass()
577end 577end
578 578
579-- Return register name for RID. 579-- Return register name for RID.
580local function regname_(r) 580local function regname(r)
581 if r < 32 then return map_gpr[r] end 581 if r < 32 then return map_gpr[r] end
582 return "f"..(r-32) 582 return "f"..(r-32)
583end 583end
584 584
585-- Public module functions. 585-- Public module functions.
586module(...) 586return {
587 587 create = create,
588create = create_ 588 disass = disass,
589disass = disass_ 589 regname = regname
590regname = regname_ 590}
591 591
diff --git a/src/jit/dis_x64.lua b/src/jit/dis_x64.lua
index 3ffd67e2..eb21f044 100644
--- a/src/jit/dis_x64.lua
+++ b/src/jit/dis_x64.lua
@@ -8,13 +8,10 @@
8-- x86/x64 disassembler module. All the interesting stuff is there. 8-- x86/x64 disassembler module. All the interesting stuff is there.
9------------------------------------------------------------------------------ 9------------------------------------------------------------------------------
10 10
11local require = require 11local dis_x86 = require((string.match(..., ".*%.") or "").."dis_x86")
12 12return {
13module(...) 13 create = dis_x86.create64,
14 14 disass = dis_x86.disass64,
15local dis_x86 = require(_PACKAGE.."dis_x86") 15 regname = dis_x86.regname64
16 16}
17create = dis_x86.create64
18disass = dis_x86.disass64
19regname = dis_x86.regname64
20 17
diff --git a/src/jit/dis_x86.lua b/src/jit/dis_x86.lua
index 77702b89..40b8218e 100644
--- a/src/jit/dis_x86.lua
+++ b/src/jit/dis_x86.lua
@@ -15,19 +15,20 @@
15-- Intel and AMD manuals. The supported instruction set is quite extensive 15-- Intel and AMD manuals. The supported instruction set is quite extensive
16-- and reflects what a current generation Intel or AMD CPU implements in 16-- and reflects what a current generation Intel or AMD CPU implements in
17-- 32 bit and 64 bit mode. Yes, this includes MMX, SSE, SSE2, SSE3, SSSE3, 17-- 32 bit and 64 bit mode. Yes, this includes MMX, SSE, SSE2, SSE3, SSSE3,
18-- SSE4.1, SSE4.2, SSE4a and even privileged and hypervisor (VMX/SVM) 18-- SSE4.1, SSE4.2, SSE4a, AVX, AVX2 and even privileged and hypervisor
19-- instructions. 19-- (VMX/SVM) instructions.
20-- 20--
21-- Notes: 21-- Notes:
22-- * The (useless) a16 prefix, 3DNow and pre-586 opcodes are unsupported. 22-- * The (useless) a16 prefix, 3DNow and pre-586 opcodes are unsupported.
23-- * No attempt at optimization has been made -- it's fast enough for my needs. 23-- * No attempt at optimization has been made -- it's fast enough for my needs.
24-- * The public API may change when more architectures are added.
25------------------------------------------------------------------------------ 24------------------------------------------------------------------------------
26 25
27local type = type 26local type = type
28local sub, byte, format = string.sub, string.byte, string.format 27local sub, byte, format = string.sub, string.byte, string.format
29local match, gmatch, gsub = string.match, string.gmatch, string.gsub 28local match, gmatch, gsub = string.match, string.gmatch, string.gsub
30local lower, rep = string.lower, string.rep 29local lower, rep = string.lower, string.rep
30local bit = require("bit")
31local tohex = bit.tohex
31 32
32-- Map for 1st opcode byte in 32 bit mode. Ugly? Well ... read on. 33-- Map for 1st opcode byte in 32 bit mode. Ugly? Well ... read on.
33local map_opc1_32 = { 34local map_opc1_32 = {
@@ -76,7 +77,7 @@ local map_opc1_32 = {
76"movBRi","movBRi","movBRi","movBRi","movBRi","movBRi","movBRi","movBRi", 77"movBRi","movBRi","movBRi","movBRi","movBRi","movBRi","movBRi","movBRi",
77"movVRI","movVRI","movVRI","movVRI","movVRI","movVRI","movVRI","movVRI", 78"movVRI","movVRI","movVRI","movVRI","movVRI","movVRI","movVRI","movVRI",
78--Cx 79--Cx
79"shift!Bmu","shift!Vmu","retBw","ret","$lesVrm","$ldsVrm","movBmi","movVmi", 80"shift!Bmu","shift!Vmu","retBw","ret","vex*3$lesVrm","vex*2$ldsVrm","movBmi","movVmi",
80"enterBwu","leave","retfBw","retf","int3","intBu","into","iretVS", 81"enterBwu","leave","retfBw","retf","int3","intBu","into","iretVS",
81--Dx 82--Dx
82"shift!Bm1","shift!Vm1","shift!Bmc","shift!Vmc","aamBu","aadBu","salc","xlatb", 83"shift!Bm1","shift!Vm1","shift!Bmc","shift!Vmc","aamBu","aadBu","salc","xlatb",
@@ -101,7 +102,7 @@ local map_opc1_64 = setmetatable({
101 [0x44]="rex*r", [0x45]="rex*rb", [0x46]="rex*rx", [0x47]="rex*rxb", 102 [0x44]="rex*r", [0x45]="rex*rb", [0x46]="rex*rx", [0x47]="rex*rxb",
102 [0x48]="rex*w", [0x49]="rex*wb", [0x4a]="rex*wx", [0x4b]="rex*wxb", 103 [0x48]="rex*w", [0x49]="rex*wb", [0x4a]="rex*wx", [0x4b]="rex*wxb",
103 [0x4c]="rex*wr", [0x4d]="rex*wrb", [0x4e]="rex*wrx", [0x4f]="rex*wrxb", 104 [0x4c]="rex*wr", [0x4d]="rex*wrb", [0x4e]="rex*wrx", [0x4f]="rex*wrxb",
104 [0x82]=false, [0x9a]=false, [0xc4]=false, [0xc5]=false, [0xce]=false, 105 [0x82]=false, [0x9a]=false, [0xc4]="vex*3", [0xc5]="vex*2", [0xce]=false,
105 [0xd4]=false, [0xd5]=false, [0xd6]=false, [0xea]=false, 106 [0xd4]=false, [0xd5]=false, [0xd6]=false, [0xea]=false,
106}, { __index = map_opc1_32 }) 107}, { __index = map_opc1_32 })
107 108
@@ -112,12 +113,12 @@ local map_opc2 = {
112[0]="sldt!Dmp","sgdt!Ump","larVrm","lslVrm",nil,"syscall","clts","sysret", 113[0]="sldt!Dmp","sgdt!Ump","larVrm","lslVrm",nil,"syscall","clts","sysret",
113"invd","wbinvd",nil,"ud1",nil,"$prefetch!Bm","femms","3dnowMrmu", 114"invd","wbinvd",nil,"ud1",nil,"$prefetch!Bm","femms","3dnowMrmu",
114--1x 115--1x
115"movupsXrm|movssXrm|movupdXrm|movsdXrm", 116"movupsXrm|movssXrvm|movupdXrm|movsdXrvm",
116"movupsXmr|movssXmr|movupdXmr|movsdXmr", 117"movupsXmr|movssXmvr|movupdXmr|movsdXmvr",
117"movhlpsXrm$movlpsXrm|movsldupXrm|movlpdXrm|movddupXrm", 118"movhlpsXrm$movlpsXrm|movsldupXrm|movlpdXrm|movddupXrm",
118"movlpsXmr||movlpdXmr", 119"movlpsXmr||movlpdXmr",
119"unpcklpsXrm||unpcklpdXrm", 120"unpcklpsXrvm||unpcklpdXrvm",
120"unpckhpsXrm||unpckhpdXrm", 121"unpckhpsXrvm||unpckhpdXrvm",
121"movlhpsXrm$movhpsXrm|movshdupXrm|movhpdXrm", 122"movlhpsXrm$movhpsXrm|movshdupXrm|movhpdXrm",
122"movhpsXmr||movhpdXmr", 123"movhpsXmr||movhpdXmr",
123"$prefetcht!Bm","hintnopVm","hintnopVm","hintnopVm", 124"$prefetcht!Bm","hintnopVm","hintnopVm","hintnopVm",
@@ -126,7 +127,7 @@ local map_opc2 = {
126"movUmx$","movUmy$","movUxm$","movUym$","movUmz$",nil,"movUzm$",nil, 127"movUmx$","movUmy$","movUxm$","movUym$","movUmz$",nil,"movUzm$",nil,
127"movapsXrm||movapdXrm", 128"movapsXrm||movapdXrm",
128"movapsXmr||movapdXmr", 129"movapsXmr||movapdXmr",
129"cvtpi2psXrMm|cvtsi2ssXrVmt|cvtpi2pdXrMm|cvtsi2sdXrVmt", 130"cvtpi2psXrMm|cvtsi2ssXrvVmt|cvtpi2pdXrMm|cvtsi2sdXrvVmt",
130"movntpsXmr|movntssXmr|movntpdXmr|movntsdXmr", 131"movntpsXmr|movntssXmr|movntpdXmr|movntsdXmr",
131"cvttps2piMrXm|cvttss2siVrXm|cvttpd2piMrXm|cvttsd2siVrXm", 132"cvttps2piMrXm|cvttss2siVrXm|cvttpd2piMrXm|cvttsd2siVrXm",
132"cvtps2piMrXm|cvtss2siVrXm|cvtpd2piMrXm|cvtsd2siVrXm", 133"cvtps2piMrXm|cvtss2siVrXm|cvtpd2piMrXm|cvtsd2siVrXm",
@@ -142,27 +143,27 @@ local map_opc2 = {
142"cmovlVrm","cmovgeVrm","cmovleVrm","cmovgVrm", 143"cmovlVrm","cmovgeVrm","cmovleVrm","cmovgVrm",
143--5x 144--5x
144"movmskpsVrXm$||movmskpdVrXm$","sqrtpsXrm|sqrtssXrm|sqrtpdXrm|sqrtsdXrm", 145"movmskpsVrXm$||movmskpdVrXm$","sqrtpsXrm|sqrtssXrm|sqrtpdXrm|sqrtsdXrm",
145"rsqrtpsXrm|rsqrtssXrm","rcppsXrm|rcpssXrm", 146"rsqrtpsXrm|rsqrtssXrvm","rcppsXrm|rcpssXrvm",
146"andpsXrm||andpdXrm","andnpsXrm||andnpdXrm", 147"andpsXrvm||andpdXrvm","andnpsXrvm||andnpdXrvm",
147"orpsXrm||orpdXrm","xorpsXrm||xorpdXrm", 148"orpsXrvm||orpdXrvm","xorpsXrvm||xorpdXrvm",
148"addpsXrm|addssXrm|addpdXrm|addsdXrm","mulpsXrm|mulssXrm|mulpdXrm|mulsdXrm", 149"addpsXrvm|addssXrvm|addpdXrvm|addsdXrvm","mulpsXrvm|mulssXrvm|mulpdXrvm|mulsdXrvm",
149"cvtps2pdXrm|cvtss2sdXrm|cvtpd2psXrm|cvtsd2ssXrm", 150"cvtps2pdXrm|cvtss2sdXrvm|cvtpd2psXrm|cvtsd2ssXrvm",
150"cvtdq2psXrm|cvttps2dqXrm|cvtps2dqXrm", 151"cvtdq2psXrm|cvttps2dqXrm|cvtps2dqXrm",
151"subpsXrm|subssXrm|subpdXrm|subsdXrm","minpsXrm|minssXrm|minpdXrm|minsdXrm", 152"subpsXrvm|subssXrvm|subpdXrvm|subsdXrvm","minpsXrvm|minssXrvm|minpdXrvm|minsdXrvm",
152"divpsXrm|divssXrm|divpdXrm|divsdXrm","maxpsXrm|maxssXrm|maxpdXrm|maxsdXrm", 153"divpsXrvm|divssXrvm|divpdXrvm|divsdXrvm","maxpsXrvm|maxssXrvm|maxpdXrvm|maxsdXrvm",
153--6x 154--6x
154"punpcklbwPrm","punpcklwdPrm","punpckldqPrm","packsswbPrm", 155"punpcklbwPrvm","punpcklwdPrvm","punpckldqPrvm","packsswbPrvm",
155"pcmpgtbPrm","pcmpgtwPrm","pcmpgtdPrm","packuswbPrm", 156"pcmpgtbPrvm","pcmpgtwPrvm","pcmpgtdPrvm","packuswbPrvm",
156"punpckhbwPrm","punpckhwdPrm","punpckhdqPrm","packssdwPrm", 157"punpckhbwPrvm","punpckhwdPrvm","punpckhdqPrvm","packssdwPrvm",
157"||punpcklqdqXrm","||punpckhqdqXrm", 158"||punpcklqdqXrvm","||punpckhqdqXrvm",
158"movPrVSm","movqMrm|movdquXrm|movdqaXrm", 159"movPrVSm","movqMrm|movdquXrm|movdqaXrm",
159--7x 160--7x
160"pshufwMrmu|pshufhwXrmu|pshufdXrmu|pshuflwXrmu","pshiftw!Pmu", 161"pshufwMrmu|pshufhwXrmu|pshufdXrmu|pshuflwXrmu","pshiftw!Pvmu",
161"pshiftd!Pmu","pshiftq!Mmu||pshiftdq!Xmu", 162"pshiftd!Pvmu","pshiftq!Mvmu||pshiftdq!Xvmu",
162"pcmpeqbPrm","pcmpeqwPrm","pcmpeqdPrm","emms|", 163"pcmpeqbPrvm","pcmpeqwPrvm","pcmpeqdPrvm","emms*|",
163"vmreadUmr||extrqXmuu$|insertqXrmuu$","vmwriteUrm||extrqXrm$|insertqXrm$", 164"vmreadUmr||extrqXmuu$|insertqXrmuu$","vmwriteUrm||extrqXrm$|insertqXrm$",
164nil,nil, 165nil,nil,
165"||haddpdXrm|haddpsXrm","||hsubpdXrm|hsubpsXrm", 166"||haddpdXrvm|haddpsXrvm","||hsubpdXrvm|hsubpsXrvm",
166"movVSmMr|movqXrm|movVSmXr","movqMmr|movdquXmr|movdqaXmr", 167"movVSmMr|movqXrm|movVSmXr","movqMmr|movdquXmr|movdqaXmr",
167--8x 168--8x
168"joVj","jnoVj","jbVj","jnbVj","jzVj","jnzVj","jbeVj","jaVj", 169"joVj","jnoVj","jbVj","jnbVj","jzVj","jnzVj","jbeVj","jaVj",
@@ -180,27 +181,27 @@ nil,nil,
180"bsfVrm","bsrVrm|lzcntVrm|bsrWrm","movsxVrBmt","movsxVrWmt", 181"bsfVrm","bsrVrm|lzcntVrm|bsrWrm","movsxVrBmt","movsxVrWmt",
181--Cx 182--Cx
182"xaddBmr","xaddVmr", 183"xaddBmr","xaddVmr",
183"cmppsXrmu|cmpssXrmu|cmppdXrmu|cmpsdXrmu","$movntiVmr|", 184"cmppsXrvmu|cmpssXrvmu|cmppdXrvmu|cmpsdXrvmu","$movntiVmr|",
184"pinsrwPrWmu","pextrwDrPmu", 185"pinsrwPrvWmu","pextrwDrPmu",
185"shufpsXrmu||shufpdXrmu","$cmpxchg!Qmp", 186"shufpsXrvmu||shufpdXrvmu","$cmpxchg!Qmp",
186"bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR", 187"bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR","bswapVR",
187--Dx 188--Dx
188"||addsubpdXrm|addsubpsXrm","psrlwPrm","psrldPrm","psrlqPrm", 189"||addsubpdXrvm|addsubpsXrvm","psrlwPrvm","psrldPrvm","psrlqPrvm",
189"paddqPrm","pmullwPrm", 190"paddqPrvm","pmullwPrvm",
190"|movq2dqXrMm|movqXmr|movdq2qMrXm$","pmovmskbVrMm||pmovmskbVrXm", 191"|movq2dqXrMm|movqXmr|movdq2qMrXm$","pmovmskbVrMm||pmovmskbVrXm",
191"psubusbPrm","psubuswPrm","pminubPrm","pandPrm", 192"psubusbPrvm","psubuswPrvm","pminubPrvm","pandPrvm",
192"paddusbPrm","padduswPrm","pmaxubPrm","pandnPrm", 193"paddusbPrvm","padduswPrvm","pmaxubPrvm","pandnPrvm",
193--Ex 194--Ex
194"pavgbPrm","psrawPrm","psradPrm","pavgwPrm", 195"pavgbPrvm","psrawPrvm","psradPrvm","pavgwPrvm",
195"pmulhuwPrm","pmulhwPrm", 196"pmulhuwPrvm","pmulhwPrvm",
196"|cvtdq2pdXrm|cvttpd2dqXrm|cvtpd2dqXrm","$movntqMmr||$movntdqXmr", 197"|cvtdq2pdXrm|cvttpd2dqXrm|cvtpd2dqXrm","$movntqMmr||$movntdqXmr",
197"psubsbPrm","psubswPrm","pminswPrm","porPrm", 198"psubsbPrvm","psubswPrvm","pminswPrvm","porPrvm",
198"paddsbPrm","paddswPrm","pmaxswPrm","pxorPrm", 199"paddsbPrvm","paddswPrvm","pmaxswPrvm","pxorPrvm",
199--Fx 200--Fx
200"|||lddquXrm","psllwPrm","pslldPrm","psllqPrm", 201"|||lddquXrm","psllwPrvm","pslldPrvm","psllqPrvm",
201"pmuludqPrm","pmaddwdPrm","psadbwPrm","maskmovqMrm||maskmovdquXrm$", 202"pmuludqPrvm","pmaddwdPrvm","psadbwPrvm","maskmovqMrm||maskmovdquXrm$",
202"psubbPrm","psubwPrm","psubdPrm","psubqPrm", 203"psubbPrvm","psubwPrvm","psubdPrvm","psubqPrvm",
203"paddbPrm","paddwPrm","padddPrm","ud", 204"paddbPrvm","paddwPrvm","padddPrvm","ud",
204} 205}
205assert(map_opc2[255] == "ud") 206assert(map_opc2[255] == "ud")
206 207
@@ -208,49 +209,91 @@ assert(map_opc2[255] == "ud")
208local map_opc3 = { 209local map_opc3 = {
209["38"] = { -- [66] 0f 38 xx 210["38"] = { -- [66] 0f 38 xx
210--0x 211--0x
211[0]="pshufbPrm","phaddwPrm","phadddPrm","phaddswPrm", 212[0]="pshufbPrvm","phaddwPrvm","phadddPrvm","phaddswPrvm",
212"pmaddubswPrm","phsubwPrm","phsubdPrm","phsubswPrm", 213"pmaddubswPrvm","phsubwPrvm","phsubdPrvm","phsubswPrvm",
213"psignbPrm","psignwPrm","psigndPrm","pmulhrswPrm", 214"psignbPrvm","psignwPrvm","psigndPrvm","pmulhrswPrvm",
214nil,nil,nil,nil, 215"||permilpsXrvm","||permilpdXrvm",nil,nil,
215--1x 216--1x
216"||pblendvbXrma",nil,nil,nil, 217"||pblendvbXrma",nil,nil,nil,
217"||blendvpsXrma","||blendvpdXrma",nil,"||ptestXrm", 218"||blendvpsXrma","||blendvpdXrma","||permpsXrvm","||ptestXrm",
218nil,nil,nil,nil, 219"||broadcastssXrm","||broadcastsdXrm","||broadcastf128XrlXm",nil,
219"pabsbPrm","pabswPrm","pabsdPrm",nil, 220"pabsbPrm","pabswPrm","pabsdPrm",nil,
220--2x 221--2x
221"||pmovsxbwXrm","||pmovsxbdXrm","||pmovsxbqXrm","||pmovsxwdXrm", 222"||pmovsxbwXrm","||pmovsxbdXrm","||pmovsxbqXrm","||pmovsxwdXrm",
222"||pmovsxwqXrm","||pmovsxdqXrm",nil,nil, 223"||pmovsxwqXrm","||pmovsxdqXrm",nil,nil,
223"||pmuldqXrm","||pcmpeqqXrm","||$movntdqaXrm","||packusdwXrm", 224"||pmuldqXrvm","||pcmpeqqXrvm","||$movntdqaXrm","||packusdwXrvm",
224nil,nil,nil,nil, 225"||maskmovpsXrvm","||maskmovpdXrvm","||maskmovpsXmvr","||maskmovpdXmvr",
225--3x 226--3x
226"||pmovzxbwXrm","||pmovzxbdXrm","||pmovzxbqXrm","||pmovzxwdXrm", 227"||pmovzxbwXrm","||pmovzxbdXrm","||pmovzxbqXrm","||pmovzxwdXrm",
227"||pmovzxwqXrm","||pmovzxdqXrm",nil,"||pcmpgtqXrm", 228"||pmovzxwqXrm","||pmovzxdqXrm","||permdXrvm","||pcmpgtqXrvm",
228"||pminsbXrm","||pminsdXrm","||pminuwXrm","||pminudXrm", 229"||pminsbXrvm","||pminsdXrvm","||pminuwXrvm","||pminudXrvm",
229"||pmaxsbXrm","||pmaxsdXrm","||pmaxuwXrm","||pmaxudXrm", 230"||pmaxsbXrvm","||pmaxsdXrvm","||pmaxuwXrvm","||pmaxudXrvm",
230--4x 231--4x
231"||pmulddXrm","||phminposuwXrm", 232"||pmulddXrvm","||phminposuwXrm",nil,nil,
233nil,"||psrlvVSXrvm","||psravdXrvm","||psllvVSXrvm",
234--5x
235[0x58] = "||pbroadcastdXrlXm",[0x59] = "||pbroadcastqXrlXm",
236[0x5a] = "||broadcasti128XrlXm",
237--7x
238[0x78] = "||pbroadcastbXrlXm",[0x79] = "||pbroadcastwXrlXm",
239--8x
240[0x8c] = "||pmaskmovXrvVSm",
241[0x8e] = "||pmaskmovVSmXvr",
242--9x
243[0x96] = "||fmaddsub132pHXrvm",[0x97] = "||fmsubadd132pHXrvm",
244[0x98] = "||fmadd132pHXrvm",[0x99] = "||fmadd132sHXrvm",
245[0x9a] = "||fmsub132pHXrvm",[0x9b] = "||fmsub132sHXrvm",
246[0x9c] = "||fnmadd132pHXrvm",[0x9d] = "||fnmadd132sHXrvm",
247[0x9e] = "||fnmsub132pHXrvm",[0x9f] = "||fnmsub132sHXrvm",
248--Ax
249[0xa6] = "||fmaddsub213pHXrvm",[0xa7] = "||fmsubadd213pHXrvm",
250[0xa8] = "||fmadd213pHXrvm",[0xa9] = "||fmadd213sHXrvm",
251[0xaa] = "||fmsub213pHXrvm",[0xab] = "||fmsub213sHXrvm",
252[0xac] = "||fnmadd213pHXrvm",[0xad] = "||fnmadd213sHXrvm",
253[0xae] = "||fnmsub213pHXrvm",[0xaf] = "||fnmsub213sHXrvm",
254--Bx
255[0xb6] = "||fmaddsub231pHXrvm",[0xb7] = "||fmsubadd231pHXrvm",
256[0xb8] = "||fmadd231pHXrvm",[0xb9] = "||fmadd231sHXrvm",
257[0xba] = "||fmsub231pHXrvm",[0xbb] = "||fmsub231sHXrvm",
258[0xbc] = "||fnmadd231pHXrvm",[0xbd] = "||fnmadd231sHXrvm",
259[0xbe] = "||fnmsub231pHXrvm",[0xbf] = "||fnmsub231sHXrvm",
260--Dx
261[0xdc] = "||aesencXrvm", [0xdd] = "||aesenclastXrvm",
262[0xde] = "||aesdecXrvm", [0xdf] = "||aesdeclastXrvm",
232--Fx 263--Fx
233[0xf0] = "|||crc32TrBmt",[0xf1] = "|||crc32TrVmt", 264[0xf0] = "|||crc32TrBmt",[0xf1] = "|||crc32TrVmt",
265[0xf7] = "| sarxVrmv| shlxVrmv| shrxVrmv",
234}, 266},
235 267
236["3a"] = { -- [66] 0f 3a xx 268["3a"] = { -- [66] 0f 3a xx
237--0x 269--0x
238[0x00]=nil,nil,nil,nil,nil,nil,nil,nil, 270[0x00]="||permqXrmu","||permpdXrmu","||pblenddXrvmu",nil,
239"||roundpsXrmu","||roundpdXrmu","||roundssXrmu","||roundsdXrmu", 271"||permilpsXrmu","||permilpdXrmu","||perm2f128Xrvmu",nil,
240"||blendpsXrmu","||blendpdXrmu","||pblendwXrmu","palignrPrmu", 272"||roundpsXrmu","||roundpdXrmu","||roundssXrvmu","||roundsdXrvmu",
273"||blendpsXrvmu","||blendpdXrvmu","||pblendwXrvmu","palignrPrvmu",
241--1x 274--1x
242nil,nil,nil,nil, 275nil,nil,nil,nil,
243"||pextrbVmXru","||pextrwVmXru","||pextrVmSXru","||extractpsVmXru", 276"||pextrbVmXru","||pextrwVmXru","||pextrVmSXru","||extractpsVmXru",
244nil,nil,nil,nil,nil,nil,nil,nil, 277"||insertf128XrvlXmu","||extractf128XlXmYru",nil,nil,
278nil,nil,nil,nil,
245--2x 279--2x
246"||pinsrbXrVmu","||insertpsXrmu","||pinsrXrVmuS",nil, 280"||pinsrbXrvVmu","||insertpsXrvmu","||pinsrXrvVmuS",nil,
281--3x
282[0x38] = "||inserti128Xrvmu",[0x39] = "||extracti128XlXmYru",
247--4x 283--4x
248[0x40] = "||dppsXrmu", 284[0x40] = "||dppsXrvmu",
249[0x41] = "||dppdXrmu", 285[0x41] = "||dppdXrvmu",
250[0x42] = "||mpsadbwXrmu", 286[0x42] = "||mpsadbwXrvmu",
287[0x44] = "||pclmulqdqXrvmu",
288[0x46] = "||perm2i128Xrvmu",
289[0x4a] = "||blendvpsXrvmb",[0x4b] = "||blendvpdXrvmb",
290[0x4c] = "||pblendvbXrvmb",
251--6x 291--6x
252[0x60] = "||pcmpestrmXrmu",[0x61] = "||pcmpestriXrmu", 292[0x60] = "||pcmpestrmXrmu",[0x61] = "||pcmpestriXrmu",
253[0x62] = "||pcmpistrmXrmu",[0x63] = "||pcmpistriXrmu", 293[0x62] = "||pcmpistrmXrmu",[0x63] = "||pcmpistriXrmu",
294[0xdf] = "||aeskeygenassistXrmu",
295--Fx
296[0xf0] = "||| rorxVrmu",
254}, 297},
255} 298}
256 299
@@ -354,17 +397,19 @@ local map_regs = {
354 "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7" }, -- No x64 ext! 397 "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7" }, -- No x64 ext!
355 X = { "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 398 X = { "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
356 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" }, 399 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" },
400 Y = { "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7",
401 "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15" },
357} 402}
358local map_segregs = { "es", "cs", "ss", "ds", "fs", "gs", "segr6", "segr7" } 403local map_segregs = { "es", "cs", "ss", "ds", "fs", "gs", "segr6", "segr7" }
359 404
360-- Maps for size names. 405-- Maps for size names.
361local map_sz2n = { 406local map_sz2n = {
362 B = 1, W = 2, D = 4, Q = 8, M = 8, X = 16, 407 B = 1, W = 2, D = 4, Q = 8, M = 8, X = 16, Y = 32,
363} 408}
364local map_sz2prefix = { 409local map_sz2prefix = {
365 B = "byte", W = "word", D = "dword", 410 B = "byte", W = "word", D = "dword",
366 Q = "qword", 411 Q = "qword",
367 M = "qword", X = "xword", 412 M = "qword", X = "xword", Y = "yword",
368 F = "dword", G = "qword", -- No need for sizes/register names for these two. 413 F = "dword", G = "qword", -- No need for sizes/register names for these two.
369} 414}
370 415
@@ -387,10 +432,13 @@ local function putop(ctx, text, operands)
387 if ctx.rep then text = ctx.rep.." "..text; ctx.rep = false end 432 if ctx.rep then text = ctx.rep.." "..text; ctx.rep = false end
388 if ctx.rex then 433 if ctx.rex then
389 local t = (ctx.rexw and "w" or "")..(ctx.rexr and "r" or "").. 434 local t = (ctx.rexw and "w" or "")..(ctx.rexr and "r" or "")..
390 (ctx.rexx and "x" or "")..(ctx.rexb and "b" or "") 435 (ctx.rexx and "x" or "")..(ctx.rexb and "b" or "")..
391 if t ~= "" then text = "rex."..t.." "..text end 436 (ctx.vexl and "l" or "")
437 if ctx.vexv and ctx.vexv ~= 0 then t = t.."v"..ctx.vexv end
438 if t ~= "" then text = ctx.rex.."."..t.." "..gsub(text, "^ ", "")
439 elseif ctx.rex == "vex" then text = gsub("v"..text, "^v ", "") end
392 ctx.rexw = false; ctx.rexr = false; ctx.rexx = false; ctx.rexb = false 440 ctx.rexw = false; ctx.rexr = false; ctx.rexx = false; ctx.rexb = false
393 ctx.rex = false 441 ctx.rex = false; ctx.vexl = false; ctx.vexv = false
394 end 442 end
395 if ctx.seg then 443 if ctx.seg then
396 local text2, n = gsub(text, "%[", "["..ctx.seg..":") 444 local text2, n = gsub(text, "%[", "["..ctx.seg..":")
@@ -405,6 +453,7 @@ local function putop(ctx, text, operands)
405 end 453 end
406 ctx.out(format("%08x %s%s\n", ctx.addr+ctx.start, hex, text)) 454 ctx.out(format("%08x %s%s\n", ctx.addr+ctx.start, hex, text))
407 ctx.mrm = false 455 ctx.mrm = false
456 ctx.vexv = false
408 ctx.start = pos 457 ctx.start = pos
409 ctx.imm = nil 458 ctx.imm = nil
410end 459end
@@ -413,7 +462,7 @@ end
413local function clearprefixes(ctx) 462local function clearprefixes(ctx)
414 ctx.o16 = false; ctx.seg = false; ctx.lock = false; ctx.rep = false 463 ctx.o16 = false; ctx.seg = false; ctx.lock = false; ctx.rep = false
415 ctx.rexw = false; ctx.rexr = false; ctx.rexx = false; ctx.rexb = false 464 ctx.rexw = false; ctx.rexr = false; ctx.rexx = false; ctx.rexb = false
416 ctx.rex = false; ctx.a32 = false 465 ctx.rex = false; ctx.a32 = false; ctx.vexl = false
417end 466end
418 467
419-- Fallback for incomplete opcodes at the end. 468-- Fallback for incomplete opcodes at the end.
@@ -450,9 +499,9 @@ end
450-- Process pattern string and generate the operands. 499-- Process pattern string and generate the operands.
451local function putpat(ctx, name, pat) 500local function putpat(ctx, name, pat)
452 local operands, regs, sz, mode, sp, rm, sc, rx, sdisp 501 local operands, regs, sz, mode, sp, rm, sc, rx, sdisp
453 local code, pos, stop = ctx.code, ctx.pos, ctx.stop 502 local code, pos, stop, vexl = ctx.code, ctx.pos, ctx.stop, ctx.vexl
454 503
455 -- Chars used: 1DFGIMPQRSTUVWXacdfgijmoprstuwxyz 504 -- Chars used: 1DFGHIMPQRSTUVWXYabcdfgijlmoprstuvwxyz
456 for p in gmatch(pat, ".") do 505 for p in gmatch(pat, ".") do
457 local x = nil 506 local x = nil
458 if p == "V" or p == "U" then 507 if p == "V" or p == "U" then
@@ -467,12 +516,17 @@ local function putpat(ctx, name, pat)
467 elseif p == "B" then 516 elseif p == "B" then
468 sz = "B" 517 sz = "B"
469 regs = ctx.rex and map_regs.B64 or map_regs.B 518 regs = ctx.rex and map_regs.B64 or map_regs.B
470 elseif match(p, "[WDQMXFG]") then 519 elseif match(p, "[WDQMXYFG]") then
471 sz = p 520 sz = p
521 if sz == "X" and vexl then sz = "Y"; ctx.vexl = false end
472 regs = map_regs[sz] 522 regs = map_regs[sz]
473 elseif p == "P" then 523 elseif p == "P" then
474 sz = ctx.o16 and "X" or "M"; ctx.o16 = false 524 sz = ctx.o16 and "X" or "M"; ctx.o16 = false
525 if sz == "X" and vexl then sz = "Y"; ctx.vexl = false end
475 regs = map_regs[sz] 526 regs = map_regs[sz]
527 elseif p == "H" then
528 name = name..(ctx.rexw and "d" or "s")
529 ctx.rexw = false
476 elseif p == "S" then 530 elseif p == "S" then
477 name = name..lower(sz) 531 name = name..lower(sz)
478 elseif p == "s" then 532 elseif p == "s" then
@@ -484,6 +538,10 @@ local function putpat(ctx, name, pat)
484 local imm = getimm(ctx, pos, 1); if not imm then return end 538 local imm = getimm(ctx, pos, 1); if not imm then return end
485 x = format("0x%02x", imm) 539 x = format("0x%02x", imm)
486 pos = pos+1 540 pos = pos+1
541 elseif p == "b" then
542 local imm = getimm(ctx, pos, 1); if not imm then return end
543 x = regs[imm/16+1]
544 pos = pos+1
487 elseif p == "w" then 545 elseif p == "w" then
488 local imm = getimm(ctx, pos, 2); if not imm then return end 546 local imm = getimm(ctx, pos, 2); if not imm then return end
489 x = format("0x%x", imm) 547 x = format("0x%x", imm)
@@ -532,7 +590,7 @@ local function putpat(ctx, name, pat)
532 local lo = imm % 0x1000000 590 local lo = imm % 0x1000000
533 x = format("0x%02x%06x", (imm-lo) / 0x1000000, lo) 591 x = format("0x%02x%06x", (imm-lo) / 0x1000000, lo)
534 else 592 else
535 x = format("0x%08x", imm) 593 x = "0x"..tohex(imm)
536 end 594 end
537 elseif p == "R" then 595 elseif p == "R" then
538 local r = byte(code, pos-1, pos-1)%8 596 local r = byte(code, pos-1, pos-1)%8
@@ -616,8 +674,13 @@ local function putpat(ctx, name, pat)
616 else 674 else
617 x = "CR"..sp 675 x = "CR"..sp
618 end 676 end
677 elseif p == "v" then
678 if ctx.vexv then
679 x = regs[ctx.vexv+1]; ctx.vexv = false
680 end
619 elseif p == "y" then x = "DR"..sp 681 elseif p == "y" then x = "DR"..sp
620 elseif p == "z" then x = "TR"..sp 682 elseif p == "z" then x = "TR"..sp
683 elseif p == "l" then vexl = false
621 elseif p == "t" then 684 elseif p == "t" then
622 else 685 else
623 error("bad pattern `"..pat.."'") 686 error("bad pattern `"..pat.."'")
@@ -692,7 +755,8 @@ map_act = {
692 B = putpat, W = putpat, D = putpat, Q = putpat, 755 B = putpat, W = putpat, D = putpat, Q = putpat,
693 V = putpat, U = putpat, T = putpat, 756 V = putpat, U = putpat, T = putpat,
694 M = putpat, X = putpat, P = putpat, 757 M = putpat, X = putpat, P = putpat,
695 F = putpat, G = putpat, 758 F = putpat, G = putpat, Y = putpat,
759 H = putpat,
696 760
697 -- Collect prefixes. 761 -- Collect prefixes.
698 [":"] = function(ctx, name, pat) 762 [":"] = function(ctx, name, pat)
@@ -753,15 +817,68 @@ map_act = {
753 817
754 -- REX prefix. 818 -- REX prefix.
755 rex = function(ctx, name, pat) 819 rex = function(ctx, name, pat)
756 if ctx.rex then return unknown(ctx) end -- Only 1 REX prefix allowed. 820 if ctx.rex then return unknown(ctx) end -- Only 1 REX or VEX prefix allowed.
757 for p in gmatch(pat, ".") do ctx["rex"..p] = true end 821 for p in gmatch(pat, ".") do ctx["rex"..p] = true end
758 ctx.rex = true 822 ctx.rex = "rex"
823 end,
824
825 -- VEX prefix.
826 vex = function(ctx, name, pat)
827 if ctx.rex then return unknown(ctx) end -- Only 1 REX or VEX prefix allowed.
828 ctx.rex = "vex"
829 local pos = ctx.pos
830 if ctx.mrm then
831 ctx.mrm = nil
832 pos = pos-1
833 end
834 local b = byte(ctx.code, pos, pos)
835 if not b then return incomplete(ctx) end
836 pos = pos+1
837 if b < 128 then ctx.rexr = true end
838 local m = 1
839 if pat == "3" then
840 m = b%32; b = (b-m)/32
841 local nb = b%2; b = (b-nb)/2
842 if nb == 0 then ctx.rexb = true end
843 local nx = b%2
844 if nx == 0 then ctx.rexx = true end
845 b = byte(ctx.code, pos, pos)
846 if not b then return incomplete(ctx) end
847 pos = pos+1
848 if b >= 128 then ctx.rexw = true end
849 end
850 ctx.pos = pos
851 local map
852 if m == 1 then map = map_opc2
853 elseif m == 2 then map = map_opc3["38"]
854 elseif m == 3 then map = map_opc3["3a"]
855 else return unknown(ctx) end
856 local p = b%4; b = (b-p)/4
857 if p == 1 then ctx.o16 = "o16"
858 elseif p == 2 then ctx.rep = "rep"
859 elseif p == 3 then ctx.rep = "repne" end
860 local l = b%2; b = (b-l)/2
861 if l ~= 0 then ctx.vexl = true end
862 ctx.vexv = (-1-b)%16
863 return dispatchmap(ctx, map)
759 end, 864 end,
760 865
761 -- Special case for nop with REX prefix. 866 -- Special case for nop with REX prefix.
762 nop = function(ctx, name, pat) 867 nop = function(ctx, name, pat)
763 return dispatch(ctx, ctx.rex and pat or "nop") 868 return dispatch(ctx, ctx.rex and pat or "nop")
764 end, 869 end,
870
871 -- Special case for 0F 77.
872 emms = function(ctx, name, pat)
873 if ctx.rex ~= "vex" then
874 return putop(ctx, "emms")
875 elseif ctx.vexl then
876 ctx.vexl = false
877 return putop(ctx, "zeroall")
878 else
879 return putop(ctx, "zeroupper")
880 end
881 end,
765} 882}
766 883
767------------------------------------------------------------------------------ 884------------------------------------------------------------------------------
@@ -782,7 +899,7 @@ local function disass_block(ctx, ofs, len)
782end 899end
783 900
784-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len). 901-- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
785local function create_(code, addr, out) 902local function create(code, addr, out)
786 local ctx = {} 903 local ctx = {}
787 ctx.code = code 904 ctx.code = code
788 ctx.addr = (addr or 0) - 1 905 ctx.addr = (addr or 0) - 1
@@ -796,8 +913,8 @@ local function create_(code, addr, out)
796 return ctx 913 return ctx
797end 914end
798 915
799local function create64_(code, addr, out) 916local function create64(code, addr, out)
800 local ctx = create_(code, addr, out) 917 local ctx = create(code, addr, out)
801 ctx.x64 = true 918 ctx.x64 = true
802 ctx.map1 = map_opc1_64 919 ctx.map1 = map_opc1_64
803 ctx.aregs = map_regs.Q 920 ctx.aregs = map_regs.Q
@@ -805,32 +922,32 @@ local function create64_(code, addr, out)
805end 922end
806 923
807-- Simple API: disassemble code (a string) at address and output via out. 924-- Simple API: disassemble code (a string) at address and output via out.
808local function disass_(code, addr, out) 925local function disass(code, addr, out)
809 create_(code, addr, out):disass() 926 create(code, addr, out):disass()
810end 927end
811 928
812local function disass64_(code, addr, out) 929local function disass64(code, addr, out)
813 create64_(code, addr, out):disass() 930 create64(code, addr, out):disass()
814end 931end
815 932
816-- Return register name for RID. 933-- Return register name for RID.
817local function regname_(r) 934local function regname(r)
818 if r < 8 then return map_regs.D[r+1] end 935 if r < 8 then return map_regs.D[r+1] end
819 return map_regs.X[r-7] 936 return map_regs.X[r-7]
820end 937end
821 938
822local function regname64_(r) 939local function regname64(r)
823 if r < 16 then return map_regs.Q[r+1] end 940 if r < 16 then return map_regs.Q[r+1] end
824 return map_regs.X[r-15] 941 return map_regs.X[r-15]
825end 942end
826 943
827-- Public module functions. 944-- Public module functions.
828module(...) 945return {
829 946 create = create,
830create = create_ 947 create64 = create64,
831create64 = create64_ 948 disass = disass,
832disass = disass_ 949 disass64 = disass64,
833disass64 = disass64_ 950 regname = regname,
834regname = regname_ 951 regname64 = regname64
835regname64 = regname64_ 952}
836 953
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 86f11e26..746732f9 100644
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -62,7 +62,7 @@ local traceinfo, traceir, tracek = jutil.traceinfo, jutil.traceir, jutil.tracek
62local tracemc, tracesnap = jutil.tracemc, jutil.tracesnap 62local tracemc, tracesnap = jutil.tracemc, jutil.tracesnap
63local traceexitstub, ircalladdr = jutil.traceexitstub, jutil.ircalladdr 63local traceexitstub, ircalladdr = jutil.traceexitstub, jutil.ircalladdr
64local bit = require("bit") 64local bit = require("bit")
65local band, shr = bit.band, bit.rshift 65local band, shr, tohex = bit.band, bit.rshift, bit.tohex
66local sub, gsub, format = string.sub, string.gsub, string.format 66local sub, gsub, format = string.sub, string.gsub, string.format
67local byte, rep = string.byte, string.rep 67local byte, rep = string.byte, string.rep
68local type, tostring = type, tostring 68local type, tostring = type, tostring
@@ -84,12 +84,13 @@ local nexitsym = 0
84local function fillsymtab_tr(tr, nexit) 84local function fillsymtab_tr(tr, nexit)
85 local t = {} 85 local t = {}
86 symtabmt.__index = t 86 symtabmt.__index = t
87 if jit.arch == "mips" or jit.arch == "mipsel" then 87 if jit.arch:sub(1, 4) == "mips" then
88 t[traceexitstub(tr, 0)] = "exit" 88 t[traceexitstub(tr, 0)] = "exit"
89 return 89 return
90 end 90 end
91 for i=0,nexit-1 do 91 for i=0,nexit-1 do
92 local addr = traceexitstub(tr, i) 92 local addr = traceexitstub(tr, i)
93 if addr < 0 then addr = addr + 2^32 end
93 t[addr] = tostring(i) 94 t[addr] = tostring(i)
94 end 95 end
95 local addr = traceexitstub(tr, nexit) 96 local addr = traceexitstub(tr, nexit)
@@ -100,10 +101,15 @@ end
100local function fillsymtab(tr, nexit) 101local function fillsymtab(tr, nexit)
101 local t = symtab 102 local t = symtab
102 if nexitsym == 0 then 103 if nexitsym == 0 then
104 local maskaddr = jit.arch == "arm" and -2
103 local ircall = vmdef.ircall 105 local ircall = vmdef.ircall
104 for i=0,#ircall do 106 for i=0,#ircall do
105 local addr = ircalladdr(i) 107 local addr = ircalladdr(i)
106 if addr ~= 0 then t[addr] = ircall[i] end 108 if addr ~= 0 then
109 if maskaddr then addr = band(addr, maskaddr) end
110 if addr < 0 then addr = addr + 2^32 end
111 t[addr] = ircall[i]
112 end
107 end 113 end
108 end 114 end
109 if nexitsym == 1000000 then -- Per-trace exit stubs. 115 if nexitsym == 1000000 then -- Per-trace exit stubs.
@@ -117,6 +123,7 @@ local function fillsymtab(tr, nexit)
117 nexit = 1000000 123 nexit = 1000000
118 break 124 break
119 end 125 end
126 if addr < 0 then addr = addr + 2^32 end
120 t[addr] = tostring(i) 127 t[addr] = tostring(i)
121 end 128 end
122 nexitsym = nexit 129 nexitsym = nexit
@@ -135,6 +142,7 @@ local function dump_mcode(tr)
135 local mcode, addr, loop = tracemc(tr) 142 local mcode, addr, loop = tracemc(tr)
136 if not mcode then return end 143 if not mcode then return end
137 if not disass then disass = require("jit.dis_"..jit.arch) end 144 if not disass then disass = require("jit.dis_"..jit.arch) end
145 if addr < 0 then addr = addr + 2^32 end
138 out:write("---- TRACE ", tr, " mcode ", #mcode, "\n") 146 out:write("---- TRACE ", tr, " mcode ", #mcode, "\n")
139 local ctx = disass.create(mcode, addr, dumpwrite) 147 local ctx = disass.create(mcode, addr, dumpwrite)
140 ctx.hexdump = 0 148 ctx.hexdump = 0
@@ -210,8 +218,10 @@ local function colorize_text(s)
210 return s 218 return s
211end 219end
212 220
213local function colorize_ansi(s, t) 221local function colorize_ansi(s, t, extra)
214 return format(colortype_ansi[t], s) 222 local out = format(colortype_ansi[t], s)
223 if extra then out = "\027[3m"..out end
224 return out
215end 225end
216 226
217local irtype_ansi = setmetatable({}, 227local irtype_ansi = setmetatable({},
@@ -220,9 +230,10 @@ local irtype_ansi = setmetatable({},
220 230
221local html_escape = { ["<"] = "&lt;", [">"] = "&gt;", ["&"] = "&amp;", } 231local html_escape = { ["<"] = "&lt;", [">"] = "&gt;", ["&"] = "&amp;", }
222 232
223local function colorize_html(s, t) 233local function colorize_html(s, t, extra)
224 s = gsub(s, "[<>&]", html_escape) 234 s = gsub(s, "[<>&]", html_escape)
225 return format('<span class="irt_%s">%s</span>', irtype_text[t], s) 235 return format('<span class="irt_%s%s">%s</span>',
236 irtype_text[t], extra and " irt_extra" or "", s)
226end 237end
227 238
228local irtype_html = setmetatable({}, 239local irtype_html = setmetatable({},
@@ -247,6 +258,7 @@ span.irt_tab { color: #c00000; }
247span.irt_udt, span.irt_lud { color: #00c0c0; } 258span.irt_udt, span.irt_lud { color: #00c0c0; }
248span.irt_num { color: #4040c0; } 259span.irt_num { color: #4040c0; }
249span.irt_int, span.irt_i8, span.irt_u8, span.irt_i16, span.irt_u16 { color: #b040b0; } 260span.irt_int, span.irt_i8, span.irt_u8, span.irt_i16, span.irt_u16 { color: #b040b0; }
261span.irt_extra { font-style: italic; }
250</style> 262</style>
251]] 263]]
252 264
@@ -262,6 +274,7 @@ local litname = {
262 if band(mode, 8) ~= 0 then s = s.."C" end 274 if band(mode, 8) ~= 0 then s = s.."C" end
263 if band(mode, 16) ~= 0 then s = s.."R" end 275 if band(mode, 16) ~= 0 then s = s.."R" end
264 if band(mode, 32) ~= 0 then s = s.."I" end 276 if band(mode, 32) ~= 0 then s = s.."I" end
277 if band(mode, 64) ~= 0 then s = s.."K" end
265 t[mode] = s 278 t[mode] = s
266 return s 279 return s
267 end}), 280 end}),
@@ -269,16 +282,20 @@ local litname = {
269 ["CONV "] = setmetatable({}, { __index = function(t, mode) 282 ["CONV "] = setmetatable({}, { __index = function(t, mode)
270 local s = irtype[band(mode, 31)] 283 local s = irtype[band(mode, 31)]
271 s = irtype[band(shr(mode, 5), 31)].."."..s 284 s = irtype[band(shr(mode, 5), 31)].."."..s
272 if band(mode, 0x400) ~= 0 then s = s.." trunc" 285 if band(mode, 0x800) ~= 0 then s = s.." sext" end
273 elseif band(mode, 0x800) ~= 0 then s = s.." sext" end
274 local c = shr(mode, 12) 286 local c = shr(mode, 12)
275 if c == 2 then s = s.." index" elseif c == 3 then s = s.." check" end 287 if c == 1 then s = s.." none"
288 elseif c == 2 then s = s.." index"
289 elseif c == 3 then s = s.." check" end
276 t[mode] = s 290 t[mode] = s
277 return s 291 return s
278 end}), 292 end}),
279 ["FLOAD "] = vmdef.irfield, 293 ["FLOAD "] = vmdef.irfield,
280 ["FREF "] = vmdef.irfield, 294 ["FREF "] = vmdef.irfield,
281 ["FPMATH"] = vmdef.irfpm, 295 ["FPMATH"] = vmdef.irfpm,
296 ["TMPREF"] = { [0] = "", "IN", "OUT", "INOUT", "", "", "OUT2", "INOUT2" },
297 ["BUFHDR"] = { [0] = "RESET", "APPEND", "WRITE" },
298 ["TOSTR "] = { [0] = "INT", "NUM", "CHAR" },
282} 299}
283 300
284local function ctlsub(c) 301local function ctlsub(c)
@@ -302,15 +319,19 @@ local function fmtfunc(func, pc)
302 end 319 end
303end 320end
304 321
305local function formatk(tr, idx) 322local function formatk(tr, idx, sn)
306 local k, t, slot = tracek(tr, idx) 323 local k, t, slot = tracek(tr, idx)
307 local tn = type(k) 324 local tn = type(k)
308 local s 325 local s
309 if tn == "number" then 326 if tn == "number" then
310 if k == 2^52+2^51 then 327 if t < 12 then
328 s = k == 0 and "NULL" or format("[0x%08x]", k)
329 elseif band(sn or 0, 0x30000) ~= 0 then
330 s = band(sn, 0x20000) ~= 0 and "contpc" or "ftsz"
331 elseif k == 2^52+2^51 then
311 s = "bias" 332 s = "bias"
312 else 333 else
313 s = format("%+.14g", k) 334 s = format(0 < k and k < 0x1p-1026 and "%+a" or "%+.14g", k)
314 end 335 end
315 elseif tn == "string" then 336 elseif tn == "string" then
316 s = format(#k > 20 and '"%.20s"~' or '"%s"', gsub(k, "%c", ctlsub)) 337 s = format(#k > 20 and '"%.20s"~' or '"%s"', gsub(k, "%c", ctlsub))
@@ -328,10 +349,12 @@ local function formatk(tr, idx)
328 elseif t == 21 then -- int64_t 349 elseif t == 21 then -- int64_t
329 s = sub(tostring(k), 1, -3) 350 s = sub(tostring(k), 1, -3)
330 if sub(s, 1, 1) ~= "-" then s = "+"..s end 351 if sub(s, 1, 1) ~= "-" then s = "+"..s end
352 elseif sn == 0x1057fff then -- SNAP(1, SNAP_FRAME | SNAP_NORESTORE, REF_NIL)
353 return "----" -- Special case for LJ_FR2 slot 1.
331 else 354 else
332 s = tostring(k) -- For primitives. 355 s = tostring(k) -- For primitives.
333 end 356 end
334 s = colorize(format("%-4s", s), t) 357 s = colorize(format("%-4s", s), t, band(sn or 0, 0x100000) ~= 0)
335 if slot then 358 if slot then
336 s = format("%s @%d", s, slot) 359 s = format("%s @%d", s, slot)
337 end 360 end
@@ -346,12 +369,12 @@ local function printsnap(tr, snap)
346 n = n + 1 369 n = n + 1
347 local ref = band(sn, 0xffff) - 0x8000 -- REF_BIAS 370 local ref = band(sn, 0xffff) - 0x8000 -- REF_BIAS
348 if ref < 0 then 371 if ref < 0 then
349 out:write(formatk(tr, ref)) 372 out:write(formatk(tr, ref, sn))
350 elseif band(sn, 0x80000) ~= 0 then -- SNAP_SOFTFPNUM 373 elseif band(sn, 0x80000) ~= 0 then -- SNAP_SOFTFPNUM
351 out:write(colorize(format("%04d/%04d", ref, ref+1), 14)) 374 out:write(colorize(format("%04d/%04d", ref, ref+1), 14))
352 else 375 else
353 local m, ot, op1, op2 = traceir(tr, ref) 376 local m, ot, op1, op2 = traceir(tr, ref)
354 out:write(colorize(format("%04d", ref), band(ot, 31))) 377 out:write(colorize(format("%04d", ref), band(ot, 31), band(sn, 0x100000) ~= 0))
355 end 378 end
356 out:write(band(sn, 0x10000) == 0 and " " or "|") -- SNAP_FRAME 379 out:write(band(sn, 0x10000) == 0 and " " or "|") -- SNAP_FRAME
357 else 380 else
@@ -544,7 +567,7 @@ local function dump_trace(what, tr, func, pc, otr, oex)
544 if what == "start" then 567 if what == "start" then
545 if dumpmode.H then out:write('<pre class="ljdump">\n') end 568 if dumpmode.H then out:write('<pre class="ljdump">\n') end
546 out:write("---- TRACE ", tr, " ", what) 569 out:write("---- TRACE ", tr, " ", what)
547 if otr then out:write(" ", otr, "/", oex) end 570 if otr then out:write(" ", otr, "/", oex == -1 and "stitch" or oex) end
548 out:write(" ", fmtfunc(func, pc), "\n") 571 out:write(" ", fmtfunc(func, pc), "\n")
549 elseif what == "stop" or what == "abort" then 572 elseif what == "stop" or what == "abort" then
550 out:write("---- TRACE ", tr, " ", what) 573 out:write("---- TRACE ", tr, " ", what)
@@ -594,23 +617,26 @@ end
594 617
595------------------------------------------------------------------------------ 618------------------------------------------------------------------------------
596 619
620local gpr64 = jit.arch:match("64")
621local fprmips32 = jit.arch == "mips" or jit.arch == "mipsel"
622
597-- Dump taken trace exits. 623-- Dump taken trace exits.
598local function dump_texit(tr, ex, ngpr, nfpr, ...) 624local function dump_texit(tr, ex, ngpr, nfpr, ...)
599 out:write("---- TRACE ", tr, " exit ", ex, "\n") 625 out:write("---- TRACE ", tr, " exit ", ex, "\n")
600 if dumpmode.X then 626 if dumpmode.X then
601 local regs = {...} 627 local regs = {...}
602 if jit.arch == "x64" then 628 if gpr64 then
603 for i=1,ngpr do 629 for i=1,ngpr do
604 out:write(format(" %016x", regs[i])) 630 out:write(format(" %016x", regs[i]))
605 if i % 4 == 0 then out:write("\n") end 631 if i % 4 == 0 then out:write("\n") end
606 end 632 end
607 else 633 else
608 for i=1,ngpr do 634 for i=1,ngpr do
609 out:write(format(" %08x", regs[i])) 635 out:write(" ", tohex(regs[i]))
610 if i % 8 == 0 then out:write("\n") end 636 if i % 8 == 0 then out:write("\n") end
611 end 637 end
612 end 638 end
613 if jit.arch == "mips" or jit.arch == "mipsel" then 639 if fprmips32 then
614 for i=1,nfpr,2 do 640 for i=1,nfpr,2 do
615 out:write(format(" %+17.14g", regs[ngpr+i])) 641 out:write(format(" %+17.14g", regs[ngpr+i]))
616 if i % 8 == 7 then out:write("\n") end 642 if i % 8 == 7 then out:write("\n") end
@@ -691,9 +717,9 @@ local function dumpon(opt, outfile)
691end 717end
692 718
693-- Public module functions. 719-- Public module functions.
694module(...) 720return {
695 721 on = dumpon,
696on = dumpon 722 off = dumpoff,
697off = dumpoff 723 start = dumpon -- For -j command line option.
698start = dumpon -- For -j command line option. 724}
699 725
diff --git a/src/jit/p.lua b/src/jit/p.lua
new file mode 100644
index 00000000..36f836c5
--- /dev/null
+++ b/src/jit/p.lua
@@ -0,0 +1,311 @@
1----------------------------------------------------------------------------
2-- LuaJIT profiler.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7--
8-- This module is a simple command line interface to the built-in
9-- low-overhead profiler of LuaJIT.
10--
11-- The lower-level API of the profiler is accessible via the "jit.profile"
12-- module or the luaJIT_profile_* C API.
13--
14-- Example usage:
15--
16-- luajit -jp myapp.lua
17-- luajit -jp=s myapp.lua
18-- luajit -jp=-s myapp.lua
19-- luajit -jp=vl myapp.lua
20-- luajit -jp=G,profile.txt myapp.lua
21--
22-- The following dump features are available:
23--
24-- f Stack dump: function name, otherwise module:line. Default mode.
25-- F Stack dump: ditto, but always prepend module.
26-- l Stack dump: module:line.
27-- <number> stack dump depth (callee < caller). Default: 1.
28-- -<number> Inverse stack dump depth (caller > callee).
29-- s Split stack dump after first stack level. Implies abs(depth) >= 2.
30-- p Show full path for module names.
31-- v Show VM states. Can be combined with stack dumps, e.g. vf or fv.
32-- z Show zones. Can be combined with stack dumps, e.g. zf or fz.
33-- r Show raw sample counts. Default: show percentages.
34-- a Annotate excerpts from source code files.
35-- A Annotate complete source code files.
36-- G Produce raw output suitable for graphical tools (e.g. flame graphs).
37-- m<number> Minimum sample percentage to be shown. Default: 3.
38-- i<number> Sampling interval in milliseconds. Default: 10.
39--
40----------------------------------------------------------------------------
41
42-- Cache some library functions and objects.
43local jit = require("jit")
44local profile = require("jit.profile")
45local vmdef = require("jit.vmdef")
46local math = math
47local pairs, ipairs, tonumber, floor = pairs, ipairs, tonumber, math.floor
48local sort, format = table.sort, string.format
49local stdout = io.stdout
50local zone -- Load jit.zone module on demand.
51
52-- Output file handle.
53local out
54
55------------------------------------------------------------------------------
56
57local prof_ud
58local prof_states, prof_split, prof_min, prof_raw, prof_fmt, prof_depth
59local prof_ann, prof_count1, prof_count2, prof_samples
60
61local map_vmmode = {
62 N = "Compiled",
63 I = "Interpreted",
64 C = "C code",
65 G = "Garbage Collector",
66 J = "JIT Compiler",
67}
68
69-- Profiler callback.
70local function prof_cb(th, samples, vmmode)
71 prof_samples = prof_samples + samples
72 local key_stack, key_stack2, key_state
73 -- Collect keys for sample.
74 if prof_states then
75 if prof_states == "v" then
76 key_state = map_vmmode[vmmode] or vmmode
77 else
78 key_state = zone:get() or "(none)"
79 end
80 end
81 if prof_fmt then
82 key_stack = profile.dumpstack(th, prof_fmt, prof_depth)
83 key_stack = key_stack:gsub("%[builtin#(%d+)%]", function(x)
84 return vmdef.ffnames[tonumber(x)]
85 end)
86 if prof_split == 2 then
87 local k1, k2 = key_stack:match("(.-) [<>] (.*)")
88 if k2 then key_stack, key_stack2 = k1, k2 end
89 elseif prof_split == 3 then
90 key_stack2 = profile.dumpstack(th, "l", 1)
91 end
92 end
93 -- Order keys.
94 local k1, k2
95 if prof_split == 1 then
96 if key_state then
97 k1 = key_state
98 if key_stack then k2 = key_stack end
99 end
100 elseif key_stack then
101 k1 = key_stack
102 if key_stack2 then k2 = key_stack2 elseif key_state then k2 = key_state end
103 end
104 -- Coalesce samples in one or two levels.
105 if k1 then
106 local t1 = prof_count1
107 t1[k1] = (t1[k1] or 0) + samples
108 if k2 then
109 local t2 = prof_count2
110 local t3 = t2[k1]
111 if not t3 then t3 = {}; t2[k1] = t3 end
112 t3[k2] = (t3[k2] or 0) + samples
113 end
114 end
115end
116
117------------------------------------------------------------------------------
118
119-- Show top N list.
120local function prof_top(count1, count2, samples, indent)
121 local t, n = {}, 0
122 for k in pairs(count1) do
123 n = n + 1
124 t[n] = k
125 end
126 sort(t, function(a, b) return count1[a] > count1[b] end)
127 for i=1,n do
128 local k = t[i]
129 local v = count1[k]
130 local pct = floor(v*100/samples + 0.5)
131 if pct < prof_min then break end
132 if not prof_raw then
133 out:write(format("%s%2d%% %s\n", indent, pct, k))
134 elseif prof_raw == "r" then
135 out:write(format("%s%5d %s\n", indent, v, k))
136 else
137 out:write(format("%s %d\n", k, v))
138 end
139 if count2 then
140 local r = count2[k]
141 if r then
142 prof_top(r, nil, v, (prof_split == 3 or prof_split == 1) and " -- " or
143 (prof_depth < 0 and " -> " or " <- "))
144 end
145 end
146 end
147end
148
149-- Annotate source code
150local function prof_annotate(count1, samples)
151 local files = {}
152 local ms = 0
153 for k, v in pairs(count1) do
154 local pct = floor(v*100/samples + 0.5)
155 ms = math.max(ms, v)
156 if pct >= prof_min then
157 local file, line = k:match("^(.*):(%d+)$")
158 if not file then file = k; line = 0 end
159 local fl = files[file]
160 if not fl then fl = {}; files[file] = fl; files[#files+1] = file end
161 line = tonumber(line)
162 fl[line] = prof_raw and v or pct
163 end
164 end
165 sort(files)
166 local fmtv, fmtn = " %3d%% | %s\n", " | %s\n"
167 if prof_raw then
168 local n = math.max(5, math.ceil(math.log10(ms)))
169 fmtv = "%"..n.."d | %s\n"
170 fmtn = (" "):rep(n).." | %s\n"
171 end
172 local ann = prof_ann
173 for _, file in ipairs(files) do
174 local f0 = file:byte()
175 if f0 == 40 or f0 == 91 then
176 out:write(format("\n====== %s ======\n[Cannot annotate non-file]\n", file))
177 break
178 end
179 local fp, err = io.open(file)
180 if not fp then
181 out:write(format("====== ERROR: %s: %s\n", file, err))
182 break
183 end
184 out:write(format("\n====== %s ======\n", file))
185 local fl = files[file]
186 local n, show = 1, false
187 if ann ~= 0 then
188 for i=1,ann do
189 if fl[i] then show = true; out:write("@@ 1 @@\n"); break end
190 end
191 end
192 for line in fp:lines() do
193 if line:byte() == 27 then
194 out:write("[Cannot annotate bytecode file]\n")
195 break
196 end
197 local v = fl[n]
198 if ann ~= 0 then
199 local v2 = fl[n+ann]
200 if show then
201 if v2 then show = n+ann elseif v then show = n
202 elseif show+ann < n then show = false end
203 elseif v2 then
204 show = n+ann
205 out:write(format("@@ %d @@\n", n))
206 end
207 if not show then goto next end
208 end
209 if v then
210 out:write(format(fmtv, v, line))
211 else
212 out:write(format(fmtn, line))
213 end
214 ::next::
215 n = n + 1
216 end
217 fp:close()
218 end
219end
220
221------------------------------------------------------------------------------
222
223-- Finish profiling and dump result.
224local function prof_finish()
225 if prof_ud then
226 profile.stop()
227 local samples = prof_samples
228 if samples == 0 then
229 if prof_raw ~= true then out:write("[No samples collected]\n") end
230 return
231 end
232 if prof_ann then
233 prof_annotate(prof_count1, samples)
234 else
235 prof_top(prof_count1, prof_count2, samples, "")
236 end
237 prof_count1 = nil
238 prof_count2 = nil
239 prof_ud = nil
240 if out ~= stdout then out:close() end
241 end
242end
243
244-- Start profiling.
245local function prof_start(mode)
246 local interval = ""
247 mode = mode:gsub("i%d*", function(s) interval = s; return "" end)
248 prof_min = 3
249 mode = mode:gsub("m(%d+)", function(s) prof_min = tonumber(s); return "" end)
250 prof_depth = 1
251 mode = mode:gsub("%-?%d+", function(s) prof_depth = tonumber(s); return "" end)
252 local m = {}
253 for c in mode:gmatch(".") do m[c] = c end
254 prof_states = m.z or m.v
255 if prof_states == "z" then zone = require("jit.zone") end
256 local scope = m.l or m.f or m.F or (prof_states and "" or "f")
257 local flags = (m.p or "")
258 prof_raw = m.r
259 if m.s then
260 prof_split = 2
261 if prof_depth == -1 or m["-"] then prof_depth = -2
262 elseif prof_depth == 1 then prof_depth = 2 end
263 elseif mode:find("[fF].*l") then
264 scope = "l"
265 prof_split = 3
266 else
267 prof_split = (scope == "" or mode:find("[zv].*[lfF]")) and 1 or 0
268 end
269 prof_ann = m.A and 0 or (m.a and 3)
270 if prof_ann then
271 scope = "l"
272 prof_fmt = "pl"
273 prof_split = 0
274 prof_depth = 1
275 elseif m.G and scope ~= "" then
276 prof_fmt = flags..scope.."Z;"
277 prof_depth = -100
278 prof_raw = true
279 prof_min = 0
280 elseif scope == "" then
281 prof_fmt = false
282 else
283 local sc = prof_split == 3 and m.f or m.F or scope
284 prof_fmt = flags..sc..(prof_depth >= 0 and "Z < " or "Z > ")
285 end
286 prof_count1 = {}
287 prof_count2 = {}
288 prof_samples = 0
289 profile.start(scope:lower()..interval, prof_cb)
290 prof_ud = newproxy(true)
291 getmetatable(prof_ud).__gc = prof_finish
292end
293
294------------------------------------------------------------------------------
295
296local function start(mode, outfile)
297 if not outfile then outfile = os.getenv("LUAJIT_PROFILEFILE") end
298 if outfile then
299 out = outfile == "-" and stdout or assert(io.open(outfile, "w"))
300 else
301 out = stdout
302 end
303 prof_start(mode or "f")
304end
305
306-- Public module functions.
307return {
308 start = start, -- For -j command line option.
309 stop = prof_finish
310}
311
diff --git a/src/jit/v.lua b/src/jit/v.lua
index 29edcf2b..8e91f494 100644
--- a/src/jit/v.lua
+++ b/src/jit/v.lua
@@ -98,7 +98,7 @@ end
98local function dump_trace(what, tr, func, pc, otr, oex) 98local function dump_trace(what, tr, func, pc, otr, oex)
99 if what == "start" then 99 if what == "start" then
100 startloc = fmtfunc(func, pc) 100 startloc = fmtfunc(func, pc)
101 startex = otr and "("..otr.."/"..oex..") " or "" 101 startex = otr and "("..otr.."/"..(oex == -1 and "stitch" or oex)..") " or ""
102 else 102 else
103 if what == "abort" then 103 if what == "abort" then
104 local loc = fmtfunc(func, pc) 104 local loc = fmtfunc(func, pc)
@@ -115,6 +115,9 @@ local function dump_trace(what, tr, func, pc, otr, oex)
115 if ltype == "interpreter" then 115 if ltype == "interpreter" then
116 out:write(format("[TRACE %3s %s%s -- fallback to interpreter]\n", 116 out:write(format("[TRACE %3s %s%s -- fallback to interpreter]\n",
117 tr, startex, startloc)) 117 tr, startex, startloc))
118 elseif ltype == "stitch" then
119 out:write(format("[TRACE %3s %s%s %s %s]\n",
120 tr, startex, startloc, ltype, fmtfunc(func, pc)))
118 elseif link == tr or link == 0 then 121 elseif link == tr or link == 0 then
119 out:write(format("[TRACE %3s %s%s %s]\n", 122 out:write(format("[TRACE %3s %s%s %s]\n",
120 tr, startex, startloc, ltype)) 123 tr, startex, startloc, ltype))
@@ -158,9 +161,9 @@ local function dumpon(outfile)
158end 161end
159 162
160-- Public module functions. 163-- Public module functions.
161module(...) 164return {
162 165 on = dumpon,
163on = dumpon 166 off = dumpoff,
164off = dumpoff 167 start = dumpon -- For -j command line option.
165start = dumpon -- For -j command line option. 168}
166 169
diff --git a/src/jit/zone.lua b/src/jit/zone.lua
new file mode 100644
index 00000000..55dc76d3
--- /dev/null
+++ b/src/jit/zone.lua
@@ -0,0 +1,45 @@
1----------------------------------------------------------------------------
2-- LuaJIT profiler zones.
3--
4-- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
5-- Released under the MIT license. See Copyright Notice in luajit.h
6----------------------------------------------------------------------------
7--
8-- This module implements a simple hierarchical zone model.
9--
10-- Example usage:
11--
12-- local zone = require("jit.zone")
13-- zone("AI")
14-- ...
15-- zone("A*")
16-- ...
17-- print(zone:get()) --> "A*"
18-- ...
19-- zone()
20-- ...
21-- print(zone:get()) --> "AI"
22-- ...
23-- zone()
24--
25----------------------------------------------------------------------------
26
27local remove = table.remove
28
29return setmetatable({
30 flush = function(t)
31 for i=#t,1,-1 do t[i] = nil end
32 end,
33 get = function(t)
34 return t[#t]
35 end
36}, {
37 __call = function(t, zone)
38 if zone then
39 t[#t+1] = zone
40 else
41 return (assert(remove(t), "empty zone stack"))
42 end
43 end
44})
45
diff --git a/src/lauxlib.h b/src/lauxlib.h
index fed1491b..a44f0272 100644
--- a/src/lauxlib.h
+++ b/src/lauxlib.h
@@ -15,9 +15,6 @@
15#include "lua.h" 15#include "lua.h"
16 16
17 17
18#define luaL_getn(L,i) ((int)lua_objlen(L, i))
19#define luaL_setn(L,i,j) ((void)0) /* no op! */
20
21/* extra error code for `luaL_load' */ 18/* extra error code for `luaL_load' */
22#define LUA_ERRFILE (LUA_ERRERR+1) 19#define LUA_ERRFILE (LUA_ERRERR+1)
23 20
@@ -58,6 +55,10 @@ LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...);
58LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def, 55LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def,
59 const char *const lst[]); 56 const char *const lst[]);
60 57
58/* pre-defined references */
59#define LUA_NOREF (-2)
60#define LUA_REFNIL (-1)
61
61LUALIB_API int (luaL_ref) (lua_State *L, int t); 62LUALIB_API int (luaL_ref) (lua_State *L, int t);
62LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref); 63LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref);
63 64
@@ -84,6 +85,11 @@ LUALIB_API int (luaL_loadbufferx) (lua_State *L, const char *buff, size_t sz,
84 const char *name, const char *mode); 85 const char *name, const char *mode);
85LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, 86LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg,
86 int level); 87 int level);
88LUALIB_API void (luaL_setfuncs) (lua_State *L, const luaL_Reg *l, int nup);
89LUALIB_API void (luaL_pushmodule) (lua_State *L, const char *modname,
90 int sizehint);
91LUALIB_API void *(luaL_testudata) (lua_State *L, int ud, const char *tname);
92LUALIB_API void (luaL_setmetatable) (lua_State *L, const char *tname);
87 93
88 94
89/* 95/*
@@ -113,6 +119,11 @@ LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg,
113 119
114#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n))) 120#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n)))
115 121
122/* From Lua 5.2. */
123#define luaL_newlibtable(L, l) \
124 lua_createtable(L, 0, sizeof(l)/sizeof((l)[0]) - 1)
125#define luaL_newlib(L, l) (luaL_newlibtable(L, l), luaL_setfuncs(L, l, 0))
126
116/* 127/*
117** {====================================================== 128** {======================================================
118** Generic Buffer manipulation 129** Generic Buffer manipulation
@@ -147,21 +158,4 @@ LUALIB_API void (luaL_pushresult) (luaL_Buffer *B);
147 158
148/* }====================================================== */ 159/* }====================================================== */
149 160
150
151/* compatibility with ref system */
152
153/* pre-defined references */
154#define LUA_NOREF (-2)
155#define LUA_REFNIL (-1)
156
157#define lua_ref(L,lock) ((lock) ? luaL_ref(L, LUA_REGISTRYINDEX) : \
158 (lua_pushstring(L, "unlocked references are obsolete"), lua_error(L), 0))
159
160#define lua_unref(L,ref) luaL_unref(L, LUA_REGISTRYINDEX, (ref))
161
162#define lua_getref(L,ref) lua_rawgeti(L, LUA_REGISTRYINDEX, (ref))
163
164
165#define luaL_reg luaL_Reg
166
167#endif 161#endif
diff --git a/src/lib_aux.c b/src/lib_aux.c
index 8a64f185..7e81ac30 100644
--- a/src/lib_aux.c
+++ b/src/lib_aux.c
@@ -21,6 +21,7 @@
21#include "lj_state.h" 21#include "lj_state.h"
22#include "lj_trace.h" 22#include "lj_trace.h"
23#include "lj_lib.h" 23#include "lj_lib.h"
24#include "lj_vmevent.h"
24 25
25#if LJ_TARGET_POSIX 26#if LJ_TARGET_POSIX
26#include <sys/wait.h> 27#include <sys/wait.h>
@@ -107,38 +108,36 @@ LUALIB_API const char *luaL_findtable(lua_State *L, int idx,
107static int libsize(const luaL_Reg *l) 108static int libsize(const luaL_Reg *l)
108{ 109{
109 int size = 0; 110 int size = 0;
110 for (; l->name; l++) size++; 111 for (; l && l->name; l++) size++;
111 return size; 112 return size;
112} 113}
113 114
115LUALIB_API void luaL_pushmodule(lua_State *L, const char *modname, int sizehint)
116{
117 luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 16);
118 lua_getfield(L, -1, modname);
119 if (!lua_istable(L, -1)) {
120 lua_pop(L, 1);
121 if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, sizehint) != NULL)
122 lj_err_callerv(L, LJ_ERR_BADMODN, modname);
123 lua_pushvalue(L, -1);
124 lua_setfield(L, -3, modname); /* _LOADED[modname] = new table. */
125 }
126 lua_remove(L, -2); /* Remove _LOADED table. */
127}
128
114LUALIB_API void luaL_openlib(lua_State *L, const char *libname, 129LUALIB_API void luaL_openlib(lua_State *L, const char *libname,
115 const luaL_Reg *l, int nup) 130 const luaL_Reg *l, int nup)
116{ 131{
117 lj_lib_checkfpu(L); 132 lj_lib_checkfpu(L);
118 if (libname) { 133 if (libname) {
119 int size = libsize(l); 134 luaL_pushmodule(L, libname, libsize(l));
120 /* check whether lib already exists */ 135 lua_insert(L, -(nup + 1)); /* Move module table below upvalues. */
121 luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 16);
122 lua_getfield(L, -1, libname); /* get _LOADED[libname] */
123 if (!lua_istable(L, -1)) { /* not found? */
124 lua_pop(L, 1); /* remove previous result */
125 /* try global variable (and create one if it does not exist) */
126 if (luaL_findtable(L, LUA_GLOBALSINDEX, libname, size) != NULL)
127 lj_err_callerv(L, LJ_ERR_BADMODN, libname);
128 lua_pushvalue(L, -1);
129 lua_setfield(L, -3, libname); /* _LOADED[libname] = new table */
130 }
131 lua_remove(L, -2); /* remove _LOADED table */
132 lua_insert(L, -(nup+1)); /* move library table to below upvalues */
133 } 136 }
134 for (; l->name; l++) { 137 if (l)
135 int i; 138 luaL_setfuncs(L, l, nup);
136 for (i = 0; i < nup; i++) /* copy upvalues to the top */ 139 else
137 lua_pushvalue(L, -nup); 140 lua_pop(L, nup); /* Remove upvalues. */
138 lua_pushcclosure(L, l->func, nup);
139 lua_setfield(L, -(nup+2), l->name);
140 }
141 lua_pop(L, nup); /* remove upvalues */
142} 141}
143 142
144LUALIB_API void luaL_register(lua_State *L, const char *libname, 143LUALIB_API void luaL_register(lua_State *L, const char *libname,
@@ -147,6 +146,19 @@ LUALIB_API void luaL_register(lua_State *L, const char *libname,
147 luaL_openlib(L, libname, l, 0); 146 luaL_openlib(L, libname, l, 0);
148} 147}
149 148
149LUALIB_API void luaL_setfuncs(lua_State *L, const luaL_Reg *l, int nup)
150{
151 luaL_checkstack(L, nup, "too many upvalues");
152 for (; l->name; l++) {
153 int i;
154 for (i = 0; i < nup; i++) /* Copy upvalues to the top. */
155 lua_pushvalue(L, -nup);
156 lua_pushcclosure(L, l->func, nup);
157 lua_setfield(L, -(nup + 2), l->name);
158 }
159 lua_pop(L, nup); /* Remove upvalues. */
160}
161
150LUALIB_API const char *luaL_gsub(lua_State *L, const char *s, 162LUALIB_API const char *luaL_gsub(lua_State *L, const char *s,
151 const char *p, const char *r) 163 const char *p, const char *r)
152{ 164{
@@ -207,8 +219,15 @@ LUALIB_API char *luaL_prepbuffer(luaL_Buffer *B)
207 219
208LUALIB_API void luaL_addlstring(luaL_Buffer *B, const char *s, size_t l) 220LUALIB_API void luaL_addlstring(luaL_Buffer *B, const char *s, size_t l)
209{ 221{
210 while (l--) 222 if (l <= bufffree(B)) {
211 luaL_addchar(B, *s++); 223 memcpy(B->p, s, l);
224 B->p += l;
225 } else {
226 emptybuffer(B);
227 lua_pushlstring(B->L, s, l);
228 B->lvl++;
229 adjuststack(B);
230 }
212} 231}
213 232
214LUALIB_API void luaL_addstring(luaL_Buffer *B, const char *s) 233LUALIB_API void luaL_addstring(luaL_Buffer *B, const char *s)
@@ -300,9 +319,21 @@ static int panic(lua_State *L)
300 return 0; 319 return 0;
301} 320}
302 321
322#ifndef LUAJIT_DISABLE_VMEVENT
323static int error_finalizer(lua_State *L)
324{
325 const char *s = lua_tostring(L, -1);
326 fputs("ERROR in finalizer: ", stderr);
327 fputs(s ? s : "?", stderr);
328 fputc('\n', stderr);
329 fflush(stderr);
330 return 0;
331}
332#endif
333
303#ifdef LUAJIT_USE_SYSMALLOC 334#ifdef LUAJIT_USE_SYSMALLOC
304 335
305#if LJ_64 && !defined(LUAJIT_USE_VALGRIND) 336#if LJ_64 && !LJ_GC64 && !defined(LUAJIT_USE_VALGRIND)
306#error "Must use builtin allocator for 64 bit target" 337#error "Must use builtin allocator for 64 bit target"
307#endif 338#endif
308 339
@@ -321,29 +352,43 @@ static void *mem_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
321LUALIB_API lua_State *luaL_newstate(void) 352LUALIB_API lua_State *luaL_newstate(void)
322{ 353{
323 lua_State *L = lua_newstate(mem_alloc, NULL); 354 lua_State *L = lua_newstate(mem_alloc, NULL);
324 if (L) G(L)->panic = panic; 355 if (L) {
356 G(L)->panic = panic;
357#ifndef LUAJIT_DISABLE_VMEVENT
358 luaL_findtable(L, LUA_REGISTRYINDEX, LJ_VMEVENTS_REGKEY, LJ_VMEVENTS_HSIZE);
359 lua_pushcfunction(L, error_finalizer);
360 lua_rawseti(L, -2, VMEVENT_HASH(LJ_VMEVENT_ERRFIN));
361 G(L)->vmevmask = VMEVENT_MASK(LJ_VMEVENT_ERRFIN);
362 L->top--;
363#endif
364 }
325 return L; 365 return L;
326} 366}
327 367
328#else 368#else
329 369
330#include "lj_alloc.h"
331
332LUALIB_API lua_State *luaL_newstate(void) 370LUALIB_API lua_State *luaL_newstate(void)
333{ 371{
334 lua_State *L; 372 lua_State *L;
335 void *ud = lj_alloc_create(); 373#if LJ_64 && !LJ_GC64
336 if (ud == NULL) return NULL; 374 L = lj_state_newstate(LJ_ALLOCF_INTERNAL, NULL);
337#if LJ_64
338 L = lj_state_newstate(lj_alloc_f, ud);
339#else 375#else
340 L = lua_newstate(lj_alloc_f, ud); 376 L = lua_newstate(LJ_ALLOCF_INTERNAL, NULL);
341#endif 377#endif
342 if (L) G(L)->panic = panic; 378 if (L) {
379 G(L)->panic = panic;
380#ifndef LUAJIT_DISABLE_VMEVENT
381 luaL_findtable(L, LUA_REGISTRYINDEX, LJ_VMEVENTS_REGKEY, LJ_VMEVENTS_HSIZE);
382 lua_pushcfunction(L, error_finalizer);
383 lua_rawseti(L, -2, VMEVENT_HASH(LJ_VMEVENT_ERRFIN));
384 G(L)->vmevmask = VMEVENT_MASK(LJ_VMEVENT_ERRFIN);
385 L->top--;
386#endif
387 }
343 return L; 388 return L;
344} 389}
345 390
346#if LJ_64 391#if LJ_64 && !LJ_GC64
347LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) 392LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
348{ 393{
349 UNUSED(f); UNUSED(ud); 394 UNUSED(f); UNUSED(ud);
diff --git a/src/lib_base.c b/src/lib_base.c
index a687411f..dd54b9f9 100644
--- a/src/lib_base.c
+++ b/src/lib_base.c
@@ -19,10 +19,12 @@
19#include "lj_gc.h" 19#include "lj_gc.h"
20#include "lj_err.h" 20#include "lj_err.h"
21#include "lj_debug.h" 21#include "lj_debug.h"
22#include "lj_buf.h"
22#include "lj_str.h" 23#include "lj_str.h"
23#include "lj_tab.h" 24#include "lj_tab.h"
24#include "lj_meta.h" 25#include "lj_meta.h"
25#include "lj_state.h" 26#include "lj_state.h"
27#include "lj_frame.h"
26#if LJ_HASFFI 28#if LJ_HASFFI
27#include "lj_ctype.h" 29#include "lj_ctype.h"
28#include "lj_cconv.h" 30#include "lj_cconv.h"
@@ -32,6 +34,7 @@
32#include "lj_dispatch.h" 34#include "lj_dispatch.h"
33#include "lj_char.h" 35#include "lj_char.h"
34#include "lj_strscan.h" 36#include "lj_strscan.h"
37#include "lj_strfmt.h"
35#include "lj_lib.h" 38#include "lj_lib.h"
36 39
37/* -- Base library: checks ------------------------------------------------ */ 40/* -- Base library: checks ------------------------------------------------ */
@@ -40,13 +43,13 @@
40 43
41LJLIB_ASM(assert) LJLIB_REC(.) 44LJLIB_ASM(assert) LJLIB_REC(.)
42{ 45{
43 GCstr *s;
44 lj_lib_checkany(L, 1); 46 lj_lib_checkany(L, 1);
45 s = lj_lib_optstr(L, 2); 47 if (L->top == L->base+1)
46 if (s)
47 lj_err_callermsg(L, strdata(s));
48 else
49 lj_err_caller(L, LJ_ERR_ASSERT); 48 lj_err_caller(L, LJ_ERR_ASSERT);
49 else if (tvisstr(L->base+1) || tvisnumber(L->base+1))
50 lj_err_callermsg(L, strdata(lj_lib_checkstr(L, 2)));
51 else
52 lj_err_run(L);
50 return FFH_UNREACHABLE; 53 return FFH_UNREACHABLE;
51} 54}
52 55
@@ -73,9 +76,10 @@ LJLIB_ASM_(type) LJLIB_REC(.)
73/* This solves a circular dependency problem -- change FF_next_N as needed. */ 76/* This solves a circular dependency problem -- change FF_next_N as needed. */
74LJ_STATIC_ASSERT((int)FF_next == FF_next_N); 77LJ_STATIC_ASSERT((int)FF_next == FF_next_N);
75 78
76LJLIB_ASM(next) 79LJLIB_ASM(next) LJLIB_REC(.)
77{ 80{
78 lj_lib_checktab(L, 1); 81 lj_lib_checktab(L, 1);
82 lj_err_msg(L, LJ_ERR_NEXTIDX);
79 return FFH_UNREACHABLE; 83 return FFH_UNREACHABLE;
80} 84}
81 85
@@ -86,10 +90,11 @@ static int ffh_pairs(lua_State *L, MMS mm)
86 cTValue *mo = lj_meta_lookup(L, o, mm); 90 cTValue *mo = lj_meta_lookup(L, o, mm);
87 if ((LJ_52 || tviscdata(o)) && !tvisnil(mo)) { 91 if ((LJ_52 || tviscdata(o)) && !tvisnil(mo)) {
88 L->top = o+1; /* Only keep one argument. */ 92 L->top = o+1; /* Only keep one argument. */
89 copyTV(L, L->base-1, mo); /* Replace callable. */ 93 copyTV(L, L->base-1-LJ_FR2, mo); /* Replace callable. */
90 return FFH_TAILCALL; 94 return FFH_TAILCALL;
91 } else { 95 } else {
92 if (!tvistab(o)) lj_err_argt(L, 1, LUA_TTABLE); 96 if (!tvistab(o)) lj_err_argt(L, 1, LUA_TTABLE);
97 if (LJ_FR2) { copyTV(L, o-1, o); o--; }
93 setfuncV(L, o-1, funcV(lj_lib_upvalue(L, 1))); 98 setfuncV(L, o-1, funcV(lj_lib_upvalue(L, 1)));
94 if (mm == MM_pairs) setnilV(o+1); else setintV(o+1, 0); 99 if (mm == MM_pairs) setnilV(o+1); else setintV(o+1, 0);
95 return FFH_RES(3); 100 return FFH_RES(3);
@@ -100,7 +105,7 @@ static int ffh_pairs(lua_State *L, MMS mm)
100#endif 105#endif
101 106
102LJLIB_PUSH(lastcl) 107LJLIB_PUSH(lastcl)
103LJLIB_ASM(pairs) 108LJLIB_ASM(pairs) LJLIB_REC(xpairs 0)
104{ 109{
105 return ffh_pairs(L, MM_pairs); 110 return ffh_pairs(L, MM_pairs);
106} 111}
@@ -113,7 +118,7 @@ LJLIB_NOREGUV LJLIB_ASM(ipairs_aux) LJLIB_REC(.)
113} 118}
114 119
115LJLIB_PUSH(lastcl) 120LJLIB_PUSH(lastcl)
116LJLIB_ASM(ipairs) LJLIB_REC(.) 121LJLIB_ASM(ipairs) LJLIB_REC(xpairs 1)
117{ 122{
118 return ffh_pairs(L, MM_ipairs); 123 return ffh_pairs(L, MM_ipairs);
119} 124}
@@ -131,11 +136,11 @@ LJLIB_ASM(setmetatable) LJLIB_REC(.)
131 lj_err_caller(L, LJ_ERR_PROTMT); 136 lj_err_caller(L, LJ_ERR_PROTMT);
132 setgcref(t->metatable, obj2gco(mt)); 137 setgcref(t->metatable, obj2gco(mt));
133 if (mt) { lj_gc_objbarriert(L, t, mt); } 138 if (mt) { lj_gc_objbarriert(L, t, mt); }
134 settabV(L, L->base-1, t); 139 settabV(L, L->base-1-LJ_FR2, t);
135 return FFH_RES(1); 140 return FFH_RES(1);
136} 141}
137 142
138LJLIB_CF(getfenv) 143LJLIB_CF(getfenv) LJLIB_REC(.)
139{ 144{
140 GCfunc *fn; 145 GCfunc *fn;
141 cTValue *o = L->base; 146 cTValue *o = L->base;
@@ -144,6 +149,7 @@ LJLIB_CF(getfenv)
144 o = lj_debug_frame(L, level, &level); 149 o = lj_debug_frame(L, level, &level);
145 if (o == NULL) 150 if (o == NULL)
146 lj_err_arg(L, 1, LJ_ERR_INVLVL); 151 lj_err_arg(L, 1, LJ_ERR_INVLVL);
152 if (LJ_FR2) o--;
147 } 153 }
148 fn = &gcval(o)->fn; 154 fn = &gcval(o)->fn;
149 settabV(L, L->top++, isluafunc(fn) ? tabref(fn->l.env) : tabref(L->env)); 155 settabV(L, L->top++, isluafunc(fn) ? tabref(fn->l.env) : tabref(L->env));
@@ -165,6 +171,7 @@ LJLIB_CF(setfenv)
165 o = lj_debug_frame(L, level, &level); 171 o = lj_debug_frame(L, level, &level);
166 if (o == NULL) 172 if (o == NULL)
167 lj_err_arg(L, 1, LJ_ERR_INVLVL); 173 lj_err_arg(L, 1, LJ_ERR_INVLVL);
174 if (LJ_FR2) o--;
168 } 175 }
169 fn = &gcval(o)->fn; 176 fn = &gcval(o)->fn;
170 if (!isluafunc(fn)) 177 if (!isluafunc(fn))
@@ -259,7 +266,7 @@ LJLIB_ASM(tonumber) LJLIB_REC(.)
259 if (base == 10) { 266 if (base == 10) {
260 TValue *o = lj_lib_checkany(L, 1); 267 TValue *o = lj_lib_checkany(L, 1);
261 if (lj_strscan_numberobj(o)) { 268 if (lj_strscan_numberobj(o)) {
262 copyTV(L, L->base-1, o); 269 copyTV(L, L->base-1-LJ_FR2, o);
263 return FFH_RES(1); 270 return FFH_RES(1);
264 } 271 }
265#if LJ_HASFFI 272#if LJ_HASFFI
@@ -272,11 +279,11 @@ LJLIB_ASM(tonumber) LJLIB_REC(.)
272 ct->size <= 4 && !(ct->size == 4 && (ct->info & CTF_UNSIGNED))) { 279 ct->size <= 4 && !(ct->size == 4 && (ct->info & CTF_UNSIGNED))) {
273 int32_t i; 280 int32_t i;
274 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_INT32), (uint8_t *)&i, o, 0); 281 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_INT32), (uint8_t *)&i, o, 0);
275 setintV(L->base-1, i); 282 setintV(L->base-1-LJ_FR2, i);
276 return FFH_RES(1); 283 return FFH_RES(1);
277 } 284 }
278 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_DOUBLE), 285 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_DOUBLE),
279 (uint8_t *)&(L->base-1)->n, o, 0); 286 (uint8_t *)&(L->base-1-LJ_FR2)->n, o, 0);
280 return FFH_RES(1); 287 return FFH_RES(1);
281 } 288 }
282 } 289 }
@@ -284,53 +291,46 @@ LJLIB_ASM(tonumber) LJLIB_REC(.)
284 } else { 291 } else {
285 const char *p = strdata(lj_lib_checkstr(L, 1)); 292 const char *p = strdata(lj_lib_checkstr(L, 1));
286 char *ep; 293 char *ep;
294 unsigned int neg = 0;
287 unsigned long ul; 295 unsigned long ul;
288 if (base < 2 || base > 36) 296 if (base < 2 || base > 36)
289 lj_err_arg(L, 2, LJ_ERR_BASERNG); 297 lj_err_arg(L, 2, LJ_ERR_BASERNG);
290 ul = strtoul(p, &ep, base); 298 while (lj_char_isspace((unsigned char)(*p))) p++;
291 if (p != ep) { 299 if (*p == '-') { p++; neg = 1; } else if (*p == '+') { p++; }
292 while (lj_char_isspace((unsigned char)(*ep))) ep++; 300 if (lj_char_isalnum((unsigned char)(*p))) {
293 if (*ep == '\0') { 301 ul = strtoul(p, &ep, base);
294 if (LJ_DUALNUM && LJ_LIKELY(ul < 0x80000000u)) 302 if (p != ep) {
295 setintV(L->base-1, (int32_t)ul); 303 while (lj_char_isspace((unsigned char)(*ep))) ep++;
296 else 304 if (*ep == '\0') {
297 setnumV(L->base-1, (lua_Number)ul); 305 if (LJ_DUALNUM && LJ_LIKELY(ul < 0x80000000u+neg)) {
298 return FFH_RES(1); 306 if (neg) ul = ~ul+1u;
307 setintV(L->base-1-LJ_FR2, (int32_t)ul);
308 } else {
309 lua_Number n = (lua_Number)ul;
310 if (neg) n = -n;
311 setnumV(L->base-1-LJ_FR2, n);
312 }
313 return FFH_RES(1);
314 }
299 } 315 }
300 } 316 }
301 } 317 }
302 setnilV(L->base-1); 318 setnilV(L->base-1-LJ_FR2);
303 return FFH_RES(1); 319 return FFH_RES(1);
304} 320}
305 321
306LJLIB_PUSH("nil")
307LJLIB_PUSH("false")
308LJLIB_PUSH("true")
309LJLIB_ASM(tostring) LJLIB_REC(.) 322LJLIB_ASM(tostring) LJLIB_REC(.)
310{ 323{
311 TValue *o = lj_lib_checkany(L, 1); 324 TValue *o = lj_lib_checkany(L, 1);
312 cTValue *mo; 325 cTValue *mo;
313 L->top = o+1; /* Only keep one argument. */ 326 L->top = o+1; /* Only keep one argument. */
314 if (!tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) { 327 if (!tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
315 copyTV(L, L->base-1, mo); /* Replace callable. */ 328 copyTV(L, L->base-1-LJ_FR2, mo); /* Replace callable. */
316 return FFH_TAILCALL; 329 return FFH_TAILCALL;
317 } else {
318 GCstr *s;
319 if (tvisnumber(o)) {
320 s = lj_str_fromnumber(L, o);
321 } else if (tvispri(o)) {
322 s = strV(lj_lib_upvalue(L, -(int32_t)itype(o)));
323 } else {
324 if (tvisfunc(o) && isffunc(funcV(o)))
325 lua_pushfstring(L, "function: builtin#%d", funcV(o)->c.ffid);
326 else
327 lua_pushfstring(L, "%s: %p", lj_typename(o), lua_topointer(L, 1));
328 /* Note: lua_pushfstring calls the GC which may invalidate o. */
329 s = strV(L->top-1);
330 }
331 setstrV(L, L->base-1, s);
332 return FFH_RES(1);
333 } 330 }
331 lj_gc_check(L);
332 setstrV(L, L->base-1-LJ_FR2, lj_strfmt_obj(L, L->base));
333 return FFH_RES(1);
334} 334}
335 335
336/* -- Base library: throw and catch errors -------------------------------- */ 336/* -- Base library: throw and catch errors -------------------------------- */
@@ -359,7 +359,7 @@ LJLIB_ASM_(xpcall) LJLIB_REC(.)
359 359
360static int load_aux(lua_State *L, int status, int envarg) 360static int load_aux(lua_State *L, int status, int envarg)
361{ 361{
362 if (status == 0) { 362 if (status == LUA_OK) {
363 if (tvistab(L->base+envarg-1)) { 363 if (tvistab(L->base+envarg-1)) {
364 GCfunc *fn = funcV(L->top-1); 364 GCfunc *fn = funcV(L->top-1);
365 GCtab *t = tabV(L->base+envarg-1); 365 GCtab *t = tabV(L->base+envarg-1);
@@ -408,10 +408,22 @@ LJLIB_CF(load)
408 GCstr *name = lj_lib_optstr(L, 2); 408 GCstr *name = lj_lib_optstr(L, 2);
409 GCstr *mode = lj_lib_optstr(L, 3); 409 GCstr *mode = lj_lib_optstr(L, 3);
410 int status; 410 int status;
411 if (L->base < L->top && (tvisstr(L->base) || tvisnumber(L->base))) { 411 if (L->base < L->top &&
412 GCstr *s = lj_lib_checkstr(L, 1); 412 (tvisstr(L->base) || tvisnumber(L->base) || tvisbuf(L->base))) {
413 const char *s;
414 MSize len;
415 if (tvisbuf(L->base)) {
416 SBufExt *sbx = bufV(L->base);
417 s = sbx->r;
418 len = sbufxlen(sbx);
419 if (!name) name = &G(L)->strempty; /* Buffers are not NUL-terminated. */
420 } else {
421 GCstr *str = lj_lib_checkstr(L, 1);
422 s = strdata(str);
423 len = str->len;
424 }
413 lua_settop(L, 4); /* Ensure env arg exists. */ 425 lua_settop(L, 4); /* Ensure env arg exists. */
414 status = luaL_loadbufferx(L, strdata(s), s->len, strdata(name ? name : s), 426 status = luaL_loadbufferx(L, s, len, name ? strdata(name) : s,
415 mode ? strdata(mode) : NULL); 427 mode ? strdata(mode) : NULL);
416 } else { 428 } else {
417 lj_lib_checkfunc(L, 1); 429 lj_lib_checkfunc(L, 1);
@@ -432,7 +444,7 @@ LJLIB_CF(dofile)
432 GCstr *fname = lj_lib_optstr(L, 1); 444 GCstr *fname = lj_lib_optstr(L, 1);
433 setnilV(L->top); 445 setnilV(L->top);
434 L->top = L->base+1; 446 L->top = L->base+1;
435 if (luaL_loadfile(L, fname ? strdata(fname) : NULL) != 0) 447 if (luaL_loadfile(L, fname ? strdata(fname) : NULL) != LUA_OK)
436 lua_error(L); 448 lua_error(L);
437 lua_call(L, 0, LUA_MULTRET); 449 lua_call(L, 0, LUA_MULTRET);
438 return (int)(L->top - L->base) - 1; 450 return (int)(L->top - L->base) - 1;
@@ -442,20 +454,20 @@ LJLIB_CF(dofile)
442 454
443LJLIB_CF(gcinfo) 455LJLIB_CF(gcinfo)
444{ 456{
445 setintV(L->top++, (G(L)->gc.total >> 10)); 457 setintV(L->top++, (int32_t)(G(L)->gc.total >> 10));
446 return 1; 458 return 1;
447} 459}
448 460
449LJLIB_CF(collectgarbage) 461LJLIB_CF(collectgarbage)
450{ 462{
451 int opt = lj_lib_checkopt(L, 1, LUA_GCCOLLECT, /* ORDER LUA_GC* */ 463 int opt = lj_lib_checkopt(L, 1, LUA_GCCOLLECT, /* ORDER LUA_GC* */
452 "\4stop\7restart\7collect\5count\1\377\4step\10setpause\12setstepmul"); 464 "\4stop\7restart\7collect\5count\1\377\4step\10setpause\12setstepmul\1\377\11isrunning");
453 int32_t data = lj_lib_optint(L, 2, 0); 465 int32_t data = lj_lib_optint(L, 2, 0);
454 if (opt == LUA_GCCOUNT) { 466 if (opt == LUA_GCCOUNT) {
455 setnumV(L->top, (lua_Number)G(L)->gc.total/1024.0); 467 setnumV(L->top, (lua_Number)G(L)->gc.total/1024.0);
456 } else { 468 } else {
457 int res = lua_gc(L, opt, data); 469 int res = lua_gc(L, opt, data);
458 if (opt == LUA_GCSTEP) 470 if (opt == LUA_GCSTEP || opt == LUA_GCISRUNNING)
459 setboolV(L->top, res); 471 setboolV(L->top, res);
460 else 472 else
461 setintV(L->top, res); 473 setintV(L->top, res);
@@ -507,23 +519,14 @@ LJLIB_CF(print)
507 tv = L->top-1; 519 tv = L->top-1;
508 } 520 }
509 shortcut = (tvisfunc(tv) && funcV(tv)->c.ffid == FF_tostring) && 521 shortcut = (tvisfunc(tv) && funcV(tv)->c.ffid == FF_tostring) &&
510 !gcrefu(basemt_it(G(L), LJ_TNUMX)); 522 !gcrefu(basemt_it(G(L), LJ_TNUMX));
511 for (i = 0; i < nargs; i++) { 523 for (i = 0; i < nargs; i++) {
524 cTValue *o = &L->base[i];
512 const char *str; 525 const char *str;
513 size_t size; 526 size_t size;
514 cTValue *o = &L->base[i]; 527 MSize len;
515 if (shortcut && tvisstr(o)) { 528 if (shortcut && (str = lj_strfmt_wstrnum(L, o, &len)) != NULL) {
516 str = strVdata(o); 529 size = len;
517 size = strV(o)->len;
518 } else if (shortcut && tvisint(o)) {
519 char buf[LJ_STR_INTBUF];
520 char *p = lj_str_bufint(buf, intV(o));
521 size = (size_t)(buf+LJ_STR_INTBUF-p);
522 str = p;
523 } else if (shortcut && tvisnum(o)) {
524 char buf[LJ_STR_NUMBUF];
525 size = lj_str_bufnum(buf, o);
526 str = buf;
527 } else { 530 } else {
528 copyTV(L, L->top+1, o); 531 copyTV(L, L->top+1, o);
529 copyTV(L, L->top, L->top-1); 532 copyTV(L, L->top, L->top-1);
@@ -560,8 +563,8 @@ LJLIB_CF(coroutine_status)
560 co = threadV(L->base); 563 co = threadV(L->base);
561 if (co == L) s = "running"; 564 if (co == L) s = "running";
562 else if (co->status == LUA_YIELD) s = "suspended"; 565 else if (co->status == LUA_YIELD) s = "suspended";
563 else if (co->status != 0) s = "dead"; 566 else if (co->status != LUA_OK) s = "dead";
564 else if (co->base > tvref(co->stack)+1) s = "normal"; 567 else if (co->base > tvref(co->stack)+1+LJ_FR2) s = "normal";
565 else if (co->top == co->base) s = "dead"; 568 else if (co->top == co->base) s = "dead";
566 else s = "suspended"; 569 else s = "suspended";
567 lua_pushstring(L, s); 570 lua_pushstring(L, s);
@@ -581,6 +584,12 @@ LJLIB_CF(coroutine_running)
581#endif 584#endif
582} 585}
583 586
587LJLIB_CF(coroutine_isyieldable)
588{
589 setboolV(L->top++, cframe_canyield(L->cframe));
590 return 1;
591}
592
584LJLIB_CF(coroutine_create) 593LJLIB_CF(coroutine_create)
585{ 594{
586 lua_State *L1; 595 lua_State *L1;
@@ -600,11 +609,11 @@ LJLIB_ASM(coroutine_yield)
600static int ffh_resume(lua_State *L, lua_State *co, int wrap) 609static int ffh_resume(lua_State *L, lua_State *co, int wrap)
601{ 610{
602 if (co->cframe != NULL || co->status > LUA_YIELD || 611 if (co->cframe != NULL || co->status > LUA_YIELD ||
603 (co->status == 0 && co->top == co->base)) { 612 (co->status == LUA_OK && co->top == co->base)) {
604 ErrMsg em = co->cframe ? LJ_ERR_CORUN : LJ_ERR_CODEAD; 613 ErrMsg em = co->cframe ? LJ_ERR_CORUN : LJ_ERR_CODEAD;
605 if (wrap) lj_err_caller(L, em); 614 if (wrap) lj_err_caller(L, em);
606 setboolV(L->base-1, 0); 615 setboolV(L->base-1-LJ_FR2, 0);
607 setstrV(L, L->base, lj_err_str(L, em)); 616 setstrV(L, L->base-LJ_FR2, lj_err_str(L, em));
608 return FFH_RES(2); 617 return FFH_RES(2);
609 } 618 }
610 lj_state_growstack(co, (MSize)(L->top - L->base)); 619 lj_state_growstack(co, (MSize)(L->top - L->base));
@@ -645,9 +654,10 @@ static void setpc_wrap_aux(lua_State *L, GCfunc *fn);
645 654
646LJLIB_CF(coroutine_wrap) 655LJLIB_CF(coroutine_wrap)
647{ 656{
657 GCfunc *fn;
648 lj_cf_coroutine_create(L); 658 lj_cf_coroutine_create(L);
649 lj_lib_pushcc(L, lj_ffh_coroutine_wrap_aux, FF_coroutine_wrap_aux, 1); 659 fn = lj_lib_pushcc(L, lj_ffh_coroutine_wrap_aux, FF_coroutine_wrap_aux, 1);
650 setpc_wrap_aux(L, funcV(L->top-1)); 660 setpc_wrap_aux(L, fn);
651 return 1; 661 return 1;
652} 662}
653 663
diff --git a/src/lib_bit.c b/src/lib_bit.c
index d7c79aeb..ce40e298 100644
--- a/src/lib_bit.c
+++ b/src/lib_bit.c
@@ -12,26 +12,99 @@
12 12
13#include "lj_obj.h" 13#include "lj_obj.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_str.h" 15#include "lj_buf.h"
16#include "lj_strscan.h"
17#include "lj_strfmt.h"
18#if LJ_HASFFI
19#include "lj_ctype.h"
20#include "lj_cdata.h"
21#include "lj_cconv.h"
22#include "lj_carith.h"
23#endif
24#include "lj_ff.h"
16#include "lj_lib.h" 25#include "lj_lib.h"
17 26
18/* ------------------------------------------------------------------------ */ 27/* ------------------------------------------------------------------------ */
19 28
20#define LJLIB_MODULE_bit 29#define LJLIB_MODULE_bit
21 30
22LJLIB_ASM(bit_tobit) LJLIB_REC(bit_unary IR_TOBIT) 31#if LJ_HASFFI
32static int bit_result64(lua_State *L, CTypeID id, uint64_t x)
23{ 33{
34 GCcdata *cd = lj_cdata_new_(L, id, 8);
35 *(uint64_t *)cdataptr(cd) = x;
36 setcdataV(L, L->base-1-LJ_FR2, cd);
37 return FFH_RES(1);
38}
39#else
40static int32_t bit_checkbit(lua_State *L, int narg)
41{
42 TValue *o = L->base + narg-1;
43 if (!(o < L->top && lj_strscan_numberobj(o)))
44 lj_err_argt(L, narg, LUA_TNUMBER);
45 if (LJ_LIKELY(tvisint(o))) {
46 return intV(o);
47 } else {
48 int32_t i = lj_num2bit(numV(o));
49 if (LJ_DUALNUM) setintV(o, i);
50 return i;
51 }
52}
53#endif
54
55LJLIB_ASM(bit_tobit) LJLIB_REC(bit_tobit)
56{
57#if LJ_HASFFI
58 CTypeID id = 0;
59 setintV(L->base-1-LJ_FR2, (int32_t)lj_carith_check64(L, 1, &id));
60 return FFH_RES(1);
61#else
62 lj_lib_checknumber(L, 1);
63 return FFH_RETRY;
64#endif
65}
66
67LJLIB_ASM(bit_bnot) LJLIB_REC(bit_unary IR_BNOT)
68{
69#if LJ_HASFFI
70 CTypeID id = 0;
71 uint64_t x = lj_carith_check64(L, 1, &id);
72 return id ? bit_result64(L, id, ~x) : FFH_RETRY;
73#else
24 lj_lib_checknumber(L, 1); 74 lj_lib_checknumber(L, 1);
25 return FFH_RETRY; 75 return FFH_RETRY;
76#endif
77}
78
79LJLIB_ASM(bit_bswap) LJLIB_REC(bit_unary IR_BSWAP)
80{
81#if LJ_HASFFI
82 CTypeID id = 0;
83 uint64_t x = lj_carith_check64(L, 1, &id);
84 return id ? bit_result64(L, id, lj_bswap64(x)) : FFH_RETRY;
85#else
86 lj_lib_checknumber(L, 1);
87 return FFH_RETRY;
88#endif
26} 89}
27LJLIB_ASM_(bit_bnot) LJLIB_REC(bit_unary IR_BNOT)
28LJLIB_ASM_(bit_bswap) LJLIB_REC(bit_unary IR_BSWAP)
29 90
30LJLIB_ASM(bit_lshift) LJLIB_REC(bit_shift IR_BSHL) 91LJLIB_ASM(bit_lshift) LJLIB_REC(bit_shift IR_BSHL)
31{ 92{
93#if LJ_HASFFI
94 CTypeID id = 0, id2 = 0;
95 uint64_t x = lj_carith_check64(L, 1, &id);
96 int32_t sh = (int32_t)lj_carith_check64(L, 2, &id2);
97 if (id) {
98 x = lj_carith_shift64(x, sh, curr_func(L)->c.ffid - (int)FF_bit_lshift);
99 return bit_result64(L, id, x);
100 }
101 if (id2) setintV(L->base+1, sh);
102 return FFH_RETRY;
103#else
32 lj_lib_checknumber(L, 1); 104 lj_lib_checknumber(L, 1);
33 lj_lib_checkbit(L, 2); 105 bit_checkbit(L, 2);
34 return FFH_RETRY; 106 return FFH_RETRY;
107#endif
35} 108}
36LJLIB_ASM_(bit_rshift) LJLIB_REC(bit_shift IR_BSHR) 109LJLIB_ASM_(bit_rshift) LJLIB_REC(bit_shift IR_BSHR)
37LJLIB_ASM_(bit_arshift) LJLIB_REC(bit_shift IR_BSAR) 110LJLIB_ASM_(bit_arshift) LJLIB_REC(bit_shift IR_BSAR)
@@ -40,25 +113,59 @@ LJLIB_ASM_(bit_ror) LJLIB_REC(bit_shift IR_BROR)
40 113
41LJLIB_ASM(bit_band) LJLIB_REC(bit_nary IR_BAND) 114LJLIB_ASM(bit_band) LJLIB_REC(bit_nary IR_BAND)
42{ 115{
116#if LJ_HASFFI
117 CTypeID id = 0;
118 TValue *o = L->base, *top = L->top;
119 int i = 0;
120 do { lj_carith_check64(L, ++i, &id); } while (++o < top);
121 if (id) {
122 CTState *cts = ctype_cts(L);
123 CType *ct = ctype_get(cts, id);
124 int op = curr_func(L)->c.ffid - (int)FF_bit_bor;
125 uint64_t x, y = op >= 0 ? 0 : ~(uint64_t)0;
126 o = L->base;
127 do {
128 lj_cconv_ct_tv(cts, ct, (uint8_t *)&x, o, 0);
129 if (op < 0) y &= x; else if (op == 0) y |= x; else y ^= x;
130 } while (++o < top);
131 return bit_result64(L, id, y);
132 }
133 return FFH_RETRY;
134#else
43 int i = 0; 135 int i = 0;
44 do { lj_lib_checknumber(L, ++i); } while (L->base+i < L->top); 136 do { lj_lib_checknumber(L, ++i); } while (L->base+i < L->top);
45 return FFH_RETRY; 137 return FFH_RETRY;
138#endif
46} 139}
47LJLIB_ASM_(bit_bor) LJLIB_REC(bit_nary IR_BOR) 140LJLIB_ASM_(bit_bor) LJLIB_REC(bit_nary IR_BOR)
48LJLIB_ASM_(bit_bxor) LJLIB_REC(bit_nary IR_BXOR) 141LJLIB_ASM_(bit_bxor) LJLIB_REC(bit_nary IR_BXOR)
49 142
50/* ------------------------------------------------------------------------ */ 143/* ------------------------------------------------------------------------ */
51 144
52LJLIB_CF(bit_tohex) 145LJLIB_CF(bit_tohex) LJLIB_REC(.)
53{ 146{
54 uint32_t b = (uint32_t)lj_lib_checkbit(L, 1); 147#if LJ_HASFFI
55 int32_t i, n = L->base+1 >= L->top ? 8 : lj_lib_checkbit(L, 2); 148 CTypeID id = 0, id2 = 0;
56 const char *hexdigits = "0123456789abcdef"; 149 uint64_t b = lj_carith_check64(L, 1, &id);
57 char buf[8]; 150 int32_t n = L->base+1>=L->top ? (id ? 16 : 8) :
58 if (n < 0) { n = (int32_t)(~(uint32_t)n+1u); hexdigits = "0123456789ABCDEF"; } 151 (int32_t)lj_carith_check64(L, 2, &id2);
59 if ((uint32_t)n > 8) n = 8; 152#else
60 for (i = n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; } 153 uint32_t b = (uint32_t)bit_checkbit(L, 1);
61 lua_pushlstring(L, buf, (size_t)n); 154 int32_t n = L->base+1>=L->top ? 8 : bit_checkbit(L, 2);
155#endif
156 SBuf *sb = lj_buf_tmp_(L);
157 SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
158 if (n < 0) { n = (int32_t)(~(uint32_t)n+1u); sf |= STRFMT_F_UPPER; }
159 if ((uint32_t)n > 254) n = 254;
160 sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
161#if LJ_HASFFI
162 if (n < 16) b &= ((uint64_t)1 << 4*n)-1;
163#else
164 if (n < 8) b &= (1u << 4*n)-1;
165#endif
166 sb = lj_strfmt_putfxint(sb, sf, b);
167 setstrV(L, L->top-1, lj_buf_str(L, sb));
168 lj_gc_check(L);
62 return 1; 169 return 1;
63} 170}
64 171
diff --git a/src/lib_buffer.c b/src/lib_buffer.c
new file mode 100644
index 00000000..e4ec9d9d
--- /dev/null
+++ b/src/lib_buffer.c
@@ -0,0 +1,360 @@
1/*
2** Buffer library.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lib_buffer_c
7#define LUA_LIB
8
9#include "lua.h"
10#include "lauxlib.h"
11#include "lualib.h"
12
13#include "lj_obj.h"
14
15#if LJ_HASBUFFER
16#include "lj_gc.h"
17#include "lj_err.h"
18#include "lj_buf.h"
19#include "lj_str.h"
20#include "lj_tab.h"
21#include "lj_udata.h"
22#include "lj_meta.h"
23#if LJ_HASFFI
24#include "lj_ctype.h"
25#include "lj_cdata.h"
26#include "lj_cconv.h"
27#endif
28#include "lj_strfmt.h"
29#include "lj_serialize.h"
30#include "lj_lib.h"
31
32/* -- Helper functions ---------------------------------------------------- */
33
34/* Check that the first argument is a string buffer. */
35static SBufExt *buffer_tobuf(lua_State *L)
36{
37 if (!(L->base < L->top && tvisbuf(L->base)))
38 lj_err_argtype(L, 1, "buffer");
39 return bufV(L->base);
40}
41
42/* Ditto, but for writers. */
43static LJ_AINLINE SBufExt *buffer_tobufw(lua_State *L)
44{
45 SBufExt *sbx = buffer_tobuf(L);
46 setsbufXL_(sbx, L);
47 return sbx;
48}
49
50#define buffer_toudata(sbx) ((GCudata *)(sbx)-1)
51
52/* -- Buffer methods ------------------------------------------------------ */
53
54#define LJLIB_MODULE_buffer_method
55
56LJLIB_CF(buffer_method_free)
57{
58 SBufExt *sbx = buffer_tobuf(L);
59 lj_bufx_free(L, sbx);
60 L->top = L->base+1; /* Chain buffer object. */
61 return 1;
62}
63
64LJLIB_CF(buffer_method_reset) LJLIB_REC(.)
65{
66 SBufExt *sbx = buffer_tobuf(L);
67 lj_bufx_reset(sbx);
68 L->top = L->base+1; /* Chain buffer object. */
69 return 1;
70}
71
72LJLIB_CF(buffer_method_skip) LJLIB_REC(.)
73{
74 SBufExt *sbx = buffer_tobuf(L);
75 MSize n = (MSize)lj_lib_checkintrange(L, 2, 0, LJ_MAX_BUF);
76 MSize len = sbufxlen(sbx);
77 if (n < len) {
78 sbx->r += n;
79 } else if (sbufiscow(sbx)) {
80 sbx->r = sbx->w;
81 } else {
82 sbx->r = sbx->w = sbx->b;
83 }
84 L->top = L->base+1; /* Chain buffer object. */
85 return 1;
86}
87
88LJLIB_CF(buffer_method_set) LJLIB_REC(.)
89{
90 SBufExt *sbx = buffer_tobuf(L);
91 GCobj *ref;
92 const char *p;
93 MSize len;
94#if LJ_HASFFI
95 if (tviscdata(L->base+1)) {
96 CTState *cts = ctype_cts(L);
97 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_P_CVOID), (uint8_t *)&p,
98 L->base+1, CCF_ARG(2));
99 len = (MSize)lj_lib_checkintrange(L, 3, 0, LJ_MAX_BUF);
100 } else
101#endif
102 {
103 GCstr *str = lj_lib_checkstrx(L, 2);
104 p = strdata(str);
105 len = str->len;
106 }
107 lj_bufx_free(L, sbx);
108 lj_bufx_set_cow(L, sbx, p, len);
109 ref = gcV(L->base+1);
110 setgcref(sbx->cowref, ref);
111 lj_gc_objbarrier(L, buffer_toudata(sbx), ref);
112 L->top = L->base+1; /* Chain buffer object. */
113 return 1;
114}
115
116LJLIB_CF(buffer_method_put) LJLIB_REC(.)
117{
118 SBufExt *sbx = buffer_tobufw(L);
119 ptrdiff_t arg, narg = L->top - L->base;
120 for (arg = 1; arg < narg; arg++) {
121 cTValue *o = &L->base[arg], *mo = NULL;
122 retry:
123 if (tvisstr(o)) {
124 lj_buf_putstr((SBuf *)sbx, strV(o));
125 } else if (tvisint(o)) {
126 lj_strfmt_putint((SBuf *)sbx, intV(o));
127 } else if (tvisnum(o)) {
128 lj_strfmt_putfnum((SBuf *)sbx, STRFMT_G14, numV(o));
129 } else if (tvisbuf(o)) {
130 SBufExt *sbx2 = bufV(o);
131 if (sbx2 == sbx) lj_err_arg(L, (int)(arg+1), LJ_ERR_BUFFER_SELF);
132 lj_buf_putmem((SBuf *)sbx, sbx2->r, sbufxlen(sbx2));
133 } else if (!mo && !tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
134 /* Call __tostring metamethod inline. */
135 copyTV(L, L->top++, mo);
136 copyTV(L, L->top++, o);
137 lua_call(L, 1, 1);
138 o = &L->base[arg]; /* The stack may have been reallocated. */
139 copyTV(L, &L->base[arg], L->top-1);
140 L->top = L->base + narg;
141 goto retry; /* Retry with the result. */
142 } else {
143 lj_err_argtype(L, (int)(arg+1), "string/number/__tostring");
144 }
145 /* Probably not useful to inline other __tostring MMs, e.g. FFI numbers. */
146 }
147 L->top = L->base+1; /* Chain buffer object. */
148 lj_gc_check(L);
149 return 1;
150}
151
152LJLIB_CF(buffer_method_putf) LJLIB_REC(.)
153{
154 SBufExt *sbx = buffer_tobufw(L);
155 lj_strfmt_putarg(L, (SBuf *)sbx, 2, 2);
156 L->top = L->base+1; /* Chain buffer object. */
157 lj_gc_check(L);
158 return 1;
159}
160
161LJLIB_CF(buffer_method_get) LJLIB_REC(.)
162{
163 SBufExt *sbx = buffer_tobuf(L);
164 ptrdiff_t arg, narg = L->top - L->base;
165 if (narg == 1) {
166 narg++;
167 setnilV(L->top++); /* get() is the same as get(nil). */
168 }
169 for (arg = 1; arg < narg; arg++) {
170 TValue *o = &L->base[arg];
171 MSize n = tvisnil(o) ? LJ_MAX_BUF :
172 (MSize) lj_lib_checkintrange(L, (int)(arg+1), 0, LJ_MAX_BUF);
173 MSize len = sbufxlen(sbx);
174 if (n > len) n = len;
175 setstrV(L, o, lj_str_new(L, sbx->r, n));
176 sbx->r += n;
177 }
178 if (sbx->r == sbx->w && !sbufiscow(sbx)) sbx->r = sbx->w = sbx->b;
179 lj_gc_check(L);
180 return (int)(narg-1);
181}
182
183#if LJ_HASFFI
184LJLIB_CF(buffer_method_putcdata) LJLIB_REC(.)
185{
186 SBufExt *sbx = buffer_tobufw(L);
187 const char *p;
188 MSize len;
189 if (tviscdata(L->base+1)) {
190 CTState *cts = ctype_cts(L);
191 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_P_CVOID), (uint8_t *)&p,
192 L->base+1, CCF_ARG(2));
193 } else {
194 lj_err_argtype(L, 2, "cdata");
195 }
196 len = (MSize)lj_lib_checkintrange(L, 3, 0, LJ_MAX_BUF);
197 lj_buf_putmem((SBuf *)sbx, p, len);
198 L->top = L->base+1; /* Chain buffer object. */
199 return 1;
200}
201
202LJLIB_CF(buffer_method_reserve) LJLIB_REC(.)
203{
204 SBufExt *sbx = buffer_tobufw(L);
205 MSize sz = (MSize)lj_lib_checkintrange(L, 2, 0, LJ_MAX_BUF);
206 GCcdata *cd;
207 lj_buf_more((SBuf *)sbx, sz);
208 ctype_loadffi(L);
209 cd = lj_cdata_new_(L, CTID_P_UINT8, CTSIZE_PTR);
210 *(void **)cdataptr(cd) = sbx->w;
211 setcdataV(L, L->top++, cd);
212 setintV(L->top++, sbufleft(sbx));
213 return 2;
214}
215
216LJLIB_CF(buffer_method_commit) LJLIB_REC(.)
217{
218 SBufExt *sbx = buffer_tobuf(L);
219 MSize len = (MSize)lj_lib_checkintrange(L, 2, 0, LJ_MAX_BUF);
220 if (len > sbufleft(sbx)) lj_err_arg(L, 2, LJ_ERR_NUMRNG);
221 sbx->w += len;
222 L->top = L->base+1; /* Chain buffer object. */
223 return 1;
224}
225
226LJLIB_CF(buffer_method_ref) LJLIB_REC(.)
227{
228 SBufExt *sbx = buffer_tobuf(L);
229 GCcdata *cd;
230 ctype_loadffi(L);
231 cd = lj_cdata_new_(L, CTID_P_UINT8, CTSIZE_PTR);
232 *(void **)cdataptr(cd) = sbx->r;
233 setcdataV(L, L->top++, cd);
234 setintV(L->top++, sbufxlen(sbx));
235 return 2;
236}
237#endif
238
239LJLIB_CF(buffer_method_encode) LJLIB_REC(.)
240{
241 SBufExt *sbx = buffer_tobufw(L);
242 cTValue *o = lj_lib_checkany(L, 2);
243 lj_serialize_put(sbx, o);
244 lj_gc_check(L);
245 L->top = L->base+1; /* Chain buffer object. */
246 return 1;
247}
248
249LJLIB_CF(buffer_method_decode) LJLIB_REC(.)
250{
251 SBufExt *sbx = buffer_tobufw(L);
252 setnilV(L->top++);
253 sbx->r = lj_serialize_get(sbx, L->top-1);
254 lj_gc_check(L);
255 return 1;
256}
257
258LJLIB_CF(buffer_method___gc)
259{
260 SBufExt *sbx = buffer_tobuf(L);
261 lj_bufx_free(L, sbx);
262 return 0;
263}
264
265LJLIB_CF(buffer_method___tostring) LJLIB_REC(.)
266{
267 SBufExt *sbx = buffer_tobuf(L);
268 setstrV(L, L->top-1, lj_str_new(L, sbx->r, sbufxlen(sbx)));
269 lj_gc_check(L);
270 return 1;
271}
272
273LJLIB_CF(buffer_method___len) LJLIB_REC(.)
274{
275 SBufExt *sbx = buffer_tobuf(L);
276 setintV(L->top-1, (int32_t)sbufxlen(sbx));
277 return 1;
278}
279
280LJLIB_PUSH("buffer") LJLIB_SET(__metatable)
281LJLIB_PUSH(top-1) LJLIB_SET(__index)
282
283/* -- Buffer library functions -------------------------------------------- */
284
285#define LJLIB_MODULE_buffer
286
287LJLIB_PUSH(top-2) LJLIB_SET(!) /* Set environment. */
288
289LJLIB_CF(buffer_new)
290{
291 MSize sz = 0;
292 int targ = 1;
293 GCtab *env, *dict_str = NULL, *dict_mt = NULL;
294 GCudata *ud;
295 SBufExt *sbx;
296 if (L->base < L->top && !tvistab(L->base)) {
297 targ = 2;
298 if (!tvisnil(L->base))
299 sz = (MSize)lj_lib_checkintrange(L, 1, 0, LJ_MAX_BUF);
300 }
301 if (L->base+targ-1 < L->top) {
302 GCtab *options = lj_lib_checktab(L, targ);
303 cTValue *opt_dict, *opt_mt;
304 opt_dict = lj_tab_getstr(options, lj_str_newlit(L, "dict"));
305 if (opt_dict && tvistab(opt_dict)) {
306 dict_str = tabV(opt_dict);
307 lj_serialize_dict_prep_str(L, dict_str);
308 }
309 opt_mt = lj_tab_getstr(options, lj_str_newlit(L, "metatable"));
310 if (opt_mt && tvistab(opt_mt)) {
311 dict_mt = tabV(opt_mt);
312 lj_serialize_dict_prep_mt(L, dict_mt);
313 }
314 }
315 env = tabref(curr_func(L)->c.env);
316 ud = lj_udata_new(L, sizeof(SBufExt), env);
317 ud->udtype = UDTYPE_BUFFER;
318 /* NOBARRIER: The GCudata is new (marked white). */
319 setgcref(ud->metatable, obj2gco(env));
320 setudataV(L, L->top++, ud);
321 sbx = (SBufExt *)uddata(ud);
322 lj_bufx_init(L, sbx);
323 setgcref(sbx->dict_str, obj2gco(dict_str));
324 setgcref(sbx->dict_mt, obj2gco(dict_mt));
325 if (sz > 0) lj_buf_need2((SBuf *)sbx, sz);
326 lj_gc_check(L);
327 return 1;
328}
329
330LJLIB_CF(buffer_encode) LJLIB_REC(.)
331{
332 cTValue *o = lj_lib_checkany(L, 1);
333 setstrV(L, L->top++, lj_serialize_encode(L, o));
334 lj_gc_check(L);
335 return 1;
336}
337
338LJLIB_CF(buffer_decode) LJLIB_REC(.)
339{
340 GCstr *str = lj_lib_checkstrx(L, 1);
341 setnilV(L->top++);
342 lj_serialize_decode(L, L->top-1, str);
343 lj_gc_check(L);
344 return 1;
345}
346
347/* ------------------------------------------------------------------------ */
348
349#include "lj_libdef.h"
350
351int luaopen_string_buffer(lua_State *L)
352{
353 LJ_LIB_REG(L, NULL, buffer_method);
354 lua_getfield(L, -1, "__tostring");
355 lua_setfield(L, -2, "tostring");
356 LJ_LIB_REG(L, NULL, buffer);
357 return 1;
358}
359
360#endif
diff --git a/src/lib_debug.c b/src/lib_debug.c
index c5f3040f..2ad51a74 100644
--- a/src/lib_debug.c
+++ b/src/lib_debug.c
@@ -29,7 +29,7 @@ LJLIB_CF(debug_getregistry)
29 return 1; 29 return 1;
30} 30}
31 31
32LJLIB_CF(debug_getmetatable) 32LJLIB_CF(debug_getmetatable) LJLIB_REC(.)
33{ 33{
34 lj_lib_checkany(L, 1); 34 lj_lib_checkany(L, 1);
35 if (!lua_getmetatable(L, 1)) { 35 if (!lua_getmetatable(L, 1)) {
@@ -231,8 +231,8 @@ LJLIB_CF(debug_upvalueid)
231 int32_t n = lj_lib_checkint(L, 2) - 1; 231 int32_t n = lj_lib_checkint(L, 2) - 1;
232 if ((uint32_t)n >= fn->l.nupvalues) 232 if ((uint32_t)n >= fn->l.nupvalues)
233 lj_err_arg(L, 2, LJ_ERR_IDXRNG); 233 lj_err_arg(L, 2, LJ_ERR_IDXRNG);
234 setlightudV(L->top-1, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) : 234 lua_pushlightuserdata(L, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
235 (void *)&fn->c.upvalue[n]); 235 (void *)&fn->c.upvalue[n]);
236 return 1; 236 return 1;
237} 237}
238 238
@@ -283,13 +283,13 @@ LJLIB_CF(debug_setuservalue)
283 283
284/* ------------------------------------------------------------------------ */ 284/* ------------------------------------------------------------------------ */
285 285
286static const char KEY_HOOK = 'h'; 286#define KEY_HOOK (U64x(80000000,00000000)|'h')
287 287
288static void hookf(lua_State *L, lua_Debug *ar) 288static void hookf(lua_State *L, lua_Debug *ar)
289{ 289{
290 static const char *const hooknames[] = 290 static const char *const hooknames[] =
291 {"call", "return", "line", "count", "tail return"}; 291 {"call", "return", "line", "count", "tail return"};
292 lua_pushlightuserdata(L, (void *)&KEY_HOOK); 292 (L->top++)->u64 = KEY_HOOK;
293 lua_rawget(L, LUA_REGISTRYINDEX); 293 lua_rawget(L, LUA_REGISTRYINDEX);
294 if (lua_isfunction(L, -1)) { 294 if (lua_isfunction(L, -1)) {
295 lua_pushstring(L, hooknames[(int)ar->event]); 295 lua_pushstring(L, hooknames[(int)ar->event]);
@@ -334,7 +334,7 @@ LJLIB_CF(debug_sethook)
334 count = luaL_optint(L, arg+3, 0); 334 count = luaL_optint(L, arg+3, 0);
335 func = hookf; mask = makemask(smask, count); 335 func = hookf; mask = makemask(smask, count);
336 } 336 }
337 lua_pushlightuserdata(L, (void *)&KEY_HOOK); 337 (L->top++)->u64 = KEY_HOOK;
338 lua_pushvalue(L, arg+1); 338 lua_pushvalue(L, arg+1);
339 lua_rawset(L, LUA_REGISTRYINDEX); 339 lua_rawset(L, LUA_REGISTRYINDEX);
340 lua_sethook(L, func, mask, count); 340 lua_sethook(L, func, mask, count);
@@ -349,7 +349,7 @@ LJLIB_CF(debug_gethook)
349 if (hook != NULL && hook != hookf) { /* external hook? */ 349 if (hook != NULL && hook != hookf) { /* external hook? */
350 lua_pushliteral(L, "external hook"); 350 lua_pushliteral(L, "external hook");
351 } else { 351 } else {
352 lua_pushlightuserdata(L, (void *)&KEY_HOOK); 352 (L->top++)->u64 = KEY_HOOK;
353 lua_rawget(L, LUA_REGISTRYINDEX); /* get hook */ 353 lua_rawget(L, LUA_REGISTRYINDEX); /* get hook */
354 } 354 }
355 lua_pushstring(L, unmakemask(mask, buff)); 355 lua_pushstring(L, unmakemask(mask, buff));
diff --git a/src/lib_ffi.c b/src/lib_ffi.c
index cf9cf9f5..6dee2e74 100644
--- a/src/lib_ffi.c
+++ b/src/lib_ffi.c
@@ -29,6 +29,7 @@
29#include "lj_ccall.h" 29#include "lj_ccall.h"
30#include "lj_ccallback.h" 30#include "lj_ccallback.h"
31#include "lj_clib.h" 31#include "lj_clib.h"
32#include "lj_strfmt.h"
32#include "lj_ff.h" 33#include "lj_ff.h"
33#include "lj_lib.h" 34#include "lj_lib.h"
34 35
@@ -137,7 +138,7 @@ static int ffi_index_meta(lua_State *L, CTState *cts, CType *ct, MMS mm)
137 } 138 }
138 } 139 }
139 copyTV(L, base, L->top); 140 copyTV(L, base, L->top);
140 tv = L->top-1; 141 tv = L->top-1-LJ_FR2;
141 } 142 }
142 return lj_meta_tailcall(L, tv); 143 return lj_meta_tailcall(L, tv);
143} 144}
@@ -318,7 +319,7 @@ LJLIB_CF(ffi_meta___tostring)
318 } 319 }
319 } 320 }
320 } 321 }
321 lj_str_pushf(L, msg, strdata(lj_ctype_repr(L, id, NULL)), p); 322 lj_strfmt_pushf(L, msg, strdata(lj_ctype_repr(L, id, NULL)), p);
322checkgc: 323checkgc:
323 lj_gc_check(L); 324 lj_gc_check(L);
324 return 1; 325 return 1;
@@ -504,10 +505,7 @@ LJLIB_CF(ffi_new) LJLIB_REC(.)
504 } 505 }
505 if (sz == CTSIZE_INVALID) 506 if (sz == CTSIZE_INVALID)
506 lj_err_arg(L, 1, LJ_ERR_FFI_INVSIZE); 507 lj_err_arg(L, 1, LJ_ERR_FFI_INVSIZE);
507 if (!(info & CTF_VLA) && ctype_align(info) <= CT_MEMALIGN) 508 cd = lj_cdata_newx(cts, id, sz, info);
508 cd = lj_cdata_new(cts, id, sz);
509 else
510 cd = lj_cdata_newv(cts, id, sz, ctype_align(info));
511 setcdataV(L, o-1, cd); /* Anchor the uninitialized cdata. */ 509 setcdataV(L, o-1, cd); /* Anchor the uninitialized cdata. */
512 lj_cconv_ct_init(cts, ct, sz, cdataptr(cd), 510 lj_cconv_ct_init(cts, ct, sz, cdataptr(cd),
513 o, (MSize)(L->top - o)); /* Initialize cdata. */ 511 o, (MSize)(L->top - o)); /* Initialize cdata. */
@@ -558,6 +556,32 @@ LJLIB_CF(ffi_typeof) LJLIB_REC(.)
558 return 1; 556 return 1;
559} 557}
560 558
559/* Internal and unsupported API. */
560LJLIB_CF(ffi_typeinfo)
561{
562 CTState *cts = ctype_cts(L);
563 CTypeID id = (CTypeID)ffi_checkint(L, 1);
564 if (id > 0 && id < cts->top) {
565 CType *ct = ctype_get(cts, id);
566 GCtab *t;
567 lua_createtable(L, 0, 4); /* Increment hash size if fields are added. */
568 t = tabV(L->top-1);
569 setintV(lj_tab_setstr(L, t, lj_str_newlit(L, "info")), (int32_t)ct->info);
570 if (ct->size != CTSIZE_INVALID)
571 setintV(lj_tab_setstr(L, t, lj_str_newlit(L, "size")), (int32_t)ct->size);
572 if (ct->sib)
573 setintV(lj_tab_setstr(L, t, lj_str_newlit(L, "sib")), (int32_t)ct->sib);
574 if (gcref(ct->name)) {
575 GCstr *s = gco2str(gcref(ct->name));
576 if (isdead(G(L), obj2gco(s))) flipwhite(obj2gco(s));
577 setstrV(L, lj_tab_setstr(L, t, lj_str_newlit(L, "name")), s);
578 }
579 lj_gc_check(L);
580 return 1;
581 }
582 return 0;
583}
584
561LJLIB_CF(ffi_istype) LJLIB_REC(.) 585LJLIB_CF(ffi_istype) LJLIB_REC(.)
562{ 586{
563 CTState *cts = ctype_cts(L); 587 CTState *cts = ctype_cts(L);
@@ -697,44 +721,50 @@ LJLIB_CF(ffi_fill) LJLIB_REC(.)
697 return 0; 721 return 0;
698} 722}
699 723
700#define H_(le, be) LJ_ENDIAN_SELECT(0x##le, 0x##be)
701
702/* Test ABI string. */ 724/* Test ABI string. */
703LJLIB_CF(ffi_abi) LJLIB_REC(.) 725LJLIB_CF(ffi_abi) LJLIB_REC(.)
704{ 726{
705 GCstr *s = lj_lib_checkstr(L, 1); 727 GCstr *s = lj_lib_checkstr(L, 1);
706 int b = 0; 728 int b = lj_cparse_case(s,
707 switch (s->hash) {
708#if LJ_64 729#if LJ_64
709 case H_(849858eb,ad35fd06): b = 1; break; /* 64bit */ 730 "\00564bit"
710#else 731#else
711 case H_(662d3c79,d0e22477): b = 1; break; /* 32bit */ 732 "\00532bit"
712#endif 733#endif
713#if LJ_ARCH_HASFPU 734#if LJ_ARCH_HASFPU
714 case H_(e33ee463,e33ee463): b = 1; break; /* fpu */ 735 "\003fpu"
715#endif 736#endif
716#if LJ_ABI_SOFTFP 737#if LJ_ABI_SOFTFP
717 case H_(61211a23,c2e8c81c): b = 1; break; /* softfp */ 738 "\006softfp"
718#else 739#else
719 case H_(539417a8,8ce0812f): b = 1; break; /* hardfp */ 740 "\006hardfp"
720#endif 741#endif
721#if LJ_ABI_EABI 742#if LJ_ABI_EABI
722 case H_(2182df8f,f2ed1152): b = 1; break; /* eabi */ 743 "\004eabi"
723#endif 744#endif
724#if LJ_ABI_WIN 745#if LJ_ABI_WIN
725 case H_(4ab624a8,4ab624a8): b = 1; break; /* win */ 746 "\003win"
726#endif 747#endif
727 case H_(3af93066,1f001464): b = 1; break; /* le/be */ 748#if LJ_ABI_PAUTH
728 default: 749 "\007pauth"
729 break; 750#endif
730 } 751#if LJ_TARGET_UWP
752 "\003uwp"
753#endif
754#if LJ_LE
755 "\002le"
756#else
757 "\002be"
758#endif
759#if LJ_GC64
760 "\004gc64"
761#endif
762 ) >= 0;
731 setboolV(L->top-1, b); 763 setboolV(L->top-1, b);
732 setboolV(&G(L)->tmptv2, b); /* Remember for trace recorder. */ 764 setboolV(&G(L)->tmptv2, b); /* Remember for trace recorder. */
733 return 1; 765 return 1;
734} 766}
735 767
736#undef H_
737
738LJLIB_PUSH(top-8) LJLIB_SET(!) /* Store reference to miscmap table. */ 768LJLIB_PUSH(top-8) LJLIB_SET(!) /* Store reference to miscmap table. */
739 769
740LJLIB_CF(ffi_metatype) 770LJLIB_CF(ffi_metatype)
@@ -768,19 +798,11 @@ LJLIB_CF(ffi_gc) LJLIB_REC(.)
768 GCcdata *cd = ffi_checkcdata(L, 1); 798 GCcdata *cd = ffi_checkcdata(L, 1);
769 TValue *fin = lj_lib_checkany(L, 2); 799 TValue *fin = lj_lib_checkany(L, 2);
770 CTState *cts = ctype_cts(L); 800 CTState *cts = ctype_cts(L);
771 GCtab *t = cts->finalizer;
772 CType *ct = ctype_raw(cts, cd->ctypeid); 801 CType *ct = ctype_raw(cts, cd->ctypeid);
773 if (!(ctype_isptr(ct->info) || ctype_isstruct(ct->info) || 802 if (!(ctype_isptr(ct->info) || ctype_isstruct(ct->info) ||
774 ctype_isrefarray(ct->info))) 803 ctype_isrefarray(ct->info)))
775 lj_err_arg(L, 1, LJ_ERR_FFI_INVTYPE); 804 lj_err_arg(L, 1, LJ_ERR_FFI_INVTYPE);
776 if (gcref(t->metatable)) { /* Update finalizer table, if still enabled. */ 805 lj_cdata_setfin(L, cd, gcval(fin), itype(fin));
777 copyTV(L, lj_tab_set(L, t, L->base), fin);
778 lj_gc_anybarriert(L, t);
779 if (!tvisnil(fin))
780 cd->marked |= LJ_GC_CDATA_FIN;
781 else
782 cd->marked &= ~LJ_GC_CDATA_FIN;
783 }
784 L->top = L->base+1; /* Pass through the cdata object. */ 806 L->top = L->base+1; /* Pass through the cdata object. */
785 return 1; 807 return 1;
786} 808}
diff --git a/src/lib_io.c b/src/lib_io.c
index f7db083e..a13d4315 100644
--- a/src/lib_io.c
+++ b/src/lib_io.c
@@ -19,8 +19,10 @@
19#include "lj_obj.h" 19#include "lj_obj.h"
20#include "lj_gc.h" 20#include "lj_gc.h"
21#include "lj_err.h" 21#include "lj_err.h"
22#include "lj_buf.h"
22#include "lj_str.h" 23#include "lj_str.h"
23#include "lj_state.h" 24#include "lj_state.h"
25#include "lj_strfmt.h"
24#include "lj_ff.h" 26#include "lj_ff.h"
25#include "lj_lib.h" 27#include "lj_lib.h"
26 28
@@ -84,7 +86,7 @@ static IOFileUD *io_file_open(lua_State *L, const char *mode)
84 IOFileUD *iof = io_file_new(L); 86 IOFileUD *iof = io_file_new(L);
85 iof->fp = fopen(fname, mode); 87 iof->fp = fopen(fname, mode);
86 if (iof->fp == NULL) 88 if (iof->fp == NULL)
87 luaL_argerror(L, 1, lj_str_pushf(L, "%s: %s", fname, strerror(errno))); 89 luaL_argerror(L, 1, lj_strfmt_pushf(L, "%s: %s", fname, strerror(errno)));
88 return iof; 90 return iof;
89} 91}
90 92
@@ -97,11 +99,8 @@ static int io_file_close(lua_State *L, IOFileUD *iof)
97 int stat = -1; 99 int stat = -1;
98#if LJ_TARGET_POSIX 100#if LJ_TARGET_POSIX
99 stat = pclose(iof->fp); 101 stat = pclose(iof->fp);
100#elif LJ_TARGET_WINDOWS 102#elif LJ_TARGET_WINDOWS && !LJ_TARGET_XBOXONE && !LJ_TARGET_UWP
101 stat = _pclose(iof->fp); 103 stat = _pclose(iof->fp);
102#else
103 lua_assert(0);
104 return 0;
105#endif 104#endif
106#if LJ_52 105#if LJ_52
107 iof->fp = NULL; 106 iof->fp = NULL;
@@ -110,7 +109,8 @@ static int io_file_close(lua_State *L, IOFileUD *iof)
110 ok = (stat != -1); 109 ok = (stat != -1);
111#endif 110#endif
112 } else { 111 } else {
113 lua_assert((iof->type & IOFILE_TYPE_MASK) == IOFILE_TYPE_STDF); 112 lj_assertL((iof->type & IOFILE_TYPE_MASK) == IOFILE_TYPE_STDF,
113 "close of unknown FILE* type");
114 setnilV(L->top++); 114 setnilV(L->top++);
115 lua_pushliteral(L, "cannot close standard file"); 115 lua_pushliteral(L, "cannot close standard file");
116 return 2; 116 return 2;
@@ -145,7 +145,7 @@ static int io_file_readline(lua_State *L, FILE *fp, MSize chop)
145 MSize m = LUAL_BUFFERSIZE, n = 0, ok = 0; 145 MSize m = LUAL_BUFFERSIZE, n = 0, ok = 0;
146 char *buf; 146 char *buf;
147 for (;;) { 147 for (;;) {
148 buf = lj_str_needbuf(L, &G(L)->tmpbuf, m); 148 buf = lj_buf_tmp(L, m);
149 if (fgets(buf+n, m-n, fp) == NULL) break; 149 if (fgets(buf+n, m-n, fp) == NULL) break;
150 n += (MSize)strlen(buf+n); 150 n += (MSize)strlen(buf+n);
151 ok |= n; 151 ok |= n;
@@ -161,7 +161,7 @@ static void io_file_readall(lua_State *L, FILE *fp)
161{ 161{
162 MSize m, n; 162 MSize m, n;
163 for (m = LUAL_BUFFERSIZE, n = 0; ; m += m) { 163 for (m = LUAL_BUFFERSIZE, n = 0; ; m += m) {
164 char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, m); 164 char *buf = lj_buf_tmp(L, m);
165 n += (MSize)fread(buf+n, 1, m-n, fp); 165 n += (MSize)fread(buf+n, 1, m-n, fp);
166 if (n != m) { 166 if (n != m) {
167 setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n)); 167 setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n));
@@ -174,7 +174,7 @@ static void io_file_readall(lua_State *L, FILE *fp)
174static int io_file_readlen(lua_State *L, FILE *fp, MSize m) 174static int io_file_readlen(lua_State *L, FILE *fp, MSize m)
175{ 175{
176 if (m) { 176 if (m) {
177 char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, m); 177 char *buf = lj_buf_tmp(L, m);
178 MSize n = (MSize)fread(buf, 1, m, fp); 178 MSize n = (MSize)fread(buf, 1, m, fp);
179 setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n)); 179 setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n));
180 lj_gc_check(L); 180 lj_gc_check(L);
@@ -202,13 +202,12 @@ static int io_file_read(lua_State *L, IOFileUD *iof, int start)
202 for (n = start; nargs-- && ok; n++) { 202 for (n = start; nargs-- && ok; n++) {
203 if (tvisstr(L->base+n)) { 203 if (tvisstr(L->base+n)) {
204 const char *p = strVdata(L->base+n); 204 const char *p = strVdata(L->base+n);
205 if (p[0] != '*') 205 if (p[0] == '*') p++;
206 lj_err_arg(L, n+1, LJ_ERR_INVOPT); 206 if (p[0] == 'n')
207 if (p[1] == 'n')
208 ok = io_file_readnum(L, fp); 207 ok = io_file_readnum(L, fp);
209 else if ((p[1] & ~0x20) == 'L') 208 else if ((p[0] & ~0x20) == 'L')
210 ok = io_file_readline(L, fp, (p[1] == 'l')); 209 ok = io_file_readline(L, fp, (p[0] == 'l'));
211 else if (p[1] == 'a') 210 else if (p[0] == 'a')
212 io_file_readall(L, fp); 211 io_file_readall(L, fp);
213 else 212 else
214 lj_err_arg(L, n+1, LJ_ERR_INVFMT); 213 lj_err_arg(L, n+1, LJ_ERR_INVFMT);
@@ -232,19 +231,11 @@ static int io_file_write(lua_State *L, IOFileUD *iof, int start)
232 cTValue *tv; 231 cTValue *tv;
233 int status = 1; 232 int status = 1;
234 for (tv = L->base+start; tv < L->top; tv++) { 233 for (tv = L->base+start; tv < L->top; tv++) {
235 if (tvisstr(tv)) { 234 MSize len;
236 MSize len = strV(tv)->len; 235 const char *p = lj_strfmt_wstrnum(L, tv, &len);
237 status = status && (fwrite(strVdata(tv), 1, len, fp) == len); 236 if (!p)
238 } else if (tvisint(tv)) {
239 char buf[LJ_STR_INTBUF];
240 char *p = lj_str_bufint(buf, intV(tv));
241 size_t len = (size_t)(buf+LJ_STR_INTBUF-p);
242 status = status && (fwrite(p, 1, len, fp) == len);
243 } else if (tvisnum(tv)) {
244 status = status && (fprintf(fp, LUA_NUMBER_FMT, numV(tv)) > 0);
245 } else {
246 lj_err_argt(L, (int)(tv - L->base) + 1, LUA_TSTRING); 237 lj_err_argt(L, (int)(tv - L->base) + 1, LUA_TSTRING);
247 } 238 status = status && (fwrite(p, 1, len, fp) == len);
248 } 239 }
249 if (LJ_52 && status) { 240 if (LJ_52 && status) {
250 L->top = L->base+1; 241 L->top = L->base+1;
@@ -319,6 +310,14 @@ LJLIB_CF(io_method_flush) LJLIB_REC(io_flush 0)
319 return luaL_fileresult(L, fflush(io_tofile(L)->fp) == 0, NULL); 310 return luaL_fileresult(L, fflush(io_tofile(L)->fp) == 0, NULL);
320} 311}
321 312
313#if LJ_32 && defined(__ANDROID__) && __ANDROID_API__ < 24
314/* The Android NDK is such an unmatched marvel of engineering. */
315extern int fseeko32(FILE *, long int, int) __asm__("fseeko");
316extern long int ftello32(FILE *) __asm__("ftello");
317#define fseeko(fp, pos, whence) (fseeko32((fp), (pos), (whence)))
318#define ftello(fp) (ftello32((fp)))
319#endif
320
322LJLIB_CF(io_method_seek) 321LJLIB_CF(io_method_seek)
323{ 322{
324 FILE *fp = io_tofile(L)->fp; 323 FILE *fp = io_tofile(L)->fp;
@@ -419,7 +418,7 @@ LJLIB_CF(io_open)
419 418
420LJLIB_CF(io_popen) 419LJLIB_CF(io_popen)
421{ 420{
422#if LJ_TARGET_POSIX || LJ_TARGET_WINDOWS 421#if LJ_TARGET_POSIX || (LJ_TARGET_WINDOWS && !LJ_TARGET_XBOXONE && !LJ_TARGET_UWP)
423 const char *fname = strdata(lj_lib_checkstr(L, 1)); 422 const char *fname = strdata(lj_lib_checkstr(L, 1));
424 GCstr *s = lj_lib_optstr(L, 2); 423 GCstr *s = lj_lib_optstr(L, 2);
425 const char *mode = s ? strdata(s) : "r"; 424 const char *mode = s ? strdata(s) : "r";
@@ -440,7 +439,7 @@ LJLIB_CF(io_popen)
440LJLIB_CF(io_tmpfile) 439LJLIB_CF(io_tmpfile)
441{ 440{
442 IOFileUD *iof = io_file_new(L); 441 IOFileUD *iof = io_file_new(L);
443#if LJ_TARGET_PS3 || LJ_TARGET_PS4 || LJ_TARGET_PSVITA 442#if LJ_TARGET_PS3 || LJ_TARGET_PS4 || LJ_TARGET_PS5 || LJ_TARGET_PSVITA || LJ_TARGET_NX
444 iof->fp = NULL; errno = ENOSYS; 443 iof->fp = NULL; errno = ENOSYS;
445#else 444#else
446 iof->fp = tmpfile(); 445 iof->fp = tmpfile();
diff --git a/src/lib_jit.c b/src/lib_jit.c
index 04a564c7..c0294927 100644
--- a/src/lib_jit.c
+++ b/src/lib_jit.c
@@ -10,13 +10,17 @@
10#include "lauxlib.h" 10#include "lauxlib.h"
11#include "lualib.h" 11#include "lualib.h"
12 12
13#include "lj_arch.h"
14#include "lj_obj.h" 13#include "lj_obj.h"
14#include "lj_gc.h"
15#include "lj_err.h" 15#include "lj_err.h"
16#include "lj_debug.h" 16#include "lj_debug.h"
17#include "lj_str.h" 17#include "lj_str.h"
18#include "lj_tab.h" 18#include "lj_tab.h"
19#include "lj_state.h"
19#include "lj_bc.h" 20#include "lj_bc.h"
21#if LJ_HASFFI
22#include "lj_ctype.h"
23#endif
20#if LJ_HASJIT 24#if LJ_HASJIT
21#include "lj_ir.h" 25#include "lj_ir.h"
22#include "lj_jit.h" 26#include "lj_jit.h"
@@ -24,6 +28,7 @@
24#include "lj_iropt.h" 28#include "lj_iropt.h"
25#include "lj_target.h" 29#include "lj_target.h"
26#endif 30#endif
31#include "lj_trace.h"
27#include "lj_dispatch.h" 32#include "lj_dispatch.h"
28#include "lj_vm.h" 33#include "lj_vm.h"
29#include "lj_vmevent.h" 34#include "lj_vmevent.h"
@@ -99,8 +104,8 @@ LJLIB_CF(jit_status)
99 jit_State *J = L2J(L); 104 jit_State *J = L2J(L);
100 L->top = L->base; 105 L->top = L->base;
101 setboolV(L->top++, (J->flags & JIT_F_ON) ? 1 : 0); 106 setboolV(L->top++, (J->flags & JIT_F_ON) ? 1 : 0);
102 flagbits_to_strings(L, J->flags, JIT_F_CPU_FIRST, JIT_F_CPUSTRING); 107 flagbits_to_strings(L, J->flags, JIT_F_CPU, JIT_F_CPUSTRING);
103 flagbits_to_strings(L, J->flags, JIT_F_OPT_FIRST, JIT_F_OPTSTRING); 108 flagbits_to_strings(L, J->flags, JIT_F_OPT, JIT_F_OPTSTRING);
104 return (int)(L->top - L->base); 109 return (int)(L->top - L->base);
105#else 110#else
106 setboolV(L->top++, 0); 111 setboolV(L->top++, 0);
@@ -108,6 +113,13 @@ LJLIB_CF(jit_status)
108#endif 113#endif
109} 114}
110 115
116LJLIB_CF(jit_security)
117{
118 int idx = lj_lib_checkopt(L, 1, -1, LJ_SECURITY_MODESTRING);
119 setintV(L->top++, ((LJ_SECURITY_MODE >> (2*idx)) & 3));
120 return 1;
121}
122
111LJLIB_CF(jit_attach) 123LJLIB_CF(jit_attach)
112{ 124{
113#ifdef LUAJIT_DISABLE_VMEVENT 125#ifdef LUAJIT_DISABLE_VMEVENT
@@ -222,7 +234,7 @@ LJLIB_CF(jit_util_funcbc)
222 if (pc < pt->sizebc) { 234 if (pc < pt->sizebc) {
223 BCIns ins = proto_bc(pt)[pc]; 235 BCIns ins = proto_bc(pt)[pc];
224 BCOp op = bc_op(ins); 236 BCOp op = bc_op(ins);
225 lua_assert(op < BC__MAX); 237 lj_assertL(op < BC__MAX, "bad bytecode op %d", op);
226 setintV(L->top, ins); 238 setintV(L->top, ins);
227 setintV(L->top+1, lj_bc_mode[op]); 239 setintV(L->top+1, lj_bc_mode[op]);
228 L->top += 2; 240 L->top += 2;
@@ -280,7 +292,7 @@ static GCtrace *jit_checktrace(lua_State *L)
280/* Names of link types. ORDER LJ_TRLINK */ 292/* Names of link types. ORDER LJ_TRLINK */
281static const char *const jit_trlinkname[] = { 293static const char *const jit_trlinkname[] = {
282 "none", "root", "loop", "tail-recursion", "up-recursion", "down-recursion", 294 "none", "root", "loop", "tail-recursion", "up-recursion", "down-recursion",
283 "interpreter", "return" 295 "interpreter", "return", "stitch"
284}; 296};
285 297
286/* local info = jit.util.traceinfo(tr) */ 298/* local info = jit.util.traceinfo(tr) */
@@ -333,6 +345,9 @@ LJLIB_CF(jit_util_tracek)
333 slot = ir->op2; 345 slot = ir->op2;
334 ir = &T->ir[ir->op1]; 346 ir = &T->ir[ir->op1];
335 } 347 }
348#if LJ_HASFFI
349 if (ir->o == IR_KINT64) ctype_loadffi(L);
350#endif
336 lj_ir_kvalue(L, L->top-2, ir); 351 lj_ir_kvalue(L, L->top-2, ir);
337 setintV(L->top-1, (int32_t)irt_type(ir->t)); 352 setintV(L->top-1, (int32_t)irt_type(ir->t));
338 if (slot == -1) 353 if (slot == -1)
@@ -407,7 +422,8 @@ LJLIB_CF(jit_util_ircalladdr)
407{ 422{
408 uint32_t idx = (uint32_t)lj_lib_checkint(L, 1); 423 uint32_t idx = (uint32_t)lj_lib_checkint(L, 1);
409 if (idx < IRCALL__MAX) { 424 if (idx < IRCALL__MAX) {
410 setintptrV(L->top-1, (intptr_t)(void *)lj_ir_callinfo[idx].func); 425 ASMFunction func = lj_ir_callinfo[idx].func;
426 setintptrV(L->top-1, (intptr_t)(void *)lj_ptr_strip(func));
411 return 1; 427 return 1;
412 } 428 }
413 return 0; 429 return 0;
@@ -417,6 +433,12 @@ LJLIB_CF(jit_util_ircalladdr)
417 433
418#include "lj_libdef.h" 434#include "lj_libdef.h"
419 435
436static int luaopen_jit_util(lua_State *L)
437{
438 LJ_LIB_REG(L, NULL, jit_util);
439 return 1;
440}
441
420/* -- jit.opt module ------------------------------------------------------ */ 442/* -- jit.opt module ------------------------------------------------------ */
421 443
422#if LJ_HASJIT 444#if LJ_HASJIT
@@ -453,7 +475,7 @@ static int jitopt_flag(jit_State *J, const char *str)
453 str += str[2] == '-' ? 3 : 2; 475 str += str[2] == '-' ? 3 : 2;
454 set = 0; 476 set = 0;
455 } 477 }
456 for (opt = JIT_F_OPT_FIRST; ; opt <<= 1) { 478 for (opt = JIT_F_OPT; ; opt <<= 1) {
457 size_t len = *(const uint8_t *)lst; 479 size_t len = *(const uint8_t *)lst;
458 if (len == 0) 480 if (len == 0)
459 break; 481 break;
@@ -473,7 +495,7 @@ static int jitopt_param(jit_State *J, const char *str)
473 int i; 495 int i;
474 for (i = 0; i < JIT_P__MAX; i++) { 496 for (i = 0; i < JIT_P__MAX; i++) {
475 size_t len = *(const uint8_t *)lst; 497 size_t len = *(const uint8_t *)lst;
476 lua_assert(len != 0); 498 lj_assertJ(len != 0, "bad JIT_P_STRING");
477 if (strncmp(str, lst+1, len) == 0 && str[len] == '=') { 499 if (strncmp(str, lst+1, len) == 0 && str[len] == '=') {
478 int32_t n = 0; 500 int32_t n = 0;
479 const char *p = &str[len+1]; 501 const char *p = &str[len+1];
@@ -514,6 +536,104 @@ LJLIB_CF(jit_opt_start)
514 536
515#endif 537#endif
516 538
539/* -- jit.profile module -------------------------------------------------- */
540
541#if LJ_HASPROFILE
542
543#define LJLIB_MODULE_jit_profile
544
545/* Not loaded by default, use: local profile = require("jit.profile") */
546
547#define KEY_PROFILE_THREAD (U64x(80000000,00000000)|'t')
548#define KEY_PROFILE_FUNC (U64x(80000000,00000000)|'f')
549
550static void jit_profile_callback(lua_State *L2, lua_State *L, int samples,
551 int vmstate)
552{
553 TValue key;
554 cTValue *tv;
555 key.u64 = KEY_PROFILE_FUNC;
556 tv = lj_tab_get(L, tabV(registry(L)), &key);
557 if (tvisfunc(tv)) {
558 char vmst = (char)vmstate;
559 int status;
560 setfuncV(L2, L2->top++, funcV(tv));
561 setthreadV(L2, L2->top++, L);
562 setintV(L2->top++, samples);
563 setstrV(L2, L2->top++, lj_str_new(L2, &vmst, 1));
564 status = lua_pcall(L2, 3, 0, 0); /* callback(thread, samples, vmstate) */
565 if (status) {
566 if (G(L2)->panic) G(L2)->panic(L2);
567 exit(EXIT_FAILURE);
568 }
569 lj_trace_abort(G(L2));
570 }
571}
572
573/* profile.start(mode, cb) */
574LJLIB_CF(jit_profile_start)
575{
576 GCtab *registry = tabV(registry(L));
577 GCstr *mode = lj_lib_optstr(L, 1);
578 GCfunc *func = lj_lib_checkfunc(L, 2);
579 lua_State *L2 = lua_newthread(L); /* Thread that runs profiler callback. */
580 TValue key;
581 /* Anchor thread and function in registry. */
582 key.u64 = KEY_PROFILE_THREAD;
583 setthreadV(L, lj_tab_set(L, registry, &key), L2);
584 key.u64 = KEY_PROFILE_FUNC;
585 setfuncV(L, lj_tab_set(L, registry, &key), func);
586 lj_gc_anybarriert(L, registry);
587 luaJIT_profile_start(L, mode ? strdata(mode) : "",
588 (luaJIT_profile_callback)jit_profile_callback, L2);
589 return 0;
590}
591
592/* profile.stop() */
593LJLIB_CF(jit_profile_stop)
594{
595 GCtab *registry;
596 TValue key;
597 luaJIT_profile_stop(L);
598 registry = tabV(registry(L));
599 key.u64 = KEY_PROFILE_THREAD;
600 setnilV(lj_tab_set(L, registry, &key));
601 key.u64 = KEY_PROFILE_FUNC;
602 setnilV(lj_tab_set(L, registry, &key));
603 lj_gc_anybarriert(L, registry);
604 return 0;
605}
606
607/* dump = profile.dumpstack([thread,] fmt, depth) */
608LJLIB_CF(jit_profile_dumpstack)
609{
610 lua_State *L2 = L;
611 int arg = 0;
612 size_t len;
613 int depth;
614 GCstr *fmt;
615 const char *p;
616 if (L->top > L->base && tvisthread(L->base)) {
617 L2 = threadV(L->base);
618 arg = 1;
619 }
620 fmt = lj_lib_checkstr(L, arg+1);
621 depth = lj_lib_checkint(L, arg+2);
622 p = luaJIT_profile_dumpstack(L2, strdata(fmt), depth, &len);
623 lua_pushlstring(L, p, len);
624 return 1;
625}
626
627#include "lj_libdef.h"
628
629static int luaopen_jit_profile(lua_State *L)
630{
631 LJ_LIB_REG(L, NULL, jit_profile);
632 return 1;
633}
634
635#endif
636
517/* -- JIT compiler initialization ----------------------------------------- */ 637/* -- JIT compiler initialization ----------------------------------------- */
518 638
519#if LJ_HASJIT 639#if LJ_HASJIT
@@ -524,66 +644,41 @@ JIT_PARAMDEF(JIT_PARAMINIT)
524#undef JIT_PARAMINIT 644#undef JIT_PARAMINIT
525 0 645 0
526}; 646};
527#endif
528 647
529#if LJ_TARGET_ARM && LJ_TARGET_LINUX 648#if LJ_TARGET_ARM && LJ_TARGET_LINUX
530#include <sys/utsname.h> 649#include <sys/utsname.h>
531#endif 650#endif
532 651
533/* Arch-dependent CPU detection. */ 652/* Arch-dependent CPU feature detection. */
534static uint32_t jit_cpudetect(lua_State *L) 653static uint32_t jit_cpudetect(void)
535{ 654{
536 uint32_t flags = 0; 655 uint32_t flags = 0;
537#if LJ_TARGET_X86ORX64 656#if LJ_TARGET_X86ORX64
657
538 uint32_t vendor[4]; 658 uint32_t vendor[4];
539 uint32_t features[4]; 659 uint32_t features[4];
540 if (lj_vm_cpuid(0, vendor) && lj_vm_cpuid(1, features)) { 660 if (lj_vm_cpuid(0, vendor) && lj_vm_cpuid(1, features)) {
541#if !LJ_HASJIT
542#define JIT_F_CMOV 1
543#define JIT_F_SSE2 2
544#endif
545 flags |= ((features[3] >> 15)&1) * JIT_F_CMOV;
546 flags |= ((features[3] >> 26)&1) * JIT_F_SSE2;
547#if LJ_HASJIT
548 flags |= ((features[2] >> 0)&1) * JIT_F_SSE3; 661 flags |= ((features[2] >> 0)&1) * JIT_F_SSE3;
549 flags |= ((features[2] >> 19)&1) * JIT_F_SSE4_1; 662 flags |= ((features[2] >> 19)&1) * JIT_F_SSE4_1;
550 if (vendor[2] == 0x6c65746e) { /* Intel. */ 663 if (vendor[0] >= 7) {
551 if ((features[0] & 0x0ff00f00) == 0x00000f00) /* P4. */ 664 uint32_t xfeatures[4];
552 flags |= JIT_F_P4; /* Currently unused. */ 665 lj_vm_cpuid(7, xfeatures);
553 else if ((features[0] & 0x0fff0ff0) == 0x000106c0) /* Atom. */ 666 flags |= ((xfeatures[1] >> 8)&1) * JIT_F_BMI2;
554 flags |= JIT_F_LEA_AGU;
555 } else if (vendor[2] == 0x444d4163) { /* AMD. */
556 uint32_t fam = (features[0] & 0x0ff00f00);
557 if (fam == 0x00000f00) /* K8. */
558 flags |= JIT_F_SPLIT_XMM;
559 if (fam >= 0x00000f00) /* K8, K10. */
560 flags |= JIT_F_PREFER_IMUL;
561 } 667 }
562#endif
563 } 668 }
564 /* Check for required instruction set support on x86 (unnecessary on x64). */ 669 /* Don't bother checking for SSE2 -- the VM will crash before getting here. */
565#if LJ_TARGET_X86 670
566#if !defined(LUAJIT_CPU_NOCMOV)
567 if (!(flags & JIT_F_CMOV))
568 luaL_error(L, "CPU not supported");
569#endif
570#if defined(LUAJIT_CPU_SSE2)
571 if (!(flags & JIT_F_SSE2))
572 luaL_error(L, "CPU does not support SSE2 (recompile without -DLUAJIT_CPU_SSE2)");
573#endif
574#endif
575#elif LJ_TARGET_ARM 671#elif LJ_TARGET_ARM
576#if LJ_HASJIT 672
577 int ver = LJ_ARCH_VERSION; /* Compile-time ARM CPU detection. */ 673 int ver = LJ_ARCH_VERSION; /* Compile-time ARM CPU detection. */
578#if LJ_TARGET_LINUX 674#if LJ_TARGET_LINUX
579 if (ver < 70) { /* Runtime ARM CPU detection. */ 675 if (ver < 70) { /* Runtime ARM CPU detection. */
580 struct utsname ut; 676 struct utsname ut;
581 uname(&ut); 677 uname(&ut);
582 if (strncmp(ut.machine, "armv", 4) == 0) { 678 if (strncmp(ut.machine, "armv", 4) == 0) {
583 if (ut.machine[4] >= '7') 679 if (ut.machine[4] >= '8') ver = 80;
584 ver = 70; 680 else if (ut.machine[4] == '7') ver = 70;
585 else if (ut.machine[4] == '6') 681 else if (ut.machine[4] == '6') ver = 60;
586 ver = 60;
587 } 682 }
588 } 683 }
589#endif 684#endif
@@ -591,74 +686,77 @@ static uint32_t jit_cpudetect(lua_State *L)
591 ver >= 61 ? JIT_F_ARMV6T2_ : 686 ver >= 61 ? JIT_F_ARMV6T2_ :
592 ver >= 60 ? JIT_F_ARMV6_ : 0; 687 ver >= 60 ? JIT_F_ARMV6_ : 0;
593 flags |= LJ_ARCH_HASFPU == 0 ? 0 : ver >= 70 ? JIT_F_VFPV3 : JIT_F_VFPV2; 688 flags |= LJ_ARCH_HASFPU == 0 ? 0 : ver >= 70 ? JIT_F_VFPV3 : JIT_F_VFPV2;
594#endif 689
690#elif LJ_TARGET_ARM64
691
692 /* No optional CPU features to detect (for now). */
693
595#elif LJ_TARGET_PPC 694#elif LJ_TARGET_PPC
596#if LJ_HASJIT 695
597#if LJ_ARCH_SQRT 696#if LJ_ARCH_SQRT
598 flags |= JIT_F_SQRT; 697 flags |= JIT_F_SQRT;
599#endif 698#endif
600#if LJ_ARCH_ROUND 699#if LJ_ARCH_ROUND
601 flags |= JIT_F_ROUND; 700 flags |= JIT_F_ROUND;
602#endif 701#endif
603#endif 702
604#elif LJ_TARGET_PPCSPE
605 /* Nothing to do. */
606#elif LJ_TARGET_MIPS 703#elif LJ_TARGET_MIPS
607#if LJ_HASJIT 704
608 /* Compile-time MIPS CPU detection. */ 705 /* Compile-time MIPS CPU detection. */
609#if LJ_ARCH_VERSION >= 20 706#if LJ_ARCH_VERSION >= 20
610 flags |= JIT_F_MIPS32R2; 707 flags |= JIT_F_MIPSXXR2;
611#endif 708#endif
612 /* Runtime MIPS CPU detection. */ 709 /* Runtime MIPS CPU detection. */
613#if defined(__GNUC__) 710#if defined(__GNUC__)
614 if (!(flags & JIT_F_MIPS32R2)) { 711 if (!(flags & JIT_F_MIPSXXR2)) {
615 int x; 712 int x;
713#ifdef __mips16
714 x = 0; /* Runtime detection is difficult. Ensure optimal -march flags. */
715#else
616 /* On MIPS32R1 rotr is treated as srl. rotr r2,r2,1 -> srl r2,r2,1. */ 716 /* On MIPS32R1 rotr is treated as srl. rotr r2,r2,1 -> srl r2,r2,1. */
617 __asm__("li $2, 1\n\t.long 0x00221042\n\tmove %0, $2" : "=r"(x) : : "$2"); 717 __asm__("li $2, 1\n\t.long 0x00221042\n\tmove %0, $2" : "=r"(x) : : "$2");
618 if (x) flags |= JIT_F_MIPS32R2; /* Either 0x80000000 (R2) or 0 (R1). */
619 }
620#endif 718#endif
719 if (x) flags |= JIT_F_MIPSXXR2; /* Either 0x80000000 (R2) or 0 (R1). */
720 }
621#endif 721#endif
722
622#else 723#else
623#error "Missing CPU detection for this architecture" 724#error "Missing CPU detection for this architecture"
624#endif 725#endif
625 UNUSED(L);
626 return flags; 726 return flags;
627} 727}
628 728
629/* Initialize JIT compiler. */ 729/* Initialize JIT compiler. */
630static void jit_init(lua_State *L) 730static void jit_init(lua_State *L)
631{ 731{
632 uint32_t flags = jit_cpudetect(L);
633#if LJ_HASJIT
634 jit_State *J = L2J(L); 732 jit_State *J = L2J(L);
635#if LJ_TARGET_X86 733 J->flags = jit_cpudetect() | JIT_F_ON | JIT_F_OPT_DEFAULT;
636 /* Silently turn off the JIT compiler on CPUs without SSE2. */
637 if ((flags & JIT_F_SSE2))
638#endif
639 J->flags = flags | JIT_F_ON | JIT_F_OPT_DEFAULT;
640 memcpy(J->param, jit_param_default, sizeof(J->param)); 734 memcpy(J->param, jit_param_default, sizeof(J->param));
641 lj_dispatch_update(G(L)); 735 lj_dispatch_update(G(L));
642#else
643 UNUSED(flags);
644#endif
645} 736}
737#endif
646 738
647LUALIB_API int luaopen_jit(lua_State *L) 739LUALIB_API int luaopen_jit(lua_State *L)
648{ 740{
741#if LJ_HASJIT
742 jit_init(L);
743#endif
649 lua_pushliteral(L, LJ_OS_NAME); 744 lua_pushliteral(L, LJ_OS_NAME);
650 lua_pushliteral(L, LJ_ARCH_NAME); 745 lua_pushliteral(L, LJ_ARCH_NAME);
651 lua_pushinteger(L, LUAJIT_VERSION_NUM); /* Deprecated. */ 746 lua_pushinteger(L, LUAJIT_VERSION_NUM); /* Deprecated. */
652 lua_pushliteral(L, LUAJIT_VERSION); 747 lua_pushliteral(L, LUAJIT_VERSION);
653 LJ_LIB_REG(L, LUA_JITLIBNAME, jit); 748 LJ_LIB_REG(L, LUA_JITLIBNAME, jit);
749#if LJ_HASPROFILE
750 lj_lib_prereg(L, LUA_JITLIBNAME ".profile", luaopen_jit_profile,
751 tabref(L->env));
752#endif
654#ifndef LUAJIT_DISABLE_JITUTIL 753#ifndef LUAJIT_DISABLE_JITUTIL
655 LJ_LIB_REG(L, "jit.util", jit_util); 754 lj_lib_prereg(L, LUA_JITLIBNAME ".util", luaopen_jit_util, tabref(L->env));
656#endif 755#endif
657#if LJ_HASJIT 756#if LJ_HASJIT
658 LJ_LIB_REG(L, "jit.opt", jit_opt); 757 LJ_LIB_REG(L, "jit.opt", jit_opt);
659#endif 758#endif
660 L->top -= 2; 759 L->top -= 2;
661 jit_init(L);
662 return 1; 760 return 1;
663} 761}
664 762
diff --git a/src/lib_math.c b/src/lib_math.c
index 56644746..4539f804 100644
--- a/src/lib_math.c
+++ b/src/lib_math.c
@@ -15,6 +15,7 @@
15#include "lj_obj.h" 15#include "lj_obj.h"
16#include "lj_lib.h" 16#include "lj_lib.h"
17#include "lj_vm.h" 17#include "lj_vm.h"
18#include "lj_prng.h"
18 19
19/* ------------------------------------------------------------------------ */ 20/* ------------------------------------------------------------------------ */
20 21
@@ -33,25 +34,19 @@ LJLIB_ASM(math_sqrt) LJLIB_REC(math_unary IRFPM_SQRT)
33 lj_lib_checknum(L, 1); 34 lj_lib_checknum(L, 1);
34 return FFH_RETRY; 35 return FFH_RETRY;
35} 36}
36LJLIB_ASM_(math_log10) LJLIB_REC(math_unary IRFPM_LOG10) 37LJLIB_ASM_(math_log10) LJLIB_REC(math_call IRCALL_log10)
37LJLIB_ASM_(math_exp) LJLIB_REC(math_unary IRFPM_EXP) 38LJLIB_ASM_(math_exp) LJLIB_REC(math_call IRCALL_exp)
38LJLIB_ASM_(math_sin) LJLIB_REC(math_unary IRFPM_SIN) 39LJLIB_ASM_(math_sin) LJLIB_REC(math_call IRCALL_sin)
39LJLIB_ASM_(math_cos) LJLIB_REC(math_unary IRFPM_COS) 40LJLIB_ASM_(math_cos) LJLIB_REC(math_call IRCALL_cos)
40LJLIB_ASM_(math_tan) LJLIB_REC(math_unary IRFPM_TAN) 41LJLIB_ASM_(math_tan) LJLIB_REC(math_call IRCALL_tan)
41LJLIB_ASM_(math_asin) LJLIB_REC(math_atrig FF_math_asin) 42LJLIB_ASM_(math_asin) LJLIB_REC(math_call IRCALL_asin)
42LJLIB_ASM_(math_acos) LJLIB_REC(math_atrig FF_math_acos) 43LJLIB_ASM_(math_acos) LJLIB_REC(math_call IRCALL_acos)
43LJLIB_ASM_(math_atan) LJLIB_REC(math_atrig FF_math_atan) 44LJLIB_ASM_(math_atan) LJLIB_REC(math_call IRCALL_atan)
44LJLIB_ASM_(math_sinh) LJLIB_REC(math_htrig IRCALL_sinh) 45LJLIB_ASM_(math_sinh) LJLIB_REC(math_call IRCALL_sinh)
45LJLIB_ASM_(math_cosh) LJLIB_REC(math_htrig IRCALL_cosh) 46LJLIB_ASM_(math_cosh) LJLIB_REC(math_call IRCALL_cosh)
46LJLIB_ASM_(math_tanh) LJLIB_REC(math_htrig IRCALL_tanh) 47LJLIB_ASM_(math_tanh) LJLIB_REC(math_call IRCALL_tanh)
47LJLIB_ASM_(math_frexp) 48LJLIB_ASM_(math_frexp)
48LJLIB_ASM_(math_modf) LJLIB_REC(.) 49LJLIB_ASM_(math_modf)
49
50LJLIB_PUSH(57.29577951308232)
51LJLIB_ASM_(math_deg) LJLIB_REC(math_degrad)
52
53LJLIB_PUSH(0.017453292519943295)
54LJLIB_ASM_(math_rad) LJLIB_REC(math_degrad)
55 50
56LJLIB_ASM(math_log) LJLIB_REC(math_log) 51LJLIB_ASM(math_log) LJLIB_REC(math_log)
57{ 52{
@@ -63,12 +58,15 @@ LJLIB_ASM(math_log) LJLIB_REC(math_log)
63#else 58#else
64 x = lj_vm_log2(x); y = 1.0 / lj_vm_log2(y); 59 x = lj_vm_log2(x); y = 1.0 / lj_vm_log2(y);
65#endif 60#endif
66 setnumV(L->base-1, x*y); /* Do NOT join the expression to x / y. */ 61 setnumV(L->base-1-LJ_FR2, x*y); /* Do NOT join the expression to x / y. */
67 return FFH_RES(1); 62 return FFH_RES(1);
68 } 63 }
69 return FFH_RETRY; 64 return FFH_RETRY;
70} 65}
71 66
67LJLIB_LUA(math_deg) /* function(x) return x * 57.29577951308232 end */
68LJLIB_LUA(math_rad) /* function(x) return x * 0.017453292519943295 end */
69
72LJLIB_ASM(math_atan2) LJLIB_REC(.) 70LJLIB_ASM(math_atan2) LJLIB_REC(.)
73{ 71{
74 lj_lib_checknum(L, 1); 72 lj_lib_checknum(L, 1);
@@ -108,34 +106,11 @@ LJLIB_PUSH(1e310) LJLIB_SET(huge)
108** Full-period ME-CF generator with L=64, J=4, k=223, N1=49. 106** Full-period ME-CF generator with L=64, J=4, k=223, N1=49.
109*/ 107*/
110 108
111/* PRNG state. */
112struct RandomState {
113 uint64_t gen[4]; /* State of the 4 LFSR generators. */
114 int valid; /* State is valid. */
115};
116
117/* Union needed for bit-pattern conversion between uint64_t and double. */ 109/* Union needed for bit-pattern conversion between uint64_t and double. */
118typedef union { uint64_t u64; double d; } U64double; 110typedef union { uint64_t u64; double d; } U64double;
119 111
120/* Update generator i and compute a running xor of all states. */ 112/* PRNG seeding function. */
121#define TW223_GEN(i, k, q, s) \ 113static void random_seed(PRNGState *rs, double d)
122 z = rs->gen[i]; \
123 z = (((z<<q)^z) >> (k-s)) ^ ((z&((uint64_t)(int64_t)-1 << (64-k)))<<s); \
124 r ^= z; rs->gen[i] = z;
125
126/* PRNG step function. Returns a double in the range 1.0 <= d < 2.0. */
127LJ_NOINLINE uint64_t LJ_FASTCALL lj_math_random_step(RandomState *rs)
128{
129 uint64_t z, r = 0;
130 TW223_GEN(0, 63, 31, 18)
131 TW223_GEN(1, 58, 19, 28)
132 TW223_GEN(2, 55, 24, 7)
133 TW223_GEN(3, 47, 21, 8)
134 return (r & U64x(000fffff,ffffffff)) | U64x(3ff00000,00000000);
135}
136
137/* PRNG initialization function. */
138static void random_init(RandomState *rs, double d)
139{ 114{
140 uint32_t r = 0x11090601; /* 64-k[i] as four 8 bit constants. */ 115 uint32_t r = 0x11090601; /* 64-k[i] as four 8 bit constants. */
141 int i; 116 int i;
@@ -144,24 +119,22 @@ static void random_init(RandomState *rs, double d)
144 uint32_t m = 1u << (r&255); 119 uint32_t m = 1u << (r&255);
145 r >>= 8; 120 r >>= 8;
146 u.d = d = d * 3.14159265358979323846 + 2.7182818284590452354; 121 u.d = d = d * 3.14159265358979323846 + 2.7182818284590452354;
147 if (u.u64 < m) u.u64 += m; /* Ensure k[i] MSB of gen[i] are non-zero. */ 122 if (u.u64 < m) u.u64 += m; /* Ensure k[i] MSB of u[i] are non-zero. */
148 rs->gen[i] = u.u64; 123 rs->u[i] = u.u64;
149 } 124 }
150 rs->valid = 1;
151 for (i = 0; i < 10; i++) 125 for (i = 0; i < 10; i++)
152 lj_math_random_step(rs); 126 (void)lj_prng_u64(rs);
153} 127}
154 128
155/* PRNG extract function. */ 129/* PRNG extract function. */
156LJLIB_PUSH(top-2) /* Upvalue holds userdata with RandomState. */ 130LJLIB_PUSH(top-2) /* Upvalue holds userdata with PRNGState. */
157LJLIB_CF(math_random) LJLIB_REC(.) 131LJLIB_CF(math_random) LJLIB_REC(.)
158{ 132{
159 int n = (int)(L->top - L->base); 133 int n = (int)(L->top - L->base);
160 RandomState *rs = (RandomState *)(uddata(udataV(lj_lib_upvalue(L, 1)))); 134 PRNGState *rs = (PRNGState *)(uddata(udataV(lj_lib_upvalue(L, 1))));
161 U64double u; 135 U64double u;
162 double d; 136 double d;
163 if (LJ_UNLIKELY(!rs->valid)) random_init(rs, 0.0); 137 u.u64 = lj_prng_u64d(rs);
164 u.u64 = lj_math_random_step(rs);
165 d = u.d - 1.0; 138 d = u.d - 1.0;
166 if (n > 0) { 139 if (n > 0) {
167#if LJ_DUALNUM 140#if LJ_DUALNUM
@@ -206,11 +179,11 @@ LJLIB_CF(math_random) LJLIB_REC(.)
206} 179}
207 180
208/* PRNG seed function. */ 181/* PRNG seed function. */
209LJLIB_PUSH(top-2) /* Upvalue holds userdata with RandomState. */ 182LJLIB_PUSH(top-2) /* Upvalue holds userdata with PRNGState. */
210LJLIB_CF(math_randomseed) 183LJLIB_CF(math_randomseed)
211{ 184{
212 RandomState *rs = (RandomState *)(uddata(udataV(lj_lib_upvalue(L, 1)))); 185 PRNGState *rs = (PRNGState *)(uddata(udataV(lj_lib_upvalue(L, 1))));
213 random_init(rs, lj_lib_checknum(L, 1)); 186 random_seed(rs, lj_lib_checknum(L, 1));
214 return 0; 187 return 0;
215} 188}
216 189
@@ -220,14 +193,9 @@ LJLIB_CF(math_randomseed)
220 193
221LUALIB_API int luaopen_math(lua_State *L) 194LUALIB_API int luaopen_math(lua_State *L)
222{ 195{
223 RandomState *rs; 196 PRNGState *rs = (PRNGState *)lua_newuserdata(L, sizeof(PRNGState));
224 rs = (RandomState *)lua_newuserdata(L, sizeof(RandomState)); 197 lj_prng_seed_fixed(rs);
225 rs->valid = 0; /* Use lazy initialization to save some time on startup. */
226 LJ_LIB_REG(L, LUA_MATHLIBNAME, math); 198 LJ_LIB_REG(L, LUA_MATHLIBNAME, math);
227#if defined(LUA_COMPAT_MOD) && !LJ_52
228 lua_getfield(L, -1, "fmod");
229 lua_setfield(L, -2, "mod");
230#endif
231 return 1; 199 return 1;
232} 200}
233 201
diff --git a/src/lib_os.c b/src/lib_os.c
index 029cb75d..cf0df281 100644
--- a/src/lib_os.c
+++ b/src/lib_os.c
@@ -17,7 +17,10 @@
17#include "lualib.h" 17#include "lualib.h"
18 18
19#include "lj_obj.h" 19#include "lj_obj.h"
20#include "lj_gc.h"
20#include "lj_err.h" 21#include "lj_err.h"
22#include "lj_buf.h"
23#include "lj_str.h"
21#include "lj_lib.h" 24#include "lj_lib.h"
22 25
23#if LJ_TARGET_POSIX 26#if LJ_TARGET_POSIX
@@ -73,7 +76,7 @@ LJLIB_CF(os_rename)
73 76
74LJLIB_CF(os_tmpname) 77LJLIB_CF(os_tmpname)
75{ 78{
76#if LJ_TARGET_PS3 || LJ_TARGET_PS4 || LJ_TARGET_PSVITA 79#if LJ_TARGET_PS3 || LJ_TARGET_PS4 || LJ_TARGET_PS5 || LJ_TARGET_PSVITA || LJ_TARGET_NX
77 lj_err_caller(L, LJ_ERR_OSUNIQF); 80 lj_err_caller(L, LJ_ERR_OSUNIQF);
78 return 0; 81 return 0;
79#else 82#else
@@ -188,7 +191,7 @@ LJLIB_CF(os_date)
188#endif 191#endif
189 } 192 }
190 if (stm == NULL) { /* Invalid date? */ 193 if (stm == NULL) { /* Invalid date? */
191 setnilV(L->top-1); 194 setnilV(L->top++);
192 } else if (strcmp(s, "*t") == 0) { 195 } else if (strcmp(s, "*t") == 0) {
193 lua_createtable(L, 0, 9); /* 9 = number of fields */ 196 lua_createtable(L, 0, 9); /* 9 = number of fields */
194 setfield(L, "sec", stm->tm_sec); 197 setfield(L, "sec", stm->tm_sec);
@@ -200,23 +203,25 @@ LJLIB_CF(os_date)
200 setfield(L, "wday", stm->tm_wday+1); 203 setfield(L, "wday", stm->tm_wday+1);
201 setfield(L, "yday", stm->tm_yday+1); 204 setfield(L, "yday", stm->tm_yday+1);
202 setboolfield(L, "isdst", stm->tm_isdst); 205 setboolfield(L, "isdst", stm->tm_isdst);
203 } else { 206 } else if (*s) {
204 char cc[3]; 207 SBuf *sb = &G(L)->tmpbuf;
205 luaL_Buffer b; 208 MSize sz = 0, retry = 4;
206 cc[0] = '%'; cc[2] = '\0'; 209 const char *q;
207 luaL_buffinit(L, &b); 210 for (q = s; *q; q++)
208 for (; *s; s++) { 211 sz += (*q == '%') ? 30 : 1; /* Overflow doesn't matter. */
209 if (*s != '%' || *(s + 1) == '\0') { /* No conversion specifier? */ 212 setsbufL(sb, L);
210 luaL_addchar(&b, *s); 213 while (retry--) { /* Limit growth for invalid format or empty result. */
211 } else { 214 char *buf = lj_buf_need(sb, sz);
212 size_t reslen; 215 size_t len = strftime(buf, sbufsz(sb), s, stm);
213 char buff[200]; /* Should be big enough for any conversion result. */ 216 if (len) {
214 cc[1] = *(++s); 217 setstrV(L, L->top++, lj_str_new(L, buf, len));
215 reslen = strftime(buff, sizeof(buff), cc, stm); 218 lj_gc_check(L);
216 luaL_addlstring(&b, buff, reslen); 219 break;
217 } 220 }
221 sz += (sz|1);
218 } 222 }
219 luaL_pushresult(&b); 223 } else {
224 setstrV(L, L->top++, &G(L)->strempty);
220 } 225 }
221 return 1; 226 return 1;
222} 227}
diff --git a/src/lib_package.c b/src/lib_package.c
index 6ec763a5..7e24afa5 100644
--- a/src/lib_package.c
+++ b/src/lib_package.c
@@ -76,6 +76,20 @@ static const char *ll_bcsym(void *lib, const char *sym)
76BOOL WINAPI GetModuleHandleExA(DWORD, LPCSTR, HMODULE*); 76BOOL WINAPI GetModuleHandleExA(DWORD, LPCSTR, HMODULE*);
77#endif 77#endif
78 78
79#if LJ_TARGET_UWP
80void *LJ_WIN_LOADLIBA(const char *path)
81{
82 DWORD err = GetLastError();
83 wchar_t wpath[256];
84 HANDLE lib = NULL;
85 if (MultiByteToWideChar(CP_ACP, 0, path, -1, wpath, 256) > 0) {
86 lib = LoadPackagedLibrary(wpath, 0);
87 }
88 SetLastError(err);
89 return lib;
90}
91#endif
92
79#undef setprogdir 93#undef setprogdir
80 94
81static void setprogdir(lua_State *L) 95static void setprogdir(lua_State *L)
@@ -96,9 +110,17 @@ static void setprogdir(lua_State *L)
96static void pusherror(lua_State *L) 110static void pusherror(lua_State *L)
97{ 111{
98 DWORD error = GetLastError(); 112 DWORD error = GetLastError();
113#if LJ_TARGET_XBOXONE
114 wchar_t wbuffer[128];
115 char buffer[128*2];
116 if (FormatMessageW(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM,
117 NULL, error, 0, wbuffer, sizeof(wbuffer)/sizeof(wchar_t), NULL) &&
118 WideCharToMultiByte(CP_ACP, 0, wbuffer, 128, buffer, 128*2, NULL, NULL))
119#else
99 char buffer[128]; 120 char buffer[128];
100 if (FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM, 121 if (FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM,
101 NULL, error, 0, buffer, sizeof(buffer), NULL)) 122 NULL, error, 0, buffer, sizeof(buffer), NULL))
123#endif
102 lua_pushstring(L, buffer); 124 lua_pushstring(L, buffer);
103 else 125 else
104 lua_pushfstring(L, "system error %d\n", error); 126 lua_pushfstring(L, "system error %d\n", error);
@@ -111,7 +133,7 @@ static void ll_unloadlib(void *lib)
111 133
112static void *ll_load(lua_State *L, const char *path, int gl) 134static void *ll_load(lua_State *L, const char *path, int gl)
113{ 135{
114 HINSTANCE lib = LoadLibraryA(path); 136 HINSTANCE lib = LJ_WIN_LOADLIBA(path);
115 if (lib == NULL) pusherror(L); 137 if (lib == NULL) pusherror(L);
116 UNUSED(gl); 138 UNUSED(gl);
117 return lib; 139 return lib;
@@ -124,17 +146,25 @@ static lua_CFunction ll_sym(lua_State *L, void *lib, const char *sym)
124 return f; 146 return f;
125} 147}
126 148
149#if LJ_TARGET_UWP
150EXTERN_C IMAGE_DOS_HEADER __ImageBase;
151#endif
152
127static const char *ll_bcsym(void *lib, const char *sym) 153static const char *ll_bcsym(void *lib, const char *sym)
128{ 154{
129 if (lib) { 155 if (lib) {
130 return (const char *)GetProcAddress((HINSTANCE)lib, sym); 156 return (const char *)GetProcAddress((HINSTANCE)lib, sym);
131 } else { 157 } else {
158#if LJ_TARGET_UWP
159 return (const char *)GetProcAddress((HINSTANCE)&__ImageBase, sym);
160#else
132 HINSTANCE h = GetModuleHandleA(NULL); 161 HINSTANCE h = GetModuleHandleA(NULL);
133 const char *p = (const char *)GetProcAddress(h, sym); 162 const char *p = (const char *)GetProcAddress(h, sym);
134 if (p == NULL && GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS|GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, 163 if (p == NULL && GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS|GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
135 (const char *)ll_bcsym, &h)) 164 (const char *)ll_bcsym, &h))
136 p = (const char *)GetProcAddress(h, sym); 165 p = (const char *)GetProcAddress(h, sym);
137 return p; 166 return p;
167#endif
138 } 168 }
139} 169}
140 170
@@ -185,8 +215,7 @@ static void **ll_register(lua_State *L, const char *path)
185 lua_pop(L, 1); 215 lua_pop(L, 1);
186 plib = (void **)lua_newuserdata(L, sizeof(void *)); 216 plib = (void **)lua_newuserdata(L, sizeof(void *));
187 *plib = NULL; 217 *plib = NULL;
188 luaL_getmetatable(L, "_LOADLIB"); 218 luaL_setmetatable(L, "_LOADLIB");
189 lua_setmetatable(L, -2);
190 lua_pushfstring(L, "LOADLIB: %s", path); 219 lua_pushfstring(L, "LOADLIB: %s", path);
191 lua_pushvalue(L, -2); 220 lua_pushvalue(L, -2);
192 lua_settable(L, LUA_REGISTRYINDEX); 221 lua_settable(L, LUA_REGISTRYINDEX);
@@ -396,8 +425,7 @@ static int lj_cf_package_loader_preload(lua_State *L)
396 425
397/* ------------------------------------------------------------------------ */ 426/* ------------------------------------------------------------------------ */
398 427
399static const int sentinel_ = 0; 428#define KEY_SENTINEL (U64x(80000000,00000000)|'s')
400#define sentinel ((void *)&sentinel_)
401 429
402static int lj_cf_package_require(lua_State *L) 430static int lj_cf_package_require(lua_State *L)
403{ 431{
@@ -407,7 +435,7 @@ static int lj_cf_package_require(lua_State *L)
407 lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); 435 lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED");
408 lua_getfield(L, 2, name); 436 lua_getfield(L, 2, name);
409 if (lua_toboolean(L, -1)) { /* is it there? */ 437 if (lua_toboolean(L, -1)) { /* is it there? */
410 if (lua_touserdata(L, -1) == sentinel) /* check loops */ 438 if ((L->top-1)->u64 == KEY_SENTINEL) /* check loops */
411 luaL_error(L, "loop or previous error loading module " LUA_QS, name); 439 luaL_error(L, "loop or previous error loading module " LUA_QS, name);
412 return 1; /* package is already loaded */ 440 return 1; /* package is already loaded */
413 } 441 }
@@ -430,14 +458,14 @@ static int lj_cf_package_require(lua_State *L)
430 else 458 else
431 lua_pop(L, 1); 459 lua_pop(L, 1);
432 } 460 }
433 lua_pushlightuserdata(L, sentinel); 461 (L->top++)->u64 = KEY_SENTINEL;
434 lua_setfield(L, 2, name); /* _LOADED[name] = sentinel */ 462 lua_setfield(L, 2, name); /* _LOADED[name] = sentinel */
435 lua_pushstring(L, name); /* pass name as argument to module */ 463 lua_pushstring(L, name); /* pass name as argument to module */
436 lua_call(L, 1, 1); /* run loaded module */ 464 lua_call(L, 1, 1); /* run loaded module */
437 if (!lua_isnil(L, -1)) /* non-nil return? */ 465 if (!lua_isnil(L, -1)) /* non-nil return? */
438 lua_setfield(L, 2, name); /* _LOADED[name] = returned value */ 466 lua_setfield(L, 2, name); /* _LOADED[name] = returned value */
439 lua_getfield(L, 2, name); 467 lua_getfield(L, 2, name);
440 if (lua_touserdata(L, -1) == sentinel) { /* module did not set a value? */ 468 if ((L->top-1)->u64 == KEY_SENTINEL) { /* module did not set a value? */
441 lua_pushboolean(L, 1); /* use true as result */ 469 lua_pushboolean(L, 1); /* use true as result */
442 lua_pushvalue(L, -1); /* extra copy to be returned */ 470 lua_pushvalue(L, -1); /* extra copy to be returned */
443 lua_setfield(L, 2, name); /* _LOADED[name] = true */ 471 lua_setfield(L, 2, name); /* _LOADED[name] = true */
@@ -487,29 +515,19 @@ static void modinit(lua_State *L, const char *modname)
487static int lj_cf_package_module(lua_State *L) 515static int lj_cf_package_module(lua_State *L)
488{ 516{
489 const char *modname = luaL_checkstring(L, 1); 517 const char *modname = luaL_checkstring(L, 1);
490 int loaded = lua_gettop(L) + 1; /* index of _LOADED table */ 518 int lastarg = (int)(L->top - L->base);
491 lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); 519 luaL_pushmodule(L, modname, 1);
492 lua_getfield(L, loaded, modname); /* get _LOADED[modname] */
493 if (!lua_istable(L, -1)) { /* not found? */
494 lua_pop(L, 1); /* remove previous result */
495 /* try global variable (and create one if it does not exist) */
496 if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, 1) != NULL)
497 lj_err_callerv(L, LJ_ERR_BADMODN, modname);
498 lua_pushvalue(L, -1);
499 lua_setfield(L, loaded, modname); /* _LOADED[modname] = new table */
500 }
501 /* check whether table already has a _NAME field */
502 lua_getfield(L, -1, "_NAME"); 520 lua_getfield(L, -1, "_NAME");
503 if (!lua_isnil(L, -1)) { /* is table an initialized module? */ 521 if (!lua_isnil(L, -1)) { /* Module already initialized? */
504 lua_pop(L, 1); 522 lua_pop(L, 1);
505 } else { /* no; initialize it */ 523 } else {
506 lua_pop(L, 1); 524 lua_pop(L, 1);
507 modinit(L, modname); 525 modinit(L, modname);
508 } 526 }
509 lua_pushvalue(L, -1); 527 lua_pushvalue(L, -1);
510 setfenv(L); 528 setfenv(L);
511 dooptions(L, loaded - 1); 529 dooptions(L, lastarg);
512 return 0; 530 return LJ_52;
513} 531}
514 532
515static int lj_cf_package_seeall(lua_State *L) 533static int lj_cf_package_seeall(lua_State *L)
@@ -580,13 +598,16 @@ LUALIB_API int luaopen_package(lua_State *L)
580 lj_lib_pushcf(L, lj_cf_package_unloadlib, 1); 598 lj_lib_pushcf(L, lj_cf_package_unloadlib, 1);
581 lua_setfield(L, -2, "__gc"); 599 lua_setfield(L, -2, "__gc");
582 luaL_register(L, LUA_LOADLIBNAME, package_lib); 600 luaL_register(L, LUA_LOADLIBNAME, package_lib);
583 lua_pushvalue(L, -1); 601 lua_copy(L, -1, LUA_ENVIRONINDEX);
584 lua_replace(L, LUA_ENVIRONINDEX);
585 lua_createtable(L, sizeof(package_loaders)/sizeof(package_loaders[0])-1, 0); 602 lua_createtable(L, sizeof(package_loaders)/sizeof(package_loaders[0])-1, 0);
586 for (i = 0; package_loaders[i] != NULL; i++) { 603 for (i = 0; package_loaders[i] != NULL; i++) {
587 lj_lib_pushcf(L, package_loaders[i], 1); 604 lj_lib_pushcf(L, package_loaders[i], 1);
588 lua_rawseti(L, -2, i+1); 605 lua_rawseti(L, -2, i+1);
589 } 606 }
607#if LJ_52
608 lua_pushvalue(L, -1);
609 lua_setfield(L, -3, "searchers");
610#endif
590 lua_setfield(L, -2, "loaders"); 611 lua_setfield(L, -2, "loaders");
591 lua_getfield(L, LUA_REGISTRYINDEX, "LUA_NOENV"); 612 lua_getfield(L, LUA_REGISTRYINDEX, "LUA_NOENV");
592 noenv = lua_toboolean(L, -1); 613 noenv = lua_toboolean(L, -1);
diff --git a/src/lib_string.c b/src/lib_string.c
index 3e3653b6..29bcb8fe 100644
--- a/src/lib_string.c
+++ b/src/lib_string.c
@@ -6,8 +6,6 @@
6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h 6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7*/ 7*/
8 8
9#include <stdio.h>
10
11#define lib_string_c 9#define lib_string_c
12#define LUA_LIB 10#define LUA_LIB
13 11
@@ -18,6 +16,7 @@
18#include "lj_obj.h" 16#include "lj_obj.h"
19#include "lj_gc.h" 17#include "lj_gc.h"
20#include "lj_err.h" 18#include "lj_err.h"
19#include "lj_buf.h"
21#include "lj_str.h" 20#include "lj_str.h"
22#include "lj_tab.h" 21#include "lj_tab.h"
23#include "lj_meta.h" 22#include "lj_meta.h"
@@ -25,17 +24,19 @@
25#include "lj_ff.h" 24#include "lj_ff.h"
26#include "lj_bcdump.h" 25#include "lj_bcdump.h"
27#include "lj_char.h" 26#include "lj_char.h"
27#include "lj_strfmt.h"
28#include "lj_lib.h" 28#include "lj_lib.h"
29 29
30/* ------------------------------------------------------------------------ */ 30/* ------------------------------------------------------------------------ */
31 31
32#define LJLIB_MODULE_string 32#define LJLIB_MODULE_string
33 33
34LJLIB_ASM(string_len) LJLIB_REC(.) 34LJLIB_LUA(string_len) /*
35{ 35 function(s)
36 lj_lib_checkstr(L, 1); 36 CHECK_str(s)
37 return FFH_RETRY; 37 return #s
38} 38 end
39*/
39 40
40LJLIB_ASM(string_byte) LJLIB_REC(string_range 0) 41LJLIB_ASM(string_byte) LJLIB_REC(string_range 0)
41{ 42{
@@ -57,21 +58,21 @@ LJLIB_ASM(string_byte) LJLIB_REC(string_range 0)
57 lj_state_checkstack(L, (MSize)n); 58 lj_state_checkstack(L, (MSize)n);
58 p = (const unsigned char *)strdata(s) + start; 59 p = (const unsigned char *)strdata(s) + start;
59 for (i = 0; i < n; i++) 60 for (i = 0; i < n; i++)
60 setintV(L->base + i-1, p[i]); 61 setintV(L->base + i-1-LJ_FR2, p[i]);
61 return FFH_RES(n); 62 return FFH_RES(n);
62} 63}
63 64
64LJLIB_ASM(string_char) 65LJLIB_ASM(string_char) LJLIB_REC(.)
65{ 66{
66 int i, nargs = (int)(L->top - L->base); 67 int i, nargs = (int)(L->top - L->base);
67 char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, (MSize)nargs); 68 char *buf = lj_buf_tmp(L, (MSize)nargs);
68 for (i = 1; i <= nargs; i++) { 69 for (i = 1; i <= nargs; i++) {
69 int32_t k = lj_lib_checkint(L, i); 70 int32_t k = lj_lib_checkint(L, i);
70 if (!checku8(k)) 71 if (!checku8(k))
71 lj_err_arg(L, i, LJ_ERR_BADVAL); 72 lj_err_arg(L, i, LJ_ERR_BADVAL);
72 buf[i-1] = (char)k; 73 buf[i-1] = (char)k;
73 } 74 }
74 setstrV(L, L->base-1, lj_str_new(L, buf, (size_t)nargs)); 75 setstrV(L, L->base-1-LJ_FR2, lj_str_new(L, buf, (size_t)nargs));
75 return FFH_RES(1); 76 return FFH_RES(1);
76} 77}
77 78
@@ -83,68 +84,38 @@ LJLIB_ASM(string_sub) LJLIB_REC(string_range 1)
83 return FFH_RETRY; 84 return FFH_RETRY;
84} 85}
85 86
86LJLIB_ASM(string_rep) 87LJLIB_CF(string_rep) LJLIB_REC(.)
87{ 88{
88 GCstr *s = lj_lib_checkstr(L, 1); 89 GCstr *s = lj_lib_checkstr(L, 1);
89 int32_t k = lj_lib_checkint(L, 2); 90 int32_t rep = lj_lib_checkint(L, 2);
90 GCstr *sep = lj_lib_optstr(L, 3); 91 GCstr *sep = lj_lib_optstr(L, 3);
91 int32_t len = (int32_t)s->len; 92 SBuf *sb = lj_buf_tmp_(L);
92 global_State *g = G(L); 93 if (sep && rep > 1) {
93 int64_t tlen; 94 GCstr *s2 = lj_buf_cat2str(L, sep, s);
94 const char *src; 95 lj_buf_reset(sb);
95 char *buf; 96 lj_buf_putstr(sb, s);
96 if (k <= 0) { 97 s = s2;
97 empty: 98 rep--;
98 setstrV(L, L->base-1, &g->strempty);
99 return FFH_RES(1);
100 }
101 if (sep) {
102 tlen = (int64_t)len + sep->len;
103 if (tlen > LJ_MAX_STR)
104 lj_err_caller(L, LJ_ERR_STROV);
105 tlen *= k;
106 if (tlen > LJ_MAX_STR)
107 lj_err_caller(L, LJ_ERR_STROV);
108 } else {
109 tlen = (int64_t)k * len;
110 if (tlen > LJ_MAX_STR)
111 lj_err_caller(L, LJ_ERR_STROV);
112 }
113 if (tlen == 0) goto empty;
114 buf = lj_str_needbuf(L, &g->tmpbuf, (MSize)tlen);
115 src = strdata(s);
116 if (sep) {
117 tlen -= sep->len; /* Ignore trailing separator. */
118 if (k > 1) { /* Paste one string and one separator. */
119 int32_t i;
120 i = 0; while (i < len) *buf++ = src[i++];
121 src = strdata(sep); len = sep->len;
122 i = 0; while (i < len) *buf++ = src[i++];
123 src = g->tmpbuf.buf; len += s->len; k--; /* Now copy that k-1 times. */
124 }
125 } 99 }
126 do { 100 sb = lj_buf_putstr_rep(sb, s, rep);
127 int32_t i = 0; 101 setstrV(L, L->top-1, lj_buf_str(L, sb));
128 do { *buf++ = src[i++]; } while (i < len); 102 lj_gc_check(L);
129 } while (--k > 0); 103 return 1;
130 setstrV(L, L->base-1, lj_str_new(L, g->tmpbuf.buf, (size_t)tlen));
131 return FFH_RES(1);
132} 104}
133 105
134LJLIB_ASM(string_reverse) 106LJLIB_ASM(string_reverse) LJLIB_REC(string_op IRCALL_lj_buf_putstr_reverse)
135{ 107{
136 GCstr *s = lj_lib_checkstr(L, 1); 108 lj_lib_checkstr(L, 1);
137 lj_str_needbuf(L, &G(L)->tmpbuf, s->len);
138 return FFH_RETRY; 109 return FFH_RETRY;
139} 110}
140LJLIB_ASM_(string_lower) 111LJLIB_ASM_(string_lower) LJLIB_REC(string_op IRCALL_lj_buf_putstr_lower)
141LJLIB_ASM_(string_upper) 112LJLIB_ASM_(string_upper) LJLIB_REC(string_op IRCALL_lj_buf_putstr_upper)
142 113
143/* ------------------------------------------------------------------------ */ 114/* ------------------------------------------------------------------------ */
144 115
145static int writer_buf(lua_State *L, const void *p, size_t size, void *b) 116static int writer_buf(lua_State *L, const void *p, size_t size, void *sb)
146{ 117{
147 luaL_addlstring((luaL_Buffer *)b, (const char *)p, size); 118 lj_buf_putmem((SBuf *)sb, p, (MSize)size);
148 UNUSED(L); 119 UNUSED(L);
149 return 0; 120 return 0;
150} 121}
@@ -153,19 +124,19 @@ LJLIB_CF(string_dump)
153{ 124{
154 GCfunc *fn = lj_lib_checkfunc(L, 1); 125 GCfunc *fn = lj_lib_checkfunc(L, 1);
155 int strip = L->base+1 < L->top && tvistruecond(L->base+1); 126 int strip = L->base+1 < L->top && tvistruecond(L->base+1);
156 luaL_Buffer b; 127 SBuf *sb = lj_buf_tmp_(L); /* Assumes lj_bcwrite() doesn't use tmpbuf. */
157 L->top = L->base+1; 128 L->top = L->base+1;
158 luaL_buffinit(L, &b); 129 if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, sb, strip))
159 if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, &b, strip))
160 lj_err_caller(L, LJ_ERR_STRDUMP); 130 lj_err_caller(L, LJ_ERR_STRDUMP);
161 luaL_pushresult(&b); 131 setstrV(L, L->top-1, lj_buf_str(L, sb));
132 lj_gc_check(L);
162 return 1; 133 return 1;
163} 134}
164 135
165/* ------------------------------------------------------------------------ */ 136/* ------------------------------------------------------------------------ */
166 137
167/* macro to `unsign' a character */ 138/* macro to `unsign' a character */
168#define uchar(c) ((unsigned char)(c)) 139#define uchar(c) ((unsigned char)(c))
169 140
170#define CAP_UNFINISHED (-1) 141#define CAP_UNFINISHED (-1)
171#define CAP_POSITION (-2) 142#define CAP_POSITION (-2)
@@ -183,7 +154,6 @@ typedef struct MatchState {
183} MatchState; 154} MatchState;
184 155
185#define L_ESC '%' 156#define L_ESC '%'
186#define SPECIALS "^$*+?.([%-"
187 157
188static int check_capture(MatchState *ms, int l) 158static int check_capture(MatchState *ms, int l)
189{ 159{
@@ -450,30 +420,6 @@ static const char *match(MatchState *ms, const char *s, const char *p)
450 return s; 420 return s;
451} 421}
452 422
453static const char *lmemfind(const char *s1, size_t l1,
454 const char *s2, size_t l2)
455{
456 if (l2 == 0) {
457 return s1; /* empty strings are everywhere */
458 } else if (l2 > l1) {
459 return NULL; /* avoids a negative `l1' */
460 } else {
461 const char *init; /* to search for a `*s2' inside `s1' */
462 l2--; /* 1st char will be checked by `memchr' */
463 l1 = l1-l2; /* `s2' cannot be found after that */
464 while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) {
465 init++; /* 1st char is already checked */
466 if (memcmp(init, s2+1, l2) == 0) {
467 return init-1;
468 } else { /* correct `l1' and `s1' to try again */
469 l1 -= (size_t)(init-s1);
470 s1 = init;
471 }
472 }
473 return NULL; /* not found */
474 }
475}
476
477static void push_onecapture(MatchState *ms, int i, const char *s, const char *e) 423static void push_onecapture(MatchState *ms, int i, const char *s, const char *e)
478{ 424{
479 if (i >= ms->level) { 425 if (i >= ms->level) {
@@ -501,64 +447,60 @@ static int push_captures(MatchState *ms, const char *s, const char *e)
501 return nlevels; /* number of strings pushed */ 447 return nlevels; /* number of strings pushed */
502} 448}
503 449
504static ptrdiff_t posrelat(ptrdiff_t pos, size_t len)
505{
506 /* relative string position: negative means back from end */
507 if (pos < 0) pos += (ptrdiff_t)len + 1;
508 return (pos >= 0) ? pos : 0;
509}
510
511static int str_find_aux(lua_State *L, int find) 450static int str_find_aux(lua_State *L, int find)
512{ 451{
513 size_t l1, l2; 452 GCstr *s = lj_lib_checkstr(L, 1);
514 const char *s = luaL_checklstring(L, 1, &l1); 453 GCstr *p = lj_lib_checkstr(L, 2);
515 const char *p = luaL_checklstring(L, 2, &l2); 454 int32_t start = lj_lib_optint(L, 3, 1);
516 ptrdiff_t init = posrelat(luaL_optinteger(L, 3, 1), l1) - 1; 455 MSize st;
517 if (init < 0) { 456 if (start < 0) start += (int32_t)s->len; else start--;
518 init = 0; 457 if (start < 0) start = 0;
519 } else if ((size_t)(init) > l1) { 458 st = (MSize)start;
459 if (st > s->len) {
520#if LJ_52 460#if LJ_52
521 setnilV(L->top-1); 461 setnilV(L->top-1);
522 return 1; 462 return 1;
523#else 463#else
524 init = (ptrdiff_t)l1; 464 st = s->len;
525#endif 465#endif
526 } 466 }
527 if (find && (lua_toboolean(L, 4) || /* explicit request? */ 467 if (find && ((L->base+3 < L->top && tvistruecond(L->base+3)) ||
528 strpbrk(p, SPECIALS) == NULL)) { /* or no special characters? */ 468 !lj_str_haspattern(p))) { /* Search for fixed string. */
529 /* do a plain search */ 469 const char *q = lj_str_find(strdata(s)+st, strdata(p), s->len-st, p->len);
530 const char *s2 = lmemfind(s+init, l1-(size_t)init, p, l2); 470 if (q) {
531 if (s2) { 471 setintV(L->top-2, (int32_t)(q-strdata(s)) + 1);
532 lua_pushinteger(L, s2-s+1); 472 setintV(L->top-1, (int32_t)(q-strdata(s)) + (int32_t)p->len);
533 lua_pushinteger(L, s2-s+(ptrdiff_t)l2);
534 return 2; 473 return 2;
535 } 474 }
536 } else { 475 } else { /* Search for pattern. */
537 MatchState ms; 476 MatchState ms;
538 int anchor = (*p == '^') ? (p++, 1) : 0; 477 const char *pstr = strdata(p);
539 const char *s1=s+init; 478 const char *sstr = strdata(s) + st;
479 int anchor = 0;
480 if (*pstr == '^') { pstr++; anchor = 1; }
540 ms.L = L; 481 ms.L = L;
541 ms.src_init = s; 482 ms.src_init = strdata(s);
542 ms.src_end = s+l1; 483 ms.src_end = strdata(s) + s->len;
543 do { 484 do { /* Loop through string and try to match the pattern. */
544 const char *res; 485 const char *q;
545 ms.level = ms.depth = 0; 486 ms.level = ms.depth = 0;
546 if ((res=match(&ms, s1, p)) != NULL) { 487 q = match(&ms, sstr, pstr);
488 if (q) {
547 if (find) { 489 if (find) {
548 lua_pushinteger(L, s1-s+1); /* start */ 490 setintV(L->top++, (int32_t)(sstr-(strdata(s)-1)));
549 lua_pushinteger(L, res-s); /* end */ 491 setintV(L->top++, (int32_t)(q-strdata(s)));
550 return push_captures(&ms, NULL, 0) + 2; 492 return push_captures(&ms, NULL, NULL) + 2;
551 } else { 493 } else {
552 return push_captures(&ms, s1, res); 494 return push_captures(&ms, sstr, q);
553 } 495 }
554 } 496 }
555 } while (s1++ < ms.src_end && !anchor); 497 } while (sstr++ < ms.src_end && !anchor);
556 } 498 }
557 lua_pushnil(L); /* not found */ 499 setnilV(L->top-1); /* Not found. */
558 return 1; 500 return 1;
559} 501}
560 502
561LJLIB_CF(string_find) 503LJLIB_CF(string_find) LJLIB_REC(.)
562{ 504{
563 return str_find_aux(L, 1); 505 return str_find_aux(L, 1);
564} 506}
@@ -698,222 +640,16 @@ LJLIB_CF(string_gsub)
698 640
699/* ------------------------------------------------------------------------ */ 641/* ------------------------------------------------------------------------ */
700 642
701/* maximum size of each formatted item (> len(format('%99.99f', -1e308))) */ 643LJLIB_CF(string_format) LJLIB_REC(.)
702#define MAX_FMTITEM 512
703/* valid flags in a format specification */
704#define FMT_FLAGS "-+ #0"
705/*
706** maximum size of each format specification (such as '%-099.99d')
707** (+10 accounts for %99.99x plus margin of error)
708*/
709#define MAX_FMTSPEC (sizeof(FMT_FLAGS) + sizeof(LUA_INTFRMLEN) + 10)
710
711static void addquoted(lua_State *L, luaL_Buffer *b, int arg)
712{
713 GCstr *str = lj_lib_checkstr(L, arg);
714 int32_t len = (int32_t)str->len;
715 const char *s = strdata(str);
716 luaL_addchar(b, '"');
717 while (len--) {
718 uint32_t c = uchar(*s);
719 if (c == '"' || c == '\\' || c == '\n') {
720 luaL_addchar(b, '\\');
721 } else if (lj_char_iscntrl(c)) { /* This can only be 0-31 or 127. */
722 uint32_t d;
723 luaL_addchar(b, '\\');
724 if (c >= 100 || lj_char_isdigit(uchar(s[1]))) {
725 luaL_addchar(b, '0'+(c >= 100)); if (c >= 100) c -= 100;
726 goto tens;
727 } else if (c >= 10) {
728 tens:
729 d = (c * 205) >> 11; c -= d * 10; luaL_addchar(b, '0'+d);
730 }
731 c += '0';
732 }
733 luaL_addchar(b, c);
734 s++;
735 }
736 luaL_addchar(b, '"');
737}
738
739static const char *scanformat(lua_State *L, const char *strfrmt, char *form)
740{
741 const char *p = strfrmt;
742 while (*p != '\0' && strchr(FMT_FLAGS, *p) != NULL) p++; /* skip flags */
743 if ((size_t)(p - strfrmt) >= sizeof(FMT_FLAGS))
744 lj_err_caller(L, LJ_ERR_STRFMTR);
745 if (lj_char_isdigit(uchar(*p))) p++; /* skip width */
746 if (lj_char_isdigit(uchar(*p))) p++; /* (2 digits at most) */
747 if (*p == '.') {
748 p++;
749 if (lj_char_isdigit(uchar(*p))) p++; /* skip precision */
750 if (lj_char_isdigit(uchar(*p))) p++; /* (2 digits at most) */
751 }
752 if (lj_char_isdigit(uchar(*p)))
753 lj_err_caller(L, LJ_ERR_STRFMTW);
754 *(form++) = '%';
755 strncpy(form, strfrmt, (size_t)(p - strfrmt + 1));
756 form += p - strfrmt + 1;
757 *form = '\0';
758 return p;
759}
760
761static void addintlen(char *form)
762{
763 size_t l = strlen(form);
764 char spec = form[l - 1];
765 strcpy(form + l - 1, LUA_INTFRMLEN);
766 form[l + sizeof(LUA_INTFRMLEN) - 2] = spec;
767 form[l + sizeof(LUA_INTFRMLEN) - 1] = '\0';
768}
769
770static unsigned LUA_INTFRM_T num2intfrm(lua_State *L, int arg)
771{
772 if (sizeof(LUA_INTFRM_T) == 4) {
773 return (LUA_INTFRM_T)lj_lib_checkbit(L, arg);
774 } else {
775 cTValue *o;
776 lj_lib_checknumber(L, arg);
777 o = L->base+arg-1;
778 if (tvisint(o))
779 return (LUA_INTFRM_T)intV(o);
780 else
781 return (LUA_INTFRM_T)numV(o);
782 }
783}
784
785static unsigned LUA_INTFRM_T num2uintfrm(lua_State *L, int arg)
786{ 644{
787 if (sizeof(LUA_INTFRM_T) == 4) { 645 int retry = 0;
788 return (unsigned LUA_INTFRM_T)lj_lib_checkbit(L, arg); 646 SBuf *sb;
789 } else { 647 do {
790 cTValue *o; 648 sb = lj_buf_tmp_(L);
791 lj_lib_checknumber(L, arg); 649 retry = lj_strfmt_putarg(L, sb, 1, -retry);
792 o = L->base+arg-1; 650 } while (retry > 0);
793 if (tvisint(o)) 651 setstrV(L, L->top-1, lj_buf_str(L, sb));
794 return (unsigned LUA_INTFRM_T)intV(o); 652 lj_gc_check(L);
795 else if ((int32_t)o->u32.hi < 0)
796 return (unsigned LUA_INTFRM_T)(LUA_INTFRM_T)numV(o);
797 else
798 return (unsigned LUA_INTFRM_T)numV(o);
799 }
800}
801
802static GCstr *meta_tostring(lua_State *L, int arg)
803{
804 TValue *o = L->base+arg-1;
805 cTValue *mo;
806 lua_assert(o < L->top); /* Caller already checks for existence. */
807 if (LJ_LIKELY(tvisstr(o)))
808 return strV(o);
809 if (!tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
810 copyTV(L, L->top++, mo);
811 copyTV(L, L->top++, o);
812 lua_call(L, 1, 1);
813 L->top--;
814 if (tvisstr(L->top))
815 return strV(L->top);
816 o = L->base+arg-1;
817 copyTV(L, o, L->top);
818 }
819 if (tvisnumber(o)) {
820 return lj_str_fromnumber(L, o);
821 } else if (tvisnil(o)) {
822 return lj_str_newlit(L, "nil");
823 } else if (tvisfalse(o)) {
824 return lj_str_newlit(L, "false");
825 } else if (tvistrue(o)) {
826 return lj_str_newlit(L, "true");
827 } else {
828 if (tvisfunc(o) && isffunc(funcV(o)))
829 lj_str_pushf(L, "function: builtin#%d", funcV(o)->c.ffid);
830 else
831 lj_str_pushf(L, "%s: %p", lj_typename(o), lua_topointer(L, arg));
832 L->top--;
833 return strV(L->top);
834 }
835}
836
837LJLIB_CF(string_format)
838{
839 int arg = 1, top = (int)(L->top - L->base);
840 GCstr *fmt = lj_lib_checkstr(L, arg);
841 const char *strfrmt = strdata(fmt);
842 const char *strfrmt_end = strfrmt + fmt->len;
843 luaL_Buffer b;
844 luaL_buffinit(L, &b);
845 while (strfrmt < strfrmt_end) {
846 if (*strfrmt != L_ESC) {
847 luaL_addchar(&b, *strfrmt++);
848 } else if (*++strfrmt == L_ESC) {
849 luaL_addchar(&b, *strfrmt++); /* %% */
850 } else { /* format item */
851 char form[MAX_FMTSPEC]; /* to store the format (`%...') */
852 char buff[MAX_FMTITEM]; /* to store the formatted item */
853 int n = 0;
854 if (++arg > top)
855 luaL_argerror(L, arg, lj_obj_typename[0]);
856 strfrmt = scanformat(L, strfrmt, form);
857 switch (*strfrmt++) {
858 case 'c':
859 n = sprintf(buff, form, lj_lib_checkint(L, arg));
860 break;
861 case 'd': case 'i':
862 addintlen(form);
863 n = sprintf(buff, form, num2intfrm(L, arg));
864 break;
865 case 'o': case 'u': case 'x': case 'X':
866 addintlen(form);
867 n = sprintf(buff, form, num2uintfrm(L, arg));
868 break;
869 case 'e': case 'E': case 'f': case 'g': case 'G': case 'a': case 'A': {
870 TValue tv;
871 tv.n = lj_lib_checknum(L, arg);
872 if (LJ_UNLIKELY((tv.u32.hi << 1) >= 0xffe00000)) {
873 /* Canonicalize output of non-finite values. */
874 char *p, nbuf[LJ_STR_NUMBUF];
875 size_t len = lj_str_bufnum(nbuf, &tv);
876 if (strfrmt[-1] < 'a') {
877 nbuf[len-3] = nbuf[len-3] - 0x20;
878 nbuf[len-2] = nbuf[len-2] - 0x20;
879 nbuf[len-1] = nbuf[len-1] - 0x20;
880 }
881 nbuf[len] = '\0';
882 for (p = form; *p < 'A' && *p != '.'; p++) ;
883 *p++ = 's'; *p = '\0';
884 n = sprintf(buff, form, nbuf);
885 break;
886 }
887 n = sprintf(buff, form, (double)tv.n);
888 break;
889 }
890 case 'q':
891 addquoted(L, &b, arg);
892 continue;
893 case 'p':
894 lj_str_pushf(L, "%p", lua_topointer(L, arg));
895 luaL_addvalue(&b);
896 continue;
897 case 's': {
898 GCstr *str = meta_tostring(L, arg);
899 if (!strchr(form, '.') && str->len >= 100) {
900 /* no precision and string is too long to be formatted;
901 keep original string */
902 setstrV(L, L->top++, str);
903 luaL_addvalue(&b);
904 continue;
905 }
906 n = sprintf(buff, form, strdata(str));
907 break;
908 }
909 default:
910 lj_err_callerv(L, LJ_ERR_STRFMTO, *(strfrmt -1));
911 break;
912 }
913 luaL_addlstring(&b, buff, n);
914 }
915 }
916 luaL_pushresult(&b);
917 return 1; 653 return 1;
918} 654}
919 655
@@ -926,16 +662,15 @@ LUALIB_API int luaopen_string(lua_State *L)
926 GCtab *mt; 662 GCtab *mt;
927 global_State *g; 663 global_State *g;
928 LJ_LIB_REG(L, LUA_STRLIBNAME, string); 664 LJ_LIB_REG(L, LUA_STRLIBNAME, string);
929#if defined(LUA_COMPAT_GFIND) && !LJ_52
930 lua_getfield(L, -1, "gmatch");
931 lua_setfield(L, -2, "gfind");
932#endif
933 mt = lj_tab_new(L, 0, 1); 665 mt = lj_tab_new(L, 0, 1);
934 /* NOBARRIER: basemt is a GC root. */ 666 /* NOBARRIER: basemt is a GC root. */
935 g = G(L); 667 g = G(L);
936 setgcref(basemt_it(g, LJ_TSTR), obj2gco(mt)); 668 setgcref(basemt_it(g, LJ_TSTR), obj2gco(mt));
937 settabV(L, lj_tab_setstr(L, mt, mmname_str(g, MM_index)), tabV(L->top-1)); 669 settabV(L, lj_tab_setstr(L, mt, mmname_str(g, MM_index)), tabV(L->top-1));
938 mt->nomm = (uint8_t)(~(1u<<MM_index)); 670 mt->nomm = (uint8_t)(~(1u<<MM_index));
671#if LJ_HASBUFFER
672 lj_lib_prereg(L, LUA_STRLIBNAME ".buffer", luaopen_string_buffer, tabV(L->top-1));
673#endif
939 return 1; 674 return 1;
940} 675}
941 676
diff --git a/src/lib_table.c b/src/lib_table.c
index c2a1f18f..97ba2847 100644
--- a/src/lib_table.c
+++ b/src/lib_table.c
@@ -16,57 +16,43 @@
16#include "lj_obj.h" 16#include "lj_obj.h"
17#include "lj_gc.h" 17#include "lj_gc.h"
18#include "lj_err.h" 18#include "lj_err.h"
19#include "lj_buf.h"
19#include "lj_tab.h" 20#include "lj_tab.h"
21#include "lj_ff.h"
20#include "lj_lib.h" 22#include "lj_lib.h"
21 23
22/* ------------------------------------------------------------------------ */ 24/* ------------------------------------------------------------------------ */
23 25
24#define LJLIB_MODULE_table 26#define LJLIB_MODULE_table
25 27
26LJLIB_CF(table_foreachi) 28LJLIB_LUA(table_foreachi) /*
27{ 29 function(t, f)
28 GCtab *t = lj_lib_checktab(L, 1); 30 CHECK_tab(t)
29 GCfunc *func = lj_lib_checkfunc(L, 2); 31 CHECK_func(f)
30 MSize i, n = lj_tab_len(t); 32 for i=1,#t do
31 for (i = 1; i <= n; i++) { 33 local r = f(i, t[i])
32 cTValue *val; 34 if r ~= nil then return r end
33 setfuncV(L, L->top, func); 35 end
34 setintV(L->top+1, i); 36 end
35 val = lj_tab_getint(t, (int32_t)i); 37*/
36 if (val) { copyTV(L, L->top+2, val); } else { setnilV(L->top+2); }
37 L->top += 3;
38 lua_call(L, 2, 1);
39 if (!tvisnil(L->top-1))
40 return 1;
41 L->top--;
42 }
43 return 0;
44}
45 38
46LJLIB_CF(table_foreach) 39LJLIB_LUA(table_foreach) /*
47{ 40 function(t, f)
48 GCtab *t = lj_lib_checktab(L, 1); 41 CHECK_tab(t)
49 GCfunc *func = lj_lib_checkfunc(L, 2); 42 CHECK_func(f)
50 L->top = L->base+3; 43 for k, v in PAIRS(t) do
51 setnilV(L->top-1); 44 local r = f(k, v)
52 while (lj_tab_next(L, t, L->top-1)) { 45 if r ~= nil then return r end
53 copyTV(L, L->top+2, L->top); 46 end
54 copyTV(L, L->top+1, L->top-1); 47 end
55 setfuncV(L, L->top, func); 48*/
56 L->top += 3;
57 lua_call(L, 2, 1);
58 if (!tvisnil(L->top-1))
59 return 1;
60 L->top--;
61 }
62 return 0;
63}
64 49
65LJLIB_ASM(table_getn) LJLIB_REC(.) 50LJLIB_LUA(table_getn) /*
66{ 51 function(t)
67 lj_lib_checktab(L, 1); 52 CHECK_tab(t)
68 return FFH_UNREACHABLE; 53 return #t
69} 54 end
55*/
70 56
71LJLIB_CF(table_maxn) 57LJLIB_CF(table_maxn)
72{ 58{
@@ -119,52 +105,67 @@ LJLIB_CF(table_insert) LJLIB_REC(.)
119 return 0; 105 return 0;
120} 106}
121 107
122LJLIB_CF(table_remove) LJLIB_REC(.) 108LJLIB_LUA(table_remove) /*
123{ 109 function(t, pos)
124 GCtab *t = lj_lib_checktab(L, 1); 110 CHECK_tab(t)
125 int32_t e = (int32_t)lj_tab_len(t); 111 local len = #t
126 int32_t pos = lj_lib_optint(L, 2, e); 112 if pos == nil then
127 if (!(1 <= pos && pos <= e)) /* Nothing to remove? */ 113 if len ~= 0 then
128 return 0; 114 local old = t[len]
129 lua_rawgeti(L, 1, pos); /* Get previous value. */ 115 t[len] = nil
130 /* NOBARRIER: This just moves existing elements around. */ 116 return old
131 for (; pos < e; pos++) { 117 end
132 cTValue *src = lj_tab_getint(t, pos+1); 118 else
133 TValue *dst = lj_tab_setint(L, t, pos); 119 CHECK_int(pos)
134 if (src) { 120 if pos >= 1 and pos <= len then
135 copyTV(L, dst, src); 121 local old = t[pos]
136 } else { 122 for i=pos+1,len do
137 setnilV(dst); 123 t[i-1] = t[i]
138 } 124 end
139 } 125 t[len] = nil
140 setnilV(lj_tab_setint(L, t, e)); /* Remove (last) value. */ 126 return old
141 return 1; /* Return previous value. */ 127 end
142} 128 end
129 end
130*/
131
132LJLIB_LUA(table_move) /*
133 function(a1, f, e, t, a2)
134 CHECK_tab(a1)
135 CHECK_int(f)
136 CHECK_int(e)
137 CHECK_int(t)
138 if a2 == nil then a2 = a1 end
139 CHECK_tab(a2)
140 if e >= f then
141 local d = t - f
142 if t > e or t <= f or a2 ~= a1 then
143 for i=f,e do a2[i+d] = a1[i] end
144 else
145 for i=e,f,-1 do a2[i+d] = a1[i] end
146 end
147 end
148 return a2
149 end
150*/
143 151
144LJLIB_CF(table_concat) 152LJLIB_CF(table_concat) LJLIB_REC(.)
145{ 153{
146 luaL_Buffer b;
147 GCtab *t = lj_lib_checktab(L, 1); 154 GCtab *t = lj_lib_checktab(L, 1);
148 GCstr *sep = lj_lib_optstr(L, 2); 155 GCstr *sep = lj_lib_optstr(L, 2);
149 MSize seplen = sep ? sep->len : 0;
150 int32_t i = lj_lib_optint(L, 3, 1); 156 int32_t i = lj_lib_optint(L, 3, 1);
151 int32_t e = (L->base+3 < L->top && !tvisnil(L->base+3)) ? 157 int32_t e = (L->base+3 < L->top && !tvisnil(L->base+3)) ?
152 lj_lib_checkint(L, 4) : (int32_t)lj_tab_len(t); 158 lj_lib_checkint(L, 4) : (int32_t)lj_tab_len(t);
153 luaL_buffinit(L, &b); 159 SBuf *sb = lj_buf_tmp_(L);
154 if (i <= e) { 160 SBuf *sbx = lj_buf_puttab(sb, t, sep, i, e);
155 for (;;) { 161 if (LJ_UNLIKELY(!sbx)) { /* Error: bad element type. */
156 cTValue *o; 162 int32_t idx = (int32_t)(intptr_t)sb->w;
157 lua_rawgeti(L, 1, i); 163 cTValue *o = lj_tab_getint(t, idx);
158 o = L->top-1; 164 lj_err_callerv(L, LJ_ERR_TABCAT,
159 if (!(tvisstr(o) || tvisnumber(o))) 165 lj_obj_itypename[o ? itypemap(o) : ~LJ_TNIL], idx);
160 lj_err_callerv(L, LJ_ERR_TABCAT, lj_typename(o), i);
161 luaL_addvalue(&b);
162 if (i++ == e) break;
163 if (seplen)
164 luaL_addlstring(&b, strdata(sep), seplen);
165 }
166 } 166 }
167 luaL_pushresult(&b); 167 setstrV(L, L->top-1, lj_buf_str(L, sbx));
168 lj_gc_check(L);
168 return 1; 169 return 1;
169} 170}
170 171
@@ -284,6 +285,30 @@ LJLIB_CF(table_pack)
284} 285}
285#endif 286#endif
286 287
288LJLIB_NOREG LJLIB_CF(table_new) LJLIB_REC(.)
289{
290 int32_t a = lj_lib_checkint(L, 1);
291 int32_t h = lj_lib_checkint(L, 2);
292 lua_createtable(L, a, h);
293 return 1;
294}
295
296LJLIB_NOREG LJLIB_CF(table_clear) LJLIB_REC(.)
297{
298 lj_tab_clear(lj_lib_checktab(L, 1));
299 return 0;
300}
301
302static int luaopen_table_new(lua_State *L)
303{
304 return lj_lib_postreg(L, lj_cf_table_new, FF_table_new, "new");
305}
306
307static int luaopen_table_clear(lua_State *L)
308{
309 return lj_lib_postreg(L, lj_cf_table_clear, FF_table_clear, "clear");
310}
311
287/* ------------------------------------------------------------------------ */ 312/* ------------------------------------------------------------------------ */
288 313
289#include "lj_libdef.h" 314#include "lj_libdef.h"
@@ -295,6 +320,8 @@ LUALIB_API int luaopen_table(lua_State *L)
295 lua_getglobal(L, "unpack"); 320 lua_getglobal(L, "unpack");
296 lua_setfield(L, -2, "unpack"); 321 lua_setfield(L, -2, "unpack");
297#endif 322#endif
323 lj_lib_prereg(L, LUA_TABLIBNAME ".new", luaopen_table_new, tabV(L->top-1));
324 lj_lib_prereg(L, LUA_TABLIBNAME ".clear", luaopen_table_clear, tabV(L->top-1));
298 return 1; 325 return 1;
299} 326}
300 327
diff --git a/src/lj.supp b/src/lj.supp
deleted file mode 100644
index 217f7c89..00000000
--- a/src/lj.supp
+++ /dev/null
@@ -1,41 +0,0 @@
1# Valgrind suppression file for LuaJIT 2.0.
2{
3 Optimized string compare
4 Memcheck:Addr4
5 fun:lj_str_cmp
6}
7{
8 Optimized string compare
9 Memcheck:Addr1
10 fun:lj_str_cmp
11}
12{
13 Optimized string compare
14 Memcheck:Addr4
15 fun:lj_str_new
16}
17{
18 Optimized string compare
19 Memcheck:Addr1
20 fun:lj_str_new
21}
22{
23 Optimized string compare
24 Memcheck:Cond
25 fun:lj_str_new
26}
27{
28 Optimized string compare
29 Memcheck:Addr4
30 fun:str_fastcmp
31}
32{
33 Optimized string compare
34 Memcheck:Addr1
35 fun:str_fastcmp
36}
37{
38 Optimized string compare
39 Memcheck:Cond
40 fun:str_fastcmp
41}
diff --git a/src/lj_alloc.c b/src/lj_alloc.c
index 9adaa0e5..20e60493 100644
--- a/src/lj_alloc.c
+++ b/src/lj_alloc.c
@@ -31,6 +31,7 @@
31#include "lj_def.h" 31#include "lj_def.h"
32#include "lj_arch.h" 32#include "lj_arch.h"
33#include "lj_alloc.h" 33#include "lj_alloc.h"
34#include "lj_prng.h"
34 35
35#ifndef LUAJIT_USE_SYSMALLOC 36#ifndef LUAJIT_USE_SYSMALLOC
36 37
@@ -72,15 +73,58 @@
72 73
73#define IS_DIRECT_BIT (SIZE_T_ONE) 74#define IS_DIRECT_BIT (SIZE_T_ONE)
74 75
76
77/* Determine system-specific block allocation method. */
75#if LJ_TARGET_WINDOWS 78#if LJ_TARGET_WINDOWS
76 79
77#define WIN32_LEAN_AND_MEAN 80#define WIN32_LEAN_AND_MEAN
78#include <windows.h> 81#include <windows.h>
79 82
83#define LJ_ALLOC_VIRTUALALLOC 1
84
85#if LJ_64 && !LJ_GC64
86#define LJ_ALLOC_NTAVM 1
87#endif
88
89#else
90
91#include <errno.h>
92/* If this include fails, then rebuild with: -DLUAJIT_USE_SYSMALLOC */
93#include <sys/mman.h>
94
95#define LJ_ALLOC_MMAP 1
96
80#if LJ_64 97#if LJ_64
81 98
99#define LJ_ALLOC_MMAP_PROBE 1
100
101#if LJ_GC64
102#define LJ_ALLOC_MBITS 47 /* 128 TB in LJ_GC64 mode. */
103#elif LJ_TARGET_X64 && LJ_HASJIT
104/* Due to limitations in the x64 compiler backend. */
105#define LJ_ALLOC_MBITS 31 /* 2 GB on x64 with !LJ_GC64. */
106#else
107#define LJ_ALLOC_MBITS 32 /* 4 GB on other archs with !LJ_GC64. */
108#endif
109
110#endif
111
112#if LJ_64 && !LJ_GC64 && defined(MAP_32BIT)
113#define LJ_ALLOC_MMAP32 1
114#endif
115
116#if LJ_TARGET_LINUX
117#define LJ_ALLOC_MREMAP 1
118#endif
119
120#endif
121
122
123#if LJ_ALLOC_VIRTUALALLOC
124
125#if LJ_ALLOC_NTAVM
82/* Undocumented, but hey, that's what we all love so much about Windows. */ 126/* Undocumented, but hey, that's what we all love so much about Windows. */
83typedef long (*PNTAVM)(HANDLE handle, void **addr, ULONG zbits, 127typedef long (*PNTAVM)(HANDLE handle, void **addr, ULONG_PTR zbits,
84 size_t *size, ULONG alloctype, ULONG prot); 128 size_t *size, ULONG alloctype, ULONG prot);
85static PNTAVM ntavm; 129static PNTAVM ntavm;
86 130
@@ -89,14 +133,15 @@ static PNTAVM ntavm;
89*/ 133*/
90#define NTAVM_ZEROBITS 1 134#define NTAVM_ZEROBITS 1
91 135
92static void INIT_MMAP(void) 136static void init_mmap(void)
93{ 137{
94 ntavm = (PNTAVM)GetProcAddress(GetModuleHandleA("ntdll.dll"), 138 ntavm = (PNTAVM)GetProcAddress(GetModuleHandleA("ntdll.dll"),
95 "NtAllocateVirtualMemory"); 139 "NtAllocateVirtualMemory");
96} 140}
141#define INIT_MMAP() init_mmap()
97 142
98/* Win64 32 bit MMAP via NtAllocateVirtualMemory. */ 143/* Win64 32 bit MMAP via NtAllocateVirtualMemory. */
99static LJ_AINLINE void *CALL_MMAP(size_t size) 144static void *mmap_plain(size_t size)
100{ 145{
101 DWORD olderr = GetLastError(); 146 DWORD olderr = GetLastError();
102 void *ptr = NULL; 147 void *ptr = NULL;
@@ -107,7 +152,7 @@ static LJ_AINLINE void *CALL_MMAP(size_t size)
107} 152}
108 153
109/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */ 154/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
110static LJ_AINLINE void *DIRECT_MMAP(size_t size) 155static void *direct_mmap(size_t size)
111{ 156{
112 DWORD olderr = GetLastError(); 157 DWORD olderr = GetLastError();
113 void *ptr = NULL; 158 void *ptr = NULL;
@@ -119,31 +164,32 @@ static LJ_AINLINE void *DIRECT_MMAP(size_t size)
119 164
120#else 165#else
121 166
122#define INIT_MMAP() ((void)0)
123
124/* Win32 MMAP via VirtualAlloc */ 167/* Win32 MMAP via VirtualAlloc */
125static LJ_AINLINE void *CALL_MMAP(size_t size) 168static void *mmap_plain(size_t size)
126{ 169{
127 DWORD olderr = GetLastError(); 170 DWORD olderr = GetLastError();
128 void *ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); 171 void *ptr = LJ_WIN_VALLOC(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
129 SetLastError(olderr); 172 SetLastError(olderr);
130 return ptr ? ptr : MFAIL; 173 return ptr ? ptr : MFAIL;
131} 174}
132 175
133/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */ 176/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
134static LJ_AINLINE void *DIRECT_MMAP(size_t size) 177static void *direct_mmap(size_t size)
135{ 178{
136 DWORD olderr = GetLastError(); 179 DWORD olderr = GetLastError();
137 void *ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, 180 void *ptr = LJ_WIN_VALLOC(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
138 PAGE_READWRITE); 181 PAGE_READWRITE);
139 SetLastError(olderr); 182 SetLastError(olderr);
140 return ptr ? ptr : MFAIL; 183 return ptr ? ptr : MFAIL;
141} 184}
142 185
143#endif 186#endif
144 187
188#define CALL_MMAP(prng, size) mmap_plain(size)
189#define DIRECT_MMAP(prng, size) direct_mmap(size)
190
145/* This function supports releasing coalesed segments */ 191/* This function supports releasing coalesed segments */
146static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size) 192static int CALL_MUNMAP(void *ptr, size_t size)
147{ 193{
148 DWORD olderr = GetLastError(); 194 DWORD olderr = GetLastError();
149 MEMORY_BASIC_INFORMATION minfo; 195 MEMORY_BASIC_INFORMATION minfo;
@@ -163,10 +209,7 @@ static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size)
163 return 0; 209 return 0;
164} 210}
165 211
166#else 212#elif LJ_ALLOC_MMAP
167
168#include <errno.h>
169#include <sys/mman.h>
170 213
171#define MMAP_PROT (PROT_READ|PROT_WRITE) 214#define MMAP_PROT (PROT_READ|PROT_WRITE)
172#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) 215#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
@@ -174,105 +217,134 @@ static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size)
174#endif 217#endif
175#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS) 218#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
176 219
177#if LJ_64 220#if LJ_ALLOC_MMAP_PROBE
178/* 64 bit mode needs special support for allocating memory in the lower 2GB. */
179 221
180#if defined(MAP_32BIT) 222#ifdef MAP_TRYFIXED
181 223#define MMAP_FLAGS_PROBE (MMAP_FLAGS|MAP_TRYFIXED)
182#if defined(__sun__)
183#define MMAP_REGION_START ((uintptr_t)0x1000)
184#else 224#else
185/* Actually this only gives us max. 1GB in current Linux kernels. */ 225#define MMAP_FLAGS_PROBE MMAP_FLAGS
186#define MMAP_REGION_START ((uintptr_t)0)
187#endif 226#endif
188 227
189static LJ_AINLINE void *CALL_MMAP(size_t size) 228#define LJ_ALLOC_MMAP_PROBE_MAX 30
229#define LJ_ALLOC_MMAP_PROBE_LINEAR 5
230
231#define LJ_ALLOC_MMAP_PROBE_LOWER ((uintptr_t)0x4000)
232
233static void *mmap_probe(PRNGState *rs, size_t size)
190{ 234{
235 /* Hint for next allocation. Doesn't need to be thread-safe. */
236 static uintptr_t hint_addr = 0;
191 int olderr = errno; 237 int olderr = errno;
192 void *ptr = mmap((void *)MMAP_REGION_START, size, MMAP_PROT, MAP_32BIT|MMAP_FLAGS, -1, 0); 238 int retry;
239 for (retry = 0; retry < LJ_ALLOC_MMAP_PROBE_MAX; retry++) {
240 void *p = mmap((void *)hint_addr, size, MMAP_PROT, MMAP_FLAGS_PROBE, -1, 0);
241 uintptr_t addr = (uintptr_t)p;
242 if ((addr >> LJ_ALLOC_MBITS) == 0 && addr >= LJ_ALLOC_MMAP_PROBE_LOWER &&
243 ((addr + size) >> LJ_ALLOC_MBITS) == 0) {
244 /* We got a suitable address. Bump the hint address. */
245 hint_addr = addr + size;
246 errno = olderr;
247 return p;
248 }
249 if (p != MFAIL) {
250 munmap(p, size);
251 } else if (errno == ENOMEM) {
252 return MFAIL;
253 }
254 if (hint_addr) {
255 /* First, try linear probing. */
256 if (retry < LJ_ALLOC_MMAP_PROBE_LINEAR) {
257 hint_addr += 0x1000000;
258 if (((hint_addr + size) >> LJ_ALLOC_MBITS) != 0)
259 hint_addr = 0;
260 continue;
261 } else if (retry == LJ_ALLOC_MMAP_PROBE_LINEAR) {
262 /* Next, try a no-hint probe to get back an ASLR address. */
263 hint_addr = 0;
264 continue;
265 }
266 }
267 /* Finally, try pseudo-random probing. */
268 do {
269 hint_addr = lj_prng_u64(rs) & (((uintptr_t)1<<LJ_ALLOC_MBITS)-LJ_PAGESIZE);
270 } while (hint_addr < LJ_ALLOC_MMAP_PROBE_LOWER);
271 }
193 errno = olderr; 272 errno = olderr;
194 return ptr; 273 return MFAIL;
195} 274}
196 275
197#elif LJ_TARGET_OSX || LJ_TARGET_PS4 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__sun__) || LJ_TARGET_CYGWIN 276#endif
277
278#if LJ_ALLOC_MMAP32
198 279
199/* OSX and FreeBSD mmap() use a naive first-fit linear search. 280#if LJ_TARGET_SOLARIS
200** That's perfect for us. Except that -pagezero_size must be set for OSX, 281#define LJ_ALLOC_MMAP32_START ((uintptr_t)0x1000)
201** otherwise the lower 4GB are blocked. And the 32GB RLIMIT_DATA needs
202** to be reduced to 250MB on FreeBSD.
203*/
204#if LJ_TARGET_OSX || defined(__DragonFly__)
205#define MMAP_REGION_START ((uintptr_t)0x10000)
206#elif LJ_TARGET_PS4
207#define MMAP_REGION_START ((uintptr_t)0x4000)
208#else 282#else
209#define MMAP_REGION_START ((uintptr_t)0x10000000) 283#define LJ_ALLOC_MMAP32_START ((uintptr_t)0)
210#endif 284#endif
211#define MMAP_REGION_END ((uintptr_t)0x80000000)
212 285
213#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) && !LJ_TARGET_PS4 286#if LJ_ALLOC_MMAP_PROBE
214#include <sys/resource.h> 287static void *mmap_map32(PRNGState *rs, size_t size)
288#else
289static void *mmap_map32(size_t size)
215#endif 290#endif
216
217static LJ_AINLINE void *CALL_MMAP(size_t size)
218{ 291{
219 int olderr = errno; 292#if LJ_ALLOC_MMAP_PROBE
220 /* Hint for next allocation. Doesn't need to be thread-safe. */ 293 static int fallback = 0;
221 static uintptr_t alloc_hint = MMAP_REGION_START; 294 if (fallback)
222 int retry = 0; 295 return mmap_probe(rs, size);
223#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) && !LJ_TARGET_PS4
224 static int rlimit_modified = 0;
225 if (LJ_UNLIKELY(rlimit_modified == 0)) {
226 struct rlimit rlim;
227 rlim.rlim_cur = rlim.rlim_max = MMAP_REGION_START;
228 setrlimit(RLIMIT_DATA, &rlim); /* Ignore result. May fail below. */
229 rlimit_modified = 1;
230 }
231#endif 296#endif
232 for (;;) { 297 {
233 void *p = mmap((void *)alloc_hint, size, MMAP_PROT, MMAP_FLAGS, -1, 0); 298 int olderr = errno;
234 if ((uintptr_t)p >= MMAP_REGION_START && 299 void *ptr = mmap((void *)LJ_ALLOC_MMAP32_START, size, MMAP_PROT, MAP_32BIT|MMAP_FLAGS, -1, 0);
235 (uintptr_t)p + size < MMAP_REGION_END) { 300 errno = olderr;
236 alloc_hint = (uintptr_t)p + size; 301 /* This only allows 1GB on Linux. So fallback to probing to get 2GB. */
237 errno = olderr; 302#if LJ_ALLOC_MMAP_PROBE
238 return p; 303 if (ptr == MFAIL) {
304 fallback = 1;
305 return mmap_probe(rs, size);
239 } 306 }
240 if (p != CMFAIL) munmap(p, size);
241#if defined(__sun__) || defined(__DragonFly__)
242 alloc_hint += 0x1000000; /* Need near-exhaustive linear scan. */
243 if (alloc_hint + size < MMAP_REGION_END) continue;
244#endif 307#endif
245 if (retry) break; 308 return ptr;
246 retry = 1;
247 alloc_hint = MMAP_REGION_START;
248 } 309 }
249 errno = olderr;
250 return CMFAIL;
251} 310}
252 311
253#else
254
255#error "NYI: need an equivalent of MAP_32BIT for this 64 bit OS"
256
257#endif 312#endif
258 313
314#if LJ_ALLOC_MMAP32
315#if LJ_ALLOC_MMAP_PROBE
316#define CALL_MMAP(prng, size) mmap_map32(prng, size)
259#else 317#else
260 318#define CALL_MMAP(prng, size) mmap_map32(size)
261/* 32 bit mode is easy. */ 319#endif
262static LJ_AINLINE void *CALL_MMAP(size_t size) 320#elif LJ_ALLOC_MMAP_PROBE
321#define CALL_MMAP(prng, size) mmap_probe(prng, size)
322#else
323static void *mmap_plain(size_t size)
263{ 324{
264 int olderr = errno; 325 int olderr = errno;
265 void *ptr = mmap(NULL, size, MMAP_PROT, MMAP_FLAGS, -1, 0); 326 void *ptr = mmap(NULL, size, MMAP_PROT, MMAP_FLAGS, -1, 0);
266 errno = olderr; 327 errno = olderr;
267 return ptr; 328 return ptr;
268} 329}
269 330#define CALL_MMAP(prng, size) mmap_plain(size)
270#endif 331#endif
271 332
272#define INIT_MMAP() ((void)0) 333#if LJ_64 && !LJ_GC64 && ((defined(__FreeBSD__) && __FreeBSD__ < 10) || defined(__FreeBSD_kernel__)) && !LJ_TARGET_PS4 && !LJ_TARGET_PS5
273#define DIRECT_MMAP(s) CALL_MMAP(s) 334
335#include <sys/resource.h>
274 336
275static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size) 337static void init_mmap(void)
338{
339 struct rlimit rlim;
340 rlim.rlim_cur = rlim.rlim_max = 0x10000;
341 setrlimit(RLIMIT_DATA, &rlim); /* Ignore result. May fail later. */
342}
343#define INIT_MMAP() init_mmap()
344
345#endif
346
347static int CALL_MUNMAP(void *ptr, size_t size)
276{ 348{
277 int olderr = errno; 349 int olderr = errno;
278 int ret = munmap(ptr, size); 350 int ret = munmap(ptr, size);
@@ -280,10 +352,9 @@ static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size)
280 return ret; 352 return ret;
281} 353}
282 354
283#if LJ_TARGET_LINUX 355#if LJ_ALLOC_MREMAP
284/* Need to define _GNU_SOURCE to get the mremap prototype. */ 356/* Need to define _GNU_SOURCE to get the mremap prototype. */
285static LJ_AINLINE void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz, 357static void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz, int flags)
286 int flags)
287{ 358{
288 int olderr = errno; 359 int olderr = errno;
289 ptr = mremap(ptr, osz, nsz, flags); 360 ptr = mremap(ptr, osz, nsz, flags);
@@ -294,7 +365,7 @@ static LJ_AINLINE void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz,
294#define CALL_MREMAP(addr, osz, nsz, mv) CALL_MREMAP_((addr), (osz), (nsz), (mv)) 365#define CALL_MREMAP(addr, osz, nsz, mv) CALL_MREMAP_((addr), (osz), (nsz), (mv))
295#define CALL_MREMAP_NOMOVE 0 366#define CALL_MREMAP_NOMOVE 0
296#define CALL_MREMAP_MAYMOVE 1 367#define CALL_MREMAP_MAYMOVE 1
297#if LJ_64 368#if LJ_64 && (!LJ_GC64 || LJ_TARGET_ARM64)
298#define CALL_MREMAP_MV CALL_MREMAP_NOMOVE 369#define CALL_MREMAP_MV CALL_MREMAP_NOMOVE
299#else 370#else
300#define CALL_MREMAP_MV CALL_MREMAP_MAYMOVE 371#define CALL_MREMAP_MV CALL_MREMAP_MAYMOVE
@@ -303,6 +374,15 @@ static LJ_AINLINE void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz,
303 374
304#endif 375#endif
305 376
377
378#ifndef INIT_MMAP
379#define INIT_MMAP() ((void)0)
380#endif
381
382#ifndef DIRECT_MMAP
383#define DIRECT_MMAP(prng, s) CALL_MMAP(prng, s)
384#endif
385
306#ifndef CALL_MREMAP 386#ifndef CALL_MREMAP
307#define CALL_MREMAP(addr, osz, nsz, mv) ((void)osz, MFAIL) 387#define CALL_MREMAP(addr, osz, nsz, mv) ((void)osz, MFAIL)
308#endif 388#endif
@@ -459,6 +539,7 @@ struct malloc_state {
459 mchunkptr smallbins[(NSMALLBINS+1)*2]; 539 mchunkptr smallbins[(NSMALLBINS+1)*2];
460 tbinptr treebins[NTREEBINS]; 540 tbinptr treebins[NTREEBINS];
461 msegment seg; 541 msegment seg;
542 PRNGState *prng;
462}; 543};
463 544
464typedef struct malloc_state *mstate; 545typedef struct malloc_state *mstate;
@@ -516,7 +597,7 @@ static int has_segment_link(mstate m, msegmentptr ss)
516 noncontiguous segments are added. 597 noncontiguous segments are added.
517*/ 598*/
518#define TOP_FOOT_SIZE\ 599#define TOP_FOOT_SIZE\
519 (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE) 600 (align_offset(TWO_SIZE_T_SIZES)+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE)
520 601
521/* ---------------------------- Indexing Bins ---------------------------- */ 602/* ---------------------------- Indexing Bins ---------------------------- */
522 603
@@ -741,11 +822,11 @@ static int has_segment_link(mstate m, msegmentptr ss)
741 822
742/* ----------------------- Direct-mmapping chunks ----------------------- */ 823/* ----------------------- Direct-mmapping chunks ----------------------- */
743 824
744static void *direct_alloc(size_t nb) 825static void *direct_alloc(mstate m, size_t nb)
745{ 826{
746 size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK); 827 size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
747 if (LJ_LIKELY(mmsize > nb)) { /* Check for wrap around 0 */ 828 if (LJ_LIKELY(mmsize > nb)) { /* Check for wrap around 0 */
748 char *mm = (char *)(DIRECT_MMAP(mmsize)); 829 char *mm = (char *)(DIRECT_MMAP(m->prng, mmsize));
749 if (mm != CMFAIL) { 830 if (mm != CMFAIL) {
750 size_t offset = align_offset(chunk2mem(mm)); 831 size_t offset = align_offset(chunk2mem(mm));
751 size_t psize = mmsize - offset - DIRECT_FOOT_PAD; 832 size_t psize = mmsize - offset - DIRECT_FOOT_PAD;
@@ -757,6 +838,7 @@ static void *direct_alloc(size_t nb)
757 return chunk2mem(p); 838 return chunk2mem(p);
758 } 839 }
759 } 840 }
841 UNUSED(m);
760 return NULL; 842 return NULL;
761} 843}
762 844
@@ -905,7 +987,7 @@ static void *alloc_sys(mstate m, size_t nb)
905 987
906 /* Directly map large chunks */ 988 /* Directly map large chunks */
907 if (LJ_UNLIKELY(nb >= DEFAULT_MMAP_THRESHOLD)) { 989 if (LJ_UNLIKELY(nb >= DEFAULT_MMAP_THRESHOLD)) {
908 void *mem = direct_alloc(nb); 990 void *mem = direct_alloc(m, nb);
909 if (mem != 0) 991 if (mem != 0)
910 return mem; 992 return mem;
911 } 993 }
@@ -914,7 +996,7 @@ static void *alloc_sys(mstate m, size_t nb)
914 size_t req = nb + TOP_FOOT_SIZE + SIZE_T_ONE; 996 size_t req = nb + TOP_FOOT_SIZE + SIZE_T_ONE;
915 size_t rsize = granularity_align(req); 997 size_t rsize = granularity_align(req);
916 if (LJ_LIKELY(rsize > nb)) { /* Fail if wraps around zero */ 998 if (LJ_LIKELY(rsize > nb)) { /* Fail if wraps around zero */
917 char *mp = (char *)(CALL_MMAP(rsize)); 999 char *mp = (char *)(CALL_MMAP(m->prng, rsize));
918 if (mp != CMFAIL) { 1000 if (mp != CMFAIL) {
919 tbase = mp; 1001 tbase = mp;
920 tsize = rsize; 1002 tsize = rsize;
@@ -1141,12 +1223,13 @@ static void *tmalloc_small(mstate m, size_t nb)
1141 1223
1142/* ----------------------------------------------------------------------- */ 1224/* ----------------------------------------------------------------------- */
1143 1225
1144void *lj_alloc_create(void) 1226void *lj_alloc_create(PRNGState *rs)
1145{ 1227{
1146 size_t tsize = DEFAULT_GRANULARITY; 1228 size_t tsize = DEFAULT_GRANULARITY;
1147 char *tbase; 1229 char *tbase;
1148 INIT_MMAP(); 1230 INIT_MMAP();
1149 tbase = (char *)(CALL_MMAP(tsize)); 1231 UNUSED(rs);
1232 tbase = (char *)(CALL_MMAP(rs, tsize));
1150 if (tbase != CMFAIL) { 1233 if (tbase != CMFAIL) {
1151 size_t msize = pad_request(sizeof(struct malloc_state)); 1234 size_t msize = pad_request(sizeof(struct malloc_state));
1152 mchunkptr mn; 1235 mchunkptr mn;
@@ -1165,6 +1248,12 @@ void *lj_alloc_create(void)
1165 return NULL; 1248 return NULL;
1166} 1249}
1167 1250
1251void lj_alloc_setprng(void *msp, PRNGState *rs)
1252{
1253 mstate ms = (mstate)msp;
1254 ms->prng = rs;
1255}
1256
1168void lj_alloc_destroy(void *msp) 1257void lj_alloc_destroy(void *msp)
1169{ 1258{
1170 mstate ms = (mstate)msp; 1259 mstate ms = (mstate)msp;
diff --git a/src/lj_alloc.h b/src/lj_alloc.h
index f87a7cf3..669f50b7 100644
--- a/src/lj_alloc.h
+++ b/src/lj_alloc.h
@@ -9,7 +9,8 @@
9#include "lj_def.h" 9#include "lj_def.h"
10 10
11#ifndef LUAJIT_USE_SYSMALLOC 11#ifndef LUAJIT_USE_SYSMALLOC
12LJ_FUNC void *lj_alloc_create(void); 12LJ_FUNC void *lj_alloc_create(PRNGState *rs);
13LJ_FUNC void lj_alloc_setprng(void *msp, PRNGState *rs);
13LJ_FUNC void lj_alloc_destroy(void *msp); 14LJ_FUNC void lj_alloc_destroy(void *msp);
14LJ_FUNC void *lj_alloc_f(void *msp, void *ptr, size_t osize, size_t nsize); 15LJ_FUNC void *lj_alloc_f(void *msp, void *ptr, size_t osize, size_t nsize);
15#endif 16#endif
diff --git a/src/lj_api.c b/src/lj_api.c
index 2018cb8f..386bcada 100644
--- a/src/lj_api.c
+++ b/src/lj_api.c
@@ -24,11 +24,12 @@
24#include "lj_trace.h" 24#include "lj_trace.h"
25#include "lj_vm.h" 25#include "lj_vm.h"
26#include "lj_strscan.h" 26#include "lj_strscan.h"
27#include "lj_strfmt.h"
27 28
28/* -- Common helper functions --------------------------------------------- */ 29/* -- Common helper functions --------------------------------------------- */
29 30
30#define api_checknelems(L, n) api_check(L, (n) <= (L->top - L->base)) 31#define lj_checkapi_slot(idx) \
31#define api_checkvalidindex(L, i) api_check(L, (i) != niltv(L)) 32 lj_checkapi((idx) <= (L->top - L->base), "stack slot %d out of range", (idx))
32 33
33static TValue *index2adr(lua_State *L, int idx) 34static TValue *index2adr(lua_State *L, int idx)
34{ 35{
@@ -36,7 +37,8 @@ static TValue *index2adr(lua_State *L, int idx)
36 TValue *o = L->base + (idx - 1); 37 TValue *o = L->base + (idx - 1);
37 return o < L->top ? o : niltv(L); 38 return o < L->top ? o : niltv(L);
38 } else if (idx > LUA_REGISTRYINDEX) { 39 } else if (idx > LUA_REGISTRYINDEX) {
39 api_check(L, idx != 0 && -idx <= L->top - L->base); 40 lj_checkapi(idx != 0 && -idx <= L->top - L->base,
41 "bad stack slot %d", idx);
40 return L->top + idx; 42 return L->top + idx;
41 } else if (idx == LUA_GLOBALSINDEX) { 43 } else if (idx == LUA_GLOBALSINDEX) {
42 TValue *o = &G(L)->tmptv; 44 TValue *o = &G(L)->tmptv;
@@ -46,7 +48,8 @@ static TValue *index2adr(lua_State *L, int idx)
46 return registry(L); 48 return registry(L);
47 } else { 49 } else {
48 GCfunc *fn = curr_func(L); 50 GCfunc *fn = curr_func(L);
49 api_check(L, fn->c.gct == ~LJ_TFUNC && !isluafunc(fn)); 51 lj_checkapi(fn->c.gct == ~LJ_TFUNC && !isluafunc(fn),
52 "calling frame is not a C function");
50 if (idx == LUA_ENVIRONINDEX) { 53 if (idx == LUA_ENVIRONINDEX) {
51 TValue *o = &G(L)->tmptv; 54 TValue *o = &G(L)->tmptv;
52 settabV(L, o, tabref(fn->c.env)); 55 settabV(L, o, tabref(fn->c.env));
@@ -58,13 +61,27 @@ static TValue *index2adr(lua_State *L, int idx)
58 } 61 }
59} 62}
60 63
61static TValue *stkindex2adr(lua_State *L, int idx) 64static LJ_AINLINE TValue *index2adr_check(lua_State *L, int idx)
65{
66 TValue *o = index2adr(L, idx);
67 lj_checkapi(o != niltv(L), "invalid stack slot %d", idx);
68 return o;
69}
70
71static TValue *index2adr_stack(lua_State *L, int idx)
62{ 72{
63 if (idx > 0) { 73 if (idx > 0) {
64 TValue *o = L->base + (idx - 1); 74 TValue *o = L->base + (idx - 1);
75 if (o < L->top) {
76 return o;
77 } else {
78 lj_checkapi(0, "invalid stack slot %d", idx);
79 return niltv(L);
80 }
65 return o < L->top ? o : niltv(L); 81 return o < L->top ? o : niltv(L);
66 } else { 82 } else {
67 api_check(L, idx != 0 && -idx <= L->top - L->base); 83 lj_checkapi(idx != 0 && -idx <= L->top - L->base,
84 "invalid stack slot %d", idx);
68 return L->top + idx; 85 return L->top + idx;
69 } 86 }
70} 87}
@@ -98,17 +115,24 @@ LUALIB_API void luaL_checkstack(lua_State *L, int size, const char *msg)
98 lj_err_callerv(L, LJ_ERR_STKOVM, msg); 115 lj_err_callerv(L, LJ_ERR_STKOVM, msg);
99} 116}
100 117
101LUA_API void lua_xmove(lua_State *from, lua_State *to, int n) 118LUA_API void lua_xmove(lua_State *L, lua_State *to, int n)
102{ 119{
103 TValue *f, *t; 120 TValue *f, *t;
104 if (from == to) return; 121 if (L == to) return;
105 api_checknelems(from, n); 122 lj_checkapi_slot(n);
106 api_check(from, G(from) == G(to)); 123 lj_checkapi(G(L) == G(to), "move across global states");
107 lj_state_checkstack(to, (MSize)n); 124 lj_state_checkstack(to, (MSize)n);
108 f = from->top; 125 f = L->top;
109 t = to->top = to->top + n; 126 t = to->top = to->top + n;
110 while (--n >= 0) copyTV(to, --t, --f); 127 while (--n >= 0) copyTV(to, --t, --f);
111 from->top = f; 128 L->top = f;
129}
130
131LUA_API const lua_Number *lua_version(lua_State *L)
132{
133 static const lua_Number version = LUA_VERSION_NUM;
134 UNUSED(L);
135 return &version;
112} 136}
113 137
114/* -- Stack manipulation -------------------------------------------------- */ 138/* -- Stack manipulation -------------------------------------------------- */
@@ -121,7 +145,7 @@ LUA_API int lua_gettop(lua_State *L)
121LUA_API void lua_settop(lua_State *L, int idx) 145LUA_API void lua_settop(lua_State *L, int idx)
122{ 146{
123 if (idx >= 0) { 147 if (idx >= 0) {
124 api_check(L, idx <= tvref(L->maxstack) - L->base); 148 lj_checkapi(idx <= tvref(L->maxstack) - L->base, "bad stack slot %d", idx);
125 if (L->base + idx > L->top) { 149 if (L->base + idx > L->top) {
126 if (L->base + idx >= tvref(L->maxstack)) 150 if (L->base + idx >= tvref(L->maxstack))
127 lj_state_growstack(L, (MSize)idx - (MSize)(L->top - L->base)); 151 lj_state_growstack(L, (MSize)idx - (MSize)(L->top - L->base));
@@ -130,51 +154,58 @@ LUA_API void lua_settop(lua_State *L, int idx)
130 L->top = L->base + idx; 154 L->top = L->base + idx;
131 } 155 }
132 } else { 156 } else {
133 api_check(L, -(idx+1) <= (L->top - L->base)); 157 lj_checkapi(-(idx+1) <= (L->top - L->base), "bad stack slot %d", idx);
134 L->top += idx+1; /* Shrinks top (idx < 0). */ 158 L->top += idx+1; /* Shrinks top (idx < 0). */
135 } 159 }
136} 160}
137 161
138LUA_API void lua_remove(lua_State *L, int idx) 162LUA_API void lua_remove(lua_State *L, int idx)
139{ 163{
140 TValue *p = stkindex2adr(L, idx); 164 TValue *p = index2adr_stack(L, idx);
141 api_checkvalidindex(L, p);
142 while (++p < L->top) copyTV(L, p-1, p); 165 while (++p < L->top) copyTV(L, p-1, p);
143 L->top--; 166 L->top--;
144} 167}
145 168
146LUA_API void lua_insert(lua_State *L, int idx) 169LUA_API void lua_insert(lua_State *L, int idx)
147{ 170{
148 TValue *q, *p = stkindex2adr(L, idx); 171 TValue *q, *p = index2adr_stack(L, idx);
149 api_checkvalidindex(L, p);
150 for (q = L->top; q > p; q--) copyTV(L, q, q-1); 172 for (q = L->top; q > p; q--) copyTV(L, q, q-1);
151 copyTV(L, p, L->top); 173 copyTV(L, p, L->top);
152} 174}
153 175
154LUA_API void lua_replace(lua_State *L, int idx) 176static void copy_slot(lua_State *L, TValue *f, int idx)
155{ 177{
156 api_checknelems(L, 1);
157 if (idx == LUA_GLOBALSINDEX) { 178 if (idx == LUA_GLOBALSINDEX) {
158 api_check(L, tvistab(L->top-1)); 179 lj_checkapi(tvistab(f), "stack slot %d is not a table", idx);
159 /* NOBARRIER: A thread (i.e. L) is never black. */ 180 /* NOBARRIER: A thread (i.e. L) is never black. */
160 setgcref(L->env, obj2gco(tabV(L->top-1))); 181 setgcref(L->env, obj2gco(tabV(f)));
161 } else if (idx == LUA_ENVIRONINDEX) { 182 } else if (idx == LUA_ENVIRONINDEX) {
162 GCfunc *fn = curr_func(L); 183 GCfunc *fn = curr_func(L);
163 if (fn->c.gct != ~LJ_TFUNC) 184 if (fn->c.gct != ~LJ_TFUNC)
164 lj_err_msg(L, LJ_ERR_NOENV); 185 lj_err_msg(L, LJ_ERR_NOENV);
165 api_check(L, tvistab(L->top-1)); 186 lj_checkapi(tvistab(f), "stack slot %d is not a table", idx);
166 setgcref(fn->c.env, obj2gco(tabV(L->top-1))); 187 setgcref(fn->c.env, obj2gco(tabV(f)));
167 lj_gc_barrier(L, fn, L->top-1); 188 lj_gc_barrier(L, fn, f);
168 } else { 189 } else {
169 TValue *o = index2adr(L, idx); 190 TValue *o = index2adr_check(L, idx);
170 api_checkvalidindex(L, o); 191 copyTV(L, o, f);
171 copyTV(L, o, L->top-1);
172 if (idx < LUA_GLOBALSINDEX) /* Need a barrier for upvalues. */ 192 if (idx < LUA_GLOBALSINDEX) /* Need a barrier for upvalues. */
173 lj_gc_barrier(L, curr_func(L), L->top-1); 193 lj_gc_barrier(L, curr_func(L), f);
174 } 194 }
195}
196
197LUA_API void lua_replace(lua_State *L, int idx)
198{
199 lj_checkapi_slot(1);
200 copy_slot(L, L->top - 1, idx);
175 L->top--; 201 L->top--;
176} 202}
177 203
204LUA_API void lua_copy(lua_State *L, int fromidx, int toidx)
205{
206 copy_slot(L, index2adr(L, fromidx), toidx);
207}
208
178LUA_API void lua_pushvalue(lua_State *L, int idx) 209LUA_API void lua_pushvalue(lua_State *L, int idx)
179{ 210{
180 copyTV(L, L->top, index2adr(L, idx)); 211 copyTV(L, L->top, index2adr(L, idx));
@@ -188,7 +219,7 @@ LUA_API int lua_type(lua_State *L, int idx)
188 cTValue *o = index2adr(L, idx); 219 cTValue *o = index2adr(L, idx);
189 if (tvisnumber(o)) { 220 if (tvisnumber(o)) {
190 return LUA_TNUMBER; 221 return LUA_TNUMBER;
191#if LJ_64 222#if LJ_64 && !LJ_GC64
192 } else if (tvislightud(o)) { 223 } else if (tvislightud(o)) {
193 return LUA_TLIGHTUSERDATA; 224 return LUA_TLIGHTUSERDATA;
194#endif 225#endif
@@ -201,7 +232,7 @@ LUA_API int lua_type(lua_State *L, int idx)
201#else 232#else
202 int tt = (int)(((t < 8 ? 0x98042110u : 0x75a06u) >> 4*(t&7)) & 15u); 233 int tt = (int)(((t < 8 ? 0x98042110u : 0x75a06u) >> 4*(t&7)) & 15u);
203#endif 234#endif
204 lua_assert(tt != LUA_TNIL || tvisnil(o)); 235 lj_assertL(tt != LUA_TNIL || tvisnil(o), "bad tag conversion");
205 return tt; 236 return tt;
206 } 237 }
207} 238}
@@ -268,7 +299,7 @@ LUA_API int lua_equal(lua_State *L, int idx1, int idx2)
268 return 0; 299 return 0;
269 } else if (tvispri(o1)) { 300 } else if (tvispri(o1)) {
270 return o1 != niltv(L) && o2 != niltv(L); 301 return o1 != niltv(L) && o2 != niltv(L);
271#if LJ_64 302#if LJ_64 && !LJ_GC64
272 } else if (tvislightud(o1)) { 303 } else if (tvislightud(o1)) {
273 return o1->u64 == o2->u64; 304 return o1->u64 == o2->u64;
274#endif 305#endif
@@ -283,8 +314,8 @@ LUA_API int lua_equal(lua_State *L, int idx1, int idx2)
283 } else { 314 } else {
284 L->top = base+2; 315 L->top = base+2;
285 lj_vm_call(L, base, 1+1); 316 lj_vm_call(L, base, 1+1);
286 L->top -= 2; 317 L->top -= 2+LJ_FR2;
287 return tvistruecond(L->top+1); 318 return tvistruecond(L->top+1+LJ_FR2);
288 } 319 }
289 } 320 }
290} 321}
@@ -306,8 +337,8 @@ LUA_API int lua_lessthan(lua_State *L, int idx1, int idx2)
306 } else { 337 } else {
307 L->top = base+2; 338 L->top = base+2;
308 lj_vm_call(L, base, 1+1); 339 lj_vm_call(L, base, 1+1);
309 L->top -= 2; 340 L->top -= 2+LJ_FR2;
310 return tvistruecond(L->top+1); 341 return tvistruecond(L->top+1+LJ_FR2);
311 } 342 }
312 } 343 }
313} 344}
@@ -324,6 +355,22 @@ LUA_API lua_Number lua_tonumber(lua_State *L, int idx)
324 return 0; 355 return 0;
325} 356}
326 357
358LUA_API lua_Number lua_tonumberx(lua_State *L, int idx, int *ok)
359{
360 cTValue *o = index2adr(L, idx);
361 TValue tmp;
362 if (LJ_LIKELY(tvisnumber(o))) {
363 if (ok) *ok = 1;
364 return numberVnum(o);
365 } else if (tvisstr(o) && lj_strscan_num(strV(o), &tmp)) {
366 if (ok) *ok = 1;
367 return numV(&tmp);
368 } else {
369 if (ok) *ok = 0;
370 return 0;
371 }
372}
373
327LUALIB_API lua_Number luaL_checknumber(lua_State *L, int idx) 374LUALIB_API lua_Number luaL_checknumber(lua_State *L, int idx)
328{ 375{
329 cTValue *o = index2adr(L, idx); 376 cTValue *o = index2adr(L, idx);
@@ -361,9 +408,38 @@ LUA_API lua_Integer lua_tointeger(lua_State *L, int idx)
361 if (!(tvisstr(o) && lj_strscan_number(strV(o), &tmp))) 408 if (!(tvisstr(o) && lj_strscan_number(strV(o), &tmp)))
362 return 0; 409 return 0;
363 if (tvisint(&tmp)) 410 if (tvisint(&tmp))
364 return (lua_Integer)intV(&tmp); 411 return intV(&tmp);
412 n = numV(&tmp);
413 }
414#if LJ_64
415 return (lua_Integer)n;
416#else
417 return lj_num2int(n);
418#endif
419}
420
421LUA_API lua_Integer lua_tointegerx(lua_State *L, int idx, int *ok)
422{
423 cTValue *o = index2adr(L, idx);
424 TValue tmp;
425 lua_Number n;
426 if (LJ_LIKELY(tvisint(o))) {
427 if (ok) *ok = 1;
428 return intV(o);
429 } else if (LJ_LIKELY(tvisnum(o))) {
430 n = numV(o);
431 } else {
432 if (!(tvisstr(o) && lj_strscan_number(strV(o), &tmp))) {
433 if (ok) *ok = 0;
434 return 0;
435 }
436 if (tvisint(&tmp)) {
437 if (ok) *ok = 1;
438 return intV(&tmp);
439 }
365 n = numV(&tmp); 440 n = numV(&tmp);
366 } 441 }
442 if (ok) *ok = 1;
367#if LJ_64 443#if LJ_64
368 return (lua_Integer)n; 444 return (lua_Integer)n;
369#else 445#else
@@ -434,7 +510,7 @@ LUA_API const char *lua_tolstring(lua_State *L, int idx, size_t *len)
434 } else if (tvisnumber(o)) { 510 } else if (tvisnumber(o)) {
435 lj_gc_check(L); 511 lj_gc_check(L);
436 o = index2adr(L, idx); /* GC may move the stack. */ 512 o = index2adr(L, idx); /* GC may move the stack. */
437 s = lj_str_fromnumber(L, o); 513 s = lj_strfmt_number(L, o);
438 setstrV(L, o, s); 514 setstrV(L, o, s);
439 } else { 515 } else {
440 if (len != NULL) *len = 0; 516 if (len != NULL) *len = 0;
@@ -453,7 +529,7 @@ LUALIB_API const char *luaL_checklstring(lua_State *L, int idx, size_t *len)
453 } else if (tvisnumber(o)) { 529 } else if (tvisnumber(o)) {
454 lj_gc_check(L); 530 lj_gc_check(L);
455 o = index2adr(L, idx); /* GC may move the stack. */ 531 o = index2adr(L, idx); /* GC may move the stack. */
456 s = lj_str_fromnumber(L, o); 532 s = lj_strfmt_number(L, o);
457 setstrV(L, o, s); 533 setstrV(L, o, s);
458 } else { 534 } else {
459 lj_err_argt(L, idx, LUA_TSTRING); 535 lj_err_argt(L, idx, LUA_TSTRING);
@@ -475,7 +551,7 @@ LUALIB_API const char *luaL_optlstring(lua_State *L, int idx,
475 } else if (tvisnumber(o)) { 551 } else if (tvisnumber(o)) {
476 lj_gc_check(L); 552 lj_gc_check(L);
477 o = index2adr(L, idx); /* GC may move the stack. */ 553 o = index2adr(L, idx); /* GC may move the stack. */
478 s = lj_str_fromnumber(L, o); 554 s = lj_strfmt_number(L, o);
479 setstrV(L, o, s); 555 setstrV(L, o, s);
480 } else { 556 } else {
481 lj_err_argt(L, idx, LUA_TSTRING); 557 lj_err_argt(L, idx, LUA_TSTRING);
@@ -507,7 +583,7 @@ LUA_API size_t lua_objlen(lua_State *L, int idx)
507 } else if (tvisudata(o)) { 583 } else if (tvisudata(o)) {
508 return udataV(o)->len; 584 return udataV(o)->len;
509 } else if (tvisnumber(o)) { 585 } else if (tvisnumber(o)) {
510 GCstr *s = lj_str_fromnumber(L, o); 586 GCstr *s = lj_strfmt_number(L, o);
511 setstrV(L, o, s); 587 setstrV(L, o, s);
512 return s->len; 588 return s->len;
513 } else { 589 } else {
@@ -532,7 +608,7 @@ LUA_API void *lua_touserdata(lua_State *L, int idx)
532 if (tvisudata(o)) 608 if (tvisudata(o))
533 return uddata(udataV(o)); 609 return uddata(udataV(o));
534 else if (tvislightud(o)) 610 else if (tvislightud(o))
535 return lightudV(o); 611 return lightudV(G(L), o);
536 else 612 else
537 return NULL; 613 return NULL;
538} 614}
@@ -545,17 +621,7 @@ LUA_API lua_State *lua_tothread(lua_State *L, int idx)
545 621
546LUA_API const void *lua_topointer(lua_State *L, int idx) 622LUA_API const void *lua_topointer(lua_State *L, int idx)
547{ 623{
548 cTValue *o = index2adr(L, idx); 624 return lj_obj_ptr(G(L), index2adr(L, idx));
549 if (tvisudata(o))
550 return uddata(udataV(o));
551 else if (tvislightud(o))
552 return lightudV(o);
553 else if (tviscdata(o))
554 return cdataptr(cdataV(o));
555 else if (tvisgcv(o))
556 return gcV(o);
557 else
558 return NULL;
559} 625}
560 626
561/* -- Stack setters (object creation) ------------------------------------- */ 627/* -- Stack setters (object creation) ------------------------------------- */
@@ -606,7 +672,7 @@ LUA_API const char *lua_pushvfstring(lua_State *L, const char *fmt,
606 va_list argp) 672 va_list argp)
607{ 673{
608 lj_gc_check(L); 674 lj_gc_check(L);
609 return lj_str_pushvf(L, fmt, argp); 675 return lj_strfmt_pushvf(L, fmt, argp);
610} 676}
611 677
612LUA_API const char *lua_pushfstring(lua_State *L, const char *fmt, ...) 678LUA_API const char *lua_pushfstring(lua_State *L, const char *fmt, ...)
@@ -615,7 +681,7 @@ LUA_API const char *lua_pushfstring(lua_State *L, const char *fmt, ...)
615 va_list argp; 681 va_list argp;
616 lj_gc_check(L); 682 lj_gc_check(L);
617 va_start(argp, fmt); 683 va_start(argp, fmt);
618 ret = lj_str_pushvf(L, fmt, argp); 684 ret = lj_strfmt_pushvf(L, fmt, argp);
619 va_end(argp); 685 va_end(argp);
620 return ret; 686 return ret;
621} 687}
@@ -624,14 +690,14 @@ LUA_API void lua_pushcclosure(lua_State *L, lua_CFunction f, int n)
624{ 690{
625 GCfunc *fn; 691 GCfunc *fn;
626 lj_gc_check(L); 692 lj_gc_check(L);
627 api_checknelems(L, n); 693 lj_checkapi_slot(n);
628 fn = lj_func_newC(L, (MSize)n, getcurrenv(L)); 694 fn = lj_func_newC(L, (MSize)n, getcurrenv(L));
629 fn->c.f = f; 695 fn->c.f = f;
630 L->top -= n; 696 L->top -= n;
631 while (n--) 697 while (n--)
632 copyTV(L, &fn->c.upvalue[n], L->top+n); 698 copyTV(L, &fn->c.upvalue[n], L->top+n);
633 setfuncV(L, L->top, fn); 699 setfuncV(L, L->top, fn);
634 lua_assert(iswhite(obj2gco(fn))); 700 lj_assertL(iswhite(obj2gco(fn)), "new GC object is not white");
635 incr_top(L); 701 incr_top(L);
636} 702}
637 703
@@ -643,16 +709,17 @@ LUA_API void lua_pushboolean(lua_State *L, int b)
643 709
644LUA_API void lua_pushlightuserdata(lua_State *L, void *p) 710LUA_API void lua_pushlightuserdata(lua_State *L, void *p)
645{ 711{
646 setlightudV(L->top, checklightudptr(L, p)); 712#if LJ_64
713 p = lj_lightud_intern(L, p);
714#endif
715 setrawlightudV(L->top, p);
647 incr_top(L); 716 incr_top(L);
648} 717}
649 718
650LUA_API void lua_createtable(lua_State *L, int narray, int nrec) 719LUA_API void lua_createtable(lua_State *L, int narray, int nrec)
651{ 720{
652 GCtab *t;
653 lj_gc_check(L); 721 lj_gc_check(L);
654 t = lj_tab_new(L, (uint32_t)(narray > 0 ? narray+1 : 0), hsize2hbits(nrec)); 722 settabV(L, L->top, lj_tab_new_ah(L, narray, nrec));
655 settabV(L, L->top, t);
656 incr_top(L); 723 incr_top(L);
657} 724}
658 725
@@ -703,7 +770,7 @@ LUA_API void *lua_newuserdata(lua_State *L, size_t size)
703 770
704LUA_API void lua_concat(lua_State *L, int n) 771LUA_API void lua_concat(lua_State *L, int n)
705{ 772{
706 api_checknelems(L, n); 773 lj_checkapi_slot(n);
707 if (n >= 2) { 774 if (n >= 2) {
708 n--; 775 n--;
709 do { 776 do {
@@ -712,11 +779,11 @@ LUA_API void lua_concat(lua_State *L, int n)
712 L->top -= n; 779 L->top -= n;
713 break; 780 break;
714 } 781 }
715 n -= (int)(L->top - top); 782 n -= (int)(L->top - (top - 2*LJ_FR2));
716 L->top = top+2; 783 L->top = top+2;
717 lj_vm_call(L, top, 1+1); 784 lj_vm_call(L, top, 1+1);
718 L->top--; 785 L->top -= 1+LJ_FR2;
719 copyTV(L, L->top-1, L->top); 786 copyTV(L, L->top-1, L->top+LJ_FR2);
720 } while (--n > 0); 787 } while (--n > 0);
721 } else if (n == 0) { /* Push empty string. */ 788 } else if (n == 0) { /* Push empty string. */
722 setstrV(L, L->top, &G(L)->strempty); 789 setstrV(L, L->top, &G(L)->strempty);
@@ -729,30 +796,28 @@ LUA_API void lua_concat(lua_State *L, int n)
729 796
730LUA_API void lua_gettable(lua_State *L, int idx) 797LUA_API void lua_gettable(lua_State *L, int idx)
731{ 798{
732 cTValue *v, *t = index2adr(L, idx); 799 cTValue *t = index2adr_check(L, idx);
733 api_checkvalidindex(L, t); 800 cTValue *v = lj_meta_tget(L, t, L->top-1);
734 v = lj_meta_tget(L, t, L->top-1);
735 if (v == NULL) { 801 if (v == NULL) {
736 L->top += 2; 802 L->top += 2;
737 lj_vm_call(L, L->top-2, 1+1); 803 lj_vm_call(L, L->top-2, 1+1);
738 L->top -= 2; 804 L->top -= 2+LJ_FR2;
739 v = L->top+1; 805 v = L->top+1+LJ_FR2;
740 } 806 }
741 copyTV(L, L->top-1, v); 807 copyTV(L, L->top-1, v);
742} 808}
743 809
744LUA_API void lua_getfield(lua_State *L, int idx, const char *k) 810LUA_API void lua_getfield(lua_State *L, int idx, const char *k)
745{ 811{
746 cTValue *v, *t = index2adr(L, idx); 812 cTValue *v, *t = index2adr_check(L, idx);
747 TValue key; 813 TValue key;
748 api_checkvalidindex(L, t);
749 setstrV(L, &key, lj_str_newz(L, k)); 814 setstrV(L, &key, lj_str_newz(L, k));
750 v = lj_meta_tget(L, t, &key); 815 v = lj_meta_tget(L, t, &key);
751 if (v == NULL) { 816 if (v == NULL) {
752 L->top += 2; 817 L->top += 2;
753 lj_vm_call(L, L->top-2, 1+1); 818 lj_vm_call(L, L->top-2, 1+1);
754 L->top -= 2; 819 L->top -= 2+LJ_FR2;
755 v = L->top+1; 820 v = L->top+1+LJ_FR2;
756 } 821 }
757 copyTV(L, L->top, v); 822 copyTV(L, L->top, v);
758 incr_top(L); 823 incr_top(L);
@@ -761,14 +826,14 @@ LUA_API void lua_getfield(lua_State *L, int idx, const char *k)
761LUA_API void lua_rawget(lua_State *L, int idx) 826LUA_API void lua_rawget(lua_State *L, int idx)
762{ 827{
763 cTValue *t = index2adr(L, idx); 828 cTValue *t = index2adr(L, idx);
764 api_check(L, tvistab(t)); 829 lj_checkapi(tvistab(t), "stack slot %d is not a table", idx);
765 copyTV(L, L->top-1, lj_tab_get(L, tabV(t), L->top-1)); 830 copyTV(L, L->top-1, lj_tab_get(L, tabV(t), L->top-1));
766} 831}
767 832
768LUA_API void lua_rawgeti(lua_State *L, int idx, int n) 833LUA_API void lua_rawgeti(lua_State *L, int idx, int n)
769{ 834{
770 cTValue *v, *t = index2adr(L, idx); 835 cTValue *v, *t = index2adr(L, idx);
771 api_check(L, tvistab(t)); 836 lj_checkapi(tvistab(t), "stack slot %d is not a table", idx);
772 v = lj_tab_getint(tabV(t), n); 837 v = lj_tab_getint(tabV(t), n);
773 if (v) { 838 if (v) {
774 copyTV(L, L->top, v); 839 copyTV(L, L->top, v);
@@ -810,8 +875,7 @@ LUALIB_API int luaL_getmetafield(lua_State *L, int idx, const char *field)
810 875
811LUA_API void lua_getfenv(lua_State *L, int idx) 876LUA_API void lua_getfenv(lua_State *L, int idx)
812{ 877{
813 cTValue *o = index2adr(L, idx); 878 cTValue *o = index2adr_check(L, idx);
814 api_checkvalidindex(L, o);
815 if (tvisfunc(o)) { 879 if (tvisfunc(o)) {
816 settabV(L, L->top, tabref(funcV(o)->c.env)); 880 settabV(L, L->top, tabref(funcV(o)->c.env));
817 } else if (tvisudata(o)) { 881 } else if (tvisudata(o)) {
@@ -828,12 +892,14 @@ LUA_API int lua_next(lua_State *L, int idx)
828{ 892{
829 cTValue *t = index2adr(L, idx); 893 cTValue *t = index2adr(L, idx);
830 int more; 894 int more;
831 api_check(L, tvistab(t)); 895 lj_checkapi(tvistab(t), "stack slot %d is not a table", idx);
832 more = lj_tab_next(L, tabV(t), L->top-1); 896 more = lj_tab_next(tabV(t), L->top-1, L->top-1);
833 if (more) { 897 if (more > 0) {
834 incr_top(L); /* Return new key and value slot. */ 898 incr_top(L); /* Return new key and value slot. */
835 } else { /* End of traversal. */ 899 } else if (!more) { /* End of traversal. */
836 L->top--; /* Remove key slot. */ 900 L->top--; /* Remove key slot. */
901 } else {
902 lj_err_msg(L, LJ_ERR_NEXTIDX);
837 } 903 }
838 return more; 904 return more;
839} 905}
@@ -854,7 +920,7 @@ LUA_API void *lua_upvalueid(lua_State *L, int idx, int n)
854{ 920{
855 GCfunc *fn = funcV(index2adr(L, idx)); 921 GCfunc *fn = funcV(index2adr(L, idx));
856 n--; 922 n--;
857 api_check(L, (uint32_t)n < fn->l.nupvalues); 923 lj_checkapi((uint32_t)n < fn->l.nupvalues, "bad upvalue %d", n);
858 return isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) : 924 return isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
859 (void *)&fn->c.upvalue[n]; 925 (void *)&fn->c.upvalue[n];
860} 926}
@@ -864,13 +930,15 @@ LUA_API void lua_upvaluejoin(lua_State *L, int idx1, int n1, int idx2, int n2)
864 GCfunc *fn1 = funcV(index2adr(L, idx1)); 930 GCfunc *fn1 = funcV(index2adr(L, idx1));
865 GCfunc *fn2 = funcV(index2adr(L, idx2)); 931 GCfunc *fn2 = funcV(index2adr(L, idx2));
866 n1--; n2--; 932 n1--; n2--;
867 api_check(L, isluafunc(fn1) && (uint32_t)n1 < fn1->l.nupvalues); 933 lj_checkapi(isluafunc(fn1), "stack slot %d is not a Lua function", idx1);
868 api_check(L, isluafunc(fn2) && (uint32_t)n2 < fn2->l.nupvalues); 934 lj_checkapi(isluafunc(fn2), "stack slot %d is not a Lua function", idx2);
935 lj_checkapi((uint32_t)n1 < fn1->l.nupvalues, "bad upvalue %d", n1+1);
936 lj_checkapi((uint32_t)n2 < fn2->l.nupvalues, "bad upvalue %d", n2+1);
869 setgcrefr(fn1->l.uvptr[n1], fn2->l.uvptr[n2]); 937 setgcrefr(fn1->l.uvptr[n1], fn2->l.uvptr[n2]);
870 lj_gc_objbarrier(L, fn1, gcref(fn1->l.uvptr[n1])); 938 lj_gc_objbarrier(L, fn1, gcref(fn1->l.uvptr[n1]));
871} 939}
872 940
873LUALIB_API void *luaL_checkudata(lua_State *L, int idx, const char *tname) 941LUALIB_API void *luaL_testudata(lua_State *L, int idx, const char *tname)
874{ 942{
875 cTValue *o = index2adr(L, idx); 943 cTValue *o = index2adr(L, idx);
876 if (tvisudata(o)) { 944 if (tvisudata(o)) {
@@ -879,8 +947,14 @@ LUALIB_API void *luaL_checkudata(lua_State *L, int idx, const char *tname)
879 if (tv && tvistab(tv) && tabV(tv) == tabref(ud->metatable)) 947 if (tv && tvistab(tv) && tabV(tv) == tabref(ud->metatable))
880 return uddata(ud); 948 return uddata(ud);
881 } 949 }
882 lj_err_argtype(L, idx, tname); 950 return NULL; /* value is not a userdata with a metatable */
883 return NULL; /* unreachable */ 951}
952
953LUALIB_API void *luaL_checkudata(lua_State *L, int idx, const char *tname)
954{
955 void *p = luaL_testudata(L, idx, tname);
956 if (!p) lj_err_argtype(L, idx, tname);
957 return p;
884} 958}
885 959
886/* -- Object setters ------------------------------------------------------ */ 960/* -- Object setters ------------------------------------------------------ */
@@ -888,19 +962,19 @@ LUALIB_API void *luaL_checkudata(lua_State *L, int idx, const char *tname)
888LUA_API void lua_settable(lua_State *L, int idx) 962LUA_API void lua_settable(lua_State *L, int idx)
889{ 963{
890 TValue *o; 964 TValue *o;
891 cTValue *t = index2adr(L, idx); 965 cTValue *t = index2adr_check(L, idx);
892 api_checknelems(L, 2); 966 lj_checkapi_slot(2);
893 api_checkvalidindex(L, t);
894 o = lj_meta_tset(L, t, L->top-2); 967 o = lj_meta_tset(L, t, L->top-2);
895 if (o) { 968 if (o) {
896 /* NOBARRIER: lj_meta_tset ensures the table is not black. */ 969 /* NOBARRIER: lj_meta_tset ensures the table is not black. */
897 copyTV(L, o, L->top-1);
898 L->top -= 2; 970 L->top -= 2;
971 copyTV(L, o, L->top+1);
899 } else { 972 } else {
900 L->top += 3; 973 TValue *base = L->top;
901 copyTV(L, L->top-1, L->top-6); 974 copyTV(L, base+2, base-3-2*LJ_FR2);
902 lj_vm_call(L, L->top-3, 0+1); 975 L->top = base+3;
903 L->top -= 3; 976 lj_vm_call(L, base, 0+1);
977 L->top -= 3+LJ_FR2;
904 } 978 }
905} 979}
906 980
@@ -908,20 +982,19 @@ LUA_API void lua_setfield(lua_State *L, int idx, const char *k)
908{ 982{
909 TValue *o; 983 TValue *o;
910 TValue key; 984 TValue key;
911 cTValue *t = index2adr(L, idx); 985 cTValue *t = index2adr_check(L, idx);
912 api_checknelems(L, 1); 986 lj_checkapi_slot(1);
913 api_checkvalidindex(L, t);
914 setstrV(L, &key, lj_str_newz(L, k)); 987 setstrV(L, &key, lj_str_newz(L, k));
915 o = lj_meta_tset(L, t, &key); 988 o = lj_meta_tset(L, t, &key);
916 if (o) { 989 if (o) {
917 L->top--;
918 /* NOBARRIER: lj_meta_tset ensures the table is not black. */ 990 /* NOBARRIER: lj_meta_tset ensures the table is not black. */
919 copyTV(L, o, L->top); 991 copyTV(L, o, --L->top);
920 } else { 992 } else {
921 L->top += 3; 993 TValue *base = L->top;
922 copyTV(L, L->top-1, L->top-6); 994 copyTV(L, base+2, base-3-2*LJ_FR2);
923 lj_vm_call(L, L->top-3, 0+1); 995 L->top = base+3;
924 L->top -= 2; 996 lj_vm_call(L, base, 0+1);
997 L->top -= 2+LJ_FR2;
925 } 998 }
926} 999}
927 1000
@@ -929,7 +1002,7 @@ LUA_API void lua_rawset(lua_State *L, int idx)
929{ 1002{
930 GCtab *t = tabV(index2adr(L, idx)); 1003 GCtab *t = tabV(index2adr(L, idx));
931 TValue *dst, *key; 1004 TValue *dst, *key;
932 api_checknelems(L, 2); 1005 lj_checkapi_slot(2);
933 key = L->top-2; 1006 key = L->top-2;
934 dst = lj_tab_set(L, t, key); 1007 dst = lj_tab_set(L, t, key);
935 copyTV(L, dst, key+1); 1008 copyTV(L, dst, key+1);
@@ -941,7 +1014,7 @@ LUA_API void lua_rawseti(lua_State *L, int idx, int n)
941{ 1014{
942 GCtab *t = tabV(index2adr(L, idx)); 1015 GCtab *t = tabV(index2adr(L, idx));
943 TValue *dst, *src; 1016 TValue *dst, *src;
944 api_checknelems(L, 1); 1017 lj_checkapi_slot(1);
945 dst = lj_tab_setint(L, t, n); 1018 dst = lj_tab_setint(L, t, n);
946 src = L->top-1; 1019 src = L->top-1;
947 copyTV(L, dst, src); 1020 copyTV(L, dst, src);
@@ -953,13 +1026,12 @@ LUA_API int lua_setmetatable(lua_State *L, int idx)
953{ 1026{
954 global_State *g; 1027 global_State *g;
955 GCtab *mt; 1028 GCtab *mt;
956 cTValue *o = index2adr(L, idx); 1029 cTValue *o = index2adr_check(L, idx);
957 api_checknelems(L, 1); 1030 lj_checkapi_slot(1);
958 api_checkvalidindex(L, o);
959 if (tvisnil(L->top-1)) { 1031 if (tvisnil(L->top-1)) {
960 mt = NULL; 1032 mt = NULL;
961 } else { 1033 } else {
962 api_check(L, tvistab(L->top-1)); 1034 lj_checkapi(tvistab(L->top-1), "top stack slot is not a table");
963 mt = tabV(L->top-1); 1035 mt = tabV(L->top-1);
964 } 1036 }
965 g = G(L); 1037 g = G(L);
@@ -988,13 +1060,18 @@ LUA_API int lua_setmetatable(lua_State *L, int idx)
988 return 1; 1060 return 1;
989} 1061}
990 1062
1063LUALIB_API void luaL_setmetatable(lua_State *L, const char *tname)
1064{
1065 lua_getfield(L, LUA_REGISTRYINDEX, tname);
1066 lua_setmetatable(L, -2);
1067}
1068
991LUA_API int lua_setfenv(lua_State *L, int idx) 1069LUA_API int lua_setfenv(lua_State *L, int idx)
992{ 1070{
993 cTValue *o = index2adr(L, idx); 1071 cTValue *o = index2adr_check(L, idx);
994 GCtab *t; 1072 GCtab *t;
995 api_checknelems(L, 1); 1073 lj_checkapi_slot(1);
996 api_checkvalidindex(L, o); 1074 lj_checkapi(tvistab(L->top-1), "top stack slot is not a table");
997 api_check(L, tvistab(L->top-1));
998 t = tabV(L->top-1); 1075 t = tabV(L->top-1);
999 if (tvisfunc(o)) { 1076 if (tvisfunc(o)) {
1000 setgcref(funcV(o)->c.env, obj2gco(t)); 1077 setgcref(funcV(o)->c.env, obj2gco(t));
@@ -1017,7 +1094,7 @@ LUA_API const char *lua_setupvalue(lua_State *L, int idx, int n)
1017 TValue *val; 1094 TValue *val;
1018 GCobj *o; 1095 GCobj *o;
1019 const char *name; 1096 const char *name;
1020 api_checknelems(L, 1); 1097 lj_checkapi_slot(1);
1021 name = lj_debug_uvnamev(f, (uint32_t)(n-1), &val, &o); 1098 name = lj_debug_uvnamev(f, (uint32_t)(n-1), &val, &o);
1022 if (name) { 1099 if (name) {
1023 L->top--; 1100 L->top--;
@@ -1029,11 +1106,25 @@ LUA_API const char *lua_setupvalue(lua_State *L, int idx, int n)
1029 1106
1030/* -- Calls --------------------------------------------------------------- */ 1107/* -- Calls --------------------------------------------------------------- */
1031 1108
1109#if LJ_FR2
1110static TValue *api_call_base(lua_State *L, int nargs)
1111{
1112 TValue *o = L->top, *base = o - nargs;
1113 L->top = o+1;
1114 for (; o > base; o--) copyTV(L, o, o-1);
1115 setnilV(o);
1116 return o+1;
1117}
1118#else
1119#define api_call_base(L, nargs) (L->top - (nargs))
1120#endif
1121
1032LUA_API void lua_call(lua_State *L, int nargs, int nresults) 1122LUA_API void lua_call(lua_State *L, int nargs, int nresults)
1033{ 1123{
1034 api_check(L, L->status == 0 || L->status == LUA_ERRERR); 1124 lj_checkapi(L->status == LUA_OK || L->status == LUA_ERRERR,
1035 api_checknelems(L, nargs+1); 1125 "thread called in wrong state %d", L->status);
1036 lj_vm_call(L, L->top - nargs, nresults+1); 1126 lj_checkapi_slot(nargs+1);
1127 lj_vm_call(L, api_call_base(L, nargs), nresults+1);
1037} 1128}
1038 1129
1039LUA_API int lua_pcall(lua_State *L, int nargs, int nresults, int errfunc) 1130LUA_API int lua_pcall(lua_State *L, int nargs, int nresults, int errfunc)
@@ -1042,16 +1133,16 @@ LUA_API int lua_pcall(lua_State *L, int nargs, int nresults, int errfunc)
1042 uint8_t oldh = hook_save(g); 1133 uint8_t oldh = hook_save(g);
1043 ptrdiff_t ef; 1134 ptrdiff_t ef;
1044 int status; 1135 int status;
1045 api_check(L, L->status == 0 || L->status == LUA_ERRERR); 1136 lj_checkapi(L->status == LUA_OK || L->status == LUA_ERRERR,
1046 api_checknelems(L, nargs+1); 1137 "thread called in wrong state %d", L->status);
1138 lj_checkapi_slot(nargs+1);
1047 if (errfunc == 0) { 1139 if (errfunc == 0) {
1048 ef = 0; 1140 ef = 0;
1049 } else { 1141 } else {
1050 cTValue *o = stkindex2adr(L, errfunc); 1142 cTValue *o = index2adr_stack(L, errfunc);
1051 api_checkvalidindex(L, o);
1052 ef = savestack(L, o); 1143 ef = savestack(L, o);
1053 } 1144 }
1054 status = lj_vm_pcall(L, L->top - nargs, nresults+1, ef); 1145 status = lj_vm_pcall(L, api_call_base(L, nargs), nresults+1, ef);
1055 if (status) hook_restore(g, oldh); 1146 if (status) hook_restore(g, oldh);
1056 return status; 1147 return status;
1057} 1148}
@@ -1059,12 +1150,17 @@ LUA_API int lua_pcall(lua_State *L, int nargs, int nresults, int errfunc)
1059static TValue *cpcall(lua_State *L, lua_CFunction func, void *ud) 1150static TValue *cpcall(lua_State *L, lua_CFunction func, void *ud)
1060{ 1151{
1061 GCfunc *fn = lj_func_newC(L, 0, getcurrenv(L)); 1152 GCfunc *fn = lj_func_newC(L, 0, getcurrenv(L));
1153 TValue *top = L->top;
1062 fn->c.f = func; 1154 fn->c.f = func;
1063 setfuncV(L, L->top, fn); 1155 setfuncV(L, top++, fn);
1064 setlightudV(L->top+1, checklightudptr(L, ud)); 1156 if (LJ_FR2) setnilV(top++);
1157#if LJ_64
1158 ud = lj_lightud_intern(L, ud);
1159#endif
1160 setrawlightudV(top++, ud);
1065 cframe_nres(L->cframe) = 1+0; /* Zero results. */ 1161 cframe_nres(L->cframe) = 1+0; /* Zero results. */
1066 L->top += 2; 1162 L->top = top;
1067 return L->top-1; /* Now call the newly allocated C function. */ 1163 return top-1; /* Now call the newly allocated C function. */
1068} 1164}
1069 1165
1070LUA_API int lua_cpcall(lua_State *L, lua_CFunction func, void *ud) 1166LUA_API int lua_cpcall(lua_State *L, lua_CFunction func, void *ud)
@@ -1072,7 +1168,8 @@ LUA_API int lua_cpcall(lua_State *L, lua_CFunction func, void *ud)
1072 global_State *g = G(L); 1168 global_State *g = G(L);
1073 uint8_t oldh = hook_save(g); 1169 uint8_t oldh = hook_save(g);
1074 int status; 1170 int status;
1075 api_check(L, L->status == 0 || L->status == LUA_ERRERR); 1171 lj_checkapi(L->status == LUA_OK || L->status == LUA_ERRERR,
1172 "thread called in wrong state %d", L->status);
1076 status = lj_vm_cpcall(L, func, ud, cpcall); 1173 status = lj_vm_cpcall(L, func, ud, cpcall);
1077 if (status) hook_restore(g, oldh); 1174 if (status) hook_restore(g, oldh);
1078 return status; 1175 return status;
@@ -1081,10 +1178,11 @@ LUA_API int lua_cpcall(lua_State *L, lua_CFunction func, void *ud)
1081LUALIB_API int luaL_callmeta(lua_State *L, int idx, const char *field) 1178LUALIB_API int luaL_callmeta(lua_State *L, int idx, const char *field)
1082{ 1179{
1083 if (luaL_getmetafield(L, idx, field)) { 1180 if (luaL_getmetafield(L, idx, field)) {
1084 TValue *base = L->top--; 1181 TValue *top = L->top--;
1085 copyTV(L, base, index2adr(L, idx)); 1182 if (LJ_FR2) setnilV(top++);
1086 L->top = base+1; 1183 copyTV(L, top++, index2adr(L, idx));
1087 lj_vm_call(L, base, 1+1); 1184 L->top = top;
1185 lj_vm_call(L, top-1, 1+1);
1088 return 1; 1186 return 1;
1089 } 1187 }
1090 return 0; 1188 return 0;
@@ -1092,6 +1190,11 @@ LUALIB_API int luaL_callmeta(lua_State *L, int idx, const char *field)
1092 1190
1093/* -- Coroutine yield and resume ------------------------------------------ */ 1191/* -- Coroutine yield and resume ------------------------------------------ */
1094 1192
1193LUA_API int lua_isyieldable(lua_State *L)
1194{
1195 return cframe_canyield(L->cframe);
1196}
1197
1095LUA_API int lua_yield(lua_State *L, int nresults) 1198LUA_API int lua_yield(lua_State *L, int nresults)
1096{ 1199{
1097 void *cf = L->cframe; 1200 void *cf = L->cframe;
@@ -1111,13 +1214,16 @@ LUA_API int lua_yield(lua_State *L, int nresults)
1111 } else { /* Yield from hook: add a pseudo-frame. */ 1214 } else { /* Yield from hook: add a pseudo-frame. */
1112 TValue *top = L->top; 1215 TValue *top = L->top;
1113 hook_leave(g); 1216 hook_leave(g);
1114 top->u64 = cframe_multres(cf); 1217 (top++)->u64 = cframe_multres(cf);
1115 setcont(top+1, lj_cont_hook); 1218 setcont(top, lj_cont_hook);
1116 setframe_pc(top+1, cframe_pc(cf)-1); 1219 if (LJ_FR2) top++;
1117 setframe_gc(top+2, obj2gco(L)); 1220 setframe_pc(top, cframe_pc(cf)-1);
1118 setframe_ftsz(top+2, (int)((char *)(top+3)-(char *)L->base)+FRAME_CONT); 1221 top++;
1119 L->top = L->base = top+3; 1222 setframe_gc(top, obj2gco(L), LJ_TTHREAD);
1120#if LJ_TARGET_X64 1223 if (LJ_FR2) top++;
1224 setframe_ftsz(top, ((char *)(top+1)-(char *)L->base)+FRAME_CONT);
1225 L->top = L->base = top+1;
1226#if ((defined(__GNUC__) || defined(__clang__)) && (LJ_TARGET_X64 || defined(LUAJIT_UNWIND_EXTERNAL)) && !LJ_NO_UNWIND) || LJ_TARGET_WINDOWS
1121 lj_err_throw(L, LUA_YIELD); 1227 lj_err_throw(L, LUA_YIELD);
1122#else 1228#else
1123 L->cframe = NULL; 1229 L->cframe = NULL;
@@ -1133,7 +1239,9 @@ LUA_API int lua_yield(lua_State *L, int nresults)
1133LUA_API int lua_resume(lua_State *L, int nargs) 1239LUA_API int lua_resume(lua_State *L, int nargs)
1134{ 1240{
1135 if (L->cframe == NULL && L->status <= LUA_YIELD) 1241 if (L->cframe == NULL && L->status <= LUA_YIELD)
1136 return lj_vm_resume(L, L->top - nargs, 0, 0); 1242 return lj_vm_resume(L,
1243 L->status == LUA_OK ? api_call_base(L, nargs) : L->top - nargs,
1244 0, 0);
1137 L->top = L->base; 1245 L->top = L->base;
1138 setstrV(L, L->top, lj_err_str(L, LJ_ERR_COSUSP)); 1246 setstrV(L, L->top, lj_err_str(L, LJ_ERR_COSUSP));
1139 incr_top(L); 1247 incr_top(L);
@@ -1163,7 +1271,7 @@ LUA_API int lua_gc(lua_State *L, int what, int data)
1163 res = (int)(g->gc.total & 0x3ff); 1271 res = (int)(g->gc.total & 0x3ff);
1164 break; 1272 break;
1165 case LUA_GCSTEP: { 1273 case LUA_GCSTEP: {
1166 MSize a = (MSize)data << 10; 1274 GCSize a = (GCSize)data << 10;
1167 g->gc.threshold = (a <= g->gc.total) ? (g->gc.total - a) : 0; 1275 g->gc.threshold = (a <= g->gc.total) ? (g->gc.total - a) : 0;
1168 while (g->gc.total >= g->gc.threshold) 1276 while (g->gc.total >= g->gc.threshold)
1169 if (lj_gc_step(L) > 0) { 1277 if (lj_gc_step(L) > 0) {
@@ -1180,6 +1288,9 @@ LUA_API int lua_gc(lua_State *L, int what, int data)
1180 res = (int)(g->gc.stepmul); 1288 res = (int)(g->gc.stepmul);
1181 g->gc.stepmul = (MSize)data; 1289 g->gc.stepmul = (MSize)data;
1182 break; 1290 break;
1291 case LUA_GCISRUNNING:
1292 res = (g->gc.threshold != LJ_MAX_MEM);
1293 break;
1183 default: 1294 default:
1184 res = -1; /* Invalid option. */ 1295 res = -1; /* Invalid option. */
1185 } 1296 }
diff --git a/src/lj_arch.h b/src/lj_arch.h
index da53b162..6e22b1b2 100644
--- a/src/lj_arch.h
+++ b/src/lj_arch.h
@@ -8,6 +8,8 @@
8 8
9#include "lua.h" 9#include "lua.h"
10 10
11/* -- Target definitions -------------------------------------------------- */
12
11/* Target endianess. */ 13/* Target endianess. */
12#define LUAJIT_LE 0 14#define LUAJIT_LE 0
13#define LUAJIT_BE 1 15#define LUAJIT_BE 1
@@ -19,12 +21,16 @@
19#define LUAJIT_ARCH_x64 2 21#define LUAJIT_ARCH_x64 2
20#define LUAJIT_ARCH_ARM 3 22#define LUAJIT_ARCH_ARM 3
21#define LUAJIT_ARCH_arm 3 23#define LUAJIT_ARCH_arm 3
22#define LUAJIT_ARCH_PPC 4 24#define LUAJIT_ARCH_ARM64 4
23#define LUAJIT_ARCH_ppc 4 25#define LUAJIT_ARCH_arm64 4
24#define LUAJIT_ARCH_PPCSPE 5 26#define LUAJIT_ARCH_PPC 5
25#define LUAJIT_ARCH_ppcspe 5 27#define LUAJIT_ARCH_ppc 5
26#define LUAJIT_ARCH_MIPS 6 28#define LUAJIT_ARCH_MIPS 6
27#define LUAJIT_ARCH_mips 6 29#define LUAJIT_ARCH_mips 6
30#define LUAJIT_ARCH_MIPS32 6
31#define LUAJIT_ARCH_mips32 6
32#define LUAJIT_ARCH_MIPS64 7
33#define LUAJIT_ARCH_mips64 7
28 34
29/* Target OS. */ 35/* Target OS. */
30#define LUAJIT_OS_OTHER 0 36#define LUAJIT_OS_OTHER 0
@@ -34,6 +40,14 @@
34#define LUAJIT_OS_BSD 4 40#define LUAJIT_OS_BSD 4
35#define LUAJIT_OS_POSIX 5 41#define LUAJIT_OS_POSIX 5
36 42
43/* Number mode. */
44#define LJ_NUMMODE_SINGLE 0 /* Single-number mode only. */
45#define LJ_NUMMODE_SINGLE_DUAL 1 /* Default to single-number mode. */
46#define LJ_NUMMODE_DUAL 2 /* Dual-number mode only. */
47#define LJ_NUMMODE_DUAL_SINGLE 3 /* Default to dual-number mode. */
48
49/* -- Target detection ---------------------------------------------------- */
50
37/* Select native target if no target defined. */ 51/* Select native target if no target defined. */
38#ifndef LUAJIT_TARGET 52#ifndef LUAJIT_TARGET
39 53
@@ -43,14 +57,14 @@
43#define LUAJIT_TARGET LUAJIT_ARCH_X64 57#define LUAJIT_TARGET LUAJIT_ARCH_X64
44#elif defined(__arm__) || defined(__arm) || defined(__ARM__) || defined(__ARM) 58#elif defined(__arm__) || defined(__arm) || defined(__ARM__) || defined(__ARM)
45#define LUAJIT_TARGET LUAJIT_ARCH_ARM 59#define LUAJIT_TARGET LUAJIT_ARCH_ARM
60#elif defined(__aarch64__)
61#define LUAJIT_TARGET LUAJIT_ARCH_ARM64
46#elif defined(__ppc__) || defined(__ppc) || defined(__PPC__) || defined(__PPC) || defined(__powerpc__) || defined(__powerpc) || defined(__POWERPC__) || defined(__POWERPC) || defined(_M_PPC) 62#elif defined(__ppc__) || defined(__ppc) || defined(__PPC__) || defined(__PPC) || defined(__powerpc__) || defined(__powerpc) || defined(__POWERPC__) || defined(__POWERPC) || defined(_M_PPC)
47#ifdef __NO_FPRS__
48#define LUAJIT_TARGET LUAJIT_ARCH_PPCSPE
49#else
50#define LUAJIT_TARGET LUAJIT_ARCH_PPC 63#define LUAJIT_TARGET LUAJIT_ARCH_PPC
51#endif 64#elif defined(__mips64__) || defined(__mips64) || defined(__MIPS64__) || defined(__MIPS64)
65#define LUAJIT_TARGET LUAJIT_ARCH_MIPS64
52#elif defined(__mips__) || defined(__mips) || defined(__MIPS__) || defined(__MIPS) 66#elif defined(__mips__) || defined(__mips) || defined(__MIPS__) || defined(__MIPS)
53#define LUAJIT_TARGET LUAJIT_ARCH_MIPS 67#define LUAJIT_TARGET LUAJIT_ARCH_MIPS32
54#else 68#else
55#error "No support for this architecture (yet)" 69#error "No support for this architecture (yet)"
56#endif 70#endif
@@ -65,16 +79,23 @@
65#elif defined(__linux__) 79#elif defined(__linux__)
66#define LUAJIT_OS LUAJIT_OS_LINUX 80#define LUAJIT_OS LUAJIT_OS_LINUX
67#elif defined(__MACH__) && defined(__APPLE__) 81#elif defined(__MACH__) && defined(__APPLE__)
82#include "TargetConditionals.h"
68#define LUAJIT_OS LUAJIT_OS_OSX 83#define LUAJIT_OS LUAJIT_OS_OSX
69#elif (defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || \ 84#elif (defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || \
70 defined(__NetBSD__) || defined(__OpenBSD__) || \ 85 defined(__NetBSD__) || defined(__OpenBSD__) || \
71 defined(__DragonFly__)) && !defined(__ORBIS__) 86 defined(__DragonFly__)) && !defined(__ORBIS__) && !defined(__PROSPERO__)
72#define LUAJIT_OS LUAJIT_OS_BSD 87#define LUAJIT_OS LUAJIT_OS_BSD
73#elif (defined(__sun__) && defined(__svr4__)) 88#elif (defined(__sun__) && defined(__svr4__))
89#define LJ_TARGET_SOLARIS 1
90#define LUAJIT_OS LUAJIT_OS_POSIX
91#elif defined(__HAIKU__)
74#define LUAJIT_OS LUAJIT_OS_POSIX 92#define LUAJIT_OS LUAJIT_OS_POSIX
75#elif defined(__CYGWIN__) 93#elif defined(__CYGWIN__)
76#define LJ_TARGET_CYGWIN 1 94#define LJ_TARGET_CYGWIN 1
77#define LUAJIT_OS LUAJIT_OS_POSIX 95#define LUAJIT_OS LUAJIT_OS_POSIX
96#elif defined(__QNX__)
97#define LJ_TARGET_QNX 1
98#define LUAJIT_OS LUAJIT_OS_POSIX
78#else 99#else
79#define LUAJIT_OS LUAJIT_OS_OTHER 100#define LUAJIT_OS LUAJIT_OS_OTHER
80#endif 101#endif
@@ -99,10 +120,16 @@
99#define LJ_TARGET_WINDOWS (LUAJIT_OS == LUAJIT_OS_WINDOWS) 120#define LJ_TARGET_WINDOWS (LUAJIT_OS == LUAJIT_OS_WINDOWS)
100#define LJ_TARGET_LINUX (LUAJIT_OS == LUAJIT_OS_LINUX) 121#define LJ_TARGET_LINUX (LUAJIT_OS == LUAJIT_OS_LINUX)
101#define LJ_TARGET_OSX (LUAJIT_OS == LUAJIT_OS_OSX) 122#define LJ_TARGET_OSX (LUAJIT_OS == LUAJIT_OS_OSX)
102#define LJ_TARGET_IOS (LJ_TARGET_OSX && LUAJIT_TARGET == LUAJIT_ARCH_ARM) 123#define LJ_TARGET_BSD (LUAJIT_OS == LUAJIT_OS_BSD)
103#define LJ_TARGET_POSIX (LUAJIT_OS > LUAJIT_OS_WINDOWS) 124#define LJ_TARGET_POSIX (LUAJIT_OS > LUAJIT_OS_WINDOWS)
104#define LJ_TARGET_DLOPEN LJ_TARGET_POSIX 125#define LJ_TARGET_DLOPEN LJ_TARGET_POSIX
105 126
127#if TARGET_OS_IPHONE
128#define LJ_TARGET_IOS 1
129#else
130#define LJ_TARGET_IOS 0
131#endif
132
106#ifdef __CELLOS_LV2__ 133#ifdef __CELLOS_LV2__
107#define LJ_TARGET_PS3 1 134#define LJ_TARGET_PS3 1
108#define LJ_TARGET_CONSOLE 1 135#define LJ_TARGET_CONSOLE 1
@@ -115,6 +142,13 @@
115#define NULL ((void*)0) 142#define NULL ((void*)0)
116#endif 143#endif
117 144
145#ifdef __PROSPERO__
146#define LJ_TARGET_PS5 1
147#define LJ_TARGET_CONSOLE 1
148#undef NULL
149#define NULL ((void*)0)
150#endif
151
118#ifdef __psp2__ 152#ifdef __psp2__
119#define LJ_TARGET_PSVITA 1 153#define LJ_TARGET_PSVITA 1
120#define LJ_TARGET_CONSOLE 1 154#define LJ_TARGET_CONSOLE 1
@@ -125,10 +159,27 @@
125#define LJ_TARGET_CONSOLE 1 159#define LJ_TARGET_CONSOLE 1
126#endif 160#endif
127 161
128#define LJ_NUMMODE_SINGLE 0 /* Single-number mode only. */ 162#ifdef _DURANGO
129#define LJ_NUMMODE_SINGLE_DUAL 1 /* Default to single-number mode. */ 163#define LJ_TARGET_XBOXONE 1
130#define LJ_NUMMODE_DUAL 2 /* Dual-number mode only. */ 164#define LJ_TARGET_CONSOLE 1
131#define LJ_NUMMODE_DUAL_SINGLE 3 /* Default to dual-number mode. */ 165#define LJ_TARGET_GC64 1
166#endif
167
168#ifdef __NX__
169#define LJ_TARGET_NX 1
170#define LJ_TARGET_CONSOLE 1
171#undef NULL
172#define NULL ((void*)0)
173#endif
174
175#ifdef _UWP
176#define LJ_TARGET_UWP 1
177#if LUAJIT_TARGET == LUAJIT_ARCH_X64
178#define LJ_TARGET_GC64 1
179#endif
180#endif
181
182/* -- Arch-specific settings ---------------------------------------------- */
132 183
133/* Set target architecture properties. */ 184/* Set target architecture properties. */
134#if LUAJIT_TARGET == LUAJIT_ARCH_X86 185#if LUAJIT_TARGET == LUAJIT_ARCH_X86
@@ -136,14 +187,10 @@
136#define LJ_ARCH_NAME "x86" 187#define LJ_ARCH_NAME "x86"
137#define LJ_ARCH_BITS 32 188#define LJ_ARCH_BITS 32
138#define LJ_ARCH_ENDIAN LUAJIT_LE 189#define LJ_ARCH_ENDIAN LUAJIT_LE
139#if LJ_TARGET_WINDOWS || LJ_TARGET_CYGWIN
140#define LJ_ABI_WIN 1
141#else
142#define LJ_ABI_WIN 0
143#endif
144#define LJ_TARGET_X86 1 190#define LJ_TARGET_X86 1
145#define LJ_TARGET_X86ORX64 1 191#define LJ_TARGET_X86ORX64 1
146#define LJ_TARGET_EHRETREG 0 192#define LJ_TARGET_EHRETREG 0
193#define LJ_TARGET_EHRAREG 8
147#define LJ_TARGET_MASKSHIFT 1 194#define LJ_TARGET_MASKSHIFT 1
148#define LJ_TARGET_MASKROT 1 195#define LJ_TARGET_MASKROT 1
149#define LJ_TARGET_UNALIGNED 1 196#define LJ_TARGET_UNALIGNED 1
@@ -154,19 +201,20 @@
154#define LJ_ARCH_NAME "x64" 201#define LJ_ARCH_NAME "x64"
155#define LJ_ARCH_BITS 64 202#define LJ_ARCH_BITS 64
156#define LJ_ARCH_ENDIAN LUAJIT_LE 203#define LJ_ARCH_ENDIAN LUAJIT_LE
157#if LJ_TARGET_WINDOWS || LJ_TARGET_CYGWIN
158#define LJ_ABI_WIN 1
159#else
160#define LJ_ABI_WIN 0
161#endif
162#define LJ_TARGET_X64 1 204#define LJ_TARGET_X64 1
163#define LJ_TARGET_X86ORX64 1 205#define LJ_TARGET_X86ORX64 1
164#define LJ_TARGET_EHRETREG 0 206#define LJ_TARGET_EHRETREG 0
207#define LJ_TARGET_EHRAREG 16
165#define LJ_TARGET_JUMPRANGE 31 /* +-2^31 = +-2GB */ 208#define LJ_TARGET_JUMPRANGE 31 /* +-2^31 = +-2GB */
166#define LJ_TARGET_MASKSHIFT 1 209#define LJ_TARGET_MASKSHIFT 1
167#define LJ_TARGET_MASKROT 1 210#define LJ_TARGET_MASKROT 1
168#define LJ_TARGET_UNALIGNED 1 211#define LJ_TARGET_UNALIGNED 1
169#define LJ_ARCH_NUMMODE LJ_NUMMODE_SINGLE_DUAL 212#define LJ_ARCH_NUMMODE LJ_NUMMODE_SINGLE_DUAL
213#ifndef LUAJIT_DISABLE_GC64
214#define LJ_TARGET_GC64 1
215#elif LJ_TARGET_OSX
216#error "macOS requires GC64 -- don't disable it"
217#endif
170 218
171#elif LUAJIT_TARGET == LUAJIT_ARCH_ARM 219#elif LUAJIT_TARGET == LUAJIT_ARCH_ARM
172 220
@@ -182,40 +230,108 @@
182#define LJ_ABI_EABI 1 230#define LJ_ABI_EABI 1
183#define LJ_TARGET_ARM 1 231#define LJ_TARGET_ARM 1
184#define LJ_TARGET_EHRETREG 0 232#define LJ_TARGET_EHRETREG 0
233#define LJ_TARGET_EHRAREG 14
185#define LJ_TARGET_JUMPRANGE 25 /* +-2^25 = +-32MB */ 234#define LJ_TARGET_JUMPRANGE 25 /* +-2^25 = +-32MB */
186#define LJ_TARGET_MASKSHIFT 0 235#define LJ_TARGET_MASKSHIFT 0
187#define LJ_TARGET_MASKROT 1 236#define LJ_TARGET_MASKROT 1
188#define LJ_TARGET_UNIFYROT 2 /* Want only IR_BROR. */ 237#define LJ_TARGET_UNIFYROT 2 /* Want only IR_BROR. */
189#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL 238#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL
190 239
191#if __ARM_ARCH____ARM_ARCH_8__ || __ARM_ARCH_8A__ 240#if __ARM_ARCH == 8 || __ARM_ARCH_8__ || __ARM_ARCH_8A__
192#define LJ_ARCH_VERSION 80 241#define LJ_ARCH_VERSION 80
193#elif __ARM_ARCH_7__ || __ARM_ARCH_7A__ || __ARM_ARCH_7R__ || __ARM_ARCH_7S__ || __ARM_ARCH_7VE__ 242#elif __ARM_ARCH == 7 || __ARM_ARCH_7__ || __ARM_ARCH_7A__ || __ARM_ARCH_7R__ || __ARM_ARCH_7S__ || __ARM_ARCH_7VE__
194#define LJ_ARCH_VERSION 70 243#define LJ_ARCH_VERSION 70
195#elif __ARM_ARCH_6T2__ 244#elif __ARM_ARCH_6T2__
196#define LJ_ARCH_VERSION 61 245#define LJ_ARCH_VERSION 61
197#elif __ARM_ARCH_6__ || __ARM_ARCH_6J__ || __ARM_ARCH_6K__ || __ARM_ARCH_6Z__ || __ARM_ARCH_6ZK__ 246#elif __ARM_ARCH == 6 || __ARM_ARCH_6__ || __ARM_ARCH_6J__ || __ARM_ARCH_6K__ || __ARM_ARCH_6Z__ || __ARM_ARCH_6ZK__
198#define LJ_ARCH_VERSION 60 247#define LJ_ARCH_VERSION 60
199#else 248#else
200#define LJ_ARCH_VERSION 50 249#define LJ_ARCH_VERSION 50
201#endif 250#endif
202 251
252#elif LUAJIT_TARGET == LUAJIT_ARCH_ARM64
253
254#define LJ_ARCH_BITS 64
255#if defined(__AARCH64EB__)
256#define LJ_ARCH_NAME "arm64be"
257#define LJ_ARCH_ENDIAN LUAJIT_BE
258#else
259#define LJ_ARCH_NAME "arm64"
260#define LJ_ARCH_ENDIAN LUAJIT_LE
261#endif
262#if !defined(LJ_ABI_PAUTH) && defined(__arm64e__)
263#define LJ_ABI_PAUTH 1
264#endif
265#define LJ_TARGET_ARM64 1
266#define LJ_TARGET_EHRETREG 0
267#define LJ_TARGET_EHRAREG 30
268#define LJ_TARGET_JUMPRANGE 27 /* +-2^27 = +-128MB */
269#define LJ_TARGET_MASKSHIFT 1
270#define LJ_TARGET_MASKROT 1
271#define LJ_TARGET_UNIFYROT 2 /* Want only IR_BROR. */
272#define LJ_TARGET_GC64 1
273#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL
274
275#define LJ_ARCH_VERSION 80
276
203#elif LUAJIT_TARGET == LUAJIT_ARCH_PPC 277#elif LUAJIT_TARGET == LUAJIT_ARCH_PPC
204 278
205#define LJ_ARCH_NAME "ppc" 279#ifndef LJ_ARCH_ENDIAN
280#if __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
281#define LJ_ARCH_ENDIAN LUAJIT_LE
282#else
283#define LJ_ARCH_ENDIAN LUAJIT_BE
284#endif
285#endif
286
206#if _LP64 287#if _LP64
207#define LJ_ARCH_BITS 64 288#define LJ_ARCH_BITS 64
289#if LJ_ARCH_ENDIAN == LUAJIT_LE
290#define LJ_ARCH_NAME "ppc64le"
291#else
292#define LJ_ARCH_NAME "ppc64"
293#endif
208#else 294#else
209#define LJ_ARCH_BITS 32 295#define LJ_ARCH_BITS 32
296#define LJ_ARCH_NAME "ppc"
297
298#if !defined(LJ_ARCH_HASFPU)
299#if defined(_SOFT_FLOAT) || defined(_SOFT_DOUBLE)
300#define LJ_ARCH_HASFPU 0
301#else
302#define LJ_ARCH_HASFPU 1
210#endif 303#endif
211#define LJ_ARCH_ENDIAN LUAJIT_BE 304#endif
305
306#if !defined(LJ_ABI_SOFTFP)
307#if defined(_SOFT_FLOAT) || defined(_SOFT_DOUBLE)
308#define LJ_ABI_SOFTFP 1
309#else
310#define LJ_ABI_SOFTFP 0
311#endif
312#endif
313#endif
314
315#if LJ_ABI_SOFTFP
316#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL
317#else
318#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL_SINGLE
319#endif
320
212#define LJ_TARGET_PPC 1 321#define LJ_TARGET_PPC 1
213#define LJ_TARGET_EHRETREG 3 322#define LJ_TARGET_EHRETREG 3
323#define LJ_TARGET_EHRAREG 65
214#define LJ_TARGET_JUMPRANGE 25 /* +-2^25 = +-32MB */ 324#define LJ_TARGET_JUMPRANGE 25 /* +-2^25 = +-32MB */
215#define LJ_TARGET_MASKSHIFT 0 325#define LJ_TARGET_MASKSHIFT 0
216#define LJ_TARGET_MASKROT 1 326#define LJ_TARGET_MASKROT 1
217#define LJ_TARGET_UNIFYROT 1 /* Want only IR_BROL. */ 327#define LJ_TARGET_UNIFYROT 1 /* Want only IR_BROL. */
218#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL_SINGLE 328
329#if LJ_TARGET_CONSOLE
330#define LJ_ARCH_PPC32ON64 1
331#define LJ_ARCH_NOFFI 1
332#elif LJ_ARCH_BITS == 64
333#error "No support for PPC64"
334#endif
219 335
220#if _ARCH_PWR7 336#if _ARCH_PWR7
221#define LJ_ARCH_VERSION 70 337#define LJ_ARCH_VERSION 70
@@ -230,10 +346,6 @@
230#else 346#else
231#define LJ_ARCH_VERSION 0 347#define LJ_ARCH_VERSION 0
232#endif 348#endif
233#if __PPC64__ || __powerpc64__ || LJ_TARGET_CONSOLE
234#define LJ_ARCH_PPC64 1
235#define LJ_ARCH_NOFFI 1
236#endif
237#if _ARCH_PPCSQ 349#if _ARCH_PPCSQ
238#define LJ_ARCH_SQRT 1 350#define LJ_ARCH_SQRT 1
239#endif 351#endif
@@ -247,44 +359,80 @@
247#define LJ_ARCH_XENON 1 359#define LJ_ARCH_XENON 1
248#endif 360#endif
249 361
250#elif LUAJIT_TARGET == LUAJIT_ARCH_PPCSPE 362#elif LUAJIT_TARGET == LUAJIT_ARCH_MIPS32 || LUAJIT_TARGET == LUAJIT_ARCH_MIPS64
251
252#define LJ_ARCH_NAME "ppcspe"
253#define LJ_ARCH_BITS 32
254#define LJ_ARCH_ENDIAN LUAJIT_BE
255#ifndef LJ_ABI_SOFTFP
256#define LJ_ABI_SOFTFP 1
257#endif
258#define LJ_ABI_EABI 1
259#define LJ_TARGET_PPCSPE 1
260#define LJ_TARGET_EHRETREG 3
261#define LJ_TARGET_JUMPRANGE 25 /* +-2^25 = +-32MB */
262#define LJ_TARGET_MASKSHIFT 0
263#define LJ_TARGET_MASKROT 1
264#define LJ_TARGET_UNIFYROT 1 /* Want only IR_BROL. */
265#define LJ_ARCH_NUMMODE LJ_NUMMODE_SINGLE
266#define LJ_ARCH_NOFFI 1 /* NYI: comparisons, calls. */
267#define LJ_ARCH_NOJIT 1
268
269#elif LUAJIT_TARGET == LUAJIT_ARCH_MIPS
270 363
271#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) 364#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL)
365#if __mips_isa_rev >= 6
366#define LJ_TARGET_MIPSR6 1
367#define LJ_TARGET_UNALIGNED 1
368#endif
369#if LUAJIT_TARGET == LUAJIT_ARCH_MIPS32
370#if LJ_TARGET_MIPSR6
371#define LJ_ARCH_NAME "mips32r6el"
372#else
272#define LJ_ARCH_NAME "mipsel" 373#define LJ_ARCH_NAME "mipsel"
374#endif
375#else
376#if LJ_TARGET_MIPSR6
377#define LJ_ARCH_NAME "mips64r6el"
378#else
379#define LJ_ARCH_NAME "mips64el"
380#endif
381#endif
273#define LJ_ARCH_ENDIAN LUAJIT_LE 382#define LJ_ARCH_ENDIAN LUAJIT_LE
274#else 383#else
384#if LUAJIT_TARGET == LUAJIT_ARCH_MIPS32
385#if LJ_TARGET_MIPSR6
386#define LJ_ARCH_NAME "mips32r6"
387#else
275#define LJ_ARCH_NAME "mips" 388#define LJ_ARCH_NAME "mips"
389#endif
390#else
391#if LJ_TARGET_MIPSR6
392#define LJ_ARCH_NAME "mips64r6"
393#else
394#define LJ_ARCH_NAME "mips64"
395#endif
396#endif
276#define LJ_ARCH_ENDIAN LUAJIT_BE 397#define LJ_ARCH_ENDIAN LUAJIT_BE
277#endif 398#endif
399
400#if !defined(LJ_ARCH_HASFPU)
401#ifdef __mips_soft_float
402#define LJ_ARCH_HASFPU 0
403#else
404#define LJ_ARCH_HASFPU 1
405#endif
406#endif
407
408#if !defined(LJ_ABI_SOFTFP)
409#ifdef __mips_soft_float
410#define LJ_ABI_SOFTFP 1
411#else
412#define LJ_ABI_SOFTFP 0
413#endif
414#endif
415
416#if LUAJIT_TARGET == LUAJIT_ARCH_MIPS32
278#define LJ_ARCH_BITS 32 417#define LJ_ARCH_BITS 32
418#define LJ_TARGET_MIPS32 1
419#else
420#define LJ_ARCH_BITS 64
421#define LJ_TARGET_MIPS64 1
422#define LJ_TARGET_GC64 1
423#endif
279#define LJ_TARGET_MIPS 1 424#define LJ_TARGET_MIPS 1
280#define LJ_TARGET_EHRETREG 4 425#define LJ_TARGET_EHRETREG 4
426#define LJ_TARGET_EHRAREG 31
281#define LJ_TARGET_JUMPRANGE 27 /* 2*2^27 = 256MB-aligned region */ 427#define LJ_TARGET_JUMPRANGE 27 /* 2*2^27 = 256MB-aligned region */
282#define LJ_TARGET_MASKSHIFT 1 428#define LJ_TARGET_MASKSHIFT 1
283#define LJ_TARGET_MASKROT 1 429#define LJ_TARGET_MASKROT 1
284#define LJ_TARGET_UNIFYROT 2 /* Want only IR_BROR. */ 430#define LJ_TARGET_UNIFYROT 2 /* Want only IR_BROR. */
285#define LJ_ARCH_NUMMODE LJ_NUMMODE_SINGLE 431#define LJ_ARCH_NUMMODE LJ_NUMMODE_DUAL
286 432
287#if _MIPS_ARCH_MIPS32R2 433#if LJ_TARGET_MIPSR6
434#define LJ_ARCH_VERSION 60
435#elif _MIPS_ARCH_MIPS32R2 || _MIPS_ARCH_MIPS64R2
288#define LJ_ARCH_VERSION 20 436#define LJ_ARCH_VERSION 20
289#else 437#else
290#define LJ_ARCH_VERSION 10 438#define LJ_ARCH_VERSION 10
@@ -294,9 +442,7 @@
294#error "No target architecture defined" 442#error "No target architecture defined"
295#endif 443#endif
296 444
297#ifndef LJ_PAGESIZE 445/* -- Checks for requirements --------------------------------------------- */
298#define LJ_PAGESIZE 4096
299#endif
300 446
301/* Check for minimum required compiler versions. */ 447/* Check for minimum required compiler versions. */
302#if defined(__GNUC__) 448#if defined(__GNUC__)
@@ -312,12 +458,28 @@
312#if (__GNUC__ < 4) || ((__GNUC__ == 4) && __GNUC_MINOR__ < 2) 458#if (__GNUC__ < 4) || ((__GNUC__ == 4) && __GNUC_MINOR__ < 2)
313#error "Need at least GCC 4.2 or newer" 459#error "Need at least GCC 4.2 or newer"
314#endif 460#endif
461#elif LJ_TARGET_ARM64
462#if __clang__
463#if ((__clang_major__ < 3) || ((__clang_major__ == 3) && __clang_minor__ < 5)) && !defined(__NX_TOOLCHAIN_MAJOR__)
464#error "Need at least Clang 3.5 or newer"
465#endif
466#else
467#if (__GNUC__ < 4) || ((__GNUC__ == 4) && __GNUC_MINOR__ < 8)
468#error "Need at least GCC 4.8 or newer"
469#endif
470#endif
315#elif !LJ_TARGET_PS3 471#elif !LJ_TARGET_PS3
472#if __clang__
473#if ((__clang_major__ < 3) || ((__clang_major__ == 3) && __clang_minor__ < 5))
474#error "Need at least Clang 3.5 or newer"
475#endif
476#else
316#if (__GNUC__ < 4) || ((__GNUC__ == 4) && __GNUC_MINOR__ < 3) 477#if (__GNUC__ < 4) || ((__GNUC__ == 4) && __GNUC_MINOR__ < 3)
317#error "Need at least GCC 4.3 or newer" 478#error "Need at least GCC 4.3 or newer"
318#endif 479#endif
319#endif 480#endif
320#endif 481#endif
482#endif
321 483
322/* Check target-specific constraints. */ 484/* Check target-specific constraints. */
323#ifndef _BUILDVM_H 485#ifndef _BUILDVM_H
@@ -335,25 +497,34 @@
335#if !(__ARM_EABI__ || LJ_TARGET_IOS) 497#if !(__ARM_EABI__ || LJ_TARGET_IOS)
336#error "Only ARM EABI or iOS 3.0+ ABI is supported" 498#error "Only ARM EABI or iOS 3.0+ ABI is supported"
337#endif 499#endif
338#elif LJ_TARGET_PPC || LJ_TARGET_PPCSPE 500#elif LJ_TARGET_ARM64
339#if defined(_SOFT_FLOAT) || defined(_SOFT_DOUBLE) 501#if defined(_ILP32)
340#error "No support for PowerPC CPUs without double-precision FPU" 502#error "No support for ILP32 model on ARM64"
341#endif 503#endif
504#elif LJ_TARGET_PPC
342#if defined(_LITTLE_ENDIAN) && (!defined(_BYTE_ORDER) || (_BYTE_ORDER == _LITTLE_ENDIAN)) 505#if defined(_LITTLE_ENDIAN) && (!defined(_BYTE_ORDER) || (_BYTE_ORDER == _LITTLE_ENDIAN))
343#error "No support for little-endian PowerPC" 506#error "No support for little-endian PPC32"
344#endif 507#endif
345#if defined(_LP64) 508#if defined(__NO_FPRS__) && !defined(_SOFT_FLOAT)
346#error "No support for PowerPC 64 bit mode" 509#error "No support for PPC/e500 anymore (use LuaJIT 2.0)"
347#endif 510#endif
348#elif LJ_TARGET_MIPS 511#elif LJ_TARGET_MIPS32
349#if defined(__mips_soft_float) 512#if !((defined(_MIPS_SIM_ABI32) && _MIPS_SIM == _MIPS_SIM_ABI32) || (defined(_ABIO32) && _MIPS_SIM == _ABIO32))
350#error "No support for MIPS CPUs without FPU" 513#error "Only o32 ABI supported for MIPS32"
351#endif 514#endif
352#if defined(_LP64) 515#if LJ_TARGET_MIPSR6
353#error "No support for MIPS64" 516/* Not that useful, since most available r6 CPUs are 64 bit. */
517#error "No support for MIPS32R6"
354#endif 518#endif
519#elif LJ_TARGET_MIPS64
520#if !((defined(_MIPS_SIM_ABI64) && _MIPS_SIM == _MIPS_SIM_ABI64) || (defined(_ABI64) && _MIPS_SIM == _ABI64))
521/* MIPS32ON64 aka n32 ABI support might be desirable, but difficult. */
522#error "Only n64 ABI supported for MIPS64"
355#endif 523#endif
356#endif 524#endif
525#endif
526
527/* -- Derived defines ----------------------------------------------------- */
357 528
358/* Enable or disable the dual-number mode for the VM. */ 529/* Enable or disable the dual-number mode for the VM. */
359#if (LJ_ARCH_NUMMODE == LJ_NUMMODE_SINGLE && LUAJIT_NUMMODE == 2) || \ 530#if (LJ_ARCH_NUMMODE == LJ_NUMMODE_SINGLE && LUAJIT_NUMMODE == 2) || \
@@ -376,6 +547,20 @@
376#endif 547#endif
377#endif 548#endif
378 549
550/* 64 bit GC references. */
551#if LJ_TARGET_GC64
552#define LJ_GC64 1
553#else
554#define LJ_GC64 0
555#endif
556
557/* 2-slot frame info. */
558#if LJ_GC64
559#define LJ_FR2 1
560#else
561#define LJ_FR2 0
562#endif
563
379/* Disable or enable the JIT compiler. */ 564/* Disable or enable the JIT compiler. */
380#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) 565#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT)
381#define LJ_HASJIT 0 566#define LJ_HASJIT 0
@@ -390,6 +575,28 @@
390#define LJ_HASFFI 1 575#define LJ_HASFFI 1
391#endif 576#endif
392 577
578/* Disable or enable the string buffer extension. */
579#if defined(LUAJIT_DISABLE_BUFFER)
580#define LJ_HASBUFFER 0
581#else
582#define LJ_HASBUFFER 1
583#endif
584
585#if defined(LUAJIT_DISABLE_PROFILE)
586#define LJ_HASPROFILE 0
587#elif LJ_TARGET_POSIX
588#define LJ_HASPROFILE 1
589#define LJ_PROFILE_SIGPROF 1
590#elif LJ_TARGET_PS3
591#define LJ_HASPROFILE 1
592#define LJ_PROFILE_PTHREAD 1
593#elif LJ_TARGET_WINDOWS || LJ_TARGET_XBOX360
594#define LJ_HASPROFILE 1
595#define LJ_PROFILE_WTHREAD 1
596#else
597#define LJ_HASPROFILE 0
598#endif
599
393#ifndef LJ_ARCH_HASFPU 600#ifndef LJ_ARCH_HASFPU
394#define LJ_ARCH_HASFPU 1 601#define LJ_ARCH_HASFPU 1
395#endif 602#endif
@@ -397,6 +604,11 @@
397#define LJ_ABI_SOFTFP 0 604#define LJ_ABI_SOFTFP 0
398#endif 605#endif
399#define LJ_SOFTFP (!LJ_ARCH_HASFPU) 606#define LJ_SOFTFP (!LJ_ARCH_HASFPU)
607#define LJ_SOFTFP32 (LJ_SOFTFP && LJ_32)
608
609#ifndef LJ_ABI_PAUTH
610#define LJ_ABI_PAUTH 0
611#endif
400 612
401#if LJ_ARCH_ENDIAN == LUAJIT_BE 613#if LJ_ARCH_ENDIAN == LUAJIT_BE
402#define LJ_LE 0 614#define LJ_LE 0
@@ -422,26 +634,52 @@
422#define LJ_TARGET_UNALIGNED 0 634#define LJ_TARGET_UNALIGNED 0
423#endif 635#endif
424 636
425/* Various workarounds for embedded operating systems. */ 637#ifndef LJ_PAGESIZE
426#if (defined(__ANDROID__) && !defined(LJ_TARGET_X86ORX64)) || defined(__symbian__) || LJ_TARGET_XBOX360 638#define LJ_PAGESIZE 4096
427#define LUAJIT_NO_LOG2
428#endif 639#endif
429#if defined(__symbian__) 640
430#define LUAJIT_NO_EXP2 641/* Various workarounds for embedded operating systems or weak C runtimes. */
642#if defined(__ANDROID__) || defined(__symbian__) || LJ_TARGET_XBOX360 || LJ_TARGET_WINDOWS
643#define LUAJIT_NO_LOG2
431#endif 644#endif
432#if LJ_TARGET_CONSOLE || (LJ_TARGET_IOS && __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0) 645#if LJ_TARGET_CONSOLE || (LJ_TARGET_IOS && __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0)
433#define LJ_NO_SYSTEM 1 646#define LJ_NO_SYSTEM 1
434#endif 647#endif
435 648
436#if !defined(LUAJIT_NO_UNWIND) && __GNU_COMPACT_EH__ 649#if LJ_TARGET_WINDOWS || LJ_TARGET_CYGWIN
437/* NYI: no support for compact unwind specification, yet. */ 650#define LJ_ABI_WIN 1
438#define LUAJIT_NO_UNWIND 1 651#else
652#define LJ_ABI_WIN 0
653#endif
654
655#if LJ_TARGET_WINDOWS
656#if LJ_TARGET_UWP
657#define LJ_WIN_VALLOC VirtualAllocFromApp
658#define LJ_WIN_VPROTECT VirtualProtectFromApp
659extern void *LJ_WIN_LOADLIBA(const char *path);
660#else
661#define LJ_WIN_VALLOC VirtualAlloc
662#define LJ_WIN_VPROTECT VirtualProtect
663#define LJ_WIN_LOADLIBA(path) LoadLibraryExA((path), NULL, 0)
664#endif
439#endif 665#endif
440 666
441#if defined(LUAJIT_NO_UNWIND) || defined(__symbian__) || LJ_TARGET_IOS || LJ_TARGET_PS3 || LJ_TARGET_PS4 667#if defined(LUAJIT_NO_UNWIND) || __GNU_COMPACT_EH__ || defined(__symbian__) || LJ_TARGET_IOS || LJ_TARGET_PS3 || LJ_TARGET_PS4 || LJ_TARGET_PS5
442#define LJ_NO_UNWIND 1 668#define LJ_NO_UNWIND 1
443#endif 669#endif
444 670
671#if !LJ_NO_UNWIND && !defined(LUAJIT_UNWIND_INTERNAL) && (LJ_ABI_WIN || (defined(LUAJIT_UNWIND_EXTERNAL) && (defined(__GNUC__) || defined(__clang__))))
672#define LJ_UNWIND_EXT 1
673#else
674#define LJ_UNWIND_EXT 0
675#endif
676
677#if LJ_UNWIND_EXT && LJ_HASJIT && !LJ_TARGET_ARM && !(LJ_ABI_WIN && LJ_TARGET_X86)
678#define LJ_UNWIND_JIT 1
679#else
680#define LJ_UNWIND_JIT 0
681#endif
682
445/* Compatibility with Lua 5.1 vs. 5.2. */ 683/* Compatibility with Lua 5.1 vs. 5.2. */
446#ifdef LUAJIT_ENABLE_LUA52COMPAT 684#ifdef LUAJIT_ENABLE_LUA52COMPAT
447#define LJ_52 1 685#define LJ_52 1
@@ -449,4 +687,46 @@
449#define LJ_52 0 687#define LJ_52 0
450#endif 688#endif
451 689
690/* -- VM security --------------------------------------------------------- */
691
692/* Don't make any changes here. Instead build with:
693** make "XCFLAGS=-DLUAJIT_SECURITY_flag=value"
694**
695** Important note to distro maintainers: DO NOT change the defaults for a
696** regular distro build -- neither upwards, nor downwards!
697** These build-time configurable security flags are intended for embedders
698** who may have specific needs wrt. security vs. performance.
699*/
700
701/* Security defaults. */
702#ifndef LUAJIT_SECURITY_PRNG
703/* PRNG init: 0 = fixed/insecure, 1 = secure from OS. */
704#define LUAJIT_SECURITY_PRNG 1
705#endif
706
707#ifndef LUAJIT_SECURITY_STRHASH
708/* String hash: 0 = sparse only, 1 = sparse + dense. */
709#define LUAJIT_SECURITY_STRHASH 1
710#endif
711
712#ifndef LUAJIT_SECURITY_STRID
713/* String IDs: 0 = linear, 1 = reseed < 255, 2 = reseed < 15, 3 = random. */
714#define LUAJIT_SECURITY_STRID 1
715#endif
716
717#ifndef LUAJIT_SECURITY_MCODE
718/* Machine code page protection: 0 = insecure RWX, 1 = secure RW^X. */
719#define LUAJIT_SECURITY_MCODE 1
720#endif
721
722#define LJ_SECURITY_MODE \
723 ( 0u \
724 | ((LUAJIT_SECURITY_PRNG & 3) << 0) \
725 | ((LUAJIT_SECURITY_STRHASH & 3) << 2) \
726 | ((LUAJIT_SECURITY_STRID & 3) << 4) \
727 | ((LUAJIT_SECURITY_MCODE & 3) << 6) \
728 )
729#define LJ_SECURITY_MODESTRING \
730 "\004prng\007strhash\005strid\005mcode"
731
452#endif 732#endif
diff --git a/src/lj_asm.c b/src/lj_asm.c
index 7c4d8f52..71079b30 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -11,6 +11,7 @@
11#if LJ_HASJIT 11#if LJ_HASJIT
12 12
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_buf.h"
14#include "lj_str.h" 15#include "lj_str.h"
15#include "lj_tab.h" 16#include "lj_tab.h"
16#include "lj_frame.h" 17#include "lj_frame.h"
@@ -71,6 +72,7 @@ typedef struct ASMState {
71 IRRef snaprename; /* Rename highwater mark for snapshot check. */ 72 IRRef snaprename; /* Rename highwater mark for snapshot check. */
72 SnapNo snapno; /* Current snapshot number. */ 73 SnapNo snapno; /* Current snapshot number. */
73 SnapNo loopsnapno; /* Loop snapshot number. */ 74 SnapNo loopsnapno; /* Loop snapshot number. */
75 int snapalloc; /* Current snapshot needs allocation. */
74 BloomFilter snapfilt1, snapfilt2; /* Filled with snapshot refs. */ 76 BloomFilter snapfilt1, snapfilt2; /* Filled with snapshot refs. */
75 77
76 IRRef fuseref; /* Fusion limit (loopref, 0 or FUSE_DISABLED). */ 78 IRRef fuseref; /* Fusion limit (loopref, 0 or FUSE_DISABLED). */
@@ -85,18 +87,25 @@ typedef struct ASMState {
85 87
86 MCode *mcbot; /* Bottom of reserved MCode. */ 88 MCode *mcbot; /* Bottom of reserved MCode. */
87 MCode *mctop; /* Top of generated MCode. */ 89 MCode *mctop; /* Top of generated MCode. */
90 MCode *mctoporig; /* Original top of generated MCode. */
88 MCode *mcloop; /* Pointer to loop MCode (or NULL). */ 91 MCode *mcloop; /* Pointer to loop MCode (or NULL). */
89 MCode *invmcp; /* Points to invertible loop branch (or NULL). */ 92 MCode *invmcp; /* Points to invertible loop branch (or NULL). */
90 MCode *flagmcp; /* Pending opportunity to merge flag setting ins. */ 93 MCode *flagmcp; /* Pending opportunity to merge flag setting ins. */
91 MCode *realign; /* Realign loop if not NULL. */ 94 MCode *realign; /* Realign loop if not NULL. */
92 95
93#ifdef RID_NUM_KREF 96#ifdef RID_NUM_KREF
94 int32_t krefk[RID_NUM_KREF]; 97 intptr_t krefk[RID_NUM_KREF];
95#endif 98#endif
96 IRRef1 phireg[RID_MAX]; /* PHI register references. */ 99 IRRef1 phireg[RID_MAX]; /* PHI register references. */
97 uint16_t parentmap[LJ_MAX_JSLOTS]; /* Parent instruction to RegSP map. */ 100 uint16_t parentmap[LJ_MAX_JSLOTS]; /* Parent instruction to RegSP map. */
98} ASMState; 101} ASMState;
99 102
103#ifdef LUA_USE_ASSERT
104#define lj_assertA(c, ...) lj_assertG_(J2G(as->J), (c), __VA_ARGS__)
105#else
106#define lj_assertA(c, ...) ((void)as)
107#endif
108
100#define IR(ref) (&as->ir[(ref)]) 109#define IR(ref) (&as->ir[(ref)])
101 110
102#define ASMREF_TMP1 REF_TRUE /* Temp. register. */ 111#define ASMREF_TMP1 REF_TRUE /* Temp. register. */
@@ -128,9 +137,8 @@ static LJ_AINLINE void checkmclim(ASMState *as)
128#ifdef LUA_USE_ASSERT 137#ifdef LUA_USE_ASSERT
129 if (as->mcp + MCLIM_REDZONE < as->mcp_prev) { 138 if (as->mcp + MCLIM_REDZONE < as->mcp_prev) {
130 IRIns *ir = IR(as->curins+1); 139 IRIns *ir = IR(as->curins+1);
131 fprintf(stderr, "RED ZONE OVERFLOW: %p IR %04d %02d %04d %04d\n", as->mcp, 140 lj_assertA(0, "red zone overflow: %p IR %04d %02d %04d %04d\n", as->mcp,
132 as->curins+1-REF_BIAS, ir->o, ir->op1-REF_BIAS, ir->op2-REF_BIAS); 141 as->curins+1-REF_BIAS, ir->o, ir->op1-REF_BIAS, ir->op2-REF_BIAS);
133 lua_assert(0);
134 } 142 }
135#endif 143#endif
136 if (LJ_UNLIKELY(as->mcp < as->mclim)) asm_mclimit(as); 144 if (LJ_UNLIKELY(as->mcp < as->mclim)) asm_mclimit(as);
@@ -144,7 +152,7 @@ static LJ_AINLINE void checkmclim(ASMState *as)
144#define ra_krefreg(ref) ((Reg)(RID_MIN_KREF + (Reg)(ref))) 152#define ra_krefreg(ref) ((Reg)(RID_MIN_KREF + (Reg)(ref)))
145#define ra_krefk(as, ref) (as->krefk[(ref)]) 153#define ra_krefk(as, ref) (as->krefk[(ref)])
146 154
147static LJ_AINLINE void ra_setkref(ASMState *as, Reg r, int32_t k) 155static LJ_AINLINE void ra_setkref(ASMState *as, Reg r, intptr_t k)
148{ 156{
149 IRRef ref = (IRRef)(r - RID_MIN_KREF); 157 IRRef ref = (IRRef)(r - RID_MIN_KREF);
150 as->krefk[ref] = k; 158 as->krefk[ref] = k;
@@ -171,6 +179,8 @@ IRFLDEF(FLOFS)
171#include "lj_emit_x86.h" 179#include "lj_emit_x86.h"
172#elif LJ_TARGET_ARM 180#elif LJ_TARGET_ARM
173#include "lj_emit_arm.h" 181#include "lj_emit_arm.h"
182#elif LJ_TARGET_ARM64
183#include "lj_emit_arm64.h"
174#elif LJ_TARGET_PPC 184#elif LJ_TARGET_PPC
175#include "lj_emit_ppc.h" 185#include "lj_emit_ppc.h"
176#elif LJ_TARGET_MIPS 186#elif LJ_TARGET_MIPS
@@ -179,6 +189,12 @@ IRFLDEF(FLOFS)
179#error "Missing instruction emitter for target CPU" 189#error "Missing instruction emitter for target CPU"
180#endif 190#endif
181 191
192/* Generic load/store of register from/to stack slot. */
193#define emit_spload(as, ir, r, ofs) \
194 emit_loadofs(as, ir, (r), RID_SP, (ofs))
195#define emit_spstore(as, ir, r, ofs) \
196 emit_storeofs(as, ir, (r), RID_SP, (ofs))
197
182/* -- Register allocator debugging ---------------------------------------- */ 198/* -- Register allocator debugging ---------------------------------------- */
183 199
184/* #define LUAJIT_DEBUG_RA */ 200/* #define LUAJIT_DEBUG_RA */
@@ -236,7 +252,7 @@ static void ra_dprintf(ASMState *as, const char *fmt, ...)
236 *p++ = *q >= 'A' && *q <= 'Z' ? *q + 0x20 : *q; 252 *p++ = *q >= 'A' && *q <= 'Z' ? *q + 0x20 : *q;
237 } else { 253 } else {
238 *p++ = '?'; 254 *p++ = '?';
239 lua_assert(0); 255 lj_assertA(0, "bad register %d for debug format \"%s\"", r, fmt);
240 } 256 }
241 } else if (e[1] == 'f' || e[1] == 'i') { 257 } else if (e[1] == 'f' || e[1] == 'i') {
242 IRRef ref; 258 IRRef ref;
@@ -254,7 +270,7 @@ static void ra_dprintf(ASMState *as, const char *fmt, ...)
254 } else if (e[1] == 'x') { 270 } else if (e[1] == 'x') {
255 p += sprintf(p, "%08x", va_arg(argp, int32_t)); 271 p += sprintf(p, "%08x", va_arg(argp, int32_t));
256 } else { 272 } else {
257 lua_assert(0); 273 lj_assertA(0, "bad debug format code");
258 } 274 }
259 fmt = e+2; 275 fmt = e+2;
260 } 276 }
@@ -313,37 +329,51 @@ static Reg ra_rematk(ASMState *as, IRRef ref)
313 Reg r; 329 Reg r;
314 if (ra_iskref(ref)) { 330 if (ra_iskref(ref)) {
315 r = ra_krefreg(ref); 331 r = ra_krefreg(ref);
316 lua_assert(!rset_test(as->freeset, r)); 332 lj_assertA(!rset_test(as->freeset, r), "rematk of free reg %d", r);
317 ra_free(as, r); 333 ra_free(as, r);
318 ra_modified(as, r); 334 ra_modified(as, r);
335#if LJ_64
336 emit_loadu64(as, r, ra_krefk(as, ref));
337#else
319 emit_loadi(as, r, ra_krefk(as, ref)); 338 emit_loadi(as, r, ra_krefk(as, ref));
339#endif
320 return r; 340 return r;
321 } 341 }
322 ir = IR(ref); 342 ir = IR(ref);
323 r = ir->r; 343 r = ir->r;
324 lua_assert(ra_hasreg(r) && !ra_hasspill(ir->s)); 344 lj_assertA(ra_hasreg(r), "rematk of K%03d has no reg", REF_BIAS - ref);
345 lj_assertA(!ra_hasspill(ir->s),
346 "rematk of K%03d has spill slot [%x]", REF_BIAS - ref, ir->s);
325 ra_free(as, r); 347 ra_free(as, r);
326 ra_modified(as, r); 348 ra_modified(as, r);
327 ir->r = RID_INIT; /* Do not keep any hint. */ 349 ir->r = RID_INIT; /* Do not keep any hint. */
328 RA_DBGX((as, "remat $i $r", ir, r)); 350 RA_DBGX((as, "remat $i $r", ir, r));
329#if !LJ_SOFTFP 351#if !LJ_SOFTFP32
330 if (ir->o == IR_KNUM) { 352 if (ir->o == IR_KNUM) {
331 emit_loadn(as, r, ir_knum(ir)); 353 emit_loadk64(as, r, ir);
332 } else 354 } else
333#endif 355#endif
334 if (emit_canremat(REF_BASE) && ir->o == IR_BASE) { 356 if (emit_canremat(REF_BASE) && ir->o == IR_BASE) {
335 ra_sethint(ir->r, RID_BASE); /* Restore BASE register hint. */ 357 ra_sethint(ir->r, RID_BASE); /* Restore BASE register hint. */
336 emit_getgl(as, r, jit_base); 358 emit_getgl(as, r, jit_base);
337 } else if (emit_canremat(ASMREF_L) && ir->o == IR_KPRI) { 359 } else if (emit_canremat(ASMREF_L) && ir->o == IR_KPRI) {
338 lua_assert(irt_isnil(ir->t)); /* REF_NIL stores ASMREF_L register. */ 360 /* REF_NIL stores ASMREF_L register. */
339 emit_getgl(as, r, jit_L); 361 lj_assertA(irt_isnil(ir->t), "rematk of bad ASMREF_L");
362 emit_getgl(as, r, cur_L);
340#if LJ_64 363#if LJ_64
341 } else if (ir->o == IR_KINT64) { 364 } else if (ir->o == IR_KINT64) {
342 emit_loadu64(as, r, ir_kint64(ir)->u64); 365 emit_loadu64(as, r, ir_kint64(ir)->u64);
366#if LJ_GC64
367 } else if (ir->o == IR_KGC) {
368 emit_loadu64(as, r, (uintptr_t)ir_kgc(ir));
369 } else if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
370 emit_loadu64(as, r, (uintptr_t)ir_kptr(ir));
371#endif
343#endif 372#endif
344 } else { 373 } else {
345 lua_assert(ir->o == IR_KINT || ir->o == IR_KGC || 374 lj_assertA(ir->o == IR_KINT || ir->o == IR_KGC ||
346 ir->o == IR_KPTR || ir->o == IR_KKPTR || ir->o == IR_KNULL); 375 ir->o == IR_KPTR || ir->o == IR_KKPTR || ir->o == IR_KNULL,
376 "rematk of bad IR op %d", ir->o);
347 emit_loadi(as, r, ir->i); 377 emit_loadi(as, r, ir->i);
348 } 378 }
349 return r; 379 return r;
@@ -353,7 +383,8 @@ static Reg ra_rematk(ASMState *as, IRRef ref)
353static int32_t ra_spill(ASMState *as, IRIns *ir) 383static int32_t ra_spill(ASMState *as, IRIns *ir)
354{ 384{
355 int32_t slot = ir->s; 385 int32_t slot = ir->s;
356 lua_assert(ir >= as->ir + REF_TRUE); 386 lj_assertA(ir >= as->ir + REF_TRUE,
387 "spill of K%03d", REF_BIAS - (int)(ir - as->ir));
357 if (!ra_hasspill(slot)) { 388 if (!ra_hasspill(slot)) {
358 if (irt_is64(ir->t)) { 389 if (irt_is64(ir->t)) {
359 slot = as->evenspill; 390 slot = as->evenspill;
@@ -378,7 +409,9 @@ static Reg ra_releasetmp(ASMState *as, IRRef ref)
378{ 409{
379 IRIns *ir = IR(ref); 410 IRIns *ir = IR(ref);
380 Reg r = ir->r; 411 Reg r = ir->r;
381 lua_assert(ra_hasreg(r) && !ra_hasspill(ir->s)); 412 lj_assertA(ra_hasreg(r), "release of TMP%d has no reg", ref-ASMREF_TMP1+1);
413 lj_assertA(!ra_hasspill(ir->s),
414 "release of TMP%d has spill slot [%x]", ref-ASMREF_TMP1+1, ir->s);
382 ra_free(as, r); 415 ra_free(as, r);
383 ra_modified(as, r); 416 ra_modified(as, r);
384 ir->r = RID_INIT; 417 ir->r = RID_INIT;
@@ -394,7 +427,7 @@ static Reg ra_restore(ASMState *as, IRRef ref)
394 IRIns *ir = IR(ref); 427 IRIns *ir = IR(ref);
395 int32_t ofs = ra_spill(as, ir); /* Force a spill slot. */ 428 int32_t ofs = ra_spill(as, ir); /* Force a spill slot. */
396 Reg r = ir->r; 429 Reg r = ir->r;
397 lua_assert(ra_hasreg(r)); 430 lj_assertA(ra_hasreg(r), "restore of IR %04d has no reg", ref - REF_BIAS);
398 ra_sethint(ir->r, r); /* Keep hint. */ 431 ra_sethint(ir->r, r); /* Keep hint. */
399 ra_free(as, r); 432 ra_free(as, r);
400 if (!rset_test(as->weakset, r)) { /* Only restore non-weak references. */ 433 if (!rset_test(as->weakset, r)) { /* Only restore non-weak references. */
@@ -423,14 +456,15 @@ static Reg ra_evict(ASMState *as, RegSet allow)
423{ 456{
424 IRRef ref; 457 IRRef ref;
425 RegCost cost = ~(RegCost)0; 458 RegCost cost = ~(RegCost)0;
426 lua_assert(allow != RSET_EMPTY); 459 lj_assertA(allow != RSET_EMPTY, "evict from empty set");
427 if (RID_NUM_FPR == 0 || allow < RID2RSET(RID_MAX_GPR)) { 460 if (RID_NUM_FPR == 0 || allow < RID2RSET(RID_MAX_GPR)) {
428 GPRDEF(MINCOST) 461 GPRDEF(MINCOST)
429 } else { 462 } else {
430 FPRDEF(MINCOST) 463 FPRDEF(MINCOST)
431 } 464 }
432 ref = regcost_ref(cost); 465 ref = regcost_ref(cost);
433 lua_assert(ra_iskref(ref) || (ref >= as->T->nk && ref < as->T->nins)); 466 lj_assertA(ra_iskref(ref) || (ref >= as->T->nk && ref < as->T->nins),
467 "evict of out-of-range IR %04d", ref - REF_BIAS);
434 /* Preferably pick any weak ref instead of a non-weak, non-const ref. */ 468 /* Preferably pick any weak ref instead of a non-weak, non-const ref. */
435 if (!irref_isk(ref) && (as->weakset & allow)) { 469 if (!irref_isk(ref) && (as->weakset & allow)) {
436 IRIns *ir = IR(ref); 470 IRIns *ir = IR(ref);
@@ -512,7 +546,7 @@ static void ra_evictk(ASMState *as)
512 546
513#ifdef RID_NUM_KREF 547#ifdef RID_NUM_KREF
514/* Allocate a register for a constant. */ 548/* Allocate a register for a constant. */
515static Reg ra_allock(ASMState *as, int32_t k, RegSet allow) 549static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow)
516{ 550{
517 /* First try to find a register which already holds the same constant. */ 551 /* First try to find a register which already holds the same constant. */
518 RegSet pick, work = ~as->freeset & RSET_GPR; 552 RegSet pick, work = ~as->freeset & RSET_GPR;
@@ -521,9 +555,31 @@ static Reg ra_allock(ASMState *as, int32_t k, RegSet allow)
521 IRRef ref; 555 IRRef ref;
522 r = rset_pickbot(work); 556 r = rset_pickbot(work);
523 ref = regcost_ref(as->cost[r]); 557 ref = regcost_ref(as->cost[r]);
558#if LJ_64
559 if (ref < ASMREF_L) {
560 if (ra_iskref(ref)) {
561 if (k == ra_krefk(as, ref))
562 return r;
563 } else {
564 IRIns *ir = IR(ref);
565 if ((ir->o == IR_KINT64 && k == (int64_t)ir_kint64(ir)->u64) ||
566#if LJ_GC64
567 (ir->o == IR_KINT && k == ir->i) ||
568 (ir->o == IR_KGC && k == (intptr_t)ir_kgc(ir)) ||
569 ((ir->o == IR_KPTR || ir->o == IR_KKPTR) &&
570 k == (intptr_t)ir_kptr(ir))
571#else
572 (ir->o != IR_KINT64 && k == ir->i)
573#endif
574 )
575 return r;
576 }
577 }
578#else
524 if (ref < ASMREF_L && 579 if (ref < ASMREF_L &&
525 k == (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i)) 580 k == (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i))
526 return r; 581 return r;
582#endif
527 rset_clear(work, r); 583 rset_clear(work, r);
528 } 584 }
529 pick = as->freeset & allow; 585 pick = as->freeset & allow;
@@ -543,7 +599,7 @@ static Reg ra_allock(ASMState *as, int32_t k, RegSet allow)
543} 599}
544 600
545/* Allocate a specific register for a constant. */ 601/* Allocate a specific register for a constant. */
546static void ra_allockreg(ASMState *as, int32_t k, Reg r) 602static void ra_allockreg(ASMState *as, intptr_t k, Reg r)
547{ 603{
548 Reg kr = ra_allock(as, k, RID2RSET(r)); 604 Reg kr = ra_allock(as, k, RID2RSET(r));
549 if (kr != r) { 605 if (kr != r) {
@@ -566,7 +622,8 @@ static Reg ra_allocref(ASMState *as, IRRef ref, RegSet allow)
566 IRIns *ir = IR(ref); 622 IRIns *ir = IR(ref);
567 RegSet pick = as->freeset & allow; 623 RegSet pick = as->freeset & allow;
568 Reg r; 624 Reg r;
569 lua_assert(ra_noreg(ir->r)); 625 lj_assertA(ra_noreg(ir->r),
626 "IR %04d already has reg %d", ref - REF_BIAS, ir->r);
570 if (pick) { 627 if (pick) {
571 /* First check register hint from propagation or PHI. */ 628 /* First check register hint from propagation or PHI. */
572 if (ra_hashint(ir->r)) { 629 if (ra_hashint(ir->r)) {
@@ -613,15 +670,27 @@ static Reg ra_alloc1(ASMState *as, IRRef ref, RegSet allow)
613 return r; 670 return r;
614} 671}
615 672
673/* Add a register rename to the IR. */
674static void ra_addrename(ASMState *as, Reg down, IRRef ref, SnapNo snapno)
675{
676 IRRef ren;
677 lj_ir_set(as->J, IRT(IR_RENAME, IRT_NIL), ref, snapno);
678 ren = tref_ref(lj_ir_emit(as->J));
679 as->J->cur.ir[ren].r = (uint8_t)down;
680 as->J->cur.ir[ren].s = SPS_NONE;
681}
682
616/* Rename register allocation and emit move. */ 683/* Rename register allocation and emit move. */
617static void ra_rename(ASMState *as, Reg down, Reg up) 684static void ra_rename(ASMState *as, Reg down, Reg up)
618{ 685{
619 IRRef ren, ref = regcost_ref(as->cost[up] = as->cost[down]); 686 IRRef ref = regcost_ref(as->cost[up] = as->cost[down]);
620 IRIns *ir = IR(ref); 687 IRIns *ir = IR(ref);
621 ir->r = (uint8_t)up; 688 ir->r = (uint8_t)up;
622 as->cost[down] = 0; 689 as->cost[down] = 0;
623 lua_assert((down < RID_MAX_GPR) == (up < RID_MAX_GPR)); 690 lj_assertA((down < RID_MAX_GPR) == (up < RID_MAX_GPR),
624 lua_assert(!rset_test(as->freeset, down) && rset_test(as->freeset, up)); 691 "rename between GPR/FPR %d and %d", down, up);
692 lj_assertA(!rset_test(as->freeset, down), "rename from free reg %d", down);
693 lj_assertA(rset_test(as->freeset, up), "rename to non-free reg %d", up);
625 ra_free(as, down); /* 'down' is free ... */ 694 ra_free(as, down); /* 'down' is free ... */
626 ra_modified(as, down); 695 ra_modified(as, down);
627 rset_clear(as->freeset, up); /* ... and 'up' is now allocated. */ 696 rset_clear(as->freeset, up); /* ... and 'up' is now allocated. */
@@ -629,11 +698,14 @@ static void ra_rename(ASMState *as, Reg down, Reg up)
629 RA_DBGX((as, "rename $f $r $r", regcost_ref(as->cost[up]), down, up)); 698 RA_DBGX((as, "rename $f $r $r", regcost_ref(as->cost[up]), down, up));
630 emit_movrr(as, ir, down, up); /* Backwards codegen needs inverse move. */ 699 emit_movrr(as, ir, down, up); /* Backwards codegen needs inverse move. */
631 if (!ra_hasspill(IR(ref)->s)) { /* Add the rename to the IR. */ 700 if (!ra_hasspill(IR(ref)->s)) { /* Add the rename to the IR. */
632 lj_ir_set(as->J, IRT(IR_RENAME, IRT_NIL), ref, as->snapno); 701 /*
633 ren = tref_ref(lj_ir_emit(as->J)); 702 ** The rename is effective at the subsequent (already emitted) exit
634 as->ir = as->T->ir; /* The IR may have been reallocated. */ 703 ** branch. This is for the current snapshot (as->snapno). Except if we
635 IR(ren)->r = (uint8_t)down; 704 ** haven't yet allocated any refs for the snapshot (as->snapalloc == 1),
636 IR(ren)->s = SPS_NONE; 705 ** then it belongs to the next snapshot.
706 ** See also the discussion at asm_snap_checkrename().
707 */
708 ra_addrename(as, down, ref, as->snapno + as->snapalloc);
637 } 709 }
638} 710}
639 711
@@ -666,7 +738,7 @@ static void ra_destreg(ASMState *as, IRIns *ir, Reg r)
666{ 738{
667 Reg dest = ra_dest(as, ir, RID2RSET(r)); 739 Reg dest = ra_dest(as, ir, RID2RSET(r));
668 if (dest != r) { 740 if (dest != r) {
669 lua_assert(rset_test(as->freeset, r)); 741 lj_assertA(rset_test(as->freeset, r), "dest reg %d is not free", r);
670 ra_modified(as, r); 742 ra_modified(as, r);
671 emit_movrr(as, ir, dest, r); 743 emit_movrr(as, ir, dest, r);
672 } 744 }
@@ -683,20 +755,25 @@ static void ra_left(ASMState *as, Reg dest, IRRef lref)
683 if (ra_noreg(left)) { 755 if (ra_noreg(left)) {
684 if (irref_isk(lref)) { 756 if (irref_isk(lref)) {
685 if (ir->o == IR_KNUM) { 757 if (ir->o == IR_KNUM) {
686 cTValue *tv = ir_knum(ir);
687 /* FP remat needs a load except for +0. Still better than eviction. */ 758 /* FP remat needs a load except for +0. Still better than eviction. */
688 if (tvispzero(tv) || !(as->freeset & RSET_FPR)) { 759 if (tvispzero(ir_knum(ir)) || !(as->freeset & RSET_FPR)) {
689 emit_loadn(as, dest, tv); 760 emit_loadk64(as, dest, ir);
690 return; 761 return;
691 } 762 }
692#if LJ_64 763#if LJ_64
693 } else if (ir->o == IR_KINT64) { 764 } else if (ir->o == IR_KINT64) {
694 emit_loadu64(as, dest, ir_kint64(ir)->u64); 765 emit_loadk64(as, dest, ir);
766 return;
767#if LJ_GC64
768 } else if (ir->o == IR_KGC || ir->o == IR_KPTR || ir->o == IR_KKPTR) {
769 emit_loadk64(as, dest, ir);
695 return; 770 return;
696#endif 771#endif
697 } else { 772#endif
698 lua_assert(ir->o == IR_KINT || ir->o == IR_KGC || 773 } else if (ir->o != IR_KPRI) {
699 ir->o == IR_KPTR || ir->o == IR_KKPTR || ir->o == IR_KNULL); 774 lj_assertA(ir->o == IR_KINT || ir->o == IR_KGC ||
775 ir->o == IR_KPTR || ir->o == IR_KKPTR || ir->o == IR_KNULL,
776 "K%03d has bad IR op %d", REF_BIAS - lref, ir->o);
700 emit_loadi(as, dest, ir->i); 777 emit_loadi(as, dest, ir->i);
701 return; 778 return;
702 } 779 }
@@ -741,11 +818,11 @@ static void ra_leftov(ASMState *as, Reg dest, IRRef lref)
741} 818}
742#endif 819#endif
743 820
744#if !LJ_64
745/* Force a RID_RETLO/RID_RETHI destination register pair (marked as free). */ 821/* Force a RID_RETLO/RID_RETHI destination register pair (marked as free). */
746static void ra_destpair(ASMState *as, IRIns *ir) 822static void ra_destpair(ASMState *as, IRIns *ir)
747{ 823{
748 Reg destlo = ir->r, desthi = (ir+1)->r; 824 Reg destlo = ir->r, desthi = (ir+1)->r;
825 IRIns *irx = (LJ_64 && !irt_is64(ir->t)) ? ir+1 : ir;
749 /* First spill unrelated refs blocking the destination registers. */ 826 /* First spill unrelated refs blocking the destination registers. */
750 if (!rset_test(as->freeset, RID_RETLO) && 827 if (!rset_test(as->freeset, RID_RETLO) &&
751 destlo != RID_RETLO && desthi != RID_RETLO) 828 destlo != RID_RETLO && desthi != RID_RETLO)
@@ -769,29 +846,29 @@ static void ra_destpair(ASMState *as, IRIns *ir)
769 /* Check for conflicts and shuffle the registers as needed. */ 846 /* Check for conflicts and shuffle the registers as needed. */
770 if (destlo == RID_RETHI) { 847 if (destlo == RID_RETHI) {
771 if (desthi == RID_RETLO) { 848 if (desthi == RID_RETLO) {
772#if LJ_TARGET_X86 849#if LJ_TARGET_X86ORX64
773 *--as->mcp = XI_XCHGa + RID_RETHI; 850 *--as->mcp = XI_XCHGa + RID_RETHI;
851 if (LJ_64 && irt_is64(irx->t)) *--as->mcp = 0x48;
774#else 852#else
775 emit_movrr(as, ir, RID_RETHI, RID_TMP); 853 emit_movrr(as, irx, RID_RETHI, RID_TMP);
776 emit_movrr(as, ir, RID_RETLO, RID_RETHI); 854 emit_movrr(as, irx, RID_RETLO, RID_RETHI);
777 emit_movrr(as, ir, RID_TMP, RID_RETLO); 855 emit_movrr(as, irx, RID_TMP, RID_RETLO);
778#endif 856#endif
779 } else { 857 } else {
780 emit_movrr(as, ir, RID_RETHI, RID_RETLO); 858 emit_movrr(as, irx, RID_RETHI, RID_RETLO);
781 if (desthi != RID_RETHI) emit_movrr(as, ir, desthi, RID_RETHI); 859 if (desthi != RID_RETHI) emit_movrr(as, irx, desthi, RID_RETHI);
782 } 860 }
783 } else if (desthi == RID_RETLO) { 861 } else if (desthi == RID_RETLO) {
784 emit_movrr(as, ir, RID_RETLO, RID_RETHI); 862 emit_movrr(as, irx, RID_RETLO, RID_RETHI);
785 if (destlo != RID_RETLO) emit_movrr(as, ir, destlo, RID_RETLO); 863 if (destlo != RID_RETLO) emit_movrr(as, irx, destlo, RID_RETLO);
786 } else { 864 } else {
787 if (desthi != RID_RETHI) emit_movrr(as, ir, desthi, RID_RETHI); 865 if (desthi != RID_RETHI) emit_movrr(as, irx, desthi, RID_RETHI);
788 if (destlo != RID_RETLO) emit_movrr(as, ir, destlo, RID_RETLO); 866 if (destlo != RID_RETLO) emit_movrr(as, irx, destlo, RID_RETLO);
789 } 867 }
790 /* Restore spill slots (if any). */ 868 /* Restore spill slots (if any). */
791 if (ra_hasspill((ir+1)->s)) ra_save(as, ir+1, RID_RETHI); 869 if (ra_hasspill((ir+1)->s)) ra_save(as, ir+1, RID_RETHI);
792 if (ra_hasspill(ir->s)) ra_save(as, ir, RID_RETLO); 870 if (ra_hasspill(ir->s)) ra_save(as, ir, RID_RETLO);
793} 871}
794#endif
795 872
796/* -- Snapshot handling --------- ----------------------------------------- */ 873/* -- Snapshot handling --------- ----------------------------------------- */
797 874
@@ -841,11 +918,14 @@ static void asm_snap_alloc1(ASMState *as, IRRef ref)
841#endif 918#endif
842 { /* Allocate stored values for TNEW, TDUP and CNEW. */ 919 { /* Allocate stored values for TNEW, TDUP and CNEW. */
843 IRIns *irs; 920 IRIns *irs;
844 lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW); 921 lj_assertA(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW,
922 "sink of IR %04d has bad op %d", ref - REF_BIAS, ir->o);
845 for (irs = IR(as->snapref-1); irs > ir; irs--) 923 for (irs = IR(as->snapref-1); irs > ir; irs--)
846 if (irs->r == RID_SINK && asm_sunk_store(as, ir, irs)) { 924 if (irs->r == RID_SINK && asm_sunk_store(as, ir, irs)) {
847 lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE || 925 lj_assertA(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
848 irs->o == IR_FSTORE || irs->o == IR_XSTORE); 926 irs->o == IR_FSTORE || irs->o == IR_XSTORE,
927 "sunk store IR %04d has bad op %d",
928 (int)(irs - as->ir) - REF_BIAS, irs->o);
849 asm_snap_alloc1(as, irs->op2); 929 asm_snap_alloc1(as, irs->op2);
850 if (LJ_32 && (irs+1)->o == IR_HIOP) 930 if (LJ_32 && (irs+1)->o == IR_HIOP)
851 asm_snap_alloc1(as, (irs+1)->op2); 931 asm_snap_alloc1(as, (irs+1)->op2);
@@ -881,9 +961,9 @@ static void asm_snap_alloc1(ASMState *as, IRRef ref)
881} 961}
882 962
883/* Allocate refs escaping to a snapshot. */ 963/* Allocate refs escaping to a snapshot. */
884static void asm_snap_alloc(ASMState *as) 964static void asm_snap_alloc(ASMState *as, int snapno)
885{ 965{
886 SnapShot *snap = &as->T->snap[as->snapno]; 966 SnapShot *snap = &as->T->snap[snapno];
887 SnapEntry *map = &as->T->snapmap[snap->mapofs]; 967 SnapEntry *map = &as->T->snapmap[snap->mapofs];
888 MSize n, nent = snap->nent; 968 MSize n, nent = snap->nent;
889 as->snapfilt1 = as->snapfilt2 = 0; 969 as->snapfilt1 = as->snapfilt2 = 0;
@@ -893,7 +973,9 @@ static void asm_snap_alloc(ASMState *as)
893 if (!irref_isk(ref)) { 973 if (!irref_isk(ref)) {
894 asm_snap_alloc1(as, ref); 974 asm_snap_alloc1(as, ref);
895 if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { 975 if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
896 lua_assert(irt_type(IR(ref+1)->t) == IRT_SOFTFP); 976 lj_assertA(irt_type(IR(ref+1)->t) == IRT_SOFTFP,
977 "snap %d[%d] points to bad SOFTFP IR %04d",
978 snapno, n, ref - REF_BIAS);
897 asm_snap_alloc1(as, ref+1); 979 asm_snap_alloc1(as, ref+1);
898 } 980 }
899 } 981 }
@@ -919,67 +1001,55 @@ static int asm_snap_checkrename(ASMState *as, IRRef ren)
919 return 0; /* Not found. */ 1001 return 0; /* Not found. */
920} 1002}
921 1003
922/* Prepare snapshot for next guard instruction. */ 1004/* Prepare snapshot for next guard or throwing instruction. */
923static void asm_snap_prep(ASMState *as) 1005static void asm_snap_prep(ASMState *as)
924{ 1006{
925 if (as->curins < as->snapref) { 1007 if (as->snapalloc) {
926 do { 1008 /* Alloc on first invocation for each snapshot. */
927 if (as->snapno == 0) return; /* Called by sunk stores before snap #0. */ 1009 as->snapalloc = 0;
928 as->snapno--; 1010 asm_snap_alloc(as, as->snapno);
929 as->snapref = as->T->snap[as->snapno].ref;
930 } while (as->curins < as->snapref);
931 asm_snap_alloc(as);
932 as->snaprename = as->T->nins; 1011 as->snaprename = as->T->nins;
933 } else { 1012 } else {
934 /* Process any renames above the highwater mark. */ 1013 /* Check any renames above the highwater mark. */
935 for (; as->snaprename < as->T->nins; as->snaprename++) { 1014 for (; as->snaprename < as->T->nins; as->snaprename++) {
936 IRIns *ir = IR(as->snaprename); 1015 IRIns *ir = &as->T->ir[as->snaprename];
937 if (asm_snap_checkrename(as, ir->op1)) 1016 if (asm_snap_checkrename(as, ir->op1))
938 ir->op2 = REF_BIAS-1; /* Kill rename. */ 1017 ir->op2 = REF_BIAS-1; /* Kill rename. */
939 } 1018 }
940 } 1019 }
941} 1020}
942 1021
943/* -- Miscellaneous helpers ----------------------------------------------- */ 1022/* Move to previous snapshot when we cross the current snapshot ref. */
944 1023static void asm_snap_prev(ASMState *as)
945/* Collect arguments from CALL* and CARG instructions. */
946static void asm_collectargs(ASMState *as, IRIns *ir,
947 const CCallInfo *ci, IRRef *args)
948{ 1024{
949 uint32_t n = CCI_NARGS(ci); 1025 if (as->curins < as->snapref) {
950 lua_assert(n <= CCI_NARGS_MAX*2); /* Account for split args. */ 1026 uintptr_t ofs = (uintptr_t)(as->mctoporig - as->mcp);
951 if ((ci->flags & CCI_L)) { *args++ = ASMREF_L; n--; } 1027 if (ofs >= 0x10000) lj_trace_err(as->J, LJ_TRERR_MCODEOV);
952 while (n-- > 1) { 1028 do {
953 ir = IR(ir->op1); 1029 if (as->snapno == 0) return;
954 lua_assert(ir->o == IR_CARG); 1030 as->snapno--;
955 args[n] = ir->op2 == REF_NIL ? 0 : ir->op2; 1031 as->snapref = as->T->snap[as->snapno].ref;
1032 as->T->snap[as->snapno].mcofs = (uint16_t)ofs; /* Remember mcode ofs. */
1033 } while (as->curins < as->snapref); /* May have no ins inbetween. */
1034 as->snapalloc = 1;
956 } 1035 }
957 args[0] = ir->op1 == REF_NIL ? 0 : ir->op1;
958 lua_assert(IR(ir->op1)->o != IR_CARG);
959} 1036}
960 1037
961/* Reconstruct CCallInfo flags for CALLX*. */ 1038/* Fixup snapshot mcode offsetst. */
962static uint32_t asm_callx_flags(ASMState *as, IRIns *ir) 1039static void asm_snap_fixup_mcofs(ASMState *as)
963{ 1040{
964 uint32_t nargs = 0; 1041 uint32_t sz = (uint32_t)(as->mctoporig - as->mcp);
965 if (ir->op1 != REF_NIL) { /* Count number of arguments first. */ 1042 SnapShot *snap = as->T->snap;
966 IRIns *ira = IR(ir->op1); 1043 SnapNo i;
967 nargs++; 1044 for (i = as->T->nsnap-1; i > 0; i--) {
968 while (ira->o == IR_CARG) { nargs++; ira = IR(ira->op1); } 1045 /* Compute offset from mcode start and store in correct snapshot. */
1046 snap[i].mcofs = (uint16_t)(sz - snap[i-1].mcofs);
969 } 1047 }
970#if LJ_HASFFI 1048 snap[0].mcofs = 0;
971 if (IR(ir->op2)->o == IR_CARG) { /* Copy calling convention info. */
972 CTypeID id = (CTypeID)IR(IR(ir->op2)->op2)->i;
973 CType *ct = ctype_get(ctype_ctsG(J2G(as->J)), id);
974 nargs |= ((ct->info & CTF_VARARG) ? CCI_VARARG : 0);
975#if LJ_TARGET_X86
976 nargs |= (ctype_cconv(ct->info) << CCI_CC_SHIFT);
977#endif
978 }
979#endif
980 return (nargs | (ir->t.irt << CCI_OTSHIFT));
981} 1049}
982 1050
1051/* -- Miscellaneous helpers ----------------------------------------------- */
1052
983/* Calculate stack adjustment. */ 1053/* Calculate stack adjustment. */
984static int32_t asm_stack_adjust(ASMState *as) 1054static int32_t asm_stack_adjust(ASMState *as)
985{ 1055{
@@ -989,21 +1059,26 @@ static int32_t asm_stack_adjust(ASMState *as)
989} 1059}
990 1060
991/* Must match with hash*() in lj_tab.c. */ 1061/* Must match with hash*() in lj_tab.c. */
992static uint32_t ir_khash(IRIns *ir) 1062static uint32_t ir_khash(ASMState *as, IRIns *ir)
993{ 1063{
994 uint32_t lo, hi; 1064 uint32_t lo, hi;
1065 UNUSED(as);
995 if (irt_isstr(ir->t)) { 1066 if (irt_isstr(ir->t)) {
996 return ir_kstr(ir)->hash; 1067 return ir_kstr(ir)->sid;
997 } else if (irt_isnum(ir->t)) { 1068 } else if (irt_isnum(ir->t)) {
998 lo = ir_knum(ir)->u32.lo; 1069 lo = ir_knum(ir)->u32.lo;
999 hi = ir_knum(ir)->u32.hi << 1; 1070 hi = ir_knum(ir)->u32.hi << 1;
1000 } else if (irt_ispri(ir->t)) { 1071 } else if (irt_ispri(ir->t)) {
1001 lua_assert(!irt_isnil(ir->t)); 1072 lj_assertA(!irt_isnil(ir->t), "hash of nil key");
1002 return irt_type(ir->t)-IRT_FALSE; 1073 return irt_type(ir->t)-IRT_FALSE;
1003 } else { 1074 } else {
1004 lua_assert(irt_isgcv(ir->t)); 1075 lj_assertA(irt_isgcv(ir->t), "hash of bad IR type %d", irt_type(ir->t));
1005 lo = u32ptr(ir_kgc(ir)); 1076 lo = u32ptr(ir_kgc(ir));
1077#if LJ_GC64
1078 hi = (uint32_t)(u64ptr(ir_kgc(ir)) >> 32) | (irt_toitype(ir->t) << 15);
1079#else
1006 hi = lo + HASH_BIAS; 1080 hi = lo + HASH_BIAS;
1081#endif
1007 } 1082 }
1008 return hashrot(lo, hi); 1083 return hashrot(lo, hi);
1009} 1084}
@@ -1017,6 +1092,7 @@ static void asm_snew(ASMState *as, IRIns *ir)
1017{ 1092{
1018 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_new]; 1093 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_new];
1019 IRRef args[3]; 1094 IRRef args[3];
1095 asm_snap_prep(as);
1020 args[0] = ASMREF_L; /* lua_State *L */ 1096 args[0] = ASMREF_L; /* lua_State *L */
1021 args[1] = ir->op1; /* const char *str */ 1097 args[1] = ir->op1; /* const char *str */
1022 args[2] = ir->op2; /* size_t len */ 1098 args[2] = ir->op2; /* size_t len */
@@ -1029,6 +1105,7 @@ static void asm_tnew(ASMState *as, IRIns *ir)
1029{ 1105{
1030 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_new1]; 1106 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_new1];
1031 IRRef args[2]; 1107 IRRef args[2];
1108 asm_snap_prep(as);
1032 args[0] = ASMREF_L; /* lua_State *L */ 1109 args[0] = ASMREF_L; /* lua_State *L */
1033 args[1] = ASMREF_TMP1; /* uint32_t ahsize */ 1110 args[1] = ASMREF_TMP1; /* uint32_t ahsize */
1034 as->gcsteps++; 1111 as->gcsteps++;
@@ -1041,6 +1118,7 @@ static void asm_tdup(ASMState *as, IRIns *ir)
1041{ 1118{
1042 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_dup]; 1119 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_dup];
1043 IRRef args[2]; 1120 IRRef args[2];
1121 asm_snap_prep(as);
1044 args[0] = ASMREF_L; /* lua_State *L */ 1122 args[0] = ASMREF_L; /* lua_State *L */
1045 args[1] = ir->op1; /* const GCtab *kt */ 1123 args[1] = ir->op1; /* const GCtab *kt */
1046 as->gcsteps++; 1124 as->gcsteps++;
@@ -1064,6 +1142,260 @@ static void asm_gcstep(ASMState *as, IRIns *ir)
1064 as->gcsteps = 0x80000000; /* Prevent implicit GC check further up. */ 1142 as->gcsteps = 0x80000000; /* Prevent implicit GC check further up. */
1065} 1143}
1066 1144
1145/* -- Buffer operations --------------------------------------------------- */
1146
1147static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode);
1148#if LJ_HASBUFFER
1149static void asm_bufhdr_write(ASMState *as, Reg sb);
1150#endif
1151
1152static void asm_bufhdr(ASMState *as, IRIns *ir)
1153{
1154 Reg sb = ra_dest(as, ir, RSET_GPR);
1155 switch (ir->op2) {
1156 case IRBUFHDR_RESET: {
1157 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
1158 IRIns irbp;
1159 irbp.ot = IRT(0, IRT_PTR); /* Buffer data pointer type. */
1160 emit_storeofs(as, &irbp, tmp, sb, offsetof(SBuf, w));
1161 emit_loadofs(as, &irbp, tmp, sb, offsetof(SBuf, b));
1162 break;
1163 }
1164 case IRBUFHDR_APPEND: {
1165 /* Rematerialize const buffer pointer instead of likely spill. */
1166 IRIns *irp = IR(ir->op1);
1167 if (!(ra_hasreg(irp->r) || irp == ir-1 ||
1168 (irp == ir-2 && !ra_used(ir-1)))) {
1169 while (!(irp->o == IR_BUFHDR && irp->op2 == IRBUFHDR_RESET))
1170 irp = IR(irp->op1);
1171 if (irref_isk(irp->op1)) {
1172 ra_weak(as, ra_allocref(as, ir->op1, RSET_GPR));
1173 ir = irp;
1174 }
1175 }
1176 break;
1177 }
1178#if LJ_HASBUFFER
1179 case IRBUFHDR_WRITE:
1180 asm_bufhdr_write(as, sb);
1181 break;
1182#endif
1183 default: lj_assertA(0, "bad BUFHDR op2 %d", ir->op2); break;
1184 }
1185#if LJ_TARGET_X86ORX64
1186 ra_left(as, sb, ir->op1);
1187#else
1188 ra_leftov(as, sb, ir->op1);
1189#endif
1190}
1191
1192static void asm_bufput(ASMState *as, IRIns *ir)
1193{
1194 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_buf_putstr];
1195 IRRef args[3];
1196 IRIns *irs;
1197 int kchar = -129;
1198 args[0] = ir->op1; /* SBuf * */
1199 args[1] = ir->op2; /* GCstr * */
1200 irs = IR(ir->op2);
1201 lj_assertA(irt_isstr(irs->t),
1202 "BUFPUT of non-string IR %04d", ir->op2 - REF_BIAS);
1203 if (irs->o == IR_KGC) {
1204 GCstr *s = ir_kstr(irs);
1205 if (s->len == 1) { /* Optimize put of single-char string constant. */
1206 kchar = (int8_t)strdata(s)[0]; /* Signed! */
1207 args[1] = ASMREF_TMP1; /* int, truncated to char */
1208 ci = &lj_ir_callinfo[IRCALL_lj_buf_putchar];
1209 }
1210 } else if (mayfuse(as, ir->op2) && ra_noreg(irs->r)) {
1211 if (irs->o == IR_TOSTR) { /* Fuse number to string conversions. */
1212 if (irs->op2 == IRTOSTR_NUM) {
1213 args[1] = ASMREF_TMP1; /* TValue * */
1214 ci = &lj_ir_callinfo[IRCALL_lj_strfmt_putnum];
1215 } else {
1216 lj_assertA(irt_isinteger(IR(irs->op1)->t),
1217 "TOSTR of non-numeric IR %04d", irs->op1);
1218 args[1] = irs->op1; /* int */
1219 if (irs->op2 == IRTOSTR_INT)
1220 ci = &lj_ir_callinfo[IRCALL_lj_strfmt_putint];
1221 else
1222 ci = &lj_ir_callinfo[IRCALL_lj_buf_putchar];
1223 }
1224 } else if (irs->o == IR_SNEW) { /* Fuse string allocation. */
1225 args[1] = irs->op1; /* const void * */
1226 args[2] = irs->op2; /* MSize */
1227 ci = &lj_ir_callinfo[IRCALL_lj_buf_putmem];
1228 }
1229 }
1230 asm_setupresult(as, ir, ci); /* SBuf * */
1231 asm_gencall(as, ci, args);
1232 if (args[1] == ASMREF_TMP1) {
1233 Reg tmp = ra_releasetmp(as, ASMREF_TMP1);
1234 if (kchar == -129)
1235 asm_tvptr(as, tmp, irs->op1, IRTMPREF_IN1);
1236 else
1237 ra_allockreg(as, kchar, tmp);
1238 }
1239}
1240
1241static void asm_bufstr(ASMState *as, IRIns *ir)
1242{
1243 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_buf_tostr];
1244 IRRef args[1];
1245 args[0] = ir->op1; /* SBuf *sb */
1246 as->gcsteps++;
1247 asm_setupresult(as, ir, ci); /* GCstr * */
1248 asm_gencall(as, ci, args);
1249}
1250
1251/* -- Type conversions ---------------------------------------------------- */
1252
1253static void asm_tostr(ASMState *as, IRIns *ir)
1254{
1255 const CCallInfo *ci;
1256 IRRef args[2];
1257 asm_snap_prep(as);
1258 args[0] = ASMREF_L;
1259 as->gcsteps++;
1260 if (ir->op2 == IRTOSTR_NUM) {
1261 args[1] = ASMREF_TMP1; /* cTValue * */
1262 ci = &lj_ir_callinfo[IRCALL_lj_strfmt_num];
1263 } else {
1264 args[1] = ir->op1; /* int32_t k */
1265 if (ir->op2 == IRTOSTR_INT)
1266 ci = &lj_ir_callinfo[IRCALL_lj_strfmt_int];
1267 else
1268 ci = &lj_ir_callinfo[IRCALL_lj_strfmt_char];
1269 }
1270 asm_setupresult(as, ir, ci); /* GCstr * */
1271 asm_gencall(as, ci, args);
1272 if (ir->op2 == IRTOSTR_NUM)
1273 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1, IRTMPREF_IN1);
1274}
1275
1276#if LJ_32 && LJ_HASFFI && !LJ_SOFTFP && !LJ_TARGET_X86
1277static void asm_conv64(ASMState *as, IRIns *ir)
1278{
1279 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
1280 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
1281 IRCallID id;
1282 IRRef args[2];
1283 lj_assertA((ir-1)->o == IR_CONV && ir->o == IR_HIOP,
1284 "not a CONV/HIOP pair at IR %04d", (int)(ir - as->ir) - REF_BIAS);
1285 args[LJ_BE] = (ir-1)->op1;
1286 args[LJ_LE] = ir->op1;
1287 if (st == IRT_NUM || st == IRT_FLOAT) {
1288 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
1289 ir--;
1290 } else {
1291 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
1292 }
1293 {
1294#if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
1295 CCallInfo cim = lj_ir_callinfo[id], *ci = &cim;
1296 cim.flags |= CCI_VARARG; /* These calls don't use the hard-float ABI! */
1297#else
1298 const CCallInfo *ci = &lj_ir_callinfo[id];
1299#endif
1300 asm_setupresult(as, ir, ci);
1301 asm_gencall(as, ci, args);
1302 }
1303}
1304#endif
1305
1306/* -- Memory references --------------------------------------------------- */
1307
1308static void asm_newref(ASMState *as, IRIns *ir)
1309{
1310 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
1311 IRRef args[3];
1312 if (ir->r == RID_SINK)
1313 return;
1314 asm_snap_prep(as);
1315 args[0] = ASMREF_L; /* lua_State *L */
1316 args[1] = ir->op1; /* GCtab *t */
1317 args[2] = ASMREF_TMP1; /* cTValue *key */
1318 asm_setupresult(as, ir, ci); /* TValue * */
1319 asm_gencall(as, ci, args);
1320 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2, IRTMPREF_IN1);
1321}
1322
1323static void asm_tmpref(ASMState *as, IRIns *ir)
1324{
1325 Reg r = ra_dest(as, ir, RSET_GPR);
1326 asm_tvptr(as, r, ir->op1, ir->op2);
1327}
1328
1329static void asm_lref(ASMState *as, IRIns *ir)
1330{
1331 Reg r = ra_dest(as, ir, RSET_GPR);
1332#if LJ_TARGET_X86ORX64
1333 ra_left(as, r, ASMREF_L);
1334#else
1335 ra_leftov(as, r, ASMREF_L);
1336#endif
1337}
1338
1339/* -- Calls --------------------------------------------------------------- */
1340
1341/* Collect arguments from CALL* and CARG instructions. */
1342static void asm_collectargs(ASMState *as, IRIns *ir,
1343 const CCallInfo *ci, IRRef *args)
1344{
1345 uint32_t n = CCI_XNARGS(ci);
1346 /* Account for split args. */
1347 lj_assertA(n <= CCI_NARGS_MAX*2, "too many args %d to collect", n);
1348 if ((ci->flags & CCI_L)) { *args++ = ASMREF_L; n--; }
1349 while (n-- > 1) {
1350 ir = IR(ir->op1);
1351 lj_assertA(ir->o == IR_CARG, "malformed CALL arg tree");
1352 args[n] = ir->op2 == REF_NIL ? 0 : ir->op2;
1353 }
1354 args[0] = ir->op1 == REF_NIL ? 0 : ir->op1;
1355 lj_assertA(IR(ir->op1)->o != IR_CARG, "malformed CALL arg tree");
1356}
1357
1358/* Reconstruct CCallInfo flags for CALLX*. */
1359static uint32_t asm_callx_flags(ASMState *as, IRIns *ir)
1360{
1361 uint32_t nargs = 0;
1362 if (ir->op1 != REF_NIL) { /* Count number of arguments first. */
1363 IRIns *ira = IR(ir->op1);
1364 nargs++;
1365 while (ira->o == IR_CARG) { nargs++; ira = IR(ira->op1); }
1366 }
1367#if LJ_HASFFI
1368 if (IR(ir->op2)->o == IR_CARG) { /* Copy calling convention info. */
1369 CTypeID id = (CTypeID)IR(IR(ir->op2)->op2)->i;
1370 CType *ct = ctype_get(ctype_ctsG(J2G(as->J)), id);
1371 nargs |= ((ct->info & CTF_VARARG) ? CCI_VARARG : 0);
1372#if LJ_TARGET_X86
1373 nargs |= (ctype_cconv(ct->info) << CCI_CC_SHIFT);
1374#endif
1375 }
1376#endif
1377 return (nargs | (ir->t.irt << CCI_OTSHIFT));
1378}
1379
1380static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
1381{
1382 const CCallInfo *ci = &lj_ir_callinfo[id];
1383 IRRef args[2];
1384 args[0] = ir->op1;
1385 args[1] = ir->op2;
1386 asm_setupresult(as, ir, ci);
1387 asm_gencall(as, ci, args);
1388}
1389
1390static void asm_call(ASMState *as, IRIns *ir)
1391{
1392 IRRef args[CCI_NARGS_MAX];
1393 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
1394 asm_collectargs(as, ir, ci, args);
1395 asm_setupresult(as, ir, ci);
1396 asm_gencall(as, ci, args);
1397}
1398
1067/* -- PHI and loop handling ----------------------------------------------- */ 1399/* -- PHI and loop handling ----------------------------------------------- */
1068 1400
1069/* Break a PHI cycle by renaming to a free register (evict if needed). */ 1401/* Break a PHI cycle by renaming to a free register (evict if needed). */
@@ -1249,12 +1581,7 @@ static void asm_phi_fixup(ASMState *as)
1249 irt_clearmark(ir->t); 1581 irt_clearmark(ir->t);
1250 /* Left PHI gained a spill slot before the loop? */ 1582 /* Left PHI gained a spill slot before the loop? */
1251 if (ra_hasspill(ir->s)) { 1583 if (ra_hasspill(ir->s)) {
1252 IRRef ren; 1584 ra_addrename(as, r, lref, as->loopsnapno);
1253 lj_ir_set(as->J, IRT(IR_RENAME, IRT_NIL), lref, as->loopsnapno);
1254 ren = tref_ref(lj_ir_emit(as->J));
1255 as->ir = as->T->ir; /* The IR may have been reallocated. */
1256 IR(ren)->r = (uint8_t)r;
1257 IR(ren)->s = SPS_NONE;
1258 } 1585 }
1259 } 1586 }
1260 rset_clear(work, r); 1587 rset_clear(work, r);
@@ -1329,6 +1656,8 @@ static void asm_loop(ASMState *as)
1329#include "lj_asm_x86.h" 1656#include "lj_asm_x86.h"
1330#elif LJ_TARGET_ARM 1657#elif LJ_TARGET_ARM
1331#include "lj_asm_arm.h" 1658#include "lj_asm_arm.h"
1659#elif LJ_TARGET_ARM64
1660#include "lj_asm_arm64.h"
1332#elif LJ_TARGET_PPC 1661#elif LJ_TARGET_PPC
1333#include "lj_asm_ppc.h" 1662#include "lj_asm_ppc.h"
1334#elif LJ_TARGET_MIPS 1663#elif LJ_TARGET_MIPS
@@ -1337,6 +1666,200 @@ static void asm_loop(ASMState *as)
1337#error "Missing assembler for target CPU" 1666#error "Missing assembler for target CPU"
1338#endif 1667#endif
1339 1668
1669/* -- Common instruction helpers ------------------------------------------ */
1670
1671#if !LJ_SOFTFP32
1672#if !LJ_TARGET_X86ORX64
1673#define asm_ldexp(as, ir) asm_callid(as, ir, IRCALL_ldexp)
1674#endif
1675
1676static void asm_pow(ASMState *as, IRIns *ir)
1677{
1678#if LJ_64 && LJ_HASFFI
1679 if (!irt_isnum(ir->t))
1680 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
1681 IRCALL_lj_carith_powu64);
1682 else
1683#endif
1684 asm_callid(as, ir, IRCALL_pow);
1685}
1686
1687static void asm_div(ASMState *as, IRIns *ir)
1688{
1689#if LJ_64 && LJ_HASFFI
1690 if (!irt_isnum(ir->t))
1691 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
1692 IRCALL_lj_carith_divu64);
1693 else
1694#endif
1695 asm_fpdiv(as, ir);
1696}
1697#endif
1698
1699static void asm_mod(ASMState *as, IRIns *ir)
1700{
1701#if LJ_64 && LJ_HASFFI
1702 if (!irt_isint(ir->t))
1703 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
1704 IRCALL_lj_carith_modu64);
1705 else
1706#endif
1707 asm_callid(as, ir, IRCALL_lj_vm_modi);
1708}
1709
1710static void asm_fuseequal(ASMState *as, IRIns *ir)
1711{
1712 /* Fuse HREF + EQ/NE. */
1713 if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
1714 as->curins--;
1715 asm_href(as, ir-1, (IROp)ir->o);
1716 } else {
1717 asm_equal(as, ir);
1718 }
1719}
1720
1721static void asm_alen(ASMState *as, IRIns *ir)
1722{
1723 asm_callid(as, ir, ir->op2 == REF_NIL ? IRCALL_lj_tab_len :
1724 IRCALL_lj_tab_len_hint);
1725}
1726
1727/* -- Instruction dispatch ------------------------------------------------ */
1728
1729/* Assemble a single instruction. */
1730static void asm_ir(ASMState *as, IRIns *ir)
1731{
1732 switch ((IROp)ir->o) {
1733 /* Miscellaneous ops. */
1734 case IR_LOOP: asm_loop(as); break;
1735 case IR_NOP: case IR_XBAR:
1736 lj_assertA(!ra_used(ir),
1737 "IR %04d not unused", (int)(ir - as->ir) - REF_BIAS);
1738 break;
1739 case IR_USE:
1740 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1741 case IR_PHI: asm_phi(as, ir); break;
1742 case IR_HIOP: asm_hiop(as, ir); break;
1743 case IR_GCSTEP: asm_gcstep(as, ir); break;
1744 case IR_PROF: asm_prof(as, ir); break;
1745
1746 /* Guarded assertions. */
1747 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1748 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1749 case IR_ABC:
1750 asm_comp(as, ir);
1751 break;
1752 case IR_EQ: case IR_NE: asm_fuseequal(as, ir); break;
1753
1754 case IR_RETF: asm_retf(as, ir); break;
1755
1756 /* Bit ops. */
1757 case IR_BNOT: asm_bnot(as, ir); break;
1758 case IR_BSWAP: asm_bswap(as, ir); break;
1759 case IR_BAND: asm_band(as, ir); break;
1760 case IR_BOR: asm_bor(as, ir); break;
1761 case IR_BXOR: asm_bxor(as, ir); break;
1762 case IR_BSHL: asm_bshl(as, ir); break;
1763 case IR_BSHR: asm_bshr(as, ir); break;
1764 case IR_BSAR: asm_bsar(as, ir); break;
1765 case IR_BROL: asm_brol(as, ir); break;
1766 case IR_BROR: asm_bror(as, ir); break;
1767
1768 /* Arithmetic ops. */
1769 case IR_ADD: asm_add(as, ir); break;
1770 case IR_SUB: asm_sub(as, ir); break;
1771 case IR_MUL: asm_mul(as, ir); break;
1772 case IR_MOD: asm_mod(as, ir); break;
1773 case IR_NEG: asm_neg(as, ir); break;
1774#if LJ_SOFTFP32
1775 case IR_DIV: case IR_POW: case IR_ABS:
1776 case IR_LDEXP: case IR_FPMATH: case IR_TOBIT:
1777 /* Unused for LJ_SOFTFP32. */
1778 lj_assertA(0, "IR %04d with unused op %d",
1779 (int)(ir - as->ir) - REF_BIAS, ir->o);
1780 break;
1781#else
1782 case IR_DIV: asm_div(as, ir); break;
1783 case IR_POW: asm_pow(as, ir); break;
1784 case IR_ABS: asm_abs(as, ir); break;
1785 case IR_LDEXP: asm_ldexp(as, ir); break;
1786 case IR_FPMATH: asm_fpmath(as, ir); break;
1787 case IR_TOBIT: asm_tobit(as, ir); break;
1788#endif
1789 case IR_MIN: asm_min(as, ir); break;
1790 case IR_MAX: asm_max(as, ir); break;
1791
1792 /* Overflow-checking arithmetic ops. */
1793 case IR_ADDOV: asm_addov(as, ir); break;
1794 case IR_SUBOV: asm_subov(as, ir); break;
1795 case IR_MULOV: asm_mulov(as, ir); break;
1796
1797 /* Memory references. */
1798 case IR_AREF: asm_aref(as, ir); break;
1799 case IR_HREF: asm_href(as, ir, 0); break;
1800 case IR_HREFK: asm_hrefk(as, ir); break;
1801 case IR_NEWREF: asm_newref(as, ir); break;
1802 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
1803 case IR_FREF: asm_fref(as, ir); break;
1804 case IR_TMPREF: asm_tmpref(as, ir); break;
1805 case IR_STRREF: asm_strref(as, ir); break;
1806 case IR_LREF: asm_lref(as, ir); break;
1807
1808 /* Loads and stores. */
1809 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
1810 asm_ahuvload(as, ir);
1811 break;
1812 case IR_FLOAD: asm_fload(as, ir); break;
1813 case IR_XLOAD: asm_xload(as, ir); break;
1814 case IR_SLOAD: asm_sload(as, ir); break;
1815 case IR_ALEN: asm_alen(as, ir); break;
1816
1817 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
1818 case IR_FSTORE: asm_fstore(as, ir); break;
1819 case IR_XSTORE: asm_xstore(as, ir); break;
1820
1821 /* Allocations. */
1822 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
1823 case IR_TNEW: asm_tnew(as, ir); break;
1824 case IR_TDUP: asm_tdup(as, ir); break;
1825 case IR_CNEW: case IR_CNEWI:
1826#if LJ_HASFFI
1827 asm_cnew(as, ir);
1828#else
1829 lj_assertA(0, "IR %04d with unused op %d",
1830 (int)(ir - as->ir) - REF_BIAS, ir->o);
1831#endif
1832 break;
1833
1834 /* Buffer operations. */
1835 case IR_BUFHDR: asm_bufhdr(as, ir); break;
1836 case IR_BUFPUT: asm_bufput(as, ir); break;
1837 case IR_BUFSTR: asm_bufstr(as, ir); break;
1838
1839 /* Write barriers. */
1840 case IR_TBAR: asm_tbar(as, ir); break;
1841 case IR_OBAR: asm_obar(as, ir); break;
1842
1843 /* Type conversions. */
1844 case IR_CONV: asm_conv(as, ir); break;
1845 case IR_TOSTR: asm_tostr(as, ir); break;
1846 case IR_STRTO: asm_strto(as, ir); break;
1847
1848 /* Calls. */
1849 case IR_CALLA:
1850 as->gcsteps++;
1851 /* fallthrough */
1852 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
1853 case IR_CALLXS: asm_callx(as, ir); break;
1854 case IR_CARG: break;
1855
1856 default:
1857 setintV(&as->J->errinfo, ir->o);
1858 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
1859 break;
1860 }
1861}
1862
1340/* -- Head of trace ------------------------------------------------------- */ 1863/* -- Head of trace ------------------------------------------------------- */
1341 1864
1342/* Head of a root trace. */ 1865/* Head of a root trace. */
@@ -1375,8 +1898,7 @@ static void asm_head_side(ASMState *as)
1375 1898
1376 if (as->snapno && as->topslot > as->parent->topslot) { 1899 if (as->snapno && as->topslot > as->parent->topslot) {
1377 /* Force snap #0 alloc to prevent register overwrite in stack check. */ 1900 /* Force snap #0 alloc to prevent register overwrite in stack check. */
1378 as->snapno = 0; 1901 asm_snap_alloc(as, 0);
1379 asm_snap_alloc(as);
1380 } 1902 }
1381 pbase = asm_head_side_base(as, irp); 1903 pbase = asm_head_side_base(as, irp);
1382 if (pbase != RID_NONE) { 1904 if (pbase != RID_NONE) {
@@ -1388,8 +1910,10 @@ static void asm_head_side(ASMState *as)
1388 for (i = as->stopins; i > REF_BASE; i--) { 1910 for (i = as->stopins; i > REF_BASE; i--) {
1389 IRIns *ir = IR(i); 1911 IRIns *ir = IR(i);
1390 RegSP rs; 1912 RegSP rs;
1391 lua_assert((ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT)) || 1913 lj_assertA((ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT)) ||
1392 (LJ_SOFTFP && ir->o == IR_HIOP) || ir->o == IR_PVAL); 1914 (LJ_SOFTFP && ir->o == IR_HIOP) || ir->o == IR_PVAL,
1915 "IR %04d has bad parent op %d",
1916 (int)(ir - as->ir) - REF_BIAS, ir->o);
1393 rs = as->parentmap[i - REF_FIRST]; 1917 rs = as->parentmap[i - REF_FIRST];
1394 if (ra_hasreg(ir->r)) { 1918 if (ra_hasreg(ir->r)) {
1395 rset_clear(allow, ir->r); 1919 rset_clear(allow, ir->r);
@@ -1542,7 +2066,7 @@ static BCReg asm_baseslot(ASMState *as, SnapShot *snap, int *gotframe)
1542 SnapEntry sn = map[n-1]; 2066 SnapEntry sn = map[n-1];
1543 if ((sn & SNAP_FRAME)) { 2067 if ((sn & SNAP_FRAME)) {
1544 *gotframe = 1; 2068 *gotframe = 1;
1545 return snap_slot(sn); 2069 return snap_slot(sn) - LJ_FR2;
1546 } 2070 }
1547 } 2071 }
1548 return 0; 2072 return 0;
@@ -1562,19 +2086,23 @@ static void asm_tail_link(ASMState *as)
1562 2086
1563 if (as->T->link == 0) { 2087 if (as->T->link == 0) {
1564 /* Setup fixed registers for exit to interpreter. */ 2088 /* Setup fixed registers for exit to interpreter. */
1565 const BCIns *pc = snap_pc(as->T->snapmap[snap->mapofs + snap->nent]); 2089 const BCIns *pc = snap_pc(&as->T->snapmap[snap->mapofs + snap->nent]);
1566 int32_t mres; 2090 int32_t mres;
1567 if (bc_op(*pc) == BC_JLOOP) { /* NYI: find a better way to do this. */ 2091 if (bc_op(*pc) == BC_JLOOP) { /* NYI: find a better way to do this. */
1568 BCIns *retpc = &traceref(as->J, bc_d(*pc))->startins; 2092 BCIns *retpc = &traceref(as->J, bc_d(*pc))->startins;
1569 if (bc_isret(bc_op(*retpc))) 2093 if (bc_isret(bc_op(*retpc)))
1570 pc = retpc; 2094 pc = retpc;
1571 } 2095 }
2096#if LJ_GC64
2097 emit_loadu64(as, RID_LPC, u64ptr(pc));
2098#else
1572 ra_allockreg(as, i32ptr(J2GG(as->J)->dispatch), RID_DISPATCH); 2099 ra_allockreg(as, i32ptr(J2GG(as->J)->dispatch), RID_DISPATCH);
1573 ra_allockreg(as, i32ptr(pc), RID_LPC); 2100 ra_allockreg(as, i32ptr(pc), RID_LPC);
1574 mres = (int32_t)(snap->nslots - baseslot); 2101#endif
2102 mres = (int32_t)(snap->nslots - baseslot - LJ_FR2);
1575 switch (bc_op(*pc)) { 2103 switch (bc_op(*pc)) {
1576 case BC_CALLM: case BC_CALLMT: 2104 case BC_CALLM: case BC_CALLMT:
1577 mres -= (int32_t)(1 + bc_a(*pc) + bc_c(*pc)); break; 2105 mres -= (int32_t)(1 + LJ_FR2 + bc_a(*pc) + bc_c(*pc)); break;
1578 case BC_RETM: mres -= (int32_t)(bc_a(*pc) + bc_d(*pc)); break; 2106 case BC_RETM: mres -= (int32_t)(bc_a(*pc) + bc_d(*pc)); break;
1579 case BC_TSETM: mres -= (int32_t)bc_a(*pc); break; 2107 case BC_TSETM: mres -= (int32_t)bc_a(*pc); break;
1580 default: if (bc_op(*pc) < BC_FUNCF) mres = 0; break; 2108 default: if (bc_op(*pc) < BC_FUNCF) mres = 0; break;
@@ -1586,6 +2114,11 @@ static void asm_tail_link(ASMState *as)
1586 } 2114 }
1587 emit_addptr(as, RID_BASE, 8*(int32_t)baseslot); 2115 emit_addptr(as, RID_BASE, 8*(int32_t)baseslot);
1588 2116
2117 if (as->J->ktrace) { /* Patch ktrace slot with the final GCtrace pointer. */
2118 setgcref(IR(as->J->ktrace)[LJ_GC64].gcr, obj2gco(as->J->curfinal));
2119 IR(as->J->ktrace)->o = IR_KGC;
2120 }
2121
1589 /* Sync the interpreter state with the on-trace state. */ 2122 /* Sync the interpreter state with the on-trace state. */
1590 asm_stack_restore(as, snap); 2123 asm_stack_restore(as, snap);
1591 2124
@@ -1609,22 +2142,32 @@ static void asm_setup_regsp(ASMState *as)
1609#endif 2142#endif
1610 2143
1611 ra_setup(as); 2144 ra_setup(as);
2145#if LJ_TARGET_ARM64
2146 ra_setkref(as, RID_GL, (intptr_t)J2G(as->J));
2147#endif
1612 2148
1613 /* Clear reg/sp for constants. */ 2149 /* Clear reg/sp for constants. */
1614 for (ir = IR(T->nk), lastir = IR(REF_BASE); ir < lastir; ir++) 2150 for (ir = IR(T->nk), lastir = IR(REF_BASE); ir < lastir; ir++) {
1615 ir->prev = REGSP_INIT; 2151 ir->prev = REGSP_INIT;
2152 if (irt_is64(ir->t) && ir->o != IR_KNULL) {
2153#if LJ_GC64
2154 /* The false-positive of irt_is64() for ASMREF_L (REF_NIL) is OK here. */
2155 ir->i = 0; /* Will become non-zero only for RIP-relative addresses. */
2156#else
2157 /* Make life easier for backends by putting address of constant in i. */
2158 ir->i = (int32_t)(intptr_t)(ir+1);
2159#endif
2160 ir++;
2161 }
2162 }
1616 2163
1617 /* REF_BASE is used for implicit references to the BASE register. */ 2164 /* REF_BASE is used for implicit references to the BASE register. */
1618 lastir->prev = REGSP_HINT(RID_BASE); 2165 lastir->prev = REGSP_HINT(RID_BASE);
1619 2166
1620 ir = IR(nins-1);
1621 if (ir->o == IR_RENAME) {
1622 do { ir--; nins--; } while (ir->o == IR_RENAME);
1623 T->nins = nins; /* Remove any renames left over from ASM restart. */
1624 }
1625 as->snaprename = nins; 2167 as->snaprename = nins;
1626 as->snapref = nins; 2168 as->snapref = nins;
1627 as->snapno = T->nsnap; 2169 as->snapno = T->nsnap;
2170 as->snapalloc = 0;
1628 2171
1629 as->stopins = REF_BASE; 2172 as->stopins = REF_BASE;
1630 as->orignins = nins; 2173 as->orignins = nins;
@@ -1634,7 +2177,7 @@ static void asm_setup_regsp(ASMState *as)
1634 ir = IR(REF_FIRST); 2177 ir = IR(REF_FIRST);
1635 if (as->parent) { 2178 if (as->parent) {
1636 uint16_t *p; 2179 uint16_t *p;
1637 lastir = lj_snap_regspmap(as->parent, as->J->exitno, ir); 2180 lastir = lj_snap_regspmap(as->J, as->parent, as->J->exitno, ir);
1638 if (lastir - ir > LJ_MAX_JSLOTS) 2181 if (lastir - ir > LJ_MAX_JSLOTS)
1639 lj_trace_err(as->J, LJ_TRERR_NYICOAL); 2182 lj_trace_err(as->J, LJ_TRERR_NYICOAL);
1640 as->stopins = (IRRef)((lastir-1) - as->ir); 2183 as->stopins = (IRRef)((lastir-1) - as->ir);
@@ -1673,6 +2216,10 @@ static void asm_setup_regsp(ASMState *as)
1673 ir->prev = (uint16_t)REGSP_HINT((rload & 15)); 2216 ir->prev = (uint16_t)REGSP_HINT((rload & 15));
1674 rload = lj_ror(rload, 4); 2217 rload = lj_ror(rload, 4);
1675 continue; 2218 continue;
2219 case IR_TMPREF:
2220 if ((ir->op2 & IRTMPREF_OUT2) && as->evenspill < 4)
2221 as->evenspill = 4; /* TMPREF OUT2 needs two TValues on the stack. */
2222 break;
1676#endif 2223#endif
1677 case IR_CALLXS: { 2224 case IR_CALLXS: {
1678 CCallInfo ci; 2225 CCallInfo ci;
@@ -1682,7 +2229,17 @@ static void asm_setup_regsp(ASMState *as)
1682 as->modset |= RSET_SCRATCH; 2229 as->modset |= RSET_SCRATCH;
1683 continue; 2230 continue;
1684 } 2231 }
1685 case IR_CALLN: case IR_CALLL: case IR_CALLS: { 2232 case IR_CALLL:
2233 /* lj_vm_next needs two TValues on the stack. */
2234#if LJ_TARGET_X64 && LJ_ABI_WIN
2235 if (ir->op2 == IRCALL_lj_vm_next && as->evenspill < SPS_FIRST + 4)
2236 as->evenspill = SPS_FIRST + 4;
2237#else
2238 if (SPS_FIRST < 4 && ir->op2 == IRCALL_lj_vm_next && as->evenspill < 4)
2239 as->evenspill = 4;
2240#endif
2241 /* fallthrough */
2242 case IR_CALLN: case IR_CALLA: case IR_CALLS: {
1686 const CCallInfo *ci = &lj_ir_callinfo[ir->op2]; 2243 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
1687 ir->prev = asm_setup_call_slots(as, ir, ci); 2244 ir->prev = asm_setup_call_slots(as, ir, ci);
1688 if (inloop) 2245 if (inloop)
@@ -1690,7 +2247,6 @@ static void asm_setup_regsp(ASMState *as)
1690 (RSET_SCRATCH & ~RSET_FPR) : RSET_SCRATCH; 2247 (RSET_SCRATCH & ~RSET_FPR) : RSET_SCRATCH;
1691 continue; 2248 continue;
1692 } 2249 }
1693#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
1694 case IR_HIOP: 2250 case IR_HIOP:
1695 switch ((ir-1)->o) { 2251 switch ((ir-1)->o) {
1696#if LJ_SOFTFP && LJ_TARGET_ARM 2252#if LJ_SOFTFP && LJ_TARGET_ARM
@@ -1701,15 +2257,15 @@ static void asm_setup_regsp(ASMState *as)
1701 } 2257 }
1702 break; 2258 break;
1703#endif 2259#endif
1704#if !LJ_SOFTFP && LJ_NEED_FP64 2260#if !LJ_SOFTFP && LJ_NEED_FP64 && LJ_32 && LJ_HASFFI
1705 case IR_CONV: 2261 case IR_CONV:
1706 if (irt_isfp((ir-1)->t)) { 2262 if (irt_isfp((ir-1)->t)) {
1707 ir->prev = REGSP_HINT(RID_FPRET); 2263 ir->prev = REGSP_HINT(RID_FPRET);
1708 continue; 2264 continue;
1709 } 2265 }
1710 /* fallthrough */
1711#endif 2266#endif
1712 case IR_CALLN: case IR_CALLXS: 2267 /* fallthrough */
2268 case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
1713#if LJ_SOFTFP 2269#if LJ_SOFTFP
1714 case IR_MIN: case IR_MAX: 2270 case IR_MIN: case IR_MAX:
1715#endif 2271#endif
@@ -1720,18 +2276,29 @@ static void asm_setup_regsp(ASMState *as)
1720 break; 2276 break;
1721 } 2277 }
1722 break; 2278 break;
1723#endif
1724#if LJ_SOFTFP 2279#if LJ_SOFTFP
1725 case IR_MIN: case IR_MAX: 2280 case IR_MIN: case IR_MAX:
1726 if ((ir+1)->o != IR_HIOP) break; 2281 if ((ir+1)->o != IR_HIOP) break;
1727#endif 2282#endif
1728 /* fallthrough */ 2283 /* fallthrough */
1729 /* C calls evict all scratch regs and return results in RID_RET. */ 2284 /* C calls evict all scratch regs and return results in RID_RET. */
1730 case IR_SNEW: case IR_XSNEW: case IR_NEWREF: 2285 case IR_SNEW: case IR_XSNEW: case IR_NEWREF: case IR_BUFPUT:
1731 if (REGARG_NUMGPR < 3 && as->evenspill < 3) 2286 if (REGARG_NUMGPR < 3 && as->evenspill < 3)
1732 as->evenspill = 3; /* lj_str_new and lj_tab_newkey need 3 args. */ 2287 as->evenspill = 3; /* lj_str_new and lj_tab_newkey need 3 args. */
2288#if LJ_TARGET_X86 && LJ_HASFFI
2289 if (0) {
2290 case IR_CNEW:
2291 if (ir->op2 != REF_NIL && as->evenspill < 4)
2292 as->evenspill = 4; /* lj_cdata_newv needs 4 args. */
2293 }
1733 /* fallthrough */ 2294 /* fallthrough */
1734 case IR_TNEW: case IR_TDUP: case IR_CNEW: case IR_CNEWI: case IR_TOSTR: 2295#else
2296 /* fallthrough */
2297 case IR_CNEW:
2298#endif
2299 /* fallthrough */
2300 case IR_TNEW: case IR_TDUP: case IR_CNEWI: case IR_TOSTR:
2301 case IR_BUFSTR:
1735 ir->prev = REGSP_HINT(RID_RET); 2302 ir->prev = REGSP_HINT(RID_RET);
1736 if (inloop) 2303 if (inloop)
1737 as->modset = RSET_SCRATCH; 2304 as->modset = RSET_SCRATCH;
@@ -1740,58 +2307,73 @@ static void asm_setup_regsp(ASMState *as)
1740 if (inloop) 2307 if (inloop)
1741 as->modset = RSET_SCRATCH; 2308 as->modset = RSET_SCRATCH;
1742 break; 2309 break;
1743#if !LJ_TARGET_X86ORX64 && !LJ_SOFTFP 2310#if !LJ_SOFTFP
1744 case IR_ATAN2: case IR_LDEXP: 2311#if !LJ_TARGET_X86ORX64
2312 case IR_LDEXP:
1745#endif 2313#endif
2314#endif
2315 /* fallthrough */
1746 case IR_POW: 2316 case IR_POW:
1747 if (!LJ_SOFTFP && irt_isnum(ir->t)) { 2317 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1748#if LJ_TARGET_X86ORX64
1749 ir->prev = REGSP_HINT(RID_XMM0);
1750 if (inloop) 2318 if (inloop)
1751 as->modset |= RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX); 2319 as->modset |= RSET_SCRATCH;
2320#if LJ_TARGET_X86
2321 if (irt_isnum(IR(ir->op2)->t)) {
2322 if (as->evenspill < 4) /* Leave room to call pow(). */
2323 as->evenspill = 4;
2324 }
2325 break;
1752#else 2326#else
1753 ir->prev = REGSP_HINT(RID_FPRET); 2327 ir->prev = REGSP_HINT(RID_FPRET);
1754 if (inloop)
1755 as->modset |= RSET_SCRATCH;
1756#endif
1757 continue; 2328 continue;
2329#endif
1758 } 2330 }
1759 /* fallthrough */ /* for integer POW */ 2331 /* fallthrough */ /* for integer POW */
1760 case IR_DIV: case IR_MOD: 2332 case IR_DIV: case IR_MOD:
1761 if (!irt_isnum(ir->t)) { 2333 if ((LJ_64 && LJ_SOFTFP) || !irt_isnum(ir->t)) {
1762 ir->prev = REGSP_HINT(RID_RET); 2334 ir->prev = REGSP_HINT(RID_RET);
1763 if (inloop) 2335 if (inloop)
1764 as->modset |= (RSET_SCRATCH & RSET_GPR); 2336 as->modset |= (RSET_SCRATCH & RSET_GPR);
1765 continue; 2337 continue;
1766 } 2338 }
1767 break; 2339 break;
1768 case IR_FPMATH: 2340#if LJ_64 && LJ_SOFTFP
1769#if LJ_TARGET_X86ORX64 2341 case IR_ADD: case IR_SUB: case IR_MUL:
1770 if (ir->op2 == IRFPM_EXP2) { /* May be joined to lj_vm_pow_sse. */ 2342 if (irt_isnum(ir->t)) {
1771 ir->prev = REGSP_HINT(RID_XMM0); 2343 ir->prev = REGSP_HINT(RID_RET);
1772#if !LJ_64
1773 if (as->evenspill < 4) /* Leave room for 16 byte scratch area. */
1774 as->evenspill = 4;
1775#endif
1776 if (inloop)
1777 as->modset |= RSET_RANGE(RID_XMM0, RID_XMM2+1)|RID2RSET(RID_EAX);
1778 continue;
1779 } else if (ir->op2 <= IRFPM_TRUNC && !(as->flags & JIT_F_SSE4_1)) {
1780 ir->prev = REGSP_HINT(RID_XMM0);
1781 if (inloop) 2344 if (inloop)
1782 as->modset |= RSET_RANGE(RID_XMM0, RID_XMM3+1)|RID2RSET(RID_EAX); 2345 as->modset |= (RSET_SCRATCH & RSET_GPR);
1783 continue; 2346 continue;
1784 } 2347 }
1785 break; 2348 break;
1786#else 2349#endif
1787 ir->prev = REGSP_HINT(RID_FPRET); 2350 case IR_FPMATH:
2351#if LJ_TARGET_X86ORX64
2352 if (ir->op2 <= IRFPM_TRUNC) {
2353 if (!(as->flags & JIT_F_SSE4_1)) {
2354 ir->prev = REGSP_HINT(RID_XMM0);
2355 if (inloop)
2356 as->modset |= RSET_RANGE(RID_XMM0, RID_XMM3+1)|RID2RSET(RID_EAX);
2357 continue;
2358 }
2359 break;
2360 }
2361#endif
1788 if (inloop) 2362 if (inloop)
1789 as->modset |= RSET_SCRATCH; 2363 as->modset |= RSET_SCRATCH;
2364#if LJ_TARGET_X86
2365 break;
2366#else
2367 ir->prev = REGSP_HINT(RID_FPRET);
1790 continue; 2368 continue;
1791#endif 2369#endif
1792#if LJ_TARGET_X86ORX64 2370#if LJ_TARGET_X86ORX64
1793 /* Non-constant shift counts need to be in RID_ECX on x86/x64. */ 2371 /* Non-constant shift counts need to be in RID_ECX on x86/x64. */
1794 case IR_BSHL: case IR_BSHR: case IR_BSAR: case IR_BROL: case IR_BROR: 2372 case IR_BSHL: case IR_BSHR: case IR_BSAR:
2373 if ((as->flags & JIT_F_BMI2)) /* Except if BMI2 is available. */
2374 break;
2375 /* fallthrough */
2376 case IR_BROL: case IR_BROR:
1795 if (!irref_isk(ir->op2) && !ra_hashint(IR(ir->op2)->r)) { 2377 if (!irref_isk(ir->op2) && !ra_hashint(IR(ir->op2)->r)) {
1796 IR(ir->op2)->r = REGSP_HINT(RID_ECX); 2378 IR(ir->op2)->r = REGSP_HINT(RID_ECX);
1797 if (inloop) 2379 if (inloop)
@@ -1835,16 +2417,26 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
1835{ 2417{
1836 ASMState as_; 2418 ASMState as_;
1837 ASMState *as = &as_; 2419 ASMState *as = &as_;
1838 MCode *origtop; 2420
2421 /* Remove nops/renames left over from ASM restart due to LJ_TRERR_MCODELM. */
2422 {
2423 IRRef nins = T->nins;
2424 IRIns *ir = &T->ir[nins-1];
2425 if (ir->o == IR_NOP || ir->o == IR_RENAME) {
2426 do { ir--; nins--; } while (ir->o == IR_NOP || ir->o == IR_RENAME);
2427 T->nins = nins;
2428 }
2429 }
1839 2430
1840 /* Ensure an initialized instruction beyond the last one for HIOP checks. */ 2431 /* Ensure an initialized instruction beyond the last one for HIOP checks. */
1841 J->cur.nins = lj_ir_nextins(J); 2432 /* This also allows one RENAME to be added without reallocating curfinal. */
1842 lj_ir_nop(&J->cur.ir[J->cur.nins]); 2433 as->orignins = lj_ir_nextins(J);
2434 lj_ir_nop(&J->cur.ir[as->orignins]);
1843 2435
1844 /* Setup initial state. Copy some fields to reduce indirections. */ 2436 /* Setup initial state. Copy some fields to reduce indirections. */
1845 as->J = J; 2437 as->J = J;
1846 as->T = T; 2438 as->T = T;
1847 as->ir = T->ir; 2439 J->curfinal = lj_trace_alloc(J->L, T); /* This copies the IR, too. */
1848 as->flags = J->flags; 2440 as->flags = J->flags;
1849 as->loopref = J->loopref; 2441 as->loopref = J->loopref;
1850 as->realign = NULL; 2442 as->realign = NULL;
@@ -1852,17 +2444,46 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
1852 as->parent = J->parent ? traceref(J, J->parent) : NULL; 2444 as->parent = J->parent ? traceref(J, J->parent) : NULL;
1853 2445
1854 /* Reserve MCode memory. */ 2446 /* Reserve MCode memory. */
1855 as->mctop = origtop = lj_mcode_reserve(J, &as->mcbot); 2447 as->mctop = as->mctoporig = lj_mcode_reserve(J, &as->mcbot);
1856 as->mcp = as->mctop; 2448 as->mcp = as->mctop;
1857 as->mclim = as->mcbot + MCLIM_REDZONE; 2449 as->mclim = as->mcbot + MCLIM_REDZONE;
1858 asm_setup_target(as); 2450 asm_setup_target(as);
1859 2451
1860 do { 2452 /*
2453 ** This is a loop, because the MCode may have to be (re-)assembled
2454 ** multiple times:
2455 **
2456 ** 1. as->realign is set (and the assembly aborted), if the arch-specific
2457 ** backend wants the MCode to be aligned differently.
2458 **
2459 ** This is currently only the case on x86/x64, where small loops get
2460 ** an aligned loop body plus a short branch. Not much effort is wasted,
2461 ** because the abort happens very quickly and only once.
2462 **
2463 ** 2. The IR is immovable, since the MCode embeds pointers to various
2464 ** constants inside the IR. But RENAMEs may need to be added to the IR
2465 ** during assembly, which might grow and reallocate the IR. We check
2466 ** at the end if the IR (in J->cur.ir) has actually grown, resize the
2467 ** copy (in J->curfinal.ir) and try again.
2468 **
2469 ** 95% of all traces have zero RENAMEs, 3% have one RENAME, 1.5% have
2470 ** 2 RENAMEs and only 0.5% have more than that. That's why we opt to
2471 ** always have one spare slot in the IR (see above), which means we
2472 ** have to redo the assembly for only ~2% of all traces.
2473 **
2474 ** Very, very rarely, this needs to be done repeatedly, since the
2475 ** location of constants inside the IR (actually, reachability from
2476 ** a global pointer) may affect register allocation and thus the
2477 ** number of RENAMEs.
2478 */
2479 for (;;) {
1861 as->mcp = as->mctop; 2480 as->mcp = as->mctop;
1862#ifdef LUA_USE_ASSERT 2481#ifdef LUA_USE_ASSERT
1863 as->mcp_prev = as->mcp; 2482 as->mcp_prev = as->mcp;
1864#endif 2483#endif
1865 as->curins = T->nins; 2484 as->ir = J->curfinal->ir; /* Use the copied IR. */
2485 as->curins = J->cur.nins = as->orignins;
2486
1866 RA_DBG_START(); 2487 RA_DBG_START();
1867 RA_DBGX((as, "===== STOP =====")); 2488 RA_DBGX((as, "===== STOP ====="));
1868 2489
@@ -1881,7 +2502,11 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
1881 /* Assemble a trace in linear backwards order. */ 2502 /* Assemble a trace in linear backwards order. */
1882 for (as->curins--; as->curins > as->stopins; as->curins--) { 2503 for (as->curins--; as->curins > as->stopins; as->curins--) {
1883 IRIns *ir = IR(as->curins); 2504 IRIns *ir = IR(as->curins);
1884 lua_assert(!(LJ_32 && irt_isint64(ir->t))); /* Handled by SPLIT. */ 2505 /* 64 bit types handled by SPLIT for 32 bit archs. */
2506 lj_assertA(!(LJ_32 && irt_isint64(ir->t)),
2507 "IR %04d has unsplit 64 bit type",
2508 (int)(ir - as->ir) - REF_BIAS);
2509 asm_snap_prev(as);
1885 if (!ra_used(ir) && !ir_sideeff(ir) && (as->flags & JIT_F_OPT_DCE)) 2510 if (!ra_used(ir) && !ir_sideeff(ir) && (as->flags & JIT_F_OPT_DCE))
1886 continue; /* Dead-code elimination can be soooo easy. */ 2511 continue; /* Dead-code elimination can be soooo easy. */
1887 if (irt_isguard(ir->t)) 2512 if (irt_isguard(ir->t))
@@ -1890,22 +2515,43 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
1890 checkmclim(as); 2515 checkmclim(as);
1891 asm_ir(as, ir); 2516 asm_ir(as, ir);
1892 } 2517 }
1893 } while (as->realign); /* Retry in case the MCode needs to be realigned. */
1894 2518
1895 /* Emit head of trace. */ 2519 if (as->realign && J->curfinal->nins >= T->nins)
1896 RA_DBG_REF(); 2520 continue; /* Retry in case only the MCode needs to be realigned. */
1897 checkmclim(as); 2521
1898 if (as->gcsteps > 0) { 2522 /* Emit head of trace. */
1899 as->curins = as->T->snap[0].ref; 2523 RA_DBG_REF();
1900 asm_snap_prep(as); /* The GC check is a guard. */ 2524 checkmclim(as);
1901 asm_gc_check(as); 2525 if (as->gcsteps > 0) {
2526 as->curins = as->T->snap[0].ref;
2527 asm_snap_prep(as); /* The GC check is a guard. */
2528 asm_gc_check(as);
2529 as->curins = as->stopins;
2530 }
2531 ra_evictk(as);
2532 if (as->parent)
2533 asm_head_side(as);
2534 else
2535 asm_head_root(as);
2536 asm_phi_fixup(as);
2537
2538 if (J->curfinal->nins >= T->nins) { /* IR didn't grow? */
2539 lj_assertA(J->curfinal->nk == T->nk, "unexpected IR constant growth");
2540 memcpy(J->curfinal->ir + as->orignins, T->ir + as->orignins,
2541 (T->nins - as->orignins) * sizeof(IRIns)); /* Copy RENAMEs. */
2542 T->nins = J->curfinal->nins;
2543 /* Fill mcofs of any unprocessed snapshots. */
2544 as->curins = REF_FIRST;
2545 asm_snap_prev(as);
2546 break; /* Done. */
2547 }
2548
2549 /* Otherwise try again with a bigger IR. */
2550 lj_trace_free(J2G(J), J->curfinal);
2551 J->curfinal = NULL; /* In case lj_trace_alloc() OOMs. */
2552 J->curfinal = lj_trace_alloc(J->L, T);
2553 as->realign = NULL;
1902 } 2554 }
1903 ra_evictk(as);
1904 if (as->parent)
1905 asm_head_side(as);
1906 else
1907 asm_head_root(as);
1908 asm_phi_fixup(as);
1909 2555
1910 RA_DBGX((as, "===== START ====")); 2556 RA_DBGX((as, "===== START ===="));
1911 RA_DBG_FLUSH(); 2557 RA_DBG_FLUSH();
@@ -1915,10 +2561,16 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
1915 /* Set trace entry point before fixing up tail to allow link to self. */ 2561 /* Set trace entry point before fixing up tail to allow link to self. */
1916 T->mcode = as->mcp; 2562 T->mcode = as->mcp;
1917 T->mcloop = as->mcloop ? (MSize)((char *)as->mcloop - (char *)as->mcp) : 0; 2563 T->mcloop = as->mcloop ? (MSize)((char *)as->mcloop - (char *)as->mcp) : 0;
1918 if (!as->loopref) 2564 if (as->loopref)
2565 asm_loop_tail_fixup(as);
2566 else
1919 asm_tail_fixup(as, T->link); /* Note: this may change as->mctop! */ 2567 asm_tail_fixup(as, T->link); /* Note: this may change as->mctop! */
1920 T->szmcode = (MSize)((char *)as->mctop - (char *)as->mcp); 2568 T->szmcode = (MSize)((char *)as->mctop - (char *)as->mcp);
1921 lj_mcode_sync(T->mcode, origtop); 2569 asm_snap_fixup_mcofs(as);
2570#if LJ_TARGET_MCODE_FIXUP
2571 asm_mcode_fixup(T->mcode, T->szmcode);
2572#endif
2573 lj_mcode_sync(T->mcode, as->mctoporig);
1922} 2574}
1923 2575
1924#undef IR 2576#undef IR
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index 20e57393..b3b1f096 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -41,7 +41,7 @@ static Reg ra_scratchpair(ASMState *as, RegSet allow)
41 } 41 }
42 } 42 }
43 } 43 }
44 lua_assert(rset_test(RSET_GPREVEN, r)); 44 lj_assertA(rset_test(RSET_GPREVEN, r), "odd reg %d", r);
45 ra_modified(as, r); 45 ra_modified(as, r);
46 ra_modified(as, r+1); 46 ra_modified(as, r+1);
47 RA_DBGX((as, "scratchpair $r $r", r, r+1)); 47 RA_DBGX((as, "scratchpair $r $r", r, r+1));
@@ -185,6 +185,9 @@ static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow,
185 *ofsp = (ofs & 255); /* Mask out less bits to allow LDRD. */ 185 *ofsp = (ofs & 255); /* Mask out less bits to allow LDRD. */
186 return ra_allock(as, (ofs & ~255), allow); 186 return ra_allock(as, (ofs & ~255), allow);
187 } 187 }
188 } else if (ir->o == IR_TMPREF) {
189 *ofsp = 0;
190 return RID_SP;
188 } 191 }
189 } 192 }
190 *ofsp = 0; 193 *ofsp = 0;
@@ -269,7 +272,7 @@ static void asm_fusexref(ASMState *as, ARMIns ai, Reg rd, IRRef ref,
269 return; 272 return;
270 } 273 }
271 } else if (ir->o == IR_STRREF && !(!LJ_SOFTFP && (ai & 0x08000000))) { 274 } else if (ir->o == IR_STRREF && !(!LJ_SOFTFP && (ai & 0x08000000))) {
272 lua_assert(ofs == 0); 275 lj_assertA(ofs == 0, "bad usage");
273 ofs = (int32_t)sizeof(GCstr); 276 ofs = (int32_t)sizeof(GCstr);
274 if (irref_isk(ir->op2)) { 277 if (irref_isk(ir->op2)) {
275 ofs += IR(ir->op2)->i; 278 ofs += IR(ir->op2)->i;
@@ -310,7 +313,11 @@ static void asm_fusexref(ASMState *as, ARMIns ai, Reg rd, IRRef ref,
310} 313}
311 314
312#if !LJ_SOFTFP 315#if !LJ_SOFTFP
313/* Fuse to multiply-add/sub instruction. */ 316/*
317** Fuse to multiply-add/sub instruction.
318** VMLA rounds twice (UMA, not FMA) -- no need to check for JIT_F_OPT_FMA.
319** VFMA needs VFPv4, which is uncommon on the remaining ARM32 targets.
320*/
314static int asm_fusemadd(ASMState *as, IRIns *ir, ARMIns ai, ARMIns air) 321static int asm_fusemadd(ASMState *as, IRIns *ir, ARMIns ai, ARMIns air)
315{ 322{
316 IRRef lref = ir->op1, rref = ir->op2; 323 IRRef lref = ir->op1, rref = ir->op2;
@@ -338,7 +345,7 @@ static int asm_fusemadd(ASMState *as, IRIns *ir, ARMIns ai, ARMIns air)
338/* Generate a call to a C function. */ 345/* Generate a call to a C function. */
339static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) 346static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
340{ 347{
341 uint32_t n, nargs = CCI_NARGS(ci); 348 uint32_t n, nargs = CCI_XNARGS(ci);
342 int32_t ofs = 0; 349 int32_t ofs = 0;
343#if LJ_SOFTFP 350#if LJ_SOFTFP
344 Reg gpr = REGARG_FIRSTGPR; 351 Reg gpr = REGARG_FIRSTGPR;
@@ -389,9 +396,11 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
389 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1)); 396 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
390 if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1u; 397 if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1u;
391 if (gpr <= REGARG_LASTGPR) { 398 if (gpr <= REGARG_LASTGPR) {
392 lua_assert(rset_test(as->freeset, gpr)); /* Must have been evicted. */ 399 lj_assertA(rset_test(as->freeset, gpr),
400 "reg %d not free", gpr); /* Must have been evicted. */
393 if (irt_isnum(ir->t)) { 401 if (irt_isnum(ir->t)) {
394 lua_assert(rset_test(as->freeset, gpr+1)); /* Ditto. */ 402 lj_assertA(rset_test(as->freeset, gpr+1),
403 "reg %d not free", gpr+1); /* Ditto. */
395 emit_dnm(as, ARMI_VMOV_RR_D, gpr, gpr+1, (src & 15)); 404 emit_dnm(as, ARMI_VMOV_RR_D, gpr, gpr+1, (src & 15));
396 gpr += 2; 405 gpr += 2;
397 } else { 406 } else {
@@ -408,7 +417,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
408#endif 417#endif
409 { 418 {
410 if (gpr <= REGARG_LASTGPR) { 419 if (gpr <= REGARG_LASTGPR) {
411 lua_assert(rset_test(as->freeset, gpr)); /* Must have been evicted. */ 420 lj_assertA(rset_test(as->freeset, gpr),
421 "reg %d not free", gpr); /* Must have been evicted. */
412 if (ref) ra_leftov(as, gpr, ref); 422 if (ref) ra_leftov(as, gpr, ref);
413 gpr++; 423 gpr++;
414 } else { 424 } else {
@@ -433,7 +443,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
433 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */ 443 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
434 ra_evictset(as, drop); /* Evictions must be performed first. */ 444 ra_evictset(as, drop); /* Evictions must be performed first. */
435 if (ra_used(ir)) { 445 if (ra_used(ir)) {
436 lua_assert(!irt_ispri(ir->t)); 446 lj_assertA(!irt_ispri(ir->t), "PRI dest");
437 if (!LJ_SOFTFP && irt_isfp(ir->t)) { 447 if (!LJ_SOFTFP && irt_isfp(ir->t)) {
438 if (LJ_ABI_SOFTFP || (ci->flags & (CCI_CASTU64|CCI_VARARG))) { 448 if (LJ_ABI_SOFTFP || (ci->flags & (CCI_CASTU64|CCI_VARARG))) {
439 Reg dest = (ra_dest(as, ir, RSET_FPR) & 15); 449 Reg dest = (ra_dest(as, ir, RSET_FPR) & 15);
@@ -453,15 +463,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
453 UNUSED(ci); 463 UNUSED(ci);
454} 464}
455 465
456static void asm_call(ASMState *as, IRIns *ir)
457{
458 IRRef args[CCI_NARGS_MAX];
459 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
460 asm_collectargs(as, ir, ci, args);
461 asm_setupresult(as, ir, ci);
462 asm_gencall(as, ci, args);
463}
464
465static void asm_callx(ASMState *as, IRIns *ir) 466static void asm_callx(ASMState *as, IRIns *ir)
466{ 467{
467 IRRef args[CCI_NARGS_MAX*2]; 468 IRRef args[CCI_NARGS_MAX*2];
@@ -490,7 +491,7 @@ static void asm_retf(ASMState *as, IRIns *ir)
490{ 491{
491 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR); 492 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
492 void *pc = ir_kptr(IR(ir->op2)); 493 void *pc = ir_kptr(IR(ir->op2));
493 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1)); 494 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
494 as->topslot -= (BCReg)delta; 495 as->topslot -= (BCReg)delta;
495 if ((int32_t)as->topslot < 0) as->topslot = 0; 496 if ((int32_t)as->topslot < 0) as->topslot = 0;
496 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */ 497 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
@@ -504,6 +505,30 @@ static void asm_retf(ASMState *as, IRIns *ir)
504 emit_lso(as, ARMI_LDR, RID_TMP, base, -4); 505 emit_lso(as, ARMI_LDR, RID_TMP, base, -4);
505} 506}
506 507
508/* -- Buffer operations --------------------------------------------------- */
509
510#if LJ_HASBUFFER
511static void asm_bufhdr_write(ASMState *as, Reg sb)
512{
513 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
514 IRIns irgc;
515 int32_t addr = i32ptr((void *)&J2G(as->J)->cur_L);
516 irgc.ot = IRT(0, IRT_PGC); /* GC type. */
517 emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
518 if ((as->flags & JIT_F_ARMV6T2)) {
519 emit_dnm(as, ARMI_BFI, RID_TMP, lj_fls(SBUF_MASK_FLAG), tmp);
520 } else {
521 emit_dnm(as, ARMI_ORR, RID_TMP, RID_TMP, tmp);
522 emit_dn(as, ARMI_AND|ARMI_K12|SBUF_MASK_FLAG, tmp, tmp);
523 }
524 emit_lso(as, ARMI_LDR, RID_TMP,
525 ra_allock(as, (addr & ~4095),
526 rset_exclude(rset_exclude(RSET_GPR, sb), tmp)),
527 (addr & 4095));
528 emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
529}
530#endif
531
507/* -- Type conversions ---------------------------------------------------- */ 532/* -- Type conversions ---------------------------------------------------- */
508 533
509#if !LJ_SOFTFP 534#if !LJ_SOFTFP
@@ -539,13 +564,17 @@ static void asm_conv(ASMState *as, IRIns *ir)
539#endif 564#endif
540 IRRef lref = ir->op1; 565 IRRef lref = ir->op1;
541 /* 64 bit integer conversions are handled by SPLIT. */ 566 /* 64 bit integer conversions are handled by SPLIT. */
542 lua_assert(!irt_isint64(ir->t) && !(st == IRT_I64 || st == IRT_U64)); 567 lj_assertA(!irt_isint64(ir->t) && !(st == IRT_I64 || st == IRT_U64),
568 "IR %04d has unsplit 64 bit type",
569 (int)(ir - as->ir) - REF_BIAS);
543#if LJ_SOFTFP 570#if LJ_SOFTFP
544 /* FP conversions are handled by SPLIT. */ 571 /* FP conversions are handled by SPLIT. */
545 lua_assert(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT)); 572 lj_assertA(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT),
573 "IR %04d has FP type",
574 (int)(ir - as->ir) - REF_BIAS);
546 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */ 575 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
547#else 576#else
548 lua_assert(irt_type(ir->t) != st); 577 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
549 if (irt_isfp(ir->t)) { 578 if (irt_isfp(ir->t)) {
550 Reg dest = ra_dest(as, ir, RSET_FPR); 579 Reg dest = ra_dest(as, ir, RSET_FPR);
551 if (stfp) { /* FP to FP conversion. */ 580 if (stfp) { /* FP to FP conversion. */
@@ -562,7 +591,8 @@ static void asm_conv(ASMState *as, IRIns *ir)
562 } else if (stfp) { /* FP to integer conversion. */ 591 } else if (stfp) { /* FP to integer conversion. */
563 if (irt_isguard(ir->t)) { 592 if (irt_isguard(ir->t)) {
564 /* Checked conversions are only supported from number to int. */ 593 /* Checked conversions are only supported from number to int. */
565 lua_assert(irt_isint(ir->t) && st == IRT_NUM); 594 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
595 "bad type for checked CONV");
566 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); 596 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
567 } else { 597 } else {
568 Reg left = ra_alloc1(as, lref, RSET_FPR); 598 Reg left = ra_alloc1(as, lref, RSET_FPR);
@@ -581,7 +611,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
581 Reg dest = ra_dest(as, ir, RSET_GPR); 611 Reg dest = ra_dest(as, ir, RSET_GPR);
582 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */ 612 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
583 Reg left = ra_alloc1(as, lref, RSET_GPR); 613 Reg left = ra_alloc1(as, lref, RSET_GPR);
584 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); 614 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
585 if ((as->flags & JIT_F_ARMV6)) { 615 if ((as->flags & JIT_F_ARMV6)) {
586 ARMIns ai = st == IRT_I8 ? ARMI_SXTB : 616 ARMIns ai = st == IRT_I8 ? ARMI_SXTB :
587 st == IRT_U8 ? ARMI_UXTB : 617 st == IRT_U8 ? ARMI_UXTB :
@@ -601,31 +631,6 @@ static void asm_conv(ASMState *as, IRIns *ir)
601 } 631 }
602} 632}
603 633
604#if !LJ_SOFTFP && LJ_HASFFI
605static void asm_conv64(ASMState *as, IRIns *ir)
606{
607 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
608 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
609 IRCallID id;
610 CCallInfo ci;
611 IRRef args[2];
612 args[0] = (ir-1)->op1;
613 args[1] = ir->op1;
614 if (st == IRT_NUM || st == IRT_FLOAT) {
615 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
616 ir--;
617 } else {
618 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
619 }
620 ci = lj_ir_callinfo[id];
621#if !LJ_ABI_SOFTFP
622 ci.flags |= CCI_VARARG; /* These calls don't use the hard-float ABI! */
623#endif
624 asm_setupresult(as, ir, &ci);
625 asm_gencall(as, &ci, args);
626}
627#endif
628
629static void asm_strto(ASMState *as, IRIns *ir) 634static void asm_strto(ASMState *as, IRIns *ir)
630{ 635{
631 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num]; 636 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
@@ -689,60 +694,61 @@ static void asm_strto(ASMState *as, IRIns *ir)
689 emit_opk(as, ARMI_ADD, tmp, RID_SP, ofs, RSET_GPR); 694 emit_opk(as, ARMI_ADD, tmp, RID_SP, ofs, RSET_GPR);
690} 695}
691 696
697/* -- Memory references --------------------------------------------------- */
698
692/* Get pointer to TValue. */ 699/* Get pointer to TValue. */
693static void asm_tvptr(ASMState *as, Reg dest, IRRef ref) 700static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
694{ 701{
695 IRIns *ir = IR(ref); 702 if ((mode & IRTMPREF_IN1)) {
696 if (irt_isnum(ir->t)) { 703 IRIns *ir = IR(ref);
697 if (irref_isk(ref)) { 704 if (irt_isnum(ir->t)) {
698 /* Use the number constant itself as a TValue. */ 705 if ((mode & IRTMPREF_OUT1)) {
699 ra_allockreg(as, i32ptr(ir_knum(ir)), dest); 706#if LJ_SOFTFP
700 } else { 707 lj_assertA(irref_isk(ref), "unsplit FP op");
708 emit_dm(as, ARMI_MOV, dest, RID_SP);
709 emit_lso(as, ARMI_STR,
710 ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, RSET_GPR),
711 RID_SP, 0);
712 emit_lso(as, ARMI_STR,
713 ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, RSET_GPR),
714 RID_SP, 4);
715#else
716 Reg src = ra_alloc1(as, ref, RSET_FPR);
717 emit_dm(as, ARMI_MOV, dest, RID_SP);
718 emit_vlso(as, ARMI_VSTR_D, src, RID_SP, 0);
719#endif
720 } else if (irref_isk(ref)) {
721 /* Use the number constant itself as a TValue. */
722 ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
723 } else {
701#if LJ_SOFTFP 724#if LJ_SOFTFP
702 lua_assert(0); 725 lj_assertA(0, "unsplit FP op");
703#else 726#else
704 /* Otherwise force a spill and use the spill slot. */ 727 /* Otherwise force a spill and use the spill slot. */
705 emit_opk(as, ARMI_ADD, dest, RID_SP, ra_spill(as, ir), RSET_GPR); 728 emit_opk(as, ARMI_ADD, dest, RID_SP, ra_spill(as, ir), RSET_GPR);
706#endif 729#endif
730 }
731 } else {
732 /* Otherwise use [sp] and [sp+4] to hold the TValue.
733 ** This assumes the following call has max. 4 args.
734 */
735 Reg type;
736 emit_dm(as, ARMI_MOV, dest, RID_SP);
737 if (!irt_ispri(ir->t)) {
738 Reg src = ra_alloc1(as, ref, RSET_GPR);
739 emit_lso(as, ARMI_STR, src, RID_SP, 0);
740 }
741 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t))
742 type = ra_alloc1(as, ref+1, RSET_GPR);
743 else
744 type = ra_allock(as, irt_toitype(ir->t), RSET_GPR);
745 emit_lso(as, ARMI_STR, type, RID_SP, 4);
707 } 746 }
708 } else { 747 } else {
709 /* Otherwise use [sp] and [sp+4] to hold the TValue. */
710 RegSet allow = rset_exclude(RSET_GPR, dest);
711 Reg type;
712 emit_dm(as, ARMI_MOV, dest, RID_SP); 748 emit_dm(as, ARMI_MOV, dest, RID_SP);
713 if (!irt_ispri(ir->t)) {
714 Reg src = ra_alloc1(as, ref, allow);
715 emit_lso(as, ARMI_STR, src, RID_SP, 0);
716 }
717 if ((ir+1)->o == IR_HIOP)
718 type = ra_alloc1(as, ref+1, allow);
719 else
720 type = ra_allock(as, irt_toitype(ir->t), allow);
721 emit_lso(as, ARMI_STR, type, RID_SP, 4);
722 } 749 }
723} 750}
724 751
725static void asm_tostr(ASMState *as, IRIns *ir)
726{
727 IRRef args[2];
728 args[0] = ASMREF_L;
729 as->gcsteps++;
730 if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
731 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
732 args[1] = ASMREF_TMP1; /* const lua_Number * */
733 asm_setupresult(as, ir, ci); /* GCstr * */
734 asm_gencall(as, ci, args);
735 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
736 } else {
737 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
738 args[1] = ir->op1; /* int32_t k */
739 asm_setupresult(as, ir, ci); /* GCstr * */
740 asm_gencall(as, ci, args);
741 }
742}
743
744/* -- Memory references --------------------------------------------------- */
745
746static void asm_aref(ASMState *as, IRIns *ir) 752static void asm_aref(ASMState *as, IRIns *ir)
747{ 753{
748 Reg dest = ra_dest(as, ir, RSET_GPR); 754 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -864,16 +870,16 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
864 *l_loop = ARMF_CC(ARMI_B, CC_NE) | ((as->mcp-l_loop-2) & 0x00ffffffu); 870 *l_loop = ARMF_CC(ARMI_B, CC_NE) | ((as->mcp-l_loop-2) & 0x00ffffffu);
865 871
866 /* Load main position relative to tab->node into dest. */ 872 /* Load main position relative to tab->node into dest. */
867 khash = irref_isk(refkey) ? ir_khash(irkey) : 1; 873 khash = irref_isk(refkey) ? ir_khash(as, irkey) : 1;
868 if (khash == 0) { 874 if (khash == 0) {
869 emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); 875 emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node));
870 } else { 876 } else {
871 emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, dest, tmp); 877 emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, dest, tmp);
872 emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 1), tmp, tmp, tmp); 878 emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 1), tmp, tmp, tmp);
873 if (irt_isstr(kt)) { /* Fetch of str->hash is cheaper than ra_allock. */ 879 if (irt_isstr(kt)) { /* Fetch of str->sid is cheaper than ra_allock. */
874 emit_dnm(as, ARMI_AND, tmp, tmp+1, RID_TMP); 880 emit_dnm(as, ARMI_AND, tmp, tmp+1, RID_TMP);
875 emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); 881 emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node));
876 emit_lso(as, ARMI_LDR, tmp+1, key, (int32_t)offsetof(GCstr, hash)); 882 emit_lso(as, ARMI_LDR, tmp+1, key, (int32_t)offsetof(GCstr, sid));
877 emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); 883 emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask));
878 } else if (irref_isk(refkey)) { 884 } else if (irref_isk(refkey)) {
879 emit_opk(as, ARMI_AND, tmp, RID_TMP, (int32_t)khash, 885 emit_opk(as, ARMI_AND, tmp, RID_TMP, (int32_t)khash,
@@ -920,7 +926,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
920 Reg node = ra_alloc1(as, ir->op1, RSET_GPR); 926 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
921 Reg key = RID_NONE, type = RID_TMP, idx = node; 927 Reg key = RID_NONE, type = RID_TMP, idx = node;
922 RegSet allow = rset_exclude(RSET_GPR, node); 928 RegSet allow = rset_exclude(RSET_GPR, node);
923 lua_assert(ofs % sizeof(Node) == 0); 929 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
924 if (ofs > 4095) { 930 if (ofs > 4095) {
925 idx = dest; 931 idx = dest;
926 rset_clear(allow, dest); 932 rset_clear(allow, dest);
@@ -960,20 +966,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
960 emit_opk(as, ARMI_ADD, dest, node, ofs, RSET_GPR); 966 emit_opk(as, ARMI_ADD, dest, node, ofs, RSET_GPR);
961} 967}
962 968
963static void asm_newref(ASMState *as, IRIns *ir)
964{
965 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
966 IRRef args[3];
967 if (ir->r == RID_SINK)
968 return;
969 args[0] = ASMREF_L; /* lua_State *L */
970 args[1] = ir->op1; /* GCtab *t */
971 args[2] = ASMREF_TMP1; /* cTValue *key */
972 asm_setupresult(as, ir, ci); /* TValue * */
973 asm_gencall(as, ci, args);
974 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
975}
976
977static void asm_uref(ASMState *as, IRIns *ir) 969static void asm_uref(ASMState *as, IRIns *ir)
978{ 970{
979 Reg dest = ra_dest(as, ir, RSET_GPR); 971 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1001,7 +993,7 @@ static void asm_uref(ASMState *as, IRIns *ir)
1001static void asm_fref(ASMState *as, IRIns *ir) 993static void asm_fref(ASMState *as, IRIns *ir)
1002{ 994{
1003 UNUSED(as); UNUSED(ir); 995 UNUSED(as); UNUSED(ir);
1004 lua_assert(!ra_used(ir)); 996 lj_assertA(!ra_used(ir), "unfused FREF");
1005} 997}
1006 998
1007static void asm_strref(ASMState *as, IRIns *ir) 999static void asm_strref(ASMState *as, IRIns *ir)
@@ -1038,25 +1030,27 @@ static void asm_strref(ASMState *as, IRIns *ir)
1038 1030
1039/* -- Loads and stores ---------------------------------------------------- */ 1031/* -- Loads and stores ---------------------------------------------------- */
1040 1032
1041static ARMIns asm_fxloadins(IRIns *ir) 1033static ARMIns asm_fxloadins(ASMState *as, IRIns *ir)
1042{ 1034{
1035 UNUSED(as);
1043 switch (irt_type(ir->t)) { 1036 switch (irt_type(ir->t)) {
1044 case IRT_I8: return ARMI_LDRSB; 1037 case IRT_I8: return ARMI_LDRSB;
1045 case IRT_U8: return ARMI_LDRB; 1038 case IRT_U8: return ARMI_LDRB;
1046 case IRT_I16: return ARMI_LDRSH; 1039 case IRT_I16: return ARMI_LDRSH;
1047 case IRT_U16: return ARMI_LDRH; 1040 case IRT_U16: return ARMI_LDRH;
1048 case IRT_NUM: lua_assert(!LJ_SOFTFP); return ARMI_VLDR_D; 1041 case IRT_NUM: lj_assertA(!LJ_SOFTFP, "unsplit FP op"); return ARMI_VLDR_D;
1049 case IRT_FLOAT: if (!LJ_SOFTFP) return ARMI_VLDR_S; /* fallthrough */ 1042 case IRT_FLOAT: if (!LJ_SOFTFP) return ARMI_VLDR_S; /* fallthrough */
1050 default: return ARMI_LDR; 1043 default: return ARMI_LDR;
1051 } 1044 }
1052} 1045}
1053 1046
1054static ARMIns asm_fxstoreins(IRIns *ir) 1047static ARMIns asm_fxstoreins(ASMState *as, IRIns *ir)
1055{ 1048{
1049 UNUSED(as);
1056 switch (irt_type(ir->t)) { 1050 switch (irt_type(ir->t)) {
1057 case IRT_I8: case IRT_U8: return ARMI_STRB; 1051 case IRT_I8: case IRT_U8: return ARMI_STRB;
1058 case IRT_I16: case IRT_U16: return ARMI_STRH; 1052 case IRT_I16: case IRT_U16: return ARMI_STRH;
1059 case IRT_NUM: lua_assert(!LJ_SOFTFP); return ARMI_VSTR_D; 1053 case IRT_NUM: lj_assertA(!LJ_SOFTFP, "unsplit FP op"); return ARMI_VSTR_D;
1060 case IRT_FLOAT: if (!LJ_SOFTFP) return ARMI_VSTR_S; /* fallthrough */ 1054 case IRT_FLOAT: if (!LJ_SOFTFP) return ARMI_VSTR_S; /* fallthrough */
1061 default: return ARMI_STR; 1055 default: return ARMI_STR;
1062 } 1056 }
@@ -1065,17 +1059,23 @@ static ARMIns asm_fxstoreins(IRIns *ir)
1065static void asm_fload(ASMState *as, IRIns *ir) 1059static void asm_fload(ASMState *as, IRIns *ir)
1066{ 1060{
1067 Reg dest = ra_dest(as, ir, RSET_GPR); 1061 Reg dest = ra_dest(as, ir, RSET_GPR);
1068 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR); 1062 ARMIns ai = asm_fxloadins(as, ir);
1069 ARMIns ai = asm_fxloadins(ir); 1063 Reg idx;
1070 int32_t ofs; 1064 int32_t ofs;
1071 if (ir->op2 == IRFL_TAB_ARRAY) { 1065 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
1072 ofs = asm_fuseabase(as, ir->op1); 1066 idx = ra_allock(as, (int32_t)(ir->op2<<2) + (int32_t)J2GG(as->J), RSET_GPR);
1073 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */ 1067 ofs = 0;
1074 emit_dn(as, ARMI_ADD|ARMI_K12|ofs, dest, idx); 1068 } else {
1075 return; 1069 idx = ra_alloc1(as, ir->op1, RSET_GPR);
1070 if (ir->op2 == IRFL_TAB_ARRAY) {
1071 ofs = asm_fuseabase(as, ir->op1);
1072 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
1073 emit_dn(as, ARMI_ADD|ARMI_K12|ofs, dest, idx);
1074 return;
1075 }
1076 } 1076 }
1077 ofs = field_ofs[ir->op2];
1077 } 1078 }
1078 ofs = field_ofs[ir->op2];
1079 if ((ai & 0x04000000)) 1079 if ((ai & 0x04000000))
1080 emit_lso(as, ai, dest, idx, ofs); 1080 emit_lso(as, ai, dest, idx, ofs);
1081 else 1081 else
@@ -1089,7 +1089,7 @@ static void asm_fstore(ASMState *as, IRIns *ir)
1089 IRIns *irf = IR(ir->op1); 1089 IRIns *irf = IR(ir->op1);
1090 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src)); 1090 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
1091 int32_t ofs = field_ofs[irf->op2]; 1091 int32_t ofs = field_ofs[irf->op2];
1092 ARMIns ai = asm_fxstoreins(ir); 1092 ARMIns ai = asm_fxstoreins(as, ir);
1093 if ((ai & 0x04000000)) 1093 if ((ai & 0x04000000))
1094 emit_lso(as, ai, src, idx, ofs); 1094 emit_lso(as, ai, src, idx, ofs);
1095 else 1095 else
@@ -1101,20 +1101,22 @@ static void asm_xload(ASMState *as, IRIns *ir)
1101{ 1101{
1102 Reg dest = ra_dest(as, ir, 1102 Reg dest = ra_dest(as, ir,
1103 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR); 1103 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1104 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED)); 1104 lj_assertA(!(ir->op2 & IRXLOAD_UNALIGNED), "unaligned XLOAD");
1105 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0); 1105 asm_fusexref(as, asm_fxloadins(as, ir), dest, ir->op1, RSET_GPR, 0);
1106} 1106}
1107 1107
1108static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs) 1108static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
1109{ 1109{
1110 if (ir->r != RID_SINK) { 1110 if (ir->r != RID_SINK) {
1111 Reg src = ra_alloc1(as, ir->op2, 1111 Reg src = ra_alloc1(as, ir->op2,
1112 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR); 1112 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1113 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1, 1113 asm_fusexref(as, asm_fxstoreins(as, ir), src, ir->op1,
1114 rset_exclude(RSET_GPR, src), ofs); 1114 rset_exclude(RSET_GPR, src), ofs);
1115 } 1115 }
1116} 1116}
1117 1117
1118#define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
1119
1118static void asm_ahuvload(ASMState *as, IRIns *ir) 1120static void asm_ahuvload(ASMState *as, IRIns *ir)
1119{ 1121{
1120 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP); 1122 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
@@ -1127,13 +1129,15 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
1127 rset_clear(allow, type); 1129 rset_clear(allow, type);
1128 } 1130 }
1129 if (ra_used(ir)) { 1131 if (ra_used(ir)) {
1130 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) || 1132 lj_assertA((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
1131 irt_isint(ir->t) || irt_isaddr(ir->t)); 1133 irt_isint(ir->t) || irt_isaddr(ir->t),
1134 "bad load type %d", irt_type(ir->t));
1132 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow); 1135 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
1133 rset_clear(allow, dest); 1136 rset_clear(allow, dest);
1134 } 1137 }
1135 idx = asm_fuseahuref(as, ir->op1, &ofs, allow, 1138 idx = asm_fuseahuref(as, ir->op1, &ofs, allow,
1136 (!LJ_SOFTFP && t == IRT_NUM) ? 1024 : 4096); 1139 (!LJ_SOFTFP && t == IRT_NUM) ? 1024 : 4096);
1140 if (ir->o == IR_VLOAD) ofs += 8 * ir->op2;
1137 if (!hiop || type == RID_NONE) { 1141 if (!hiop || type == RID_NONE) {
1138 rset_clear(allow, idx); 1142 rset_clear(allow, idx);
1139 if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 && 1143 if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 &&
@@ -1194,10 +1198,13 @@ static void asm_sload(ASMState *as, IRIns *ir)
1194 IRType t = hiop ? IRT_NUM : irt_type(ir->t); 1198 IRType t = hiop ? IRT_NUM : irt_type(ir->t);
1195 Reg dest = RID_NONE, type = RID_NONE, base; 1199 Reg dest = RID_NONE, type = RID_NONE, base;
1196 RegSet allow = RSET_GPR; 1200 RegSet allow = RSET_GPR;
1197 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ 1201 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1198 lua_assert(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK)); 1202 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1203 lj_assertA(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1204 "inconsistent SLOAD variant");
1199#if LJ_SOFTFP 1205#if LJ_SOFTFP
1200 lua_assert(!(ir->op2 & IRSLOAD_CONVERT)); /* Handled by LJ_SOFTFP SPLIT. */ 1206 lj_assertA(!(ir->op2 & IRSLOAD_CONVERT),
1207 "unsplit SLOAD convert"); /* Handled by LJ_SOFTFP SPLIT. */
1201 if (hiop && ra_used(ir+1)) { 1208 if (hiop && ra_used(ir+1)) {
1202 type = ra_dest(as, ir+1, allow); 1209 type = ra_dest(as, ir+1, allow);
1203 rset_clear(allow, type); 1210 rset_clear(allow, type);
@@ -1213,8 +1220,9 @@ static void asm_sload(ASMState *as, IRIns *ir)
1213 Reg tmp = RID_NONE; 1220 Reg tmp = RID_NONE;
1214 if ((ir->op2 & IRSLOAD_CONVERT)) 1221 if ((ir->op2 & IRSLOAD_CONVERT))
1215 tmp = ra_scratch(as, t == IRT_INT ? RSET_FPR : RSET_GPR); 1222 tmp = ra_scratch(as, t == IRT_INT ? RSET_FPR : RSET_GPR);
1216 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) || 1223 lj_assertA((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
1217 irt_isint(ir->t) || irt_isaddr(ir->t)); 1224 irt_isint(ir->t) || irt_isaddr(ir->t),
1225 "bad SLOAD type %d", irt_type(ir->t));
1218 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow); 1226 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
1219 rset_clear(allow, dest); 1227 rset_clear(allow, dest);
1220 base = ra_alloc1(as, REF_BASE, allow); 1228 base = ra_alloc1(as, REF_BASE, allow);
@@ -1246,7 +1254,12 @@ dotypecheck:
1246 } 1254 }
1247 } 1255 }
1248 asm_guardcc(as, t == IRT_NUM ? CC_HS : CC_NE); 1256 asm_guardcc(as, t == IRT_NUM ? CC_HS : CC_NE);
1249 emit_n(as, ARMI_CMN|ARMI_K12|-irt_toitype_(t), type); 1257 if ((ir->op2 & IRSLOAD_KEYINDEX)) {
1258 emit_n(as, ARMI_CMN|ARMI_K12|1, type);
1259 emit_dn(as, ARMI_EOR^emit_isk12(ARMI_EOR, ~LJ_KEYINDEX), type, type);
1260 } else {
1261 emit_n(as, ARMI_CMN|ARMI_K12|-irt_toitype_(t), type);
1262 }
1250 } 1263 }
1251 if (ra_hasreg(dest)) { 1264 if (ra_hasreg(dest)) {
1252#if !LJ_SOFTFP 1265#if !LJ_SOFTFP
@@ -1272,19 +1285,17 @@ dotypecheck:
1272static void asm_cnew(ASMState *as, IRIns *ir) 1285static void asm_cnew(ASMState *as, IRIns *ir)
1273{ 1286{
1274 CTState *cts = ctype_ctsG(J2G(as->J)); 1287 CTState *cts = ctype_ctsG(J2G(as->J));
1275 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i; 1288 CTypeID id = (CTypeID)IR(ir->op1)->i;
1276 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ? 1289 CTSize sz;
1277 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i; 1290 CTInfo info = lj_ctype_info(cts, id, &sz);
1278 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; 1291 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1279 IRRef args[2]; 1292 IRRef args[4];
1280 RegSet allow = (RSET_GPR & ~RSET_SCRATCH); 1293 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1281 RegSet drop = RSET_SCRATCH; 1294 RegSet drop = RSET_SCRATCH;
1282 lua_assert(sz != CTSIZE_INVALID); 1295 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1296 "bad CNEW/CNEWI operands");
1283 1297
1284 args[0] = ASMREF_L; /* lua_State *L */
1285 args[1] = ASMREF_TMP1; /* MSize size */
1286 as->gcsteps++; 1298 as->gcsteps++;
1287
1288 if (ra_hasreg(ir->r)) 1299 if (ra_hasreg(ir->r))
1289 rset_clear(drop, ir->r); /* Dest reg handled below. */ 1300 rset_clear(drop, ir->r); /* Dest reg handled below. */
1290 ra_evictset(as, drop); 1301 ra_evictset(as, drop);
@@ -1294,10 +1305,10 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1294 /* Initialize immutable cdata object. */ 1305 /* Initialize immutable cdata object. */
1295 if (ir->o == IR_CNEWI) { 1306 if (ir->o == IR_CNEWI) {
1296 int32_t ofs = sizeof(GCcdata); 1307 int32_t ofs = sizeof(GCcdata);
1297 lua_assert(sz == 4 || sz == 8); 1308 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1298 if (sz == 8) { 1309 if (sz == 8) {
1299 ofs += 4; ir++; 1310 ofs += 4; ir++;
1300 lua_assert(ir->o == IR_HIOP); 1311 lj_assertA(ir->o == IR_HIOP, "expected HIOP for CNEWI");
1301 } 1312 }
1302 for (;;) { 1313 for (;;) {
1303 Reg r = ra_alloc1(as, ir->op2, allow); 1314 Reg r = ra_alloc1(as, ir->op2, allow);
@@ -1306,22 +1317,32 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1306 if (ofs == sizeof(GCcdata)) break; 1317 if (ofs == sizeof(GCcdata)) break;
1307 ofs -= 4; ir--; 1318 ofs -= 4; ir--;
1308 } 1319 }
1320 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1321 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1322 args[0] = ASMREF_L; /* lua_State *L */
1323 args[1] = ir->op1; /* CTypeID id */
1324 args[2] = ir->op2; /* CTSize sz */
1325 args[3] = ASMREF_TMP1; /* CTSize align */
1326 asm_gencall(as, ci, args);
1327 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1328 return;
1309 } 1329 }
1330
1310 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ 1331 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1311 { 1332 {
1312 uint32_t k = emit_isk12(ARMI_MOV, ctypeid); 1333 uint32_t k = emit_isk12(ARMI_MOV, id);
1313 Reg r = k ? RID_R1 : ra_allock(as, ctypeid, allow); 1334 Reg r = k ? RID_R1 : ra_allock(as, id, allow);
1314 emit_lso(as, ARMI_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct)); 1335 emit_lso(as, ARMI_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct));
1315 emit_lsox(as, ARMI_STRH, r, RID_RET, offsetof(GCcdata, ctypeid)); 1336 emit_lsox(as, ARMI_STRH, r, RID_RET, offsetof(GCcdata, ctypeid));
1316 emit_d(as, ARMI_MOV|ARMI_K12|~LJ_TCDATA, RID_TMP); 1337 emit_d(as, ARMI_MOV|ARMI_K12|~LJ_TCDATA, RID_TMP);
1317 if (k) emit_d(as, ARMI_MOV^k, RID_R1); 1338 if (k) emit_d(as, ARMI_MOV^k, RID_R1);
1318 } 1339 }
1340 args[0] = ASMREF_L; /* lua_State *L */
1341 args[1] = ASMREF_TMP1; /* MSize size */
1319 asm_gencall(as, ci, args); 1342 asm_gencall(as, ci, args);
1320 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)), 1343 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1321 ra_releasetmp(as, ASMREF_TMP1)); 1344 ra_releasetmp(as, ASMREF_TMP1));
1322} 1345}
1323#else
1324#define asm_cnew(as, ir) ((void)0)
1325#endif 1346#endif
1326 1347
1327/* -- Write barriers ------------------------------------------------------ */ 1348/* -- Write barriers ------------------------------------------------------ */
@@ -1353,7 +1374,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1353 MCLabel l_end; 1374 MCLabel l_end;
1354 Reg obj, val, tmp; 1375 Reg obj, val, tmp;
1355 /* No need for other object barriers (yet). */ 1376 /* No need for other object barriers (yet). */
1356 lua_assert(IR(ir->op1)->o == IR_UREFC); 1377 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1357 ra_evictset(as, RSET_SCRATCH); 1378 ra_evictset(as, RSET_SCRATCH);
1358 l_end = emit_label(as); 1379 l_end = emit_label(as);
1359 args[0] = ASMREF_TMP1; /* global_State *g */ 1380 args[0] = ASMREF_TMP1; /* global_State *g */
@@ -1392,23 +1413,36 @@ static void asm_fpunary(ASMState *as, IRIns *ir, ARMIns ai)
1392 emit_dm(as, ai, (dest & 15), (left & 15)); 1413 emit_dm(as, ai, (dest & 15), (left & 15));
1393} 1414}
1394 1415
1395static int asm_fpjoin_pow(ASMState *as, IRIns *ir) 1416static void asm_callround(ASMState *as, IRIns *ir, int id)
1396{ 1417{
1397 IRIns *irp = IR(ir->op1); 1418 /* The modified regs must match with the *.dasc implementation. */
1398 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) { 1419 RegSet drop = RID2RSET(RID_R0)|RID2RSET(RID_R1)|RID2RSET(RID_R2)|
1399 IRIns *irpp = IR(irp->op1); 1420 RID2RSET(RID_R3)|RID2RSET(RID_R12);
1400 if (irpp == ir-2 && irpp->o == IR_FPMATH && 1421 RegSet of;
1401 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) { 1422 Reg dest, src;
1402 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow]; 1423 ra_evictset(as, drop);
1403 IRRef args[2]; 1424 dest = ra_dest(as, ir, RSET_FPR);
1404 args[0] = irpp->op1; 1425 emit_dnm(as, ARMI_VMOV_D_RR, RID_RETLO, RID_RETHI, (dest & 15));
1405 args[1] = irp->op2; 1426 emit_call(as, id == IRFPM_FLOOR ? (void *)lj_vm_floor_sf :
1406 asm_setupresult(as, ir, ci); 1427 id == IRFPM_CEIL ? (void *)lj_vm_ceil_sf :
1407 asm_gencall(as, ci, args); 1428 (void *)lj_vm_trunc_sf);
1408 return 1; 1429 /* Workaround to protect argument GPRs from being used for remat. */
1409 } 1430 of = as->freeset;
1410 } 1431 as->freeset &= ~RSET_RANGE(RID_R0, RID_R1+1);
1411 return 0; 1432 as->cost[RID_R0] = as->cost[RID_R1] = REGCOST(~0u, ASMREF_L);
1433 src = ra_alloc1(as, ir->op1, RSET_FPR); /* May alloc GPR to remat FPR. */
1434 as->freeset |= (of & RSET_RANGE(RID_R0, RID_R1+1));
1435 emit_dnm(as, ARMI_VMOV_RR_D, RID_R0, RID_R1, (src & 15));
1436}
1437
1438static void asm_fpmath(ASMState *as, IRIns *ir)
1439{
1440 if (ir->op2 <= IRFPM_TRUNC)
1441 asm_callround(as, ir, ir->op2);
1442 else if (ir->op2 == IRFPM_SQRT)
1443 asm_fpunary(as, ir, ARMI_VSQRT_D);
1444 else
1445 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1412} 1446}
1413#endif 1447#endif
1414 1448
@@ -1474,19 +1508,6 @@ static void asm_intop_s(ASMState *as, IRIns *ir, ARMIns ai)
1474 asm_intop(as, ir, asm_drop_cmp0(as, ai)); 1508 asm_intop(as, ir, asm_drop_cmp0(as, ai));
1475} 1509}
1476 1510
1477static void asm_bitop(ASMState *as, IRIns *ir, ARMIns ai)
1478{
1479 ai = asm_drop_cmp0(as, ai);
1480 if (ir->op2 == 0) {
1481 Reg dest = ra_dest(as, ir, RSET_GPR);
1482 uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
1483 emit_d(as, ai^m, dest);
1484 } else {
1485 /* NYI: Turn BAND !k12 into uxtb, uxth or bfc or shl+shr. */
1486 asm_intop(as, ir, ai);
1487 }
1488}
1489
1490static void asm_intneg(ASMState *as, IRIns *ir, ARMIns ai) 1511static void asm_intneg(ASMState *as, IRIns *ir, ARMIns ai)
1491{ 1512{
1492 Reg dest = ra_dest(as, ir, RSET_GPR); 1513 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1552,6 +1573,15 @@ static void asm_mul(ASMState *as, IRIns *ir)
1552 asm_intmul(as, ir); 1573 asm_intmul(as, ir);
1553} 1574}
1554 1575
1576#define asm_addov(as, ir) asm_add(as, ir)
1577#define asm_subov(as, ir) asm_sub(as, ir)
1578#define asm_mulov(as, ir) asm_mul(as, ir)
1579
1580#if !LJ_SOFTFP
1581#define asm_fpdiv(as, ir) asm_fparith(as, ir, ARMI_VDIV_D)
1582#define asm_abs(as, ir) asm_fpunary(as, ir, ARMI_VABS_D)
1583#endif
1584
1555static void asm_neg(ASMState *as, IRIns *ir) 1585static void asm_neg(ASMState *as, IRIns *ir)
1556{ 1586{
1557#if !LJ_SOFTFP 1587#if !LJ_SOFTFP
@@ -1563,41 +1593,22 @@ static void asm_neg(ASMState *as, IRIns *ir)
1563 asm_intneg(as, ir, ARMI_RSB); 1593 asm_intneg(as, ir, ARMI_RSB);
1564} 1594}
1565 1595
1566static void asm_callid(ASMState *as, IRIns *ir, IRCallID id) 1596static void asm_bitop(ASMState *as, IRIns *ir, ARMIns ai)
1567{ 1597{
1568 const CCallInfo *ci = &lj_ir_callinfo[id]; 1598 ai = asm_drop_cmp0(as, ai);
1569 IRRef args[2]; 1599 if (ir->op2 == 0) {
1570 args[0] = ir->op1; 1600 Reg dest = ra_dest(as, ir, RSET_GPR);
1571 args[1] = ir->op2; 1601 uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
1572 asm_setupresult(as, ir, ci); 1602 emit_d(as, ai^m, dest);
1573 asm_gencall(as, ci, args); 1603 } else {
1604 /* NYI: Turn BAND !k12 into uxtb, uxth or bfc or shl+shr. */
1605 asm_intop(as, ir, ai);
1606 }
1574} 1607}
1575 1608
1576#if !LJ_SOFTFP 1609#define asm_bnot(as, ir) asm_bitop(as, ir, ARMI_MVN)
1577static void asm_callround(ASMState *as, IRIns *ir, int id)
1578{
1579 /* The modified regs must match with the *.dasc implementation. */
1580 RegSet drop = RID2RSET(RID_R0)|RID2RSET(RID_R1)|RID2RSET(RID_R2)|
1581 RID2RSET(RID_R3)|RID2RSET(RID_R12);
1582 RegSet of;
1583 Reg dest, src;
1584 ra_evictset(as, drop);
1585 dest = ra_dest(as, ir, RSET_FPR);
1586 emit_dnm(as, ARMI_VMOV_D_RR, RID_RETLO, RID_RETHI, (dest & 15));
1587 emit_call(as, id == IRFPM_FLOOR ? (void *)lj_vm_floor_sf :
1588 id == IRFPM_CEIL ? (void *)lj_vm_ceil_sf :
1589 (void *)lj_vm_trunc_sf);
1590 /* Workaround to protect argument GPRs from being used for remat. */
1591 of = as->freeset;
1592 as->freeset &= ~RSET_RANGE(RID_R0, RID_R1+1);
1593 as->cost[RID_R0] = as->cost[RID_R1] = REGCOST(~0u, ASMREF_L);
1594 src = ra_alloc1(as, ir->op1, RSET_FPR); /* May alloc GPR to remat FPR. */
1595 as->freeset |= (of & RSET_RANGE(RID_R0, RID_R1+1));
1596 emit_dnm(as, ARMI_VMOV_RR_D, RID_R0, RID_R1, (src & 15));
1597}
1598#endif
1599 1610
1600static void asm_bitswap(ASMState *as, IRIns *ir) 1611static void asm_bswap(ASMState *as, IRIns *ir)
1601{ 1612{
1602 Reg dest = ra_dest(as, ir, RSET_GPR); 1613 Reg dest = ra_dest(as, ir, RSET_GPR);
1603 Reg left = ra_alloc1(as, ir->op1, RSET_GPR); 1614 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
@@ -1614,6 +1625,10 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
1614 } 1625 }
1615} 1626}
1616 1627
1628#define asm_band(as, ir) asm_bitop(as, ir, ARMI_AND)
1629#define asm_bor(as, ir) asm_bitop(as, ir, ARMI_ORR)
1630#define asm_bxor(as, ir) asm_bitop(as, ir, ARMI_EOR)
1631
1617static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh) 1632static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh)
1618{ 1633{
1619 if (irref_isk(ir->op2)) { /* Constant shifts. */ 1634 if (irref_isk(ir->op2)) { /* Constant shifts. */
@@ -1631,6 +1646,12 @@ static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh)
1631 } 1646 }
1632} 1647}
1633 1648
1649#define asm_bshl(as, ir) asm_bitshift(as, ir, ARMSH_LSL)
1650#define asm_bshr(as, ir) asm_bitshift(as, ir, ARMSH_LSR)
1651#define asm_bsar(as, ir) asm_bitshift(as, ir, ARMSH_ASR)
1652#define asm_bror(as, ir) asm_bitshift(as, ir, ARMSH_ROR)
1653#define asm_brol(as, ir) lj_assertA(0, "unexpected BROL")
1654
1634static void asm_intmin_max(ASMState *as, IRIns *ir, int cc) 1655static void asm_intmin_max(ASMState *as, IRIns *ir, int cc)
1635{ 1656{
1636 uint32_t kcmp = 0, kmov = 0; 1657 uint32_t kcmp = 0, kmov = 0;
@@ -1704,6 +1725,9 @@ static void asm_min_max(ASMState *as, IRIns *ir, int cc, int fcc)
1704 asm_intmin_max(as, ir, cc); 1725 asm_intmin_max(as, ir, cc);
1705} 1726}
1706 1727
1728#define asm_min(as, ir) asm_min_max(as, ir, CC_GT, CC_PL)
1729#define asm_max(as, ir) asm_min_max(as, ir, CC_LT, CC_LE)
1730
1707/* -- Comparisons --------------------------------------------------------- */ 1731/* -- Comparisons --------------------------------------------------------- */
1708 1732
1709/* Map of comparisons to flags. ORDER IR. */ 1733/* Map of comparisons to flags. ORDER IR. */
@@ -1777,7 +1801,8 @@ static void asm_intcomp(ASMState *as, IRIns *ir)
1777 Reg left; 1801 Reg left;
1778 uint32_t m; 1802 uint32_t m;
1779 int cmpprev0 = 0; 1803 int cmpprev0 = 0;
1780 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); 1804 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t),
1805 "bad comparison data type %d", irt_type(ir->t));
1781 if (asm_swapops(as, lref, rref)) { 1806 if (asm_swapops(as, lref, rref)) {
1782 Reg tmp = lref; lref = rref; rref = tmp; 1807 Reg tmp = lref; lref = rref; rref = tmp;
1783 if (cc >= CC_GE) cc ^= 7; /* LT <-> GT, LE <-> GE */ 1808 if (cc >= CC_GE) cc ^= 7; /* LT <-> GT, LE <-> GE */
@@ -1819,6 +1844,18 @@ notst:
1819 as->flagmcp = as->mcp; /* Allow elimination of the compare. */ 1844 as->flagmcp = as->mcp; /* Allow elimination of the compare. */
1820} 1845}
1821 1846
1847static void asm_comp(ASMState *as, IRIns *ir)
1848{
1849#if !LJ_SOFTFP
1850 if (irt_isnum(ir->t))
1851 asm_fpcomp(as, ir);
1852 else
1853#endif
1854 asm_intcomp(as, ir);
1855}
1856
1857#define asm_equal(as, ir) asm_comp(as, ir)
1858
1822#if LJ_HASFFI 1859#if LJ_HASFFI
1823/* 64 bit integer comparisons. */ 1860/* 64 bit integer comparisons. */
1824static void asm_int64comp(ASMState *as, IRIns *ir) 1861static void asm_int64comp(ASMState *as, IRIns *ir)
@@ -1857,15 +1894,15 @@ static void asm_int64comp(ASMState *as, IRIns *ir)
1857} 1894}
1858#endif 1895#endif
1859 1896
1860/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */ 1897/* -- Split register ops -------------------------------------------------- */
1861 1898
1862/* Hiword op of a split 64 bit op. Previous op must be the loword op. */ 1899/* Hiword op of a split 32/32 bit op. Previous op is the loword op. */
1863static void asm_hiop(ASMState *as, IRIns *ir) 1900static void asm_hiop(ASMState *as, IRIns *ir)
1864{ 1901{
1865#if LJ_HASFFI || LJ_SOFTFP
1866 /* HIOP is marked as a store because it needs its own DCE logic. */ 1902 /* HIOP is marked as a store because it needs its own DCE logic. */
1867 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */ 1903 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1868 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1; 1904 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1905#if LJ_HASFFI || LJ_SOFTFP
1869 if ((ir-1)->o <= IR_NE) { /* 64 bit integer or FP comparisons. ORDER IR. */ 1906 if ((ir-1)->o <= IR_NE) { /* 64 bit integer or FP comparisons. ORDER IR. */
1870 as->curins--; /* Always skip the loword comparison. */ 1907 as->curins--; /* Always skip the loword comparison. */
1871#if LJ_SOFTFP 1908#if LJ_SOFTFP
@@ -1882,7 +1919,7 @@ static void asm_hiop(ASMState *as, IRIns *ir)
1882 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) { 1919 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
1883 as->curins--; /* Always skip the loword min/max. */ 1920 as->curins--; /* Always skip the loword min/max. */
1884 if (uselo || usehi) 1921 if (uselo || usehi)
1885 asm_sfpmin_max(as, ir-1, (ir-1)->o == IR_MIN ? CC_HI : CC_LO); 1922 asm_sfpmin_max(as, ir-1, (ir-1)->o == IR_MIN ? CC_PL : CC_LE);
1886 return; 1923 return;
1887#elif LJ_HASFFI 1924#elif LJ_HASFFI
1888 } else if ((ir-1)->o == IR_CONV) { 1925 } else if ((ir-1)->o == IR_CONV) {
@@ -1893,9 +1930,10 @@ static void asm_hiop(ASMState *as, IRIns *ir)
1893#endif 1930#endif
1894 } else if ((ir-1)->o == IR_XSTORE) { 1931 } else if ((ir-1)->o == IR_XSTORE) {
1895 if ((ir-1)->r != RID_SINK) 1932 if ((ir-1)->r != RID_SINK)
1896 asm_xstore(as, ir, 4); 1933 asm_xstore_(as, ir, 4);
1897 return; 1934 return;
1898 } 1935 }
1936#endif
1899 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */ 1937 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1900 switch ((ir-1)->o) { 1938 switch ((ir-1)->o) {
1901#if LJ_HASFFI 1939#if LJ_HASFFI
@@ -1914,6 +1952,9 @@ static void asm_hiop(ASMState *as, IRIns *ir)
1914 asm_intneg(as, ir, ARMI_RSC); 1952 asm_intneg(as, ir, ARMI_RSC);
1915 asm_intneg(as, ir-1, ARMI_RSB|ARMI_S); 1953 asm_intneg(as, ir-1, ARMI_RSB|ARMI_S);
1916 break; 1954 break;
1955 case IR_CNEWI:
1956 /* Nothing to do here. Handled by lo op itself. */
1957 break;
1917#endif 1958#endif
1918#if LJ_SOFTFP 1959#if LJ_SOFTFP
1919 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD: 1960 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
@@ -1921,24 +1962,26 @@ static void asm_hiop(ASMState *as, IRIns *ir)
1921 if (!uselo) 1962 if (!uselo)
1922 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */ 1963 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */
1923 break; 1964 break;
1965 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR: case IR_TMPREF:
1966 /* Nothing to do here. Handled by lo op itself. */
1967 break;
1924#endif 1968#endif
1925 case IR_CALLN: 1969 case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
1926 case IR_CALLS:
1927 case IR_CALLXS:
1928 if (!uselo) 1970 if (!uselo)
1929 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */ 1971 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
1930 break; 1972 break;
1931#if LJ_SOFTFP 1973 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
1932 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR:
1933#endif
1934 case IR_CNEWI:
1935 /* Nothing to do here. Handled by lo op itself. */
1936 break;
1937 default: lua_assert(0); break;
1938 } 1974 }
1939#else 1975}
1940 UNUSED(as); UNUSED(ir); lua_assert(0); 1976
1941#endif 1977/* -- Profiling ----------------------------------------------------------- */
1978
1979static void asm_prof(ASMState *as, IRIns *ir)
1980{
1981 UNUSED(ir);
1982 asm_guardcc(as, CC_NE);
1983 emit_n(as, ARMI_TST|ARMI_K12|HOOK_PROFILE, RID_TMP);
1984 emit_lsptr(as, ARMI_LDRB, RID_TMP, (void *)&J2G(as->J)->hookmask);
1942} 1985}
1943 1986
1944/* -- Stack handling ------------------------------------------------------ */ 1987/* -- Stack handling ------------------------------------------------------ */
@@ -1952,7 +1995,7 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
1952 if (irp) { 1995 if (irp) {
1953 if (!ra_hasspill(irp->s)) { 1996 if (!ra_hasspill(irp->s)) {
1954 pbase = irp->r; 1997 pbase = irp->r;
1955 lua_assert(ra_hasreg(pbase)); 1998 lj_assertA(ra_hasreg(pbase), "base reg lost");
1956 } else if (allow) { 1999 } else if (allow) {
1957 pbase = rset_pickbot(allow); 2000 pbase = rset_pickbot(allow);
1958 } else { 2001 } else {
@@ -1964,13 +2007,13 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
1964 } 2007 }
1965 emit_branch(as, ARMF_CC(ARMI_BL, CC_LS), exitstub_addr(as->J, exitno)); 2008 emit_branch(as, ARMF_CC(ARMI_BL, CC_LS), exitstub_addr(as->J, exitno));
1966 k = emit_isk12(0, (int32_t)(8*topslot)); 2009 k = emit_isk12(0, (int32_t)(8*topslot));
1967 lua_assert(k); 2010 lj_assertA(k, "slot offset %d does not fit in K12", 8*topslot);
1968 emit_n(as, ARMI_CMP^k, RID_TMP); 2011 emit_n(as, ARMI_CMP^k, RID_TMP);
1969 emit_dnm(as, ARMI_SUB, RID_TMP, RID_TMP, pbase); 2012 emit_dnm(as, ARMI_SUB, RID_TMP, RID_TMP, pbase);
1970 emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP, 2013 emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP,
1971 (int32_t)offsetof(lua_State, maxstack)); 2014 (int32_t)offsetof(lua_State, maxstack));
1972 if (irp) { /* Must not spill arbitrary registers in head of side trace. */ 2015 if (irp) { /* Must not spill arbitrary registers in head of side trace. */
1973 int32_t i = i32ptr(&J2G(as->J)->jit_L); 2016 int32_t i = i32ptr(&J2G(as->J)->cur_L);
1974 if (ra_hasspill(irp->s)) 2017 if (ra_hasspill(irp->s))
1975 emit_lso(as, ARMI_LDR, pbase, RID_SP, sps_scale(irp->s)); 2018 emit_lso(as, ARMI_LDR, pbase, RID_SP, sps_scale(irp->s));
1976 emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP, (i & 4095)); 2019 emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP, (i & 4095));
@@ -1978,7 +2021,7 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
1978 emit_lso(as, ARMI_STR, RID_RET, RID_SP, 0); /* Save temp. register. */ 2021 emit_lso(as, ARMI_STR, RID_RET, RID_SP, 0); /* Save temp. register. */
1979 emit_loadi(as, RID_TMP, (i & ~4095)); 2022 emit_loadi(as, RID_TMP, (i & ~4095));
1980 } else { 2023 } else {
1981 emit_getgl(as, RID_TMP, jit_L); 2024 emit_getgl(as, RID_TMP, cur_L);
1982 } 2025 }
1983} 2026}
1984 2027
@@ -2001,7 +2044,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2001#if LJ_SOFTFP 2044#if LJ_SOFTFP
2002 RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE); 2045 RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE);
2003 Reg tmp; 2046 Reg tmp;
2004 lua_assert(irref_isk(ref)); /* LJ_SOFTFP: must be a number constant. */ 2047 /* LJ_SOFTFP: must be a number constant. */
2048 lj_assertA(irref_isk(ref), "unsplit FP op");
2005 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, 2049 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo,
2006 rset_exclude(RSET_GPREVEN, RID_BASE)); 2050 rset_exclude(RSET_GPREVEN, RID_BASE));
2007 emit_lso(as, ARMI_STR, tmp, RID_BASE, ofs); 2051 emit_lso(as, ARMI_STR, tmp, RID_BASE, ofs);
@@ -2015,7 +2059,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2015 } else { 2059 } else {
2016 RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE); 2060 RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE);
2017 Reg type; 2061 Reg type;
2018 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t)); 2062 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
2063 "restore of IR type %d", irt_type(ir->t));
2019 if (!irt_ispri(ir->t)) { 2064 if (!irt_ispri(ir->t)) {
2020 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPREVEN, RID_BASE)); 2065 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPREVEN, RID_BASE));
2021 emit_lso(as, ARMI_STR, src, RID_BASE, ofs); 2066 emit_lso(as, ARMI_STR, src, RID_BASE, ofs);
@@ -2028,6 +2073,8 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2028 } else if ((sn & SNAP_SOFTFPNUM)) { 2073 } else if ((sn & SNAP_SOFTFPNUM)) {
2029 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPRODD, RID_BASE)); 2074 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPRODD, RID_BASE));
2030#endif 2075#endif
2076 } else if ((sn & SNAP_KEYINDEX)) {
2077 type = ra_allock(as, (int32_t)LJ_KEYINDEX, odd);
2031 } else { 2078 } else {
2032 type = ra_allock(as, (int32_t)irt_toitype(ir->t), odd); 2079 type = ra_allock(as, (int32_t)irt_toitype(ir->t), odd);
2033 } 2080 }
@@ -2035,7 +2082,7 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2035 } 2082 }
2036 checkmclim(as); 2083 checkmclim(as);
2037 } 2084 }
2038 lua_assert(map + nent == flinks); 2085 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
2039} 2086}
2040 2087
2041/* -- GC handling --------------------------------------------------------- */ 2088/* -- GC handling --------------------------------------------------------- */
@@ -2089,15 +2136,21 @@ static void asm_loop_fixup(ASMState *as)
2089 } 2136 }
2090} 2137}
2091 2138
2139/* Fixup the tail of the loop. */
2140static void asm_loop_tail_fixup(ASMState *as)
2141{
2142 UNUSED(as); /* Nothing to do. */
2143}
2144
2092/* -- Head of trace ------------------------------------------------------- */ 2145/* -- Head of trace ------------------------------------------------------- */
2093 2146
2094/* Reload L register from g->jit_L. */ 2147/* Reload L register from g->cur_L. */
2095static void asm_head_lreg(ASMState *as) 2148static void asm_head_lreg(ASMState *as)
2096{ 2149{
2097 IRIns *ir = IR(ASMREF_L); 2150 IRIns *ir = IR(ASMREF_L);
2098 if (ra_used(ir)) { 2151 if (ra_used(ir)) {
2099 Reg r = ra_dest(as, ir, RSET_GPR); 2152 Reg r = ra_dest(as, ir, RSET_GPR);
2100 emit_getgl(as, r, jit_L); 2153 emit_getgl(as, r, cur_L);
2101 ra_evictk(as); 2154 ra_evictk(as);
2102 } 2155 }
2103} 2156}
@@ -2125,7 +2178,7 @@ static Reg asm_head_side_base(ASMState *as, IRIns *irp)
2125 return ra_dest(as, ir, RSET_GPR); 2178 return ra_dest(as, ir, RSET_GPR);
2126 } else { 2179 } else {
2127 Reg r = irp->r; 2180 Reg r = irp->r;
2128 lua_assert(ra_hasreg(r)); 2181 lj_assertA(ra_hasreg(r), "base reg lost");
2129 if (r != ir->r && !rset_test(as->freeset, r)) 2182 if (r != ir->r && !rset_test(as->freeset, r))
2130 ra_restore(as, regcost_ref(as->cost[r])); 2183 ra_restore(as, regcost_ref(as->cost[r]));
2131 ra_destreg(as, ir, r); 2184 ra_destreg(as, ir, r);
@@ -2146,7 +2199,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2146 } else { 2199 } else {
2147 /* Patch stack adjustment. */ 2200 /* Patch stack adjustment. */
2148 uint32_t k = emit_isk12(ARMI_ADD, spadj); 2201 uint32_t k = emit_isk12(ARMI_ADD, spadj);
2149 lua_assert(k); 2202 lj_assertA(k, "stack adjustment %d does not fit in K12", spadj);
2150 p[-2] = (ARMI_ADD^k) | ARMF_D(RID_SP) | ARMF_N(RID_SP); 2203 p[-2] = (ARMI_ADD^k) | ARMF_D(RID_SP) | ARMF_N(RID_SP);
2151 } 2204 }
2152 /* Patch exit branch. */ 2205 /* Patch exit branch. */
@@ -2167,143 +2220,13 @@ static void asm_tail_prep(ASMState *as)
2167 *p = 0; /* Prevent load/store merging. */ 2220 *p = 0; /* Prevent load/store merging. */
2168} 2221}
2169 2222
2170/* -- Instruction dispatch ------------------------------------------------ */
2171
2172/* Assemble a single instruction. */
2173static void asm_ir(ASMState *as, IRIns *ir)
2174{
2175 switch ((IROp)ir->o) {
2176 /* Miscellaneous ops. */
2177 case IR_LOOP: asm_loop(as); break;
2178 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
2179 case IR_USE:
2180 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
2181 case IR_PHI: asm_phi(as, ir); break;
2182 case IR_HIOP: asm_hiop(as, ir); break;
2183 case IR_GCSTEP: asm_gcstep(as, ir); break;
2184
2185 /* Guarded assertions. */
2186 case IR_EQ: case IR_NE:
2187 if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
2188 as->curins--;
2189 asm_href(as, ir-1, (IROp)ir->o);
2190 break;
2191 }
2192 /* fallthrough */
2193 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
2194 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
2195 case IR_ABC:
2196#if !LJ_SOFTFP
2197 if (irt_isnum(ir->t)) { asm_fpcomp(as, ir); break; }
2198#endif
2199 asm_intcomp(as, ir);
2200 break;
2201
2202 case IR_RETF: asm_retf(as, ir); break;
2203
2204 /* Bit ops. */
2205 case IR_BNOT: asm_bitop(as, ir, ARMI_MVN); break;
2206 case IR_BSWAP: asm_bitswap(as, ir); break;
2207
2208 case IR_BAND: asm_bitop(as, ir, ARMI_AND); break;
2209 case IR_BOR: asm_bitop(as, ir, ARMI_ORR); break;
2210 case IR_BXOR: asm_bitop(as, ir, ARMI_EOR); break;
2211
2212 case IR_BSHL: asm_bitshift(as, ir, ARMSH_LSL); break;
2213 case IR_BSHR: asm_bitshift(as, ir, ARMSH_LSR); break;
2214 case IR_BSAR: asm_bitshift(as, ir, ARMSH_ASR); break;
2215 case IR_BROR: asm_bitshift(as, ir, ARMSH_ROR); break;
2216 case IR_BROL: lua_assert(0); break;
2217
2218 /* Arithmetic ops. */
2219 case IR_ADD: case IR_ADDOV: asm_add(as, ir); break;
2220 case IR_SUB: case IR_SUBOV: asm_sub(as, ir); break;
2221 case IR_MUL: case IR_MULOV: asm_mul(as, ir); break;
2222 case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
2223 case IR_NEG: asm_neg(as, ir); break;
2224
2225#if LJ_SOFTFP
2226 case IR_DIV: case IR_POW: case IR_ABS:
2227 case IR_ATAN2: case IR_LDEXP: case IR_FPMATH: case IR_TOBIT:
2228 lua_assert(0); /* Unused for LJ_SOFTFP. */
2229 break;
2230#else
2231 case IR_DIV: asm_fparith(as, ir, ARMI_VDIV_D); break;
2232 case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
2233 case IR_ABS: asm_fpunary(as, ir, ARMI_VABS_D); break;
2234 case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
2235 case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
2236 case IR_FPMATH:
2237 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
2238 break;
2239 if (ir->op2 <= IRFPM_TRUNC)
2240 asm_callround(as, ir, ir->op2);
2241 else if (ir->op2 == IRFPM_SQRT)
2242 asm_fpunary(as, ir, ARMI_VSQRT_D);
2243 else
2244 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
2245 break;
2246 case IR_TOBIT: asm_tobit(as, ir); break;
2247#endif
2248
2249 case IR_MIN: asm_min_max(as, ir, CC_GT, CC_HI); break;
2250 case IR_MAX: asm_min_max(as, ir, CC_LT, CC_LO); break;
2251
2252 /* Memory references. */
2253 case IR_AREF: asm_aref(as, ir); break;
2254 case IR_HREF: asm_href(as, ir, 0); break;
2255 case IR_HREFK: asm_hrefk(as, ir); break;
2256 case IR_NEWREF: asm_newref(as, ir); break;
2257 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
2258 case IR_FREF: asm_fref(as, ir); break;
2259 case IR_STRREF: asm_strref(as, ir); break;
2260
2261 /* Loads and stores. */
2262 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2263 asm_ahuvload(as, ir);
2264 break;
2265 case IR_FLOAD: asm_fload(as, ir); break;
2266 case IR_XLOAD: asm_xload(as, ir); break;
2267 case IR_SLOAD: asm_sload(as, ir); break;
2268
2269 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
2270 case IR_FSTORE: asm_fstore(as, ir); break;
2271 case IR_XSTORE: asm_xstore(as, ir, 0); break;
2272
2273 /* Allocations. */
2274 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
2275 case IR_TNEW: asm_tnew(as, ir); break;
2276 case IR_TDUP: asm_tdup(as, ir); break;
2277 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
2278
2279 /* Write barriers. */
2280 case IR_TBAR: asm_tbar(as, ir); break;
2281 case IR_OBAR: asm_obar(as, ir); break;
2282
2283 /* Type conversions. */
2284 case IR_CONV: asm_conv(as, ir); break;
2285 case IR_TOSTR: asm_tostr(as, ir); break;
2286 case IR_STRTO: asm_strto(as, ir); break;
2287
2288 /* Calls. */
2289 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
2290 case IR_CALLXS: asm_callx(as, ir); break;
2291 case IR_CARG: break;
2292
2293 default:
2294 setintV(&as->J->errinfo, ir->o);
2295 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
2296 break;
2297 }
2298}
2299
2300/* -- Trace setup --------------------------------------------------------- */ 2223/* -- Trace setup --------------------------------------------------------- */
2301 2224
2302/* Ensure there are enough stack slots for call arguments. */ 2225/* Ensure there are enough stack slots for call arguments. */
2303static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci) 2226static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2304{ 2227{
2305 IRRef args[CCI_NARGS_MAX*2]; 2228 IRRef args[CCI_NARGS_MAX*2];
2306 uint32_t i, nargs = (int)CCI_NARGS(ci); 2229 uint32_t i, nargs = CCI_XNARGS(ci);
2307 int nslots = 0, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR, fprodd = 0; 2230 int nslots = 0, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR, fprodd = 0;
2308 asm_collectargs(as, ir, ci, args); 2231 asm_collectargs(as, ir, ci, args);
2309 for (i = 0; i < nargs; i++) { 2232 for (i = 0; i < nargs; i++) {
@@ -2359,7 +2282,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2359 if (!cstart) cstart = p; 2282 if (!cstart) cstart = p;
2360 } 2283 }
2361 } 2284 }
2362 lua_assert(cstart != NULL); 2285 lj_assertJ(cstart != NULL, "exit stub %d not found", exitno);
2363 lj_mcode_sync(cstart, cend); 2286 lj_mcode_sync(cstart, cend);
2364 lj_mcode_patch(J, mcarea, 1); 2287 lj_mcode_patch(J, mcarea, 1);
2365} 2288}
diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h
new file mode 100644
index 00000000..34960d7c
--- /dev/null
+++ b/src/lj_asm_arm64.h
@@ -0,0 +1,2070 @@
1/*
2** ARM64 IR assembler (SSA IR -> machine code).
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4**
5** Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com.
6** Sponsored by Cisco Systems, Inc.
7*/
8
9/* -- Register allocator extensions --------------------------------------- */
10
11/* Allocate a register with a hint. */
12static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
13{
14 Reg r = IR(ref)->r;
15 if (ra_noreg(r)) {
16 if (!ra_hashint(r) && !iscrossref(as, ref))
17 ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */
18 r = ra_allocref(as, ref, allow);
19 }
20 ra_noweak(as, r);
21 return r;
22}
23
24/* Allocate two source registers for three-operand instructions. */
25static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
26{
27 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
28 Reg left = irl->r, right = irr->r;
29 if (ra_hasreg(left)) {
30 ra_noweak(as, left);
31 if (ra_noreg(right))
32 right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
33 else
34 ra_noweak(as, right);
35 } else if (ra_hasreg(right)) {
36 ra_noweak(as, right);
37 left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
38 } else if (ra_hashint(right)) {
39 right = ra_allocref(as, ir->op2, allow);
40 left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
41 } else {
42 left = ra_allocref(as, ir->op1, allow);
43 right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
44 }
45 return left | (right << 8);
46}
47
48/* -- Guard handling ------------------------------------------------------ */
49
50/* Setup all needed exit stubs. */
51static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
52{
53 ExitNo i;
54 MCode *mxp = as->mctop;
55 if (mxp - (nexits + 3 + MCLIM_REDZONE) < as->mclim)
56 asm_mclimit(as);
57 /* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */
58 for (i = nexits-1; (int32_t)i >= 0; i--)
59 *--mxp = A64I_LE(A64I_BL | A64F_S26(-3-i));
60 *--mxp = A64I_LE(A64I_MOVZw | A64F_U16(as->T->traceno));
61 mxp--;
62 *mxp = A64I_LE(A64I_BL | A64F_S26(((MCode *)(void *)lj_vm_exit_handler-mxp)));
63 *--mxp = A64I_LE(A64I_STRx | A64F_D(RID_LR) | A64F_N(RID_SP));
64 as->mctop = mxp;
65}
66
67static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
68{
69 /* Keep this in-sync with exitstub_trace_addr(). */
70 return as->mctop + exitno + 3;
71}
72
73/* Emit conditional branch to exit for guard. */
74static void asm_guardcc(ASMState *as, A64CC cc)
75{
76 MCode *target = asm_exitstub_addr(as, as->snapno);
77 MCode *p = as->mcp;
78 if (LJ_UNLIKELY(p == as->invmcp)) {
79 as->loopinv = 1;
80 *p = A64I_B | A64F_S26(target-p);
81 emit_cond_branch(as, cc^1, p-1);
82 return;
83 }
84 emit_cond_branch(as, cc, target);
85}
86
87/* Emit test and branch instruction to exit for guard. */
88static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
89{
90 MCode *target = asm_exitstub_addr(as, as->snapno);
91 MCode *p = as->mcp;
92 if (LJ_UNLIKELY(p == as->invmcp)) {
93 as->loopinv = 1;
94 *p = A64I_B | A64F_S26(target-p);
95 emit_tnb(as, ai^0x01000000u, r, bit, p-1);
96 return;
97 }
98 emit_tnb(as, ai, r, bit, target);
99}
100
101/* Emit compare and branch instruction to exit for guard. */
102static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
103{
104 MCode *target = asm_exitstub_addr(as, as->snapno);
105 MCode *p = as->mcp;
106 if (LJ_UNLIKELY(p == as->invmcp)) {
107 as->loopinv = 1;
108 *p = A64I_B | A64F_S26(target-p);
109 emit_cnb(as, ai^0x01000000u, r, p-1);
110 return;
111 }
112 emit_cnb(as, ai, r, target);
113}
114
115/* -- Operand fusion ------------------------------------------------------ */
116
117/* Limit linear search to this distance. Avoids O(n^2) behavior. */
118#define CONFLICT_SEARCH_LIM 31
119
120static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
121{
122 if (irref_isk(ref)) {
123 IRIns *ir = IR(ref);
124 if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
125 *k = ir->i;
126 return 1;
127 } else if (checki32((int64_t)ir_k64(ir)->u64)) {
128 *k = (int32_t)ir_k64(ir)->u64;
129 return 1;
130 }
131 }
132 return 0;
133}
134
135/* Check if there's no conflicting instruction between curins and ref. */
136static int noconflict(ASMState *as, IRRef ref, IROp conflict)
137{
138 IRIns *ir = as->ir;
139 IRRef i = as->curins;
140 if (i > ref + CONFLICT_SEARCH_LIM)
141 return 0; /* Give up, ref is too far away. */
142 while (--i > ref)
143 if (ir[i].o == conflict)
144 return 0; /* Conflict found. */
145 return 1; /* Ok, no conflict. */
146}
147
148/* Fuse the array base of colocated arrays. */
149static int32_t asm_fuseabase(ASMState *as, IRRef ref)
150{
151 IRIns *ir = IR(ref);
152 if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
153 !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
154 return (int32_t)sizeof(GCtab);
155 return 0;
156}
157
158#define FUSE_REG 0x40000000
159
160/* Fuse array/hash/upvalue reference into register+offset operand. */
161static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow,
162 A64Ins ins)
163{
164 IRIns *ir = IR(ref);
165 if (ra_noreg(ir->r)) {
166 if (ir->o == IR_AREF) {
167 if (mayfuse(as, ref)) {
168 if (irref_isk(ir->op2)) {
169 IRRef tab = IR(ir->op1)->op1;
170 int32_t ofs = asm_fuseabase(as, tab);
171 IRRef refa = ofs ? tab : ir->op1;
172 ofs += 8*IR(ir->op2)->i;
173 if (emit_checkofs(ins, ofs)) {
174 *ofsp = ofs;
175 return ra_alloc1(as, refa, allow);
176 }
177 } else {
178 Reg base = ra_alloc1(as, ir->op1, allow);
179 *ofsp = FUSE_REG|ra_alloc1(as, ir->op2, rset_exclude(allow, base));
180 return base;
181 }
182 }
183 } else if (ir->o == IR_HREFK) {
184 if (mayfuse(as, ref)) {
185 int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
186 if (emit_checkofs(ins, ofs)) {
187 *ofsp = ofs;
188 return ra_alloc1(as, ir->op1, allow);
189 }
190 }
191 } else if (ir->o == IR_UREFC) {
192 if (irref_isk(ir->op1)) {
193 GCfunc *fn = ir_kfunc(IR(ir->op1));
194 GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
195 int64_t ofs = glofs(as, &uv->tv);
196 if (emit_checkofs(ins, ofs)) {
197 *ofsp = (int32_t)ofs;
198 return RID_GL;
199 }
200 }
201 } else if (ir->o == IR_TMPREF) {
202 *ofsp = (int32_t)glofs(as, &J2G(as->J)->tmptv);
203 return RID_GL;
204 }
205 }
206 *ofsp = 0;
207 return ra_alloc1(as, ref, allow);
208}
209
210/* Fuse m operand into arithmetic/logic instructions. */
211static uint32_t asm_fuseopm(ASMState *as, A64Ins ai, IRRef ref, RegSet allow)
212{
213 IRIns *ir = IR(ref);
214 if (ra_hasreg(ir->r)) {
215 ra_noweak(as, ir->r);
216 return A64F_M(ir->r);
217 } else if (irref_isk(ref)) {
218 uint32_t m;
219 int64_t k = get_k64val(as, ref);
220 if ((ai & 0x1f000000) == 0x0a000000)
221 m = emit_isk13(k, irt_is64(ir->t));
222 else
223 m = emit_isk12(k);
224 if (m)
225 return m;
226 } else if (mayfuse(as, ref)) {
227 if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR && irref_isk(ir->op2)) ||
228 (ir->o == IR_ADD && ir->op1 == ir->op2)) {
229 A64Shift sh = ir->o == IR_BSHR ? A64SH_LSR :
230 ir->o == IR_BSAR ? A64SH_ASR : A64SH_LSL;
231 int shift = ir->o == IR_ADD ? 1 :
232 (IR(ir->op2)->i & (irt_is64(ir->t) ? 63 : 31));
233 IRIns *irl = IR(ir->op1);
234 if (sh == A64SH_LSL &&
235 irl->o == IR_CONV &&
236 irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
237 shift <= 4 &&
238 canfuse(as, irl)) {
239 Reg m = ra_alloc1(as, irl->op1, allow);
240 return A64F_M(m) | A64F_EXSH(A64EX_SXTW, shift);
241 } else {
242 Reg m = ra_alloc1(as, ir->op1, allow);
243 return A64F_M(m) | A64F_SH(sh, shift);
244 }
245 } else if (ir->o == IR_CONV &&
246 ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)) {
247 Reg m = ra_alloc1(as, ir->op1, allow);
248 return A64F_M(m) | A64F_EX(A64EX_SXTW);
249 }
250 }
251 return A64F_M(ra_allocref(as, ref, allow));
252}
253
254/* Fuse XLOAD/XSTORE reference into load/store operand. */
255static void asm_fusexref(ASMState *as, A64Ins ai, Reg rd, IRRef ref,
256 RegSet allow)
257{
258 IRIns *ir = IR(ref);
259 Reg base;
260 int32_t ofs = 0;
261 if (ra_noreg(ir->r) && canfuse(as, ir)) {
262 if (ir->o == IR_ADD) {
263 if (asm_isk32(as, ir->op2, &ofs) && emit_checkofs(ai, ofs)) {
264 ref = ir->op1;
265 } else {
266 Reg rn, rm;
267 IRRef lref = ir->op1, rref = ir->op2;
268 IRIns *irl = IR(lref);
269 if (mayfuse(as, irl->op1)) {
270 unsigned int shift = 4;
271 if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
272 shift = (IR(irl->op2)->i & 63);
273 } else if (irl->o == IR_ADD && irl->op1 == irl->op2) {
274 shift = 1;
275 }
276 if ((ai >> 30) == shift) {
277 lref = irl->op1;
278 irl = IR(lref);
279 ai |= A64I_LS_SH;
280 }
281 }
282 if (irl->o == IR_CONV &&
283 irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
284 canfuse(as, irl)) {
285 lref = irl->op1;
286 ai |= A64I_LS_SXTWx;
287 } else {
288 ai |= A64I_LS_LSLx;
289 }
290 rm = ra_alloc1(as, lref, allow);
291 rn = ra_alloc1(as, rref, rset_exclude(allow, rm));
292 emit_dnm(as, (ai^A64I_LS_R), (rd & 31), rn, rm);
293 return;
294 }
295 } else if (ir->o == IR_STRREF) {
296 if (asm_isk32(as, ir->op2, &ofs)) {
297 ref = ir->op1;
298 } else if (asm_isk32(as, ir->op1, &ofs)) {
299 ref = ir->op2;
300 } else {
301 Reg refk = irref_isk(ir->op1) ? ir->op1 : ir->op2;
302 Reg refv = irref_isk(ir->op1) ? ir->op2 : ir->op1;
303 Reg rn = ra_alloc1(as, refv, allow);
304 IRIns *irr = IR(refk);
305 uint32_t m;
306 if (irr+1 == ir && !ra_used(irr) &&
307 irr->o == IR_ADD && irref_isk(irr->op2)) {
308 ofs = sizeof(GCstr) + IR(irr->op2)->i;
309 if (emit_checkofs(ai, ofs)) {
310 Reg rm = ra_alloc1(as, irr->op1, rset_exclude(allow, rn));
311 m = A64F_M(rm) | A64F_EX(A64EX_SXTW);
312 goto skipopm;
313 }
314 }
315 m = asm_fuseopm(as, 0, refk, rset_exclude(allow, rn));
316 ofs = sizeof(GCstr);
317 skipopm:
318 emit_lso(as, ai, rd, rd, ofs);
319 emit_dn(as, A64I_ADDx^m, rd, rn);
320 return;
321 }
322 ofs += sizeof(GCstr);
323 if (!emit_checkofs(ai, ofs)) {
324 Reg rn = ra_alloc1(as, ref, allow);
325 Reg rm = ra_allock(as, ofs, rset_exclude(allow, rn));
326 emit_dnm(as, (ai^A64I_LS_R)|A64I_LS_UXTWx, rd, rn, rm);
327 return;
328 }
329 }
330 }
331 base = ra_alloc1(as, ref, allow);
332 emit_lso(as, ai, (rd & 31), base, ofs);
333}
334
335/* Fuse FP multiply-add/sub. */
336static int asm_fusemadd(ASMState *as, IRIns *ir, A64Ins ai, A64Ins air)
337{
338 IRRef lref = ir->op1, rref = ir->op2;
339 IRIns *irm;
340 if ((as->flags & JIT_F_OPT_FMA) &&
341 lref != rref &&
342 ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) &&
343 ra_noreg(irm->r)) ||
344 (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) &&
345 (rref = lref, ai = air, ra_noreg(irm->r))))) {
346 Reg dest = ra_dest(as, ir, RSET_FPR);
347 Reg add = ra_hintalloc(as, rref, dest, RSET_FPR);
348 Reg left = ra_alloc2(as, irm,
349 rset_exclude(rset_exclude(RSET_FPR, dest), add));
350 Reg right = (left >> 8); left &= 255;
351 emit_dnma(as, ai, (dest & 31), (left & 31), (right & 31), (add & 31));
352 return 1;
353 }
354 return 0;
355}
356
357/* Fuse BAND + BSHL/BSHR into UBFM. */
358static int asm_fuseandshift(ASMState *as, IRIns *ir)
359{
360 IRIns *irl = IR(ir->op1);
361 lj_assertA(ir->o == IR_BAND, "bad usage");
362 if (canfuse(as, irl) && irref_isk(ir->op2)) {
363 uint64_t mask = get_k64val(as, ir->op2);
364 if (irref_isk(irl->op2) && (irl->o == IR_BSHR || irl->o == IR_BSHL)) {
365 int32_t shmask = irt_is64(irl->t) ? 63 : 31;
366 int32_t shift = (IR(irl->op2)->i & shmask);
367 int32_t imms = shift;
368 if (irl->o == IR_BSHL) {
369 mask >>= shift;
370 shift = (shmask-shift+1) & shmask;
371 imms = 0;
372 }
373 if (mask && !((mask+1) & mask)) { /* Contiguous 1-bits at the bottom. */
374 Reg dest = ra_dest(as, ir, RSET_GPR);
375 Reg left = ra_alloc1(as, irl->op1, RSET_GPR);
376 A64Ins ai = shmask == 63 ? A64I_UBFMx : A64I_UBFMw;
377 imms += 63 - emit_clz64(mask);
378 if (imms > shmask) imms = shmask;
379 emit_dn(as, ai | A64F_IMMS(imms) | A64F_IMMR(shift), dest, left);
380 return 1;
381 }
382 }
383 }
384 return 0;
385}
386
387/* Fuse BOR(BSHL, BSHR) into EXTR/ROR. */
388static int asm_fuseorshift(ASMState *as, IRIns *ir)
389{
390 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
391 lj_assertA(ir->o == IR_BOR, "bad usage");
392 if (canfuse(as, irl) && canfuse(as, irr) &&
393 ((irl->o == IR_BSHR && irr->o == IR_BSHL) ||
394 (irl->o == IR_BSHL && irr->o == IR_BSHR))) {
395 if (irref_isk(irl->op2) && irref_isk(irr->op2)) {
396 IRRef lref = irl->op1, rref = irr->op1;
397 uint32_t lshift = IR(irl->op2)->i, rshift = IR(irr->op2)->i;
398 if (irl->o == IR_BSHR) { /* BSHR needs to be the right operand. */
399 uint32_t tmp2;
400 IRRef tmp1 = lref; lref = rref; rref = tmp1;
401 tmp2 = lshift; lshift = rshift; rshift = tmp2;
402 }
403 if (rshift + lshift == (irt_is64(ir->t) ? 64 : 32)) {
404 A64Ins ai = irt_is64(ir->t) ? A64I_EXTRx : A64I_EXTRw;
405 Reg dest = ra_dest(as, ir, RSET_GPR);
406 Reg left = ra_alloc1(as, lref, RSET_GPR);
407 Reg right = ra_alloc1(as, rref, rset_exclude(RSET_GPR, left));
408 emit_dnm(as, ai | A64F_IMMS(rshift), dest, left, right);
409 return 1;
410 }
411 }
412 }
413 return 0;
414}
415
416/* -- Calls --------------------------------------------------------------- */
417
418/* Generate a call to a C function. */
419static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
420{
421 uint32_t n, nargs = CCI_XNARGS(ci);
422 int32_t ofs = 0;
423 Reg gpr, fpr = REGARG_FIRSTFPR;
424 if (ci->func)
425 emit_call(as, ci->func);
426 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
427 as->cost[gpr] = REGCOST(~0u, ASMREF_L);
428 gpr = REGARG_FIRSTGPR;
429 for (n = 0; n < nargs; n++) { /* Setup args. */
430 IRRef ref = args[n];
431 IRIns *ir = IR(ref);
432 if (ref) {
433 if (irt_isfp(ir->t)) {
434 if (fpr <= REGARG_LASTFPR) {
435 lj_assertA(rset_test(as->freeset, fpr),
436 "reg %d not free", fpr); /* Must have been evicted. */
437 ra_leftov(as, fpr, ref);
438 fpr++;
439 } else {
440 Reg r = ra_alloc1(as, ref, RSET_FPR);
441 emit_spstore(as, ir, r, ofs + ((LJ_BE && !irt_isnum(ir->t)) ? 4 : 0));
442 ofs += 8;
443 }
444 } else {
445 if (gpr <= REGARG_LASTGPR) {
446 lj_assertA(rset_test(as->freeset, gpr),
447 "reg %d not free", gpr); /* Must have been evicted. */
448 ra_leftov(as, gpr, ref);
449 gpr++;
450 } else {
451 Reg r = ra_alloc1(as, ref, RSET_GPR);
452 emit_spstore(as, ir, r, ofs + ((LJ_BE && !irt_is64(ir->t)) ? 4 : 0));
453 ofs += 8;
454 }
455 }
456 }
457 }
458}
459
460/* Setup result reg/sp for call. Evict scratch regs. */
461static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
462{
463 RegSet drop = RSET_SCRATCH;
464 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
465 if (ra_hasreg(ir->r))
466 rset_clear(drop, ir->r); /* Dest reg handled below. */
467 if (hiop && ra_hasreg((ir+1)->r))
468 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
469 ra_evictset(as, drop); /* Evictions must be performed first. */
470 if (ra_used(ir)) {
471 lj_assertA(!irt_ispri(ir->t), "PRI dest");
472 if (irt_isfp(ir->t)) {
473 if (ci->flags & CCI_CASTU64) {
474 Reg dest = ra_dest(as, ir, RSET_FPR) & 31;
475 emit_dn(as, irt_isnum(ir->t) ? A64I_FMOV_D_R : A64I_FMOV_S_R,
476 dest, RID_RET);
477 } else {
478 ra_destreg(as, ir, RID_FPRET);
479 }
480 } else if (hiop) {
481 ra_destpair(as, ir);
482 } else {
483 ra_destreg(as, ir, RID_RET);
484 }
485 }
486 UNUSED(ci);
487}
488
489static void asm_callx(ASMState *as, IRIns *ir)
490{
491 IRRef args[CCI_NARGS_MAX*2];
492 CCallInfo ci;
493 IRRef func;
494 IRIns *irf;
495 ci.flags = asm_callx_flags(as, ir);
496 asm_collectargs(as, ir, &ci, args);
497 asm_setupresult(as, ir, &ci);
498 func = ir->op2; irf = IR(func);
499 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
500 if (irref_isk(func)) { /* Call to constant address. */
501 ci.func = (ASMFunction)(ir_k64(irf)->u64);
502 } else { /* Need a non-argument register for indirect calls. */
503 Reg freg = ra_alloc1(as, func, RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED);
504 emit_n(as, A64I_BLR_AUTH, freg);
505 ci.func = (ASMFunction)(void *)0;
506 }
507 asm_gencall(as, &ci, args);
508}
509
510/* -- Returns ------------------------------------------------------------- */
511
512/* Return to lower frame. Guard that it goes to the right spot. */
513static void asm_retf(ASMState *as, IRIns *ir)
514{
515 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
516 void *pc = ir_kptr(IR(ir->op2));
517 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
518 as->topslot -= (BCReg)delta;
519 if ((int32_t)as->topslot < 0) as->topslot = 0;
520 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
521 /* Need to force a spill on REF_BASE now to update the stack slot. */
522 emit_lso(as, A64I_STRx, base, RID_SP, ra_spill(as, IR(REF_BASE)));
523 emit_setgl(as, base, jit_base);
524 emit_addptr(as, base, -8*delta);
525 asm_guardcc(as, CC_NE);
526 emit_nm(as, A64I_CMPx, RID_TMP,
527 ra_allock(as, i64ptr(pc), rset_exclude(RSET_GPR, base)));
528 emit_lso(as, A64I_LDRx, RID_TMP, base, -8);
529}
530
531/* -- Buffer operations --------------------------------------------------- */
532
533#if LJ_HASBUFFER
534static void asm_bufhdr_write(ASMState *as, Reg sb)
535{
536 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
537 IRIns irgc;
538 irgc.ot = IRT(0, IRT_PGC); /* GC type. */
539 emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
540 emit_dn(as, A64I_BFMx | A64F_IMMS(lj_fls(SBUF_MASK_FLAG)) | A64F_IMMR(0), RID_TMP, tmp);
541 emit_getgl(as, RID_TMP, cur_L);
542 emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
543}
544#endif
545
546/* -- Type conversions ---------------------------------------------------- */
547
548static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
549{
550 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
551 Reg dest = ra_dest(as, ir, RSET_GPR);
552 asm_guardcc(as, CC_NE);
553 emit_nm(as, A64I_FCMPd, (tmp & 31), (left & 31));
554 emit_dn(as, A64I_FCVT_F64_S32, (tmp & 31), dest);
555 emit_dn(as, A64I_FCVT_S32_F64, dest, (left & 31));
556}
557
558static void asm_tobit(ASMState *as, IRIns *ir)
559{
560 RegSet allow = RSET_FPR;
561 Reg left = ra_alloc1(as, ir->op1, allow);
562 Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
563 Reg tmp = ra_scratch(as, rset_clear(allow, right));
564 Reg dest = ra_dest(as, ir, RSET_GPR);
565 emit_dn(as, A64I_FMOV_R_S, dest, (tmp & 31));
566 emit_dnm(as, A64I_FADDd, (tmp & 31), (left & 31), (right & 31));
567}
568
569static void asm_conv(ASMState *as, IRIns *ir)
570{
571 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
572 int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
573 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
574 IRRef lref = ir->op1;
575 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
576 if (irt_isfp(ir->t)) {
577 Reg dest = ra_dest(as, ir, RSET_FPR);
578 if (stfp) { /* FP to FP conversion. */
579 emit_dn(as, st == IRT_NUM ? A64I_FCVT_F32_F64 : A64I_FCVT_F64_F32,
580 (dest & 31), (ra_alloc1(as, lref, RSET_FPR) & 31));
581 } else { /* Integer to FP conversion. */
582 Reg left = ra_alloc1(as, lref, RSET_GPR);
583 A64Ins ai = irt_isfloat(ir->t) ?
584 (((IRT_IS64 >> st) & 1) ?
585 (st == IRT_I64 ? A64I_FCVT_F32_S64 : A64I_FCVT_F32_U64) :
586 (st == IRT_INT ? A64I_FCVT_F32_S32 : A64I_FCVT_F32_U32)) :
587 (((IRT_IS64 >> st) & 1) ?
588 (st == IRT_I64 ? A64I_FCVT_F64_S64 : A64I_FCVT_F64_U64) :
589 (st == IRT_INT ? A64I_FCVT_F64_S32 : A64I_FCVT_F64_U32));
590 emit_dn(as, ai, (dest & 31), left);
591 }
592 } else if (stfp) { /* FP to integer conversion. */
593 if (irt_isguard(ir->t)) {
594 /* Checked conversions are only supported from number to int. */
595 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
596 "bad type for checked CONV");
597 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
598 } else {
599 Reg left = ra_alloc1(as, lref, RSET_FPR);
600 Reg dest = ra_dest(as, ir, RSET_GPR);
601 A64Ins ai = irt_is64(ir->t) ?
602 (st == IRT_NUM ?
603 (irt_isi64(ir->t) ? A64I_FCVT_S64_F64 : A64I_FCVT_U64_F64) :
604 (irt_isi64(ir->t) ? A64I_FCVT_S64_F32 : A64I_FCVT_U64_F32)) :
605 (st == IRT_NUM ?
606 (irt_isint(ir->t) ? A64I_FCVT_S32_F64 : A64I_FCVT_U32_F64) :
607 (irt_isint(ir->t) ? A64I_FCVT_S32_F32 : A64I_FCVT_U32_F32));
608 emit_dn(as, ai, dest, (left & 31));
609 }
610 } else if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
611 Reg dest = ra_dest(as, ir, RSET_GPR);
612 Reg left = ra_alloc1(as, lref, RSET_GPR);
613 A64Ins ai = st == IRT_I8 ? A64I_SXTBw :
614 st == IRT_U8 ? A64I_UXTBw :
615 st == IRT_I16 ? A64I_SXTHw : A64I_UXTHw;
616 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
617 emit_dn(as, ai, dest, left);
618 } else {
619 Reg dest = ra_dest(as, ir, RSET_GPR);
620 if (irt_is64(ir->t)) {
621 if (st64 || !(ir->op2 & IRCONV_SEXT)) {
622 /* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
623 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
624 } else { /* 32 to 64 bit sign extension. */
625 Reg left = ra_alloc1(as, lref, RSET_GPR);
626 emit_dn(as, A64I_SXTW, dest, left);
627 }
628 } else {
629 if (st64 && !(ir->op2 & IRCONV_NONE)) {
630 /* This is either a 32 bit reg/reg mov which zeroes the hiword
631 ** or a load of the loword from a 64 bit address.
632 */
633 Reg left = ra_alloc1(as, lref, RSET_GPR);
634 emit_dm(as, A64I_MOVw, dest, left);
635 } else { /* 32/32 bit no-op (cast). */
636 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
637 }
638 }
639 }
640}
641
642static void asm_strto(ASMState *as, IRIns *ir)
643{
644 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
645 IRRef args[2];
646 Reg dest = 0, tmp;
647 int destused = ra_used(ir);
648 int32_t ofs = 0;
649 ra_evictset(as, RSET_SCRATCH);
650 if (destused) {
651 if (ra_hasspill(ir->s)) {
652 ofs = sps_scale(ir->s);
653 destused = 0;
654 if (ra_hasreg(ir->r)) {
655 ra_free(as, ir->r);
656 ra_modified(as, ir->r);
657 emit_spload(as, ir, ir->r, ofs);
658 }
659 } else {
660 dest = ra_dest(as, ir, RSET_FPR);
661 }
662 }
663 if (destused)
664 emit_lso(as, A64I_LDRd, (dest & 31), RID_SP, 0);
665 asm_guardcnb(as, A64I_CBZ, RID_RET);
666 args[0] = ir->op1; /* GCstr *str */
667 args[1] = ASMREF_TMP1; /* TValue *n */
668 asm_gencall(as, ci, args);
669 tmp = ra_releasetmp(as, ASMREF_TMP1);
670 emit_opk(as, A64I_ADDx, tmp, RID_SP, ofs, RSET_GPR);
671}
672
673/* -- Memory references --------------------------------------------------- */
674
675/* Store tagged value for ref at base+ofs. */
676static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
677{
678 RegSet allow = rset_exclude(RSET_GPR, base);
679 IRIns *ir = IR(ref);
680 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
681 "store of IR type %d", irt_type(ir->t));
682 if (irref_isk(ref)) {
683 TValue k;
684 lj_ir_kvalue(as->J->L, &k, ir);
685 emit_lso(as, A64I_STRx, ra_allock(as, k.u64, allow), base, ofs);
686 } else {
687 Reg src = ra_alloc1(as, ref, allow);
688 rset_clear(allow, src);
689 if (irt_isinteger(ir->t)) {
690 Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
691 emit_lso(as, A64I_STRx, RID_TMP, base, ofs);
692 emit_dnm(as, A64I_ADDx | A64F_EX(A64EX_UXTW), RID_TMP, type, src);
693 } else {
694 Reg type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
695 emit_lso(as, A64I_STRx, RID_TMP, base, ofs);
696 emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 47), RID_TMP, src, type);
697 }
698 }
699}
700
701/* Get pointer to TValue. */
702static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
703{
704 if ((mode & IRTMPREF_IN1)) {
705 IRIns *ir = IR(ref);
706 if (irt_isnum(ir->t)) {
707 if (irref_isk(ref) && !(mode & IRTMPREF_OUT1)) {
708 /* Use the number constant itself as a TValue. */
709 ra_allockreg(as, i64ptr(ir_knum(ir)), dest);
710 return;
711 }
712 emit_lso(as, A64I_STRd, (ra_alloc1(as, ref, RSET_FPR) & 31), dest, 0);
713 } else {
714 asm_tvstore64(as, dest, 0, ref);
715 }
716 }
717 /* g->tmptv holds the TValue(s). */
718 emit_dn(as, A64I_ADDx^emit_isk12(glofs(as, &J2G(as->J)->tmptv)), dest, RID_GL);
719}
720
721static void asm_aref(ASMState *as, IRIns *ir)
722{
723 Reg dest = ra_dest(as, ir, RSET_GPR);
724 Reg idx, base;
725 if (irref_isk(ir->op2)) {
726 IRRef tab = IR(ir->op1)->op1;
727 int32_t ofs = asm_fuseabase(as, tab);
728 IRRef refa = ofs ? tab : ir->op1;
729 uint32_t k = emit_isk12(ofs + 8*IR(ir->op2)->i);
730 if (k) {
731 base = ra_alloc1(as, refa, RSET_GPR);
732 emit_dn(as, A64I_ADDx^k, dest, base);
733 return;
734 }
735 }
736 base = ra_alloc1(as, ir->op1, RSET_GPR);
737 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
738 emit_dnm(as, A64I_ADDx | A64F_EXSH(A64EX_UXTW, 3), dest, base, idx);
739}
740
741/* Inlined hash lookup. Specialized for key type and for const keys.
742** The equivalent C code is:
743** Node *n = hashkey(t, key);
744** do {
745** if (lj_obj_equal(&n->key, key)) return &n->val;
746** } while ((n = nextnode(n)));
747** return niltv(L);
748*/
749static void asm_href(ASMState *as, IRIns *ir, IROp merge)
750{
751 RegSet allow = RSET_GPR;
752 int destused = ra_used(ir);
753 Reg dest = ra_dest(as, ir, allow);
754 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
755 Reg key = 0, tmp = RID_TMP;
756 Reg ftmp = RID_NONE, type = RID_NONE, scr = RID_NONE, tisnum = RID_NONE;
757 IRRef refkey = ir->op2;
758 IRIns *irkey = IR(refkey);
759 int isk = irref_isk(ir->op2);
760 IRType1 kt = irkey->t;
761 uint32_t k = 0;
762 uint32_t khash;
763 MCLabel l_end, l_loop, l_next;
764 rset_clear(allow, tab);
765
766 if (!isk) {
767 key = ra_alloc1(as, ir->op2, irt_isnum(kt) ? RSET_FPR : allow);
768 rset_clear(allow, key);
769 if (!irt_isstr(kt)) {
770 tmp = ra_scratch(as, allow);
771 rset_clear(allow, tmp);
772 }
773 } else if (irt_isnum(kt)) {
774 int64_t val = (int64_t)ir_knum(irkey)->u64;
775 if (!(k = emit_isk12(val))) {
776 key = ra_allock(as, val, allow);
777 rset_clear(allow, key);
778 }
779 } else if (!irt_ispri(kt)) {
780 if (!(k = emit_isk12(irkey->i))) {
781 key = ra_alloc1(as, refkey, allow);
782 rset_clear(allow, key);
783 }
784 }
785
786 /* Allocate constants early. */
787 if (irt_isnum(kt)) {
788 if (!isk) {
789 tisnum = ra_allock(as, LJ_TISNUM << 15, allow);
790 ftmp = ra_scratch(as, rset_exclude(RSET_FPR, key));
791 rset_clear(allow, tisnum);
792 }
793 } else if (irt_isaddr(kt)) {
794 if (isk) {
795 int64_t kk = ((int64_t)irt_toitype(kt) << 47) | irkey[1].tv.u64;
796 scr = ra_allock(as, kk, allow);
797 } else {
798 scr = ra_scratch(as, allow);
799 }
800 rset_clear(allow, scr);
801 } else {
802 lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
803 type = ra_allock(as, ~((int64_t)~irt_toitype(kt) << 47), allow);
804 scr = ra_scratch(as, rset_clear(allow, type));
805 rset_clear(allow, scr);
806 }
807
808 /* Key not found in chain: jump to exit (if merged) or load niltv. */
809 l_end = emit_label(as);
810 as->invmcp = NULL;
811 if (merge == IR_NE)
812 asm_guardcc(as, CC_AL);
813 else if (destused)
814 emit_loada(as, dest, niltvg(J2G(as->J)));
815
816 /* Follow hash chain until the end. */
817 l_loop = --as->mcp;
818 emit_n(as, A64I_CMPx^A64I_K12^0, dest);
819 emit_lso(as, A64I_LDRx, dest, dest, offsetof(Node, next));
820 l_next = emit_label(as);
821
822 /* Type and value comparison. */
823 if (merge == IR_EQ)
824 asm_guardcc(as, CC_EQ);
825 else
826 emit_cond_branch(as, CC_EQ, l_end);
827
828 if (irt_isnum(kt)) {
829 if (isk) {
830 /* Assumes -0.0 is already canonicalized to +0.0. */
831 if (k)
832 emit_n(as, A64I_CMPx^k, tmp);
833 else
834 emit_nm(as, A64I_CMPx, key, tmp);
835 emit_lso(as, A64I_LDRx, tmp, dest, offsetof(Node, key.u64));
836 } else {
837 emit_nm(as, A64I_FCMPd, key, ftmp);
838 emit_dn(as, A64I_FMOV_D_R, (ftmp & 31), (tmp & 31));
839 emit_cond_branch(as, CC_LO, l_next);
840 emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32), tisnum, tmp);
841 emit_lso(as, A64I_LDRx, tmp, dest, offsetof(Node, key.n));
842 }
843 } else if (irt_isaddr(kt)) {
844 if (isk) {
845 emit_nm(as, A64I_CMPx, scr, tmp);
846 emit_lso(as, A64I_LDRx, tmp, dest, offsetof(Node, key.u64));
847 } else {
848 emit_nm(as, A64I_CMPx, tmp, scr);
849 emit_lso(as, A64I_LDRx, scr, dest, offsetof(Node, key.u64));
850 }
851 } else {
852 emit_nm(as, A64I_CMPx, scr, type);
853 emit_lso(as, A64I_LDRx, scr, dest, offsetof(Node, key));
854 }
855
856 *l_loop = A64I_BCC | A64F_S19(as->mcp - l_loop) | CC_NE;
857 if (!isk && irt_isaddr(kt)) {
858 type = ra_allock(as, (int32_t)irt_toitype(kt), allow);
859 emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 47), tmp, key, type);
860 rset_clear(allow, type);
861 }
862 /* Load main position relative to tab->node into dest. */
863 khash = isk ? ir_khash(as, irkey) : 1;
864 if (khash == 0) {
865 emit_lso(as, A64I_LDRx, dest, tab, offsetof(GCtab, node));
866 } else {
867 emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 3), dest, tmp, dest);
868 emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 1), dest, dest, dest);
869 emit_lso(as, A64I_LDRx, tmp, tab, offsetof(GCtab, node));
870 if (isk) {
871 Reg tmphash = ra_allock(as, khash, allow);
872 emit_dnm(as, A64I_ANDw, dest, dest, tmphash);
873 emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask));
874 } else if (irt_isstr(kt)) {
875 /* Fetch of str->sid is cheaper than ra_allock. */
876 emit_dnm(as, A64I_ANDw, dest, dest, tmp);
877 emit_lso(as, A64I_LDRw, tmp, key, offsetof(GCstr, sid));
878 emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask));
879 } else { /* Must match with hash*() in lj_tab.c. */
880 emit_dnm(as, A64I_ANDw, dest, dest, tmp);
881 emit_lso(as, A64I_LDRw, tmp, tab, offsetof(GCtab, hmask));
882 emit_dnm(as, A64I_SUBw, dest, dest, tmp);
883 emit_dnm(as, A64I_EXTRw | (A64F_IMMS(32-HASH_ROT3)), tmp, tmp, tmp);
884 emit_dnm(as, A64I_EORw, dest, dest, tmp);
885 emit_dnm(as, A64I_EXTRw | (A64F_IMMS(32-HASH_ROT2)), dest, dest, dest);
886 emit_dnm(as, A64I_SUBw, tmp, tmp, dest);
887 emit_dnm(as, A64I_EXTRw | (A64F_IMMS(32-HASH_ROT1)), dest, dest, dest);
888 emit_dnm(as, A64I_EORw, tmp, tmp, dest);
889 if (irt_isnum(kt)) {
890 emit_dnm(as, A64I_ADDw, dest, dest, dest);
891 emit_dn(as, A64I_LSRx | A64F_IMMR(32)|A64F_IMMS(32), dest, dest);
892 emit_dm(as, A64I_MOVw, tmp, dest);
893 emit_dn(as, A64I_FMOV_R_D, dest, (key & 31));
894 } else {
895 checkmclim(as);
896 emit_dm(as, A64I_MOVw, tmp, key);
897 emit_dnm(as, A64I_EORw, dest, dest,
898 ra_allock(as, irt_toitype(kt) << 15, allow));
899 emit_dn(as, A64I_LSRx | A64F_IMMR(32)|A64F_IMMS(32), dest, dest);
900 emit_dm(as, A64I_MOVx, dest, key);
901 }
902 }
903 }
904}
905
906static void asm_hrefk(ASMState *as, IRIns *ir)
907{
908 IRIns *kslot = IR(ir->op2);
909 IRIns *irkey = IR(kslot->op1);
910 int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
911 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
912 int bigofs = !emit_checkofs(A64I_LDRx, kofs);
913 Reg dest = (ra_used(ir) || bigofs) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
914 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
915 Reg key, idx = node;
916 RegSet allow = rset_exclude(RSET_GPR, node);
917 uint64_t k;
918 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
919 if (bigofs) {
920 idx = dest;
921 rset_clear(allow, dest);
922 kofs = (int32_t)offsetof(Node, key);
923 } else if (ra_hasreg(dest)) {
924 emit_opk(as, A64I_ADDx, dest, node, ofs, allow);
925 }
926 asm_guardcc(as, CC_NE);
927 if (irt_ispri(irkey->t)) {
928 k = ~((int64_t)~irt_toitype(irkey->t) << 47);
929 } else if (irt_isnum(irkey->t)) {
930 k = ir_knum(irkey)->u64;
931 } else {
932 k = ((uint64_t)irt_toitype(irkey->t) << 47) | (uint64_t)ir_kgc(irkey);
933 }
934 key = ra_scratch(as, allow);
935 emit_nm(as, A64I_CMPx, key, ra_allock(as, k, rset_exclude(allow, key)));
936 emit_lso(as, A64I_LDRx, key, idx, kofs);
937 if (bigofs)
938 emit_opk(as, A64I_ADDx, dest, node, ofs, rset_exclude(RSET_GPR, node));
939}
940
941static void asm_uref(ASMState *as, IRIns *ir)
942{
943 Reg dest = ra_dest(as, ir, RSET_GPR);
944 if (irref_isk(ir->op1)) {
945 GCfunc *fn = ir_kfunc(IR(ir->op1));
946 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
947 emit_lsptr(as, A64I_LDRx, dest, v);
948 } else {
949 Reg uv = ra_scratch(as, RSET_GPR);
950 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
951 if (ir->o == IR_UREFC) {
952 asm_guardcc(as, CC_NE);
953 emit_n(as, (A64I_CMPx^A64I_K12) | A64F_U12(1), RID_TMP);
954 emit_opk(as, A64I_ADDx, dest, uv,
955 (int32_t)offsetof(GCupval, tv), RSET_GPR);
956 emit_lso(as, A64I_LDRB, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
957 } else {
958 emit_lso(as, A64I_LDRx, dest, uv, (int32_t)offsetof(GCupval, v));
959 }
960 emit_lso(as, A64I_LDRx, uv, func,
961 (int32_t)offsetof(GCfuncL, uvptr) + 8*(int32_t)(ir->op2 >> 8));
962 }
963}
964
965static void asm_fref(ASMState *as, IRIns *ir)
966{
967 UNUSED(as); UNUSED(ir);
968 lj_assertA(!ra_used(ir), "unfused FREF");
969}
970
971static void asm_strref(ASMState *as, IRIns *ir)
972{
973 RegSet allow = RSET_GPR;
974 Reg dest = ra_dest(as, ir, allow);
975 Reg base = ra_alloc1(as, ir->op1, allow);
976 IRIns *irr = IR(ir->op2);
977 int32_t ofs = sizeof(GCstr);
978 uint32_t m;
979 rset_clear(allow, base);
980 if (irref_isk(ir->op2) && (m = emit_isk12(ofs + irr->i))) {
981 emit_dn(as, A64I_ADDx^m, dest, base);
982 } else {
983 emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(ofs), dest, dest);
984 emit_dnm(as, A64I_ADDx, dest, base, ra_alloc1(as, ir->op2, allow));
985 }
986}
987
988/* -- Loads and stores ---------------------------------------------------- */
989
990static A64Ins asm_fxloadins(IRIns *ir)
991{
992 switch (irt_type(ir->t)) {
993 case IRT_I8: return A64I_LDRB ^ A64I_LS_S;
994 case IRT_U8: return A64I_LDRB;
995 case IRT_I16: return A64I_LDRH ^ A64I_LS_S;
996 case IRT_U16: return A64I_LDRH;
997 case IRT_NUM: return A64I_LDRd;
998 case IRT_FLOAT: return A64I_LDRs;
999 default: return irt_is64(ir->t) ? A64I_LDRx : A64I_LDRw;
1000 }
1001}
1002
1003static A64Ins asm_fxstoreins(IRIns *ir)
1004{
1005 switch (irt_type(ir->t)) {
1006 case IRT_I8: case IRT_U8: return A64I_STRB;
1007 case IRT_I16: case IRT_U16: return A64I_STRH;
1008 case IRT_NUM: return A64I_STRd;
1009 case IRT_FLOAT: return A64I_STRs;
1010 default: return irt_is64(ir->t) ? A64I_STRx : A64I_STRw;
1011 }
1012}
1013
1014static void asm_fload(ASMState *as, IRIns *ir)
1015{
1016 Reg dest = ra_dest(as, ir, RSET_GPR);
1017 Reg idx;
1018 A64Ins ai = asm_fxloadins(ir);
1019 int32_t ofs;
1020 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
1021 idx = RID_GL;
1022 ofs = (ir->op2 << 2) - GG_OFS(g);
1023 } else {
1024 idx = ra_alloc1(as, ir->op1, RSET_GPR);
1025 if (ir->op2 == IRFL_TAB_ARRAY) {
1026 ofs = asm_fuseabase(as, ir->op1);
1027 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
1028 emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(ofs), dest, idx);
1029 return;
1030 }
1031 }
1032 ofs = field_ofs[ir->op2];
1033 }
1034 emit_lso(as, ai, (dest & 31), idx, ofs);
1035}
1036
1037static void asm_fstore(ASMState *as, IRIns *ir)
1038{
1039 if (ir->r != RID_SINK) {
1040 Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
1041 IRIns *irf = IR(ir->op1);
1042 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
1043 int32_t ofs = field_ofs[irf->op2];
1044 emit_lso(as, asm_fxstoreins(ir), (src & 31), idx, ofs);
1045 }
1046}
1047
1048static void asm_xload(ASMState *as, IRIns *ir)
1049{
1050 Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
1051 lj_assertA(!(ir->op2 & IRXLOAD_UNALIGNED), "unaligned XLOAD");
1052 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR);
1053}
1054
1055static void asm_xstore(ASMState *as, IRIns *ir)
1056{
1057 if (ir->r != RID_SINK) {
1058 Reg src = ra_alloc1(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
1059 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
1060 rset_exclude(RSET_GPR, src));
1061 }
1062}
1063
1064static void asm_ahuvload(ASMState *as, IRIns *ir)
1065{
1066 Reg idx, tmp, type;
1067 int32_t ofs = 0;
1068 RegSet gpr = RSET_GPR, allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
1069 lj_assertA(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
1070 irt_isint(ir->t),
1071 "bad load type %d", irt_type(ir->t));
1072 if (ra_used(ir)) {
1073 Reg dest = ra_dest(as, ir, allow);
1074 tmp = irt_isnum(ir->t) ? ra_scratch(as, rset_clear(gpr, dest)) : dest;
1075 if (irt_isaddr(ir->t)) {
1076 emit_dn(as, A64I_ANDx^emit_isk13(LJ_GCVMASK, 1), dest, dest);
1077 } else if (irt_isnum(ir->t)) {
1078 emit_dn(as, A64I_FMOV_D_R, (dest & 31), tmp);
1079 } else if (irt_isint(ir->t)) {
1080 emit_dm(as, A64I_MOVw, dest, dest);
1081 }
1082 } else {
1083 tmp = ra_scratch(as, gpr);
1084 }
1085 type = ra_scratch(as, rset_clear(gpr, tmp));
1086 idx = asm_fuseahuref(as, ir->op1, &ofs, rset_clear(gpr, type), A64I_LDRx);
1087 if (ir->o == IR_VLOAD) ofs += 8 * ir->op2;
1088 /* Always do the type check, even if the load result is unused. */
1089 asm_guardcc(as, irt_isnum(ir->t) ? CC_LS : CC_NE);
1090 if (irt_type(ir->t) >= IRT_NUM) {
1091 lj_assertA(irt_isinteger(ir->t) || irt_isnum(ir->t),
1092 "bad load type %d", irt_type(ir->t));
1093 emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1094 ra_allock(as, LJ_TISNUM << 15, rset_exclude(gpr, idx)), tmp);
1095 } else if (irt_isaddr(ir->t)) {
1096 emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(-irt_toitype(ir->t)), type);
1097 emit_dn(as, A64I_ASRx | A64F_IMMR(47), type, tmp);
1098 } else if (irt_isnil(ir->t)) {
1099 emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(1), tmp);
1100 } else {
1101 emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1102 ra_allock(as, (irt_toitype(ir->t) << 15) | 0x7fff, gpr), tmp);
1103 }
1104 if (ofs & FUSE_REG)
1105 emit_dnm(as, (A64I_LDRx^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, tmp, idx, (ofs & 31));
1106 else
1107 emit_lso(as, A64I_LDRx, tmp, idx, ofs);
1108}
1109
1110static void asm_ahustore(ASMState *as, IRIns *ir)
1111{
1112 if (ir->r != RID_SINK) {
1113 RegSet allow = RSET_GPR;
1114 Reg idx, src = RID_NONE, tmp = RID_TMP, type = RID_NONE;
1115 int32_t ofs = 0;
1116 if (irt_isnum(ir->t)) {
1117 src = ra_alloc1(as, ir->op2, RSET_FPR);
1118 idx = asm_fuseahuref(as, ir->op1, &ofs, allow, A64I_STRd);
1119 if (ofs & FUSE_REG)
1120 emit_dnm(as, (A64I_STRd^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, (src & 31), idx, (ofs &31));
1121 else
1122 emit_lso(as, A64I_STRd, (src & 31), idx, ofs);
1123 } else {
1124 if (!irt_ispri(ir->t)) {
1125 src = ra_alloc1(as, ir->op2, allow);
1126 rset_clear(allow, src);
1127 if (irt_isinteger(ir->t))
1128 type = ra_allock(as, (uint64_t)(int32_t)LJ_TISNUM << 47, allow);
1129 else
1130 type = ra_allock(as, irt_toitype(ir->t), allow);
1131 } else {
1132 tmp = type = ra_allock(as, ~((int64_t)~irt_toitype(ir->t)<<47), allow);
1133 }
1134 idx = asm_fuseahuref(as, ir->op1, &ofs, rset_exclude(allow, type),
1135 A64I_STRx);
1136 if (ofs & FUSE_REG)
1137 emit_dnm(as, (A64I_STRx^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, tmp, idx, (ofs & 31));
1138 else
1139 emit_lso(as, A64I_STRx, tmp, idx, ofs);
1140 if (ra_hasreg(src)) {
1141 if (irt_isinteger(ir->t)) {
1142 emit_dnm(as, A64I_ADDx | A64F_EX(A64EX_UXTW), tmp, type, src);
1143 } else {
1144 emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 47), tmp, src, type);
1145 }
1146 }
1147 }
1148 }
1149}
1150
1151static void asm_sload(ASMState *as, IRIns *ir)
1152{
1153 int32_t ofs = 8*((int32_t)ir->op1-2);
1154 IRType1 t = ir->t;
1155 Reg dest = RID_NONE, base;
1156 RegSet allow = RSET_GPR;
1157 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1158 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1159 lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1160 "inconsistent SLOAD variant");
1161 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1162 dest = ra_scratch(as, RSET_FPR);
1163 asm_tointg(as, ir, dest);
1164 t.irt = IRT_NUM; /* Continue with a regular number type check. */
1165 } else if (ra_used(ir)) {
1166 Reg tmp = RID_NONE;
1167 if ((ir->op2 & IRSLOAD_CONVERT))
1168 tmp = ra_scratch(as, irt_isint(t) ? RSET_FPR : RSET_GPR);
1169 lj_assertA((irt_isnum(t)) || irt_isint(t) || irt_isaddr(t),
1170 "bad SLOAD type %d", irt_type(t));
1171 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : allow);
1172 base = ra_alloc1(as, REF_BASE, rset_clear(allow, dest));
1173 if (irt_isaddr(t)) {
1174 emit_dn(as, A64I_ANDx^emit_isk13(LJ_GCVMASK, 1), dest, dest);
1175 } else if ((ir->op2 & IRSLOAD_CONVERT)) {
1176 if (irt_isint(t)) {
1177 emit_dn(as, A64I_FCVT_S32_F64, dest, (tmp & 31));
1178 /* If value is already loaded for type check, move it to FPR. */
1179 if ((ir->op2 & IRSLOAD_TYPECHECK))
1180 emit_dn(as, A64I_FMOV_D_R, (tmp & 31), dest);
1181 else
1182 dest = tmp;
1183 t.irt = IRT_NUM; /* Check for original type. */
1184 } else {
1185 emit_dn(as, A64I_FCVT_F64_S32, (dest & 31), tmp);
1186 dest = tmp;
1187 t.irt = IRT_INT; /* Check for original type. */
1188 }
1189 } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
1190 emit_dm(as, A64I_MOVw, dest, dest);
1191 }
1192 goto dotypecheck;
1193 }
1194 base = ra_alloc1(as, REF_BASE, allow);
1195dotypecheck:
1196 rset_clear(allow, base);
1197 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1198 Reg tmp;
1199 if (ra_hasreg(dest) && rset_test(RSET_GPR, dest)) {
1200 tmp = dest;
1201 } else {
1202 tmp = ra_scratch(as, allow);
1203 rset_clear(allow, tmp);
1204 }
1205 if (ra_hasreg(dest) && tmp != dest)
1206 emit_dn(as, A64I_FMOV_D_R, (dest & 31), tmp);
1207 /* Need type check, even if the load result is unused. */
1208 asm_guardcc(as, irt_isnum(t) ? CC_LS : CC_NE);
1209 if (irt_type(t) >= IRT_NUM) {
1210 lj_assertA(irt_isinteger(t) || irt_isnum(t),
1211 "bad SLOAD type %d", irt_type(t));
1212 emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1213 ra_allock(as, (ir->op2 & IRSLOAD_KEYINDEX) ? LJ_KEYINDEX : (LJ_TISNUM << 15), allow), tmp);
1214 } else if (irt_isnil(t)) {
1215 emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(1), tmp);
1216 } else if (irt_ispri(t)) {
1217 emit_nm(as, A64I_CMPx,
1218 ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow), tmp);
1219 } else {
1220 Reg type = ra_scratch(as, allow);
1221 emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(-irt_toitype(t)), type);
1222 emit_dn(as, A64I_ASRx | A64F_IMMR(47), type, tmp);
1223 }
1224 emit_lso(as, A64I_LDRx, tmp, base, ofs);
1225 return;
1226 }
1227 if (ra_hasreg(dest)) {
1228 emit_lso(as, irt_isnum(t) ? A64I_LDRd :
1229 (irt_isint(t) ? A64I_LDRw : A64I_LDRx), (dest & 31), base,
1230 ofs ^ ((LJ_BE && irt_isint(t) ? 4 : 0)));
1231 }
1232}
1233
1234/* -- Allocations --------------------------------------------------------- */
1235
1236#if LJ_HASFFI
1237static void asm_cnew(ASMState *as, IRIns *ir)
1238{
1239 CTState *cts = ctype_ctsG(J2G(as->J));
1240 CTypeID id = (CTypeID)IR(ir->op1)->i;
1241 CTSize sz;
1242 CTInfo info = lj_ctype_info(cts, id, &sz);
1243 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1244 IRRef args[4];
1245 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1246 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1247 "bad CNEW/CNEWI operands");
1248
1249 as->gcsteps++;
1250 asm_setupresult(as, ir, ci); /* GCcdata * */
1251 /* Initialize immutable cdata object. */
1252 if (ir->o == IR_CNEWI) {
1253 int32_t ofs = sizeof(GCcdata);
1254 Reg r = ra_alloc1(as, ir->op2, allow);
1255 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1256 emit_lso(as, sz == 8 ? A64I_STRx : A64I_STRw, r, RID_RET, ofs);
1257 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1258 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1259 args[0] = ASMREF_L; /* lua_State *L */
1260 args[1] = ir->op1; /* CTypeID id */
1261 args[2] = ir->op2; /* CTSize sz */
1262 args[3] = ASMREF_TMP1; /* CTSize align */
1263 asm_gencall(as, ci, args);
1264 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1265 return;
1266 }
1267
1268 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1269 {
1270 Reg r = (id < 65536) ? RID_X1 : ra_allock(as, id, allow);
1271 emit_lso(as, A64I_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct));
1272 emit_lso(as, A64I_STRH, r, RID_RET, offsetof(GCcdata, ctypeid));
1273 emit_d(as, A64I_MOVZw | A64F_U16(~LJ_TCDATA), RID_TMP);
1274 if (id < 65536) emit_d(as, A64I_MOVZw | A64F_U16(id), RID_X1);
1275 }
1276 args[0] = ASMREF_L; /* lua_State *L */
1277 args[1] = ASMREF_TMP1; /* MSize size */
1278 asm_gencall(as, ci, args);
1279 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1280 ra_releasetmp(as, ASMREF_TMP1));
1281}
1282#endif
1283
1284/* -- Write barriers ------------------------------------------------------ */
1285
1286static void asm_tbar(ASMState *as, IRIns *ir)
1287{
1288 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1289 Reg link = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1290 Reg mark = RID_TMP;
1291 MCLabel l_end = emit_label(as);
1292 emit_lso(as, A64I_STRx, link, tab, (int32_t)offsetof(GCtab, gclist));
1293 emit_lso(as, A64I_STRB, mark, tab, (int32_t)offsetof(GCtab, marked));
1294 emit_setgl(as, tab, gc.grayagain);
1295 emit_dn(as, A64I_ANDw^emit_isk13(~LJ_GC_BLACK, 0), mark, mark);
1296 emit_getgl(as, link, gc.grayagain);
1297 emit_cond_branch(as, CC_EQ, l_end);
1298 emit_n(as, A64I_TSTw^emit_isk13(LJ_GC_BLACK, 0), mark);
1299 emit_lso(as, A64I_LDRB, mark, tab, (int32_t)offsetof(GCtab, marked));
1300}
1301
1302static void asm_obar(ASMState *as, IRIns *ir)
1303{
1304 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1305 IRRef args[2];
1306 MCLabel l_end;
1307 RegSet allow = RSET_GPR;
1308 Reg obj, val, tmp;
1309 /* No need for other object barriers (yet). */
1310 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1311 ra_evictset(as, RSET_SCRATCH);
1312 l_end = emit_label(as);
1313 args[0] = ASMREF_TMP1; /* global_State *g */
1314 args[1] = ir->op1; /* TValue *tv */
1315 asm_gencall(as, ci, args);
1316 emit_dm(as, A64I_MOVx, ra_releasetmp(as, ASMREF_TMP1), RID_GL);
1317 obj = IR(ir->op1)->r;
1318 tmp = ra_scratch(as, rset_exclude(allow, obj));
1319 emit_cond_branch(as, CC_EQ, l_end);
1320 emit_n(as, A64I_TSTw^emit_isk13(LJ_GC_BLACK, 0), tmp);
1321 emit_cond_branch(as, CC_EQ, l_end);
1322 emit_n(as, A64I_TSTw^emit_isk13(LJ_GC_WHITES, 0), RID_TMP);
1323 val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1324 emit_lso(as, A64I_LDRB, tmp, obj,
1325 (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1326 emit_lso(as, A64I_LDRB, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1327}
1328
1329/* -- Arithmetic and logic operations ------------------------------------- */
1330
1331static void asm_fparith(ASMState *as, IRIns *ir, A64Ins ai)
1332{
1333 Reg dest = ra_dest(as, ir, RSET_FPR);
1334 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1335 right = (left >> 8); left &= 255;
1336 emit_dnm(as, ai, (dest & 31), (left & 31), (right & 31));
1337}
1338
1339static void asm_fpunary(ASMState *as, IRIns *ir, A64Ins ai)
1340{
1341 Reg dest = ra_dest(as, ir, RSET_FPR);
1342 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1343 emit_dn(as, ai, (dest & 31), (left & 31));
1344}
1345
1346static void asm_fpmath(ASMState *as, IRIns *ir)
1347{
1348 IRFPMathOp fpm = (IRFPMathOp)ir->op2;
1349 if (fpm == IRFPM_SQRT) {
1350 asm_fpunary(as, ir, A64I_FSQRTd);
1351 } else if (fpm <= IRFPM_TRUNC) {
1352 asm_fpunary(as, ir, fpm == IRFPM_FLOOR ? A64I_FRINTMd :
1353 fpm == IRFPM_CEIL ? A64I_FRINTPd : A64I_FRINTZd);
1354 } else {
1355 asm_callid(as, ir, IRCALL_lj_vm_floor + fpm);
1356 }
1357}
1358
1359static int asm_swapops(ASMState *as, IRRef lref, IRRef rref)
1360{
1361 IRIns *ir;
1362 if (irref_isk(rref))
1363 return 0; /* Don't swap constants to the left. */
1364 if (irref_isk(lref))
1365 return 1; /* But swap constants to the right. */
1366 ir = IR(rref);
1367 if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR) ||
1368 (ir->o == IR_ADD && ir->op1 == ir->op2) ||
1369 (ir->o == IR_CONV && ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)))
1370 return 0; /* Don't swap fusable operands to the left. */
1371 ir = IR(lref);
1372 if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR) ||
1373 (ir->o == IR_ADD && ir->op1 == ir->op2) ||
1374 (ir->o == IR_CONV && ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)))
1375 return 1; /* But swap fusable operands to the right. */
1376 return 0; /* Otherwise don't swap. */
1377}
1378
1379static void asm_intop(ASMState *as, IRIns *ir, A64Ins ai)
1380{
1381 IRRef lref = ir->op1, rref = ir->op2;
1382 Reg left, dest = ra_dest(as, ir, RSET_GPR);
1383 uint32_t m;
1384 if ((ai & ~A64I_S) != A64I_SUBw && asm_swapops(as, lref, rref)) {
1385 IRRef tmp = lref; lref = rref; rref = tmp;
1386 }
1387 left = ra_hintalloc(as, lref, dest, RSET_GPR);
1388 if (irt_is64(ir->t)) ai |= A64I_X;
1389 m = asm_fuseopm(as, ai, rref, rset_exclude(RSET_GPR, left));
1390 if (irt_isguard(ir->t)) { /* For IR_ADDOV etc. */
1391 asm_guardcc(as, CC_VS);
1392 ai |= A64I_S;
1393 }
1394 emit_dn(as, ai^m, dest, left);
1395}
1396
1397static void asm_intop_s(ASMState *as, IRIns *ir, A64Ins ai)
1398{
1399 if (as->flagmcp == as->mcp) { /* Drop cmp r, #0. */
1400 as->flagmcp = NULL;
1401 as->mcp++;
1402 ai |= A64I_S;
1403 }
1404 asm_intop(as, ir, ai);
1405}
1406
1407static void asm_intneg(ASMState *as, IRIns *ir)
1408{
1409 Reg dest = ra_dest(as, ir, RSET_GPR);
1410 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1411 emit_dm(as, irt_is64(ir->t) ? A64I_NEGx : A64I_NEGw, dest, left);
1412}
1413
1414/* NYI: use add/shift for MUL(OV) with constants. FOLD only does 2^k. */
1415static void asm_intmul(ASMState *as, IRIns *ir)
1416{
1417 Reg dest = ra_dest(as, ir, RSET_GPR);
1418 Reg left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, dest));
1419 Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1420 if (irt_isguard(ir->t)) { /* IR_MULOV */
1421 asm_guardcc(as, CC_NE);
1422 emit_dm(as, A64I_MOVw, dest, dest); /* Zero-extend. */
1423 emit_nm(as, A64I_CMPw | A64F_SH(A64SH_ASR, 31), RID_TMP, dest);
1424 emit_dn(as, A64I_ASRx | A64F_IMMR(32), RID_TMP, dest);
1425 emit_dnm(as, A64I_SMULL, dest, right, left);
1426 } else {
1427 emit_dnm(as, irt_is64(ir->t) ? A64I_MULx : A64I_MULw, dest, left, right);
1428 }
1429}
1430
1431static void asm_add(ASMState *as, IRIns *ir)
1432{
1433 if (irt_isnum(ir->t)) {
1434 if (!asm_fusemadd(as, ir, A64I_FMADDd, A64I_FMADDd))
1435 asm_fparith(as, ir, A64I_FADDd);
1436 return;
1437 }
1438 asm_intop_s(as, ir, A64I_ADDw);
1439}
1440
1441static void asm_sub(ASMState *as, IRIns *ir)
1442{
1443 if (irt_isnum(ir->t)) {
1444 if (!asm_fusemadd(as, ir, A64I_FNMSUBd, A64I_FMSUBd))
1445 asm_fparith(as, ir, A64I_FSUBd);
1446 return;
1447 }
1448 asm_intop_s(as, ir, A64I_SUBw);
1449}
1450
1451static void asm_mul(ASMState *as, IRIns *ir)
1452{
1453 if (irt_isnum(ir->t)) {
1454 asm_fparith(as, ir, A64I_FMULd);
1455 return;
1456 }
1457 asm_intmul(as, ir);
1458}
1459
1460#define asm_addov(as, ir) asm_add(as, ir)
1461#define asm_subov(as, ir) asm_sub(as, ir)
1462#define asm_mulov(as, ir) asm_mul(as, ir)
1463
1464#define asm_fpdiv(as, ir) asm_fparith(as, ir, A64I_FDIVd)
1465#define asm_abs(as, ir) asm_fpunary(as, ir, A64I_FABS)
1466
1467static void asm_neg(ASMState *as, IRIns *ir)
1468{
1469 if (irt_isnum(ir->t)) {
1470 asm_fpunary(as, ir, A64I_FNEGd);
1471 return;
1472 }
1473 asm_intneg(as, ir);
1474}
1475
1476static void asm_band(ASMState *as, IRIns *ir)
1477{
1478 A64Ins ai = A64I_ANDw;
1479 if (asm_fuseandshift(as, ir))
1480 return;
1481 if (as->flagmcp == as->mcp) {
1482 /* Try to drop cmp r, #0. */
1483 as->flagmcp = NULL;
1484 as->mcp++;
1485 ai = A64I_ANDSw;
1486 }
1487 asm_intop(as, ir, ai);
1488}
1489
1490static void asm_borbxor(ASMState *as, IRIns *ir, A64Ins ai)
1491{
1492 IRRef lref = ir->op1, rref = ir->op2;
1493 IRIns *irl = IR(lref), *irr = IR(rref);
1494 if ((canfuse(as, irl) && irl->o == IR_BNOT && !irref_isk(rref)) ||
1495 (canfuse(as, irr) && irr->o == IR_BNOT && !irref_isk(lref))) {
1496 Reg left, dest = ra_dest(as, ir, RSET_GPR);
1497 uint32_t m;
1498 if (irl->o == IR_BNOT) {
1499 IRRef tmp = lref; lref = rref; rref = tmp;
1500 }
1501 left = ra_alloc1(as, lref, RSET_GPR);
1502 ai |= A64I_ON;
1503 if (irt_is64(ir->t)) ai |= A64I_X;
1504 m = asm_fuseopm(as, ai, IR(rref)->op1, rset_exclude(RSET_GPR, left));
1505 emit_dn(as, ai^m, dest, left);
1506 } else {
1507 asm_intop(as, ir, ai);
1508 }
1509}
1510
1511static void asm_bor(ASMState *as, IRIns *ir)
1512{
1513 if (asm_fuseorshift(as, ir))
1514 return;
1515 asm_borbxor(as, ir, A64I_ORRw);
1516}
1517
1518#define asm_bxor(as, ir) asm_borbxor(as, ir, A64I_EORw)
1519
1520static void asm_bnot(ASMState *as, IRIns *ir)
1521{
1522 A64Ins ai = A64I_MVNw;
1523 Reg dest = ra_dest(as, ir, RSET_GPR);
1524 uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
1525 if (irt_is64(ir->t)) ai |= A64I_X;
1526 emit_d(as, ai^m, dest);
1527}
1528
1529static void asm_bswap(ASMState *as, IRIns *ir)
1530{
1531 Reg dest = ra_dest(as, ir, RSET_GPR);
1532 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1533 emit_dn(as, irt_is64(ir->t) ? A64I_REVx : A64I_REVw, dest, left);
1534}
1535
1536static void asm_bitshift(ASMState *as, IRIns *ir, A64Ins ai, A64Shift sh)
1537{
1538 int32_t shmask = irt_is64(ir->t) ? 63 : 31;
1539 if (irref_isk(ir->op2)) { /* Constant shifts. */
1540 Reg left, dest = ra_dest(as, ir, RSET_GPR);
1541 int32_t shift = (IR(ir->op2)->i & shmask);
1542 IRIns *irl = IR(ir->op1);
1543 if (shmask == 63) ai += A64I_UBFMx - A64I_UBFMw;
1544
1545 /* Fuse BSHL + BSHR/BSAR into UBFM/SBFM aka UBFX/SBFX/UBFIZ/SBFIZ. */
1546 if ((sh == A64SH_LSR || sh == A64SH_ASR) && canfuse(as, irl)) {
1547 if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
1548 int32_t shift2 = (IR(irl->op2)->i & shmask);
1549 shift = ((shift - shift2) & shmask);
1550 shmask -= shift2;
1551 ir = irl;
1552 }
1553 }
1554
1555 left = ra_alloc1(as, ir->op1, RSET_GPR);
1556 switch (sh) {
1557 case A64SH_LSL:
1558 emit_dn(as, ai | A64F_IMMS(shmask-shift) |
1559 A64F_IMMR((shmask-shift+1)&shmask), dest, left);
1560 break;
1561 case A64SH_LSR: case A64SH_ASR:
1562 emit_dn(as, ai | A64F_IMMS(shmask) | A64F_IMMR(shift), dest, left);
1563 break;
1564 case A64SH_ROR:
1565 emit_dnm(as, ai | A64F_IMMS(shift), dest, left, left);
1566 break;
1567 }
1568 } else { /* Variable-length shifts. */
1569 Reg dest = ra_dest(as, ir, RSET_GPR);
1570 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1571 Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1572 emit_dnm(as, (shmask == 63 ? A64I_SHRx : A64I_SHRw) | A64F_BSH(sh), dest, left, right);
1573 }
1574}
1575
1576#define asm_bshl(as, ir) asm_bitshift(as, ir, A64I_UBFMw, A64SH_LSL)
1577#define asm_bshr(as, ir) asm_bitshift(as, ir, A64I_UBFMw, A64SH_LSR)
1578#define asm_bsar(as, ir) asm_bitshift(as, ir, A64I_SBFMw, A64SH_ASR)
1579#define asm_bror(as, ir) asm_bitshift(as, ir, A64I_EXTRw, A64SH_ROR)
1580#define asm_brol(as, ir) lj_assertA(0, "unexpected BROL")
1581
1582static void asm_intmin_max(ASMState *as, IRIns *ir, A64CC cc)
1583{
1584 Reg dest = ra_dest(as, ir, RSET_GPR);
1585 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1586 Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1587 emit_dnm(as, A64I_CSELw|A64F_CC(cc), dest, left, right);
1588 emit_nm(as, A64I_CMPw, left, right);
1589}
1590
1591static void asm_fpmin_max(ASMState *as, IRIns *ir, A64CC fcc)
1592{
1593 Reg dest = (ra_dest(as, ir, RSET_FPR) & 31);
1594 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1595 right = ((left >> 8) & 31); left &= 31;
1596 emit_dnm(as, A64I_FCSELd | A64F_CC(fcc), dest, right, left);
1597 emit_nm(as, A64I_FCMPd, left, right);
1598}
1599
1600static void asm_min_max(ASMState *as, IRIns *ir, A64CC cc, A64CC fcc)
1601{
1602 if (irt_isnum(ir->t))
1603 asm_fpmin_max(as, ir, fcc);
1604 else
1605 asm_intmin_max(as, ir, cc);
1606}
1607
1608#define asm_min(as, ir) asm_min_max(as, ir, CC_LT, CC_PL)
1609#define asm_max(as, ir) asm_min_max(as, ir, CC_GT, CC_LE)
1610
1611/* -- Comparisons --------------------------------------------------------- */
1612
1613/* Map of comparisons to flags. ORDER IR. */
1614static const uint8_t asm_compmap[IR_ABC+1] = {
1615 /* op FP swp int cc FP cc */
1616 /* LT */ CC_GE + (CC_HS << 4),
1617 /* GE x */ CC_LT + (CC_HI << 4),
1618 /* LE */ CC_GT + (CC_HI << 4),
1619 /* GT x */ CC_LE + (CC_HS << 4),
1620 /* ULT x */ CC_HS + (CC_LS << 4),
1621 /* UGE */ CC_LO + (CC_LO << 4),
1622 /* ULE x */ CC_HI + (CC_LO << 4),
1623 /* UGT */ CC_LS + (CC_LS << 4),
1624 /* EQ */ CC_NE + (CC_NE << 4),
1625 /* NE */ CC_EQ + (CC_EQ << 4),
1626 /* ABC */ CC_LS + (CC_LS << 4) /* Same as UGT. */
1627};
1628
1629/* FP comparisons. */
1630static void asm_fpcomp(ASMState *as, IRIns *ir)
1631{
1632 Reg left, right;
1633 A64Ins ai;
1634 int swp = ((ir->o ^ (ir->o >> 2)) & ~(ir->o >> 3) & 1);
1635 if (!swp && irref_isk(ir->op2) && ir_knum(IR(ir->op2))->u64 == 0) {
1636 left = (ra_alloc1(as, ir->op1, RSET_FPR) & 31);
1637 right = 0;
1638 ai = A64I_FCMPZd;
1639 } else {
1640 left = ra_alloc2(as, ir, RSET_FPR);
1641 if (swp) {
1642 right = (left & 31); left = ((left >> 8) & 31);
1643 } else {
1644 right = ((left >> 8) & 31); left &= 31;
1645 }
1646 ai = A64I_FCMPd;
1647 }
1648 asm_guardcc(as, (asm_compmap[ir->o] >> 4));
1649 emit_nm(as, ai, left, right);
1650}
1651
1652/* Integer comparisons. */
1653static void asm_intcomp(ASMState *as, IRIns *ir)
1654{
1655 A64CC oldcc, cc = (asm_compmap[ir->o] & 15);
1656 A64Ins ai = irt_is64(ir->t) ? A64I_CMPx : A64I_CMPw;
1657 IRRef lref = ir->op1, rref = ir->op2;
1658 Reg left;
1659 uint32_t m;
1660 int cmpprev0 = 0;
1661 lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) ||
1662 irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t),
1663 "bad comparison data type %d", irt_type(ir->t));
1664 if (asm_swapops(as, lref, rref)) {
1665 IRRef tmp = lref; lref = rref; rref = tmp;
1666 if (cc >= CC_GE) cc ^= 7; /* LT <-> GT, LE <-> GE */
1667 else if (cc > CC_NE) cc ^= 11; /* LO <-> HI, LS <-> HS */
1668 }
1669 oldcc = cc;
1670 if (irref_isk(rref) && get_k64val(as, rref) == 0) {
1671 IRIns *irl = IR(lref);
1672 if (cc == CC_GE) cc = CC_PL;
1673 else if (cc == CC_LT) cc = CC_MI;
1674 else if (cc > CC_NE) goto nocombine; /* Other conds don't work with tst. */
1675 cmpprev0 = (irl+1 == ir);
1676 /* Combine and-cmp-bcc into tbz/tbnz or and-cmp into tst. */
1677 if (cmpprev0 && irl->o == IR_BAND && !ra_used(irl)) {
1678 IRRef blref = irl->op1, brref = irl->op2;
1679 uint32_t m2 = 0;
1680 Reg bleft;
1681 if (asm_swapops(as, blref, brref)) {
1682 Reg tmp = blref; blref = brref; brref = tmp;
1683 }
1684 if (irref_isk(brref)) {
1685 uint64_t k = get_k64val(as, brref);
1686 if (k && !(k & (k-1)) && (cc == CC_EQ || cc == CC_NE)) {
1687 asm_guardtnb(as, cc == CC_EQ ? A64I_TBZ : A64I_TBNZ,
1688 ra_alloc1(as, blref, RSET_GPR), emit_ctz64(k));
1689 return;
1690 }
1691 m2 = emit_isk13(k, irt_is64(irl->t));
1692 }
1693 bleft = ra_alloc1(as, blref, RSET_GPR);
1694 ai = (irt_is64(irl->t) ? A64I_TSTx : A64I_TSTw);
1695 if (!m2)
1696 m2 = asm_fuseopm(as, ai, brref, rset_exclude(RSET_GPR, bleft));
1697 asm_guardcc(as, cc);
1698 emit_n(as, ai^m2, bleft);
1699 return;
1700 }
1701 if (cc == CC_EQ || cc == CC_NE) {
1702 /* Combine cmp-bcc into cbz/cbnz. */
1703 ai = cc == CC_EQ ? A64I_CBZ : A64I_CBNZ;
1704 if (irt_is64(ir->t)) ai |= A64I_X;
1705 asm_guardcnb(as, ai, ra_alloc1(as, lref, RSET_GPR));
1706 return;
1707 }
1708 }
1709nocombine:
1710 left = ra_alloc1(as, lref, RSET_GPR);
1711 m = asm_fuseopm(as, ai, rref, rset_exclude(RSET_GPR, left));
1712 asm_guardcc(as, cc);
1713 emit_n(as, ai^m, left);
1714 /* Signed comparison with zero and referencing previous ins? */
1715 if (cmpprev0 && (oldcc <= CC_NE || oldcc >= CC_GE))
1716 as->flagmcp = as->mcp; /* Allow elimination of the compare. */
1717}
1718
1719static void asm_comp(ASMState *as, IRIns *ir)
1720{
1721 if (irt_isnum(ir->t))
1722 asm_fpcomp(as, ir);
1723 else
1724 asm_intcomp(as, ir);
1725}
1726
1727#define asm_equal(as, ir) asm_comp(as, ir)
1728
1729/* -- Split register ops -------------------------------------------------- */
1730
1731/* Hiword op of a split 64/64 bit op. Previous op is the loword op. */
1732static void asm_hiop(ASMState *as, IRIns *ir)
1733{
1734 /* HIOP is marked as a store because it needs its own DCE logic. */
1735 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1736 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1737 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1738 switch ((ir-1)->o) {
1739 case IR_CALLN:
1740 case IR_CALLL:
1741 case IR_CALLS:
1742 case IR_CALLXS:
1743 if (!uselo)
1744 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
1745 break;
1746 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
1747 }
1748}
1749
1750/* -- Profiling ----------------------------------------------------------- */
1751
1752static void asm_prof(ASMState *as, IRIns *ir)
1753{
1754 uint32_t k = emit_isk13(HOOK_PROFILE, 0);
1755 lj_assertA(k != 0, "HOOK_PROFILE does not fit in K13");
1756 UNUSED(ir);
1757 asm_guardcc(as, CC_NE);
1758 emit_n(as, A64I_TSTw^k, RID_TMP);
1759 emit_lsptr(as, A64I_LDRB, RID_TMP, (void *)&J2G(as->J)->hookmask);
1760}
1761
1762/* -- Stack handling ------------------------------------------------------ */
1763
1764/* Check Lua stack size for overflow. Use exit handler as fallback. */
1765static void asm_stack_check(ASMState *as, BCReg topslot,
1766 IRIns *irp, RegSet allow, ExitNo exitno)
1767{
1768 Reg pbase;
1769 uint32_t k;
1770 if (irp) {
1771 if (!ra_hasspill(irp->s)) {
1772 pbase = irp->r;
1773 lj_assertA(ra_hasreg(pbase), "base reg lost");
1774 } else if (allow) {
1775 pbase = rset_pickbot(allow);
1776 } else {
1777 pbase = RID_RET;
1778 emit_lso(as, A64I_LDRx, RID_RET, RID_SP, 0); /* Restore temp register. */
1779 }
1780 } else {
1781 pbase = RID_BASE;
1782 }
1783 emit_cond_branch(as, CC_LS, asm_exitstub_addr(as, exitno));
1784 k = emit_isk12((8*topslot));
1785 lj_assertA(k, "slot offset %d does not fit in K12", 8*topslot);
1786 emit_n(as, A64I_CMPx^k, RID_TMP);
1787 emit_dnm(as, A64I_SUBx, RID_TMP, RID_TMP, pbase);
1788 emit_lso(as, A64I_LDRx, RID_TMP, RID_TMP,
1789 (int32_t)offsetof(lua_State, maxstack));
1790 if (irp) { /* Must not spill arbitrary registers in head of side trace. */
1791 if (ra_hasspill(irp->s))
1792 emit_lso(as, A64I_LDRx, pbase, RID_SP, sps_scale(irp->s));
1793 emit_lso(as, A64I_LDRx, RID_TMP, RID_GL, glofs(as, &J2G(as->J)->cur_L));
1794 if (ra_hasspill(irp->s) && !allow)
1795 emit_lso(as, A64I_STRx, RID_RET, RID_SP, 0); /* Save temp register. */
1796 } else {
1797 emit_getgl(as, RID_TMP, cur_L);
1798 }
1799}
1800
1801/* Restore Lua stack from on-trace state. */
1802static void asm_stack_restore(ASMState *as, SnapShot *snap)
1803{
1804 SnapEntry *map = &as->T->snapmap[snap->mapofs];
1805#ifdef LUA_USE_ASSERT
1806 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
1807#endif
1808 MSize n, nent = snap->nent;
1809 /* Store the value of all modified slots to the Lua stack. */
1810 for (n = 0; n < nent; n++) {
1811 SnapEntry sn = map[n];
1812 BCReg s = snap_slot(sn);
1813 int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
1814 IRRef ref = snap_ref(sn);
1815 IRIns *ir = IR(ref);
1816 if ((sn & SNAP_NORESTORE))
1817 continue;
1818 if ((sn & SNAP_KEYINDEX)) {
1819 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1820 Reg r = irref_isk(ref) ? ra_allock(as, ir->i, allow) :
1821 ra_alloc1(as, ref, allow);
1822 rset_clear(allow, r);
1823 emit_lso(as, A64I_STRw, r, RID_BASE, ofs);
1824 emit_lso(as, A64I_STRw, ra_allock(as, LJ_KEYINDEX, allow), RID_BASE, ofs+4);
1825 } else if (irt_isnum(ir->t)) {
1826 Reg src = ra_alloc1(as, ref, RSET_FPR);
1827 emit_lso(as, A64I_STRd, (src & 31), RID_BASE, ofs);
1828 } else {
1829 asm_tvstore64(as, RID_BASE, ofs, ref);
1830 }
1831 checkmclim(as);
1832 }
1833 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
1834}
1835
1836/* -- GC handling --------------------------------------------------------- */
1837
1838/* Marker to prevent patching the GC check exit. */
1839#define ARM64_NOPATCH_GC_CHECK \
1840 (A64I_ORRx|A64F_D(RID_TMP)|A64F_M(RID_TMP)|A64F_N(RID_TMP))
1841
1842/* Check GC threshold and do one or more GC steps. */
1843static void asm_gc_check(ASMState *as)
1844{
1845 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1846 IRRef args[2];
1847 MCLabel l_end;
1848 Reg tmp2;
1849 ra_evictset(as, RSET_SCRATCH);
1850 l_end = emit_label(as);
1851 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1852 asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */
1853 *--as->mcp = ARM64_NOPATCH_GC_CHECK;
1854 args[0] = ASMREF_TMP1; /* global_State *g */
1855 args[1] = ASMREF_TMP2; /* MSize steps */
1856 asm_gencall(as, ci, args);
1857 emit_dm(as, A64I_MOVx, ra_releasetmp(as, ASMREF_TMP1), RID_GL);
1858 tmp2 = ra_releasetmp(as, ASMREF_TMP2);
1859 emit_loadi(as, tmp2, as->gcsteps);
1860 /* Jump around GC step if GC total < GC threshold. */
1861 emit_cond_branch(as, CC_LS, l_end);
1862 emit_nm(as, A64I_CMPx, RID_TMP, tmp2);
1863 emit_getgl(as, tmp2, gc.threshold);
1864 emit_getgl(as, RID_TMP, gc.total);
1865 as->gcsteps = 0;
1866 checkmclim(as);
1867}
1868
1869/* -- Loop handling ------------------------------------------------------- */
1870
1871/* Fixup the loop branch. */
1872static void asm_loop_fixup(ASMState *as)
1873{
1874 MCode *p = as->mctop;
1875 MCode *target = as->mcp;
1876 if (as->loopinv) { /* Inverted loop branch? */
1877 uint32_t mask = (p[-2] & 0x7e000000) == 0x36000000 ? 0x3fffu : 0x7ffffu;
1878 ptrdiff_t delta = target - (p - 2);
1879 /* asm_guard* already inverted the bcc/tnb/cnb and patched the final b. */
1880 p[-2] |= ((uint32_t)delta & mask) << 5;
1881 } else {
1882 ptrdiff_t delta = target - (p - 1);
1883 p[-1] = A64I_B | A64F_S26(delta);
1884 }
1885}
1886
1887/* Fixup the tail of the loop. */
1888static void asm_loop_tail_fixup(ASMState *as)
1889{
1890 UNUSED(as); /* Nothing to do. */
1891}
1892
1893/* -- Head of trace ------------------------------------------------------- */
1894
1895/* Reload L register from g->cur_L. */
1896static void asm_head_lreg(ASMState *as)
1897{
1898 IRIns *ir = IR(ASMREF_L);
1899 if (ra_used(ir)) {
1900 Reg r = ra_dest(as, ir, RSET_GPR);
1901 emit_getgl(as, r, cur_L);
1902 ra_evictk(as);
1903 }
1904}
1905
1906/* Coalesce BASE register for a root trace. */
1907static void asm_head_root_base(ASMState *as)
1908{
1909 IRIns *ir;
1910 asm_head_lreg(as);
1911 ir = IR(REF_BASE);
1912 if (ra_hasreg(ir->r) && (rset_test(as->modset, ir->r) || irt_ismarked(ir->t)))
1913 ra_spill(as, ir);
1914 ra_destreg(as, ir, RID_BASE);
1915}
1916
1917/* Coalesce BASE register for a side trace. */
1918static Reg asm_head_side_base(ASMState *as, IRIns *irp)
1919{
1920 IRIns *ir;
1921 asm_head_lreg(as);
1922 ir = IR(REF_BASE);
1923 if (ra_hasreg(ir->r) && (rset_test(as->modset, ir->r) || irt_ismarked(ir->t)))
1924 ra_spill(as, ir);
1925 if (ra_hasspill(irp->s)) {
1926 return ra_dest(as, ir, RSET_GPR);
1927 } else {
1928 Reg r = irp->r;
1929 lj_assertA(ra_hasreg(r), "base reg lost");
1930 if (r != ir->r && !rset_test(as->freeset, r))
1931 ra_restore(as, regcost_ref(as->cost[r]));
1932 ra_destreg(as, ir, r);
1933 return r;
1934 }
1935}
1936
1937/* -- Tail of trace ------------------------------------------------------- */
1938
1939/* Fixup the tail code. */
1940static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1941{
1942 MCode *p = as->mctop;
1943 MCode *target;
1944 /* Undo the sp adjustment in BC_JLOOP when exiting to the interpreter. */
1945 int32_t spadj = as->T->spadjust + (lnk ? 0 : sps_scale(SPS_FIXED));
1946 if (spadj == 0) {
1947 *--p = A64I_LE(A64I_NOP);
1948 as->mctop = p;
1949 } else {
1950 /* Patch stack adjustment. */
1951 uint32_t k = emit_isk12(spadj);
1952 lj_assertA(k, "stack adjustment %d does not fit in K12", spadj);
1953 p[-2] = (A64I_ADDx^k) | A64F_D(RID_SP) | A64F_N(RID_SP);
1954 }
1955 /* Patch exit branch. */
1956 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
1957 p[-1] = A64I_B | A64F_S26((target-p)+1);
1958}
1959
1960/* Prepare tail of code. */
1961static void asm_tail_prep(ASMState *as)
1962{
1963 MCode *p = as->mctop - 1; /* Leave room for exit branch. */
1964 if (as->loopref) {
1965 as->invmcp = as->mcp = p;
1966 } else {
1967 as->mcp = p-1; /* Leave room for stack pointer adjustment. */
1968 as->invmcp = NULL;
1969 }
1970 *p = 0; /* Prevent load/store merging. */
1971}
1972
1973/* -- Trace setup --------------------------------------------------------- */
1974
1975/* Ensure there are enough stack slots for call arguments. */
1976static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
1977{
1978 IRRef args[CCI_NARGS_MAX*2];
1979 uint32_t i, nargs = CCI_XNARGS(ci);
1980 int nslots = 0, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
1981 asm_collectargs(as, ir, ci, args);
1982 for (i = 0; i < nargs; i++) {
1983 if (args[i] && irt_isfp(IR(args[i])->t)) {
1984 if (nfpr > 0) nfpr--; else nslots += 2;
1985 } else {
1986 if (ngpr > 0) ngpr--; else nslots += 2;
1987 }
1988 }
1989 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
1990 as->evenspill = nslots;
1991 return REGSP_HINT(RID_RET);
1992}
1993
1994static void asm_setup_target(ASMState *as)
1995{
1996 /* May need extra exit for asm_stack_check on side traces. */
1997 asm_exitstub_setup(as, as->T->nsnap + (as->parent ? 1 : 0));
1998}
1999
2000#if LJ_BE
2001/* ARM64 instructions are always little-endian. Swap for ARM64BE. */
2002static void asm_mcode_fixup(MCode *mcode, MSize size)
2003{
2004 MCode *pe = (MCode *)((char *)mcode + size);
2005 while (mcode < pe) {
2006 MCode ins = *mcode;
2007 *mcode++ = lj_bswap(ins);
2008 }
2009}
2010#define LJ_TARGET_MCODE_FIXUP 1
2011#endif
2012
2013/* -- Trace patching ------------------------------------------------------ */
2014
2015/* Patch exit jumps of existing machine code to a new target. */
2016void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2017{
2018 MCode *p = T->mcode;
2019 MCode *pe = (MCode *)((char *)p + T->szmcode);
2020 MCode *cstart = NULL;
2021 MCode *mcarea = lj_mcode_patch(J, p, 0);
2022 MCode *px = exitstub_trace_addr(T, exitno);
2023 int patchlong = 1;
2024 /* Note: this assumes a trace exit is only ever patched once. */
2025 for (; p < pe; p++) {
2026 /* Look for exitstub branch, replace with branch to target. */
2027 ptrdiff_t delta = target - p;
2028 MCode ins = A64I_LE(*p);
2029 if ((ins & 0xff000000u) == 0x54000000u &&
2030 ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
2031 /* Patch bcc, if within range. */
2032 if (A64F_S_OK(delta, 19)) {
2033 *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
2034 if (!cstart) cstart = p;
2035 }
2036 } else if ((ins & 0xfc000000u) == 0x14000000u &&
2037 ((ins ^ (px-p)) & 0x03ffffffu) == 0) {
2038 /* Patch b. */
2039 lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
2040 *p = A64I_LE((ins & 0xfc000000u) | A64F_S26(delta));
2041 if (!cstart) cstart = p;
2042 } else if ((ins & 0x7e000000u) == 0x34000000u &&
2043 ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
2044 /* Patch cbz/cbnz, if within range. */
2045 if (p[-1] == ARM64_NOPATCH_GC_CHECK) {
2046 patchlong = 0;
2047 } else if (A64F_S_OK(delta, 19)) {
2048 *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
2049 if (!cstart) cstart = p;
2050 }
2051 } else if ((ins & 0x7e000000u) == 0x36000000u &&
2052 ((ins ^ ((px-p)<<5)) & 0x0007ffe0u) == 0) {
2053 /* Patch tbz/tbnz, if within range. */
2054 if (A64F_S_OK(delta, 14)) {
2055 *p = A64I_LE((ins & 0xfff8001fu) | A64F_S14(delta));
2056 if (!cstart) cstart = p;
2057 }
2058 }
2059 }
2060 /* Always patch long-range branch in exit stub itself. Except, if we can't. */
2061 if (patchlong) {
2062 ptrdiff_t delta = target - px;
2063 lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
2064 *px = A64I_B | A64F_S26(delta);
2065 if (!cstart) cstart = px;
2066 }
2067 if (cstart) lj_mcode_sync(cstart, px+1);
2068 lj_mcode_patch(J, mcarea, 1);
2069}
2070
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index 3c99a843..b02da663 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -23,7 +23,7 @@ static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
23{ 23{
24 Reg r = IR(ref)->r; 24 Reg r = IR(ref)->r;
25 if (ra_noreg(r)) { 25 if (ra_noreg(r)) {
26 if (!(allow & RSET_FPR) && irref_isk(ref) && IR(ref)->i == 0) 26 if (!(allow & RSET_FPR) && irref_isk(ref) && get_kval(as, ref) == 0)
27 return RID_ZERO; 27 return RID_ZERO;
28 r = ra_allocref(as, ref, allow); 28 r = ra_allocref(as, ref, allow);
29 } else { 29 } else {
@@ -64,17 +64,29 @@ static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
64/* Setup spare long-range jump slots per mcarea. */ 64/* Setup spare long-range jump slots per mcarea. */
65static void asm_sparejump_setup(ASMState *as) 65static void asm_sparejump_setup(ASMState *as)
66{ 66{
67 MCode *mxp = as->mcbot; 67 MCode *mxp = as->mctop;
68 if (((uintptr_t)mxp & (LJ_PAGESIZE-1)) == sizeof(MCLink)) { 68 if ((char *)mxp == (char *)as->J->mcarea + as->J->szmcarea) {
69 lua_assert(MIPSI_NOP == 0); 69 mxp -= MIPS_SPAREJUMP*2;
70 lj_assertA(MIPSI_NOP == 0, "bad NOP");
70 memset(mxp, 0, MIPS_SPAREJUMP*2*sizeof(MCode)); 71 memset(mxp, 0, MIPS_SPAREJUMP*2*sizeof(MCode));
71 mxp += MIPS_SPAREJUMP*2; 72 as->mctop = mxp;
72 lua_assert(mxp < as->mctop); 73 }
73 lj_mcode_sync(as->mcbot, mxp); 74}
74 lj_mcode_commitbot(as->J, mxp); 75
75 as->mcbot = mxp; 76static MCode *asm_sparejump_use(MCode *mcarea, MCode tjump)
76 as->mclim = as->mcbot + MCLIM_REDZONE; 77{
78 MCode *mxp = (MCode *)((char *)mcarea + ((MCLink *)mcarea)->size);
79 int slot = MIPS_SPAREJUMP;
80 while (slot--) {
81 mxp -= 2;
82 if (*mxp == tjump) {
83 return mxp;
84 } else if (*mxp == MIPSI_NOP) {
85 *mxp = tjump;
86 return mxp;
87 }
77 } 88 }
89 return NULL;
78} 90}
79 91
80/* Setup exit stub after the end of each trace. */ 92/* Setup exit stub after the end of each trace. */
@@ -84,7 +96,8 @@ static void asm_exitstub_setup(ASMState *as)
84 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */ 96 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
85 *--mxp = MIPSI_LI|MIPSF_T(RID_TMP)|as->T->traceno; 97 *--mxp = MIPSI_LI|MIPSF_T(RID_TMP)|as->T->traceno;
86 *--mxp = MIPSI_J|((((uintptr_t)(void *)lj_vm_exit_handler)>>2)&0x03ffffffu); 98 *--mxp = MIPSI_J|((((uintptr_t)(void *)lj_vm_exit_handler)>>2)&0x03ffffffu);
87 lua_assert(((uintptr_t)mxp ^ (uintptr_t)(void *)lj_vm_exit_handler)>>28 == 0); 99 lj_assertA(((uintptr_t)mxp ^ (uintptr_t)(void *)lj_vm_exit_handler)>>28 == 0,
100 "branch target out of range");
88 *--mxp = MIPSI_SW|MIPSF_T(RID_TMP)|MIPSF_S(RID_SP)|0; 101 *--mxp = MIPSI_SW|MIPSF_T(RID_TMP)|MIPSF_S(RID_SP)|0;
89 as->mctop = mxp; 102 as->mctop = mxp;
90} 103}
@@ -101,7 +114,12 @@ static void asm_guard(ASMState *as, MIPSIns mi, Reg rs, Reg rt)
101 as->invmcp = NULL; 114 as->invmcp = NULL;
102 as->loopinv = 1; 115 as->loopinv = 1;
103 as->mcp = p+1; 116 as->mcp = p+1;
117#if !LJ_TARGET_MIPSR6
104 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u); /* Invert cond. */ 118 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u); /* Invert cond. */
119#else
120 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u :
121 (mi>>28) == 4 ? 0x00800000u : 0x00010000u); /* Invert cond. */
122#endif
105 target = p; /* Patch target later in asm_loop_fixup. */ 123 target = p; /* Patch target later in asm_loop_fixup. */
106 } 124 }
107 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno); 125 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
@@ -165,9 +183,9 @@ static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
165 } else if (ir->o == IR_UREFC) { 183 } else if (ir->o == IR_UREFC) {
166 if (irref_isk(ir->op1)) { 184 if (irref_isk(ir->op1)) {
167 GCfunc *fn = ir_kfunc(IR(ir->op1)); 185 GCfunc *fn = ir_kfunc(IR(ir->op1));
168 int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv); 186 intptr_t ofs = (intptr_t)&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv;
169 int32_t jgl = (intptr_t)J2G(as->J); 187 intptr_t jgl = (intptr_t)J2G(as->J);
170 if ((uint32_t)(ofs-jgl) < 65536) { 188 if ((uintptr_t)(ofs-jgl) < 65536) {
171 *ofsp = ofs-jgl-32768; 189 *ofsp = ofs-jgl-32768;
172 return RID_JGL; 190 return RID_JGL;
173 } else { 191 } else {
@@ -175,6 +193,9 @@ static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
175 return ra_allock(as, ofs-(int16_t)ofs, allow); 193 return ra_allock(as, ofs-(int16_t)ofs, allow);
176 } 194 }
177 } 195 }
196 } else if (ir->o == IR_TMPREF) {
197 *ofsp = (int32_t)(offsetof(global_State, tmptv)-32768);
198 return RID_JGL;
178 } 199 }
179 } 200 }
180 *ofsp = 0; 201 *ofsp = 0;
@@ -189,20 +210,21 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
189 Reg base; 210 Reg base;
190 if (ra_noreg(ir->r) && canfuse(as, ir)) { 211 if (ra_noreg(ir->r) && canfuse(as, ir)) {
191 if (ir->o == IR_ADD) { 212 if (ir->o == IR_ADD) {
192 int32_t ofs2; 213 intptr_t ofs2;
193 if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) { 214 if (irref_isk(ir->op2) && (ofs2 = ofs + get_kval(as, ir->op2),
215 checki16(ofs2))) {
194 ref = ir->op1; 216 ref = ir->op1;
195 ofs = ofs2; 217 ofs = (int32_t)ofs2;
196 } 218 }
197 } else if (ir->o == IR_STRREF) { 219 } else if (ir->o == IR_STRREF) {
198 int32_t ofs2 = 65536; 220 intptr_t ofs2 = 65536;
199 lua_assert(ofs == 0); 221 lj_assertA(ofs == 0, "bad usage");
200 ofs = (int32_t)sizeof(GCstr); 222 ofs = (int32_t)sizeof(GCstr);
201 if (irref_isk(ir->op2)) { 223 if (irref_isk(ir->op2)) {
202 ofs2 = ofs + IR(ir->op2)->i; 224 ofs2 = ofs + get_kval(as, ir->op2);
203 ref = ir->op1; 225 ref = ir->op1;
204 } else if (irref_isk(ir->op1)) { 226 } else if (irref_isk(ir->op1)) {
205 ofs2 = ofs + IR(ir->op1)->i; 227 ofs2 = ofs + get_kval(as, ir->op1);
206 ref = ir->op2; 228 ref = ir->op2;
207 } 229 }
208 if (!checki16(ofs2)) { 230 if (!checki16(ofs2)) {
@@ -210,7 +232,7 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
210 Reg right, left = ra_alloc2(as, ir, allow); 232 Reg right, left = ra_alloc2(as, ir, allow);
211 right = (left >> 8); left &= 255; 233 right = (left >> 8); left &= 255;
212 emit_hsi(as, mi, rt, RID_TMP, ofs); 234 emit_hsi(as, mi, rt, RID_TMP, ofs);
213 emit_dst(as, MIPSI_ADDU, RID_TMP, left, right); 235 emit_dst(as, MIPSI_AADDU, RID_TMP, left, right);
214 return; 236 return;
215 } 237 }
216 ofs = ofs2; 238 ofs = ofs2;
@@ -225,29 +247,43 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
225/* Generate a call to a C function. */ 247/* Generate a call to a C function. */
226static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) 248static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
227{ 249{
228 uint32_t n, nargs = CCI_NARGS(ci); 250 uint32_t n, nargs = CCI_XNARGS(ci);
229 int32_t ofs = 16; 251 int32_t ofs = LJ_32 ? 16 : 0;
252#if LJ_SOFTFP
253 Reg gpr = REGARG_FIRSTGPR;
254#else
230 Reg gpr, fpr = REGARG_FIRSTFPR; 255 Reg gpr, fpr = REGARG_FIRSTFPR;
256#endif
231 if ((void *)ci->func) 257 if ((void *)ci->func)
232 emit_call(as, (void *)ci->func); 258 emit_call(as, (void *)ci->func, 1);
259#if !LJ_SOFTFP
233 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++) 260 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
234 as->cost[gpr] = REGCOST(~0u, ASMREF_L); 261 as->cost[gpr] = REGCOST(~0u, ASMREF_L);
235 gpr = REGARG_FIRSTGPR; 262 gpr = REGARG_FIRSTGPR;
263#endif
236 for (n = 0; n < nargs; n++) { /* Setup args. */ 264 for (n = 0; n < nargs; n++) { /* Setup args. */
237 IRRef ref = args[n]; 265 IRRef ref = args[n];
238 if (ref) { 266 if (ref) {
239 IRIns *ir = IR(ref); 267 IRIns *ir = IR(ref);
268#if !LJ_SOFTFP
240 if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR && 269 if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR &&
241 !(ci->flags & CCI_VARARG)) { 270 !(ci->flags & CCI_VARARG)) {
242 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */ 271 lj_assertA(rset_test(as->freeset, fpr),
272 "reg %d not free", fpr); /* Already evicted. */
243 ra_leftov(as, fpr, ref); 273 ra_leftov(as, fpr, ref);
244 fpr += 2; 274 fpr += LJ_32 ? 2 : 1;
245 gpr += irt_isnum(ir->t) ? 2 : 1; 275 gpr += (LJ_32 && irt_isnum(ir->t)) ? 2 : 1;
246 } else { 276 } else
277#endif
278 {
279#if LJ_32 && !LJ_SOFTFP
247 fpr = REGARG_LASTFPR+1; 280 fpr = REGARG_LASTFPR+1;
248 if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1; 281#endif
282 if (LJ_32 && irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
249 if (gpr <= REGARG_LASTGPR) { 283 if (gpr <= REGARG_LASTGPR) {
250 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */ 284 lj_assertA(rset_test(as->freeset, gpr),
285 "reg %d not free", gpr); /* Already evicted. */
286#if !LJ_SOFTFP
251 if (irt_isfp(ir->t)) { 287 if (irt_isfp(ir->t)) {
252 RegSet of = as->freeset; 288 RegSet of = as->freeset;
253 Reg r; 289 Reg r;
@@ -256,31 +292,56 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
256 r = ra_alloc1(as, ref, RSET_FPR); 292 r = ra_alloc1(as, ref, RSET_FPR);
257 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1)); 293 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
258 if (irt_isnum(ir->t)) { 294 if (irt_isnum(ir->t)) {
295#if LJ_32
259 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1); 296 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
260 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r); 297 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
261 lua_assert(rset_test(as->freeset, gpr+1)); /* Already evicted. */ 298 lj_assertA(rset_test(as->freeset, gpr+1),
299 "reg %d not free", gpr+1); /* Already evicted. */
262 gpr += 2; 300 gpr += 2;
301#else
302 emit_tg(as, MIPSI_DMFC1, gpr, r);
303 gpr++; fpr++;
304#endif
263 } else if (irt_isfloat(ir->t)) { 305 } else if (irt_isfloat(ir->t)) {
264 emit_tg(as, MIPSI_MFC1, gpr, r); 306 emit_tg(as, MIPSI_MFC1, gpr, r);
265 gpr++; 307 gpr++;
308#if LJ_64
309 fpr++;
310#endif
266 } 311 }
267 } else { 312 } else
313#endif
314 {
268 ra_leftov(as, gpr, ref); 315 ra_leftov(as, gpr, ref);
269 gpr++; 316 gpr++;
317#if LJ_64 && !LJ_SOFTFP
318 fpr++;
319#endif
270 } 320 }
271 } else { 321 } else {
272 Reg r = ra_alloc1z(as, ref, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); 322 Reg r = ra_alloc1z(as, ref, !LJ_SOFTFP && irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
323#if LJ_32
273 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4; 324 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
274 emit_spstore(as, ir, r, ofs); 325 emit_spstore(as, ir, r, ofs);
275 ofs += irt_isnum(ir->t) ? 8 : 4; 326 ofs += irt_isnum(ir->t) ? 8 : 4;
327#else
328 emit_spstore(as, ir, r, ofs + ((LJ_BE && !irt_isfp(ir->t) && !irt_is64(ir->t)) ? 4 : 0));
329 ofs += 8;
330#endif
276 } 331 }
277 } 332 }
278 } else { 333 } else {
334#if !LJ_SOFTFP
279 fpr = REGARG_LASTFPR+1; 335 fpr = REGARG_LASTFPR+1;
280 if (gpr <= REGARG_LASTGPR) 336#endif
337 if (gpr <= REGARG_LASTGPR) {
281 gpr++; 338 gpr++;
282 else 339#if LJ_64 && !LJ_SOFTFP
283 ofs += 4; 340 fpr++;
341#endif
342 } else {
343 ofs += LJ_32 ? 4 : 8;
344 }
284 } 345 }
285 checkmclim(as); 346 checkmclim(as);
286 } 347 }
@@ -291,28 +352,38 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
291{ 352{
292 RegSet drop = RSET_SCRATCH; 353 RegSet drop = RSET_SCRATCH;
293 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t)); 354 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
355#if !LJ_SOFTFP
294 if ((ci->flags & CCI_NOFPRCLOBBER)) 356 if ((ci->flags & CCI_NOFPRCLOBBER))
295 drop &= ~RSET_FPR; 357 drop &= ~RSET_FPR;
358#endif
296 if (ra_hasreg(ir->r)) 359 if (ra_hasreg(ir->r))
297 rset_clear(drop, ir->r); /* Dest reg handled below. */ 360 rset_clear(drop, ir->r); /* Dest reg handled below. */
298 if (hiop && ra_hasreg((ir+1)->r)) 361 if (hiop && ra_hasreg((ir+1)->r))
299 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */ 362 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
300 ra_evictset(as, drop); /* Evictions must be performed first. */ 363 ra_evictset(as, drop); /* Evictions must be performed first. */
301 if (ra_used(ir)) { 364 if (ra_used(ir)) {
302 lua_assert(!irt_ispri(ir->t)); 365 lj_assertA(!irt_ispri(ir->t), "PRI dest");
303 if (irt_isfp(ir->t)) { 366 if (!LJ_SOFTFP && irt_isfp(ir->t)) {
304 if ((ci->flags & CCI_CASTU64)) { 367 if ((ci->flags & CCI_CASTU64)) {
305 int32_t ofs = sps_scale(ir->s); 368 int32_t ofs = sps_scale(ir->s);
306 Reg dest = ir->r; 369 Reg dest = ir->r;
307 if (ra_hasreg(dest)) { 370 if (ra_hasreg(dest)) {
308 ra_free(as, dest); 371 ra_free(as, dest);
309 ra_modified(as, dest); 372 ra_modified(as, dest);
373#if LJ_32
310 emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1); 374 emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
311 emit_tg(as, MIPSI_MTC1, RID_RETLO, dest); 375 emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
376#else
377 emit_tg(as, MIPSI_DMTC1, RID_RET, dest);
378#endif
312 } 379 }
313 if (ofs) { 380 if (ofs) {
381#if LJ_32
314 emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0)); 382 emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
315 emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4)); 383 emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
384#else
385 emit_tsi(as, MIPSI_SD, RID_RET, RID_SP, ofs);
386#endif
316 } 387 }
317 } else { 388 } else {
318 ra_destreg(as, ir, RID_FPRET); 389 ra_destreg(as, ir, RID_FPRET);
@@ -325,15 +396,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
325 } 396 }
326} 397}
327 398
328static void asm_call(ASMState *as, IRIns *ir)
329{
330 IRRef args[CCI_NARGS_MAX];
331 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
332 asm_collectargs(as, ir, ci, args);
333 asm_setupresult(as, ir, ci);
334 asm_gencall(as, ci, args);
335}
336
337static void asm_callx(ASMState *as, IRIns *ir) 399static void asm_callx(ASMState *as, IRIns *ir)
338{ 400{
339 IRRef args[CCI_NARGS_MAX*2]; 401 IRRef args[CCI_NARGS_MAX*2];
@@ -346,7 +408,7 @@ static void asm_callx(ASMState *as, IRIns *ir)
346 func = ir->op2; irf = IR(func); 408 func = ir->op2; irf = IR(func);
347 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); } 409 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
348 if (irref_isk(func)) { /* Call to constant address. */ 410 if (irref_isk(func)) { /* Call to constant address. */
349 ci.func = (ASMFunction)(void *)(irf->i); 411 ci.func = (ASMFunction)(void *)get_kval(as, func);
350 } else { /* Need specific register for indirect calls. */ 412 } else { /* Need specific register for indirect calls. */
351 Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR)); 413 Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
352 MCode *p = as->mcp; 414 MCode *p = as->mcp;
@@ -361,27 +423,23 @@ static void asm_callx(ASMState *as, IRIns *ir)
361 asm_gencall(as, &ci, args); 423 asm_gencall(as, &ci, args);
362} 424}
363 425
364static void asm_callid(ASMState *as, IRIns *ir, IRCallID id) 426#if !LJ_SOFTFP
365{
366 const CCallInfo *ci = &lj_ir_callinfo[id];
367 IRRef args[2];
368 args[0] = ir->op1;
369 args[1] = ir->op2;
370 asm_setupresult(as, ir, ci);
371 asm_gencall(as, ci, args);
372}
373
374static void asm_callround(ASMState *as, IRIns *ir, IRCallID id) 427static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
375{ 428{
376 /* The modified regs must match with the *.dasc implementation. */ 429 /* The modified regs must match with the *.dasc implementation. */
377 RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)| 430 RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)|
378 RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR); 431 RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR)
432#if LJ_TARGET_MIPSR6
433 |RID2RSET(RID_F21)
434#endif
435 ;
379 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r); 436 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
380 ra_evictset(as, drop); 437 ra_evictset(as, drop);
381 ra_destreg(as, ir, RID_FPRET); 438 ra_destreg(as, ir, RID_FPRET);
382 emit_call(as, (void *)lj_ir_callinfo[id].func); 439 emit_call(as, (void *)lj_ir_callinfo[id].func, 0);
383 ra_leftov(as, REGARG_FIRSTFPR, ir->op1); 440 ra_leftov(as, REGARG_FIRSTFPR, ir->op1);
384} 441}
442#endif
385 443
386/* -- Returns ------------------------------------------------------------- */ 444/* -- Returns ------------------------------------------------------------- */
387 445
@@ -390,25 +448,52 @@ static void asm_retf(ASMState *as, IRIns *ir)
390{ 448{
391 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR); 449 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
392 void *pc = ir_kptr(IR(ir->op2)); 450 void *pc = ir_kptr(IR(ir->op2));
393 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1)); 451 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
394 as->topslot -= (BCReg)delta; 452 as->topslot -= (BCReg)delta;
395 if ((int32_t)as->topslot < 0) as->topslot = 0; 453 if ((int32_t)as->topslot < 0) as->topslot = 0;
396 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */ 454 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
397 emit_setgl(as, base, jit_base); 455 emit_setgl(as, base, jit_base);
398 emit_addptr(as, base, -8*delta); 456 emit_addptr(as, base, -8*delta);
399 asm_guard(as, MIPSI_BNE, RID_TMP, 457 asm_guard(as, MIPSI_BNE, RID_TMP,
400 ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base))); 458 ra_allock(as, igcptr(pc), rset_exclude(RSET_GPR, base)));
401 emit_tsi(as, MIPSI_LW, RID_TMP, base, -8); 459 emit_tsi(as, MIPSI_AL, RID_TMP, base, -8);
402} 460}
403 461
462/* -- Buffer operations --------------------------------------------------- */
463
464#if LJ_HASBUFFER
465static void asm_bufhdr_write(ASMState *as, Reg sb)
466{
467 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
468 IRIns irgc;
469 irgc.ot = IRT(0, IRT_PGC); /* GC type. */
470 emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
471 if ((as->flags & JIT_F_MIPSXXR2)) {
472 emit_tsml(as, LJ_64 ? MIPSI_DINS : MIPSI_INS, RID_TMP, tmp,
473 lj_fls(SBUF_MASK_FLAG), 0);
474 } else {
475 emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
476 emit_tsi(as, MIPSI_ANDI, tmp, tmp, SBUF_MASK_FLAG);
477 }
478 emit_getgl(as, RID_TMP, cur_L);
479 emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
480}
481#endif
482
404/* -- Type conversions ---------------------------------------------------- */ 483/* -- Type conversions ---------------------------------------------------- */
405 484
485#if !LJ_SOFTFP
406static void asm_tointg(ASMState *as, IRIns *ir, Reg left) 486static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
407{ 487{
408 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left)); 488 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
409 Reg dest = ra_dest(as, ir, RSET_GPR); 489 Reg dest = ra_dest(as, ir, RSET_GPR);
490#if !LJ_TARGET_MIPSR6
410 asm_guard(as, MIPSI_BC1F, 0, 0); 491 asm_guard(as, MIPSI_BC1F, 0, 0);
411 emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left); 492 emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left);
493#else
494 asm_guard(as, MIPSI_BC1EQZ, 0, (tmp&31));
495 emit_fgh(as, MIPSI_CMP_EQ_D, tmp, tmp, left);
496#endif
412 emit_fg(as, MIPSI_CVT_D_W, tmp, tmp); 497 emit_fg(as, MIPSI_CVT_D_W, tmp, tmp);
413 emit_tg(as, MIPSI_MFC1, dest, tmp); 498 emit_tg(as, MIPSI_MFC1, dest, tmp);
414 emit_fg(as, MIPSI_CVT_W_D, tmp, left); 499 emit_fg(as, MIPSI_CVT_W_D, tmp, left);
@@ -424,15 +509,57 @@ static void asm_tobit(ASMState *as, IRIns *ir)
424 emit_tg(as, MIPSI_MFC1, dest, tmp); 509 emit_tg(as, MIPSI_MFC1, dest, tmp);
425 emit_fgh(as, MIPSI_ADD_D, tmp, left, right); 510 emit_fgh(as, MIPSI_ADD_D, tmp, left, right);
426} 511}
512#elif LJ_64 /* && LJ_SOFTFP */
513static void asm_tointg(ASMState *as, IRIns *ir, Reg r)
514{
515 /* The modified regs must match with the *.dasc implementation. */
516 RegSet drop = RID2RSET(REGARG_FIRSTGPR)|RID2RSET(RID_RET)|RID2RSET(RID_RET+1)|
517 RID2RSET(RID_R1)|RID2RSET(RID_R12);
518 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
519 ra_evictset(as, drop);
520 /* Return values are in RID_RET (converted value) and RID_RET+1 (status). */
521 ra_destreg(as, ir, RID_RET);
522 asm_guard(as, MIPSI_BNE, RID_RET+1, RID_ZERO);
523 emit_call(as, (void *)lj_ir_callinfo[IRCALL_lj_vm_tointg].func, 0);
524 if (r == RID_NONE)
525 ra_leftov(as, REGARG_FIRSTGPR, ir->op1);
526 else if (r != REGARG_FIRSTGPR)
527 emit_move(as, REGARG_FIRSTGPR, r);
528}
529
530static void asm_tobit(ASMState *as, IRIns *ir)
531{
532 Reg dest = ra_dest(as, ir, RSET_GPR);
533 emit_dta(as, MIPSI_SLL, dest, dest, 0);
534 asm_callid(as, ir, IRCALL_lj_vm_tobit);
535}
536#endif
427 537
428static void asm_conv(ASMState *as, IRIns *ir) 538static void asm_conv(ASMState *as, IRIns *ir)
429{ 539{
430 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK); 540 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
541#if !LJ_SOFTFP32
431 int stfp = (st == IRT_NUM || st == IRT_FLOAT); 542 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
543#endif
544#if LJ_64
545 int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
546#endif
432 IRRef lref = ir->op1; 547 IRRef lref = ir->op1;
433 lua_assert(irt_type(ir->t) != st); 548#if LJ_32
434 lua_assert(!(irt_isint64(ir->t) || 549 /* 64 bit integer conversions are handled by SPLIT. */
435 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */ 550 lj_assertA(!(irt_isint64(ir->t) || (st == IRT_I64 || st == IRT_U64)),
551 "IR %04d has unsplit 64 bit type",
552 (int)(ir - as->ir) - REF_BIAS);
553#endif
554#if LJ_SOFTFP32
555 /* FP conversions are handled by SPLIT. */
556 lj_assertA(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT),
557 "IR %04d has FP type",
558 (int)(ir - as->ir) - REF_BIAS);
559 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
560#else
561 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
562#if !LJ_SOFTFP
436 if (irt_isfp(ir->t)) { 563 if (irt_isfp(ir->t)) {
437 Reg dest = ra_dest(as, ir, RSET_FPR); 564 Reg dest = ra_dest(as, ir, RSET_FPR);
438 if (stfp) { /* FP to FP conversion. */ 565 if (stfp) { /* FP to FP conversion. */
@@ -448,27 +575,56 @@ static void asm_conv(ASMState *as, IRIns *ir)
448 emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp); 575 emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
449 emit_fg(as, MIPSI_CVT_D_W, dest, dest); 576 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
450 emit_lsptr(as, MIPSI_LDC1, (tmp & 31), 577 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
451 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)), 578 (void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
452 RSET_GPR);
453 emit_tg(as, MIPSI_MTC1, RID_TMP, dest); 579 emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
454 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left); 580 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
455 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000); 581 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
582#if LJ_64
583 } else if(st == IRT_U64) { /* U64 to FP conversion. */
584 /* if (x >= 1u<<63) y = (double)(int64_t)(x&(1u<<63)-1) + pow(2.0, 63) */
585 Reg left = ra_alloc1(as, lref, RSET_GPR);
586 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
587 MCLabel l_end = emit_label(as);
588 if (irt_isfloat(ir->t)) {
589 emit_fgh(as, MIPSI_ADD_S, dest, dest, tmp);
590 emit_lsptr(as, MIPSI_LWC1, (tmp & 31), (void *)&as->J->k32[LJ_K32_2P63],
591 rset_exclude(RSET_GPR, left));
592 emit_fg(as, MIPSI_CVT_S_L, dest, dest);
593 } else {
594 emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
595 emit_lsptr(as, MIPSI_LDC1, (tmp & 31), (void *)&as->J->k64[LJ_K64_2P63],
596 rset_exclude(RSET_GPR, left));
597 emit_fg(as, MIPSI_CVT_D_L, dest, dest);
598 }
599 emit_branch(as, MIPSI_BGEZ, left, RID_ZERO, l_end);
600 emit_tg(as, MIPSI_DMTC1, RID_TMP, dest);
601 emit_tsml(as, MIPSI_DEXTM, RID_TMP, left, 30, 0);
602#endif
456 } else { /* Integer to FP conversion. */ 603 } else { /* Integer to FP conversion. */
457 Reg left = ra_alloc1(as, lref, RSET_GPR); 604 Reg left = ra_alloc1(as, lref, RSET_GPR);
605#if LJ_32
458 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W, 606 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
459 dest, dest); 607 dest, dest);
460 emit_tg(as, MIPSI_MTC1, left, dest); 608 emit_tg(as, MIPSI_MTC1, left, dest);
609#else
610 MIPSIns mi = irt_isfloat(ir->t) ?
611 (st64 ? MIPSI_CVT_S_L : MIPSI_CVT_S_W) :
612 (st64 ? MIPSI_CVT_D_L : MIPSI_CVT_D_W);
613 emit_fg(as, mi, dest, dest);
614 emit_tg(as, st64 ? MIPSI_DMTC1 : MIPSI_MTC1, left, dest);
615#endif
461 } 616 }
462 } else if (stfp) { /* FP to integer conversion. */ 617 } else if (stfp) { /* FP to integer conversion. */
463 if (irt_isguard(ir->t)) { 618 if (irt_isguard(ir->t)) {
464 /* Checked conversions are only supported from number to int. */ 619 /* Checked conversions are only supported from number to int. */
465 lua_assert(irt_isint(ir->t) && st == IRT_NUM); 620 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
621 "bad type for checked CONV");
466 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); 622 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
467 } else { 623 } else {
468 Reg dest = ra_dest(as, ir, RSET_GPR); 624 Reg dest = ra_dest(as, ir, RSET_GPR);
469 Reg left = ra_alloc1(as, lref, RSET_FPR); 625 Reg left = ra_alloc1(as, lref, RSET_FPR);
470 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left)); 626 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
471 if (irt_isu32(ir->t)) { 627 if (irt_isu32(ir->t)) { /* FP to U32 conversion. */
472 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */ 628 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
473 emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP); 629 emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP);
474 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000); 630 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
@@ -479,25 +635,112 @@ static void asm_conv(ASMState *as, IRIns *ir)
479 tmp, left, tmp); 635 tmp, left, tmp);
480 if (st == IRT_FLOAT) 636 if (st == IRT_FLOAT)
481 emit_lsptr(as, MIPSI_LWC1, (tmp & 31), 637 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
482 (void *)lj_ir_k64_find(as->J, U64x(4f000000,4f000000)), 638 (void *)&as->J->k32[LJ_K32_2P31], RSET_GPR);
483 RSET_GPR);
484 else 639 else
485 emit_lsptr(as, MIPSI_LDC1, (tmp & 31), 640 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
486 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)), 641 (void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
487 RSET_GPR); 642#if LJ_64
643 } else if (irt_isu64(ir->t)) { /* FP to U64 conversion. */
644 MCLabel l_end;
645 emit_tg(as, MIPSI_DMFC1, dest, tmp);
646 l_end = emit_label(as);
647 /* For inputs >= 2^63 add -2^64 and convert again. */
648 if (st == IRT_NUM) {
649 emit_fg(as, MIPSI_TRUNC_L_D, tmp, tmp);
650 emit_fgh(as, MIPSI_ADD_D, tmp, left, tmp);
651 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
652 (void *)&as->J->k64[LJ_K64_M2P64],
653 rset_exclude(RSET_GPR, dest));
654 emit_fg(as, MIPSI_TRUNC_L_D, tmp, left); /* Delay slot. */
655#if !LJ_TARGET_MIPSR6
656 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
657 emit_fgh(as, MIPSI_C_OLT_D, 0, left, tmp);
658#else
659 emit_branch(as, MIPSI_BC1NEZ, 0, (left&31), l_end);
660 emit_fgh(as, MIPSI_CMP_LT_D, left, left, tmp);
661#endif
662 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
663 (void *)&as->J->k64[LJ_K64_2P63],
664 rset_exclude(RSET_GPR, dest));
665 } else {
666 emit_fg(as, MIPSI_TRUNC_L_S, tmp, tmp);
667 emit_fgh(as, MIPSI_ADD_S, tmp, left, tmp);
668 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
669 (void *)&as->J->k32[LJ_K32_M2P64],
670 rset_exclude(RSET_GPR, dest));
671 emit_fg(as, MIPSI_TRUNC_L_S, tmp, left); /* Delay slot. */
672#if !LJ_TARGET_MIPSR6
673 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
674 emit_fgh(as, MIPSI_C_OLT_S, 0, left, tmp);
675#else
676 emit_branch(as, MIPSI_BC1NEZ, 0, (left&31), l_end);
677 emit_fgh(as, MIPSI_CMP_LT_S, left, left, tmp);
678#endif
679 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
680 (void *)&as->J->k32[LJ_K32_2P63],
681 rset_exclude(RSET_GPR, dest));
682 }
683#endif
488 } else { 684 } else {
685#if LJ_32
489 emit_tg(as, MIPSI_MFC1, dest, tmp); 686 emit_tg(as, MIPSI_MFC1, dest, tmp);
490 emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D, 687 emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
491 tmp, left); 688 tmp, left);
689#else
690 MIPSIns mi = irt_is64(ir->t) ?
691 (st == IRT_NUM ? MIPSI_TRUNC_L_D : MIPSI_TRUNC_L_S) :
692 (st == IRT_NUM ? MIPSI_TRUNC_W_D : MIPSI_TRUNC_W_S);
693 emit_tg(as, irt_is64(ir->t) ? MIPSI_DMFC1 : MIPSI_MFC1, dest, left);
694 emit_fg(as, mi, left, left);
695#endif
492 } 696 }
493 } 697 }
494 } else { 698 } else
699#else
700 if (irt_isfp(ir->t)) {
701#if LJ_64 && LJ_HASFFI
702 if (stfp) { /* FP to FP conversion. */
703 asm_callid(as, ir, irt_isnum(ir->t) ? IRCALL_softfp_f2d :
704 IRCALL_softfp_d2f);
705 } else { /* Integer to FP conversion. */
706 IRCallID cid = ((IRT_IS64 >> st) & 1) ?
707 (irt_isnum(ir->t) ?
708 (st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d) :
709 (st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f)) :
710 (irt_isnum(ir->t) ?
711 (st == IRT_INT ? IRCALL_softfp_i2d : IRCALL_softfp_ui2d) :
712 (st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f));
713 asm_callid(as, ir, cid);
714 }
715#else
716 asm_callid(as, ir, IRCALL_softfp_i2d);
717#endif
718 } else if (stfp) { /* FP to integer conversion. */
719 if (irt_isguard(ir->t)) {
720 /* Checked conversions are only supported from number to int. */
721 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
722 "bad type for checked CONV");
723 asm_tointg(as, ir, RID_NONE);
724 } else {
725 IRCallID cid = irt_is64(ir->t) ?
726 ((st == IRT_NUM) ?
727 (irt_isi64(ir->t) ? IRCALL_fp64_d2l : IRCALL_fp64_d2ul) :
728 (irt_isi64(ir->t) ? IRCALL_fp64_f2l : IRCALL_fp64_f2ul)) :
729 ((st == IRT_NUM) ?
730 (irt_isint(ir->t) ? IRCALL_softfp_d2i : IRCALL_softfp_d2ui) :
731 (irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui));
732 asm_callid(as, ir, cid);
733 }
734 } else
735#endif
736#endif
737 {
495 Reg dest = ra_dest(as, ir, RSET_GPR); 738 Reg dest = ra_dest(as, ir, RSET_GPR);
496 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */ 739 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
497 Reg left = ra_alloc1(as, ir->op1, RSET_GPR); 740 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
498 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); 741 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
499 if ((ir->op2 & IRCONV_SEXT)) { 742 if ((ir->op2 & IRCONV_SEXT)) {
500 if ((as->flags & JIT_F_MIPS32R2)) { 743 if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
501 emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left); 744 emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
502 } else { 745 } else {
503 uint32_t shift = st == IRT_I8 ? 24 : 16; 746 uint32_t shift = st == IRT_I8 ? 24 : 16;
@@ -509,94 +752,171 @@ static void asm_conv(ASMState *as, IRIns *ir)
509 (int32_t)(st == IRT_U8 ? 0xff : 0xffff)); 752 (int32_t)(st == IRT_U8 ? 0xff : 0xffff));
510 } 753 }
511 } else { /* 32/64 bit integer conversions. */ 754 } else { /* 32/64 bit integer conversions. */
755#if LJ_32
512 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */ 756 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
513 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */ 757 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
758#else
759 if (irt_is64(ir->t)) {
760 if (st64) {
761 /* 64/64 bit no-op (cast)*/
762 ra_leftov(as, dest, lref);
763 } else {
764 Reg left = ra_alloc1(as, lref, RSET_GPR);
765 if ((ir->op2 & IRCONV_SEXT)) { /* 32 to 64 bit sign extension. */
766 emit_dta(as, MIPSI_SLL, dest, left, 0);
767 } else { /* 32 to 64 bit zero extension. */
768 emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
769 }
770 }
771 } else {
772 if (st64 && !(ir->op2 & IRCONV_NONE)) {
773 /* This is either a 32 bit reg/reg mov which zeroes the hiword
774 ** or a load of the loword from a 64 bit address.
775 */
776 Reg left = ra_alloc1(as, lref, RSET_GPR);
777 emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
778 } else { /* 32/32 bit no-op (cast). */
779 /* Do nothing, but may need to move regs. */
780 ra_leftov(as, dest, lref);
781 }
782 }
783#endif
514 } 784 }
515 } 785 }
516} 786}
517 787
518#if LJ_HASFFI
519static void asm_conv64(ASMState *as, IRIns *ir)
520{
521 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
522 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
523 IRCallID id;
524 const CCallInfo *ci;
525 IRRef args[2];
526 args[LJ_BE?0:1] = ir->op1;
527 args[LJ_BE?1:0] = (ir-1)->op1;
528 if (st == IRT_NUM || st == IRT_FLOAT) {
529 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
530 ir--;
531 } else {
532 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
533 }
534 ci = &lj_ir_callinfo[id];
535 asm_setupresult(as, ir, ci);
536 asm_gencall(as, ci, args);
537}
538#endif
539
540static void asm_strto(ASMState *as, IRIns *ir) 788static void asm_strto(ASMState *as, IRIns *ir)
541{ 789{
542 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num]; 790 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
543 IRRef args[2]; 791 IRRef args[2];
792 int32_t ofs = 0;
793#if LJ_SOFTFP32
794 ra_evictset(as, RSET_SCRATCH);
795 if (ra_used(ir)) {
796 if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) &&
797 (ir->s & 1) == LJ_BE && (ir->s ^ 1) == (ir+1)->s) {
798 int i;
799 for (i = 0; i < 2; i++) {
800 Reg r = (ir+i)->r;
801 if (ra_hasreg(r)) {
802 ra_free(as, r);
803 ra_modified(as, r);
804 emit_spload(as, ir+i, r, sps_scale((ir+i)->s));
805 }
806 }
807 ofs = sps_scale(ir->s & ~1);
808 } else {
809 Reg rhi = ra_dest(as, ir+1, RSET_GPR);
810 Reg rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi));
811 emit_tsi(as, MIPSI_LW, rhi, RID_SP, ofs+(LJ_BE?0:4));
812 emit_tsi(as, MIPSI_LW, rlo, RID_SP, ofs+(LJ_BE?4:0));
813 }
814 }
815#else
544 RegSet drop = RSET_SCRATCH; 816 RegSet drop = RSET_SCRATCH;
545 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */ 817 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
546 ra_evictset(as, drop); 818 ra_evictset(as, drop);
819 ofs = sps_scale(ir->s);
820#endif
547 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO); /* Test return status. */ 821 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO); /* Test return status. */
548 args[0] = ir->op1; /* GCstr *str */ 822 args[0] = ir->op1; /* GCstr *str */
549 args[1] = ASMREF_TMP1; /* TValue *n */ 823 args[1] = ASMREF_TMP1; /* TValue *n */
550 asm_gencall(as, ci, args); 824 asm_gencall(as, ci, args);
551 /* Store the result to the spill slot or temp slots. */ 825 /* Store the result to the spill slot or temp slots. */
552 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), 826 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1),
553 RID_SP, sps_scale(ir->s)); 827 RID_SP, ofs);
554} 828}
555 829
556/* Get pointer to TValue. */ 830/* -- Memory references --------------------------------------------------- */
557static void asm_tvptr(ASMState *as, Reg dest, IRRef ref) 831
832#if LJ_64
833/* Store tagged value for ref at base+ofs. */
834static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
558{ 835{
836 RegSet allow = rset_exclude(RSET_GPR, base);
559 IRIns *ir = IR(ref); 837 IRIns *ir = IR(ref);
560 if (irt_isnum(ir->t)) { 838 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
561 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */ 839 "store of IR type %d", irt_type(ir->t));
562 ra_allockreg(as, i32ptr(ir_knum(ir)), dest); 840 if (irref_isk(ref)) {
563 else /* Otherwise force a spill and use the spill slot. */ 841 TValue k;
564 emit_tsi(as, MIPSI_ADDIU, dest, RID_SP, ra_spill(as, ir)); 842 lj_ir_kvalue(as->J->L, &k, ir);
843 emit_tsi(as, MIPSI_SD, ra_allock(as, (int64_t)k.u64, allow), base, ofs);
565 } else { 844 } else {
566 /* Otherwise use g->tmptv to hold the TValue. */ 845 Reg src = ra_alloc1(as, ref, allow);
567 RegSet allow = rset_exclude(RSET_GPR, dest); 846 Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47,
568 Reg type; 847 rset_exclude(allow, src));
569 emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, offsetof(global_State, tmptv)-32768); 848 emit_tsi(as, MIPSI_SD, RID_TMP, base, ofs);
570 if (!irt_ispri(ir->t)) { 849 if (irt_isinteger(ir->t)) {
571 Reg src = ra_alloc1(as, ref, allow); 850 emit_dst(as, MIPSI_DADDU, RID_TMP, RID_TMP, type);
572 emit_setgl(as, src, tmptv.gcr); 851 emit_tsml(as, MIPSI_DEXT, RID_TMP, src, 31, 0);
852 } else {
853 emit_dst(as, MIPSI_DADDU, RID_TMP, src, type);
573 } 854 }
574 type = ra_allock(as, irt_toitype(ir->t), allow);
575 emit_setgl(as, type, tmptv.it);
576 } 855 }
577} 856}
857#endif
578 858
579static void asm_tostr(ASMState *as, IRIns *ir) 859/* Get pointer to TValue. */
860static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
580{ 861{
581 IRRef args[2]; 862 int32_t tmpofs = (int32_t)(offsetof(global_State, tmptv)-32768);
582 args[0] = ASMREF_L; 863 if ((mode & IRTMPREF_IN1)) {
583 as->gcsteps++; 864 IRIns *ir = IR(ref);
584 if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) { 865 if (irt_isnum(ir->t)) {
585 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum]; 866 if ((mode & IRTMPREF_OUT1)) {
586 args[1] = ASMREF_TMP1; /* const lua_Number * */ 867#if LJ_SOFTFP
587 asm_setupresult(as, ir, ci); /* GCstr * */ 868 emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
588 asm_gencall(as, ci, args); 869#if LJ_64
589 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1); 870 emit_setgl(as, ra_alloc1(as, ref, RSET_GPR), tmptv.u64);
871#else
872 lj_assertA(irref_isk(ref), "unsplit FP op");
873 emit_setgl(as,
874 ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, RSET_GPR),
875 tmptv.u32.lo);
876 emit_setgl(as,
877 ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, RSET_GPR),
878 tmptv.u32.hi);
879#endif
880#else
881 Reg src = ra_alloc1(as, ref, RSET_FPR);
882 emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
883 emit_tsi(as, MIPSI_SDC1, (src & 31), RID_JGL, tmpofs);
884#endif
885 } else if (irref_isk(ref)) {
886 /* Use the number constant itself as a TValue. */
887 ra_allockreg(as, igcptr(ir_knum(ir)), dest);
888 } else {
889#if LJ_SOFTFP32
890 lj_assertA(0, "unsplit FP op");
891#else
892 /* Otherwise force a spill and use the spill slot. */
893 emit_tsi(as, MIPSI_AADDIU, dest, RID_SP, ra_spill(as, ir));
894#endif
895 }
896 } else {
897 /* Otherwise use g->tmptv to hold the TValue. */
898#if LJ_32
899 Reg type;
900 emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, tmpofs);
901 if (!irt_ispri(ir->t)) {
902 Reg src = ra_alloc1(as, ref, RSET_GPR);
903 emit_setgl(as, src, tmptv.gcr);
904 }
905 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t))
906 type = ra_alloc1(as, ref+1, RSET_GPR);
907 else
908 type = ra_allock(as, (int32_t)irt_toitype(ir->t), RSET_GPR);
909 emit_setgl(as, type, tmptv.it);
910#else
911 asm_tvstore64(as, dest, 0, ref);
912 emit_tsi(as, MIPSI_DADDIU, dest, RID_JGL, tmpofs);
913#endif
914 }
590 } else { 915 } else {
591 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint]; 916 emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
592 args[1] = ir->op1; /* int32_t k */
593 asm_setupresult(as, ir, ci); /* GCstr * */
594 asm_gencall(as, ci, args);
595 } 917 }
596} 918}
597 919
598/* -- Memory references --------------------------------------------------- */
599
600static void asm_aref(ASMState *as, IRIns *ir) 920static void asm_aref(ASMState *as, IRIns *ir)
601{ 921{
602 Reg dest = ra_dest(as, ir, RSET_GPR); 922 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -608,14 +928,18 @@ static void asm_aref(ASMState *as, IRIns *ir)
608 ofs += 8*IR(ir->op2)->i; 928 ofs += 8*IR(ir->op2)->i;
609 if (checki16(ofs)) { 929 if (checki16(ofs)) {
610 base = ra_alloc1(as, refa, RSET_GPR); 930 base = ra_alloc1(as, refa, RSET_GPR);
611 emit_tsi(as, MIPSI_ADDIU, dest, base, ofs); 931 emit_tsi(as, MIPSI_AADDIU, dest, base, ofs);
612 return; 932 return;
613 } 933 }
614 } 934 }
615 base = ra_alloc1(as, ir->op1, RSET_GPR); 935 base = ra_alloc1(as, ir->op1, RSET_GPR);
616 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base)); 936 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
617 emit_dst(as, MIPSI_ADDU, dest, RID_TMP, base); 937#if !LJ_TARGET_MIPSR6
938 emit_dst(as, MIPSI_AADDU, dest, RID_TMP, base);
618 emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3); 939 emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
940#else
941 emit_dst(as, MIPSI_ALSA | MIPSF_A(3-1), dest, idx, base);
942#endif
619} 943}
620 944
621/* Inlined hash lookup. Specialized for key type and for const keys. 945/* Inlined hash lookup. Specialized for key type and for const keys.
@@ -626,21 +950,25 @@ static void asm_aref(ASMState *as, IRIns *ir)
626** } while ((n = nextnode(n))); 950** } while ((n = nextnode(n)));
627** return niltv(L); 951** return niltv(L);
628*/ 952*/
629static void asm_href(ASMState *as, IRIns *ir) 953static void asm_href(ASMState *as, IRIns *ir, IROp merge)
630{ 954{
631 RegSet allow = RSET_GPR; 955 RegSet allow = RSET_GPR;
632 int destused = ra_used(ir); 956 int destused = ra_used(ir);
633 Reg dest = ra_dest(as, ir, allow); 957 Reg dest = ra_dest(as, ir, allow);
634 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest)); 958 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
635 Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2; 959 Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
960#if LJ_64
961 Reg cmp64 = RID_NONE;
962#endif
636 IRRef refkey = ir->op2; 963 IRRef refkey = ir->op2;
637 IRIns *irkey = IR(refkey); 964 IRIns *irkey = IR(refkey);
965 int isk = irref_isk(refkey);
638 IRType1 kt = irkey->t; 966 IRType1 kt = irkey->t;
639 uint32_t khash; 967 uint32_t khash;
640 MCLabel l_end, l_loop, l_next; 968 MCLabel l_end, l_loop, l_next;
641 969
642 rset_clear(allow, tab); 970 rset_clear(allow, tab);
643 if (irt_isnum(kt)) { 971 if (!LJ_SOFTFP && irt_isnum(kt)) {
644 key = ra_alloc1(as, refkey, RSET_FPR); 972 key = ra_alloc1(as, refkey, RSET_FPR);
645 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key)); 973 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
646 } else { 974 } else {
@@ -648,31 +976,76 @@ static void asm_href(ASMState *as, IRIns *ir)
648 key = ra_alloc1(as, refkey, allow); 976 key = ra_alloc1(as, refkey, allow);
649 rset_clear(allow, key); 977 rset_clear(allow, key);
650 } 978 }
651 type = ra_allock(as, irt_toitype(irkey->t), allow); 979#if LJ_32
652 rset_clear(allow, type); 980 if (LJ_SOFTFP && irkey[1].o == IR_HIOP) {
981 if (ra_hasreg((irkey+1)->r)) {
982 type = tmpnum = (irkey+1)->r;
983 tmp1 = ra_scratch(as, allow);
984 rset_clear(allow, tmp1);
985 ra_noweak(as, tmpnum);
986 } else {
987 type = tmpnum = ra_allocref(as, refkey+1, allow);
988 }
989 rset_clear(allow, tmpnum);
990 } else {
991 type = ra_allock(as, (int32_t)irt_toitype(kt), allow);
992 rset_clear(allow, type);
993 }
994#endif
653 } 995 }
654 tmp2 = ra_scratch(as, allow); 996 tmp2 = ra_scratch(as, allow);
655 rset_clear(allow, tmp2); 997 rset_clear(allow, tmp2);
998#if LJ_64
999 if (LJ_SOFTFP || !irt_isnum(kt)) {
1000 /* Allocate cmp64 register used for 64-bit comparisons */
1001 if (LJ_SOFTFP && irt_isnum(kt)) {
1002 cmp64 = key;
1003 } else if (!isk && irt_isaddr(kt)) {
1004 cmp64 = tmp2;
1005 } else {
1006 int64_t k;
1007 if (isk && irt_isaddr(kt)) {
1008 k = ((int64_t)irt_toitype(kt) << 47) | irkey[1].tv.u64;
1009 } else {
1010 lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
1011 k = ~((int64_t)~irt_toitype(kt) << 47);
1012 }
1013 cmp64 = ra_allock(as, k, allow);
1014 rset_clear(allow, cmp64);
1015 }
1016 }
1017#endif
656 1018
657 /* Key not found in chain: load niltv. */ 1019 /* Key not found in chain: jump to exit (if merged) or load niltv. */
658 l_end = emit_label(as); 1020 l_end = emit_label(as);
659 if (destused) 1021 as->invmcp = NULL;
1022 if (merge == IR_NE)
1023 asm_guard(as, MIPSI_B, RID_ZERO, RID_ZERO);
1024 else if (destused)
660 emit_loada(as, dest, niltvg(J2G(as->J))); 1025 emit_loada(as, dest, niltvg(J2G(as->J)));
661 else
662 *--as->mcp = MIPSI_NOP;
663 /* Follow hash chain until the end. */ 1026 /* Follow hash chain until the end. */
664 emit_move(as, dest, tmp1); 1027 emit_move(as, dest, tmp1);
665 l_loop = --as->mcp; 1028 l_loop = --as->mcp;
666 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, next)); 1029 emit_tsi(as, MIPSI_AL, tmp1, dest, (int32_t)offsetof(Node, next));
667 l_next = emit_label(as); 1030 l_next = emit_label(as);
668 1031
669 /* Type and value comparison. */ 1032 /* Type and value comparison. */
670 if (irt_isnum(kt)) { 1033 if (merge == IR_EQ) { /* Must match asm_guard(). */
1034 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
1035 l_end = asm_exitstub_addr(as);
1036 }
1037 if (!LJ_SOFTFP && irt_isnum(kt)) {
1038#if !LJ_TARGET_MIPSR6
671 emit_branch(as, MIPSI_BC1T, 0, 0, l_end); 1039 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
672 emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key); 1040 emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
673 emit_tg(as, MIPSI_MFC1, tmp1, key+1); 1041#else
1042 emit_branch(as, MIPSI_BC1NEZ, 0, (tmpnum&31), l_end);
1043 emit_fgh(as, MIPSI_CMP_EQ_D, tmpnum, tmpnum, key);
1044#endif
1045 *--as->mcp = MIPSI_NOP; /* Avoid NaN comparison overhead. */
674 emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next); 1046 emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
675 emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM); 1047 emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
1048#if LJ_32
676 emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n)); 1049 emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
677 } else { 1050 } else {
678 if (irt_ispri(kt)) { 1051 if (irt_ispri(kt)) {
@@ -685,36 +1058,52 @@ static void asm_href(ASMState *as, IRIns *ir)
685 } 1058 }
686 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it)); 1059 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
687 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu); 1060 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
1061#else
1062 emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 15);
1063 emit_tg(as, MIPSI_DMTC1, tmp1, tmpnum);
1064 emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
1065 } else {
1066 emit_branch(as, MIPSI_BEQ, tmp1, cmp64, l_end);
1067 emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
1068 }
1069 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
1070 if (!isk && irt_isaddr(kt)) {
1071 type = ra_allock(as, (int64_t)irt_toitype(kt) << 47, allow);
1072 emit_dst(as, MIPSI_DADDU, tmp2, key, type);
1073 rset_clear(allow, type);
1074 }
1075#endif
688 1076
689 /* Load main position relative to tab->node into dest. */ 1077 /* Load main position relative to tab->node into dest. */
690 khash = irref_isk(refkey) ? ir_khash(irkey) : 1; 1078 khash = isk ? ir_khash(as, irkey) : 1;
691 if (khash == 0) { 1079 if (khash == 0) {
692 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node)); 1080 emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
693 } else { 1081 } else {
694 Reg tmphash = tmp1; 1082 Reg tmphash = tmp1;
695 if (irref_isk(refkey)) 1083 if (isk)
696 tmphash = ra_allock(as, khash, allow); 1084 tmphash = ra_allock(as, khash, allow);
697 emit_dst(as, MIPSI_ADDU, dest, dest, tmp1); 1085 emit_dst(as, MIPSI_AADDU, dest, dest, tmp1);
698 lua_assert(sizeof(Node) == 24); 1086 lj_assertA(sizeof(Node) == 24, "bad Node size");
699 emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1); 1087 emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
700 emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3); 1088 emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
701 emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5); 1089 emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
702 emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash); 1090 emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
703 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node)); 1091 emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
704 emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask)); 1092 emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
705 if (irref_isk(refkey)) { 1093 if (isk) {
706 /* Nothing to do. */ 1094 /* Nothing to do. */
707 } else if (irt_isstr(kt)) { 1095 } else if (irt_isstr(kt)) {
708 emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash)); 1096 emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, sid));
709 } else { /* Must match with hash*() in lj_tab.c. */ 1097 } else { /* Must match with hash*() in lj_tab.c. */
710 emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2); 1098 emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2);
711 emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31); 1099 emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31);
712 emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2); 1100 emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
713 emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31); 1101 emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
714 emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest); 1102 emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
715 if (irt_isnum(kt)) { 1103#if LJ_32
1104 if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
716 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1); 1105 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
717 if ((as->flags & JIT_F_MIPS32R2)) { 1106 if ((as->flags & JIT_F_MIPSXXR2)) {
718 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31); 1107 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
719 } else { 1108 } else {
720 emit_dst(as, MIPSI_OR, dest, dest, tmp1); 1109 emit_dst(as, MIPSI_OR, dest, dest, tmp1);
@@ -722,13 +1111,35 @@ static void asm_href(ASMState *as, IRIns *ir)
722 emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31); 1111 emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31);
723 } 1112 }
724 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1); 1113 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
1114#if LJ_SOFTFP
1115 emit_ds(as, MIPSI_MOVE, tmp1, type);
1116 emit_ds(as, MIPSI_MOVE, tmp2, key);
1117#else
725 emit_tg(as, MIPSI_MFC1, tmp2, key); 1118 emit_tg(as, MIPSI_MFC1, tmp2, key);
726 emit_tg(as, MIPSI_MFC1, tmp1, key+1); 1119 emit_tg(as, MIPSI_MFC1, tmp1, key+1);
1120#endif
727 } else { 1121 } else {
728 emit_dst(as, MIPSI_XOR, tmp2, key, tmp1); 1122 emit_dst(as, MIPSI_XOR, tmp2, key, tmp1);
729 emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31); 1123 emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
730 emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow)); 1124 emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
731 } 1125 }
1126#else
1127 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
1128 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
1129 if (irt_isnum(kt)) {
1130 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
1131 emit_dta(as, MIPSI_DSRA32, tmp1, LJ_SOFTFP ? key : tmp1, 0);
1132 emit_dta(as, MIPSI_SLL, tmp2, LJ_SOFTFP ? key : tmp1, 0);
1133#if !LJ_SOFTFP
1134 emit_tg(as, MIPSI_DMFC1, tmp1, key);
1135#endif
1136 } else {
1137 checkmclim(as);
1138 emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
1139 emit_dta(as, MIPSI_SLL, tmp2, key, 0);
1140 emit_dst(as, MIPSI_DADDU, tmp1, key, type);
1141 }
1142#endif
732 } 1143 }
733 } 1144 }
734} 1145}
@@ -741,17 +1152,24 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
741 int32_t kofs = ofs + (int32_t)offsetof(Node, key); 1152 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
742 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE; 1153 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
743 Reg node = ra_alloc1(as, ir->op1, RSET_GPR); 1154 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
744 Reg key = RID_NONE, type = RID_TMP, idx = node;
745 RegSet allow = rset_exclude(RSET_GPR, node); 1155 RegSet allow = rset_exclude(RSET_GPR, node);
1156 Reg idx = node;
1157#if LJ_32
1158 Reg key = RID_NONE, type = RID_TMP;
746 int32_t lo, hi; 1159 int32_t lo, hi;
747 lua_assert(ofs % sizeof(Node) == 0); 1160#else
1161 Reg key = ra_scratch(as, allow);
1162 int64_t k;
1163#endif
1164 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
748 if (ofs > 32736) { 1165 if (ofs > 32736) {
749 idx = dest; 1166 idx = dest;
750 rset_clear(allow, dest); 1167 rset_clear(allow, dest);
751 kofs = (int32_t)offsetof(Node, key); 1168 kofs = (int32_t)offsetof(Node, key);
752 } else if (ra_hasreg(dest)) { 1169 } else if (ra_hasreg(dest)) {
753 emit_tsi(as, MIPSI_ADDIU, dest, node, ofs); 1170 emit_tsi(as, MIPSI_AADDIU, dest, node, ofs);
754 } 1171 }
1172#if LJ_32
755 if (!irt_ispri(irkey->t)) { 1173 if (!irt_ispri(irkey->t)) {
756 key = ra_scratch(as, allow); 1174 key = ra_scratch(as, allow);
757 rset_clear(allow, key); 1175 rset_clear(allow, key);
@@ -770,22 +1188,20 @@ nolo:
770 asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO); 1188 asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
771 if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0)); 1189 if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
772 emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4)); 1190 emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
773 if (ofs > 32736) 1191#else
774 emit_tsi(as, MIPSI_ADDU, dest, node, ra_allock(as, ofs, allow)); 1192 if (irt_ispri(irkey->t)) {
775} 1193 lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
776 1194 k = ~((int64_t)~irt_toitype(irkey->t) << 47);
777static void asm_newref(ASMState *as, IRIns *ir) 1195 } else if (irt_isnum(irkey->t)) {
778{ 1196 k = (int64_t)ir_knum(irkey)->u64;
779 if (ir->r != RID_SINK) { 1197 } else {
780 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey]; 1198 k = ((int64_t)irt_toitype(irkey->t) << 47) | (int64_t)ir_kgc(irkey);
781 IRRef args[3];
782 args[0] = ASMREF_L; /* lua_State *L */
783 args[1] = ir->op1; /* GCtab *t */
784 args[2] = ASMREF_TMP1; /* cTValue *key */
785 asm_setupresult(as, ir, ci); /* TValue * */
786 asm_gencall(as, ci, args);
787 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
788 } 1199 }
1200 asm_guard(as, MIPSI_BNE, key, ra_allock(as, k, allow));
1201 emit_tsi(as, MIPSI_LD, key, idx, kofs);
1202#endif
1203 if (ofs > 32736)
1204 emit_tsi(as, MIPSI_AADDU, dest, node, ra_allock(as, ofs, allow));
789} 1205}
790 1206
791static void asm_uref(ASMState *as, IRIns *ir) 1207static void asm_uref(ASMState *as, IRIns *ir)
@@ -794,30 +1210,31 @@ static void asm_uref(ASMState *as, IRIns *ir)
794 if (irref_isk(ir->op1)) { 1210 if (irref_isk(ir->op1)) {
795 GCfunc *fn = ir_kfunc(IR(ir->op1)); 1211 GCfunc *fn = ir_kfunc(IR(ir->op1));
796 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v; 1212 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
797 emit_lsptr(as, MIPSI_LW, dest, v, RSET_GPR); 1213 emit_lsptr(as, MIPSI_AL, dest, v, RSET_GPR);
798 } else { 1214 } else {
799 Reg uv = ra_scratch(as, RSET_GPR); 1215 Reg uv = ra_scratch(as, RSET_GPR);
800 Reg func = ra_alloc1(as, ir->op1, RSET_GPR); 1216 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
801 if (ir->o == IR_UREFC) { 1217 if (ir->o == IR_UREFC) {
802 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO); 1218 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
803 emit_tsi(as, MIPSI_ADDIU, dest, uv, (int32_t)offsetof(GCupval, tv)); 1219 emit_tsi(as, MIPSI_AADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
804 emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed)); 1220 emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
805 } else { 1221 } else {
806 emit_tsi(as, MIPSI_LW, dest, uv, (int32_t)offsetof(GCupval, v)); 1222 emit_tsi(as, MIPSI_AL, dest, uv, (int32_t)offsetof(GCupval, v));
807 } 1223 }
808 emit_tsi(as, MIPSI_LW, uv, func, 1224 emit_tsi(as, MIPSI_AL, uv, func, (int32_t)offsetof(GCfuncL, uvptr) +
809 (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8)); 1225 (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
810 } 1226 }
811} 1227}
812 1228
813static void asm_fref(ASMState *as, IRIns *ir) 1229static void asm_fref(ASMState *as, IRIns *ir)
814{ 1230{
815 UNUSED(as); UNUSED(ir); 1231 UNUSED(as); UNUSED(ir);
816 lua_assert(!ra_used(ir)); 1232 lj_assertA(!ra_used(ir), "unfused FREF");
817} 1233}
818 1234
819static void asm_strref(ASMState *as, IRIns *ir) 1235static void asm_strref(ASMState *as, IRIns *ir)
820{ 1236{
1237#if LJ_32
821 Reg dest = ra_dest(as, ir, RSET_GPR); 1238 Reg dest = ra_dest(as, ir, RSET_GPR);
822 IRRef ref = ir->op2, refk = ir->op1; 1239 IRRef ref = ir->op2, refk = ir->op1;
823 int32_t ofs = (int32_t)sizeof(GCstr); 1240 int32_t ofs = (int32_t)sizeof(GCstr);
@@ -849,49 +1266,79 @@ static void asm_strref(ASMState *as, IRIns *ir)
849 else 1266 else
850 emit_dst(as, MIPSI_ADDU, dest, r, 1267 emit_dst(as, MIPSI_ADDU, dest, r,
851 ra_allock(as, ofs, rset_exclude(RSET_GPR, r))); 1268 ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
1269#else
1270 RegSet allow = RSET_GPR;
1271 Reg dest = ra_dest(as, ir, allow);
1272 Reg base = ra_alloc1(as, ir->op1, allow);
1273 IRIns *irr = IR(ir->op2);
1274 int32_t ofs = sizeof(GCstr);
1275 rset_clear(allow, base);
1276 if (irref_isk(ir->op2) && checki16(ofs + irr->i)) {
1277 emit_tsi(as, MIPSI_DADDIU, dest, base, ofs + irr->i);
1278 } else {
1279 emit_tsi(as, MIPSI_DADDIU, dest, dest, ofs);
1280 emit_dst(as, MIPSI_DADDU, dest, base, ra_alloc1(as, ir->op2, allow));
1281 }
1282#endif
852} 1283}
853 1284
854/* -- Loads and stores ---------------------------------------------------- */ 1285/* -- Loads and stores ---------------------------------------------------- */
855 1286
856static MIPSIns asm_fxloadins(IRIns *ir) 1287static MIPSIns asm_fxloadins(ASMState *as, IRIns *ir)
857{ 1288{
1289 UNUSED(as);
858 switch (irt_type(ir->t)) { 1290 switch (irt_type(ir->t)) {
859 case IRT_I8: return MIPSI_LB; 1291 case IRT_I8: return MIPSI_LB;
860 case IRT_U8: return MIPSI_LBU; 1292 case IRT_U8: return MIPSI_LBU;
861 case IRT_I16: return MIPSI_LH; 1293 case IRT_I16: return MIPSI_LH;
862 case IRT_U16: return MIPSI_LHU; 1294 case IRT_U16: return MIPSI_LHU;
863 case IRT_NUM: return MIPSI_LDC1; 1295 case IRT_NUM:
864 case IRT_FLOAT: return MIPSI_LWC1; 1296 lj_assertA(!LJ_SOFTFP32, "unsplit FP op");
865 default: return MIPSI_LW; 1297 if (!LJ_SOFTFP) return MIPSI_LDC1;
1298 /* fallthrough */
1299 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_LWC1;
1300 /* fallthrough */
1301 default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_LD : MIPSI_LW;
866 } 1302 }
867} 1303}
868 1304
869static MIPSIns asm_fxstoreins(IRIns *ir) 1305static MIPSIns asm_fxstoreins(ASMState *as, IRIns *ir)
870{ 1306{
1307 UNUSED(as);
871 switch (irt_type(ir->t)) { 1308 switch (irt_type(ir->t)) {
872 case IRT_I8: case IRT_U8: return MIPSI_SB; 1309 case IRT_I8: case IRT_U8: return MIPSI_SB;
873 case IRT_I16: case IRT_U16: return MIPSI_SH; 1310 case IRT_I16: case IRT_U16: return MIPSI_SH;
874 case IRT_NUM: return MIPSI_SDC1; 1311 case IRT_NUM:
875 case IRT_FLOAT: return MIPSI_SWC1; 1312 lj_assertA(!LJ_SOFTFP32, "unsplit FP op");
876 default: return MIPSI_SW; 1313 if (!LJ_SOFTFP) return MIPSI_SDC1;
1314 /* fallthrough */
1315 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_SWC1;
1316 /* fallthrough */
1317 default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_SD : MIPSI_SW;
877 } 1318 }
878} 1319}
879 1320
880static void asm_fload(ASMState *as, IRIns *ir) 1321static void asm_fload(ASMState *as, IRIns *ir)
881{ 1322{
882 Reg dest = ra_dest(as, ir, RSET_GPR); 1323 Reg dest = ra_dest(as, ir, RSET_GPR);
883 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR); 1324 MIPSIns mi = asm_fxloadins(as, ir);
884 MIPSIns mi = asm_fxloadins(ir); 1325 Reg idx;
885 int32_t ofs; 1326 int32_t ofs;
886 if (ir->op2 == IRFL_TAB_ARRAY) { 1327 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
887 ofs = asm_fuseabase(as, ir->op1); 1328 idx = RID_JGL;
888 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */ 1329 ofs = (ir->op2 << 2) - 32768 - GG_OFS(g);
889 emit_tsi(as, MIPSI_ADDIU, dest, idx, ofs); 1330 } else {
890 return; 1331 idx = ra_alloc1(as, ir->op1, RSET_GPR);
1332 if (ir->op2 == IRFL_TAB_ARRAY) {
1333 ofs = asm_fuseabase(as, ir->op1);
1334 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
1335 emit_tsi(as, MIPSI_AADDIU, dest, idx, ofs);
1336 return;
1337 }
891 } 1338 }
1339 ofs = field_ofs[ir->op2];
1340 lj_assertA(!irt_isfp(ir->t), "bad FP FLOAD");
892 } 1341 }
893 ofs = field_ofs[ir->op2];
894 lua_assert(!irt_isfp(ir->t));
895 emit_tsi(as, mi, dest, idx, ofs); 1342 emit_tsi(as, mi, dest, idx, ofs);
896} 1343}
897 1344
@@ -902,51 +1349,90 @@ static void asm_fstore(ASMState *as, IRIns *ir)
902 IRIns *irf = IR(ir->op1); 1349 IRIns *irf = IR(ir->op1);
903 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src)); 1350 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
904 int32_t ofs = field_ofs[irf->op2]; 1351 int32_t ofs = field_ofs[irf->op2];
905 MIPSIns mi = asm_fxstoreins(ir); 1352 MIPSIns mi = asm_fxstoreins(as, ir);
906 lua_assert(!irt_isfp(ir->t)); 1353 lj_assertA(!irt_isfp(ir->t), "bad FP FSTORE");
907 emit_tsi(as, mi, src, idx, ofs); 1354 emit_tsi(as, mi, src, idx, ofs);
908 } 1355 }
909} 1356}
910 1357
911static void asm_xload(ASMState *as, IRIns *ir) 1358static void asm_xload(ASMState *as, IRIns *ir)
912{ 1359{
913 Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); 1360 Reg dest = ra_dest(as, ir,
914 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED)); 1361 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
915 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0); 1362 lj_assertA(LJ_TARGET_UNALIGNED || !(ir->op2 & IRXLOAD_UNALIGNED),
1363 "unaligned XLOAD");
1364 asm_fusexref(as, asm_fxloadins(as, ir), dest, ir->op1, RSET_GPR, 0);
916} 1365}
917 1366
918static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs) 1367static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
919{ 1368{
920 if (ir->r != RID_SINK) { 1369 if (ir->r != RID_SINK) {
921 Reg src = ra_alloc1z(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); 1370 Reg src = ra_alloc1z(as, ir->op2,
922 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1, 1371 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1372 asm_fusexref(as, asm_fxstoreins(as, ir), src, ir->op1,
923 rset_exclude(RSET_GPR, src), ofs); 1373 rset_exclude(RSET_GPR, src), ofs);
924 } 1374 }
925} 1375}
926 1376
1377#define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
1378
927static void asm_ahuvload(ASMState *as, IRIns *ir) 1379static void asm_ahuvload(ASMState *as, IRIns *ir)
928{ 1380{
929 IRType1 t = ir->t; 1381 int hiop = (LJ_SOFTFP32 && (ir+1)->o == IR_HIOP);
930 Reg dest = RID_NONE, type = RID_TMP, idx; 1382 Reg dest = RID_NONE, type = RID_TMP, idx;
931 RegSet allow = RSET_GPR; 1383 RegSet allow = RSET_GPR;
932 int32_t ofs = 0; 1384 int32_t ofs = 0;
1385 IRType1 t = ir->t;
1386 if (hiop) {
1387 t.irt = IRT_NUM;
1388 if (ra_used(ir+1)) {
1389 type = ra_dest(as, ir+1, allow);
1390 rset_clear(allow, type);
1391 }
1392 }
933 if (ra_used(ir)) { 1393 if (ra_used(ir)) {
934 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1394 lj_assertA((LJ_SOFTFP32 ? 0 : irt_isnum(ir->t)) ||
935 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR); 1395 irt_isint(ir->t) || irt_isaddr(ir->t),
1396 "bad load type %d", irt_type(ir->t));
1397 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
936 rset_clear(allow, dest); 1398 rset_clear(allow, dest);
1399#if LJ_64
1400 if (irt_isaddr(t))
1401 emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1402 else if (irt_isint(t))
1403 emit_dta(as, MIPSI_SLL, dest, dest, 0);
1404#endif
937 } 1405 }
938 idx = asm_fuseahuref(as, ir->op1, &ofs, allow); 1406 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1407 if (ir->o == IR_VLOAD) ofs += 8 * ir->op2;
939 rset_clear(allow, idx); 1408 rset_clear(allow, idx);
940 if (irt_isnum(t)) { 1409 if (irt_isnum(t)) {
941 asm_guard(as, MIPSI_BEQ, type, RID_ZERO); 1410 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
942 emit_tsi(as, MIPSI_SLTIU, type, type, (int32_t)LJ_TISNUM); 1411 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
943 if (ra_hasreg(dest))
944 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
945 } else { 1412 } else {
946 asm_guard(as, MIPSI_BNE, type, ra_allock(as, irt_toitype(t), allow)); 1413 asm_guard(as, MIPSI_BNE, type,
947 if (ra_hasreg(dest)) emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0)); 1414 ra_allock(as, (int32_t)irt_toitype(t), allow));
1415 }
1416#if LJ_32
1417 if (ra_hasreg(dest)) {
1418 if (!LJ_SOFTFP && irt_isnum(t))
1419 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1420 else
1421 emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
948 } 1422 }
949 emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4)); 1423 emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
1424#else
1425 if (ra_hasreg(dest)) {
1426 if (!LJ_SOFTFP && irt_isnum(t)) {
1427 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1428 dest = type;
1429 }
1430 } else {
1431 dest = type;
1432 }
1433 emit_dta(as, MIPSI_DSRA32, type, dest, 15);
1434 emit_tsi(as, MIPSI_LD, dest, idx, ofs);
1435#endif
950} 1436}
951 1437
952static void asm_ahustore(ASMState *as, IRIns *ir) 1438static void asm_ahustore(ASMState *as, IRIns *ir)
@@ -956,81 +1442,184 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
956 int32_t ofs = 0; 1442 int32_t ofs = 0;
957 if (ir->r == RID_SINK) 1443 if (ir->r == RID_SINK)
958 return; 1444 return;
959 if (irt_isnum(ir->t)) { 1445 if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
960 src = ra_alloc1(as, ir->op2, RSET_FPR); 1446 src = ra_alloc1(as, ir->op2, LJ_SOFTFP ? RSET_GPR : RSET_FPR);
1447 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1448 emit_hsi(as, LJ_SOFTFP ? MIPSI_SD : MIPSI_SDC1, src, idx, ofs);
961 } else { 1449 } else {
1450#if LJ_32
962 if (!irt_ispri(ir->t)) { 1451 if (!irt_ispri(ir->t)) {
963 src = ra_alloc1(as, ir->op2, allow); 1452 src = ra_alloc1(as, ir->op2, allow);
964 rset_clear(allow, src); 1453 rset_clear(allow, src);
965 } 1454 }
966 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow); 1455 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
1456 type = ra_alloc1(as, (ir+1)->op2, allow);
1457 else
1458 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
967 rset_clear(allow, type); 1459 rset_clear(allow, type);
968 } 1460 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
969 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
970 if (irt_isnum(ir->t)) {
971 emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
972 } else {
973 if (ra_hasreg(src)) 1461 if (ra_hasreg(src))
974 emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0)); 1462 emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
975 emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4)); 1463 emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
1464#else
1465 Reg tmp = RID_TMP;
1466 if (irt_ispri(ir->t)) {
1467 tmp = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
1468 rset_clear(allow, tmp);
1469 } else {
1470 src = ra_alloc1(as, ir->op2, allow);
1471 rset_clear(allow, src);
1472 type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
1473 rset_clear(allow, type);
1474 }
1475 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1476 emit_tsi(as, MIPSI_SD, tmp, idx, ofs);
1477 if (ra_hasreg(src)) {
1478 if (irt_isinteger(ir->t)) {
1479 emit_dst(as, MIPSI_DADDU, tmp, tmp, type);
1480 emit_tsml(as, MIPSI_DEXT, tmp, src, 31, 0);
1481 } else {
1482 emit_dst(as, MIPSI_DADDU, tmp, src, type);
1483 }
1484 }
1485#endif
976 } 1486 }
977} 1487}
978 1488
979static void asm_sload(ASMState *as, IRIns *ir) 1489static void asm_sload(ASMState *as, IRIns *ir)
980{ 1490{
981 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
982 IRType1 t = ir->t;
983 Reg dest = RID_NONE, type = RID_NONE, base; 1491 Reg dest = RID_NONE, type = RID_NONE, base;
984 RegSet allow = RSET_GPR; 1492 RegSet allow = RSET_GPR;
985 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ 1493 IRType1 t = ir->t;
986 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK)); 1494#if LJ_32
987 lua_assert(!irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME))); 1495 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1496 int hiop = (LJ_SOFTFP32 && (ir+1)->o == IR_HIOP);
1497 if (hiop)
1498 t.irt = IRT_NUM;
1499#else
1500 int32_t ofs = 8*((int32_t)ir->op1-2);
1501#endif
1502 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1503 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1504 lj_assertA(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1505 "inconsistent SLOAD variant");
1506#if LJ_SOFTFP32
1507 lj_assertA(!(ir->op2 & IRSLOAD_CONVERT),
1508 "unsplit SLOAD convert"); /* Handled by LJ_SOFTFP SPLIT. */
1509 if (hiop && ra_used(ir+1)) {
1510 type = ra_dest(as, ir+1, allow);
1511 rset_clear(allow, type);
1512 }
1513#else
988 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { 1514 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
989 dest = ra_scratch(as, RSET_FPR); 1515 dest = ra_scratch(as, LJ_SOFTFP ? allow : RSET_FPR);
990 asm_tointg(as, ir, dest); 1516 asm_tointg(as, ir, dest);
991 t.irt = IRT_NUM; /* Continue with a regular number type check. */ 1517 t.irt = IRT_NUM; /* Continue with a regular number type check. */
992 } else if (ra_used(ir)) { 1518 } else
993 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1519#endif
994 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR); 1520 if (ra_used(ir)) {
1521 lj_assertA((LJ_SOFTFP32 ? 0 : irt_isnum(ir->t)) ||
1522 irt_isint(ir->t) || irt_isaddr(ir->t),
1523 "bad SLOAD type %d", irt_type(ir->t));
1524 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
995 rset_clear(allow, dest); 1525 rset_clear(allow, dest);
996 base = ra_alloc1(as, REF_BASE, allow); 1526 base = ra_alloc1(as, REF_BASE, allow);
997 rset_clear(allow, base); 1527 rset_clear(allow, base);
998 if ((ir->op2 & IRSLOAD_CONVERT)) { 1528 if (!LJ_SOFTFP32 && (ir->op2 & IRSLOAD_CONVERT)) {
999 if (irt_isint(t)) { 1529 if (irt_isint(t)) {
1000 Reg tmp = ra_scratch(as, RSET_FPR); 1530 Reg tmp = ra_scratch(as, LJ_SOFTFP ? RSET_GPR : RSET_FPR);
1531#if LJ_SOFTFP
1532 ra_evictset(as, rset_exclude(RSET_SCRATCH, dest));
1533 ra_destreg(as, ir, RID_RET);
1534 emit_call(as, (void *)lj_ir_callinfo[IRCALL_softfp_d2i].func, 0);
1535 if (tmp != REGARG_FIRSTGPR)
1536 emit_move(as, REGARG_FIRSTGPR, tmp);
1537#else
1001 emit_tg(as, MIPSI_MFC1, dest, tmp); 1538 emit_tg(as, MIPSI_MFC1, dest, tmp);
1002 emit_fg(as, MIPSI_CVT_W_D, tmp, tmp); 1539 emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
1540#endif
1003 dest = tmp; 1541 dest = tmp;
1004 t.irt = IRT_NUM; /* Check for original type. */ 1542 t.irt = IRT_NUM; /* Check for original type. */
1005 } else { 1543 } else {
1006 Reg tmp = ra_scratch(as, RSET_GPR); 1544 Reg tmp = ra_scratch(as, RSET_GPR);
1545#if LJ_SOFTFP
1546 ra_evictset(as, rset_exclude(RSET_SCRATCH, dest));
1547 ra_destreg(as, ir, RID_RET);
1548 emit_call(as, (void *)lj_ir_callinfo[IRCALL_softfp_i2d].func, 0);
1549 emit_dta(as, MIPSI_SLL, REGARG_FIRSTGPR, tmp, 0);
1550#else
1007 emit_fg(as, MIPSI_CVT_D_W, dest, dest); 1551 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
1008 emit_tg(as, MIPSI_MTC1, tmp, dest); 1552 emit_tg(as, MIPSI_MTC1, tmp, dest);
1553#endif
1009 dest = tmp; 1554 dest = tmp;
1010 t.irt = IRT_INT; /* Check for original type. */ 1555 t.irt = IRT_INT; /* Check for original type. */
1011 } 1556 }
1012 } 1557 }
1558#if LJ_64
1559 else if (irt_isaddr(t)) {
1560 /* Clear type from pointers. */
1561 emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1562 } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
1563 /* Sign-extend integers. */
1564 emit_dta(as, MIPSI_SLL, dest, dest, 0);
1565 }
1566#endif
1013 goto dotypecheck; 1567 goto dotypecheck;
1014 } 1568 }
1015 base = ra_alloc1(as, REF_BASE, allow); 1569 base = ra_alloc1(as, REF_BASE, allow);
1016 rset_clear(allow, base); 1570 rset_clear(allow, base);
1017dotypecheck: 1571dotypecheck:
1018 if (irt_isnum(t)) { 1572#if LJ_32
1019 if ((ir->op2 & IRSLOAD_TYPECHECK)) { 1573 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1020 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO); 1574 if (ra_noreg(type))
1021 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
1022 type = RID_TMP; 1575 type = RID_TMP;
1576 if (irt_isnum(t)) {
1577 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1578 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1579 } else {
1580 Reg ktype = ra_allock(as, (ir->op2 & IRSLOAD_KEYINDEX) ? LJ_KEYINDEX : irt_toitype(t), allow);
1581 asm_guard(as, MIPSI_BNE, type, ktype);
1023 } 1582 }
1024 if (ra_hasreg(dest)) emit_hsi(as, MIPSI_LDC1, dest, base, ofs); 1583 }
1025 } else { 1584 if (ra_hasreg(dest)) {
1026 if ((ir->op2 & IRSLOAD_TYPECHECK)) { 1585 if (!LJ_SOFTFP && irt_isnum(t))
1027 Reg ktype = ra_allock(as, irt_toitype(t), allow); 1586 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1028 asm_guard(as, MIPSI_BNE, RID_TMP, ktype); 1587 else
1029 type = RID_TMP; 1588 emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
1589 }
1590 if (ra_hasreg(type))
1591 emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
1592#else
1593 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1594 type = dest < RID_MAX_GPR ? dest : RID_TMP;
1595 if (irt_ispri(t)) {
1596 asm_guard(as, MIPSI_BNE, type,
1597 ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow));
1598 } else if ((ir->op2 & IRSLOAD_KEYINDEX)) {
1599 asm_guard(as, MIPSI_BNE, RID_TMP,
1600 ra_allock(as, (int32_t)LJ_KEYINDEX, allow));
1601 emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 0);
1602 } else {
1603 if (irt_isnum(t)) {
1604 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1605 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
1606 if (!LJ_SOFTFP && ra_hasreg(dest))
1607 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1608 } else {
1609 asm_guard(as, MIPSI_BNE, RID_TMP,
1610 ra_allock(as, (int32_t)irt_toitype(t), allow));
1611 }
1612 emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 15);
1030 } 1613 }
1031 if (ra_hasreg(dest)) emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0)); 1614 emit_tsi(as, MIPSI_LD, type, base, ofs);
1615 } else if (ra_hasreg(dest)) {
1616 if (!LJ_SOFTFP && irt_isnum(t))
1617 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1618 else
1619 emit_tsi(as, irt_isint(t) ? MIPSI_LW : MIPSI_LD, dest, base,
1620 ofs ^ ((LJ_BE && irt_isint(t)) ? 4 : 0));
1032 } 1621 }
1033 if (ra_hasreg(type)) emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4)); 1622#endif
1034} 1623}
1035 1624
1036/* -- Allocations --------------------------------------------------------- */ 1625/* -- Allocations --------------------------------------------------------- */
@@ -1039,19 +1628,16 @@ dotypecheck:
1039static void asm_cnew(ASMState *as, IRIns *ir) 1628static void asm_cnew(ASMState *as, IRIns *ir)
1040{ 1629{
1041 CTState *cts = ctype_ctsG(J2G(as->J)); 1630 CTState *cts = ctype_ctsG(J2G(as->J));
1042 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i; 1631 CTypeID id = (CTypeID)IR(ir->op1)->i;
1043 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ? 1632 CTSize sz;
1044 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i; 1633 CTInfo info = lj_ctype_info(cts, id, &sz);
1045 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; 1634 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1046 IRRef args[2]; 1635 IRRef args[4];
1047 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1048 RegSet drop = RSET_SCRATCH; 1636 RegSet drop = RSET_SCRATCH;
1049 lua_assert(sz != CTSIZE_INVALID); 1637 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1638 "bad CNEW/CNEWI operands");
1050 1639
1051 args[0] = ASMREF_L; /* lua_State *L */
1052 args[1] = ASMREF_TMP1; /* MSize size */
1053 as->gcsteps++; 1640 as->gcsteps++;
1054
1055 if (ra_hasreg(ir->r)) 1641 if (ra_hasreg(ir->r))
1056 rset_clear(drop, ir->r); /* Dest reg handled below. */ 1642 rset_clear(drop, ir->r); /* Dest reg handled below. */
1057 ra_evictset(as, drop); 1643 ra_evictset(as, drop);
@@ -1060,11 +1646,12 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1060 1646
1061 /* Initialize immutable cdata object. */ 1647 /* Initialize immutable cdata object. */
1062 if (ir->o == IR_CNEWI) { 1648 if (ir->o == IR_CNEWI) {
1649 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1650#if LJ_32
1063 int32_t ofs = sizeof(GCcdata); 1651 int32_t ofs = sizeof(GCcdata);
1064 lua_assert(sz == 4 || sz == 8);
1065 if (sz == 8) { 1652 if (sz == 8) {
1066 ofs += 4; 1653 ofs += 4;
1067 lua_assert((ir+1)->o == IR_HIOP); 1654 lj_assertA((ir+1)->o == IR_HIOP, "expected HIOP for CNEWI");
1068 if (LJ_LE) ir++; 1655 if (LJ_LE) ir++;
1069 } 1656 }
1070 for (;;) { 1657 for (;;) {
@@ -1074,18 +1661,33 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1074 if (ofs == sizeof(GCcdata)) break; 1661 if (ofs == sizeof(GCcdata)) break;
1075 ofs -= 4; if (LJ_BE) ir++; else ir--; 1662 ofs -= 4; if (LJ_BE) ir++; else ir--;
1076 } 1663 }
1664#else
1665 emit_tsi(as, sz == 8 ? MIPSI_SD : MIPSI_SW, ra_alloc1(as, ir->op2, allow),
1666 RID_RET, sizeof(GCcdata));
1667#endif
1668 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1669 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1670 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1671 args[0] = ASMREF_L; /* lua_State *L */
1672 args[1] = ir->op1; /* CTypeID id */
1673 args[2] = ir->op2; /* CTSize sz */
1674 args[3] = ASMREF_TMP1; /* CTSize align */
1675 asm_gencall(as, ci, args);
1676 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1677 return;
1077 } 1678 }
1679
1078 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ 1680 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1079 emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct)); 1681 emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1080 emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid)); 1682 emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1081 emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA); 1683 emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
1082 emit_ti(as, MIPSI_LI, RID_TMP, ctypeid); /* Lower 16 bit used. Sign-ext ok. */ 1684 emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
1685 args[0] = ASMREF_L; /* lua_State *L */
1686 args[1] = ASMREF_TMP1; /* MSize size */
1083 asm_gencall(as, ci, args); 1687 asm_gencall(as, ci, args);
1084 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)), 1688 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1085 ra_releasetmp(as, ASMREF_TMP1)); 1689 ra_releasetmp(as, ASMREF_TMP1));
1086} 1690}
1087#else
1088#define asm_cnew(as, ir) ((void)0)
1089#endif 1691#endif
1090 1692
1091/* -- Write barriers ------------------------------------------------------ */ 1693/* -- Write barriers ------------------------------------------------------ */
@@ -1096,7 +1698,7 @@ static void asm_tbar(ASMState *as, IRIns *ir)
1096 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab)); 1698 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1097 Reg link = RID_TMP; 1699 Reg link = RID_TMP;
1098 MCLabel l_end = emit_label(as); 1700 MCLabel l_end = emit_label(as);
1099 emit_tsi(as, MIPSI_SW, link, tab, (int32_t)offsetof(GCtab, gclist)); 1701 emit_tsi(as, MIPSI_AS, link, tab, (int32_t)offsetof(GCtab, gclist));
1100 emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked)); 1702 emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
1101 emit_setgl(as, tab, gc.grayagain); 1703 emit_setgl(as, tab, gc.grayagain);
1102 emit_getgl(as, link, gc.grayagain); 1704 emit_getgl(as, link, gc.grayagain);
@@ -1113,13 +1715,13 @@ static void asm_obar(ASMState *as, IRIns *ir)
1113 MCLabel l_end; 1715 MCLabel l_end;
1114 Reg obj, val, tmp; 1716 Reg obj, val, tmp;
1115 /* No need for other object barriers (yet). */ 1717 /* No need for other object barriers (yet). */
1116 lua_assert(IR(ir->op1)->o == IR_UREFC); 1718 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1117 ra_evictset(as, RSET_SCRATCH); 1719 ra_evictset(as, RSET_SCRATCH);
1118 l_end = emit_label(as); 1720 l_end = emit_label(as);
1119 args[0] = ASMREF_TMP1; /* global_State *g */ 1721 args[0] = ASMREF_TMP1; /* global_State *g */
1120 args[1] = ir->op1; /* TValue *tv */ 1722 args[1] = ir->op1; /* TValue *tv */
1121 asm_gencall(as, ci, args); 1723 asm_gencall(as, ci, args);
1122 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768); 1724 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1123 obj = IR(ir->op1)->r; 1725 obj = IR(ir->op1)->r;
1124 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj)); 1726 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1125 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end); 1727 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
@@ -1134,6 +1736,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1134 1736
1135/* -- Arithmetic and logic operations ------------------------------------- */ 1737/* -- Arithmetic and logic operations ------------------------------------- */
1136 1738
1739#if !LJ_SOFTFP
1137static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi) 1740static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi)
1138{ 1741{
1139 Reg dest = ra_dest(as, ir, RSET_FPR); 1742 Reg dest = ra_dest(as, ir, RSET_FPR);
@@ -1148,83 +1751,147 @@ static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
1148 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR); 1751 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1149 emit_fg(as, mi, dest, left); 1752 emit_fg(as, mi, dest, left);
1150} 1753}
1754#endif
1151 1755
1152static int asm_fpjoin_pow(ASMState *as, IRIns *ir) 1756#if !LJ_SOFTFP32
1153{ 1757static void asm_fpmath(ASMState *as, IRIns *ir)
1154 IRIns *irp = IR(ir->op1); 1758{
1155 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) { 1759#if !LJ_SOFTFP
1156 IRIns *irpp = IR(irp->op1); 1760 if (ir->op2 <= IRFPM_TRUNC)
1157 if (irpp == ir-2 && irpp->o == IR_FPMATH && 1761 asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1158 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) { 1762 else if (ir->op2 == IRFPM_SQRT)
1159 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow]; 1763 asm_fpunary(as, ir, MIPSI_SQRT_D);
1160 IRRef args[2]; 1764 else
1161 args[0] = irpp->op1; 1765#endif
1162 args[1] = irp->op2; 1766 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1163 asm_setupresult(as, ir, ci);
1164 asm_gencall(as, ci, args);
1165 return 1;
1166 }
1167 }
1168 return 0;
1169} 1767}
1768#endif
1769
1770#if !LJ_SOFTFP
1771#define asm_fpadd(as, ir) asm_fparith(as, ir, MIPSI_ADD_D)
1772#define asm_fpsub(as, ir) asm_fparith(as, ir, MIPSI_SUB_D)
1773#define asm_fpmul(as, ir) asm_fparith(as, ir, MIPSI_MUL_D)
1774#elif LJ_64 /* && LJ_SOFTFP */
1775#define asm_fpadd(as, ir) asm_callid(as, ir, IRCALL_softfp_add)
1776#define asm_fpsub(as, ir) asm_callid(as, ir, IRCALL_softfp_sub)
1777#define asm_fpmul(as, ir) asm_callid(as, ir, IRCALL_softfp_mul)
1778#endif
1170 1779
1171static void asm_add(ASMState *as, IRIns *ir) 1780static void asm_add(ASMState *as, IRIns *ir)
1172{ 1781{
1173 if (irt_isnum(ir->t)) { 1782 IRType1 t = ir->t;
1174 asm_fparith(as, ir, MIPSI_ADD_D); 1783#if !LJ_SOFTFP32
1175 } else { 1784 if (irt_isnum(t)) {
1785 asm_fpadd(as, ir);
1786 } else
1787#endif
1788 {
1789 /* TODO MIPSR6: Fuse ADD(BSHL(a,1-4),b) or ADD(ADD(a,a),b) to MIPSI_ALSA. */
1176 Reg dest = ra_dest(as, ir, RSET_GPR); 1790 Reg dest = ra_dest(as, ir, RSET_GPR);
1177 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); 1791 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1178 if (irref_isk(ir->op2)) { 1792 if (irref_isk(ir->op2)) {
1179 int32_t k = IR(ir->op2)->i; 1793 intptr_t k = get_kval(as, ir->op2);
1180 if (checki16(k)) { 1794 if (checki16(k)) {
1181 emit_tsi(as, MIPSI_ADDIU, dest, left, k); 1795 emit_tsi(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDIU : MIPSI_ADDIU, dest,
1796 left, k);
1182 return; 1797 return;
1183 } 1798 }
1184 } 1799 }
1185 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left)); 1800 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1186 emit_dst(as, MIPSI_ADDU, dest, left, right); 1801 emit_dst(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDU : MIPSI_ADDU, dest,
1802 left, right);
1187 } 1803 }
1188} 1804}
1189 1805
1190static void asm_sub(ASMState *as, IRIns *ir) 1806static void asm_sub(ASMState *as, IRIns *ir)
1191{ 1807{
1808#if !LJ_SOFTFP32
1192 if (irt_isnum(ir->t)) { 1809 if (irt_isnum(ir->t)) {
1193 asm_fparith(as, ir, MIPSI_SUB_D); 1810 asm_fpsub(as, ir);
1194 } else { 1811 } else
1812#endif
1813 {
1195 Reg dest = ra_dest(as, ir, RSET_GPR); 1814 Reg dest = ra_dest(as, ir, RSET_GPR);
1196 Reg right, left = ra_alloc2(as, ir, RSET_GPR); 1815 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1197 right = (left >> 8); left &= 255; 1816 right = (left >> 8); left &= 255;
1198 emit_dst(as, MIPSI_SUBU, dest, left, right); 1817 emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1818 left, right);
1199 } 1819 }
1200} 1820}
1201 1821
1202static void asm_mul(ASMState *as, IRIns *ir) 1822static void asm_mul(ASMState *as, IRIns *ir)
1203{ 1823{
1824#if !LJ_SOFTFP32
1204 if (irt_isnum(ir->t)) { 1825 if (irt_isnum(ir->t)) {
1205 asm_fparith(as, ir, MIPSI_MUL_D); 1826 asm_fpmul(as, ir);
1206 } else { 1827 } else
1828#endif
1829 {
1207 Reg dest = ra_dest(as, ir, RSET_GPR); 1830 Reg dest = ra_dest(as, ir, RSET_GPR);
1208 Reg right, left = ra_alloc2(as, ir, RSET_GPR); 1831 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1209 right = (left >> 8); left &= 255; 1832 right = (left >> 8); left &= 255;
1210 emit_dst(as, MIPSI_MUL, dest, left, right); 1833 if (LJ_64 && irt_is64(ir->t)) {
1834#if !LJ_TARGET_MIPSR6
1835 emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1836 emit_dst(as, MIPSI_DMULT, 0, left, right);
1837#else
1838 emit_dst(as, MIPSI_DMUL, dest, left, right);
1839#endif
1840 } else {
1841 emit_dst(as, MIPSI_MUL, dest, left, right);
1842 }
1211 } 1843 }
1212} 1844}
1213 1845
1846#if !LJ_SOFTFP32
1847static void asm_fpdiv(ASMState *as, IRIns *ir)
1848{
1849#if !LJ_SOFTFP
1850 asm_fparith(as, ir, MIPSI_DIV_D);
1851#else
1852 asm_callid(as, ir, IRCALL_softfp_div);
1853#endif
1854}
1855#endif
1856
1214static void asm_neg(ASMState *as, IRIns *ir) 1857static void asm_neg(ASMState *as, IRIns *ir)
1215{ 1858{
1859#if !LJ_SOFTFP
1216 if (irt_isnum(ir->t)) { 1860 if (irt_isnum(ir->t)) {
1217 asm_fpunary(as, ir, MIPSI_NEG_D); 1861 asm_fpunary(as, ir, MIPSI_NEG_D);
1218 } else { 1862 } else
1863#elif LJ_64 /* && LJ_SOFTFP */
1864 if (irt_isnum(ir->t)) {
1865 Reg dest = ra_dest(as, ir, RSET_GPR);
1866 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1867 emit_dst(as, MIPSI_XOR, dest, left,
1868 ra_allock(as, 0x8000000000000000ll, rset_exclude(RSET_GPR, dest)));
1869 } else
1870#endif
1871 {
1219 Reg dest = ra_dest(as, ir, RSET_GPR); 1872 Reg dest = ra_dest(as, ir, RSET_GPR);
1220 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); 1873 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1221 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left); 1874 emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1875 RID_ZERO, left);
1222 } 1876 }
1223} 1877}
1224 1878
1879#if !LJ_SOFTFP
1880#define asm_abs(as, ir) asm_fpunary(as, ir, MIPSI_ABS_D)
1881#elif LJ_64 /* && LJ_SOFTFP */
1882static void asm_abs(ASMState *as, IRIns *ir)
1883{
1884 Reg dest = ra_dest(as, ir, RSET_GPR);
1885 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1886 emit_tsml(as, MIPSI_DEXTM, dest, left, 30, 0);
1887}
1888#endif
1889
1225static void asm_arithov(ASMState *as, IRIns *ir) 1890static void asm_arithov(ASMState *as, IRIns *ir)
1226{ 1891{
1892 /* TODO MIPSR6: bovc/bnvc. Caveat: no delay slot to load RID_TMP. */
1227 Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR); 1893 Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
1894 lj_assertA(!irt_is64(ir->t), "bad usage");
1228 if (irref_isk(ir->op2)) { 1895 if (irref_isk(ir->op2)) {
1229 int k = IR(ir->op2)->i; 1896 int k = IR(ir->op2)->i;
1230 if (ir->o == IR_SUBOV) k = (int)(~(unsigned int)k+1u); 1897 if (ir->o == IR_SUBOV) k = (int)(~(unsigned int)k+1u);
@@ -1255,16 +1922,29 @@ static void asm_arithov(ASMState *as, IRIns *ir)
1255 emit_move(as, RID_TMP, dest == left ? left : right); 1922 emit_move(as, RID_TMP, dest == left ? left : right);
1256} 1923}
1257 1924
1925#define asm_addov(as, ir) asm_arithov(as, ir)
1926#define asm_subov(as, ir) asm_arithov(as, ir)
1927
1258static void asm_mulov(ASMState *as, IRIns *ir) 1928static void asm_mulov(ASMState *as, IRIns *ir)
1259{ 1929{
1260#if LJ_DUALNUM 1930 Reg dest = ra_dest(as, ir, RSET_GPR);
1261#error "NYI: MULOV" 1931 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1932 right = (left >> 8); left &= 255;
1933 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1934 right), dest));
1935 asm_guard(as, MIPSI_BNE, RID_TMP, tmp);
1936 emit_dta(as, MIPSI_SRA, RID_TMP, dest, 31);
1937#if !LJ_TARGET_MIPSR6
1938 emit_dst(as, MIPSI_MFHI, tmp, 0, 0);
1939 emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1940 emit_dst(as, MIPSI_MULT, 0, left, right);
1262#else 1941#else
1263 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused in single-number mode. */ 1942 emit_dst(as, MIPSI_MUL, dest, left, right);
1943 emit_dst(as, MIPSI_MUH, tmp, left, right);
1264#endif 1944#endif
1265} 1945}
1266 1946
1267#if LJ_HASFFI 1947#if LJ_32 && LJ_HASFFI
1268static void asm_add64(ASMState *as, IRIns *ir) 1948static void asm_add64(ASMState *as, IRIns *ir)
1269{ 1949{
1270 Reg dest = ra_dest(as, ir, RSET_GPR); 1950 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1348,7 +2028,7 @@ static void asm_neg64(ASMState *as, IRIns *ir)
1348} 2028}
1349#endif 2029#endif
1350 2030
1351static void asm_bitnot(ASMState *as, IRIns *ir) 2031static void asm_bnot(ASMState *as, IRIns *ir)
1352{ 2032{
1353 Reg left, right, dest = ra_dest(as, ir, RSET_GPR); 2033 Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
1354 IRIns *irl = IR(ir->op1); 2034 IRIns *irl = IR(ir->op1);
@@ -1362,11 +2042,12 @@ static void asm_bitnot(ASMState *as, IRIns *ir)
1362 emit_dst(as, MIPSI_NOR, dest, left, right); 2042 emit_dst(as, MIPSI_NOR, dest, left, right);
1363} 2043}
1364 2044
1365static void asm_bitswap(ASMState *as, IRIns *ir) 2045static void asm_bswap(ASMState *as, IRIns *ir)
1366{ 2046{
1367 Reg dest = ra_dest(as, ir, RSET_GPR); 2047 Reg dest = ra_dest(as, ir, RSET_GPR);
1368 Reg left = ra_alloc1(as, ir->op1, RSET_GPR); 2048 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1369 if ((as->flags & JIT_F_MIPS32R2)) { 2049#if LJ_32
2050 if ((as->flags & JIT_F_MIPSXXR2)) {
1370 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16); 2051 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
1371 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left); 2052 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
1372 } else { 2053 } else {
@@ -1381,6 +2062,15 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
1381 emit_dta(as, MIPSI_SRL, tmp, left, 24); 2062 emit_dta(as, MIPSI_SRL, tmp, left, 24);
1382 emit_dta(as, MIPSI_SLL, RID_TMP, left, 24); 2063 emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
1383 } 2064 }
2065#else
2066 if (irt_is64(ir->t)) {
2067 emit_dst(as, MIPSI_DSHD, dest, 0, RID_TMP);
2068 emit_dst(as, MIPSI_DSBH, RID_TMP, 0, left);
2069 } else {
2070 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
2071 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
2072 }
2073#endif
1384} 2074}
1385 2075
1386static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik) 2076static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
@@ -1388,7 +2078,7 @@ static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1388 Reg dest = ra_dest(as, ir, RSET_GPR); 2078 Reg dest = ra_dest(as, ir, RSET_GPR);
1389 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); 2079 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1390 if (irref_isk(ir->op2)) { 2080 if (irref_isk(ir->op2)) {
1391 int32_t k = IR(ir->op2)->i; 2081 intptr_t k = get_kval(as, ir->op2);
1392 if (checku16(k)) { 2082 if (checku16(k)) {
1393 emit_tsi(as, mik, dest, left, k); 2083 emit_tsi(as, mik, dest, left, k);
1394 return; 2084 return;
@@ -1398,22 +2088,34 @@ static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1398 emit_dst(as, mi, dest, left, right); 2088 emit_dst(as, mi, dest, left, right);
1399} 2089}
1400 2090
2091#define asm_band(as, ir) asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
2092#define asm_bor(as, ir) asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
2093#define asm_bxor(as, ir) asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
2094
1401static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik) 2095static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1402{ 2096{
1403 Reg dest = ra_dest(as, ir, RSET_GPR); 2097 Reg dest = ra_dest(as, ir, RSET_GPR);
1404 if (irref_isk(ir->op2)) { /* Constant shifts. */ 2098 if (irref_isk(ir->op2)) { /* Constant shifts. */
1405 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31); 2099 uint32_t shift = (uint32_t)IR(ir->op2)->i;
1406 emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR), shift); 2100 if (LJ_64 && irt_is64(ir->t)) mik |= (shift & 32) ? MIPSI_D32 : MIPSI_D;
2101 emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR),
2102 (shift & 31));
1407 } else { 2103 } else {
1408 Reg right, left = ra_alloc2(as, ir, RSET_GPR); 2104 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1409 right = (left >> 8); left &= 255; 2105 right = (left >> 8); left &= 255;
2106 if (LJ_64 && irt_is64(ir->t)) mi |= MIPSI_DV;
1410 emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */ 2107 emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */
1411 } 2108 }
1412} 2109}
1413 2110
1414static void asm_bitror(ASMState *as, IRIns *ir) 2111#define asm_bshl(as, ir) asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
2112#define asm_bshr(as, ir) asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
2113#define asm_bsar(as, ir) asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
2114#define asm_brol(as, ir) lj_assertA(0, "unexpected BROL")
2115
2116static void asm_bror(ASMState *as, IRIns *ir)
1415{ 2117{
1416 if ((as->flags & JIT_F_MIPS32R2)) { 2118 if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
1417 asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR); 2119 asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
1418 } else { 2120 } else {
1419 Reg dest = ra_dest(as, ir, RSET_GPR); 2121 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1432,55 +2134,182 @@ static void asm_bitror(ASMState *as, IRIns *ir)
1432 } 2134 }
1433} 2135}
1434 2136
2137#if LJ_SOFTFP
2138static void asm_sfpmin_max(ASMState *as, IRIns *ir)
2139{
2140 CCallInfo ci = lj_ir_callinfo[(IROp)ir->o == IR_MIN ? IRCALL_lj_vm_sfmin : IRCALL_lj_vm_sfmax];
2141#if LJ_64
2142 IRRef args[2];
2143 args[0] = ir->op1;
2144 args[1] = ir->op2;
2145#else
2146 IRRef args[4];
2147 args[0^LJ_BE] = ir->op1;
2148 args[1^LJ_BE] = (ir+1)->op1;
2149 args[2^LJ_BE] = ir->op2;
2150 args[3^LJ_BE] = (ir+1)->op2;
2151#endif
2152 asm_setupresult(as, ir, &ci);
2153 emit_call(as, (void *)ci.func, 0);
2154 ci.func = NULL;
2155 asm_gencall(as, &ci, args);
2156}
2157#endif
2158
1435static void asm_min_max(ASMState *as, IRIns *ir, int ismax) 2159static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1436{ 2160{
1437 if (irt_isnum(ir->t)) { 2161 if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2162#if LJ_SOFTFP
2163 asm_sfpmin_max(as, ir);
2164#else
1438 Reg dest = ra_dest(as, ir, RSET_FPR); 2165 Reg dest = ra_dest(as, ir, RSET_FPR);
1439 Reg right, left = ra_alloc2(as, ir, RSET_FPR); 2166 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1440 right = (left >> 8); left &= 255; 2167 right = (left >> 8); left &= 255;
2168#if !LJ_TARGET_MIPSR6
1441 if (dest == left) { 2169 if (dest == left) {
1442 emit_fg(as, MIPSI_MOVT_D, dest, right); 2170 emit_fg(as, MIPSI_MOVF_D, dest, right);
1443 } else { 2171 } else {
1444 emit_fg(as, MIPSI_MOVF_D, dest, left); 2172 emit_fg(as, MIPSI_MOVT_D, dest, left);
1445 if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right); 2173 if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right);
1446 } 2174 }
1447 emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? left : right, ismax ? right : left); 2175 emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? right : left, ismax ? left : right);
2176#else
2177 emit_fgh(as, ismax ? MIPSI_MAX_D : MIPSI_MIN_D, dest, left, right);
2178#endif
2179#endif
1448 } else { 2180 } else {
1449 Reg dest = ra_dest(as, ir, RSET_GPR); 2181 Reg dest = ra_dest(as, ir, RSET_GPR);
1450 Reg right, left = ra_alloc2(as, ir, RSET_GPR); 2182 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1451 right = (left >> 8); left &= 255; 2183 right = (left >> 8); left &= 255;
1452 if (dest == left) { 2184 if (left == right) {
1453 emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP); 2185 if (dest != left) emit_move(as, dest, left);
1454 } else { 2186 } else {
1455 emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP); 2187#if !LJ_TARGET_MIPSR6
1456 if (dest != right) emit_move(as, dest, right); 2188 if (dest == left) {
2189 emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP);
2190 } else {
2191 emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP);
2192 if (dest != right) emit_move(as, dest, right);
2193 }
2194#else
2195 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
2196 if (dest != right) {
2197 emit_dst(as, MIPSI_SELNEZ, RID_TMP, right, RID_TMP);
2198 emit_dst(as, MIPSI_SELEQZ, dest, left, RID_TMP);
2199 } else {
2200 emit_dst(as, MIPSI_SELEQZ, RID_TMP, left, RID_TMP);
2201 emit_dst(as, MIPSI_SELNEZ, dest, right, RID_TMP);
2202 }
2203#endif
2204 emit_dst(as, MIPSI_SLT, RID_TMP,
2205 ismax ? left : right, ismax ? right : left);
1457 } 2206 }
1458 emit_dst(as, MIPSI_SLT, RID_TMP,
1459 ismax ? left : right, ismax ? right : left);
1460 } 2207 }
1461} 2208}
1462 2209
2210#define asm_min(as, ir) asm_min_max(as, ir, 0)
2211#define asm_max(as, ir) asm_min_max(as, ir, 1)
2212
1463/* -- Comparisons --------------------------------------------------------- */ 2213/* -- Comparisons --------------------------------------------------------- */
1464 2214
2215#if LJ_SOFTFP
2216/* SFP comparisons. */
2217static void asm_sfpcomp(ASMState *as, IRIns *ir)
2218{
2219 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp];
2220 RegSet drop = RSET_SCRATCH;
2221 Reg r;
2222#if LJ_64
2223 IRRef args[2];
2224 args[0] = ir->op1;
2225 args[1] = ir->op2;
2226#else
2227 IRRef args[4];
2228 args[LJ_LE ? 0 : 1] = ir->op1; args[LJ_LE ? 1 : 0] = (ir+1)->op1;
2229 args[LJ_LE ? 2 : 3] = ir->op2; args[LJ_LE ? 3 : 2] = (ir+1)->op2;
2230#endif
2231
2232 for (r = REGARG_FIRSTGPR; r <= REGARG_FIRSTGPR+(LJ_64?1:3); r++) {
2233 if (!rset_test(as->freeset, r) &&
2234 regcost_ref(as->cost[r]) == args[r-REGARG_FIRSTGPR])
2235 rset_clear(drop, r);
2236 }
2237 ra_evictset(as, drop);
2238
2239 asm_setupresult(as, ir, ci);
2240
2241 switch ((IROp)ir->o) {
2242 case IR_LT:
2243 asm_guard(as, MIPSI_BGEZ, RID_RET, 0);
2244 break;
2245 case IR_ULT:
2246 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2247 emit_loadi(as, RID_TMP, 1);
2248 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO);
2249 break;
2250 case IR_GE:
2251 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2252 emit_loadi(as, RID_TMP, 2);
2253 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2254 break;
2255 case IR_LE:
2256 asm_guard(as, MIPSI_BGTZ, RID_RET, 0);
2257 break;
2258 case IR_GT:
2259 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2260 emit_loadi(as, RID_TMP, 2);
2261 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2262 break;
2263 case IR_UGE:
2264 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2265 break;
2266 case IR_ULE:
2267 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2268 emit_loadi(as, RID_TMP, 1);
2269 break;
2270 case IR_UGT: case IR_ABC:
2271 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2272 break;
2273 case IR_EQ: case IR_NE:
2274 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_RET, RID_ZERO);
2275 default:
2276 break;
2277 }
2278 asm_gencall(as, ci, args);
2279}
2280#endif
2281
1465static void asm_comp(ASMState *as, IRIns *ir) 2282static void asm_comp(ASMState *as, IRIns *ir)
1466{ 2283{
1467 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */ 2284 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1468 IROp op = ir->o; 2285 IROp op = ir->o;
1469 if (irt_isnum(ir->t)) { 2286 if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2287#if LJ_SOFTFP
2288 asm_sfpcomp(as, ir);
2289#else
2290#if !LJ_TARGET_MIPSR6
1470 Reg right, left = ra_alloc2(as, ir, RSET_FPR); 2291 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1471 right = (left >> 8); left &= 255; 2292 right = (left >> 8); left &= 255;
1472 asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0); 2293 asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1473 emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right); 2294 emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right);
2295#else
2296 Reg tmp, right, left = ra_alloc2(as, ir, RSET_FPR);
2297 right = (left >> 8); left &= 255;
2298 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_FPR, left), right));
2299 asm_guard(as, (op&1) ? MIPSI_BC1NEZ : MIPSI_BC1EQZ, 0, (tmp&31));
2300 emit_fgh(as, MIPSI_CMP_LT_D + ((op&3) ^ ((op>>2)&1)), tmp, left, right);
2301#endif
2302#endif
1474 } else { 2303 } else {
1475 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR); 2304 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1476 if (op == IR_ABC) op = IR_UGT; 2305 if (op == IR_ABC) op = IR_UGT;
1477 if ((op&4) == 0 && irref_isk(ir->op2) && IR(ir->op2)->i == 0) { 2306 if ((op&4) == 0 && irref_isk(ir->op2) && get_kval(as, ir->op2) == 0) {
1478 MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) : 2307 MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
1479 ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ); 2308 ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
1480 asm_guard(as, mi, left, 0); 2309 asm_guard(as, mi, left, 0);
1481 } else { 2310 } else {
1482 if (irref_isk(ir->op2)) { 2311 if (irref_isk(ir->op2)) {
1483 int32_t k = IR(ir->op2)->i; 2312 intptr_t k = get_kval(as, ir->op2);
1484 if ((op&2)) k++; 2313 if ((op&2)) k++;
1485 if (checki16(k)) { 2314 if (checki16(k)) {
1486 asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO); 2315 asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
@@ -1497,19 +2326,28 @@ static void asm_comp(ASMState *as, IRIns *ir)
1497 } 2326 }
1498} 2327}
1499 2328
1500static void asm_compeq(ASMState *as, IRIns *ir) 2329static void asm_equal(ASMState *as, IRIns *ir)
1501{ 2330{
1502 Reg right, left = ra_alloc2(as, ir, irt_isnum(ir->t) ? RSET_FPR : RSET_GPR); 2331 Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ?
2332 RSET_FPR : RSET_GPR);
1503 right = (left >> 8); left &= 255; 2333 right = (left >> 8); left &= 255;
1504 if (irt_isnum(ir->t)) { 2334 if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2335#if LJ_SOFTFP
2336 asm_sfpcomp(as, ir);
2337#elif !LJ_TARGET_MIPSR6
1505 asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0); 2338 asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1506 emit_fgh(as, MIPSI_C_EQ_D, 0, left, right); 2339 emit_fgh(as, MIPSI_C_EQ_D, 0, left, right);
2340#else
2341 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_FPR, left), right));
2342 asm_guard(as, (ir->o & 1) ? MIPSI_BC1NEZ : MIPSI_BC1EQZ, 0, (tmp&31));
2343 emit_fgh(as, MIPSI_CMP_EQ_D, tmp, left, right);
2344#endif
1507 } else { 2345 } else {
1508 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right); 2346 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right);
1509 } 2347 }
1510} 2348}
1511 2349
1512#if LJ_HASFFI 2350#if LJ_32 && LJ_HASFFI
1513/* 64 bit integer comparisons. */ 2351/* 64 bit integer comparisons. */
1514static void asm_comp64(ASMState *as, IRIns *ir) 2352static void asm_comp64(ASMState *as, IRIns *ir)
1515{ 2353{
@@ -1546,54 +2384,99 @@ static void asm_comp64eq(ASMState *as, IRIns *ir)
1546} 2384}
1547#endif 2385#endif
1548 2386
1549/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */ 2387/* -- Split register ops -------------------------------------------------- */
1550 2388
1551/* Hiword op of a split 64 bit op. Previous op must be the loword op. */ 2389/* Hiword op of a split 32/32 or 64/64 bit op. Previous op is the loword op. */
1552static void asm_hiop(ASMState *as, IRIns *ir) 2390static void asm_hiop(ASMState *as, IRIns *ir)
1553{ 2391{
1554#if LJ_HASFFI
1555 /* HIOP is marked as a store because it needs its own DCE logic. */ 2392 /* HIOP is marked as a store because it needs its own DCE logic. */
1556 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */ 2393 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1557 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1; 2394 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
2395#if LJ_32 && (LJ_HASFFI || LJ_SOFTFP)
1558 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */ 2396 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
1559 as->curins--; /* Always skip the CONV. */ 2397 as->curins--; /* Always skip the CONV. */
2398#if LJ_HASFFI && !LJ_SOFTFP
1560 if (usehi || uselo) 2399 if (usehi || uselo)
1561 asm_conv64(as, ir); 2400 asm_conv64(as, ir);
1562 return; 2401 return;
2402#endif
1563 } else if ((ir-1)->o < IR_EQ) { /* 64 bit integer comparisons. ORDER IR. */ 2403 } else if ((ir-1)->o < IR_EQ) { /* 64 bit integer comparisons. ORDER IR. */
1564 as->curins--; /* Always skip the loword comparison. */ 2404 as->curins--; /* Always skip the loword comparison. */
2405#if LJ_SOFTFP
2406 if (!irt_isint(ir->t)) {
2407 asm_sfpcomp(as, ir-1);
2408 return;
2409 }
2410#endif
2411#if LJ_HASFFI
1565 asm_comp64(as, ir); 2412 asm_comp64(as, ir);
2413#endif
1566 return; 2414 return;
1567 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */ 2415 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
1568 as->curins--; /* Always skip the loword comparison. */ 2416 as->curins--; /* Always skip the loword comparison. */
2417#if LJ_SOFTFP
2418 if (!irt_isint(ir->t)) {
2419 asm_sfpcomp(as, ir-1);
2420 return;
2421 }
2422#endif
2423#if LJ_HASFFI
1569 asm_comp64eq(as, ir); 2424 asm_comp64eq(as, ir);
2425#endif
2426 return;
2427#if LJ_SOFTFP
2428 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
2429 as->curins--; /* Always skip the loword min/max. */
2430 if (uselo || usehi)
2431 asm_sfpmin_max(as, ir-1);
1570 return; 2432 return;
2433#endif
1571 } else if ((ir-1)->o == IR_XSTORE) { 2434 } else if ((ir-1)->o == IR_XSTORE) {
1572 as->curins--; /* Handle both stores here. */ 2435 as->curins--; /* Handle both stores here. */
1573 if ((ir-1)->r != RID_SINK) { 2436 if ((ir-1)->r != RID_SINK) {
1574 asm_xstore(as, ir, LJ_LE ? 4 : 0); 2437 asm_xstore_(as, ir, LJ_LE ? 4 : 0);
1575 asm_xstore(as, ir-1, LJ_LE ? 0 : 4); 2438 asm_xstore_(as, ir-1, LJ_LE ? 0 : 4);
1576 } 2439 }
1577 return; 2440 return;
1578 } 2441 }
2442#endif
1579 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */ 2443 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1580 switch ((ir-1)->o) { 2444 switch ((ir-1)->o) {
2445#if LJ_32 && LJ_HASFFI
1581 case IR_ADD: as->curins--; asm_add64(as, ir); break; 2446 case IR_ADD: as->curins--; asm_add64(as, ir); break;
1582 case IR_SUB: as->curins--; asm_sub64(as, ir); break; 2447 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1583 case IR_NEG: as->curins--; asm_neg64(as, ir); break; 2448 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1584 case IR_CALLN: 2449 case IR_CNEWI:
1585 case IR_CALLXS: 2450 /* Nothing to do here. Handled by lo op itself. */
2451 break;
2452#endif
2453#if LJ_32 && LJ_SOFTFP
2454 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2455 case IR_STRTO:
1586 if (!uselo) 2456 if (!uselo)
1587 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */ 2457 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */
1588 break; 2458 break;
1589 case IR_CNEWI: 2459 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR: case IR_TMPREF:
1590 /* Nothing to do here. Handled by lo op itself. */ 2460 /* Nothing to do here. Handled by lo op itself. */
1591 break; 2461 break;
1592 default: lua_assert(0); break;
1593 }
1594#else
1595 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
1596#endif 2462#endif
2463 case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
2464 if (!uselo)
2465 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
2466 break;
2467 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
2468 }
2469}
2470
2471/* -- Profiling ----------------------------------------------------------- */
2472
2473static void asm_prof(ASMState *as, IRIns *ir)
2474{
2475 UNUSED(ir);
2476 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
2477 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, HOOK_PROFILE);
2478 emit_lsglptr(as, MIPSI_LBU, RID_TMP,
2479 (int32_t)offsetof(global_State, hookmask));
1597} 2480}
1598 2481
1599/* -- Stack handling ------------------------------------------------------ */ 2482/* -- Stack handling ------------------------------------------------------ */
@@ -1606,47 +2489,70 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
1606 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE; 2489 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
1607 ExitNo oldsnap = as->snapno; 2490 ExitNo oldsnap = as->snapno;
1608 rset_clear(allow, pbase); 2491 rset_clear(allow, pbase);
2492#if LJ_32
1609 tmp = allow ? rset_pickbot(allow) : 2493 tmp = allow ? rset_pickbot(allow) :
1610 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI); 2494 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
2495#else
2496 tmp = allow ? rset_pickbot(allow) : RID_RET;
2497#endif
1611 as->snapno = exitno; 2498 as->snapno = exitno;
1612 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO); 2499 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
1613 as->snapno = oldsnap; 2500 as->snapno = oldsnap;
1614 if (allow == RSET_EMPTY) /* Restore temp. register. */ 2501 if (allow == RSET_EMPTY) /* Restore temp. register. */
1615 emit_tsi(as, MIPSI_LW, tmp, RID_SP, 0); 2502 emit_tsi(as, MIPSI_AL, tmp, RID_SP, 0);
1616 else 2503 else
1617 ra_modified(as, tmp); 2504 ra_modified(as, tmp);
1618 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot)); 2505 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
1619 emit_dst(as, MIPSI_SUBU, RID_TMP, tmp, pbase); 2506 emit_dst(as, MIPSI_ASUBU, RID_TMP, tmp, pbase);
1620 emit_tsi(as, MIPSI_LW, tmp, tmp, offsetof(lua_State, maxstack)); 2507 emit_tsi(as, MIPSI_AL, tmp, tmp, offsetof(lua_State, maxstack));
1621 if (pbase == RID_TMP) 2508 if (pbase == RID_TMP)
1622 emit_getgl(as, RID_TMP, jit_base); 2509 emit_getgl(as, RID_TMP, jit_base);
1623 emit_getgl(as, tmp, jit_L); 2510 emit_getgl(as, tmp, cur_L);
1624 if (allow == RSET_EMPTY) /* Spill temp. register. */ 2511 if (allow == RSET_EMPTY) /* Spill temp. register. */
1625 emit_tsi(as, MIPSI_SW, tmp, RID_SP, 0); 2512 emit_tsi(as, MIPSI_AS, tmp, RID_SP, 0);
1626} 2513}
1627 2514
1628/* Restore Lua stack from on-trace state. */ 2515/* Restore Lua stack from on-trace state. */
1629static void asm_stack_restore(ASMState *as, SnapShot *snap) 2516static void asm_stack_restore(ASMState *as, SnapShot *snap)
1630{ 2517{
1631 SnapEntry *map = &as->T->snapmap[snap->mapofs]; 2518 SnapEntry *map = &as->T->snapmap[snap->mapofs];
1632 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1]; 2519#if LJ_32 || defined(LUA_USE_ASSERT)
2520 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
2521#endif
1633 MSize n, nent = snap->nent; 2522 MSize n, nent = snap->nent;
1634 /* Store the value of all modified slots to the Lua stack. */ 2523 /* Store the value of all modified slots to the Lua stack. */
1635 for (n = 0; n < nent; n++) { 2524 for (n = 0; n < nent; n++) {
1636 SnapEntry sn = map[n]; 2525 SnapEntry sn = map[n];
1637 BCReg s = snap_slot(sn); 2526 BCReg s = snap_slot(sn);
1638 int32_t ofs = 8*((int32_t)s-1); 2527 int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
1639 IRRef ref = snap_ref(sn); 2528 IRRef ref = snap_ref(sn);
1640 IRIns *ir = IR(ref); 2529 IRIns *ir = IR(ref);
1641 if ((sn & SNAP_NORESTORE)) 2530 if ((sn & SNAP_NORESTORE))
1642 continue; 2531 continue;
1643 if (irt_isnum(ir->t)) { 2532 if (irt_isnum(ir->t)) {
2533#if LJ_SOFTFP32
2534 Reg tmp;
2535 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2536 /* LJ_SOFTFP: must be a number constant. */
2537 lj_assertA(irref_isk(ref), "unsplit FP op");
2538 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, allow);
2539 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?4:0));
2540 if (rset_test(as->freeset, tmp+1)) allow = RID2RSET(tmp+1);
2541 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, allow);
2542 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?0:4));
2543#elif LJ_SOFTFP /* && LJ_64 */
2544 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2545 emit_tsi(as, MIPSI_SD, src, RID_BASE, ofs);
2546#else
1644 Reg src = ra_alloc1(as, ref, RSET_FPR); 2547 Reg src = ra_alloc1(as, ref, RSET_FPR);
1645 emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs); 2548 emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
2549#endif
1646 } else { 2550 } else {
1647 Reg type; 2551#if LJ_32
1648 RegSet allow = rset_exclude(RSET_GPR, RID_BASE); 2552 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1649 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t)); 2553 Reg type;
2554 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
2555 "restore of IR type %d", irt_type(ir->t));
1650 if (!irt_ispri(ir->t)) { 2556 if (!irt_ispri(ir->t)) {
1651 Reg src = ra_alloc1(as, ref, allow); 2557 Reg src = ra_alloc1(as, ref, allow);
1652 rset_clear(allow, src); 2558 rset_clear(allow, src);
@@ -1655,14 +2561,38 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
1655 if ((sn & (SNAP_CONT|SNAP_FRAME))) { 2561 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1656 if (s == 0) continue; /* Do not overwrite link to previous frame. */ 2562 if (s == 0) continue; /* Do not overwrite link to previous frame. */
1657 type = ra_allock(as, (int32_t)(*flinks--), allow); 2563 type = ra_allock(as, (int32_t)(*flinks--), allow);
2564#if LJ_SOFTFP
2565 } else if ((sn & SNAP_SOFTFPNUM)) {
2566 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPR, RID_BASE));
2567#endif
2568 } else if ((sn & SNAP_KEYINDEX)) {
2569 type = ra_allock(as, (int32_t)LJ_KEYINDEX, allow);
1658 } else { 2570 } else {
1659 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow); 2571 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1660 } 2572 }
1661 emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4)); 2573 emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
2574#else
2575 if ((sn & SNAP_KEYINDEX)) {
2576 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2577 int64_t kki = (int64_t)LJ_KEYINDEX << 32;
2578 if (irref_isk(ref)) {
2579 emit_tsi(as, MIPSI_SD,
2580 ra_allock(as, kki | (int64_t)(uint32_t)ir->i, allow),
2581 RID_BASE, ofs);
2582 } else {
2583 Reg src = ra_alloc1(as, ref, allow);
2584 Reg rki = ra_allock(as, kki, rset_exclude(allow, src));
2585 emit_tsi(as, MIPSI_SD, RID_TMP, RID_BASE, ofs);
2586 emit_dst(as, MIPSI_DADDU, RID_TMP, src, rki);
2587 }
2588 } else {
2589 asm_tvstore64(as, RID_BASE, ofs, ref);
2590 }
2591#endif
1662 } 2592 }
1663 checkmclim(as); 2593 checkmclim(as);
1664 } 2594 }
1665 lua_assert(map + nent == flinks); 2595 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
1666} 2596}
1667 2597
1668/* -- GC handling --------------------------------------------------------- */ 2598/* -- GC handling --------------------------------------------------------- */
@@ -1686,7 +2616,7 @@ static void asm_gc_check(ASMState *as)
1686 args[1] = ASMREF_TMP2; /* MSize steps */ 2616 args[1] = ASMREF_TMP2; /* MSize steps */
1687 asm_gencall(as, ci, args); 2617 asm_gencall(as, ci, args);
1688 l_end[-3] = MIPS_NOPATCH_GC_CHECK; /* Replace the nop after the call. */ 2618 l_end[-3] = MIPS_NOPATCH_GC_CHECK; /* Replace the nop after the call. */
1689 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768); 2619 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1690 tmp = ra_releasetmp(as, ASMREF_TMP2); 2620 tmp = ra_releasetmp(as, ASMREF_TMP2);
1691 emit_loadi(as, tmp, as->gcsteps); 2621 emit_loadi(as, tmp, as->gcsteps);
1692 /* Jump around GC step if GC total < GC threshold. */ 2622 /* Jump around GC step if GC total < GC threshold. */
@@ -1714,6 +2644,12 @@ static void asm_loop_fixup(ASMState *as)
1714 } 2644 }
1715} 2645}
1716 2646
2647/* Fixup the tail of the loop. */
2648static void asm_loop_tail_fixup(ASMState *as)
2649{
2650 if (as->loopinv) as->mctop--;
2651}
2652
1717/* -- Head of trace ------------------------------------------------------- */ 2653/* -- Head of trace ------------------------------------------------------- */
1718 2654
1719/* Coalesce BASE register for a root trace. */ 2655/* Coalesce BASE register for a root trace. */
@@ -1721,7 +2657,6 @@ static void asm_head_root_base(ASMState *as)
1721{ 2657{
1722 IRIns *ir = IR(REF_BASE); 2658 IRIns *ir = IR(REF_BASE);
1723 Reg r = ir->r; 2659 Reg r = ir->r;
1724 if (as->loopinv) as->mctop--;
1725 if (ra_hasreg(r)) { 2660 if (ra_hasreg(r)) {
1726 ra_free(as, r); 2661 ra_free(as, r);
1727 if (rset_test(as->modset, r) || irt_ismarked(ir->t)) 2662 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
@@ -1736,7 +2671,6 @@ static Reg asm_head_side_base(ASMState *as, IRIns *irp)
1736{ 2671{
1737 IRIns *ir = IR(REF_BASE); 2672 IRIns *ir = IR(REF_BASE);
1738 Reg r = ir->r; 2673 Reg r = ir->r;
1739 if (as->loopinv) as->mctop--;
1740 if (ra_hasreg(r)) { 2674 if (ra_hasreg(r)) {
1741 ra_free(as, r); 2675 ra_free(as, r);
1742 if (rset_test(as->modset, r) || irt_ismarked(ir->t)) 2676 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
@@ -1761,7 +2695,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1761 MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp; 2695 MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp;
1762 int32_t spadj = as->T->spadjust; 2696 int32_t spadj = as->T->spadjust;
1763 MCode *p = as->mctop-1; 2697 MCode *p = as->mctop-1;
1764 *p = spadj ? (MIPSI_ADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP; 2698 *p = spadj ? (MIPSI_AADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
1765 p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu); 2699 p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
1766} 2700}
1767 2701
@@ -1772,139 +2706,26 @@ static void asm_tail_prep(ASMState *as)
1772 as->invmcp = as->loopref ? as->mcp : NULL; 2706 as->invmcp = as->loopref ? as->mcp : NULL;
1773} 2707}
1774 2708
1775/* -- Instruction dispatch ------------------------------------------------ */
1776
1777/* Assemble a single instruction. */
1778static void asm_ir(ASMState *as, IRIns *ir)
1779{
1780 switch ((IROp)ir->o) {
1781 /* Miscellaneous ops. */
1782 case IR_LOOP: asm_loop(as); break;
1783 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
1784 case IR_USE:
1785 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1786 case IR_PHI: asm_phi(as, ir); break;
1787 case IR_HIOP: asm_hiop(as, ir); break;
1788 case IR_GCSTEP: asm_gcstep(as, ir); break;
1789
1790 /* Guarded assertions. */
1791 case IR_EQ: case IR_NE: asm_compeq(as, ir); break;
1792 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1793 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1794 case IR_ABC:
1795 asm_comp(as, ir);
1796 break;
1797
1798 case IR_RETF: asm_retf(as, ir); break;
1799
1800 /* Bit ops. */
1801 case IR_BNOT: asm_bitnot(as, ir); break;
1802 case IR_BSWAP: asm_bitswap(as, ir); break;
1803
1804 case IR_BAND: asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI); break;
1805 case IR_BOR: asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI); break;
1806 case IR_BXOR: asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI); break;
1807
1808 case IR_BSHL: asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL); break;
1809 case IR_BSHR: asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL); break;
1810 case IR_BSAR: asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA); break;
1811 case IR_BROL: lua_assert(0); break;
1812 case IR_BROR: asm_bitror(as, ir); break;
1813
1814 /* Arithmetic ops. */
1815 case IR_ADD: asm_add(as, ir); break;
1816 case IR_SUB: asm_sub(as, ir); break;
1817 case IR_MUL: asm_mul(as, ir); break;
1818 case IR_DIV: asm_fparith(as, ir, MIPSI_DIV_D); break;
1819 case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
1820 case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
1821 case IR_NEG: asm_neg(as, ir); break;
1822
1823 case IR_ABS: asm_fpunary(as, ir, MIPSI_ABS_D); break;
1824 case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
1825 case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
1826 case IR_MIN: asm_min_max(as, ir, 0); break;
1827 case IR_MAX: asm_min_max(as, ir, 1); break;
1828 case IR_FPMATH:
1829 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
1830 break;
1831 if (ir->op2 <= IRFPM_TRUNC)
1832 asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1833 else if (ir->op2 == IRFPM_SQRT)
1834 asm_fpunary(as, ir, MIPSI_SQRT_D);
1835 else
1836 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1837 break;
1838
1839 /* Overflow-checking arithmetic ops. */
1840 case IR_ADDOV: asm_arithov(as, ir); break;
1841 case IR_SUBOV: asm_arithov(as, ir); break;
1842 case IR_MULOV: asm_mulov(as, ir); break;
1843
1844 /* Memory references. */
1845 case IR_AREF: asm_aref(as, ir); break;
1846 case IR_HREF: asm_href(as, ir); break;
1847 case IR_HREFK: asm_hrefk(as, ir); break;
1848 case IR_NEWREF: asm_newref(as, ir); break;
1849 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
1850 case IR_FREF: asm_fref(as, ir); break;
1851 case IR_STRREF: asm_strref(as, ir); break;
1852
1853 /* Loads and stores. */
1854 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
1855 asm_ahuvload(as, ir);
1856 break;
1857 case IR_FLOAD: asm_fload(as, ir); break;
1858 case IR_XLOAD: asm_xload(as, ir); break;
1859 case IR_SLOAD: asm_sload(as, ir); break;
1860
1861 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
1862 case IR_FSTORE: asm_fstore(as, ir); break;
1863 case IR_XSTORE: asm_xstore(as, ir, 0); break;
1864
1865 /* Allocations. */
1866 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
1867 case IR_TNEW: asm_tnew(as, ir); break;
1868 case IR_TDUP: asm_tdup(as, ir); break;
1869 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
1870
1871 /* Write barriers. */
1872 case IR_TBAR: asm_tbar(as, ir); break;
1873 case IR_OBAR: asm_obar(as, ir); break;
1874
1875 /* Type conversions. */
1876 case IR_CONV: asm_conv(as, ir); break;
1877 case IR_TOBIT: asm_tobit(as, ir); break;
1878 case IR_TOSTR: asm_tostr(as, ir); break;
1879 case IR_STRTO: asm_strto(as, ir); break;
1880
1881 /* Calls. */
1882 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
1883 case IR_CALLXS: asm_callx(as, ir); break;
1884 case IR_CARG: break;
1885
1886 default:
1887 setintV(&as->J->errinfo, ir->o);
1888 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
1889 break;
1890 }
1891}
1892
1893/* -- Trace setup --------------------------------------------------------- */ 2709/* -- Trace setup --------------------------------------------------------- */
1894 2710
1895/* Ensure there are enough stack slots for call arguments. */ 2711/* Ensure there are enough stack slots for call arguments. */
1896static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci) 2712static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
1897{ 2713{
1898 IRRef args[CCI_NARGS_MAX*2]; 2714 IRRef args[CCI_NARGS_MAX*2];
1899 uint32_t i, nargs = (int)CCI_NARGS(ci); 2715 uint32_t i, nargs = CCI_XNARGS(ci);
2716#if LJ_32
1900 int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR; 2717 int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2718#else
2719 int nslots = 0, ngpr = REGARG_NUMGPR;
2720#endif
1901 asm_collectargs(as, ir, ci, args); 2721 asm_collectargs(as, ir, ci, args);
1902 for (i = 0; i < nargs; i++) { 2722 for (i = 0; i < nargs; i++) {
1903 if (args[i] && irt_isfp(IR(args[i])->t) && 2723#if LJ_32
2724 if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t) &&
1904 nfpr > 0 && !(ci->flags & CCI_VARARG)) { 2725 nfpr > 0 && !(ci->flags & CCI_VARARG)) {
1905 nfpr--; 2726 nfpr--;
1906 ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1; 2727 ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1;
1907 } else if (args[i] && irt_isnum(IR(args[i])->t)) { 2728 } else if (!LJ_SOFTFP && args[i] && irt_isnum(IR(args[i])->t)) {
1908 nfpr = 0; 2729 nfpr = 0;
1909 ngpr = ngpr & ~1; 2730 ngpr = ngpr & ~1;
1910 if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1; 2731 if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1;
@@ -1912,6 +2733,9 @@ static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
1912 nfpr = 0; 2733 nfpr = 0;
1913 if (ngpr > 0) ngpr--; else nslots++; 2734 if (ngpr > 0) ngpr--; else nslots++;
1914 } 2735 }
2736#else
2737 if (ngpr > 0) ngpr--; else nslots += 2;
2738#endif
1915 } 2739 }
1916 if (nslots > as->evenspill) /* Leave room for args in stack slots. */ 2740 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
1917 as->evenspill = nslots; 2741 as->evenspill = nslots;
@@ -1942,35 +2766,35 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
1942 if (((p[-1] ^ (px-p)) & 0xffffu) == 0 && 2766 if (((p[-1] ^ (px-p)) & 0xffffu) == 0 &&
1943 ((p[-1] & 0xf0000000u) == MIPSI_BEQ || 2767 ((p[-1] & 0xf0000000u) == MIPSI_BEQ ||
1944 (p[-1] & 0xfc1e0000u) == MIPSI_BLTZ || 2768 (p[-1] & 0xfc1e0000u) == MIPSI_BLTZ ||
1945 (p[-1] & 0xffe00000u) == MIPSI_BC1F) && 2769#if !LJ_TARGET_MIPSR6
1946 p[-2] != MIPS_NOPATCH_GC_CHECK) { 2770 (p[-1] & 0xffe00000u) == MIPSI_BC1F
2771#else
2772 (p[-1] & 0xff600000u) == MIPSI_BC1EQZ
2773#endif
2774 ) && p[-2] != MIPS_NOPATCH_GC_CHECK) {
1947 ptrdiff_t delta = target - p; 2775 ptrdiff_t delta = target - p;
1948 if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */ 2776 if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
1949 patchbranch: 2777 patchbranch:
1950 p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu); 2778 p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu);
1951 *p = MIPSI_NOP; /* Replace the load of the exit number. */ 2779 *p = MIPSI_NOP; /* Replace the load of the exit number. */
1952 cstop = p; 2780 cstop = p+1;
1953 if (!cstart) cstart = p-1; 2781 if (!cstart) cstart = p-1;
1954 } else { /* Branch out of range. Use spare jump slot in mcarea. */ 2782 } else { /* Branch out of range. Use spare jump slot in mcarea. */
1955 int i; 2783 MCode *mcjump = asm_sparejump_use(mcarea, tjump);
1956 for (i = (int)(sizeof(MCLink)/sizeof(MCode)); 2784 if (mcjump) {
1957 i < (int)(sizeof(MCLink)/sizeof(MCode)+MIPS_SPAREJUMP*2); 2785 lj_mcode_sync(mcjump, mcjump+1);
1958 i += 2) { 2786 delta = mcjump - p;
1959 if (mcarea[i] == tjump) { 2787 if (((delta + 0x8000) >> 16) == 0) {
1960 delta = mcarea+i - p;
1961 goto patchbranch;
1962 } else if (mcarea[i] == MIPSI_NOP) {
1963 mcarea[i] = tjump;
1964 cstart = mcarea+i;
1965 delta = mcarea+i - p;
1966 goto patchbranch; 2788 goto patchbranch;
2789 } else {
2790 lj_assertJ(0, "spare jump out of range: -Osizemcode too big");
1967 } 2791 }
1968 } 2792 }
1969 /* Ignore jump slot overflow. Child trace is simply not attached. */ 2793 /* Ignore jump slot overflow. Child trace is simply not attached. */
1970 } 2794 }
1971 } else if (p+1 == pe) { 2795 } else if (p+1 == pe) {
1972 /* Patch NOP after code for inverted loop branch. Use of J is ok. */ 2796 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
1973 lua_assert(p[1] == MIPSI_NOP); 2797 lj_assertJ(p[1] == MIPSI_NOP, "expected NOP");
1974 p[1] = tjump; 2798 p[1] = tjump;
1975 *p = MIPSI_NOP; /* Replace the load of the exit number. */ 2799 *p = MIPSI_NOP; /* Replace the load of the exit number. */
1976 cstop = p+2; 2800 cstop = p+2;
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index bd9374cc..6555312d 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -156,6 +156,9 @@ static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
156 return ra_allock(as, ofs-(int16_t)ofs, allow); 156 return ra_allock(as, ofs-(int16_t)ofs, allow);
157 } 157 }
158 } 158 }
159 } else if (ir->o == IR_TMPREF) {
160 *ofsp = (int32_t)(offsetof(global_State, tmptv)-32768);
161 return RID_JGL;
159 } 162 }
160 } 163 }
161 *ofsp = 0; 164 *ofsp = 0;
@@ -181,7 +184,7 @@ static void asm_fusexref(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
181 return; 184 return;
182 } 185 }
183 } else if (ir->o == IR_STRREF) { 186 } else if (ir->o == IR_STRREF) {
184 lua_assert(ofs == 0); 187 lj_assertA(ofs == 0, "bad usage");
185 ofs = (int32_t)sizeof(GCstr); 188 ofs = (int32_t)sizeof(GCstr);
186 if (irref_isk(ir->op2)) { 189 if (irref_isk(ir->op2)) {
187 ofs += IR(ir->op2)->i; 190 ofs += IR(ir->op2)->i;
@@ -226,12 +229,14 @@ static void asm_fusexrefx(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
226 emit_tab(as, pi, rt, left, right); 229 emit_tab(as, pi, rt, left, right);
227} 230}
228 231
232#if !LJ_SOFTFP
229/* Fuse to multiply-add/sub instruction. */ 233/* Fuse to multiply-add/sub instruction. */
230static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir) 234static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir)
231{ 235{
232 IRRef lref = ir->op1, rref = ir->op2; 236 IRRef lref = ir->op1, rref = ir->op2;
233 IRIns *irm; 237 IRIns *irm;
234 if (lref != rref && 238 if ((as->flags & JIT_F_OPT_FMA) &&
239 lref != rref &&
235 ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) && 240 ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) &&
236 ra_noreg(irm->r)) || 241 ra_noreg(irm->r)) ||
237 (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) && 242 (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) &&
@@ -245,24 +250,30 @@ static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir)
245 } 250 }
246 return 0; 251 return 0;
247} 252}
253#endif
248 254
249/* -- Calls --------------------------------------------------------------- */ 255/* -- Calls --------------------------------------------------------------- */
250 256
251/* Generate a call to a C function. */ 257/* Generate a call to a C function. */
252static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) 258static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
253{ 259{
254 uint32_t n, nargs = CCI_NARGS(ci); 260 uint32_t n, nargs = CCI_XNARGS(ci);
255 int32_t ofs = 8; 261 int32_t ofs = 8;
256 Reg gpr = REGARG_FIRSTGPR, fpr = REGARG_FIRSTFPR; 262 Reg gpr = REGARG_FIRSTGPR;
263#if !LJ_SOFTFP
264 Reg fpr = REGARG_FIRSTFPR;
265#endif
257 if ((void *)ci->func) 266 if ((void *)ci->func)
258 emit_call(as, (void *)ci->func); 267 emit_call(as, (void *)ci->func);
259 for (n = 0; n < nargs; n++) { /* Setup args. */ 268 for (n = 0; n < nargs; n++) { /* Setup args. */
260 IRRef ref = args[n]; 269 IRRef ref = args[n];
261 if (ref) { 270 if (ref) {
262 IRIns *ir = IR(ref); 271 IRIns *ir = IR(ref);
272#if !LJ_SOFTFP
263 if (irt_isfp(ir->t)) { 273 if (irt_isfp(ir->t)) {
264 if (fpr <= REGARG_LASTFPR) { 274 if (fpr <= REGARG_LASTFPR) {
265 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */ 275 lj_assertA(rset_test(as->freeset, fpr),
276 "reg %d not free", fpr); /* Already evicted. */
266 ra_leftov(as, fpr, ref); 277 ra_leftov(as, fpr, ref);
267 fpr++; 278 fpr++;
268 } else { 279 } else {
@@ -271,9 +282,12 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
271 emit_spstore(as, ir, r, ofs); 282 emit_spstore(as, ir, r, ofs);
272 ofs += irt_isnum(ir->t) ? 8 : 4; 283 ofs += irt_isnum(ir->t) ? 8 : 4;
273 } 284 }
274 } else { 285 } else
286#endif
287 {
275 if (gpr <= REGARG_LASTGPR) { 288 if (gpr <= REGARG_LASTGPR) {
276 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */ 289 lj_assertA(rset_test(as->freeset, gpr),
290 "reg %d not free", gpr); /* Already evicted. */
277 ra_leftov(as, gpr, ref); 291 ra_leftov(as, gpr, ref);
278 gpr++; 292 gpr++;
279 } else { 293 } else {
@@ -290,8 +304,10 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
290 } 304 }
291 checkmclim(as); 305 checkmclim(as);
292 } 306 }
307#if !LJ_SOFTFP
293 if ((ci->flags & CCI_VARARG)) /* Vararg calls need to know about FPR use. */ 308 if ((ci->flags & CCI_VARARG)) /* Vararg calls need to know about FPR use. */
294 emit_tab(as, fpr == REGARG_FIRSTFPR ? PPCI_CRXOR : PPCI_CREQV, 6, 6, 6); 309 emit_tab(as, fpr == REGARG_FIRSTFPR ? PPCI_CRXOR : PPCI_CREQV, 6, 6, 6);
310#endif
295} 311}
296 312
297/* Setup result reg/sp for call. Evict scratch regs. */ 313/* Setup result reg/sp for call. Evict scratch regs. */
@@ -299,16 +315,18 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
299{ 315{
300 RegSet drop = RSET_SCRATCH; 316 RegSet drop = RSET_SCRATCH;
301 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t)); 317 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
318#if !LJ_SOFTFP
302 if ((ci->flags & CCI_NOFPRCLOBBER)) 319 if ((ci->flags & CCI_NOFPRCLOBBER))
303 drop &= ~RSET_FPR; 320 drop &= ~RSET_FPR;
321#endif
304 if (ra_hasreg(ir->r)) 322 if (ra_hasreg(ir->r))
305 rset_clear(drop, ir->r); /* Dest reg handled below. */ 323 rset_clear(drop, ir->r); /* Dest reg handled below. */
306 if (hiop && ra_hasreg((ir+1)->r)) 324 if (hiop && ra_hasreg((ir+1)->r))
307 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */ 325 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
308 ra_evictset(as, drop); /* Evictions must be performed first. */ 326 ra_evictset(as, drop); /* Evictions must be performed first. */
309 if (ra_used(ir)) { 327 if (ra_used(ir)) {
310 lua_assert(!irt_ispri(ir->t)); 328 lj_assertA(!irt_ispri(ir->t), "PRI dest");
311 if (irt_isfp(ir->t)) { 329 if (!LJ_SOFTFP && irt_isfp(ir->t)) {
312 if ((ci->flags & CCI_CASTU64)) { 330 if ((ci->flags & CCI_CASTU64)) {
313 /* Use spill slot or temp slots. */ 331 /* Use spill slot or temp slots. */
314 int32_t ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP; 332 int32_t ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
@@ -331,15 +349,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
331 } 349 }
332} 350}
333 351
334static void asm_call(ASMState *as, IRIns *ir)
335{
336 IRRef args[CCI_NARGS_MAX];
337 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
338 asm_collectargs(as, ir, ci, args);
339 asm_setupresult(as, ir, ci);
340 asm_gencall(as, ci, args);
341}
342
343static void asm_callx(ASMState *as, IRIns *ir) 352static void asm_callx(ASMState *as, IRIns *ir)
344{ 353{
345 IRRef args[CCI_NARGS_MAX*2]; 354 IRRef args[CCI_NARGS_MAX*2];
@@ -352,7 +361,7 @@ static void asm_callx(ASMState *as, IRIns *ir)
352 func = ir->op2; irf = IR(func); 361 func = ir->op2; irf = IR(func);
353 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); } 362 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
354 if (irref_isk(func)) { /* Call to constant address. */ 363 if (irref_isk(func)) { /* Call to constant address. */
355 ci.func = (ASMFunction)(void *)(irf->i); 364 ci.func = (ASMFunction)(void *)(intptr_t)(irf->i);
356 } else { /* Need a non-argument register for indirect calls. */ 365 } else { /* Need a non-argument register for indirect calls. */
357 RegSet allow = RSET_GPR & ~RSET_RANGE(RID_R0, REGARG_LASTGPR+1); 366 RegSet allow = RSET_GPR & ~RSET_RANGE(RID_R0, REGARG_LASTGPR+1);
358 Reg freg = ra_alloc1(as, func, allow); 367 Reg freg = ra_alloc1(as, func, allow);
@@ -363,16 +372,6 @@ static void asm_callx(ASMState *as, IRIns *ir)
363 asm_gencall(as, &ci, args); 372 asm_gencall(as, &ci, args);
364} 373}
365 374
366static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
367{
368 const CCallInfo *ci = &lj_ir_callinfo[id];
369 IRRef args[2];
370 args[0] = ir->op1;
371 args[1] = ir->op2;
372 asm_setupresult(as, ir, ci);
373 asm_gencall(as, ci, args);
374}
375
376/* -- Returns ------------------------------------------------------------- */ 375/* -- Returns ------------------------------------------------------------- */
377 376
378/* Return to lower frame. Guard that it goes to the right spot. */ 377/* Return to lower frame. Guard that it goes to the right spot. */
@@ -380,7 +379,7 @@ static void asm_retf(ASMState *as, IRIns *ir)
380{ 379{
381 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR); 380 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
382 void *pc = ir_kptr(IR(ir->op2)); 381 void *pc = ir_kptr(IR(ir->op2));
383 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1)); 382 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
384 as->topslot -= (BCReg)delta; 383 as->topslot -= (BCReg)delta;
385 if ((int32_t)as->topslot < 0) as->topslot = 0; 384 if ((int32_t)as->topslot < 0) as->topslot = 0;
386 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */ 385 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
@@ -392,8 +391,24 @@ static void asm_retf(ASMState *as, IRIns *ir)
392 emit_tai(as, PPCI_LWZ, RID_TMP, base, -8); 391 emit_tai(as, PPCI_LWZ, RID_TMP, base, -8);
393} 392}
394 393
394/* -- Buffer operations --------------------------------------------------- */
395
396#if LJ_HASBUFFER
397static void asm_bufhdr_write(ASMState *as, Reg sb)
398{
399 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
400 IRIns irgc;
401 irgc.ot = IRT(0, IRT_PGC); /* GC type. */
402 emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
403 emit_rot(as, PPCI_RLWIMI, RID_TMP, tmp, 0, 31-lj_fls(SBUF_MASK_FLAG), 31);
404 emit_getgl(as, RID_TMP, cur_L);
405 emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
406}
407#endif
408
395/* -- Type conversions ---------------------------------------------------- */ 409/* -- Type conversions ---------------------------------------------------- */
396 410
411#if !LJ_SOFTFP
397static void asm_tointg(ASMState *as, IRIns *ir, Reg left) 412static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
398{ 413{
399 RegSet allow = RSET_FPR; 414 RegSet allow = RSET_FPR;
@@ -410,8 +425,7 @@ static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
410 emit_asi(as, PPCI_XORIS, RID_TMP, dest, 0x8000); 425 emit_asi(as, PPCI_XORIS, RID_TMP, dest, 0x8000);
411 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO); 426 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
412 emit_lsptr(as, PPCI_LFS, (fbias & 31), 427 emit_lsptr(as, PPCI_LFS, (fbias & 31),
413 (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)), 428 (void *)&as->J->k32[LJ_K32_2P52_2P31], RSET_GPR);
414 RSET_GPR);
415 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP); 429 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
416 emit_fb(as, PPCI_FCTIWZ, tmp, left); 430 emit_fb(as, PPCI_FCTIWZ, tmp, left);
417} 431}
@@ -427,15 +441,27 @@ static void asm_tobit(ASMState *as, IRIns *ir)
427 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP); 441 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
428 emit_fab(as, PPCI_FADD, tmp, left, right); 442 emit_fab(as, PPCI_FADD, tmp, left, right);
429} 443}
444#endif
430 445
431static void asm_conv(ASMState *as, IRIns *ir) 446static void asm_conv(ASMState *as, IRIns *ir)
432{ 447{
433 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK); 448 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
449#if !LJ_SOFTFP
434 int stfp = (st == IRT_NUM || st == IRT_FLOAT); 450 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
451#endif
435 IRRef lref = ir->op1; 452 IRRef lref = ir->op1;
436 lua_assert(irt_type(ir->t) != st); 453 /* 64 bit integer conversions are handled by SPLIT. */
437 lua_assert(!(irt_isint64(ir->t) || 454 lj_assertA(!(irt_isint64(ir->t) || (st == IRT_I64 || st == IRT_U64)),
438 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */ 455 "IR %04d has unsplit 64 bit type",
456 (int)(ir - as->ir) - REF_BIAS);
457#if LJ_SOFTFP
458 /* FP conversions are handled by SPLIT. */
459 lj_assertA(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT),
460 "IR %04d has FP type",
461 (int)(ir - as->ir) - REF_BIAS);
462 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
463#else
464 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
439 if (irt_isfp(ir->t)) { 465 if (irt_isfp(ir->t)) {
440 Reg dest = ra_dest(as, ir, RSET_FPR); 466 Reg dest = ra_dest(as, ir, RSET_FPR);
441 if (stfp) { /* FP to FP conversion. */ 467 if (stfp) { /* FP to FP conversion. */
@@ -450,13 +476,11 @@ static void asm_conv(ASMState *as, IRIns *ir)
450 Reg left = ra_alloc1(as, lref, allow); 476 Reg left = ra_alloc1(as, lref, allow);
451 Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, left)); 477 Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, left));
452 Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest)); 478 Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
453 const float *kbias;
454 if (irt_isfloat(ir->t)) emit_fb(as, PPCI_FRSP, dest, dest); 479 if (irt_isfloat(ir->t)) emit_fb(as, PPCI_FRSP, dest, dest);
455 emit_fab(as, PPCI_FSUB, dest, dest, fbias); 480 emit_fab(as, PPCI_FSUB, dest, dest, fbias);
456 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP); 481 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
457 kbias = (const float *)lj_ir_k64_find(as->J, U64x(59800004,59800000)); 482 emit_lsptr(as, PPCI_LFS, (fbias & 31),
458 if (st == IRT_U32) kbias++; 483 &as->J->k32[st == IRT_U32 ? LJ_K32_2P52 : LJ_K32_2P52_2P31],
459 emit_lsptr(as, PPCI_LFS, (fbias & 31), (void *)kbias,
460 rset_clear(allow, hibias)); 484 rset_clear(allow, hibias));
461 emit_tai(as, PPCI_STW, st == IRT_U32 ? left : RID_TMP, 485 emit_tai(as, PPCI_STW, st == IRT_U32 ? left : RID_TMP,
462 RID_SP, SPOFS_TMPLO); 486 RID_SP, SPOFS_TMPLO);
@@ -466,7 +490,8 @@ static void asm_conv(ASMState *as, IRIns *ir)
466 } else if (stfp) { /* FP to integer conversion. */ 490 } else if (stfp) { /* FP to integer conversion. */
467 if (irt_isguard(ir->t)) { 491 if (irt_isguard(ir->t)) {
468 /* Checked conversions are only supported from number to int. */ 492 /* Checked conversions are only supported from number to int. */
469 lua_assert(irt_isint(ir->t) && st == IRT_NUM); 493 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
494 "bad type for checked CONV");
470 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); 495 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
471 } else { 496 } else {
472 Reg dest = ra_dest(as, ir, RSET_GPR); 497 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -489,19 +514,20 @@ static void asm_conv(ASMState *as, IRIns *ir)
489 emit_fb(as, PPCI_FCTIWZ, tmp, tmp); 514 emit_fb(as, PPCI_FCTIWZ, tmp, tmp);
490 emit_fab(as, PPCI_FSUB, tmp, left, tmp); 515 emit_fab(as, PPCI_FSUB, tmp, left, tmp);
491 emit_lsptr(as, PPCI_LFS, (tmp & 31), 516 emit_lsptr(as, PPCI_LFS, (tmp & 31),
492 (void *)lj_ir_k64_find(as->J, U64x(4f000000,00000000)), 517 (void *)&as->J->k32[LJ_K32_2P31], RSET_GPR);
493 RSET_GPR);
494 } else { 518 } else {
495 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO); 519 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
496 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP); 520 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
497 emit_fb(as, PPCI_FCTIWZ, tmp, left); 521 emit_fb(as, PPCI_FCTIWZ, tmp, left);
498 } 522 }
499 } 523 }
500 } else { 524 } else
525#endif
526 {
501 Reg dest = ra_dest(as, ir, RSET_GPR); 527 Reg dest = ra_dest(as, ir, RSET_GPR);
502 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */ 528 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
503 Reg left = ra_alloc1(as, ir->op1, RSET_GPR); 529 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
504 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); 530 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
505 if ((ir->op2 & IRCONV_SEXT)) 531 if ((ir->op2 & IRCONV_SEXT))
506 emit_as(as, st == IRT_I8 ? PPCI_EXTSB : PPCI_EXTSH, dest, left); 532 emit_as(as, st == IRT_I8 ? PPCI_EXTSB : PPCI_EXTSH, dest, left);
507 else 533 else
@@ -513,90 +539,102 @@ static void asm_conv(ASMState *as, IRIns *ir)
513 } 539 }
514} 540}
515 541
516#if LJ_HASFFI
517static void asm_conv64(ASMState *as, IRIns *ir)
518{
519 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
520 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
521 IRCallID id;
522 const CCallInfo *ci;
523 IRRef args[2];
524 args[0] = ir->op1;
525 args[1] = (ir-1)->op1;
526 if (st == IRT_NUM || st == IRT_FLOAT) {
527 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
528 ir--;
529 } else {
530 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
531 }
532 ci = &lj_ir_callinfo[id];
533 asm_setupresult(as, ir, ci);
534 asm_gencall(as, ci, args);
535}
536#endif
537
538static void asm_strto(ASMState *as, IRIns *ir) 542static void asm_strto(ASMState *as, IRIns *ir)
539{ 543{
540 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num]; 544 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
541 IRRef args[2]; 545 IRRef args[2];
542 int32_t ofs; 546 int32_t ofs = SPOFS_TMP;
547#if LJ_SOFTFP
548 ra_evictset(as, RSET_SCRATCH);
549 if (ra_used(ir)) {
550 if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) &&
551 (ir->s & 1) == LJ_BE && (ir->s ^ 1) == (ir+1)->s) {
552 int i;
553 for (i = 0; i < 2; i++) {
554 Reg r = (ir+i)->r;
555 if (ra_hasreg(r)) {
556 ra_free(as, r);
557 ra_modified(as, r);
558 emit_spload(as, ir+i, r, sps_scale((ir+i)->s));
559 }
560 }
561 ofs = sps_scale(ir->s & ~1);
562 } else {
563 Reg rhi = ra_dest(as, ir+1, RSET_GPR);
564 Reg rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi));
565 emit_tai(as, PPCI_LWZ, rhi, RID_SP, ofs);
566 emit_tai(as, PPCI_LWZ, rlo, RID_SP, ofs+4);
567 }
568 }
569#else
543 RegSet drop = RSET_SCRATCH; 570 RegSet drop = RSET_SCRATCH;
544 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */ 571 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
545 ra_evictset(as, drop); 572 ra_evictset(as, drop);
573 if (ir->s) ofs = sps_scale(ir->s);
574#endif
546 asm_guardcc(as, CC_EQ); 575 asm_guardcc(as, CC_EQ);
547 emit_ai(as, PPCI_CMPWI, RID_RET, 0); /* Test return status. */ 576 emit_ai(as, PPCI_CMPWI, RID_RET, 0); /* Test return status. */
548 args[0] = ir->op1; /* GCstr *str */ 577 args[0] = ir->op1; /* GCstr *str */
549 args[1] = ASMREF_TMP1; /* TValue *n */ 578 args[1] = ASMREF_TMP1; /* TValue *n */
550 asm_gencall(as, ci, args); 579 asm_gencall(as, ci, args);
551 /* Store the result to the spill slot or temp slots. */ 580 /* Store the result to the spill slot or temp slots. */
552 ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
553 emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_SP, ofs); 581 emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_SP, ofs);
554} 582}
555 583
584/* -- Memory references --------------------------------------------------- */
585
556/* Get pointer to TValue. */ 586/* Get pointer to TValue. */
557static void asm_tvptr(ASMState *as, Reg dest, IRRef ref) 587static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
558{ 588{
559 IRIns *ir = IR(ref); 589 int32_t tmpofs = (int32_t)(offsetof(global_State, tmptv)-32768);
560 if (irt_isnum(ir->t)) { 590 if ((mode & IRTMPREF_IN1)) {
561 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */ 591 IRIns *ir = IR(ref);
562 ra_allockreg(as, i32ptr(ir_knum(ir)), dest); 592 if (irt_isnum(ir->t)) {
563 else /* Otherwise force a spill and use the spill slot. */ 593 if ((mode & IRTMPREF_OUT1)) {
564 emit_tai(as, PPCI_ADDI, dest, RID_SP, ra_spill(as, ir)); 594#if LJ_SOFTFP
565 } else { 595 lj_assertA(irref_isk(ref), "unsplit FP op");
566 /* Otherwise use g->tmptv to hold the TValue. */ 596 emit_tai(as, PPCI_ADDI, dest, RID_JGL, tmpofs);
567 RegSet allow = rset_exclude(RSET_GPR, dest); 597 emit_setgl(as,
568 Reg type; 598 ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, RSET_GPR),
569 emit_tai(as, PPCI_ADDI, dest, RID_JGL, offsetof(global_State, tmptv)-32768); 599 tmptv.u32.lo);
570 if (!irt_ispri(ir->t)) { 600 emit_setgl(as,
571 Reg src = ra_alloc1(as, ref, allow); 601 ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, RSET_GPR),
572 emit_setgl(as, src, tmptv.gcr); 602 tmptv.u32.hi);
603#else
604 Reg src = ra_alloc1(as, ref, RSET_FPR);
605 emit_tai(as, PPCI_ADDI, dest, RID_JGL, tmpofs);
606 emit_fai(as, PPCI_STFD, src, RID_JGL, tmpofs);
607#endif
608 } else if (irref_isk(ref)) {
609 /* Use the number constant itself as a TValue. */
610 ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
611 } else {
612#if LJ_SOFTFP
613 lj_assertA(0, "unsplit FP op");
614#else
615 /* Otherwise force a spill and use the spill slot. */
616 emit_tai(as, PPCI_ADDI, dest, RID_SP, ra_spill(as, ir));
617#endif
618 }
619 } else {
620 /* Otherwise use g->tmptv to hold the TValue. */
621 Reg type;
622 emit_tai(as, PPCI_ADDI, dest, RID_JGL, tmpofs);
623 if (!irt_ispri(ir->t)) {
624 Reg src = ra_alloc1(as, ref, RSET_GPR);
625 emit_setgl(as, src, tmptv.gcr);
626 }
627 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t))
628 type = ra_alloc1(as, ref+1, RSET_GPR);
629 else
630 type = ra_allock(as, irt_toitype(ir->t), RSET_GPR);
631 emit_setgl(as, type, tmptv.it);
573 } 632 }
574 type = ra_allock(as, irt_toitype(ir->t), allow);
575 emit_setgl(as, type, tmptv.it);
576 }
577}
578
579static void asm_tostr(ASMState *as, IRIns *ir)
580{
581 IRRef args[2];
582 args[0] = ASMREF_L;
583 as->gcsteps++;
584 if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
585 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
586 args[1] = ASMREF_TMP1; /* const lua_Number * */
587 asm_setupresult(as, ir, ci); /* GCstr * */
588 asm_gencall(as, ci, args);
589 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
590 } else { 633 } else {
591 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint]; 634 emit_tai(as, PPCI_ADDI, dest, RID_JGL, tmpofs);
592 args[1] = ir->op1; /* int32_t k */
593 asm_setupresult(as, ir, ci); /* GCstr * */
594 asm_gencall(as, ci, args);
595 } 635 }
596} 636}
597 637
598/* -- Memory references --------------------------------------------------- */
599
600static void asm_aref(ASMState *as, IRIns *ir) 638static void asm_aref(ASMState *as, IRIns *ir)
601{ 639{
602 Reg dest = ra_dest(as, ir, RSET_GPR); 640 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -636,11 +674,27 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
636 Reg tisnum = RID_NONE, tmpnum = RID_NONE; 674 Reg tisnum = RID_NONE, tmpnum = RID_NONE;
637 IRRef refkey = ir->op2; 675 IRRef refkey = ir->op2;
638 IRIns *irkey = IR(refkey); 676 IRIns *irkey = IR(refkey);
677 int isk = irref_isk(refkey);
639 IRType1 kt = irkey->t; 678 IRType1 kt = irkey->t;
640 uint32_t khash; 679 uint32_t khash;
641 MCLabel l_end, l_loop, l_next; 680 MCLabel l_end, l_loop, l_next;
642 681
643 rset_clear(allow, tab); 682 rset_clear(allow, tab);
683#if LJ_SOFTFP
684 if (!isk) {
685 key = ra_alloc1(as, refkey, allow);
686 rset_clear(allow, key);
687 if (irkey[1].o == IR_HIOP) {
688 if (ra_hasreg((irkey+1)->r)) {
689 tmpnum = (irkey+1)->r;
690 ra_noweak(as, tmpnum);
691 } else {
692 tmpnum = ra_allocref(as, refkey+1, allow);
693 }
694 rset_clear(allow, tmpnum);
695 }
696 }
697#else
644 if (irt_isnum(kt)) { 698 if (irt_isnum(kt)) {
645 key = ra_alloc1(as, refkey, RSET_FPR); 699 key = ra_alloc1(as, refkey, RSET_FPR);
646 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key)); 700 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
@@ -650,6 +704,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
650 key = ra_alloc1(as, refkey, allow); 704 key = ra_alloc1(as, refkey, allow);
651 rset_clear(allow, key); 705 rset_clear(allow, key);
652 } 706 }
707#endif
653 tmp2 = ra_scratch(as, allow); 708 tmp2 = ra_scratch(as, allow);
654 rset_clear(allow, tmp2); 709 rset_clear(allow, tmp2);
655 710
@@ -672,7 +727,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
672 asm_guardcc(as, CC_EQ); 727 asm_guardcc(as, CC_EQ);
673 else 728 else
674 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end); 729 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
675 if (irt_isnum(kt)) { 730 if (!LJ_SOFTFP && irt_isnum(kt)) {
676 emit_fab(as, PPCI_FCMPU, 0, tmpnum, key); 731 emit_fab(as, PPCI_FCMPU, 0, tmpnum, key);
677 emit_condbranch(as, PPCI_BC, CC_GE, l_next); 732 emit_condbranch(as, PPCI_BC, CC_GE, l_next);
678 emit_ab(as, PPCI_CMPLW, tmp1, tisnum); 733 emit_ab(as, PPCI_CMPLW, tmp1, tisnum);
@@ -682,7 +737,10 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
682 emit_ab(as, PPCI_CMPW, tmp2, key); 737 emit_ab(as, PPCI_CMPW, tmp2, key);
683 emit_condbranch(as, PPCI_BC, CC_NE, l_next); 738 emit_condbranch(as, PPCI_BC, CC_NE, l_next);
684 } 739 }
685 emit_ai(as, PPCI_CMPWI, tmp1, irt_toitype(irkey->t)); 740 if (LJ_SOFTFP && ra_hasreg(tmpnum))
741 emit_ab(as, PPCI_CMPW, tmp1, tmpnum);
742 else
743 emit_ai(as, PPCI_CMPWI, tmp1, irt_toitype(irkey->t));
686 if (!irt_ispri(kt)) 744 if (!irt_ispri(kt))
687 emit_tai(as, PPCI_LWZ, tmp2, dest, (int32_t)offsetof(Node, key.gcr)); 745 emit_tai(as, PPCI_LWZ, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
688 } 746 }
@@ -691,35 +749,41 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
691 (((char *)as->mcp-(char *)l_loop) & 0xffffu); 749 (((char *)as->mcp-(char *)l_loop) & 0xffffu);
692 750
693 /* Load main position relative to tab->node into dest. */ 751 /* Load main position relative to tab->node into dest. */
694 khash = irref_isk(refkey) ? ir_khash(irkey) : 1; 752 khash = isk ? ir_khash(as, irkey) : 1;
695 if (khash == 0) { 753 if (khash == 0) {
696 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node)); 754 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
697 } else { 755 } else {
698 Reg tmphash = tmp1; 756 Reg tmphash = tmp1;
699 if (irref_isk(refkey)) 757 if (isk)
700 tmphash = ra_allock(as, khash, allow); 758 tmphash = ra_allock(as, khash, allow);
701 emit_tab(as, PPCI_ADD, dest, dest, tmp1); 759 emit_tab(as, PPCI_ADD, dest, dest, tmp1);
702 emit_tai(as, PPCI_MULLI, tmp1, tmp1, sizeof(Node)); 760 emit_tai(as, PPCI_MULLI, tmp1, tmp1, sizeof(Node));
703 emit_asb(as, PPCI_AND, tmp1, tmp2, tmphash); 761 emit_asb(as, PPCI_AND, tmp1, tmp2, tmphash);
704 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node)); 762 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
705 emit_tai(as, PPCI_LWZ, tmp2, tab, (int32_t)offsetof(GCtab, hmask)); 763 emit_tai(as, PPCI_LWZ, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
706 if (irref_isk(refkey)) { 764 if (isk) {
707 /* Nothing to do. */ 765 /* Nothing to do. */
708 } else if (irt_isstr(kt)) { 766 } else if (irt_isstr(kt)) {
709 emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, hash)); 767 emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, sid));
710 } else { /* Must match with hash*() in lj_tab.c. */ 768 } else { /* Must match with hash*() in lj_tab.c. */
711 emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1); 769 emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1);
712 emit_rotlwi(as, tmp2, tmp2, HASH_ROT3); 770 emit_rotlwi(as, tmp2, tmp2, HASH_ROT3);
713 emit_asb(as, PPCI_XOR, tmp1, tmp1, tmp2); 771 emit_asb(as, PPCI_XOR, tmp1, tmp1, tmp2);
714 emit_rotlwi(as, tmp1, tmp1, (HASH_ROT2+HASH_ROT1)&31); 772 emit_rotlwi(as, tmp1, tmp1, (HASH_ROT2+HASH_ROT1)&31);
715 emit_tab(as, PPCI_SUBF, tmp2, dest, tmp2); 773 emit_tab(as, PPCI_SUBF, tmp2, dest, tmp2);
716 if (irt_isnum(kt)) { 774 if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
775#if LJ_SOFTFP
776 emit_asb(as, PPCI_XOR, tmp2, key, tmp1);
777 emit_rotlwi(as, dest, tmp1, HASH_ROT1);
778 emit_tab(as, PPCI_ADD, tmp1, tmpnum, tmpnum);
779#else
717 int32_t ofs = ra_spill(as, irkey); 780 int32_t ofs = ra_spill(as, irkey);
718 emit_asb(as, PPCI_XOR, tmp2, tmp2, tmp1); 781 emit_asb(as, PPCI_XOR, tmp2, tmp2, tmp1);
719 emit_rotlwi(as, dest, tmp1, HASH_ROT1); 782 emit_rotlwi(as, dest, tmp1, HASH_ROT1);
720 emit_tab(as, PPCI_ADD, tmp1, tmp1, tmp1); 783 emit_tab(as, PPCI_ADD, tmp1, tmp1, tmp1);
721 emit_tai(as, PPCI_LWZ, tmp2, RID_SP, ofs+4); 784 emit_tai(as, PPCI_LWZ, tmp2, RID_SP, ofs+4);
722 emit_tai(as, PPCI_LWZ, tmp1, RID_SP, ofs); 785 emit_tai(as, PPCI_LWZ, tmp1, RID_SP, ofs);
786#endif
723 } else { 787 } else {
724 emit_asb(as, PPCI_XOR, tmp2, key, tmp1); 788 emit_asb(as, PPCI_XOR, tmp2, key, tmp1);
725 emit_rotlwi(as, dest, tmp1, HASH_ROT1); 789 emit_rotlwi(as, dest, tmp1, HASH_ROT1);
@@ -740,7 +804,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
740 Reg node = ra_alloc1(as, ir->op1, RSET_GPR); 804 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
741 Reg key = RID_NONE, type = RID_TMP, idx = node; 805 Reg key = RID_NONE, type = RID_TMP, idx = node;
742 RegSet allow = rset_exclude(RSET_GPR, node); 806 RegSet allow = rset_exclude(RSET_GPR, node);
743 lua_assert(ofs % sizeof(Node) == 0); 807 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
744 if (ofs > 32736) { 808 if (ofs > 32736) {
745 idx = dest; 809 idx = dest;
746 rset_clear(allow, dest); 810 rset_clear(allow, dest);
@@ -773,20 +837,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
773 } 837 }
774} 838}
775 839
776static void asm_newref(ASMState *as, IRIns *ir)
777{
778 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
779 IRRef args[3];
780 if (ir->r == RID_SINK)
781 return;
782 args[0] = ASMREF_L; /* lua_State *L */
783 args[1] = ir->op1; /* GCtab *t */
784 args[2] = ASMREF_TMP1; /* cTValue *key */
785 asm_setupresult(as, ir, ci); /* TValue * */
786 asm_gencall(as, ci, args);
787 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
788}
789
790static void asm_uref(ASMState *as, IRIns *ir) 840static void asm_uref(ASMState *as, IRIns *ir)
791{ 841{
792 Reg dest = ra_dest(as, ir, RSET_GPR); 842 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -813,7 +863,7 @@ static void asm_uref(ASMState *as, IRIns *ir)
813static void asm_fref(ASMState *as, IRIns *ir) 863static void asm_fref(ASMState *as, IRIns *ir)
814{ 864{
815 UNUSED(as); UNUSED(ir); 865 UNUSED(as); UNUSED(ir);
816 lua_assert(!ra_used(ir)); 866 lj_assertA(!ra_used(ir), "unfused FREF");
817} 867}
818 868
819static void asm_strref(ASMState *as, IRIns *ir) 869static void asm_strref(ASMState *as, IRIns *ir)
@@ -853,26 +903,28 @@ static void asm_strref(ASMState *as, IRIns *ir)
853 903
854/* -- Loads and stores ---------------------------------------------------- */ 904/* -- Loads and stores ---------------------------------------------------- */
855 905
856static PPCIns asm_fxloadins(IRIns *ir) 906static PPCIns asm_fxloadins(ASMState *as, IRIns *ir)
857{ 907{
908 UNUSED(as);
858 switch (irt_type(ir->t)) { 909 switch (irt_type(ir->t)) {
859 case IRT_I8: return PPCI_LBZ; /* Needs sign-extension. */ 910 case IRT_I8: return PPCI_LBZ; /* Needs sign-extension. */
860 case IRT_U8: return PPCI_LBZ; 911 case IRT_U8: return PPCI_LBZ;
861 case IRT_I16: return PPCI_LHA; 912 case IRT_I16: return PPCI_LHA;
862 case IRT_U16: return PPCI_LHZ; 913 case IRT_U16: return PPCI_LHZ;
863 case IRT_NUM: return PPCI_LFD; 914 case IRT_NUM: lj_assertA(!LJ_SOFTFP, "unsplit FP op"); return PPCI_LFD;
864 case IRT_FLOAT: return PPCI_LFS; 915 case IRT_FLOAT: if (!LJ_SOFTFP) return PPCI_LFS;
865 default: return PPCI_LWZ; 916 default: return PPCI_LWZ;
866 } 917 }
867} 918}
868 919
869static PPCIns asm_fxstoreins(IRIns *ir) 920static PPCIns asm_fxstoreins(ASMState *as, IRIns *ir)
870{ 921{
922 UNUSED(as);
871 switch (irt_type(ir->t)) { 923 switch (irt_type(ir->t)) {
872 case IRT_I8: case IRT_U8: return PPCI_STB; 924 case IRT_I8: case IRT_U8: return PPCI_STB;
873 case IRT_I16: case IRT_U16: return PPCI_STH; 925 case IRT_I16: case IRT_U16: return PPCI_STH;
874 case IRT_NUM: return PPCI_STFD; 926 case IRT_NUM: lj_assertA(!LJ_SOFTFP, "unsplit FP op"); return PPCI_STFD;
875 case IRT_FLOAT: return PPCI_STFS; 927 case IRT_FLOAT: if (!LJ_SOFTFP) return PPCI_STFS;
876 default: return PPCI_STW; 928 default: return PPCI_STW;
877 } 929 }
878} 930}
@@ -880,18 +932,24 @@ static PPCIns asm_fxstoreins(IRIns *ir)
880static void asm_fload(ASMState *as, IRIns *ir) 932static void asm_fload(ASMState *as, IRIns *ir)
881{ 933{
882 Reg dest = ra_dest(as, ir, RSET_GPR); 934 Reg dest = ra_dest(as, ir, RSET_GPR);
883 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR); 935 PPCIns pi = asm_fxloadins(as, ir);
884 PPCIns pi = asm_fxloadins(ir); 936 Reg idx;
885 int32_t ofs; 937 int32_t ofs;
886 if (ir->op2 == IRFL_TAB_ARRAY) { 938 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
887 ofs = asm_fuseabase(as, ir->op1); 939 idx = RID_JGL;
888 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */ 940 ofs = (ir->op2 << 2) - 32768 - GG_OFS(g);
889 emit_tai(as, PPCI_ADDI, dest, idx, ofs); 941 } else {
890 return; 942 idx = ra_alloc1(as, ir->op1, RSET_GPR);
943 if (ir->op2 == IRFL_TAB_ARRAY) {
944 ofs = asm_fuseabase(as, ir->op1);
945 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
946 emit_tai(as, PPCI_ADDI, dest, idx, ofs);
947 return;
948 }
891 } 949 }
950 ofs = field_ofs[ir->op2];
892 } 951 }
893 ofs = field_ofs[ir->op2]; 952 lj_assertA(!irt_isi8(ir->t), "unsupported FLOAD I8");
894 lua_assert(!irt_isi8(ir->t));
895 emit_tai(as, pi, dest, idx, ofs); 953 emit_tai(as, pi, dest, idx, ofs);
896} 954}
897 955
@@ -902,21 +960,22 @@ static void asm_fstore(ASMState *as, IRIns *ir)
902 IRIns *irf = IR(ir->op1); 960 IRIns *irf = IR(ir->op1);
903 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src)); 961 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
904 int32_t ofs = field_ofs[irf->op2]; 962 int32_t ofs = field_ofs[irf->op2];
905 PPCIns pi = asm_fxstoreins(ir); 963 PPCIns pi = asm_fxstoreins(as, ir);
906 emit_tai(as, pi, src, idx, ofs); 964 emit_tai(as, pi, src, idx, ofs);
907 } 965 }
908} 966}
909 967
910static void asm_xload(ASMState *as, IRIns *ir) 968static void asm_xload(ASMState *as, IRIns *ir)
911{ 969{
912 Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); 970 Reg dest = ra_dest(as, ir,
913 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED)); 971 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
972 lj_assertA(!(ir->op2 & IRXLOAD_UNALIGNED), "unaligned XLOAD");
914 if (irt_isi8(ir->t)) 973 if (irt_isi8(ir->t))
915 emit_as(as, PPCI_EXTSB, dest, dest); 974 emit_as(as, PPCI_EXTSB, dest, dest);
916 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0); 975 asm_fusexref(as, asm_fxloadins(as, ir), dest, ir->op1, RSET_GPR, 0);
917} 976}
918 977
919static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs) 978static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
920{ 979{
921 IRIns *irb; 980 IRIns *irb;
922 if (ir->r == RID_SINK) 981 if (ir->r == RID_SINK)
@@ -927,36 +986,54 @@ static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
927 Reg src = ra_alloc1(as, irb->op1, RSET_GPR); 986 Reg src = ra_alloc1(as, irb->op1, RSET_GPR);
928 asm_fusexrefx(as, PPCI_STWBRX, src, ir->op1, rset_exclude(RSET_GPR, src)); 987 asm_fusexrefx(as, PPCI_STWBRX, src, ir->op1, rset_exclude(RSET_GPR, src));
929 } else { 988 } else {
930 Reg src = ra_alloc1(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); 989 Reg src = ra_alloc1(as, ir->op2,
931 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1, 990 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
991 asm_fusexref(as, asm_fxstoreins(as, ir), src, ir->op1,
932 rset_exclude(RSET_GPR, src), ofs); 992 rset_exclude(RSET_GPR, src), ofs);
933 } 993 }
934} 994}
935 995
996#define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
997
936static void asm_ahuvload(ASMState *as, IRIns *ir) 998static void asm_ahuvload(ASMState *as, IRIns *ir)
937{ 999{
938 IRType1 t = ir->t; 1000 IRType1 t = ir->t;
939 Reg dest = RID_NONE, type = RID_TMP, tmp = RID_TMP, idx; 1001 Reg dest = RID_NONE, type = RID_TMP, tmp = RID_TMP, idx;
940 RegSet allow = RSET_GPR; 1002 RegSet allow = RSET_GPR;
941 int32_t ofs = AHUREF_LSX; 1003 int32_t ofs = AHUREF_LSX;
1004 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP) {
1005 t.irt = IRT_NUM;
1006 if (ra_used(ir+1)) {
1007 type = ra_dest(as, ir+1, allow);
1008 rset_clear(allow, type);
1009 }
1010 ofs = 0;
1011 }
942 if (ra_used(ir)) { 1012 if (ra_used(ir)) {
943 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1013 lj_assertA((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
944 if (!irt_isnum(t)) ofs = 0; 1014 irt_isint(ir->t) || irt_isaddr(ir->t),
945 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR); 1015 "bad load type %d", irt_type(ir->t));
1016 if (LJ_SOFTFP || !irt_isnum(t)) ofs = 0;
1017 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
946 rset_clear(allow, dest); 1018 rset_clear(allow, dest);
947 } 1019 }
948 idx = asm_fuseahuref(as, ir->op1, &ofs, allow); 1020 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1021 if (ir->o == IR_VLOAD) {
1022 ofs = ofs != AHUREF_LSX ? ofs + 8 * ir->op2 :
1023 ir->op2 ? 8 * ir->op2 : AHUREF_LSX;
1024 }
949 if (irt_isnum(t)) { 1025 if (irt_isnum(t)) {
950 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, rset_exclude(allow, idx)); 1026 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, rset_exclude(allow, idx));
951 asm_guardcc(as, CC_GE); 1027 asm_guardcc(as, CC_GE);
952 emit_ab(as, PPCI_CMPLW, type, tisnum); 1028 emit_ab(as, PPCI_CMPLW, type, tisnum);
953 if (ra_hasreg(dest)) { 1029 if (ra_hasreg(dest)) {
954 if (ofs == AHUREF_LSX) { 1030 if (!LJ_SOFTFP && ofs == AHUREF_LSX) {
955 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, 1031 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR,
956 (idx&255)), (idx>>8))); 1032 (idx&255)), (idx>>8)));
957 emit_fab(as, PPCI_LFDX, dest, (idx&255), tmp); 1033 emit_fab(as, PPCI_LFDX, dest, (idx&255), tmp);
958 } else { 1034 } else {
959 emit_fai(as, PPCI_LFD, dest, idx, ofs); 1035 emit_fai(as, LJ_SOFTFP ? PPCI_LWZ : PPCI_LFD, dest, idx,
1036 ofs+4*LJ_SOFTFP);
960 } 1037 }
961 } 1038 }
962 } else { 1039 } else {
@@ -979,7 +1056,7 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
979 int32_t ofs = AHUREF_LSX; 1056 int32_t ofs = AHUREF_LSX;
980 if (ir->r == RID_SINK) 1057 if (ir->r == RID_SINK)
981 return; 1058 return;
982 if (irt_isnum(ir->t)) { 1059 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
983 src = ra_alloc1(as, ir->op2, RSET_FPR); 1060 src = ra_alloc1(as, ir->op2, RSET_FPR);
984 } else { 1061 } else {
985 if (!irt_ispri(ir->t)) { 1062 if (!irt_ispri(ir->t)) {
@@ -987,11 +1064,14 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
987 rset_clear(allow, src); 1064 rset_clear(allow, src);
988 ofs = 0; 1065 ofs = 0;
989 } 1066 }
990 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow); 1067 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
1068 type = ra_alloc1(as, (ir+1)->op2, allow);
1069 else
1070 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
991 rset_clear(allow, type); 1071 rset_clear(allow, type);
992 } 1072 }
993 idx = asm_fuseahuref(as, ir->op1, &ofs, allow); 1073 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
994 if (irt_isnum(ir->t)) { 1074 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
995 if (ofs == AHUREF_LSX) { 1075 if (ofs == AHUREF_LSX) {
996 emit_fab(as, PPCI_STFDX, src, (idx&255), RID_TMP); 1076 emit_fab(as, PPCI_STFDX, src, (idx&255), RID_TMP);
997 emit_slwi(as, RID_TMP, (idx>>8), 3); 1077 emit_slwi(as, RID_TMP, (idx>>8), 3);
@@ -1016,21 +1096,39 @@ static void asm_sload(ASMState *as, IRIns *ir)
1016 IRType1 t = ir->t; 1096 IRType1 t = ir->t;
1017 Reg dest = RID_NONE, type = RID_NONE, base; 1097 Reg dest = RID_NONE, type = RID_NONE, base;
1018 RegSet allow = RSET_GPR; 1098 RegSet allow = RSET_GPR;
1019 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ 1099 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
1020 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK)); 1100 if (hiop)
1021 lua_assert(LJ_DUALNUM || 1101 t.irt = IRT_NUM;
1022 !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME))); 1102 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1103 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1104 lj_assertA(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1105 "inconsistent SLOAD variant");
1106 lj_assertA(LJ_DUALNUM ||
1107 !irt_isint(t) ||
1108 (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME|IRSLOAD_KEYINDEX)),
1109 "bad SLOAD type");
1110#if LJ_SOFTFP
1111 lj_assertA(!(ir->op2 & IRSLOAD_CONVERT),
1112 "unsplit SLOAD convert"); /* Handled by LJ_SOFTFP SPLIT. */
1113 if (hiop && ra_used(ir+1)) {
1114 type = ra_dest(as, ir+1, allow);
1115 rset_clear(allow, type);
1116 }
1117#else
1023 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { 1118 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1024 dest = ra_scratch(as, RSET_FPR); 1119 dest = ra_scratch(as, RSET_FPR);
1025 asm_tointg(as, ir, dest); 1120 asm_tointg(as, ir, dest);
1026 t.irt = IRT_NUM; /* Continue with a regular number type check. */ 1121 t.irt = IRT_NUM; /* Continue with a regular number type check. */
1027 } else if (ra_used(ir)) { 1122 } else
1028 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1123#endif
1029 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR); 1124 if (ra_used(ir)) {
1125 lj_assertA(irt_isnum(t) || irt_isint(t) || irt_isaddr(t),
1126 "bad SLOAD type %d", irt_type(ir->t));
1127 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
1030 rset_clear(allow, dest); 1128 rset_clear(allow, dest);
1031 base = ra_alloc1(as, REF_BASE, allow); 1129 base = ra_alloc1(as, REF_BASE, allow);
1032 rset_clear(allow, base); 1130 rset_clear(allow, base);
1033 if ((ir->op2 & IRSLOAD_CONVERT)) { 1131 if (!LJ_SOFTFP && (ir->op2 & IRSLOAD_CONVERT)) {
1034 if (irt_isint(t)) { 1132 if (irt_isint(t)) {
1035 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO); 1133 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
1036 dest = ra_scratch(as, RSET_FPR); 1134 dest = ra_scratch(as, RSET_FPR);
@@ -1044,7 +1142,7 @@ static void asm_sload(ASMState *as, IRIns *ir)
1044 emit_fab(as, PPCI_FSUB, dest, dest, fbias); 1142 emit_fab(as, PPCI_FSUB, dest, dest, fbias);
1045 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP); 1143 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
1046 emit_lsptr(as, PPCI_LFS, (fbias & 31), 1144 emit_lsptr(as, PPCI_LFS, (fbias & 31),
1047 (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)), 1145 (void *)&as->J->k32[LJ_K32_2P52_2P31],
1048 rset_clear(allow, hibias)); 1146 rset_clear(allow, hibias));
1049 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPLO); 1147 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPLO);
1050 emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI); 1148 emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
@@ -1062,14 +1160,22 @@ dotypecheck:
1062 if ((ir->op2 & IRSLOAD_TYPECHECK)) { 1160 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1063 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow); 1161 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow);
1064 asm_guardcc(as, CC_GE); 1162 asm_guardcc(as, CC_GE);
1065 emit_ab(as, PPCI_CMPLW, RID_TMP, tisnum); 1163#if !LJ_SOFTFP
1066 type = RID_TMP; 1164 type = RID_TMP;
1165#endif
1166 emit_ab(as, PPCI_CMPLW, type, tisnum);
1067 } 1167 }
1068 if (ra_hasreg(dest)) emit_fai(as, PPCI_LFD, dest, base, ofs-4); 1168 if (ra_hasreg(dest)) emit_fai(as, LJ_SOFTFP ? PPCI_LWZ : PPCI_LFD, dest,
1169 base, ofs-(LJ_SOFTFP?0:4));
1069 } else { 1170 } else {
1070 if ((ir->op2 & IRSLOAD_TYPECHECK)) { 1171 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1071 asm_guardcc(as, CC_NE); 1172 asm_guardcc(as, CC_NE);
1072 emit_ai(as, PPCI_CMPWI, RID_TMP, irt_toitype(t)); 1173 if ((ir->op2 & IRSLOAD_KEYINDEX)) {
1174 emit_ai(as, PPCI_CMPWI, RID_TMP, (LJ_KEYINDEX & 0xffff));
1175 emit_asi(as, PPCI_XORIS, RID_TMP, RID_TMP, (LJ_KEYINDEX >> 16));
1176 } else {
1177 emit_ai(as, PPCI_CMPWI, RID_TMP, irt_toitype(t));
1178 }
1073 type = RID_TMP; 1179 type = RID_TMP;
1074 } 1180 }
1075 if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, base, ofs); 1181 if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, base, ofs);
@@ -1083,19 +1189,16 @@ dotypecheck:
1083static void asm_cnew(ASMState *as, IRIns *ir) 1189static void asm_cnew(ASMState *as, IRIns *ir)
1084{ 1190{
1085 CTState *cts = ctype_ctsG(J2G(as->J)); 1191 CTState *cts = ctype_ctsG(J2G(as->J));
1086 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i; 1192 CTypeID id = (CTypeID)IR(ir->op1)->i;
1087 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ? 1193 CTSize sz;
1088 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i; 1194 CTInfo info = lj_ctype_info(cts, id, &sz);
1089 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; 1195 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1090 IRRef args[2]; 1196 IRRef args[4];
1091 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1092 RegSet drop = RSET_SCRATCH; 1197 RegSet drop = RSET_SCRATCH;
1093 lua_assert(sz != CTSIZE_INVALID); 1198 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1199 "bad CNEW/CNEWI operands");
1094 1200
1095 args[0] = ASMREF_L; /* lua_State *L */
1096 args[1] = ASMREF_TMP1; /* MSize size */
1097 as->gcsteps++; 1201 as->gcsteps++;
1098
1099 if (ra_hasreg(ir->r)) 1202 if (ra_hasreg(ir->r))
1100 rset_clear(drop, ir->r); /* Dest reg handled below. */ 1203 rset_clear(drop, ir->r); /* Dest reg handled below. */
1101 ra_evictset(as, drop); 1204 ra_evictset(as, drop);
@@ -1104,11 +1207,12 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1104 1207
1105 /* Initialize immutable cdata object. */ 1208 /* Initialize immutable cdata object. */
1106 if (ir->o == IR_CNEWI) { 1209 if (ir->o == IR_CNEWI) {
1210 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1107 int32_t ofs = sizeof(GCcdata); 1211 int32_t ofs = sizeof(GCcdata);
1108 lua_assert(sz == 4 || sz == 8); 1212 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1109 if (sz == 8) { 1213 if (sz == 8) {
1110 ofs += 4; 1214 ofs += 4;
1111 lua_assert((ir+1)->o == IR_HIOP); 1215 lj_assertA((ir+1)->o == IR_HIOP, "expected HIOP for CNEWI");
1112 } 1216 }
1113 for (;;) { 1217 for (;;) {
1114 Reg r = ra_alloc1(as, ir->op2, allow); 1218 Reg r = ra_alloc1(as, ir->op2, allow);
@@ -1117,18 +1221,28 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1117 if (ofs == sizeof(GCcdata)) break; 1221 if (ofs == sizeof(GCcdata)) break;
1118 ofs -= 4; ir++; 1222 ofs -= 4; ir++;
1119 } 1223 }
1224 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1225 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1226 args[0] = ASMREF_L; /* lua_State *L */
1227 args[1] = ir->op1; /* CTypeID id */
1228 args[2] = ir->op2; /* CTSize sz */
1229 args[3] = ASMREF_TMP1; /* CTSize align */
1230 asm_gencall(as, ci, args);
1231 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1232 return;
1120 } 1233 }
1234
1121 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ 1235 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1122 emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct)); 1236 emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1123 emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid)); 1237 emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1124 emit_ti(as, PPCI_LI, RID_RET+1, ~LJ_TCDATA); 1238 emit_ti(as, PPCI_LI, RID_RET+1, ~LJ_TCDATA);
1125 emit_ti(as, PPCI_LI, RID_TMP, ctypeid); /* Lower 16 bit used. Sign-ext ok. */ 1239 emit_ti(as, PPCI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
1240 args[0] = ASMREF_L; /* lua_State *L */
1241 args[1] = ASMREF_TMP1; /* MSize size */
1126 asm_gencall(as, ci, args); 1242 asm_gencall(as, ci, args);
1127 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)), 1243 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1128 ra_releasetmp(as, ASMREF_TMP1)); 1244 ra_releasetmp(as, ASMREF_TMP1));
1129} 1245}
1130#else
1131#define asm_cnew(as, ir) ((void)0)
1132#endif 1246#endif
1133 1247
1134/* -- Write barriers ------------------------------------------------------ */ 1248/* -- Write barriers ------------------------------------------------------ */
@@ -1142,7 +1256,7 @@ static void asm_tbar(ASMState *as, IRIns *ir)
1142 emit_tai(as, PPCI_STW, link, tab, (int32_t)offsetof(GCtab, gclist)); 1256 emit_tai(as, PPCI_STW, link, tab, (int32_t)offsetof(GCtab, gclist));
1143 emit_tai(as, PPCI_STB, mark, tab, (int32_t)offsetof(GCtab, marked)); 1257 emit_tai(as, PPCI_STB, mark, tab, (int32_t)offsetof(GCtab, marked));
1144 emit_setgl(as, tab, gc.grayagain); 1258 emit_setgl(as, tab, gc.grayagain);
1145 lua_assert(LJ_GC_BLACK == 0x04); 1259 lj_assertA(LJ_GC_BLACK == 0x04, "bad LJ_GC_BLACK");
1146 emit_rot(as, PPCI_RLWINM, mark, mark, 0, 30, 28); /* Clear black bit. */ 1260 emit_rot(as, PPCI_RLWINM, mark, mark, 0, 30, 28); /* Clear black bit. */
1147 emit_getgl(as, link, gc.grayagain); 1261 emit_getgl(as, link, gc.grayagain);
1148 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end); 1262 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
@@ -1157,7 +1271,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1157 MCLabel l_end; 1271 MCLabel l_end;
1158 Reg obj, val, tmp; 1272 Reg obj, val, tmp;
1159 /* No need for other object barriers (yet). */ 1273 /* No need for other object barriers (yet). */
1160 lua_assert(IR(ir->op1)->o == IR_UREFC); 1274 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1161 ra_evictset(as, RSET_SCRATCH); 1275 ra_evictset(as, RSET_SCRATCH);
1162 l_end = emit_label(as); 1276 l_end = emit_label(as);
1163 args[0] = ASMREF_TMP1; /* global_State *g */ 1277 args[0] = ASMREF_TMP1; /* global_State *g */
@@ -1178,6 +1292,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1178 1292
1179/* -- Arithmetic and logic operations ------------------------------------- */ 1293/* -- Arithmetic and logic operations ------------------------------------- */
1180 1294
1295#if !LJ_SOFTFP
1181static void asm_fparith(ASMState *as, IRIns *ir, PPCIns pi) 1296static void asm_fparith(ASMState *as, IRIns *ir, PPCIns pi)
1182{ 1297{
1183 Reg dest = ra_dest(as, ir, RSET_FPR); 1298 Reg dest = ra_dest(as, ir, RSET_FPR);
@@ -1196,31 +1311,24 @@ static void asm_fpunary(ASMState *as, IRIns *ir, PPCIns pi)
1196 emit_fb(as, pi, dest, left); 1311 emit_fb(as, pi, dest, left);
1197} 1312}
1198 1313
1199static int asm_fpjoin_pow(ASMState *as, IRIns *ir) 1314static void asm_fpmath(ASMState *as, IRIns *ir)
1200{ 1315{
1201 IRIns *irp = IR(ir->op1); 1316 if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
1202 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) { 1317 asm_fpunary(as, ir, PPCI_FSQRT);
1203 IRIns *irpp = IR(irp->op1); 1318 else
1204 if (irpp == ir-2 && irpp->o == IR_FPMATH && 1319 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1205 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
1206 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
1207 IRRef args[2];
1208 args[0] = irpp->op1;
1209 args[1] = irp->op2;
1210 asm_setupresult(as, ir, ci);
1211 asm_gencall(as, ci, args);
1212 return 1;
1213 }
1214 }
1215 return 0;
1216} 1320}
1321#endif
1217 1322
1218static void asm_add(ASMState *as, IRIns *ir) 1323static void asm_add(ASMState *as, IRIns *ir)
1219{ 1324{
1325#if !LJ_SOFTFP
1220 if (irt_isnum(ir->t)) { 1326 if (irt_isnum(ir->t)) {
1221 if (!asm_fusemadd(as, ir, PPCI_FMADD, PPCI_FMADD)) 1327 if (!asm_fusemadd(as, ir, PPCI_FMADD, PPCI_FMADD))
1222 asm_fparith(as, ir, PPCI_FADD); 1328 asm_fparith(as, ir, PPCI_FADD);
1223 } else { 1329 } else
1330#endif
1331 {
1224 Reg dest = ra_dest(as, ir, RSET_GPR); 1332 Reg dest = ra_dest(as, ir, RSET_GPR);
1225 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); 1333 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1226 PPCIns pi; 1334 PPCIns pi;
@@ -1259,10 +1367,13 @@ static void asm_add(ASMState *as, IRIns *ir)
1259 1367
1260static void asm_sub(ASMState *as, IRIns *ir) 1368static void asm_sub(ASMState *as, IRIns *ir)
1261{ 1369{
1370#if !LJ_SOFTFP
1262 if (irt_isnum(ir->t)) { 1371 if (irt_isnum(ir->t)) {
1263 if (!asm_fusemadd(as, ir, PPCI_FMSUB, PPCI_FNMSUB)) 1372 if (!asm_fusemadd(as, ir, PPCI_FMSUB, PPCI_FNMSUB))
1264 asm_fparith(as, ir, PPCI_FSUB); 1373 asm_fparith(as, ir, PPCI_FSUB);
1265 } else { 1374 } else
1375#endif
1376 {
1266 PPCIns pi = PPCI_SUBF; 1377 PPCIns pi = PPCI_SUBF;
1267 Reg dest = ra_dest(as, ir, RSET_GPR); 1378 Reg dest = ra_dest(as, ir, RSET_GPR);
1268 Reg left, right; 1379 Reg left, right;
@@ -1288,9 +1399,12 @@ static void asm_sub(ASMState *as, IRIns *ir)
1288 1399
1289static void asm_mul(ASMState *as, IRIns *ir) 1400static void asm_mul(ASMState *as, IRIns *ir)
1290{ 1401{
1402#if !LJ_SOFTFP
1291 if (irt_isnum(ir->t)) { 1403 if (irt_isnum(ir->t)) {
1292 asm_fparith(as, ir, PPCI_FMUL); 1404 asm_fparith(as, ir, PPCI_FMUL);
1293 } else { 1405 } else
1406#endif
1407 {
1294 PPCIns pi = PPCI_MULLW; 1408 PPCIns pi = PPCI_MULLW;
1295 Reg dest = ra_dest(as, ir, RSET_GPR); 1409 Reg dest = ra_dest(as, ir, RSET_GPR);
1296 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); 1410 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
@@ -1312,11 +1426,16 @@ static void asm_mul(ASMState *as, IRIns *ir)
1312 } 1426 }
1313} 1427}
1314 1428
1429#define asm_fpdiv(as, ir) asm_fparith(as, ir, PPCI_FDIV)
1430
1315static void asm_neg(ASMState *as, IRIns *ir) 1431static void asm_neg(ASMState *as, IRIns *ir)
1316{ 1432{
1433#if !LJ_SOFTFP
1317 if (irt_isnum(ir->t)) { 1434 if (irt_isnum(ir->t)) {
1318 asm_fpunary(as, ir, PPCI_FNEG); 1435 asm_fpunary(as, ir, PPCI_FNEG);
1319 } else { 1436 } else
1437#endif
1438 {
1320 Reg dest, left; 1439 Reg dest, left;
1321 PPCIns pi = PPCI_NEG; 1440 PPCIns pi = PPCI_NEG;
1322 if (as->flagmcp == as->mcp) { 1441 if (as->flagmcp == as->mcp) {
@@ -1330,6 +1449,8 @@ static void asm_neg(ASMState *as, IRIns *ir)
1330 } 1449 }
1331} 1450}
1332 1451
1452#define asm_abs(as, ir) asm_fpunary(as, ir, PPCI_FABS)
1453
1333static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi) 1454static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
1334{ 1455{
1335 Reg dest, left, right; 1456 Reg dest, left, right;
@@ -1345,6 +1466,10 @@ static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
1345 emit_tab(as, pi|PPCF_DOT, dest, left, right); 1466 emit_tab(as, pi|PPCF_DOT, dest, left, right);
1346} 1467}
1347 1468
1469#define asm_addov(as, ir) asm_arithov(as, ir, PPCI_ADDO)
1470#define asm_subov(as, ir) asm_arithov(as, ir, PPCI_SUBFO)
1471#define asm_mulov(as, ir) asm_arithov(as, ir, PPCI_MULLWO)
1472
1348#if LJ_HASFFI 1473#if LJ_HASFFI
1349static void asm_add64(ASMState *as, IRIns *ir) 1474static void asm_add64(ASMState *as, IRIns *ir)
1350{ 1475{
@@ -1424,7 +1549,7 @@ static void asm_neg64(ASMState *as, IRIns *ir)
1424} 1549}
1425#endif 1550#endif
1426 1551
1427static void asm_bitnot(ASMState *as, IRIns *ir) 1552static void asm_bnot(ASMState *as, IRIns *ir)
1428{ 1553{
1429 Reg dest, left, right; 1554 Reg dest, left, right;
1430 PPCIns pi = PPCI_NOR; 1555 PPCIns pi = PPCI_NOR;
@@ -1451,7 +1576,7 @@ nofuse:
1451 emit_asb(as, pi, dest, left, right); 1576 emit_asb(as, pi, dest, left, right);
1452} 1577}
1453 1578
1454static void asm_bitswap(ASMState *as, IRIns *ir) 1579static void asm_bswap(ASMState *as, IRIns *ir)
1455{ 1580{
1456 Reg dest = ra_dest(as, ir, RSET_GPR); 1581 Reg dest = ra_dest(as, ir, RSET_GPR);
1457 IRIns *irx; 1582 IRIns *irx;
@@ -1472,32 +1597,6 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
1472 } 1597 }
1473} 1598}
1474 1599
1475static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1476{
1477 Reg dest = ra_dest(as, ir, RSET_GPR);
1478 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1479 if (irref_isk(ir->op2)) {
1480 int32_t k = IR(ir->op2)->i;
1481 Reg tmp = left;
1482 if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
1483 if (!checku16(k)) {
1484 emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
1485 if ((k & 0xffff) == 0) return;
1486 }
1487 emit_asi(as, pik, dest, left, k);
1488 return;
1489 }
1490 }
1491 /* May fail due to spills/restores above, but simplifies the logic. */
1492 if (as->flagmcp == as->mcp) {
1493 as->flagmcp = NULL;
1494 as->mcp++;
1495 pi |= PPCF_DOT;
1496 }
1497 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1498 emit_asb(as, pi, dest, left, right);
1499}
1500
1501/* Fuse BAND with contiguous bitmask and a shift to rlwinm. */ 1600/* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1502static void asm_fuseandsh(ASMState *as, PPCIns pi, int32_t mask, IRRef ref) 1601static void asm_fuseandsh(ASMState *as, PPCIns pi, int32_t mask, IRRef ref)
1503{ 1602{
@@ -1528,7 +1627,7 @@ nofuse:
1528 *--as->mcp = pi | PPCF_T(left); 1627 *--as->mcp = pi | PPCF_T(left);
1529} 1628}
1530 1629
1531static void asm_bitand(ASMState *as, IRIns *ir) 1630static void asm_band(ASMState *as, IRIns *ir)
1532{ 1631{
1533 Reg dest, left, right; 1632 Reg dest, left, right;
1534 IRRef lref = ir->op1; 1633 IRRef lref = ir->op1;
@@ -1583,6 +1682,35 @@ static void asm_bitand(ASMState *as, IRIns *ir)
1583 emit_asb(as, PPCI_AND ^ dot, dest, left, right); 1682 emit_asb(as, PPCI_AND ^ dot, dest, left, right);
1584} 1683}
1585 1684
1685static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1686{
1687 Reg dest = ra_dest(as, ir, RSET_GPR);
1688 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1689 if (irref_isk(ir->op2)) {
1690 int32_t k = IR(ir->op2)->i;
1691 Reg tmp = left;
1692 if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
1693 if (!checku16(k)) {
1694 emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
1695 if ((k & 0xffff) == 0) return;
1696 }
1697 emit_asi(as, pik, dest, left, k);
1698 return;
1699 }
1700 }
1701 /* May fail due to spills/restores above, but simplifies the logic. */
1702 if (as->flagmcp == as->mcp) {
1703 as->flagmcp = NULL;
1704 as->mcp++;
1705 pi |= PPCF_DOT;
1706 }
1707 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1708 emit_asb(as, pi, dest, left, right);
1709}
1710
1711#define asm_bor(as, ir) asm_bitop(as, ir, PPCI_OR, PPCI_ORI)
1712#define asm_bxor(as, ir) asm_bitop(as, ir, PPCI_XOR, PPCI_XORI)
1713
1586static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik) 1714static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1587{ 1715{
1588 Reg dest, left; 1716 Reg dest, left;
@@ -1608,9 +1736,48 @@ static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1608 } 1736 }
1609} 1737}
1610 1738
1739#define asm_bshl(as, ir) asm_bitshift(as, ir, PPCI_SLW, 0)
1740#define asm_bshr(as, ir) asm_bitshift(as, ir, PPCI_SRW, 1)
1741#define asm_bsar(as, ir) asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI)
1742#define asm_brol(as, ir) \
1743 asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31), \
1744 PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31))
1745#define asm_bror(as, ir) lj_assertA(0, "unexpected BROR")
1746
1747#if LJ_SOFTFP
1748static void asm_sfpmin_max(ASMState *as, IRIns *ir)
1749{
1750 CCallInfo ci = lj_ir_callinfo[IRCALL_softfp_cmp];
1751 IRRef args[4];
1752 MCLabel l_right, l_end;
1753 Reg desthi = ra_dest(as, ir, RSET_GPR), destlo = ra_dest(as, ir+1, RSET_GPR);
1754 Reg righthi, lefthi = ra_alloc2(as, ir, RSET_GPR);
1755 Reg rightlo, leftlo = ra_alloc2(as, ir+1, RSET_GPR);
1756 PPCCC cond = (IROp)ir->o == IR_MIN ? CC_EQ : CC_NE;
1757 righthi = (lefthi >> 8); lefthi &= 255;
1758 rightlo = (leftlo >> 8); leftlo &= 255;
1759 args[0^LJ_BE] = ir->op1; args[1^LJ_BE] = (ir+1)->op1;
1760 args[2^LJ_BE] = ir->op2; args[3^LJ_BE] = (ir+1)->op2;
1761 l_end = emit_label(as);
1762 if (desthi != righthi) emit_mr(as, desthi, righthi);
1763 if (destlo != rightlo) emit_mr(as, destlo, rightlo);
1764 l_right = emit_label(as);
1765 if (l_end != l_right) emit_jmp(as, l_end);
1766 if (desthi != lefthi) emit_mr(as, desthi, lefthi);
1767 if (destlo != leftlo) emit_mr(as, destlo, leftlo);
1768 if (l_right == as->mcp+1) {
1769 cond ^= 4; l_right = l_end; ++as->mcp;
1770 }
1771 emit_condbranch(as, PPCI_BC, cond, l_right);
1772 ra_evictset(as, RSET_SCRATCH);
1773 emit_cmpi(as, RID_RET, 1);
1774 asm_gencall(as, &ci, args);
1775}
1776#endif
1777
1611static void asm_min_max(ASMState *as, IRIns *ir, int ismax) 1778static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1612{ 1779{
1613 if (irt_isnum(ir->t)) { 1780 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1614 Reg dest = ra_dest(as, ir, RSET_FPR); 1781 Reg dest = ra_dest(as, ir, RSET_FPR);
1615 Reg tmp = dest; 1782 Reg tmp = dest;
1616 Reg right, left = ra_alloc2(as, ir, RSET_FPR); 1783 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
@@ -1618,9 +1785,8 @@ static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1618 if (tmp == left || tmp == right) 1785 if (tmp == left || tmp == right)
1619 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_FPR, 1786 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_FPR,
1620 dest), left), right)); 1787 dest), left), right));
1621 emit_facb(as, PPCI_FSEL, dest, tmp, 1788 emit_facb(as, PPCI_FSEL, dest, tmp, left, right);
1622 ismax ? left : right, ismax ? right : left); 1789 emit_fab(as, PPCI_FSUB, tmp, ismax ? left : right, ismax ? right : left);
1623 emit_fab(as, PPCI_FSUB, tmp, left, right);
1624 } else { 1790 } else {
1625 Reg dest = ra_dest(as, ir, RSET_GPR); 1791 Reg dest = ra_dest(as, ir, RSET_GPR);
1626 Reg tmp1 = RID_TMP, tmp2 = dest; 1792 Reg tmp1 = RID_TMP, tmp2 = dest;
@@ -1638,6 +1804,9 @@ static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1638 } 1804 }
1639} 1805}
1640 1806
1807#define asm_min(as, ir) asm_min_max(as, ir, 0)
1808#define asm_max(as, ir) asm_min_max(as, ir, 1)
1809
1641/* -- Comparisons --------------------------------------------------------- */ 1810/* -- Comparisons --------------------------------------------------------- */
1642 1811
1643#define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */ 1812#define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
@@ -1695,7 +1864,7 @@ static void asm_intcomp_(ASMState *as, IRRef lref, IRRef rref, Reg cr, PPCCC cc)
1695static void asm_comp(ASMState *as, IRIns *ir) 1864static void asm_comp(ASMState *as, IRIns *ir)
1696{ 1865{
1697 PPCCC cc = asm_compmap[ir->o]; 1866 PPCCC cc = asm_compmap[ir->o];
1698 if (irt_isnum(ir->t)) { 1867 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1699 Reg right, left = ra_alloc2(as, ir, RSET_FPR); 1868 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1700 right = (left >> 8); left &= 255; 1869 right = (left >> 8); left &= 255;
1701 asm_guardcc(as, (cc >> 4)); 1870 asm_guardcc(as, (cc >> 4));
@@ -1714,6 +1883,46 @@ static void asm_comp(ASMState *as, IRIns *ir)
1714 } 1883 }
1715} 1884}
1716 1885
1886#define asm_equal(as, ir) asm_comp(as, ir)
1887
1888#if LJ_SOFTFP
1889/* SFP comparisons. */
1890static void asm_sfpcomp(ASMState *as, IRIns *ir)
1891{
1892 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp];
1893 RegSet drop = RSET_SCRATCH;
1894 Reg r;
1895 IRRef args[4];
1896 args[0^LJ_BE] = ir->op1; args[1^LJ_BE] = (ir+1)->op1;
1897 args[2^LJ_BE] = ir->op2; args[3^LJ_BE] = (ir+1)->op2;
1898
1899 for (r = REGARG_FIRSTGPR; r <= REGARG_FIRSTGPR+3; r++) {
1900 if (!rset_test(as->freeset, r) &&
1901 regcost_ref(as->cost[r]) == args[r-REGARG_FIRSTGPR])
1902 rset_clear(drop, r);
1903 }
1904 ra_evictset(as, drop);
1905 asm_setupresult(as, ir, ci);
1906 switch ((IROp)ir->o) {
1907 case IR_ULT:
1908 asm_guardcc(as, CC_EQ);
1909 emit_ai(as, PPCI_CMPWI, RID_RET, 0);
1910 case IR_ULE:
1911 asm_guardcc(as, CC_EQ);
1912 emit_ai(as, PPCI_CMPWI, RID_RET, 1);
1913 break;
1914 case IR_GE: case IR_GT:
1915 asm_guardcc(as, CC_EQ);
1916 emit_ai(as, PPCI_CMPWI, RID_RET, 2);
1917 default:
1918 asm_guardcc(as, (asm_compmap[ir->o] & 0xf));
1919 emit_ai(as, PPCI_CMPWI, RID_RET, 0);
1920 break;
1921 }
1922 asm_gencall(as, ci, args);
1923}
1924#endif
1925
1717#if LJ_HASFFI 1926#if LJ_HASFFI
1718/* 64 bit integer comparisons. */ 1927/* 64 bit integer comparisons. */
1719static void asm_comp64(ASMState *as, IRIns *ir) 1928static void asm_comp64(ASMState *as, IRIns *ir)
@@ -1738,50 +1947,87 @@ static void asm_comp64(ASMState *as, IRIns *ir)
1738} 1947}
1739#endif 1948#endif
1740 1949
1741/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */ 1950/* -- Split register ops -------------------------------------------------- */
1742 1951
1743/* Hiword op of a split 64 bit op. Previous op must be the loword op. */ 1952/* Hiword op of a split 32/32 bit op. Previous op is be the loword op. */
1744static void asm_hiop(ASMState *as, IRIns *ir) 1953static void asm_hiop(ASMState *as, IRIns *ir)
1745{ 1954{
1746#if LJ_HASFFI
1747 /* HIOP is marked as a store because it needs its own DCE logic. */ 1955 /* HIOP is marked as a store because it needs its own DCE logic. */
1748 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */ 1956 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1749 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1; 1957 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1958#if LJ_HASFFI || LJ_SOFTFP
1750 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */ 1959 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
1751 as->curins--; /* Always skip the CONV. */ 1960 as->curins--; /* Always skip the CONV. */
1961#if LJ_HASFFI && !LJ_SOFTFP
1752 if (usehi || uselo) 1962 if (usehi || uselo)
1753 asm_conv64(as, ir); 1963 asm_conv64(as, ir);
1754 return; 1964 return;
1965#endif
1755 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */ 1966 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
1756 as->curins--; /* Always skip the loword comparison. */ 1967 as->curins--; /* Always skip the loword comparison. */
1968#if LJ_SOFTFP
1969 if (!irt_isint(ir->t)) {
1970 asm_sfpcomp(as, ir-1);
1971 return;
1972 }
1973#endif
1974#if LJ_HASFFI
1757 asm_comp64(as, ir); 1975 asm_comp64(as, ir);
1976#endif
1758 return; 1977 return;
1978#if LJ_SOFTFP
1979 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
1980 as->curins--; /* Always skip the loword min/max. */
1981 if (uselo || usehi)
1982 asm_sfpmin_max(as, ir-1);
1983 return;
1984#endif
1759 } else if ((ir-1)->o == IR_XSTORE) { 1985 } else if ((ir-1)->o == IR_XSTORE) {
1760 as->curins--; /* Handle both stores here. */ 1986 as->curins--; /* Handle both stores here. */
1761 if ((ir-1)->r != RID_SINK) { 1987 if ((ir-1)->r != RID_SINK) {
1762 asm_xstore(as, ir, 0); 1988 asm_xstore_(as, ir, 0);
1763 asm_xstore(as, ir-1, 4); 1989 asm_xstore_(as, ir-1, 4);
1764 } 1990 }
1765 return; 1991 return;
1766 } 1992 }
1993#endif
1767 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */ 1994 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1768 switch ((ir-1)->o) { 1995 switch ((ir-1)->o) {
1996#if LJ_HASFFI
1769 case IR_ADD: as->curins--; asm_add64(as, ir); break; 1997 case IR_ADD: as->curins--; asm_add64(as, ir); break;
1770 case IR_SUB: as->curins--; asm_sub64(as, ir); break; 1998 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1771 case IR_NEG: as->curins--; asm_neg64(as, ir); break; 1999 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1772 case IR_CALLN: 2000 case IR_CNEWI:
1773 case IR_CALLXS: 2001 /* Nothing to do here. Handled by lo op itself. */
2002 break;
2003#endif
2004#if LJ_SOFTFP
2005 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2006 case IR_STRTO:
1774 if (!uselo) 2007 if (!uselo)
1775 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */ 2008 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */
1776 break; 2009 break;
1777 case IR_CNEWI: 2010 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR: case IR_TMPREF:
1778 /* Nothing to do here. Handled by lo op itself. */ 2011 /* Nothing to do here. Handled by lo op itself. */
1779 break; 2012 break;
1780 default: lua_assert(0); break;
1781 }
1782#else
1783 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
1784#endif 2013#endif
2014 case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
2015 if (!uselo)
2016 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
2017 break;
2018 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
2019 }
2020}
2021
2022/* -- Profiling ----------------------------------------------------------- */
2023
2024static void asm_prof(ASMState *as, IRIns *ir)
2025{
2026 UNUSED(ir);
2027 asm_guardcc(as, CC_NE);
2028 emit_asi(as, PPCI_ANDIDOT, RID_TMP, RID_TMP, HOOK_PROFILE);
2029 emit_lsglptr(as, PPCI_LBZ, RID_TMP,
2030 (int32_t)offsetof(global_State, hookmask));
1785} 2031}
1786 2032
1787/* -- Stack handling ------------------------------------------------------ */ 2033/* -- Stack handling ------------------------------------------------------ */
@@ -1805,7 +2051,7 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
1805 emit_tai(as, PPCI_LWZ, tmp, tmp, offsetof(lua_State, maxstack)); 2051 emit_tai(as, PPCI_LWZ, tmp, tmp, offsetof(lua_State, maxstack));
1806 if (pbase == RID_TMP) 2052 if (pbase == RID_TMP)
1807 emit_getgl(as, RID_TMP, jit_base); 2053 emit_getgl(as, RID_TMP, jit_base);
1808 emit_getgl(as, tmp, jit_L); 2054 emit_getgl(as, tmp, cur_L);
1809 if (allow == RSET_EMPTY) /* Spill temp. register. */ 2055 if (allow == RSET_EMPTY) /* Spill temp. register. */
1810 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPW); 2056 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPW);
1811} 2057}
@@ -1826,12 +2072,25 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
1826 if ((sn & SNAP_NORESTORE)) 2072 if ((sn & SNAP_NORESTORE))
1827 continue; 2073 continue;
1828 if (irt_isnum(ir->t)) { 2074 if (irt_isnum(ir->t)) {
2075#if LJ_SOFTFP
2076 Reg tmp;
2077 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2078 /* LJ_SOFTFP: must be a number constant. */
2079 lj_assertA(irref_isk(ref), "unsplit FP op");
2080 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, allow);
2081 emit_tai(as, PPCI_STW, tmp, RID_BASE, ofs+(LJ_BE?4:0));
2082 if (rset_test(as->freeset, tmp+1)) allow = RID2RSET(tmp+1);
2083 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, allow);
2084 emit_tai(as, PPCI_STW, tmp, RID_BASE, ofs+(LJ_BE?0:4));
2085#else
1829 Reg src = ra_alloc1(as, ref, RSET_FPR); 2086 Reg src = ra_alloc1(as, ref, RSET_FPR);
1830 emit_fai(as, PPCI_STFD, src, RID_BASE, ofs); 2087 emit_fai(as, PPCI_STFD, src, RID_BASE, ofs);
2088#endif
1831 } else { 2089 } else {
1832 Reg type; 2090 Reg type;
1833 RegSet allow = rset_exclude(RSET_GPR, RID_BASE); 2091 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1834 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t)); 2092 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
2093 "restore of IR type %d", irt_type(ir->t));
1835 if (!irt_ispri(ir->t)) { 2094 if (!irt_ispri(ir->t)) {
1836 Reg src = ra_alloc1(as, ref, allow); 2095 Reg src = ra_alloc1(as, ref, allow);
1837 rset_clear(allow, src); 2096 rset_clear(allow, src);
@@ -1840,6 +2099,12 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
1840 if ((sn & (SNAP_CONT|SNAP_FRAME))) { 2099 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1841 if (s == 0) continue; /* Do not overwrite link to previous frame. */ 2100 if (s == 0) continue; /* Do not overwrite link to previous frame. */
1842 type = ra_allock(as, (int32_t)(*flinks--), allow); 2101 type = ra_allock(as, (int32_t)(*flinks--), allow);
2102#if LJ_SOFTFP
2103 } else if ((sn & SNAP_SOFTFPNUM)) {
2104 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPR, RID_BASE));
2105#endif
2106 } else if ((sn & SNAP_KEYINDEX)) {
2107 type = ra_allock(as, (int32_t)LJ_KEYINDEX, allow);
1843 } else { 2108 } else {
1844 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow); 2109 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1845 } 2110 }
@@ -1847,7 +2112,7 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
1847 } 2112 }
1848 checkmclim(as); 2113 checkmclim(as);
1849 } 2114 }
1850 lua_assert(map + nent == flinks); 2115 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
1851} 2116}
1852 2117
1853/* -- GC handling --------------------------------------------------------- */ 2118/* -- GC handling --------------------------------------------------------- */
@@ -1898,6 +2163,12 @@ static void asm_loop_fixup(ASMState *as)
1898 } 2163 }
1899} 2164}
1900 2165
2166/* Fixup the tail of the loop. */
2167static void asm_loop_tail_fixup(ASMState *as)
2168{
2169 UNUSED(as); /* Nothing to do. */
2170}
2171
1901/* -- Head of trace ------------------------------------------------------- */ 2172/* -- Head of trace ------------------------------------------------------- */
1902 2173
1903/* Coalesce BASE register for a root trace. */ 2174/* Coalesce BASE register for a root trace. */
@@ -1949,7 +2220,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1949 as->mctop = p; 2220 as->mctop = p;
1950 } else { 2221 } else {
1951 /* Patch stack adjustment. */ 2222 /* Patch stack adjustment. */
1952 lua_assert(checki16(CFRAME_SIZE+spadj)); 2223 lj_assertA(checki16(CFRAME_SIZE+spadj), "stack adjustment out of range");
1953 p[-3] = PPCI_ADDI | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | (CFRAME_SIZE+spadj); 2224 p[-3] = PPCI_ADDI | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | (CFRAME_SIZE+spadj);
1954 p[-2] = PPCI_STWU | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | spadj; 2225 p[-2] = PPCI_STWU | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | spadj;
1955 } 2226 }
@@ -1970,147 +2241,25 @@ static void asm_tail_prep(ASMState *as)
1970 } 2241 }
1971} 2242}
1972 2243
1973/* -- Instruction dispatch ------------------------------------------------ */
1974
1975/* Assemble a single instruction. */
1976static void asm_ir(ASMState *as, IRIns *ir)
1977{
1978 switch ((IROp)ir->o) {
1979 /* Miscellaneous ops. */
1980 case IR_LOOP: asm_loop(as); break;
1981 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
1982 case IR_USE:
1983 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1984 case IR_PHI: asm_phi(as, ir); break;
1985 case IR_HIOP: asm_hiop(as, ir); break;
1986 case IR_GCSTEP: asm_gcstep(as, ir); break;
1987
1988 /* Guarded assertions. */
1989 case IR_EQ: case IR_NE:
1990 if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
1991 as->curins--;
1992 asm_href(as, ir-1, (IROp)ir->o);
1993 break;
1994 }
1995 /* fallthrough */
1996 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1997 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1998 case IR_ABC:
1999 asm_comp(as, ir);
2000 break;
2001
2002 case IR_RETF: asm_retf(as, ir); break;
2003
2004 /* Bit ops. */
2005 case IR_BNOT: asm_bitnot(as, ir); break;
2006 case IR_BSWAP: asm_bitswap(as, ir); break;
2007
2008 case IR_BAND: asm_bitand(as, ir); break;
2009 case IR_BOR: asm_bitop(as, ir, PPCI_OR, PPCI_ORI); break;
2010 case IR_BXOR: asm_bitop(as, ir, PPCI_XOR, PPCI_XORI); break;
2011
2012 case IR_BSHL: asm_bitshift(as, ir, PPCI_SLW, 0); break;
2013 case IR_BSHR: asm_bitshift(as, ir, PPCI_SRW, 1); break;
2014 case IR_BSAR: asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI); break;
2015 case IR_BROL: asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31),
2016 PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31)); break;
2017 case IR_BROR: lua_assert(0); break;
2018
2019 /* Arithmetic ops. */
2020 case IR_ADD: asm_add(as, ir); break;
2021 case IR_SUB: asm_sub(as, ir); break;
2022 case IR_MUL: asm_mul(as, ir); break;
2023 case IR_DIV: asm_fparith(as, ir, PPCI_FDIV); break;
2024 case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
2025 case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
2026 case IR_NEG: asm_neg(as, ir); break;
2027
2028 case IR_ABS: asm_fpunary(as, ir, PPCI_FABS); break;
2029 case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
2030 case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
2031 case IR_MIN: asm_min_max(as, ir, 0); break;
2032 case IR_MAX: asm_min_max(as, ir, 1); break;
2033 case IR_FPMATH:
2034 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
2035 break;
2036 if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
2037 asm_fpunary(as, ir, PPCI_FSQRT);
2038 else
2039 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
2040 break;
2041
2042 /* Overflow-checking arithmetic ops. */
2043 case IR_ADDOV: asm_arithov(as, ir, PPCI_ADDO); break;
2044 case IR_SUBOV: asm_arithov(as, ir, PPCI_SUBFO); break;
2045 case IR_MULOV: asm_arithov(as, ir, PPCI_MULLWO); break;
2046
2047 /* Memory references. */
2048 case IR_AREF: asm_aref(as, ir); break;
2049 case IR_HREF: asm_href(as, ir, 0); break;
2050 case IR_HREFK: asm_hrefk(as, ir); break;
2051 case IR_NEWREF: asm_newref(as, ir); break;
2052 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
2053 case IR_FREF: asm_fref(as, ir); break;
2054 case IR_STRREF: asm_strref(as, ir); break;
2055
2056 /* Loads and stores. */
2057 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2058 asm_ahuvload(as, ir);
2059 break;
2060 case IR_FLOAD: asm_fload(as, ir); break;
2061 case IR_XLOAD: asm_xload(as, ir); break;
2062 case IR_SLOAD: asm_sload(as, ir); break;
2063
2064 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
2065 case IR_FSTORE: asm_fstore(as, ir); break;
2066 case IR_XSTORE: asm_xstore(as, ir, 0); break;
2067
2068 /* Allocations. */
2069 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
2070 case IR_TNEW: asm_tnew(as, ir); break;
2071 case IR_TDUP: asm_tdup(as, ir); break;
2072 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
2073
2074 /* Write barriers. */
2075 case IR_TBAR: asm_tbar(as, ir); break;
2076 case IR_OBAR: asm_obar(as, ir); break;
2077
2078 /* Type conversions. */
2079 case IR_CONV: asm_conv(as, ir); break;
2080 case IR_TOBIT: asm_tobit(as, ir); break;
2081 case IR_TOSTR: asm_tostr(as, ir); break;
2082 case IR_STRTO: asm_strto(as, ir); break;
2083
2084 /* Calls. */
2085 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
2086 case IR_CALLXS: asm_callx(as, ir); break;
2087 case IR_CARG: break;
2088
2089 default:
2090 setintV(&as->J->errinfo, ir->o);
2091 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
2092 break;
2093 }
2094}
2095
2096/* -- Trace setup --------------------------------------------------------- */ 2244/* -- Trace setup --------------------------------------------------------- */
2097 2245
2098/* Ensure there are enough stack slots for call arguments. */ 2246/* Ensure there are enough stack slots for call arguments. */
2099static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci) 2247static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2100{ 2248{
2101 IRRef args[CCI_NARGS_MAX*2]; 2249 IRRef args[CCI_NARGS_MAX*2];
2102 uint32_t i, nargs = (int)CCI_NARGS(ci); 2250 uint32_t i, nargs = CCI_XNARGS(ci);
2103 int nslots = 2, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR; 2251 int nslots = 2, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2104 asm_collectargs(as, ir, ci, args); 2252 asm_collectargs(as, ir, ci, args);
2105 for (i = 0; i < nargs; i++) 2253 for (i = 0; i < nargs; i++)
2106 if (args[i] && irt_isfp(IR(args[i])->t)) { 2254 if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t)) {
2107 if (nfpr > 0) nfpr--; else nslots = (nslots+3) & ~1; 2255 if (nfpr > 0) nfpr--; else nslots = (nslots+3) & ~1;
2108 } else { 2256 } else {
2109 if (ngpr > 0) ngpr--; else nslots++; 2257 if (ngpr > 0) ngpr--; else nslots++;
2110 } 2258 }
2111 if (nslots > as->evenspill) /* Leave room for args in stack slots. */ 2259 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
2112 as->evenspill = nslots; 2260 as->evenspill = nslots;
2113 return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET); 2261 return (!LJ_SOFTFP && irt_isfp(ir->t)) ? REGSP_HINT(RID_FPRET) :
2262 REGSP_HINT(RID_RET);
2114} 2263}
2115 2264
2116static void asm_setup_target(ASMState *as) 2265static void asm_setup_target(ASMState *as)
@@ -2150,7 +2299,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2150 } else if ((ins & 0xfc000000u) == PPCI_B && 2299 } else if ((ins & 0xfc000000u) == PPCI_B &&
2151 ((ins ^ ((char *)px-(char *)p)) & 0x03ffffffu) == 0) { 2300 ((ins ^ ((char *)px-(char *)p)) & 0x03ffffffu) == 0) {
2152 ptrdiff_t delta = (char *)target - (char *)p; 2301 ptrdiff_t delta = (char *)target - (char *)p;
2153 lua_assert(((delta + 0x02000000) >> 26) == 0); 2302 lj_assertJ(((delta + 0x02000000) >> 26) == 0,
2303 "branch target out of range");
2154 *p = PPCI_B | ((uint32_t)delta & 0x03ffffffu); 2304 *p = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2155 if (!cstart) cstart = p; 2305 if (!cstart) cstart = p;
2156 } 2306 }
@@ -2158,7 +2308,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2158 /* Always patch long-range branch in exit stub itself. Except, if we can't. */ 2308 /* Always patch long-range branch in exit stub itself. Except, if we can't. */
2159 if (patchlong) { 2309 if (patchlong) {
2160 ptrdiff_t delta = (char *)target - (char *)px - clearso; 2310 ptrdiff_t delta = (char *)target - (char *)px - clearso;
2161 lua_assert(((delta + 0x02000000) >> 26) == 0); 2311 lj_assertJ(((delta + 0x02000000) >> 26) == 0,
2312 "branch target out of range");
2162 *px = PPCI_B | ((uint32_t)delta & 0x03ffffffu); 2313 *px = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2163 } 2314 }
2164 if (!cstart) cstart = px; 2315 if (!cstart) cstart = px;
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index e01def59..9f779bf5 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -21,15 +21,17 @@ static MCode *asm_exitstub_gen(ASMState *as, ExitNo group)
21 } 21 }
22 /* Push the high byte of the exitno for each exit stub group. */ 22 /* Push the high byte of the exitno for each exit stub group. */
23 *mxp++ = XI_PUSHi8; *mxp++ = (MCode)((group*EXITSTUBS_PER_GROUP)>>8); 23 *mxp++ = XI_PUSHi8; *mxp++ = (MCode)((group*EXITSTUBS_PER_GROUP)>>8);
24#if !LJ_GC64
24 /* Store DISPATCH at original stack slot 0. Account for the two push ops. */ 25 /* Store DISPATCH at original stack slot 0. Account for the two push ops. */
25 *mxp++ = XI_MOVmi; 26 *mxp++ = XI_MOVmi;
26 *mxp++ = MODRM(XM_OFS8, 0, RID_ESP); 27 *mxp++ = MODRM(XM_OFS8, 0, RID_ESP);
27 *mxp++ = MODRM(XM_SCALE1, RID_ESP, RID_ESP); 28 *mxp++ = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
28 *mxp++ = 2*sizeof(void *); 29 *mxp++ = 2*sizeof(void *);
29 *(int32_t *)mxp = ptr2addr(J2GG(as->J)->dispatch); mxp += 4; 30 *(int32_t *)mxp = ptr2addr(J2GG(as->J)->dispatch); mxp += 4;
31#endif
30 /* Jump to exit handler which fills in the ExitState. */ 32 /* Jump to exit handler which fills in the ExitState. */
31 *mxp++ = XI_JMP; mxp += 4; 33 *mxp++ = XI_JMP; mxp += 4;
32 *((int32_t *)(mxp-4)) = jmprel(mxp, (MCode *)(void *)lj_vm_exit_handler); 34 *((int32_t *)(mxp-4)) = jmprel(as->J, mxp, (MCode *)(void *)lj_vm_exit_handler);
33 /* Commit the code for this group (even if assembly fails later on). */ 35 /* Commit the code for this group (even if assembly fails later on). */
34 lj_mcode_commitbot(as->J, mxp); 36 lj_mcode_commitbot(as->J, mxp);
35 as->mcbot = mxp; 37 as->mcbot = mxp;
@@ -58,14 +60,18 @@ static void asm_guardcc(ASMState *as, int cc)
58 MCode *p = as->mcp; 60 MCode *p = as->mcp;
59 if (LJ_UNLIKELY(p == as->invmcp)) { 61 if (LJ_UNLIKELY(p == as->invmcp)) {
60 as->loopinv = 1; 62 as->loopinv = 1;
61 *(int32_t *)(p+1) = jmprel(p+5, target); 63 *(int32_t *)(p+1) = jmprel(as->J, p+5, target);
62 target = p; 64 target = p;
63 cc ^= 1; 65 cc ^= 1;
64 if (as->realign) { 66 if (as->realign) {
67 if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
68 as->mrm.ofs += 2; /* Fixup RIP offset for pending fused load. */
65 emit_sjcc(as, cc, target); 69 emit_sjcc(as, cc, target);
66 return; 70 return;
67 } 71 }
68 } 72 }
73 if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
74 as->mrm.ofs += 6; /* Fixup RIP offset for pending fused load. */
69 emit_jcc(as, cc, target); 75 emit_jcc(as, cc, target);
70} 76}
71 77
@@ -79,6 +85,15 @@ static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
79{ 85{
80 if (irref_isk(ref)) { 86 if (irref_isk(ref)) {
81 IRIns *ir = IR(ref); 87 IRIns *ir = IR(ref);
88#if LJ_GC64
89 if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
90 *k = ir->i;
91 return 1;
92 } else if (checki32((int64_t)ir_k64(ir)->u64)) {
93 *k = (int32_t)ir_k64(ir)->u64;
94 return 1;
95 }
96#else
82 if (ir->o != IR_KINT64) { 97 if (ir->o != IR_KINT64) {
83 *k = ir->i; 98 *k = ir->i;
84 return 1; 99 return 1;
@@ -86,6 +101,7 @@ static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
86 *k = (int32_t)ir_kint64(ir)->u64; 101 *k = (int32_t)ir_kint64(ir)->u64;
87 return 1; 102 return 1;
88 } 103 }
104#endif
89 } 105 }
90 return 0; 106 return 0;
91} 107}
@@ -115,7 +131,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref)
115 as->mrm.ofs = 0; 131 as->mrm.ofs = 0;
116 if (irb->o == IR_FLOAD) { 132 if (irb->o == IR_FLOAD) {
117 IRIns *ira = IR(irb->op1); 133 IRIns *ira = IR(irb->op1);
118 lua_assert(irb->op2 == IRFL_TAB_ARRAY); 134 lj_assertA(irb->op2 == IRFL_TAB_ARRAY, "expected FLOAD TAB_ARRAY");
119 /* We can avoid the FLOAD of t->array for colocated arrays. */ 135 /* We can avoid the FLOAD of t->array for colocated arrays. */
120 if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE && 136 if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE &&
121 !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) { 137 !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) {
@@ -134,7 +150,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref)
134static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow) 150static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow)
135{ 151{
136 IRIns *irx; 152 IRIns *irx;
137 lua_assert(ir->o == IR_AREF); 153 lj_assertA(ir->o == IR_AREF, "expected AREF");
138 as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow); 154 as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow);
139 irx = IR(ir->op2); 155 irx = IR(ir->op2);
140 if (irref_isk(ir->op2)) { 156 if (irref_isk(ir->op2)) {
@@ -185,14 +201,32 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
185 if (irref_isk(ir->op1)) { 201 if (irref_isk(ir->op1)) {
186 GCfunc *fn = ir_kfunc(IR(ir->op1)); 202 GCfunc *fn = ir_kfunc(IR(ir->op1));
187 GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv; 203 GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
204#if LJ_GC64
205 int64_t ofs = dispofs(as, &uv->tv);
206 if (checki32(ofs) && checki32(ofs+4)) {
207 as->mrm.ofs = (int32_t)ofs;
208 as->mrm.base = RID_DISPATCH;
209 as->mrm.idx = RID_NONE;
210 return;
211 }
212#else
188 as->mrm.ofs = ptr2addr(&uv->tv); 213 as->mrm.ofs = ptr2addr(&uv->tv);
189 as->mrm.base = as->mrm.idx = RID_NONE; 214 as->mrm.base = as->mrm.idx = RID_NONE;
190 return; 215 return;
216#endif
191 } 217 }
192 break; 218 break;
219 case IR_TMPREF:
220#if LJ_GC64
221 as->mrm.ofs = (int32_t)dispofs(as, &J2G(as->J)->tmptv);
222 as->mrm.base = RID_DISPATCH;
223 as->mrm.idx = RID_NONE;
224#else
225 as->mrm.ofs = igcptr(&J2G(as->J)->tmptv);
226 as->mrm.base = as->mrm.idx = RID_NONE;
227#endif
228 return;
193 default: 229 default:
194 lua_assert(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO ||
195 ir->o == IR_KKPTR);
196 break; 230 break;
197 } 231 }
198 } 232 }
@@ -204,26 +238,53 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
204/* Fuse FLOAD/FREF reference into memory operand. */ 238/* Fuse FLOAD/FREF reference into memory operand. */
205static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow) 239static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow)
206{ 240{
207 lua_assert(ir->o == IR_FLOAD || ir->o == IR_FREF); 241 lj_assertA(ir->o == IR_FLOAD || ir->o == IR_FREF,
208 as->mrm.ofs = field_ofs[ir->op2]; 242 "bad IR op %d", ir->o);
209 as->mrm.idx = RID_NONE; 243 as->mrm.idx = RID_NONE;
244 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
245#if LJ_GC64
246 as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch);
247 as->mrm.base = RID_DISPATCH;
248#else
249 as->mrm.ofs = (int32_t)(ir->op2 << 2) + ptr2addr(J2GG(as->J));
250 as->mrm.base = RID_NONE;
251#endif
252 return;
253 }
254 as->mrm.ofs = field_ofs[ir->op2];
210 if (irref_isk(ir->op1)) { 255 if (irref_isk(ir->op1)) {
211 as->mrm.ofs += IR(ir->op1)->i; 256 IRIns *op1 = IR(ir->op1);
257#if LJ_GC64
258 if (ir->op1 == REF_NIL) {
259 as->mrm.ofs -= GG_OFS(dispatch);
260 as->mrm.base = RID_DISPATCH;
261 return;
262 } else if (op1->o == IR_KPTR || op1->o == IR_KKPTR) {
263 intptr_t ofs = dispofs(as, ir_kptr(op1));
264 if (checki32(as->mrm.ofs + ofs)) {
265 as->mrm.ofs += (int32_t)ofs;
266 as->mrm.base = RID_DISPATCH;
267 return;
268 }
269 }
270#else
271 as->mrm.ofs += op1->i;
212 as->mrm.base = RID_NONE; 272 as->mrm.base = RID_NONE;
213 } else { 273 return;
214 as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow); 274#endif
215 } 275 }
276 as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow);
216} 277}
217 278
218/* Fuse string reference into memory operand. */ 279/* Fuse string reference into memory operand. */
219static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow) 280static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow)
220{ 281{
221 IRIns *irr; 282 IRIns *irr;
222 lua_assert(ir->o == IR_STRREF); 283 lj_assertA(ir->o == IR_STRREF, "bad IR op %d", ir->o);
223 as->mrm.base = as->mrm.idx = RID_NONE; 284 as->mrm.base = as->mrm.idx = RID_NONE;
224 as->mrm.scale = XM_SCALE1; 285 as->mrm.scale = XM_SCALE1;
225 as->mrm.ofs = sizeof(GCstr); 286 as->mrm.ofs = sizeof(GCstr);
226 if (irref_isk(ir->op1)) { 287 if (!LJ_GC64 && irref_isk(ir->op1)) {
227 as->mrm.ofs += IR(ir->op1)->i; 288 as->mrm.ofs += IR(ir->op1)->i;
228 } else { 289 } else {
229 Reg r = ra_alloc1(as, ir->op1, allow); 290 Reg r = ra_alloc1(as, ir->op1, allow);
@@ -255,10 +316,20 @@ static void asm_fusexref(ASMState *as, IRRef ref, RegSet allow)
255 IRIns *ir = IR(ref); 316 IRIns *ir = IR(ref);
256 as->mrm.idx = RID_NONE; 317 as->mrm.idx = RID_NONE;
257 if (ir->o == IR_KPTR || ir->o == IR_KKPTR) { 318 if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
319#if LJ_GC64
320 intptr_t ofs = dispofs(as, ir_kptr(ir));
321 if (checki32(ofs)) {
322 as->mrm.ofs = (int32_t)ofs;
323 as->mrm.base = RID_DISPATCH;
324 return;
325 }
326 } if (0) {
327#else
258 as->mrm.ofs = ir->i; 328 as->mrm.ofs = ir->i;
259 as->mrm.base = RID_NONE; 329 as->mrm.base = RID_NONE;
260 } else if (ir->o == IR_STRREF) { 330 } else if (ir->o == IR_STRREF) {
261 asm_fusestrref(as, ir, allow); 331 asm_fusestrref(as, ir, allow);
332#endif
262 } else { 333 } else {
263 as->mrm.ofs = 0; 334 as->mrm.ofs = 0;
264 if (canfuse(as, ir) && ir->o == IR_ADD && ra_noreg(ir->r)) { 335 if (canfuse(as, ir) && ir->o == IR_ADD && ra_noreg(ir->r)) {
@@ -301,7 +372,47 @@ static void asm_fusexref(ASMState *as, IRRef ref, RegSet allow)
301 } 372 }
302} 373}
303 374
304/* Fuse load into memory operand. */ 375/* Fuse load of 64 bit IR constant into memory operand. */
376static Reg asm_fuseloadk64(ASMState *as, IRIns *ir)
377{
378 const uint64_t *k = &ir_k64(ir)->u64;
379 if (!LJ_GC64 || checki32((intptr_t)k)) {
380 as->mrm.ofs = ptr2addr(k);
381 as->mrm.base = RID_NONE;
382#if LJ_GC64
383 } else if (checki32(dispofs(as, k))) {
384 as->mrm.ofs = (int32_t)dispofs(as, k);
385 as->mrm.base = RID_DISPATCH;
386 } else if (checki32(mcpofs(as, k)) && checki32(mcpofs(as, k+1)) &&
387 checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) {
388 as->mrm.ofs = (int32_t)mcpofs(as, k);
389 as->mrm.base = RID_RIP;
390 } else { /* Intern 64 bit constant at bottom of mcode. */
391 if (ir->i) {
392 lj_assertA(*k == *(uint64_t*)(as->mctop - ir->i),
393 "bad interned 64 bit constant");
394 } else {
395 while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3;
396 *(uint64_t*)as->mcbot = *k;
397 ir->i = (int32_t)(as->mctop - as->mcbot);
398 as->mcbot += 8;
399 as->mclim = as->mcbot + MCLIM_REDZONE;
400 lj_mcode_commitbot(as->J, as->mcbot);
401 }
402 as->mrm.ofs = (int32_t)mcpofs(as, as->mctop - ir->i);
403 as->mrm.base = RID_RIP;
404#endif
405 }
406 as->mrm.idx = RID_NONE;
407 return RID_MRM;
408}
409
410/* Fuse load into memory operand.
411**
412** Important caveat: this may emit RIP-relative loads! So don't place any
413** code emitters between this function and the use of its result.
414** The only permitted exception is asm_guardcc().
415*/
305static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow) 416static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
306{ 417{
307 IRIns *ir = IR(ref); 418 IRIns *ir = IR(ref);
@@ -319,27 +430,36 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
319 } 430 }
320 if (ir->o == IR_KNUM) { 431 if (ir->o == IR_KNUM) {
321 RegSet avail = as->freeset & ~as->modset & RSET_FPR; 432 RegSet avail = as->freeset & ~as->modset & RSET_FPR;
322 lua_assert(allow != RSET_EMPTY); 433 lj_assertA(allow != RSET_EMPTY, "no register allowed");
323 if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */ 434 if (!(avail & (avail-1))) /* Fuse if less than two regs available. */
324 as->mrm.ofs = ptr2addr(ir_knum(ir)); 435 return asm_fuseloadk64(as, ir);
325 as->mrm.base = as->mrm.idx = RID_NONE;
326 return RID_MRM;
327 }
328 } else if (ref == REF_BASE || ir->o == IR_KINT64) { 436 } else if (ref == REF_BASE || ir->o == IR_KINT64) {
329 RegSet avail = as->freeset & ~as->modset & RSET_GPR; 437 RegSet avail = as->freeset & ~as->modset & RSET_GPR;
330 lua_assert(allow != RSET_EMPTY); 438 lj_assertA(allow != RSET_EMPTY, "no register allowed");
331 if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */ 439 if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */
332 as->mrm.ofs = ptr2addr(ref == REF_BASE ? (void *)&J2G(as->J)->jit_base : (void *)ir_kint64(ir)); 440 if (ref == REF_BASE) {
333 as->mrm.base = as->mrm.idx = RID_NONE; 441#if LJ_GC64
334 return RID_MRM; 442 as->mrm.ofs = (int32_t)dispofs(as, &J2G(as->J)->jit_base);
443 as->mrm.base = RID_DISPATCH;
444#else
445 as->mrm.ofs = ptr2addr(&J2G(as->J)->jit_base);
446 as->mrm.base = RID_NONE;
447#endif
448 as->mrm.idx = RID_NONE;
449 return RID_MRM;
450 } else {
451 return asm_fuseloadk64(as, ir);
452 }
335 } 453 }
336 } else if (mayfuse(as, ref)) { 454 } else if (mayfuse(as, ref)) {
337 RegSet xallow = (allow & RSET_GPR) ? allow : RSET_GPR; 455 RegSet xallow = (allow & RSET_GPR) ? allow : RSET_GPR;
338 if (ir->o == IR_SLOAD) { 456 if (ir->o == IR_SLOAD) {
339 if (!(ir->op2 & (IRSLOAD_PARENT|IRSLOAD_CONVERT)) && 457 if (!(ir->op2 & (IRSLOAD_PARENT|IRSLOAD_CONVERT)) &&
340 noconflict(as, ref, IR_RETF, 0)) { 458 noconflict(as, ref, IR_RETF, 0) &&
459 !(LJ_GC64 && irt_isaddr(ir->t))) {
341 as->mrm.base = (uint8_t)ra_alloc1(as, REF_BASE, xallow); 460 as->mrm.base = (uint8_t)ra_alloc1(as, REF_BASE, xallow);
342 as->mrm.ofs = 8*((int32_t)ir->op1-1) + ((ir->op2&IRSLOAD_FRAME)?4:0); 461 as->mrm.ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
462 (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
343 as->mrm.idx = RID_NONE; 463 as->mrm.idx = RID_NONE;
344 return RID_MRM; 464 return RID_MRM;
345 } 465 }
@@ -351,7 +471,8 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
351 return RID_MRM; 471 return RID_MRM;
352 } 472 }
353 } else if (ir->o == IR_ALOAD || ir->o == IR_HLOAD || ir->o == IR_ULOAD) { 473 } else if (ir->o == IR_ALOAD || ir->o == IR_HLOAD || ir->o == IR_ULOAD) {
354 if (noconflict(as, ref, ir->o + IRDELTA_L2S, 0)) { 474 if (noconflict(as, ref, ir->o + IRDELTA_L2S, 0) &&
475 !(LJ_GC64 && irt_isaddr(ir->t))) {
355 asm_fuseahuref(as, ir->op1, xallow); 476 asm_fuseahuref(as, ir->op1, xallow);
356 return RID_MRM; 477 return RID_MRM;
357 } 478 }
@@ -364,11 +485,17 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
364 asm_fusexref(as, ir->op1, xallow); 485 asm_fusexref(as, ir->op1, xallow);
365 return RID_MRM; 486 return RID_MRM;
366 } 487 }
367 } else if (ir->o == IR_VLOAD) { 488 } else if (ir->o == IR_VLOAD && IR(ir->op1)->o == IR_AREF &&
489 !(LJ_GC64 && irt_isaddr(ir->t))) {
368 asm_fuseahuref(as, ir->op1, xallow); 490 asm_fuseahuref(as, ir->op1, xallow);
491 as->mrm.ofs += 8 * ir->op2;
369 return RID_MRM; 492 return RID_MRM;
370 } 493 }
371 } 494 }
495 if (ir->o == IR_FLOAD && ir->op1 == REF_NIL) {
496 asm_fusefref(as, ir, RSET_EMPTY);
497 return RID_MRM;
498 }
372 if (!(as->freeset & allow) && !emit_canremat(ref) && 499 if (!(as->freeset & allow) && !emit_canremat(ref) &&
373 (allow == RSET_EMPTY || ra_hasspill(ir->s) || iscrossref(as, ref))) 500 (allow == RSET_EMPTY || ra_hasspill(ir->s) || iscrossref(as, ref)))
374 goto fusespill; 501 goto fusespill;
@@ -392,7 +519,7 @@ static Reg asm_fuseloadm(ASMState *as, IRRef ref, RegSet allow, int is64)
392/* Count the required number of stack slots for a call. */ 519/* Count the required number of stack slots for a call. */
393static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args) 520static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args)
394{ 521{
395 uint32_t i, nargs = CCI_NARGS(ci); 522 uint32_t i, nargs = CCI_XNARGS(ci);
396 int nslots = 0; 523 int nslots = 0;
397#if LJ_64 524#if LJ_64
398 if (LJ_ABI_WIN) { 525 if (LJ_ABI_WIN) {
@@ -425,7 +552,7 @@ static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args)
425/* Generate a call to a C function. */ 552/* Generate a call to a C function. */
426static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) 553static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
427{ 554{
428 uint32_t n, nargs = CCI_NARGS(ci); 555 uint32_t n, nargs = CCI_XNARGS(ci);
429 int32_t ofs = STACKARG_OFS; 556 int32_t ofs = STACKARG_OFS;
430#if LJ_64 557#if LJ_64
431 uint32_t gprs = REGARG_GPRS; 558 uint32_t gprs = REGARG_GPRS;
@@ -485,13 +612,14 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
485 if (r) { /* Argument is in a register. */ 612 if (r) { /* Argument is in a register. */
486 if (r < RID_MAX_GPR && ref < ASMREF_TMP1) { 613 if (r < RID_MAX_GPR && ref < ASMREF_TMP1) {
487#if LJ_64 614#if LJ_64
488 if (ir->o == IR_KINT64) 615 if (LJ_GC64 ? !(ir->o == IR_KINT || ir->o == IR_KNULL) : ir->o == IR_KINT64)
489 emit_loadu64(as, r, ir_kint64(ir)->u64); 616 emit_loadu64(as, r, ir_k64(ir)->u64);
490 else 617 else
491#endif 618#endif
492 emit_loadi(as, r, ir->i); 619 emit_loadi(as, r, ir->i);
493 } else { 620 } else {
494 lua_assert(rset_test(as->freeset, r)); /* Must have been evicted. */ 621 /* Must have been evicted. */
622 lj_assertA(rset_test(as->freeset, r), "reg %d not free", r);
495 if (ra_hasreg(ir->r)) { 623 if (ra_hasreg(ir->r)) {
496 ra_noweak(as, ir->r); 624 ra_noweak(as, ir->r);
497 emit_movrr(as, ir, r, ir->r); 625 emit_movrr(as, ir, r, ir->r);
@@ -500,7 +628,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
500 } 628 }
501 } 629 }
502 } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */ 630 } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */
503 lua_assert(!(irt_isfloat(ir->t) && irref_isk(ref))); /* No float k. */ 631 lj_assertA(!(irt_isfloat(ir->t) && irref_isk(ref)),
632 "unexpected float constant");
504 if (LJ_32 && (ofs & 4) && irref_isk(ref)) { 633 if (LJ_32 && (ofs & 4) && irref_isk(ref)) {
505 /* Split stores for unaligned FP consts. */ 634 /* Split stores for unaligned FP consts. */
506 emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo); 635 emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo);
@@ -531,7 +660,7 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
531static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci) 660static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
532{ 661{
533 RegSet drop = RSET_SCRATCH; 662 RegSet drop = RSET_SCRATCH;
534 int hiop = (LJ_32 && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t)); 663 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
535 if ((ci->flags & CCI_NOFPRCLOBBER)) 664 if ((ci->flags & CCI_NOFPRCLOBBER))
536 drop &= ~RSET_FPR; 665 drop &= ~RSET_FPR;
537 if (ra_hasreg(ir->r)) 666 if (ra_hasreg(ir->r))
@@ -560,7 +689,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
560 if (ra_hasreg(dest)) { 689 if (ra_hasreg(dest)) {
561 ra_free(as, dest); 690 ra_free(as, dest);
562 ra_modified(as, dest); 691 ra_modified(as, dest);
563 emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS, 692 emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS,
564 dest, RID_ESP, ofs); 693 dest, RID_ESP, ofs);
565 } 694 }
566 if ((ci->flags & CCI_CASTU64)) { 695 if ((ci->flags & CCI_CASTU64)) {
@@ -571,12 +700,10 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
571 irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs); 700 irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
572 } 701 }
573#endif 702#endif
574#if LJ_32
575 } else if (hiop) { 703 } else if (hiop) {
576 ra_destpair(as, ir); 704 ra_destpair(as, ir);
577#endif
578 } else { 705 } else {
579 lua_assert(!irt_ispri(ir->t)); 706 lj_assertA(!irt_ispri(ir->t), "PRI dest");
580 ra_destreg(as, ir, RID_RET); 707 ra_destreg(as, ir, RID_RET);
581 } 708 }
582 } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) { 709 } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) {
@@ -584,15 +711,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
584 } 711 }
585} 712}
586 713
587static void asm_call(ASMState *as, IRIns *ir)
588{
589 IRRef args[CCI_NARGS_MAX];
590 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
591 asm_collectargs(as, ir, ci, args);
592 asm_setupresult(as, ir, ci);
593 asm_gencall(as, ci, args);
594}
595
596/* Return a constant function pointer or NULL for indirect calls. */ 714/* Return a constant function pointer or NULL for indirect calls. */
597static void *asm_callx_func(ASMState *as, IRIns *irf, IRRef func) 715static void *asm_callx_func(ASMState *as, IRIns *irf, IRRef func)
598{ 716{
@@ -651,16 +769,39 @@ static void asm_callx(ASMState *as, IRIns *ir)
651static void asm_retf(ASMState *as, IRIns *ir) 769static void asm_retf(ASMState *as, IRIns *ir)
652{ 770{
653 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR); 771 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
772#if LJ_FR2
773 Reg rpc = ra_scratch(as, rset_exclude(RSET_GPR, base));
774#endif
654 void *pc = ir_kptr(IR(ir->op2)); 775 void *pc = ir_kptr(IR(ir->op2));
655 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1)); 776 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
656 as->topslot -= (BCReg)delta; 777 as->topslot -= (BCReg)delta;
657 if ((int32_t)as->topslot < 0) as->topslot = 0; 778 if ((int32_t)as->topslot < 0) as->topslot = 0;
658 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */ 779 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
659 emit_setgl(as, base, jit_base); 780 emit_setgl(as, base, jit_base);
660 emit_addptr(as, base, -8*delta); 781 emit_addptr(as, base, -8*delta);
661 asm_guardcc(as, CC_NE); 782 asm_guardcc(as, CC_NE);
783#if LJ_FR2
784 emit_rmro(as, XO_CMP, rpc|REX_GC64, base, -8);
785 emit_loadu64(as, rpc, u64ptr(pc));
786#else
662 emit_gmroi(as, XG_ARITHi(XOg_CMP), base, -4, ptr2addr(pc)); 787 emit_gmroi(as, XG_ARITHi(XOg_CMP), base, -4, ptr2addr(pc));
788#endif
789}
790
791/* -- Buffer operations --------------------------------------------------- */
792
793#if LJ_HASBUFFER
794static void asm_bufhdr_write(ASMState *as, Reg sb)
795{
796 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
797 IRIns irgc;
798 irgc.ot = IRT(0, IRT_PGC); /* GC type. */
799 emit_storeofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
800 emit_opgl(as, XO_ARITH(XOg_OR), tmp|REX_GC64, cur_L);
801 emit_gri(as, XG_ARITHi(XOg_AND), tmp, SBUF_MASK_FLAG);
802 emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
663} 803}
804#endif
664 805
665/* -- Type conversions ---------------------------------------------------- */ 806/* -- Type conversions ---------------------------------------------------- */
666 807
@@ -672,8 +813,7 @@ static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
672 asm_guardcc(as, CC_NE); 813 asm_guardcc(as, CC_NE);
673 emit_rr(as, XO_UCOMISD, left, tmp); 814 emit_rr(as, XO_UCOMISD, left, tmp);
674 emit_rr(as, XO_CVTSI2SD, tmp, dest); 815 emit_rr(as, XO_CVTSI2SD, tmp, dest);
675 if (!(as->flags & JIT_F_SPLIT_XMM)) 816 emit_rr(as, XO_XORPS, tmp, tmp); /* Avoid partial register stall. */
676 emit_rr(as, XO_XORPS, tmp, tmp); /* Avoid partial register stall. */
677 emit_rr(as, XO_CVTTSD2SI, dest, left); 817 emit_rr(as, XO_CVTTSD2SI, dest, left);
678 /* Can't fuse since left is needed twice. */ 818 /* Can't fuse since left is needed twice. */
679} 819}
@@ -684,8 +824,9 @@ static void asm_tobit(ASMState *as, IRIns *ir)
684 Reg tmp = ra_noreg(IR(ir->op1)->r) ? 824 Reg tmp = ra_noreg(IR(ir->op1)->r) ?
685 ra_alloc1(as, ir->op1, RSET_FPR) : 825 ra_alloc1(as, ir->op1, RSET_FPR) :
686 ra_scratch(as, RSET_FPR); 826 ra_scratch(as, RSET_FPR);
687 Reg right = asm_fuseload(as, ir->op2, rset_exclude(RSET_FPR, tmp)); 827 Reg right;
688 emit_rr(as, XO_MOVDto, tmp, dest); 828 emit_rr(as, XO_MOVDto, tmp, dest);
829 right = asm_fuseload(as, ir->op2, rset_exclude(RSET_FPR, tmp));
689 emit_mrm(as, XO_ADDSD, tmp, right); 830 emit_mrm(as, XO_ADDSD, tmp, right);
690 ra_left(as, tmp, ir->op1); 831 ra_left(as, tmp, ir->op1);
691} 832}
@@ -696,8 +837,10 @@ static void asm_conv(ASMState *as, IRIns *ir)
696 int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64)); 837 int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64));
697 int stfp = (st == IRT_NUM || st == IRT_FLOAT); 838 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
698 IRRef lref = ir->op1; 839 IRRef lref = ir->op1;
699 lua_assert(irt_type(ir->t) != st); 840 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
700 lua_assert(!(LJ_32 && (irt_isint64(ir->t) || st64))); /* Handled by SPLIT. */ 841 lj_assertA(!(LJ_32 && (irt_isint64(ir->t) || st64)),
842 "IR %04d has unsplit 64 bit type",
843 (int)(ir - as->ir) - REF_BIAS);
701 if (irt_isfp(ir->t)) { 844 if (irt_isfp(ir->t)) {
702 Reg dest = ra_dest(as, ir, RSET_FPR); 845 Reg dest = ra_dest(as, ir, RSET_FPR);
703 if (stfp) { /* FP to FP conversion. */ 846 if (stfp) { /* FP to FP conversion. */
@@ -706,13 +849,13 @@ static void asm_conv(ASMState *as, IRIns *ir)
706 if (left == dest) return; /* Avoid the XO_XORPS. */ 849 if (left == dest) return; /* Avoid the XO_XORPS. */
707 } else if (LJ_32 && st == IRT_U32) { /* U32 to FP conversion on x86. */ 850 } else if (LJ_32 && st == IRT_U32) { /* U32 to FP conversion on x86. */
708 /* number = (2^52+2^51 .. u32) - (2^52+2^51) */ 851 /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
709 cTValue *k = lj_ir_k64_find(as->J, U64x(43380000,00000000)); 852 cTValue *k = &as->J->k64[LJ_K64_TOBIT];
710 Reg bias = ra_scratch(as, rset_exclude(RSET_FPR, dest)); 853 Reg bias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
711 if (irt_isfloat(ir->t)) 854 if (irt_isfloat(ir->t))
712 emit_rr(as, XO_CVTSD2SS, dest, dest); 855 emit_rr(as, XO_CVTSD2SS, dest, dest);
713 emit_rr(as, XO_SUBSD, dest, bias); /* Subtract 2^52+2^51 bias. */ 856 emit_rr(as, XO_SUBSD, dest, bias); /* Subtract 2^52+2^51 bias. */
714 emit_rr(as, XO_XORPS, dest, bias); /* Merge bias and integer. */ 857 emit_rr(as, XO_XORPS, dest, bias); /* Merge bias and integer. */
715 emit_loadn(as, bias, k); 858 emit_rma(as, XO_MOVSD, bias, k);
716 emit_mrm(as, XO_MOVD, dest, asm_fuseload(as, lref, RSET_GPR)); 859 emit_mrm(as, XO_MOVD, dest, asm_fuseload(as, lref, RSET_GPR));
717 return; 860 return;
718 } else { /* Integer to FP conversion. */ 861 } else { /* Integer to FP conversion. */
@@ -721,7 +864,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
721 asm_fuseloadm(as, lref, RSET_GPR, st64); 864 asm_fuseloadm(as, lref, RSET_GPR, st64);
722 if (LJ_64 && st == IRT_U64) { 865 if (LJ_64 && st == IRT_U64) {
723 MCLabel l_end = emit_label(as); 866 MCLabel l_end = emit_label(as);
724 const void *k = lj_ir_k64_find(as->J, U64x(43f00000,00000000)); 867 cTValue *k = &as->J->k64[LJ_K64_2P64];
725 emit_rma(as, XO_ADDSD, dest, k); /* Add 2^64 to compensate. */ 868 emit_rma(as, XO_ADDSD, dest, k); /* Add 2^64 to compensate. */
726 emit_sjcc(as, CC_NS, l_end); 869 emit_sjcc(as, CC_NS, l_end);
727 emit_rr(as, XO_TEST, left|REX_64, left); /* Check if u64 >= 2^63. */ 870 emit_rr(as, XO_TEST, left|REX_64, left); /* Check if u64 >= 2^63. */
@@ -729,18 +872,16 @@ static void asm_conv(ASMState *as, IRIns *ir)
729 emit_mrm(as, irt_isnum(ir->t) ? XO_CVTSI2SD : XO_CVTSI2SS, 872 emit_mrm(as, irt_isnum(ir->t) ? XO_CVTSI2SD : XO_CVTSI2SS,
730 dest|((LJ_64 && (st64 || st == IRT_U32)) ? REX_64 : 0), left); 873 dest|((LJ_64 && (st64 || st == IRT_U32)) ? REX_64 : 0), left);
731 } 874 }
732 if (!(as->flags & JIT_F_SPLIT_XMM)) 875 emit_rr(as, XO_XORPS, dest, dest); /* Avoid partial register stall. */
733 emit_rr(as, XO_XORPS, dest, dest); /* Avoid partial register stall. */
734 } else if (stfp) { /* FP to integer conversion. */ 876 } else if (stfp) { /* FP to integer conversion. */
735 if (irt_isguard(ir->t)) { 877 if (irt_isguard(ir->t)) {
736 /* Checked conversions are only supported from number to int. */ 878 /* Checked conversions are only supported from number to int. */
737 lua_assert(irt_isint(ir->t) && st == IRT_NUM); 879 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
880 "bad type for checked CONV");
738 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); 881 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
739 } else { 882 } else {
740 Reg dest = ra_dest(as, ir, RSET_GPR); 883 Reg dest = ra_dest(as, ir, RSET_GPR);
741 x86Op op = st == IRT_NUM ? 884 x86Op op = st == IRT_NUM ? XO_CVTTSD2SI : XO_CVTTSS2SI;
742 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSD2SI : XO_CVTSD2SI) :
743 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSS2SI : XO_CVTSS2SI);
744 if (LJ_64 ? irt_isu64(ir->t) : irt_isu32(ir->t)) { 885 if (LJ_64 ? irt_isu64(ir->t) : irt_isu32(ir->t)) {
745 /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */ 886 /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */
746 /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */ 887 /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */
@@ -751,30 +892,27 @@ static void asm_conv(ASMState *as, IRIns *ir)
751 emit_gri(as, XG_ARITHi(XOg_ADD), dest, (int32_t)0x80000000); 892 emit_gri(as, XG_ARITHi(XOg_ADD), dest, (int32_t)0x80000000);
752 emit_rr(as, op, dest|REX_64, tmp); 893 emit_rr(as, op, dest|REX_64, tmp);
753 if (st == IRT_NUM) 894 if (st == IRT_NUM)
754 emit_rma(as, XO_ADDSD, tmp, lj_ir_k64_find(as->J, 895 emit_rma(as, XO_ADDSD, tmp, &as->J->k64[LJ_K64_M2P64_31]);
755 LJ_64 ? U64x(c3f00000,00000000) : U64x(c1e00000,00000000)));
756 else 896 else
757 emit_rma(as, XO_ADDSS, tmp, lj_ir_k64_find(as->J, 897 emit_rma(as, XO_ADDSS, tmp, &as->J->k32[LJ_K32_M2P64_31]);
758 LJ_64 ? U64x(00000000,df800000) : U64x(00000000,cf000000)));
759 emit_sjcc(as, CC_NS, l_end); 898 emit_sjcc(as, CC_NS, l_end);
760 emit_rr(as, XO_TEST, dest|REX_64, dest); /* Check if dest negative. */ 899 emit_rr(as, XO_TEST, dest|REX_64, dest); /* Check if dest negative. */
761 emit_rr(as, op, dest|REX_64, tmp); 900 emit_rr(as, op, dest|REX_64, tmp);
762 ra_left(as, tmp, lref); 901 ra_left(as, tmp, lref);
763 } else { 902 } else {
764 Reg left = asm_fuseload(as, lref, RSET_FPR);
765 if (LJ_64 && irt_isu32(ir->t)) 903 if (LJ_64 && irt_isu32(ir->t))
766 emit_rr(as, XO_MOV, dest, dest); /* Zero hiword. */ 904 emit_rr(as, XO_MOV, dest, dest); /* Zero hiword. */
767 emit_mrm(as, op, 905 emit_mrm(as, op,
768 dest|((LJ_64 && 906 dest|((LJ_64 &&
769 (irt_is64(ir->t) || irt_isu32(ir->t))) ? REX_64 : 0), 907 (irt_is64(ir->t) || irt_isu32(ir->t))) ? REX_64 : 0),
770 left); 908 asm_fuseload(as, lref, RSET_FPR));
771 } 909 }
772 } 910 }
773 } else if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */ 911 } else if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
774 Reg left, dest = ra_dest(as, ir, RSET_GPR); 912 Reg left, dest = ra_dest(as, ir, RSET_GPR);
775 RegSet allow = RSET_GPR; 913 RegSet allow = RSET_GPR;
776 x86Op op; 914 x86Op op;
777 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); 915 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
778 if (st == IRT_I8) { 916 if (st == IRT_I8) {
779 op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX; 917 op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX;
780 } else if (st == IRT_U8) { 918 } else if (st == IRT_U8) {
@@ -808,7 +946,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
808 } 946 }
809 } else { 947 } else {
810 Reg dest = ra_dest(as, ir, RSET_GPR); 948 Reg dest = ra_dest(as, ir, RSET_GPR);
811 if (st64) { 949 if (st64 && !(ir->op2 & IRCONV_NONE)) {
812 Reg left = asm_fuseload(as, lref, RSET_GPR); 950 Reg left = asm_fuseload(as, lref, RSET_GPR);
813 /* This is either a 32 bit reg/reg mov which zeroes the hiword 951 /* This is either a 32 bit reg/reg mov which zeroes the hiword
814 ** or a load of the loword from a 64 bit address. 952 ** or a load of the loword from a 64 bit address.
@@ -834,20 +972,18 @@ static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
834 if (ra_hasreg(dest)) { 972 if (ra_hasreg(dest)) {
835 ra_free(as, dest); 973 ra_free(as, dest);
836 ra_modified(as, dest); 974 ra_modified(as, dest);
837 emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS, 975 emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS, dest, RID_ESP, ofs);
838 dest, RID_ESP, ofs);
839 } 976 }
840 emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd, 977 emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd,
841 irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs); 978 irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
842 if (((ir-1)->op2 & IRCONV_SRCMASK) == IRT_U64) { 979 if (((ir-1)->op2 & IRCONV_SRCMASK) == IRT_U64) {
843 /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */ 980 /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
844 MCLabel l_end = emit_label(as); 981 MCLabel l_end = emit_label(as);
845 emit_rma(as, XO_FADDq, XOg_FADDq, 982 emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_2P64]);
846 lj_ir_k64_find(as->J, U64x(43f00000,00000000)));
847 emit_sjcc(as, CC_NS, l_end); 983 emit_sjcc(as, CC_NS, l_end);
848 emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */ 984 emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */
849 } else { 985 } else {
850 lua_assert(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64); 986 lj_assertA(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64, "bad type for CONV");
851 } 987 }
852 emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0); 988 emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0);
853 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */ 989 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
@@ -861,9 +997,8 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
861 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK); 997 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
862 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH); 998 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
863 Reg lo, hi; 999 Reg lo, hi;
864 lua_assert(st == IRT_NUM || st == IRT_FLOAT); 1000 lj_assertA(st == IRT_NUM || st == IRT_FLOAT, "bad type for CONV");
865 lua_assert(dt == IRT_I64 || dt == IRT_U64); 1001 lj_assertA(dt == IRT_I64 || dt == IRT_U64, "bad type for CONV");
866 lua_assert(((ir-1)->op2 & IRCONV_TRUNC));
867 hi = ra_dest(as, ir, RSET_GPR); 1002 hi = ra_dest(as, ir, RSET_GPR);
868 lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi)); 1003 lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi));
869 if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0); 1004 if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0);
@@ -884,8 +1019,7 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
884 emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0); 1019 emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
885 else 1020 else
886 emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0); 1021 emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
887 emit_rma(as, XO_FADDq, XOg_FADDq, 1022 emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_M2P64]);
888 lj_ir_k64_find(as->J, U64x(c3f00000,00000000)));
889 emit_sjcc(as, CC_NS, l_pop); 1023 emit_sjcc(as, CC_NS, l_pop);
890 emit_rr(as, XO_TEST, hi, hi); /* Check if out-of-range (2^63). */ 1024 emit_rr(as, XO_TEST, hi, hi); /* Check if out-of-range (2^63). */
891 } 1025 }
@@ -906,6 +1040,14 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
906 st == IRT_NUM ? XOg_FLDq: XOg_FLDd, 1040 st == IRT_NUM ? XOg_FLDq: XOg_FLDd,
907 asm_fuseload(as, ir->op1, RSET_EMPTY)); 1041 asm_fuseload(as, ir->op1, RSET_EMPTY));
908} 1042}
1043
1044static void asm_conv64(ASMState *as, IRIns *ir)
1045{
1046 if (irt_isfp(ir->t))
1047 asm_conv_fp_int64(as, ir);
1048 else
1049 asm_conv_int64_fp(as, ir);
1050}
909#endif 1051#endif
910 1052
911static void asm_strto(ASMState *as, IRIns *ir) 1053static void asm_strto(ASMState *as, IRIns *ir)
@@ -927,54 +1069,61 @@ static void asm_strto(ASMState *as, IRIns *ir)
927 RID_ESP, sps_scale(ir->s)); 1069 RID_ESP, sps_scale(ir->s));
928} 1070}
929 1071
930static void asm_tostr(ASMState *as, IRIns *ir) 1072/* -- Memory references --------------------------------------------------- */
1073
1074/* Get pointer to TValue. */
1075static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
931{ 1076{
932 IRIns *irl = IR(ir->op1); 1077 if ((mode & IRTMPREF_IN1)) {
933 IRRef args[2]; 1078 IRIns *ir = IR(ref);
934 args[0] = ASMREF_L; 1079 if (irt_isnum(ir->t)) {
935 as->gcsteps++; 1080 if (irref_isk(ref) && !(mode & IRTMPREF_OUT1)) {
936 if (irt_isnum(irl->t)) { 1081 /* Use the number constant itself as a TValue. */
937 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum]; 1082 emit_loada(as, dest, ir_knum(ir));
938 args[1] = ASMREF_TMP1; /* const lua_Number * */ 1083 return;
939 asm_setupresult(as, ir, ci); /* GCstr * */ 1084 }
940 asm_gencall(as, ci, args); 1085 emit_rmro(as, XO_MOVSDto, ra_alloc1(as, ref, RSET_FPR), dest, 0);
941 emit_rmro(as, XO_LEA, ra_releasetmp(as, ASMREF_TMP1)|REX_64, 1086 } else {
942 RID_ESP, ra_spill(as, irl)); 1087#if LJ_GC64
943 } else { 1088 if (irref_isk(ref)) {
944 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint]; 1089 TValue k;
945 args[1] = ir->op1; /* int32_t k */ 1090 lj_ir_kvalue(as->J->L, &k, ir);
946 asm_setupresult(as, ir, ci); /* GCstr * */ 1091 emit_movmroi(as, dest, 4, k.u32.hi);
947 asm_gencall(as, ci, args); 1092 emit_movmroi(as, dest, 0, k.u32.lo);
1093 } else {
1094 /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
1095 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
1096 if (irt_is64(ir->t)) {
1097 emit_u32(as, irt_toitype(ir->t) << 15);
1098 emit_rmro(as, XO_ARITHi, XOg_OR, dest, 4);
1099 } else {
1100 emit_movmroi(as, dest, 4, (irt_toitype(ir->t) << 15));
1101 }
1102 emit_movtomro(as, REX_64IR(ir, src), dest, 0);
1103 }
1104#else
1105 if (!irref_isk(ref)) {
1106 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
1107 emit_movtomro(as, REX_64IR(ir, src), dest, 0);
1108 } else if (!irt_ispri(ir->t)) {
1109 emit_movmroi(as, dest, 0, ir->i);
1110 }
1111 if (!(LJ_64 && irt_islightud(ir->t)))
1112 emit_movmroi(as, dest, 4, irt_toitype(ir->t));
1113#endif
1114 }
948 } 1115 }
1116 emit_loada(as, dest, &J2G(as->J)->tmptv); /* g->tmptv holds the TValue(s). */
949} 1117}
950 1118
951/* -- Memory references --------------------------------------------------- */
952
953static void asm_aref(ASMState *as, IRIns *ir) 1119static void asm_aref(ASMState *as, IRIns *ir)
954{ 1120{
955 Reg dest = ra_dest(as, ir, RSET_GPR); 1121 Reg dest = ra_dest(as, ir, RSET_GPR);
956 asm_fusearef(as, ir, RSET_GPR); 1122 asm_fusearef(as, ir, RSET_GPR);
957 if (!(as->mrm.idx == RID_NONE && as->mrm.ofs == 0)) 1123 if (!(as->mrm.idx == RID_NONE && as->mrm.ofs == 0))
958 emit_mrm(as, XO_LEA, dest, RID_MRM); 1124 emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
959 else if (as->mrm.base != dest) 1125 else if (as->mrm.base != dest)
960 emit_rr(as, XO_MOV, dest, as->mrm.base); 1126 emit_rr(as, XO_MOV, dest|REX_GC64, as->mrm.base);
961}
962
963/* Merge NE(HREF, niltv) check. */
964static MCode *merge_href_niltv(ASMState *as, IRIns *ir)
965{
966 /* Assumes nothing else generates NE of HREF. */
967 if ((ir[1].o == IR_NE || ir[1].o == IR_EQ) && ir[1].op1 == as->curins &&
968 ra_hasreg(ir->r)) {
969 MCode *p = as->mcp;
970 p += (LJ_64 && *p != XI_ARITHi) ? 7+6 : 6+6;
971 /* Ensure no loop branch inversion happened. */
972 if (p[-6] == 0x0f && p[-5] == XI_JCCn+(CC_NE^(ir[1].o & 1))) {
973 as->mcp = p; /* Kill cmp reg, imm32 + jz exit. */
974 return p + *(int32_t *)(p-4); /* Return exit address. */
975 }
976 }
977 return NULL;
978} 1127}
979 1128
980/* Inlined hash lookup. Specialized for key type and for const keys. 1129/* Inlined hash lookup. Specialized for key type and for const keys.
@@ -985,10 +1134,10 @@ static MCode *merge_href_niltv(ASMState *as, IRIns *ir)
985** } while ((n = nextnode(n))); 1134** } while ((n = nextnode(n)));
986** return niltv(L); 1135** return niltv(L);
987*/ 1136*/
988static void asm_href(ASMState *as, IRIns *ir) 1137static void asm_href(ASMState *as, IRIns *ir, IROp merge)
989{ 1138{
990 MCode *nilexit = merge_href_niltv(as, ir); /* Do this before any restores. */
991 RegSet allow = RSET_GPR; 1139 RegSet allow = RSET_GPR;
1140 int destused = ra_used(ir);
992 Reg dest = ra_dest(as, ir, allow); 1141 Reg dest = ra_dest(as, ir, allow);
993 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest)); 1142 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
994 Reg key = RID_NONE, tmp = RID_NONE; 1143 Reg key = RID_NONE, tmp = RID_NONE;
@@ -1001,28 +1150,26 @@ static void asm_href(ASMState *as, IRIns *ir)
1001 if (!isk) { 1150 if (!isk) {
1002 rset_clear(allow, tab); 1151 rset_clear(allow, tab);
1003 key = ra_alloc1(as, ir->op2, irt_isnum(kt) ? RSET_FPR : allow); 1152 key = ra_alloc1(as, ir->op2, irt_isnum(kt) ? RSET_FPR : allow);
1004 if (!irt_isstr(kt)) 1153 if (LJ_GC64 || !irt_isstr(kt))
1005 tmp = ra_scratch(as, rset_exclude(allow, key)); 1154 tmp = ra_scratch(as, rset_exclude(allow, key));
1006 } 1155 }
1007 1156
1008 /* Key not found in chain: jump to exit (if merged with NE) or load niltv. */ 1157 /* Key not found in chain: jump to exit (if merged) or load niltv. */
1009 l_end = emit_label(as); 1158 l_end = emit_label(as);
1010 if (nilexit && ir[1].o == IR_NE) { 1159 if (merge == IR_NE)
1011 emit_jcc(as, CC_E, nilexit); /* XI_JMP is not found by lj_asm_patchexit. */ 1160 asm_guardcc(as, CC_E); /* XI_JMP is not found by lj_asm_patchexit. */
1012 nilexit = NULL; 1161 else if (destused)
1013 } else {
1014 emit_loada(as, dest, niltvg(J2G(as->J))); 1162 emit_loada(as, dest, niltvg(J2G(as->J)));
1015 }
1016 1163
1017 /* Follow hash chain until the end. */ 1164 /* Follow hash chain until the end. */
1018 l_loop = emit_sjcc_label(as, CC_NZ); 1165 l_loop = emit_sjcc_label(as, CC_NZ);
1019 emit_rr(as, XO_TEST, dest, dest); 1166 emit_rr(as, XO_TEST, dest|REX_GC64, dest);
1020 emit_rmro(as, XO_MOV, dest, dest, offsetof(Node, next)); 1167 emit_rmro(as, XO_MOV, dest|REX_GC64, dest, offsetof(Node, next));
1021 l_next = emit_label(as); 1168 l_next = emit_label(as);
1022 1169
1023 /* Type and value comparison. */ 1170 /* Type and value comparison. */
1024 if (nilexit) 1171 if (merge == IR_EQ)
1025 emit_jcc(as, CC_E, nilexit); 1172 asm_guardcc(as, CC_E);
1026 else 1173 else
1027 emit_sjcc(as, CC_E, l_end); 1174 emit_sjcc(as, CC_E, l_end);
1028 if (irt_isnum(kt)) { 1175 if (irt_isnum(kt)) {
@@ -1038,7 +1185,7 @@ static void asm_href(ASMState *as, IRIns *ir)
1038 emit_rmro(as, XO_UCOMISD, key, dest, offsetof(Node, key.n)); 1185 emit_rmro(as, XO_UCOMISD, key, dest, offsetof(Node, key.n));
1039 emit_sjcc(as, CC_AE, l_next); 1186 emit_sjcc(as, CC_AE, l_next);
1040 /* The type check avoids NaN penalties and complaints from Valgrind. */ 1187 /* The type check avoids NaN penalties and complaints from Valgrind. */
1041#if LJ_64 1188#if LJ_64 && !LJ_GC64
1042 emit_u32(as, LJ_TISNUM); 1189 emit_u32(as, LJ_TISNUM);
1043 emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it)); 1190 emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
1044#else 1191#else
@@ -1046,13 +1193,31 @@ static void asm_href(ASMState *as, IRIns *ir)
1046 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it)); 1193 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
1047#endif 1194#endif
1048 } 1195 }
1049#if LJ_64 1196#if LJ_64 && !LJ_GC64
1050 } else if (irt_islightud(kt)) { 1197 } else if (irt_islightud(kt)) {
1051 emit_rmro(as, XO_CMP, key|REX_64, dest, offsetof(Node, key.u64)); 1198 emit_rmro(as, XO_CMP, key|REX_64, dest, offsetof(Node, key.u64));
1052#endif 1199#endif
1200#if LJ_GC64
1201 } else if (irt_isaddr(kt)) {
1202 if (isk) {
1203 TValue k;
1204 k.u64 = ((uint64_t)irt_toitype(irkey->t) << 47) | irkey[1].tv.u64;
1205 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.lo),
1206 k.u32.lo);
1207 emit_sjcc(as, CC_NE, l_next);
1208 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.hi),
1209 k.u32.hi);
1210 } else {
1211 emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64));
1212 }
1213 } else {
1214 lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
1215 emit_u32(as, (irt_toitype(kt)<<15)|0x7fff);
1216 emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
1217#else
1053 } else { 1218 } else {
1054 if (!irt_ispri(kt)) { 1219 if (!irt_ispri(kt)) {
1055 lua_assert(irt_isaddr(kt)); 1220 lj_assertA(irt_isaddr(kt), "bad HREF key type");
1056 if (isk) 1221 if (isk)
1057 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr), 1222 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr),
1058 ptr2addr(ir_kgc(irkey))); 1223 ptr2addr(ir_kgc(irkey)));
@@ -1060,31 +1225,33 @@ static void asm_href(ASMState *as, IRIns *ir)
1060 emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr)); 1225 emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr));
1061 emit_sjcc(as, CC_NE, l_next); 1226 emit_sjcc(as, CC_NE, l_next);
1062 } 1227 }
1063 lua_assert(!irt_isnil(kt)); 1228 lj_assertA(!irt_isnil(kt), "bad HREF key type");
1064 emit_i8(as, irt_toitype(kt)); 1229 emit_i8(as, irt_toitype(kt));
1065 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it)); 1230 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
1231#endif
1066 } 1232 }
1067 emit_sfixup(as, l_loop); 1233 emit_sfixup(as, l_loop);
1068 checkmclim(as); 1234 checkmclim(as);
1235#if LJ_GC64
1236 if (!isk && irt_isaddr(kt)) {
1237 emit_rr(as, XO_OR, tmp|REX_64, key);
1238 emit_loadu64(as, tmp, (uint64_t)irt_toitype(kt) << 47);
1239 }
1240#endif
1069 1241
1070 /* Load main position relative to tab->node into dest. */ 1242 /* Load main position relative to tab->node into dest. */
1071 khash = isk ? ir_khash(irkey) : 1; 1243 khash = isk ? ir_khash(as, irkey) : 1;
1072 if (khash == 0) { 1244 if (khash == 0) {
1073 emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, node)); 1245 emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node));
1074 } else { 1246 } else {
1075 emit_rmro(as, XO_ARITH(XOg_ADD), dest, tab, offsetof(GCtab, node)); 1247 emit_rmro(as, XO_ARITH(XOg_ADD), dest|REX_GC64, tab, offsetof(GCtab,node));
1076 if ((as->flags & JIT_F_PREFER_IMUL)) { 1248 emit_shifti(as, XOg_SHL, dest, 3);
1077 emit_i8(as, sizeof(Node)); 1249 emit_rmrxo(as, XO_LEA, dest, dest, dest, XM_SCALE2, 0);
1078 emit_rr(as, XO_IMULi8, dest, dest);
1079 } else {
1080 emit_shifti(as, XOg_SHL, dest, 3);
1081 emit_rmrxo(as, XO_LEA, dest, dest, dest, XM_SCALE2, 0);
1082 }
1083 if (isk) { 1250 if (isk) {
1084 emit_gri(as, XG_ARITHi(XOg_AND), dest, (int32_t)khash); 1251 emit_gri(as, XG_ARITHi(XOg_AND), dest, (int32_t)khash);
1085 emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); 1252 emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask));
1086 } else if (irt_isstr(kt)) { 1253 } else if (irt_isstr(kt)) {
1087 emit_rmro(as, XO_ARITH(XOg_AND), dest, key, offsetof(GCstr, hash)); 1254 emit_rmro(as, XO_ARITH(XOg_AND), dest, key, offsetof(GCstr, sid));
1088 emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); 1255 emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask));
1089 } else { /* Must match with hashrot() in lj_tab.c. */ 1256 } else { /* Must match with hashrot() in lj_tab.c. */
1090 emit_rmro(as, XO_ARITH(XOg_AND), dest, tab, offsetof(GCtab, hmask)); 1257 emit_rmro(as, XO_ARITH(XOg_AND), dest, tab, offsetof(GCtab, hmask));
@@ -1107,7 +1274,19 @@ static void asm_href(ASMState *as, IRIns *ir)
1107#endif 1274#endif
1108 } else { 1275 } else {
1109 emit_rr(as, XO_MOV, tmp, key); 1276 emit_rr(as, XO_MOV, tmp, key);
1277#if LJ_GC64
1278 checkmclim(as);
1279 emit_gri(as, XG_ARITHi(XOg_XOR), dest, irt_toitype(kt) << 15);
1280 if ((as->flags & JIT_F_BMI2)) {
1281 emit_i8(as, 32);
1282 emit_mrm(as, XV_RORX|VEX_64, dest, key);
1283 } else {
1284 emit_shifti(as, XOg_SHR|REX_64, dest, 32);
1285 emit_rr(as, XO_MOV, dest|REX_64, key|REX_64);
1286 }
1287#else
1110 emit_rmro(as, XO_LEA, dest, key, HASH_BIAS); 1288 emit_rmro(as, XO_LEA, dest, key, HASH_BIAS);
1289#endif
1111 } 1290 }
1112 } 1291 }
1113 } 1292 }
@@ -1123,15 +1302,15 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1123#if !LJ_64 1302#if !LJ_64
1124 MCLabel l_exit; 1303 MCLabel l_exit;
1125#endif 1304#endif
1126 lua_assert(ofs % sizeof(Node) == 0); 1305 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
1127 if (ra_hasreg(dest)) { 1306 if (ra_hasreg(dest)) {
1128 if (ofs != 0) { 1307 if (ofs != 0) {
1129 if (dest == node && !(as->flags & JIT_F_LEA_AGU)) 1308 if (dest == node)
1130 emit_gri(as, XG_ARITHi(XOg_ADD), dest, ofs); 1309 emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, ofs);
1131 else 1310 else
1132 emit_rmro(as, XO_LEA, dest, node, ofs); 1311 emit_rmro(as, XO_LEA, dest|REX_GC64, node, ofs);
1133 } else if (dest != node) { 1312 } else if (dest != node) {
1134 emit_rr(as, XO_MOV, dest, node); 1313 emit_rr(as, XO_MOV, dest|REX_GC64, node);
1135 } 1314 }
1136 } 1315 }
1137 asm_guardcc(as, CC_NE); 1316 asm_guardcc(as, CC_NE);
@@ -1140,16 +1319,28 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1140 Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node)); 1319 Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node));
1141 emit_rmro(as, XO_CMP, key|REX_64, node, 1320 emit_rmro(as, XO_CMP, key|REX_64, node,
1142 ofs + (int32_t)offsetof(Node, key.u64)); 1321 ofs + (int32_t)offsetof(Node, key.u64));
1143 lua_assert(irt_isnum(irkey->t) || irt_isgcv(irkey->t)); 1322 lj_assertA(irt_isnum(irkey->t) || irt_isgcv(irkey->t),
1323 "bad HREFK key type");
1144 /* Assumes -0.0 is already canonicalized to +0.0. */ 1324 /* Assumes -0.0 is already canonicalized to +0.0. */
1145 emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 : 1325 emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 :
1326#if LJ_GC64
1327 ((uint64_t)irt_toitype(irkey->t) << 47) |
1328 (uint64_t)ir_kgc(irkey));
1329#else
1146 ((uint64_t)irt_toitype(irkey->t) << 32) | 1330 ((uint64_t)irt_toitype(irkey->t) << 32) |
1147 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey))); 1331 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey)));
1332#endif
1148 } else { 1333 } else {
1149 lua_assert(!irt_isnil(irkey->t)); 1334 lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
1335#if LJ_GC64
1336 emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff);
1337 emit_rmro(as, XO_ARITHi, XOg_CMP, node,
1338 ofs + (int32_t)offsetof(Node, key.it));
1339#else
1150 emit_i8(as, irt_toitype(irkey->t)); 1340 emit_i8(as, irt_toitype(irkey->t));
1151 emit_rmro(as, XO_ARITHi8, XOg_CMP, node, 1341 emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
1152 ofs + (int32_t)offsetof(Node, key.it)); 1342 ofs + (int32_t)offsetof(Node, key.it));
1343#endif
1153 } 1344 }
1154#else 1345#else
1155 l_exit = emit_label(as); 1346 l_exit = emit_label(as);
@@ -1164,13 +1355,13 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1164 (int32_t)ir_knum(irkey)->u32.hi); 1355 (int32_t)ir_knum(irkey)->u32.hi);
1165 } else { 1356 } else {
1166 if (!irt_ispri(irkey->t)) { 1357 if (!irt_ispri(irkey->t)) {
1167 lua_assert(irt_isgcv(irkey->t)); 1358 lj_assertA(irt_isgcv(irkey->t), "bad HREFK key type");
1168 emit_gmroi(as, XG_ARITHi(XOg_CMP), node, 1359 emit_gmroi(as, XG_ARITHi(XOg_CMP), node,
1169 ofs + (int32_t)offsetof(Node, key.gcr), 1360 ofs + (int32_t)offsetof(Node, key.gcr),
1170 ptr2addr(ir_kgc(irkey))); 1361 ptr2addr(ir_kgc(irkey)));
1171 emit_sjcc(as, CC_NE, l_exit); 1362 emit_sjcc(as, CC_NE, l_exit);
1172 } 1363 }
1173 lua_assert(!irt_isnil(irkey->t)); 1364 lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
1174 emit_i8(as, irt_toitype(irkey->t)); 1365 emit_i8(as, irt_toitype(irkey->t));
1175 emit_rmro(as, XO_ARITHi8, XOg_CMP, node, 1366 emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
1176 ofs + (int32_t)offsetof(Node, key.it)); 1367 ofs + (int32_t)offsetof(Node, key.it));
@@ -1178,61 +1369,27 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1178#endif 1369#endif
1179} 1370}
1180 1371
1181static void asm_newref(ASMState *as, IRIns *ir)
1182{
1183 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
1184 IRRef args[3];
1185 IRIns *irkey;
1186 Reg tmp;
1187 if (ir->r == RID_SINK)
1188 return;
1189 args[0] = ASMREF_L; /* lua_State *L */
1190 args[1] = ir->op1; /* GCtab *t */
1191 args[2] = ASMREF_TMP1; /* cTValue *key */
1192 asm_setupresult(as, ir, ci); /* TValue * */
1193 asm_gencall(as, ci, args);
1194 tmp = ra_releasetmp(as, ASMREF_TMP1);
1195 irkey = IR(ir->op2);
1196 if (irt_isnum(irkey->t)) {
1197 /* For numbers use the constant itself or a spill slot as a TValue. */
1198 if (irref_isk(ir->op2))
1199 emit_loada(as, tmp, ir_knum(irkey));
1200 else
1201 emit_rmro(as, XO_LEA, tmp|REX_64, RID_ESP, ra_spill(as, irkey));
1202 } else {
1203 /* Otherwise use g->tmptv to hold the TValue. */
1204 if (!irref_isk(ir->op2)) {
1205 Reg src = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, tmp));
1206 emit_movtomro(as, REX_64IR(irkey, src), tmp, 0);
1207 } else if (!irt_ispri(irkey->t)) {
1208 emit_movmroi(as, tmp, 0, irkey->i);
1209 }
1210 if (!(LJ_64 && irt_islightud(irkey->t)))
1211 emit_movmroi(as, tmp, 4, irt_toitype(irkey->t));
1212 emit_loada(as, tmp, &J2G(as->J)->tmptv);
1213 }
1214}
1215
1216static void asm_uref(ASMState *as, IRIns *ir) 1372static void asm_uref(ASMState *as, IRIns *ir)
1217{ 1373{
1218 Reg dest = ra_dest(as, ir, RSET_GPR); 1374 Reg dest = ra_dest(as, ir, RSET_GPR);
1219 if (irref_isk(ir->op1)) { 1375 if (irref_isk(ir->op1)) {
1220 GCfunc *fn = ir_kfunc(IR(ir->op1)); 1376 GCfunc *fn = ir_kfunc(IR(ir->op1));
1221 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v; 1377 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
1222 emit_rma(as, XO_MOV, dest, v); 1378 emit_rma(as, XO_MOV, dest|REX_GC64, v);
1223 } else { 1379 } else {
1224 Reg uv = ra_scratch(as, RSET_GPR); 1380 Reg uv = ra_scratch(as, RSET_GPR);
1225 Reg func = ra_alloc1(as, ir->op1, RSET_GPR); 1381 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
1226 if (ir->o == IR_UREFC) { 1382 if (ir->o == IR_UREFC) {
1227 emit_rmro(as, XO_LEA, dest, uv, offsetof(GCupval, tv)); 1383 emit_rmro(as, XO_LEA, dest|REX_GC64, uv, offsetof(GCupval, tv));
1228 asm_guardcc(as, CC_NE); 1384 asm_guardcc(as, CC_NE);
1229 emit_i8(as, 1); 1385 emit_i8(as, 1);
1230 emit_rmro(as, XO_ARITHib, XOg_CMP, uv, offsetof(GCupval, closed)); 1386 emit_rmro(as, XO_ARITHib, XOg_CMP, uv, offsetof(GCupval, closed));
1231 } else { 1387 } else {
1232 emit_rmro(as, XO_MOV, dest, uv, offsetof(GCupval, v)); 1388 emit_rmro(as, XO_MOV, dest|REX_GC64, uv, offsetof(GCupval, v));
1233 } 1389 }
1234 emit_rmro(as, XO_MOV, uv, func, 1390 emit_rmro(as, XO_MOV, uv|REX_GC64, func,
1235 (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8)); 1391 (int32_t)offsetof(GCfuncL, uvptr) +
1392 (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
1236 } 1393 }
1237} 1394}
1238 1395
@@ -1250,9 +1407,9 @@ static void asm_strref(ASMState *as, IRIns *ir)
1250 if (as->mrm.base == RID_NONE) 1407 if (as->mrm.base == RID_NONE)
1251 emit_loadi(as, dest, as->mrm.ofs); 1408 emit_loadi(as, dest, as->mrm.ofs);
1252 else if (as->mrm.base == dest && as->mrm.idx == RID_NONE) 1409 else if (as->mrm.base == dest && as->mrm.idx == RID_NONE)
1253 emit_gri(as, XG_ARITHi(XOg_ADD), dest, as->mrm.ofs); 1410 emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, as->mrm.ofs);
1254 else 1411 else
1255 emit_mrm(as, XO_LEA, dest, RID_MRM); 1412 emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
1256} 1413}
1257 1414
1258/* -- Loads and stores ---------------------------------------------------- */ 1415/* -- Loads and stores ---------------------------------------------------- */
@@ -1271,19 +1428,23 @@ static void asm_fxload(ASMState *as, IRIns *ir)
1271 case IRT_U8: xo = XO_MOVZXb; break; 1428 case IRT_U8: xo = XO_MOVZXb; break;
1272 case IRT_I16: xo = XO_MOVSXw; break; 1429 case IRT_I16: xo = XO_MOVSXw; break;
1273 case IRT_U16: xo = XO_MOVZXw; break; 1430 case IRT_U16: xo = XO_MOVZXw; break;
1274 case IRT_NUM: xo = XMM_MOVRM(as); break; 1431 case IRT_NUM: xo = XO_MOVSD; break;
1275 case IRT_FLOAT: xo = XO_MOVSS; break; 1432 case IRT_FLOAT: xo = XO_MOVSS; break;
1276 default: 1433 default:
1277 if (LJ_64 && irt_is64(ir->t)) 1434 if (LJ_64 && irt_is64(ir->t))
1278 dest |= REX_64; 1435 dest |= REX_64;
1279 else 1436 else
1280 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); 1437 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t),
1438 "unsplit 64 bit load");
1281 xo = XO_MOV; 1439 xo = XO_MOV;
1282 break; 1440 break;
1283 } 1441 }
1284 emit_mrm(as, xo, dest, RID_MRM); 1442 emit_mrm(as, xo, dest, RID_MRM);
1285} 1443}
1286 1444
1445#define asm_fload(as, ir) asm_fxload(as, ir)
1446#define asm_xload(as, ir) asm_fxload(as, ir)
1447
1287static void asm_fxstore(ASMState *as, IRIns *ir) 1448static void asm_fxstore(ASMState *as, IRIns *ir)
1288{ 1449{
1289 RegSet allow = RSET_GPR; 1450 RegSet allow = RSET_GPR;
@@ -1318,14 +1479,17 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
1318 case IRT_I16: case IRT_U16: xo = XO_MOVtow; break; 1479 case IRT_I16: case IRT_U16: xo = XO_MOVtow; break;
1319 case IRT_NUM: xo = XO_MOVSDto; break; 1480 case IRT_NUM: xo = XO_MOVSDto; break;
1320 case IRT_FLOAT: xo = XO_MOVSSto; break; 1481 case IRT_FLOAT: xo = XO_MOVSSto; break;
1321#if LJ_64 1482#if LJ_64 && !LJ_GC64
1322 case IRT_LIGHTUD: lua_assert(0); /* NYI: mask 64 bit lightuserdata. */ 1483 case IRT_LIGHTUD:
1484 /* NYI: mask 64 bit lightuserdata. */
1485 lj_assertA(0, "store of lightuserdata");
1323#endif 1486#endif
1324 default: 1487 default:
1325 if (LJ_64 && irt_is64(ir->t)) 1488 if (LJ_64 && irt_is64(ir->t))
1326 src |= REX_64; 1489 src |= REX_64;
1327 else 1490 else
1328 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); 1491 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t),
1492 "unsplit 64 bit store");
1329 xo = XO_MOVto; 1493 xo = XO_MOVto;
1330 break; 1494 break;
1331 } 1495 }
@@ -1339,15 +1503,18 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
1339 emit_i8(as, k); 1503 emit_i8(as, k);
1340 emit_mrm(as, XO_MOVmib, 0, RID_MRM); 1504 emit_mrm(as, XO_MOVmib, 0, RID_MRM);
1341 } else { 1505 } else {
1342 lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) || 1506 lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) ||
1343 irt_isaddr(ir->t)); 1507 irt_isaddr(ir->t), "bad store type");
1344 emit_i32(as, k); 1508 emit_i32(as, k);
1345 emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM); 1509 emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM);
1346 } 1510 }
1347 } 1511 }
1348} 1512}
1349 1513
1350#if LJ_64 1514#define asm_fstore(as, ir) asm_fxstore(as, ir)
1515#define asm_xstore(as, ir) asm_fxstore(as, ir)
1516
1517#if LJ_64 && !LJ_GC64
1351static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck) 1518static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
1352{ 1519{
1353 if (ra_used(ir) || typecheck) { 1520 if (ra_used(ir) || typecheck) {
@@ -1369,13 +1536,18 @@ static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
1369 1536
1370static void asm_ahuvload(ASMState *as, IRIns *ir) 1537static void asm_ahuvload(ASMState *as, IRIns *ir)
1371{ 1538{
1372 lua_assert(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) || 1539#if LJ_GC64
1373 (LJ_DUALNUM && irt_isint(ir->t))); 1540 Reg tmp = RID_NONE;
1374#if LJ_64 1541#endif
1542 lj_assertA(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
1543 (LJ_DUALNUM && irt_isint(ir->t)),
1544 "bad load type %d", irt_type(ir->t));
1545#if LJ_64 && !LJ_GC64
1375 if (irt_islightud(ir->t)) { 1546 if (irt_islightud(ir->t)) {
1376 Reg dest = asm_load_lightud64(as, ir, 1); 1547 Reg dest = asm_load_lightud64(as, ir, 1);
1377 if (ra_hasreg(dest)) { 1548 if (ra_hasreg(dest)) {
1378 asm_fuseahuref(as, ir->op1, RSET_GPR); 1549 asm_fuseahuref(as, ir->op1, RSET_GPR);
1550 if (ir->o == IR_VLOAD) as->mrm.ofs += 8 * ir->op2;
1379 emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM); 1551 emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM);
1380 } 1552 }
1381 return; 1553 return;
@@ -1385,20 +1557,67 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
1385 RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR; 1557 RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
1386 Reg dest = ra_dest(as, ir, allow); 1558 Reg dest = ra_dest(as, ir, allow);
1387 asm_fuseahuref(as, ir->op1, RSET_GPR); 1559 asm_fuseahuref(as, ir->op1, RSET_GPR);
1388 emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XMM_MOVRM(as), dest, RID_MRM); 1560 if (ir->o == IR_VLOAD) as->mrm.ofs += 8 * ir->op2;
1561#if LJ_GC64
1562 if (irt_isaddr(ir->t)) {
1563 emit_shifti(as, XOg_SHR|REX_64, dest, 17);
1564 asm_guardcc(as, CC_NE);
1565 emit_i8(as, irt_toitype(ir->t));
1566 emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
1567 emit_i8(as, XI_O16);
1568 if ((as->flags & JIT_F_BMI2)) {
1569 emit_i8(as, 47);
1570 emit_mrm(as, XV_RORX|VEX_64, dest, RID_MRM);
1571 } else {
1572 emit_shifti(as, XOg_ROR|REX_64, dest, 47);
1573 emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM);
1574 }
1575 return;
1576 } else
1577#endif
1578 emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XO_MOVSD, dest, RID_MRM);
1389 } else { 1579 } else {
1390 asm_fuseahuref(as, ir->op1, RSET_GPR); 1580 RegSet gpr = RSET_GPR;
1581#if LJ_GC64
1582 if (irt_isaddr(ir->t)) {
1583 tmp = ra_scratch(as, RSET_GPR);
1584 gpr = rset_exclude(gpr, tmp);
1585 }
1586#endif
1587 asm_fuseahuref(as, ir->op1, gpr);
1588 if (ir->o == IR_VLOAD) as->mrm.ofs += 8 * ir->op2;
1391 } 1589 }
1392 /* Always do the type check, even if the load result is unused. */ 1590 /* Always do the type check, even if the load result is unused. */
1393 as->mrm.ofs += 4; 1591 as->mrm.ofs += 4;
1394 asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE); 1592 asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE);
1395 if (LJ_64 && irt_type(ir->t) >= IRT_NUM) { 1593 if (LJ_64 && irt_type(ir->t) >= IRT_NUM) {
1396 lua_assert(irt_isinteger(ir->t) || irt_isnum(ir->t)); 1594 lj_assertA(irt_isinteger(ir->t) || irt_isnum(ir->t),
1595 "bad load type %d", irt_type(ir->t));
1596#if LJ_GC64
1597 emit_u32(as, LJ_TISNUM << 15);
1598#else
1397 emit_u32(as, LJ_TISNUM); 1599 emit_u32(as, LJ_TISNUM);
1600#endif
1601 emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
1602#if LJ_GC64
1603 } else if (irt_isaddr(ir->t)) {
1604 as->mrm.ofs -= 4;
1605 emit_i8(as, irt_toitype(ir->t));
1606 emit_mrm(as, XO_ARITHi8, XOg_CMP, tmp);
1607 emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
1608 emit_mrm(as, XO_MOV, tmp|REX_64, RID_MRM);
1609 } else if (irt_isnil(ir->t)) {
1610 as->mrm.ofs -= 4;
1611 emit_i8(as, -1);
1612 emit_mrm(as, XO_ARITHi8, XOg_CMP|REX_64, RID_MRM);
1613 } else {
1614 emit_u32(as, (irt_toitype(ir->t) << 15) | 0x7fff);
1398 emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM); 1615 emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
1616#else
1399 } else { 1617 } else {
1400 emit_i8(as, irt_toitype(ir->t)); 1618 emit_i8(as, irt_toitype(ir->t));
1401 emit_mrm(as, XO_ARITHi8, XOg_CMP, RID_MRM); 1619 emit_mrm(as, XO_ARITHi8, XOg_CMP, RID_MRM);
1620#endif
1402 } 1621 }
1403} 1622}
1404 1623
@@ -1410,12 +1629,28 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
1410 Reg src = ra_alloc1(as, ir->op2, RSET_FPR); 1629 Reg src = ra_alloc1(as, ir->op2, RSET_FPR);
1411 asm_fuseahuref(as, ir->op1, RSET_GPR); 1630 asm_fuseahuref(as, ir->op1, RSET_GPR);
1412 emit_mrm(as, XO_MOVSDto, src, RID_MRM); 1631 emit_mrm(as, XO_MOVSDto, src, RID_MRM);
1413#if LJ_64 1632#if LJ_64 && !LJ_GC64
1414 } else if (irt_islightud(ir->t)) { 1633 } else if (irt_islightud(ir->t)) {
1415 Reg src = ra_alloc1(as, ir->op2, RSET_GPR); 1634 Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
1416 asm_fuseahuref(as, ir->op1, rset_exclude(RSET_GPR, src)); 1635 asm_fuseahuref(as, ir->op1, rset_exclude(RSET_GPR, src));
1417 emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM); 1636 emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
1418#endif 1637#endif
1638#if LJ_GC64
1639 } else if (irref_isk(ir->op2)) {
1640 TValue k;
1641 lj_ir_kvalue(as->J->L, &k, IR(ir->op2));
1642 asm_fuseahuref(as, ir->op1, RSET_GPR);
1643 if (tvisnil(&k)) {
1644 emit_i32(as, -1);
1645 emit_mrm(as, XO_MOVmi, REX_64, RID_MRM);
1646 } else {
1647 emit_u32(as, k.u32.lo);
1648 emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1649 as->mrm.ofs += 4;
1650 emit_u32(as, k.u32.hi);
1651 emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1652 }
1653#endif
1419 } else { 1654 } else {
1420 IRIns *irr = IR(ir->op2); 1655 IRIns *irr = IR(ir->op2);
1421 RegSet allow = RSET_GPR; 1656 RegSet allow = RSET_GPR;
@@ -1426,34 +1661,56 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
1426 } 1661 }
1427 asm_fuseahuref(as, ir->op1, allow); 1662 asm_fuseahuref(as, ir->op1, allow);
1428 if (ra_hasreg(src)) { 1663 if (ra_hasreg(src)) {
1664#if LJ_GC64
1665 if (!(LJ_DUALNUM && irt_isinteger(ir->t))) {
1666 /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
1667 as->mrm.ofs += 4;
1668 emit_u32(as, irt_toitype(ir->t) << 15);
1669 emit_mrm(as, XO_ARITHi, XOg_OR, RID_MRM);
1670 as->mrm.ofs -= 4;
1671 emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
1672 return;
1673 }
1674#endif
1429 emit_mrm(as, XO_MOVto, src, RID_MRM); 1675 emit_mrm(as, XO_MOVto, src, RID_MRM);
1430 } else if (!irt_ispri(irr->t)) { 1676 } else if (!irt_ispri(irr->t)) {
1431 lua_assert(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t))); 1677 lj_assertA(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t)),
1678 "bad store type");
1432 emit_i32(as, irr->i); 1679 emit_i32(as, irr->i);
1433 emit_mrm(as, XO_MOVmi, 0, RID_MRM); 1680 emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1434 } 1681 }
1435 as->mrm.ofs += 4; 1682 as->mrm.ofs += 4;
1683#if LJ_GC64
1684 lj_assertA(LJ_DUALNUM && irt_isinteger(ir->t), "bad store type");
1685 emit_i32(as, LJ_TNUMX << 15);
1686#else
1436 emit_i32(as, (int32_t)irt_toitype(ir->t)); 1687 emit_i32(as, (int32_t)irt_toitype(ir->t));
1688#endif
1437 emit_mrm(as, XO_MOVmi, 0, RID_MRM); 1689 emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1438 } 1690 }
1439} 1691}
1440 1692
1441static void asm_sload(ASMState *as, IRIns *ir) 1693static void asm_sload(ASMState *as, IRIns *ir)
1442{ 1694{
1443 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0); 1695 int32_t ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
1696 (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1444 IRType1 t = ir->t; 1697 IRType1 t = ir->t;
1445 Reg base; 1698 Reg base;
1446 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ 1699 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1447 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK)); 1700 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1448 lua_assert(LJ_DUALNUM || 1701 lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1449 !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME))); 1702 "inconsistent SLOAD variant");
1703 lj_assertA(LJ_DUALNUM ||
1704 !irt_isint(t) ||
1705 (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME|IRSLOAD_KEYINDEX)),
1706 "bad SLOAD type");
1450 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { 1707 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1451 Reg left = ra_scratch(as, RSET_FPR); 1708 Reg left = ra_scratch(as, RSET_FPR);
1452 asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */ 1709 asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */
1453 base = ra_alloc1(as, REF_BASE, RSET_GPR); 1710 base = ra_alloc1(as, REF_BASE, RSET_GPR);
1454 emit_rmro(as, XMM_MOVRM(as), left, base, ofs); 1711 emit_rmro(as, XO_MOVSD, left, base, ofs);
1455 t.irt = IRT_NUM; /* Continue with a regular number type check. */ 1712 t.irt = IRT_NUM; /* Continue with a regular number type check. */
1456#if LJ_64 1713#if LJ_64 && !LJ_GC64
1457 } else if (irt_islightud(t)) { 1714 } else if (irt_islightud(t)) {
1458 Reg dest = asm_load_lightud64(as, ir, (ir->op2 & IRSLOAD_TYPECHECK)); 1715 Reg dest = asm_load_lightud64(as, ir, (ir->op2 & IRSLOAD_TYPECHECK));
1459 if (ra_hasreg(dest)) { 1716 if (ra_hasreg(dest)) {
@@ -1466,14 +1723,43 @@ static void asm_sload(ASMState *as, IRIns *ir)
1466 RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR; 1723 RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR;
1467 Reg dest = ra_dest(as, ir, allow); 1724 Reg dest = ra_dest(as, ir, allow);
1468 base = ra_alloc1(as, REF_BASE, RSET_GPR); 1725 base = ra_alloc1(as, REF_BASE, RSET_GPR);
1469 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1726 lj_assertA(irt_isnum(t) || irt_isint(t) || irt_isaddr(t),
1727 "bad SLOAD type %d", irt_type(t));
1470 if ((ir->op2 & IRSLOAD_CONVERT)) { 1728 if ((ir->op2 & IRSLOAD_CONVERT)) {
1471 t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */ 1729 t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */
1472 emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTSD2SI, dest, base, ofs); 1730 emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs);
1473 } else if (irt_isnum(t)) {
1474 emit_rmro(as, XMM_MOVRM(as), dest, base, ofs);
1475 } else { 1731 } else {
1476 emit_rmro(as, XO_MOV, dest, base, ofs); 1732#if LJ_GC64
1733 if (irt_isaddr(t)) {
1734 /* LJ_GC64 type check + tag removal without BMI2 and with BMI2:
1735 **
1736 ** mov r64, [addr] rorx r64, [addr], 47
1737 ** ror r64, 47
1738 ** cmp r16, itype cmp r16, itype
1739 ** jne ->exit jne ->exit
1740 ** shr r64, 16 shr r64, 16
1741 */
1742 emit_shifti(as, XOg_SHR|REX_64, dest, 17);
1743 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1744 asm_guardcc(as, CC_NE);
1745 emit_i8(as, irt_toitype(t));
1746 emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
1747 emit_i8(as, XI_O16);
1748 }
1749 if ((as->flags & JIT_F_BMI2)) {
1750 emit_i8(as, 47);
1751 emit_rmro(as, XV_RORX|VEX_64, dest, base, ofs);
1752 } else {
1753 if ((ir->op2 & IRSLOAD_TYPECHECK))
1754 emit_shifti(as, XOg_ROR|REX_64, dest, 47);
1755 else
1756 emit_shifti(as, XOg_SHL|REX_64, dest, 17);
1757 emit_rmro(as, XO_MOV, dest|REX_64, base, ofs);
1758 }
1759 return;
1760 } else
1761#endif
1762 emit_rmro(as, irt_isnum(t) ? XO_MOVSD : XO_MOV, dest, base, ofs);
1477 } 1763 }
1478 } else { 1764 } else {
1479 if (!(ir->op2 & IRSLOAD_TYPECHECK)) 1765 if (!(ir->op2 & IRSLOAD_TYPECHECK))
@@ -1483,13 +1769,42 @@ static void asm_sload(ASMState *as, IRIns *ir)
1483 if ((ir->op2 & IRSLOAD_TYPECHECK)) { 1769 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1484 /* Need type check, even if the load result is unused. */ 1770 /* Need type check, even if the load result is unused. */
1485 asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE); 1771 asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE);
1486 if (LJ_64 && irt_type(t) >= IRT_NUM) { 1772 if ((LJ_64 && irt_type(t) >= IRT_NUM) || (ir->op2 & IRSLOAD_KEYINDEX)) {
1487 lua_assert(irt_isinteger(t) || irt_isnum(t)); 1773 lj_assertA(irt_isinteger(t) || irt_isnum(t),
1488 emit_u32(as, LJ_TISNUM); 1774 "bad SLOAD type %d", irt_type(t));
1775 emit_u32(as, (ir->op2 & IRSLOAD_KEYINDEX) ? LJ_KEYINDEX :
1776 LJ_GC64 ? (LJ_TISNUM << 15) : LJ_TISNUM);
1489 emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4); 1777 emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
1778#if LJ_GC64
1779 } else if (irt_isnil(t)) {
1780 /* LJ_GC64 type check for nil:
1781 **
1782 ** cmp qword [addr], -1
1783 ** jne ->exit
1784 */
1785 emit_i8(as, -1);
1786 emit_rmro(as, XO_ARITHi8, XOg_CMP|REX_64, base, ofs);
1787 } else if (irt_ispri(t)) {
1788 emit_u32(as, (irt_toitype(t) << 15) | 0x7fff);
1789 emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
1790 } else {
1791 /* LJ_GC64 type check only:
1792 **
1793 ** mov r64, [addr]
1794 ** sar r64, 47
1795 ** cmp r32, itype
1796 ** jne ->exit
1797 */
1798 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, base));
1799 emit_i8(as, irt_toitype(t));
1800 emit_rr(as, XO_ARITHi8, XOg_CMP, tmp);
1801 emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
1802 emit_rmro(as, XO_MOV, tmp|REX_64, base, ofs);
1803#else
1490 } else { 1804 } else {
1491 emit_i8(as, irt_toitype(t)); 1805 emit_i8(as, irt_toitype(t));
1492 emit_rmro(as, XO_ARITHi8, XOg_CMP, base, ofs+4); 1806 emit_rmro(as, XO_ARITHi8, XOg_CMP, base, ofs+4);
1807#endif
1493 } 1808 }
1494 } 1809 }
1495} 1810}
@@ -1500,15 +1815,14 @@ static void asm_sload(ASMState *as, IRIns *ir)
1500static void asm_cnew(ASMState *as, IRIns *ir) 1815static void asm_cnew(ASMState *as, IRIns *ir)
1501{ 1816{
1502 CTState *cts = ctype_ctsG(J2G(as->J)); 1817 CTState *cts = ctype_ctsG(J2G(as->J));
1503 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i; 1818 CTypeID id = (CTypeID)IR(ir->op1)->i;
1504 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ? 1819 CTSize sz;
1505 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i; 1820 CTInfo info = lj_ctype_info(cts, id, &sz);
1506 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; 1821 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1507 IRRef args[2]; 1822 IRRef args[4];
1508 lua_assert(sz != CTSIZE_INVALID); 1823 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1824 "bad CNEW/CNEWI operands");
1509 1825
1510 args[0] = ASMREF_L; /* lua_State *L */
1511 args[1] = ASMREF_TMP1; /* MSize size */
1512 as->gcsteps++; 1826 as->gcsteps++;
1513 asm_setupresult(as, ir, ci); /* GCcdata * */ 1827 asm_setupresult(as, ir, ci); /* GCcdata * */
1514 1828
@@ -1519,8 +1833,9 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1519 Reg r64 = sz == 8 ? REX_64 : 0; 1833 Reg r64 = sz == 8 ? REX_64 : 0;
1520 if (irref_isk(ir->op2)) { 1834 if (irref_isk(ir->op2)) {
1521 IRIns *irk = IR(ir->op2); 1835 IRIns *irk = IR(ir->op2);
1522 uint64_t k = irk->o == IR_KINT64 ? ir_k64(irk)->u64 : 1836 uint64_t k = (irk->o == IR_KINT64 ||
1523 (uint64_t)(uint32_t)irk->i; 1837 (LJ_GC64 && (irk->o == IR_KPTR || irk->o == IR_KKPTR))) ?
1838 ir_k64(irk)->u64 : (uint64_t)(uint32_t)irk->i;
1524 if (sz == 4 || checki32((int64_t)k)) { 1839 if (sz == 4 || checki32((int64_t)k)) {
1525 emit_i32(as, (int32_t)k); 1840 emit_i32(as, (int32_t)k);
1526 emit_rmro(as, XO_MOVmi, r64, RID_RET, sizeof(GCcdata)); 1841 emit_rmro(as, XO_MOVmi, r64, RID_RET, sizeof(GCcdata));
@@ -1536,7 +1851,7 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1536 int32_t ofs = sizeof(GCcdata); 1851 int32_t ofs = sizeof(GCcdata);
1537 if (sz == 8) { 1852 if (sz == 8) {
1538 ofs += 4; ir++; 1853 ofs += 4; ir++;
1539 lua_assert(ir->o == IR_HIOP); 1854 lj_assertA(ir->o == IR_HIOP, "missing CNEWI HIOP");
1540 } 1855 }
1541 do { 1856 do {
1542 if (irref_isk(ir->op2)) { 1857 if (irref_isk(ir->op2)) {
@@ -1550,21 +1865,30 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1550 ofs -= 4; ir--; 1865 ofs -= 4; ir--;
1551 } while (1); 1866 } while (1);
1552#endif 1867#endif
1553 lua_assert(sz == 4 || sz == 8); 1868 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1869 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1870 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1871 args[0] = ASMREF_L; /* lua_State *L */
1872 args[1] = ir->op1; /* CTypeID id */
1873 args[2] = ir->op2; /* CTSize sz */
1874 args[3] = ASMREF_TMP1; /* CTSize align */
1875 asm_gencall(as, ci, args);
1876 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1877 return;
1554 } 1878 }
1555 1879
1556 /* Combine initialization of marked, gct and ctypeid. */ 1880 /* Combine initialization of marked, gct and ctypeid. */
1557 emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked)); 1881 emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked));
1558 emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX, 1882 emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX,
1559 (int32_t)((~LJ_TCDATA<<8)+(ctypeid<<16))); 1883 (int32_t)((~LJ_TCDATA<<8)+(id<<16)));
1560 emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES); 1884 emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES);
1561 emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite); 1885 emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite);
1562 1886
1887 args[0] = ASMREF_L; /* lua_State *L */
1888 args[1] = ASMREF_TMP1; /* MSize size */
1563 asm_gencall(as, ci, args); 1889 asm_gencall(as, ci, args);
1564 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata))); 1890 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata)));
1565} 1891}
1566#else
1567#define asm_cnew(as, ir) ((void)0)
1568#endif 1892#endif
1569 1893
1570/* -- Write barriers ------------------------------------------------------ */ 1894/* -- Write barriers ------------------------------------------------------ */
@@ -1574,7 +1898,7 @@ static void asm_tbar(ASMState *as, IRIns *ir)
1574 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR); 1898 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1575 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab)); 1899 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1576 MCLabel l_end = emit_label(as); 1900 MCLabel l_end = emit_label(as);
1577 emit_movtomro(as, tmp, tab, offsetof(GCtab, gclist)); 1901 emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist));
1578 emit_setgl(as, tab, gc.grayagain); 1902 emit_setgl(as, tab, gc.grayagain);
1579 emit_getgl(as, tmp, gc.grayagain); 1903 emit_getgl(as, tmp, gc.grayagain);
1580 emit_i8(as, ~LJ_GC_BLACK); 1904 emit_i8(as, ~LJ_GC_BLACK);
@@ -1591,7 +1915,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1591 MCLabel l_end; 1915 MCLabel l_end;
1592 Reg obj; 1916 Reg obj;
1593 /* No need for other object barriers (yet). */ 1917 /* No need for other object barriers (yet). */
1594 lua_assert(IR(ir->op1)->o == IR_UREFC); 1918 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1595 ra_evictset(as, RSET_SCRATCH); 1919 ra_evictset(as, RSET_SCRATCH);
1596 l_end = emit_label(as); 1920 l_end = emit_label(as);
1597 args[0] = ASMREF_TMP1; /* global_State *g */ 1921 args[0] = ASMREF_TMP1; /* global_State *g */
@@ -1637,36 +1961,9 @@ static void asm_x87load(ASMState *as, IRRef ref)
1637 } 1961 }
1638} 1962}
1639 1963
1640/* Try to rejoin pow from EXP2, MUL and LOG2 (if still unsplit). */
1641static int fpmjoin_pow(ASMState *as, IRIns *ir)
1642{
1643 IRIns *irp = IR(ir->op1);
1644 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
1645 IRIns *irpp = IR(irp->op1);
1646 if (irpp == ir-2 && irpp->o == IR_FPMATH &&
1647 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
1648 /* The modified regs must match with the *.dasc implementation. */
1649 RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM2+1)|RID2RSET(RID_EAX);
1650 IRIns *irx;
1651 if (ra_hasreg(ir->r))
1652 rset_clear(drop, ir->r); /* Dest reg handled below. */
1653 ra_evictset(as, drop);
1654 ra_destreg(as, ir, RID_XMM0);
1655 emit_call(as, lj_vm_pow_sse);
1656 irx = IR(irpp->op1);
1657 if (ra_noreg(irx->r) && ra_gethint(irx->r) == RID_XMM1)
1658 irx->r = RID_INIT; /* Avoid allocating xmm1 for x. */
1659 ra_left(as, RID_XMM0, irpp->op1);
1660 ra_left(as, RID_XMM1, irp->op2);
1661 return 1;
1662 }
1663 }
1664 return 0;
1665}
1666
1667static void asm_fpmath(ASMState *as, IRIns *ir) 1964static void asm_fpmath(ASMState *as, IRIns *ir)
1668{ 1965{
1669 IRFPMathOp fpm = ir->o == IR_FPMATH ? (IRFPMathOp)ir->op2 : IRFPM_OTHER; 1966 IRFPMathOp fpm = (IRFPMathOp)ir->op2;
1670 if (fpm == IRFPM_SQRT) { 1967 if (fpm == IRFPM_SQRT) {
1671 Reg dest = ra_dest(as, ir, RSET_FPR); 1968 Reg dest = ra_dest(as, ir, RSET_FPR);
1672 Reg left = asm_fuseload(as, ir->op1, RSET_FPR); 1969 Reg left = asm_fuseload(as, ir->op1, RSET_FPR);
@@ -1697,93 +1994,32 @@ static void asm_fpmath(ASMState *as, IRIns *ir)
1697 fpm == IRFPM_CEIL ? lj_vm_ceil_sse : lj_vm_trunc_sse); 1994 fpm == IRFPM_CEIL ? lj_vm_ceil_sse : lj_vm_trunc_sse);
1698 ra_left(as, RID_XMM0, ir->op1); 1995 ra_left(as, RID_XMM0, ir->op1);
1699 } 1996 }
1700 } else if (fpm == IRFPM_EXP2 && fpmjoin_pow(as, ir)) { 1997 } else {
1701 /* Rejoined to pow(). */ 1998 asm_callid(as, ir, IRCALL_lj_vm_floor + fpm);
1702 } else { /* Handle x87 ops. */
1703 int32_t ofs = sps_scale(ir->s); /* Use spill slot or temp slots. */
1704 Reg dest = ir->r;
1705 if (ra_hasreg(dest)) {
1706 ra_free(as, dest);
1707 ra_modified(as, dest);
1708 emit_rmro(as, XMM_MOVRM(as), dest, RID_ESP, ofs);
1709 }
1710 emit_rmro(as, XO_FSTPq, XOg_FSTPq, RID_ESP, ofs);
1711 switch (fpm) { /* st0 = lj_vm_*(st0) */
1712 case IRFPM_EXP: emit_call(as, lj_vm_exp_x87); break;
1713 case IRFPM_EXP2: emit_call(as, lj_vm_exp2_x87); break;
1714 case IRFPM_SIN: emit_x87op(as, XI_FSIN); break;
1715 case IRFPM_COS: emit_x87op(as, XI_FCOS); break;
1716 case IRFPM_TAN: emit_x87op(as, XI_FPOP); emit_x87op(as, XI_FPTAN); break;
1717 case IRFPM_LOG: case IRFPM_LOG2: case IRFPM_LOG10:
1718 /* Note: the use of fyl2xp1 would be pointless here. When computing
1719 ** log(1.0+eps) the precision is already lost after 1.0 is added.
1720 ** Subtracting 1.0 won't recover it. OTOH math.log1p would make sense.
1721 */
1722 emit_x87op(as, XI_FYL2X); break;
1723 case IRFPM_OTHER:
1724 switch (ir->o) {
1725 case IR_ATAN2:
1726 emit_x87op(as, XI_FPATAN); asm_x87load(as, ir->op2); break;
1727 case IR_LDEXP:
1728 emit_x87op(as, XI_FPOP1); emit_x87op(as, XI_FSCALE); break;
1729 default: lua_assert(0); break;
1730 }
1731 break;
1732 default: lua_assert(0); break;
1733 }
1734 asm_x87load(as, ir->op1);
1735 switch (fpm) {
1736 case IRFPM_LOG: emit_x87op(as, XI_FLDLN2); break;
1737 case IRFPM_LOG2: emit_x87op(as, XI_FLD1); break;
1738 case IRFPM_LOG10: emit_x87op(as, XI_FLDLG2); break;
1739 case IRFPM_OTHER:
1740 if (ir->o == IR_LDEXP) asm_x87load(as, ir->op2);
1741 break;
1742 default: break;
1743 }
1744 } 1999 }
1745} 2000}
1746 2001
1747static void asm_fppowi(ASMState *as, IRIns *ir) 2002static void asm_ldexp(ASMState *as, IRIns *ir)
1748{
1749 /* The modified regs must match with the *.dasc implementation. */
1750 RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX);
1751 if (ra_hasreg(ir->r))
1752 rset_clear(drop, ir->r); /* Dest reg handled below. */
1753 ra_evictset(as, drop);
1754 ra_destreg(as, ir, RID_XMM0);
1755 emit_call(as, lj_vm_powi_sse);
1756 ra_left(as, RID_XMM0, ir->op1);
1757 ra_left(as, RID_EAX, ir->op2);
1758}
1759
1760#if LJ_64 && LJ_HASFFI
1761static void asm_arith64(ASMState *as, IRIns *ir, IRCallID id)
1762{
1763 const CCallInfo *ci = &lj_ir_callinfo[id];
1764 IRRef args[2];
1765 args[0] = ir->op1;
1766 args[1] = ir->op2;
1767 asm_setupresult(as, ir, ci);
1768 asm_gencall(as, ci, args);
1769}
1770#endif
1771
1772static void asm_intmod(ASMState *as, IRIns *ir)
1773{ 2003{
1774 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_vm_modi]; 2004 int32_t ofs = sps_scale(ir->s); /* Use spill slot or temp slots. */
1775 IRRef args[2]; 2005 Reg dest = ir->r;
1776 args[0] = ir->op1; 2006 if (ra_hasreg(dest)) {
1777 args[1] = ir->op2; 2007 ra_free(as, dest);
1778 asm_setupresult(as, ir, ci); 2008 ra_modified(as, dest);
1779 asm_gencall(as, ci, args); 2009 emit_rmro(as, XO_MOVSD, dest, RID_ESP, ofs);
2010 }
2011 emit_rmro(as, XO_FSTPq, XOg_FSTPq, RID_ESP, ofs);
2012 emit_x87op(as, XI_FPOP1);
2013 emit_x87op(as, XI_FSCALE);
2014 asm_x87load(as, ir->op1);
2015 asm_x87load(as, ir->op2);
1780} 2016}
1781 2017
1782static int asm_swapops(ASMState *as, IRIns *ir) 2018static int asm_swapops(ASMState *as, IRIns *ir)
1783{ 2019{
1784 IRIns *irl = IR(ir->op1); 2020 IRIns *irl = IR(ir->op1);
1785 IRIns *irr = IR(ir->op2); 2021 IRIns *irr = IR(ir->op2);
1786 lua_assert(ra_noreg(irr->r)); 2022 lj_assertA(ra_noreg(irr->r), "bad usage");
1787 if (!irm_iscomm(lj_ir_mode[ir->o])) 2023 if (!irm_iscomm(lj_ir_mode[ir->o]))
1788 return 0; /* Can't swap non-commutative operations. */ 2024 return 0; /* Can't swap non-commutative operations. */
1789 if (irref_isk(ir->op2)) 2025 if (irref_isk(ir->op2))
@@ -1955,11 +2191,28 @@ static void asm_add(ASMState *as, IRIns *ir)
1955{ 2191{
1956 if (irt_isnum(ir->t)) 2192 if (irt_isnum(ir->t))
1957 asm_fparith(as, ir, XO_ADDSD); 2193 asm_fparith(as, ir, XO_ADDSD);
1958 else if ((as->flags & JIT_F_LEA_AGU) || as->flagmcp == as->mcp || 2194 else if (as->flagmcp == as->mcp || irt_is64(ir->t) || !asm_lea(as, ir))
1959 irt_is64(ir->t) || !asm_lea(as, ir))
1960 asm_intarith(as, ir, XOg_ADD); 2195 asm_intarith(as, ir, XOg_ADD);
1961} 2196}
1962 2197
2198static void asm_sub(ASMState *as, IRIns *ir)
2199{
2200 if (irt_isnum(ir->t))
2201 asm_fparith(as, ir, XO_SUBSD);
2202 else /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
2203 asm_intarith(as, ir, XOg_SUB);
2204}
2205
2206static void asm_mul(ASMState *as, IRIns *ir)
2207{
2208 if (irt_isnum(ir->t))
2209 asm_fparith(as, ir, XO_MULSD);
2210 else
2211 asm_intarith(as, ir, XOg_X_IMUL);
2212}
2213
2214#define asm_fpdiv(as, ir) asm_fparith(as, ir, XO_DIVSD)
2215
1963static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg) 2216static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg)
1964{ 2217{
1965 Reg dest = ra_dest(as, ir, RSET_GPR); 2218 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1967,7 +2220,17 @@ static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg)
1967 ra_left(as, dest, ir->op1); 2220 ra_left(as, dest, ir->op1);
1968} 2221}
1969 2222
1970static void asm_min_max(ASMState *as, IRIns *ir, int cc) 2223static void asm_neg(ASMState *as, IRIns *ir)
2224{
2225 if (irt_isnum(ir->t))
2226 asm_fparith(as, ir, XO_XORPS);
2227 else
2228 asm_neg_not(as, ir, XOg_NEG);
2229}
2230
2231#define asm_abs(as, ir) asm_fparith(as, ir, XO_ANDPS)
2232
2233static void asm_intmin_max(ASMState *as, IRIns *ir, int cc)
1971{ 2234{
1972 Reg right, dest = ra_dest(as, ir, RSET_GPR); 2235 Reg right, dest = ra_dest(as, ir, RSET_GPR);
1973 IRRef lref = ir->op1, rref = ir->op2; 2236 IRRef lref = ir->op1, rref = ir->op2;
@@ -1978,7 +2241,30 @@ static void asm_min_max(ASMState *as, IRIns *ir, int cc)
1978 ra_left(as, dest, lref); 2241 ra_left(as, dest, lref);
1979} 2242}
1980 2243
1981static void asm_bitswap(ASMState *as, IRIns *ir) 2244static void asm_min(ASMState *as, IRIns *ir)
2245{
2246 if (irt_isnum(ir->t))
2247 asm_fparith(as, ir, XO_MINSD);
2248 else
2249 asm_intmin_max(as, ir, CC_G);
2250}
2251
2252static void asm_max(ASMState *as, IRIns *ir)
2253{
2254 if (irt_isnum(ir->t))
2255 asm_fparith(as, ir, XO_MAXSD);
2256 else
2257 asm_intmin_max(as, ir, CC_L);
2258}
2259
2260/* Note: don't use LEA for overflow-checking arithmetic! */
2261#define asm_addov(as, ir) asm_intarith(as, ir, XOg_ADD)
2262#define asm_subov(as, ir) asm_intarith(as, ir, XOg_SUB)
2263#define asm_mulov(as, ir) asm_intarith(as, ir, XOg_X_IMUL)
2264
2265#define asm_bnot(as, ir) asm_neg_not(as, ir, XOg_NOT)
2266
2267static void asm_bswap(ASMState *as, IRIns *ir)
1982{ 2268{
1983 Reg dest = ra_dest(as, ir, RSET_GPR); 2269 Reg dest = ra_dest(as, ir, RSET_GPR);
1984 as->mcp = emit_op(XO_BSWAP + ((dest&7) << 24), 2270 as->mcp = emit_op(XO_BSWAP + ((dest&7) << 24),
@@ -1986,7 +2272,11 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
1986 ra_left(as, dest, ir->op1); 2272 ra_left(as, dest, ir->op1);
1987} 2273}
1988 2274
1989static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs) 2275#define asm_band(as, ir) asm_intarith(as, ir, XOg_AND)
2276#define asm_bor(as, ir) asm_intarith(as, ir, XOg_OR)
2277#define asm_bxor(as, ir) asm_intarith(as, ir, XOg_XOR)
2278
2279static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs, x86Op xv)
1990{ 2280{
1991 IRRef rref = ir->op2; 2281 IRRef rref = ir->op2;
1992 IRIns *irr = IR(rref); 2282 IRIns *irr = IR(rref);
@@ -1995,17 +2285,33 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
1995 int shift; 2285 int shift;
1996 dest = ra_dest(as, ir, RSET_GPR); 2286 dest = ra_dest(as, ir, RSET_GPR);
1997 shift = irr->i & (irt_is64(ir->t) ? 63 : 31); 2287 shift = irr->i & (irt_is64(ir->t) ? 63 : 31);
2288 if (!xv && shift && (as->flags & JIT_F_BMI2)) {
2289 Reg left = asm_fuseloadm(as, ir->op1, RSET_GPR, irt_is64(ir->t));
2290 if (left != dest) { /* BMI2 rotate right by constant. */
2291 emit_i8(as, xs == XOg_ROL ? -shift : shift);
2292 emit_mrm(as, VEX_64IR(ir, XV_RORX), dest, left);
2293 return;
2294 }
2295 }
1998 switch (shift) { 2296 switch (shift) {
1999 case 0: break; 2297 case 0: break;
2000 case 1: emit_rr(as, XO_SHIFT1, REX_64IR(ir, xs), dest); break; 2298 case 1: emit_rr(as, XO_SHIFT1, REX_64IR(ir, xs), dest); break;
2001 default: emit_shifti(as, REX_64IR(ir, xs), dest, shift); break; 2299 default: emit_shifti(as, REX_64IR(ir, xs), dest, shift); break;
2002 } 2300 }
2301 } else if ((as->flags & JIT_F_BMI2) && xv) { /* BMI2 variable shifts. */
2302 Reg left, right;
2303 dest = ra_dest(as, ir, RSET_GPR);
2304 right = ra_alloc1(as, rref, RSET_GPR);
2305 left = asm_fuseloadm(as, ir->op1, rset_exclude(RSET_GPR, right),
2306 irt_is64(ir->t));
2307 emit_mrm(as, VEX_64IR(ir, xv) ^ (right << 19), dest, left);
2308 return;
2003 } else { /* Variable shifts implicitly use register cl (i.e. ecx). */ 2309 } else { /* Variable shifts implicitly use register cl (i.e. ecx). */
2004 Reg right; 2310 Reg right;
2005 dest = ra_dest(as, ir, rset_exclude(RSET_GPR, RID_ECX)); 2311 dest = ra_dest(as, ir, rset_exclude(RSET_GPR, RID_ECX));
2006 if (dest == RID_ECX) { 2312 if (dest == RID_ECX) {
2007 dest = ra_scratch(as, rset_exclude(RSET_GPR, RID_ECX)); 2313 dest = ra_scratch(as, rset_exclude(RSET_GPR, RID_ECX));
2008 emit_rr(as, XO_MOV, RID_ECX, dest); 2314 emit_rr(as, XO_MOV, REX_64IR(ir, RID_ECX), dest);
2009 } 2315 }
2010 right = irr->r; 2316 right = irr->r;
2011 if (ra_noreg(right)) 2317 if (ra_noreg(right))
@@ -2025,6 +2331,12 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
2025 */ 2331 */
2026} 2332}
2027 2333
2334#define asm_bshl(as, ir) asm_bitshift(as, ir, XOg_SHL, XV_SHLX)
2335#define asm_bshr(as, ir) asm_bitshift(as, ir, XOg_SHR, XV_SHRX)
2336#define asm_bsar(as, ir) asm_bitshift(as, ir, XOg_SAR, XV_SARX)
2337#define asm_brol(as, ir) asm_bitshift(as, ir, XOg_ROL, 0)
2338#define asm_bror(as, ir) asm_bitshift(as, ir, XOg_ROR, 0)
2339
2028/* -- Comparisons --------------------------------------------------------- */ 2340/* -- Comparisons --------------------------------------------------------- */
2029 2341
2030/* Virtual flags for unordered FP comparisons. */ 2342/* Virtual flags for unordered FP comparisons. */
@@ -2051,8 +2363,9 @@ static const uint16_t asm_compmap[IR_ABC+1] = {
2051}; 2363};
2052 2364
2053/* FP and integer comparisons. */ 2365/* FP and integer comparisons. */
2054static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc) 2366static void asm_comp(ASMState *as, IRIns *ir)
2055{ 2367{
2368 uint32_t cc = asm_compmap[ir->o];
2056 if (irt_isnum(ir->t)) { 2369 if (irt_isnum(ir->t)) {
2057 IRRef lref = ir->op1; 2370 IRRef lref = ir->op1;
2058 IRRef rref = ir->op2; 2371 IRRef rref = ir->op2;
@@ -2073,7 +2386,6 @@ static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
2073 cc ^= (VCC_PS|(5<<4)); /* A <-> B, AE <-> BE, PS <-> none */ 2386 cc ^= (VCC_PS|(5<<4)); /* A <-> B, AE <-> BE, PS <-> none */
2074 } 2387 }
2075 left = ra_alloc1(as, lref, RSET_FPR); 2388 left = ra_alloc1(as, lref, RSET_FPR);
2076 right = asm_fuseload(as, rref, rset_exclude(RSET_FPR, left));
2077 l_around = emit_label(as); 2389 l_around = emit_label(as);
2078 asm_guardcc(as, cc >> 4); 2390 asm_guardcc(as, cc >> 4);
2079 if (cc & VCC_P) { /* Extra CC_P branch required? */ 2391 if (cc & VCC_P) { /* Extra CC_P branch required? */
@@ -2090,14 +2402,16 @@ static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
2090 emit_jcc(as, CC_P, as->mcp); 2402 emit_jcc(as, CC_P, as->mcp);
2091 } 2403 }
2092 } 2404 }
2405 right = asm_fuseload(as, rref, rset_exclude(RSET_FPR, left));
2093 emit_mrm(as, XO_UCOMISD, left, right); 2406 emit_mrm(as, XO_UCOMISD, left, right);
2094 } else { 2407 } else {
2095 IRRef lref = ir->op1, rref = ir->op2; 2408 IRRef lref = ir->op1, rref = ir->op2;
2096 IROp leftop = (IROp)(IR(lref)->o); 2409 IROp leftop = (IROp)(IR(lref)->o);
2097 Reg r64 = REX_64IR(ir, 0); 2410 Reg r64 = REX_64IR(ir, 0);
2098 int32_t imm = 0; 2411 int32_t imm = 0;
2099 lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || 2412 lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) ||
2100 irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t)); 2413 irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t),
2414 "bad comparison data type %d", irt_type(ir->t));
2101 /* Swap constants (only for ABC) and fusable loads to the right. */ 2415 /* Swap constants (only for ABC) and fusable loads to the right. */
2102 if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) { 2416 if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) {
2103 if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */ 2417 if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */
@@ -2179,7 +2493,7 @@ static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
2179 /* Use test r,r instead of cmp r,0. */ 2493 /* Use test r,r instead of cmp r,0. */
2180 x86Op xo = XO_TEST; 2494 x86Op xo = XO_TEST;
2181 if (irt_isu8(ir->t)) { 2495 if (irt_isu8(ir->t)) {
2182 lua_assert(ir->o == IR_EQ || ir->o == IR_NE); 2496 lj_assertA(ir->o == IR_EQ || ir->o == IR_NE, "bad usage");
2183 xo = XO_TESTb; 2497 xo = XO_TESTb;
2184 if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) { 2498 if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) {
2185 if (LJ_64) { 2499 if (LJ_64) {
@@ -2207,6 +2521,8 @@ static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
2207 } 2521 }
2208} 2522}
2209 2523
2524#define asm_equal(as, ir) asm_comp(as, ir)
2525
2210#if LJ_32 && LJ_HASFFI 2526#if LJ_32 && LJ_HASFFI
2211/* 64 bit integer comparisons in 32 bit mode. */ 2527/* 64 bit integer comparisons in 32 bit mode. */
2212static void asm_comp_int64(ASMState *as, IRIns *ir) 2528static void asm_comp_int64(ASMState *as, IRIns *ir)
@@ -2279,23 +2595,19 @@ static void asm_comp_int64(ASMState *as, IRIns *ir)
2279} 2595}
2280#endif 2596#endif
2281 2597
2282/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */ 2598/* -- Split register ops -------------------------------------------------- */
2283 2599
2284/* Hiword op of a split 64 bit op. Previous op must be the loword op. */ 2600/* Hiword op of a split 32/32 or 64/64 bit op. Previous op is the loword op. */
2285static void asm_hiop(ASMState *as, IRIns *ir) 2601static void asm_hiop(ASMState *as, IRIns *ir)
2286{ 2602{
2287#if LJ_32 && LJ_HASFFI
2288 /* HIOP is marked as a store because it needs its own DCE logic. */ 2603 /* HIOP is marked as a store because it needs its own DCE logic. */
2289 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */ 2604 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
2290 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1; 2605 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
2606#if LJ_32 && LJ_HASFFI
2291 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */ 2607 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
2292 if (usehi || uselo) {
2293 if (irt_isfp(ir->t))
2294 asm_conv_fp_int64(as, ir);
2295 else
2296 asm_conv_int64_fp(as, ir);
2297 }
2298 as->curins--; /* Always skip the CONV. */ 2608 as->curins--; /* Always skip the CONV. */
2609 if (usehi || uselo)
2610 asm_conv64(as, ir);
2299 return; 2611 return;
2300 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */ 2612 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
2301 asm_comp_int64(as, ir); 2613 asm_comp_int64(as, ir);
@@ -2305,8 +2617,10 @@ static void asm_hiop(ASMState *as, IRIns *ir)
2305 asm_fxstore(as, ir); 2617 asm_fxstore(as, ir);
2306 return; 2618 return;
2307 } 2619 }
2620#endif
2308 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */ 2621 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
2309 switch ((ir-1)->o) { 2622 switch ((ir-1)->o) {
2623#if LJ_32 && LJ_HASFFI
2310 case IR_ADD: 2624 case IR_ADD:
2311 as->flagmcp = NULL; 2625 as->flagmcp = NULL;
2312 as->curins--; 2626 as->curins--;
@@ -2329,19 +2643,26 @@ static void asm_hiop(ASMState *as, IRIns *ir)
2329 asm_neg_not(as, ir-1, XOg_NEG); 2643 asm_neg_not(as, ir-1, XOg_NEG);
2330 break; 2644 break;
2331 } 2645 }
2332 case IR_CALLN:
2333 case IR_CALLXS:
2334 if (!uselo)
2335 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
2336 break;
2337 case IR_CNEWI: 2646 case IR_CNEWI:
2338 /* Nothing to do here. Handled by CNEWI itself. */ 2647 /* Nothing to do here. Handled by CNEWI itself. */
2339 break; 2648 break;
2340 default: lua_assert(0); break;
2341 }
2342#else
2343 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused on x64 or without FFI. */
2344#endif 2649#endif
2650 case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
2651 if (!uselo)
2652 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
2653 break;
2654 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
2655 }
2656}
2657
2658/* -- Profiling ----------------------------------------------------------- */
2659
2660static void asm_prof(ASMState *as, IRIns *ir)
2661{
2662 UNUSED(ir);
2663 asm_guardcc(as, CC_NE);
2664 emit_i8(as, HOOK_PROFILE);
2665 emit_rma(as, XO_GROUP3b, XOg_TEST, &J2G(as->J)->hookmask);
2345} 2666}
2346 2667
2347/* -- Stack handling ------------------------------------------------------ */ 2668/* -- Stack handling ------------------------------------------------------ */
@@ -2358,14 +2679,19 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
2358 emit_rmro(as, XO_MOV, r|REX_64, RID_ESP, 0); 2679 emit_rmro(as, XO_MOV, r|REX_64, RID_ESP, 0);
2359 else 2680 else
2360 ra_modified(as, r); 2681 ra_modified(as, r);
2361 emit_gri(as, XG_ARITHi(XOg_CMP), r, (int32_t)(8*topslot)); 2682 emit_gri(as, XG_ARITHi(XOg_CMP), r|REX_GC64, (int32_t)(8*topslot));
2362 if (ra_hasreg(pbase) && pbase != r) 2683 if (ra_hasreg(pbase) && pbase != r)
2363 emit_rr(as, XO_ARITH(XOg_SUB), r, pbase); 2684 emit_rr(as, XO_ARITH(XOg_SUB), r|REX_GC64, pbase);
2364 else 2685 else
2686#if LJ_GC64
2687 emit_rmro(as, XO_ARITH(XOg_SUB), r|REX_64, RID_DISPATCH,
2688 (int32_t)dispofs(as, &J2G(as->J)->jit_base));
2689#else
2365 emit_rmro(as, XO_ARITH(XOg_SUB), r, RID_NONE, 2690 emit_rmro(as, XO_ARITH(XOg_SUB), r, RID_NONE,
2366 ptr2addr(&J2G(as->J)->jit_base)); 2691 ptr2addr(&J2G(as->J)->jit_base));
2367 emit_rmro(as, XO_MOV, r, r, offsetof(lua_State, maxstack)); 2692#endif
2368 emit_getgl(as, r, jit_L); 2693 emit_rmro(as, XO_MOV, r|REX_GC64, r, offsetof(lua_State, maxstack));
2694 emit_getgl(as, r, cur_L);
2369 if (allow == RSET_EMPTY) /* Spill temp. register. */ 2695 if (allow == RSET_EMPTY) /* Spill temp. register. */
2370 emit_rmro(as, XO_MOVto, r|REX_64, RID_ESP, 0); 2696 emit_rmro(as, XO_MOVto, r|REX_64, RID_ESP, 0);
2371} 2697}
@@ -2374,40 +2700,79 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
2374static void asm_stack_restore(ASMState *as, SnapShot *snap) 2700static void asm_stack_restore(ASMState *as, SnapShot *snap)
2375{ 2701{
2376 SnapEntry *map = &as->T->snapmap[snap->mapofs]; 2702 SnapEntry *map = &as->T->snapmap[snap->mapofs];
2377 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1]; 2703#if !LJ_FR2 || defined(LUA_USE_ASSERT)
2704 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
2705#endif
2378 MSize n, nent = snap->nent; 2706 MSize n, nent = snap->nent;
2379 /* Store the value of all modified slots to the Lua stack. */ 2707 /* Store the value of all modified slots to the Lua stack. */
2380 for (n = 0; n < nent; n++) { 2708 for (n = 0; n < nent; n++) {
2381 SnapEntry sn = map[n]; 2709 SnapEntry sn = map[n];
2382 BCReg s = snap_slot(sn); 2710 BCReg s = snap_slot(sn);
2383 int32_t ofs = 8*((int32_t)s-1); 2711 int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
2384 IRRef ref = snap_ref(sn); 2712 IRRef ref = snap_ref(sn);
2385 IRIns *ir = IR(ref); 2713 IRIns *ir = IR(ref);
2386 if ((sn & SNAP_NORESTORE)) 2714 if ((sn & SNAP_NORESTORE))
2387 continue; 2715 continue;
2388 if (irt_isnum(ir->t)) { 2716 if ((sn & SNAP_KEYINDEX)) {
2717 emit_movmroi(as, RID_BASE, ofs+4, LJ_KEYINDEX);
2718 if (irref_isk(ref)) {
2719 emit_movmroi(as, RID_BASE, ofs, ir->i);
2720 } else {
2721 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2722 emit_movtomro(as, src, RID_BASE, ofs);
2723 }
2724 } else if (irt_isnum(ir->t)) {
2389 Reg src = ra_alloc1(as, ref, RSET_FPR); 2725 Reg src = ra_alloc1(as, ref, RSET_FPR);
2390 emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs); 2726 emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs);
2391 } else { 2727 } else {
2392 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || 2728 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) ||
2393 (LJ_DUALNUM && irt_isinteger(ir->t))); 2729 (LJ_DUALNUM && irt_isinteger(ir->t)),
2730 "restore of IR type %d", irt_type(ir->t));
2394 if (!irref_isk(ref)) { 2731 if (!irref_isk(ref)) {
2395 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE)); 2732 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2733#if LJ_GC64
2734 if (irt_is64(ir->t)) {
2735 /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
2736 emit_u32(as, irt_toitype(ir->t) << 15);
2737 emit_rmro(as, XO_ARITHi, XOg_OR, RID_BASE, ofs+4);
2738 } else if (LJ_DUALNUM && irt_isinteger(ir->t)) {
2739 emit_movmroi(as, RID_BASE, ofs+4, LJ_TISNUM << 15);
2740 } else {
2741 emit_movmroi(as, RID_BASE, ofs+4, (irt_toitype(ir->t)<<15)|0x7fff);
2742 }
2743#endif
2396 emit_movtomro(as, REX_64IR(ir, src), RID_BASE, ofs); 2744 emit_movtomro(as, REX_64IR(ir, src), RID_BASE, ofs);
2745#if LJ_GC64
2746 } else {
2747 TValue k;
2748 lj_ir_kvalue(as->J->L, &k, ir);
2749 if (tvisnil(&k)) {
2750 emit_i32(as, -1);
2751 emit_rmro(as, XO_MOVmi, REX_64, RID_BASE, ofs);
2752 } else {
2753 emit_movmroi(as, RID_BASE, ofs+4, k.u32.hi);
2754 emit_movmroi(as, RID_BASE, ofs, k.u32.lo);
2755 }
2756#else
2397 } else if (!irt_ispri(ir->t)) { 2757 } else if (!irt_ispri(ir->t)) {
2398 emit_movmroi(as, RID_BASE, ofs, ir->i); 2758 emit_movmroi(as, RID_BASE, ofs, ir->i);
2759#endif
2399 } 2760 }
2400 if ((sn & (SNAP_CONT|SNAP_FRAME))) { 2761 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
2762#if !LJ_FR2
2401 if (s != 0) /* Do not overwrite link to previous frame. */ 2763 if (s != 0) /* Do not overwrite link to previous frame. */
2402 emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*flinks--)); 2764 emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*flinks--));
2765#endif
2766#if !LJ_GC64
2403 } else { 2767 } else {
2404 if (!(LJ_64 && irt_islightud(ir->t))) 2768 if (!(LJ_64 && irt_islightud(ir->t)))
2405 emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t)); 2769 emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t));
2770#endif
2406 } 2771 }
2407 } 2772 }
2408 checkmclim(as); 2773 checkmclim(as);
2409 } 2774 }
2410 lua_assert(map + nent == flinks); 2775 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
2411} 2776}
2412 2777
2413/* -- GC handling --------------------------------------------------------- */ 2778/* -- GC handling --------------------------------------------------------- */
@@ -2428,11 +2793,15 @@ static void asm_gc_check(ASMState *as)
2428 args[1] = ASMREF_TMP2; /* MSize steps */ 2793 args[1] = ASMREF_TMP2; /* MSize steps */
2429 asm_gencall(as, ci, args); 2794 asm_gencall(as, ci, args);
2430 tmp = ra_releasetmp(as, ASMREF_TMP1); 2795 tmp = ra_releasetmp(as, ASMREF_TMP1);
2796#if LJ_GC64
2797 emit_rmro(as, XO_LEA, tmp|REX_64, RID_DISPATCH, GG_DISP2G);
2798#else
2431 emit_loada(as, tmp, J2G(as->J)); 2799 emit_loada(as, tmp, J2G(as->J));
2800#endif
2432 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), as->gcsteps); 2801 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), as->gcsteps);
2433 /* Jump around GC step if GC total < GC threshold. */ 2802 /* Jump around GC step if GC total < GC threshold. */
2434 emit_sjcc(as, CC_B, l_end); 2803 emit_sjcc(as, CC_B, l_end);
2435 emit_opgl(as, XO_ARITH(XOg_CMP), tmp, gc.threshold); 2804 emit_opgl(as, XO_ARITH(XOg_CMP), tmp|REX_GC64, gc.threshold);
2436 emit_getgl(as, tmp, gc.total); 2805 emit_getgl(as, tmp, gc.total);
2437 as->gcsteps = 0; 2806 as->gcsteps = 0;
2438 checkmclim(as); 2807 checkmclim(as);
@@ -2447,16 +2816,16 @@ static void asm_loop_fixup(ASMState *as)
2447 MCode *target = as->mcp; 2816 MCode *target = as->mcp;
2448 if (as->realign) { /* Realigned loops use short jumps. */ 2817 if (as->realign) { /* Realigned loops use short jumps. */
2449 as->realign = NULL; /* Stop another retry. */ 2818 as->realign = NULL; /* Stop another retry. */
2450 lua_assert(((intptr_t)target & 15) == 0); 2819 lj_assertA(((intptr_t)target & 15) == 0, "loop realign failed");
2451 if (as->loopinv) { /* Inverted loop branch? */ 2820 if (as->loopinv) { /* Inverted loop branch? */
2452 p -= 5; 2821 p -= 5;
2453 p[0] = XI_JMP; 2822 p[0] = XI_JMP;
2454 lua_assert(target - p >= -128); 2823 lj_assertA(target - p >= -128, "loop realign failed");
2455 p[-1] = (MCode)(target - p); /* Patch sjcc. */ 2824 p[-1] = (MCode)(target - p); /* Patch sjcc. */
2456 if (as->loopinv == 2) 2825 if (as->loopinv == 2)
2457 p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */ 2826 p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */
2458 } else { 2827 } else {
2459 lua_assert(target - p >= -128); 2828 lj_assertA(target - p >= -128, "loop realign failed");
2460 p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */ 2829 p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */
2461 p[-2] = XI_JMPs; 2830 p[-2] = XI_JMPs;
2462 } 2831 }
@@ -2485,6 +2854,12 @@ static void asm_loop_fixup(ASMState *as)
2485 } 2854 }
2486} 2855}
2487 2856
2857/* Fixup the tail of the loop. */
2858static void asm_loop_tail_fixup(ASMState *as)
2859{
2860 UNUSED(as); /* Nothing to do. */
2861}
2862
2488/* -- Head of trace ------------------------------------------------------- */ 2863/* -- Head of trace ------------------------------------------------------- */
2489 2864
2490/* Coalesce BASE register for a root trace. */ 2865/* Coalesce BASE register for a root trace. */
@@ -2497,7 +2872,7 @@ static void asm_head_root_base(ASMState *as)
2497 if (rset_test(as->modset, r) || irt_ismarked(ir->t)) 2872 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2498 ir->r = RID_INIT; /* No inheritance for modified BASE register. */ 2873 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
2499 if (r != RID_BASE) 2874 if (r != RID_BASE)
2500 emit_rr(as, XO_MOV, r, RID_BASE); 2875 emit_rr(as, XO_MOV, r|REX_GC64, RID_BASE);
2501 } 2876 }
2502} 2877}
2503 2878
@@ -2513,7 +2888,8 @@ static Reg asm_head_side_base(ASMState *as, IRIns *irp)
2513 if (irp->r == r) { 2888 if (irp->r == r) {
2514 return r; /* Same BASE register already coalesced. */ 2889 return r; /* Same BASE register already coalesced. */
2515 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) { 2890 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
2516 emit_rr(as, XO_MOV, r, irp->r); /* Move from coalesced parent reg. */ 2891 /* Move from coalesced parent reg. */
2892 emit_rr(as, XO_MOV, r|REX_GC64, irp->r);
2517 return irp->r; 2893 return irp->r;
2518 } else { 2894 } else {
2519 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */ 2895 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */
@@ -2532,7 +2908,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2532 MCode *target, *q; 2908 MCode *target, *q;
2533 int32_t spadj = as->T->spadjust; 2909 int32_t spadj = as->T->spadjust;
2534 if (spadj == 0) { 2910 if (spadj == 0) {
2535 p -= ((as->flags & JIT_F_LEA_AGU) ? 7 : 6) + (LJ_64 ? 1 : 0); 2911 p -= LJ_64 ? 7 : 6;
2536 } else { 2912 } else {
2537 MCode *p1; 2913 MCode *p1;
2538 /* Patch stack adjustment. */ 2914 /* Patch stack adjustment. */
@@ -2544,24 +2920,15 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2544 p1 = p-9; 2920 p1 = p-9;
2545 *(int32_t *)p1 = spadj; 2921 *(int32_t *)p1 = spadj;
2546 } 2922 }
2547 if ((as->flags & JIT_F_LEA_AGU)) {
2548#if LJ_64 2923#if LJ_64
2549 p1[-4] = 0x48; 2924 p1[-3] = 0x48;
2550#endif 2925#endif
2551 p1[-3] = (MCode)XI_LEA; 2926 p1[-2] = (MCode)(checki8(spadj) ? XI_ARITHi8 : XI_ARITHi);
2552 p1[-2] = MODRM(checki8(spadj) ? XM_OFS8 : XM_OFS32, RID_ESP, RID_ESP); 2927 p1[-1] = MODRM(XM_REG, XOg_ADD, RID_ESP);
2553 p1[-1] = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
2554 } else {
2555#if LJ_64
2556 p1[-3] = 0x48;
2557#endif
2558 p1[-2] = (MCode)(checki8(spadj) ? XI_ARITHi8 : XI_ARITHi);
2559 p1[-1] = MODRM(XM_REG, XOg_ADD, RID_ESP);
2560 }
2561 } 2928 }
2562 /* Patch exit branch. */ 2929 /* Patch exit branch. */
2563 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; 2930 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
2564 *(int32_t *)(p-4) = jmprel(p, target); 2931 *(int32_t *)(p-4) = jmprel(as->J, p, target);
2565 p[-5] = XI_JMP; 2932 p[-5] = XI_JMP;
2566 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */ 2933 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
2567 for (q = as->mctop-1; q >= p; q--) 2934 for (q = as->mctop-1; q >= p; q--)
@@ -2588,168 +2955,11 @@ static void asm_tail_prep(ASMState *as)
2588 as->invmcp = as->mcp = p; 2955 as->invmcp = as->mcp = p;
2589 } else { 2956 } else {
2590 /* Leave room for ESP adjustment: add esp, imm or lea esp, [esp+imm] */ 2957 /* Leave room for ESP adjustment: add esp, imm or lea esp, [esp+imm] */
2591 as->mcp = p - (((as->flags & JIT_F_LEA_AGU) ? 7 : 6) + (LJ_64 ? 1 : 0)); 2958 as->mcp = p - (LJ_64 ? 7 : 6);
2592 as->invmcp = NULL; 2959 as->invmcp = NULL;
2593 } 2960 }
2594} 2961}
2595 2962
2596/* -- Instruction dispatch ------------------------------------------------ */
2597
2598/* Assemble a single instruction. */
2599static void asm_ir(ASMState *as, IRIns *ir)
2600{
2601 switch ((IROp)ir->o) {
2602 /* Miscellaneous ops. */
2603 case IR_LOOP: asm_loop(as); break;
2604 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
2605 case IR_USE:
2606 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
2607 case IR_PHI: asm_phi(as, ir); break;
2608 case IR_HIOP: asm_hiop(as, ir); break;
2609 case IR_GCSTEP: asm_gcstep(as, ir); break;
2610
2611 /* Guarded assertions. */
2612 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
2613 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
2614 case IR_EQ: case IR_NE: case IR_ABC:
2615 asm_comp(as, ir, asm_compmap[ir->o]);
2616 break;
2617
2618 case IR_RETF: asm_retf(as, ir); break;
2619
2620 /* Bit ops. */
2621 case IR_BNOT: asm_neg_not(as, ir, XOg_NOT); break;
2622 case IR_BSWAP: asm_bitswap(as, ir); break;
2623
2624 case IR_BAND: asm_intarith(as, ir, XOg_AND); break;
2625 case IR_BOR: asm_intarith(as, ir, XOg_OR); break;
2626 case IR_BXOR: asm_intarith(as, ir, XOg_XOR); break;
2627
2628 case IR_BSHL: asm_bitshift(as, ir, XOg_SHL); break;
2629 case IR_BSHR: asm_bitshift(as, ir, XOg_SHR); break;
2630 case IR_BSAR: asm_bitshift(as, ir, XOg_SAR); break;
2631 case IR_BROL: asm_bitshift(as, ir, XOg_ROL); break;
2632 case IR_BROR: asm_bitshift(as, ir, XOg_ROR); break;
2633
2634 /* Arithmetic ops. */
2635 case IR_ADD: asm_add(as, ir); break;
2636 case IR_SUB:
2637 if (irt_isnum(ir->t))
2638 asm_fparith(as, ir, XO_SUBSD);
2639 else /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
2640 asm_intarith(as, ir, XOg_SUB);
2641 break;
2642 case IR_MUL:
2643 if (irt_isnum(ir->t))
2644 asm_fparith(as, ir, XO_MULSD);
2645 else
2646 asm_intarith(as, ir, XOg_X_IMUL);
2647 break;
2648 case IR_DIV:
2649#if LJ_64 && LJ_HASFFI
2650 if (!irt_isnum(ir->t))
2651 asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
2652 IRCALL_lj_carith_divu64);
2653 else
2654#endif
2655 asm_fparith(as, ir, XO_DIVSD);
2656 break;
2657 case IR_MOD:
2658#if LJ_64 && LJ_HASFFI
2659 if (!irt_isint(ir->t))
2660 asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
2661 IRCALL_lj_carith_modu64);
2662 else
2663#endif
2664 asm_intmod(as, ir);
2665 break;
2666
2667 case IR_NEG:
2668 if (irt_isnum(ir->t))
2669 asm_fparith(as, ir, XO_XORPS);
2670 else
2671 asm_neg_not(as, ir, XOg_NEG);
2672 break;
2673 case IR_ABS: asm_fparith(as, ir, XO_ANDPS); break;
2674
2675 case IR_MIN:
2676 if (irt_isnum(ir->t))
2677 asm_fparith(as, ir, XO_MINSD);
2678 else
2679 asm_min_max(as, ir, CC_G);
2680 break;
2681 case IR_MAX:
2682 if (irt_isnum(ir->t))
2683 asm_fparith(as, ir, XO_MAXSD);
2684 else
2685 asm_min_max(as, ir, CC_L);
2686 break;
2687
2688 case IR_FPMATH: case IR_ATAN2: case IR_LDEXP:
2689 asm_fpmath(as, ir);
2690 break;
2691 case IR_POW:
2692#if LJ_64 && LJ_HASFFI
2693 if (!irt_isnum(ir->t))
2694 asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
2695 IRCALL_lj_carith_powu64);
2696 else
2697#endif
2698 asm_fppowi(as, ir);
2699 break;
2700
2701 /* Overflow-checking arithmetic ops. Note: don't use LEA here! */
2702 case IR_ADDOV: asm_intarith(as, ir, XOg_ADD); break;
2703 case IR_SUBOV: asm_intarith(as, ir, XOg_SUB); break;
2704 case IR_MULOV: asm_intarith(as, ir, XOg_X_IMUL); break;
2705
2706 /* Memory references. */
2707 case IR_AREF: asm_aref(as, ir); break;
2708 case IR_HREF: asm_href(as, ir); break;
2709 case IR_HREFK: asm_hrefk(as, ir); break;
2710 case IR_NEWREF: asm_newref(as, ir); break;
2711 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
2712 case IR_FREF: asm_fref(as, ir); break;
2713 case IR_STRREF: asm_strref(as, ir); break;
2714
2715 /* Loads and stores. */
2716 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2717 asm_ahuvload(as, ir);
2718 break;
2719 case IR_FLOAD: case IR_XLOAD: asm_fxload(as, ir); break;
2720 case IR_SLOAD: asm_sload(as, ir); break;
2721
2722 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
2723 case IR_FSTORE: case IR_XSTORE: asm_fxstore(as, ir); break;
2724
2725 /* Allocations. */
2726 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
2727 case IR_TNEW: asm_tnew(as, ir); break;
2728 case IR_TDUP: asm_tdup(as, ir); break;
2729 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
2730
2731 /* Write barriers. */
2732 case IR_TBAR: asm_tbar(as, ir); break;
2733 case IR_OBAR: asm_obar(as, ir); break;
2734
2735 /* Type conversions. */
2736 case IR_TOBIT: asm_tobit(as, ir); break;
2737 case IR_CONV: asm_conv(as, ir); break;
2738 case IR_TOSTR: asm_tostr(as, ir); break;
2739 case IR_STRTO: asm_strto(as, ir); break;
2740
2741 /* Calls. */
2742 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
2743 case IR_CALLXS: asm_callx(as, ir); break;
2744 case IR_CARG: break;
2745
2746 default:
2747 setintV(&as->J->errinfo, ir->o);
2748 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
2749 break;
2750 }
2751}
2752
2753/* -- Trace setup --------------------------------------------------------- */ 2963/* -- Trace setup --------------------------------------------------------- */
2754 2964
2755/* Ensure there are enough stack slots for call arguments. */ 2965/* Ensure there are enough stack slots for call arguments. */
@@ -2772,6 +2982,7 @@ static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2772static void asm_setup_target(ASMState *as) 2982static void asm_setup_target(ASMState *as)
2773{ 2983{
2774 asm_exitstub_setup(as, as->T->nsnap); 2984 asm_exitstub_setup(as, as->T->nsnap);
2985 as->mrm.base = 0;
2775} 2986}
2776 2987
2777/* -- Trace patching ------------------------------------------------------ */ 2988/* -- Trace patching ------------------------------------------------------ */
@@ -2885,18 +3096,24 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2885 MCode *px = exitstub_addr(J, exitno) - 6; 3096 MCode *px = exitstub_addr(J, exitno) - 6;
2886 MCode *pe = p+len-6; 3097 MCode *pe = p+len-6;
2887 MCode *pgc = NULL; 3098 MCode *pgc = NULL;
2888 uint32_t stateaddr = u32ptr(&J2G(J)->vmstate); 3099#if LJ_GC64
3100 uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
3101#else
3102 uint32_t statei = u32ptr(&J2G(J)->vmstate);
3103#endif
2889 if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px) 3104 if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px)
2890 *(int32_t *)(p+len-4) = jmprel(p+len, target); 3105 *(int32_t *)(p+len-4) = jmprel(J, p+len, target);
2891 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */ 3106 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
2892 for (; p < pe; p += asm_x86_inslen(p)) 3107 for (; p < pe; p += asm_x86_inslen(p)) {
2893 if (*(uint32_t *)(p+(LJ_64 ? 3 : 2)) == stateaddr && p[0] == XI_MOVmi) 3108 intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64;
3109 if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi)
2894 break; 3110 break;
2895 lua_assert(p < pe); 3111 }
3112 lj_assertJ(p < pe, "instruction length decoder failed");
2896 for (; p < pe; p += asm_x86_inslen(p)) { 3113 for (; p < pe; p += asm_x86_inslen(p)) {
2897 if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px && 3114 if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px &&
2898 p != pgc) { 3115 p != pgc) {
2899 *(int32_t *)(p+2) = jmprel(p+6, target); 3116 *(int32_t *)(p+2) = jmprel(J, p+6, target);
2900 } else if (*p == XI_CALL && 3117 } else if (*p == XI_CALL &&
2901 (void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) { 3118 (void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) {
2902 pgc = p+7; /* Do not patch GC check exit. */ 3119 pgc = p+7; /* Do not patch GC check exit. */
diff --git a/src/lj_assert.c b/src/lj_assert.c
new file mode 100644
index 00000000..5c948b41
--- /dev/null
+++ b/src/lj_assert.c
@@ -0,0 +1,28 @@
1/*
2** Internal assertions.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_assert_c
7#define LUA_CORE
8
9#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
10
11#include <stdio.h>
12
13#include "lj_obj.h"
14
15void lj_assert_fail(global_State *g, const char *file, int line,
16 const char *func, const char *fmt, ...)
17{
18 va_list argp;
19 va_start(argp, fmt);
20 fprintf(stderr, "LuaJIT ASSERT %s:%d: %s: ", file, line, func);
21 vfprintf(stderr, fmt, argp);
22 fputc('\n', stderr);
23 va_end(argp);
24 UNUSED(g); /* May be NULL. TODO: optionally dump state. */
25 abort();
26}
27
28#endif
diff --git a/src/lj_bc.h b/src/lj_bc.h
index 5f6146b6..97e19a1b 100644
--- a/src/lj_bc.h
+++ b/src/lj_bc.h
@@ -89,6 +89,8 @@
89 _(ISFC, dst, ___, var, ___) \ 89 _(ISFC, dst, ___, var, ___) \
90 _(IST, ___, ___, var, ___) \ 90 _(IST, ___, ___, var, ___) \
91 _(ISF, ___, ___, var, ___) \ 91 _(ISF, ___, ___, var, ___) \
92 _(ISTYPE, var, ___, lit, ___) \
93 _(ISNUM, var, ___, lit, ___) \
92 \ 94 \
93 /* Unary ops. */ \ 95 /* Unary ops. */ \
94 _(MOV, dst, ___, var, ___) \ 96 _(MOV, dst, ___, var, ___) \
@@ -143,10 +145,12 @@
143 _(TGETV, dst, var, var, index) \ 145 _(TGETV, dst, var, var, index) \
144 _(TGETS, dst, var, str, index) \ 146 _(TGETS, dst, var, str, index) \
145 _(TGETB, dst, var, lit, index) \ 147 _(TGETB, dst, var, lit, index) \
148 _(TGETR, dst, var, var, index) \
146 _(TSETV, var, var, var, newindex) \ 149 _(TSETV, var, var, var, newindex) \
147 _(TSETS, var, var, str, newindex) \ 150 _(TSETS, var, var, str, newindex) \
148 _(TSETB, var, var, lit, newindex) \ 151 _(TSETB, var, var, lit, newindex) \
149 _(TSETM, base, ___, num, newindex) \ 152 _(TSETM, base, ___, num, newindex) \
153 _(TSETR, var, var, var, newindex) \
150 \ 154 \
151 /* Calls and vararg handling. T = tail call. */ \ 155 /* Calls and vararg handling. T = tail call. */ \
152 _(CALLM, base, lit, lit, call) \ 156 _(CALLM, base, lit, lit, call) \
diff --git a/src/lj_bcdump.h b/src/lj_bcdump.h
index 5cca36a9..6ba71e25 100644
--- a/src/lj_bcdump.h
+++ b/src/lj_bcdump.h
@@ -36,14 +36,15 @@
36/* If you perform *any* kind of private modifications to the bytecode itself 36/* If you perform *any* kind of private modifications to the bytecode itself
37** or to the dump format, you *must* set BCDUMP_VERSION to 0x80 or higher. 37** or to the dump format, you *must* set BCDUMP_VERSION to 0x80 or higher.
38*/ 38*/
39#define BCDUMP_VERSION 1 39#define BCDUMP_VERSION 2
40 40
41/* Compatibility flags. */ 41/* Compatibility flags. */
42#define BCDUMP_F_BE 0x01 42#define BCDUMP_F_BE 0x01
43#define BCDUMP_F_STRIP 0x02 43#define BCDUMP_F_STRIP 0x02
44#define BCDUMP_F_FFI 0x04 44#define BCDUMP_F_FFI 0x04
45#define BCDUMP_F_FR2 0x08
45 46
46#define BCDUMP_F_KNOWN (BCDUMP_F_FFI*2-1) 47#define BCDUMP_F_KNOWN (BCDUMP_F_FR2*2-1)
47 48
48/* Type codes for the GC constants of a prototype. Plus length for strings. */ 49/* Type codes for the GC constants of a prototype. Plus length for strings. */
49enum { 50enum {
@@ -61,6 +62,7 @@ enum {
61 62
62LJ_FUNC int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, 63LJ_FUNC int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer,
63 void *data, int strip); 64 void *data, int strip);
65LJ_FUNC GCproto *lj_bcread_proto(LexState *ls);
64LJ_FUNC GCproto *lj_bcread(LexState *ls); 66LJ_FUNC GCproto *lj_bcread(LexState *ls);
65 67
66#endif 68#endif
diff --git a/src/lj_bcread.c b/src/lj_bcread.c
index ad6b2a5c..c98c0d42 100644
--- a/src/lj_bcread.c
+++ b/src/lj_bcread.c
@@ -9,6 +9,7 @@
9#include "lj_obj.h" 9#include "lj_obj.h"
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_buf.h"
12#include "lj_str.h" 13#include "lj_str.h"
13#include "lj_tab.h" 14#include "lj_tab.h"
14#include "lj_bc.h" 15#include "lj_bc.h"
@@ -20,6 +21,7 @@
20#include "lj_lex.h" 21#include "lj_lex.h"
21#include "lj_bcdump.h" 22#include "lj_bcdump.h"
22#include "lj_state.h" 23#include "lj_state.h"
24#include "lj_strfmt.h"
23 25
24/* Reuse some lexer fields for our own purposes. */ 26/* Reuse some lexer fields for our own purposes. */
25#define bcread_flags(ls) ls->level 27#define bcread_flags(ls) ls->level
@@ -38,85 +40,74 @@ static LJ_NOINLINE void bcread_error(LexState *ls, ErrMsg em)
38 const char *name = ls->chunkarg; 40 const char *name = ls->chunkarg;
39 if (*name == BCDUMP_HEAD1) name = "(binary)"; 41 if (*name == BCDUMP_HEAD1) name = "(binary)";
40 else if (*name == '@' || *name == '=') name++; 42 else if (*name == '@' || *name == '=') name++;
41 lj_str_pushf(L, "%s: %s", name, err2msg(em)); 43 lj_strfmt_pushf(L, "%s: %s", name, err2msg(em));
42 lj_err_throw(L, LUA_ERRSYNTAX); 44 lj_err_throw(L, LUA_ERRSYNTAX);
43} 45}
44 46
45/* Resize input buffer. */ 47/* Refill buffer. */
46static void bcread_resize(LexState *ls, MSize len)
47{
48 if (ls->sb.sz < len) {
49 MSize sz = ls->sb.sz * 2;
50 while (len > sz) sz = sz * 2;
51 lj_str_resizebuf(ls->L, &ls->sb, sz);
52 /* Caveat: this may change ls->sb.buf which may affect ls->p. */
53 }
54}
55
56/* Refill buffer if needed. */
57static LJ_NOINLINE void bcread_fill(LexState *ls, MSize len, int need) 48static LJ_NOINLINE void bcread_fill(LexState *ls, MSize len, int need)
58{ 49{
59 lua_assert(len != 0); 50 lj_assertLS(len != 0, "empty refill");
60 if (len > LJ_MAX_MEM || ls->current < 0) 51 if (len > LJ_MAX_BUF || ls->c < 0)
61 bcread_error(ls, LJ_ERR_BCBAD); 52 bcread_error(ls, LJ_ERR_BCBAD);
62 do { 53 do {
63 const char *buf; 54 const char *buf;
64 size_t size; 55 size_t sz;
65 if (ls->n) { /* Copy remainder to buffer. */ 56 char *p = ls->sb.b;
66 if (ls->sb.n) { /* Move down in buffer. */ 57 MSize n = (MSize)(ls->pe - ls->p);
67 lua_assert(ls->p + ls->n == ls->sb.buf + ls->sb.n); 58 if (n) { /* Copy remainder to buffer. */
68 if (ls->n != ls->sb.n) 59 if (sbuflen(&ls->sb)) { /* Move down in buffer. */
69 memmove(ls->sb.buf, ls->p, ls->n); 60 lj_assertLS(ls->pe == ls->sb.w, "bad buffer pointer");
61 if (ls->p != p) memmove(p, ls->p, n);
70 } else { /* Copy from buffer provided by reader. */ 62 } else { /* Copy from buffer provided by reader. */
71 bcread_resize(ls, len); 63 p = lj_buf_need(&ls->sb, len);
72 memcpy(ls->sb.buf, ls->p, ls->n); 64 memcpy(p, ls->p, n);
73 } 65 }
74 ls->p = ls->sb.buf; 66 ls->p = p;
67 ls->pe = p + n;
75 } 68 }
76 ls->sb.n = ls->n; 69 ls->sb.w = p + n;
77 buf = ls->rfunc(ls->L, ls->rdata, &size); /* Get more data from reader. */ 70 buf = ls->rfunc(ls->L, ls->rdata, &sz); /* Get more data from reader. */
78 if (buf == NULL || size == 0) { /* EOF? */ 71 if (buf == NULL || sz == 0) { /* EOF? */
79 if (need) bcread_error(ls, LJ_ERR_BCBAD); 72 if (need) bcread_error(ls, LJ_ERR_BCBAD);
80 ls->current = -1; /* Only bad if we get called again. */ 73 ls->c = -1; /* Only bad if we get called again. */
81 break; 74 break;
82 } 75 }
83 if (size >= LJ_MAX_MEM - ls->sb.n) lj_err_mem(ls->L); 76 if (sz >= LJ_MAX_BUF - n) lj_err_mem(ls->L);
84 if (ls->sb.n) { /* Append to buffer. */ 77 if (n) { /* Append to buffer. */
85 MSize n = ls->sb.n + (MSize)size; 78 n += (MSize)sz;
86 bcread_resize(ls, n < len ? len : n); 79 p = lj_buf_need(&ls->sb, n < len ? len : n);
87 memcpy(ls->sb.buf + ls->sb.n, buf, size); 80 memcpy(ls->sb.w, buf, sz);
88 ls->n = ls->sb.n = n; 81 ls->sb.w = p + n;
89 ls->p = ls->sb.buf; 82 ls->p = p;
83 ls->pe = p + n;
90 } else { /* Return buffer provided by reader. */ 84 } else { /* Return buffer provided by reader. */
91 ls->n = (MSize)size;
92 ls->p = buf; 85 ls->p = buf;
86 ls->pe = buf + sz;
93 } 87 }
94 } while (ls->n < len); 88 } while ((MSize)(ls->pe - ls->p) < len);
95} 89}
96 90
97/* Need a certain number of bytes. */ 91/* Need a certain number of bytes. */
98static LJ_AINLINE void bcread_need(LexState *ls, MSize len) 92static LJ_AINLINE void bcread_need(LexState *ls, MSize len)
99{ 93{
100 if (LJ_UNLIKELY(ls->n < len)) 94 if (LJ_UNLIKELY((MSize)(ls->pe - ls->p) < len))
101 bcread_fill(ls, len, 1); 95 bcread_fill(ls, len, 1);
102} 96}
103 97
104/* Want to read up to a certain number of bytes, but may need less. */ 98/* Want to read up to a certain number of bytes, but may need less. */
105static LJ_AINLINE void bcread_want(LexState *ls, MSize len) 99static LJ_AINLINE void bcread_want(LexState *ls, MSize len)
106{ 100{
107 if (LJ_UNLIKELY(ls->n < len)) 101 if (LJ_UNLIKELY((MSize)(ls->pe - ls->p) < len))
108 bcread_fill(ls, len, 0); 102 bcread_fill(ls, len, 0);
109} 103}
110 104
111#define bcread_dec(ls) check_exp(ls->n > 0, ls->n--)
112#define bcread_consume(ls, len) check_exp(ls->n >= (len), ls->n -= (len))
113
114/* Return memory block from buffer. */ 105/* Return memory block from buffer. */
115static uint8_t *bcread_mem(LexState *ls, MSize len) 106static LJ_AINLINE uint8_t *bcread_mem(LexState *ls, MSize len)
116{ 107{
117 uint8_t *p = (uint8_t *)ls->p; 108 uint8_t *p = (uint8_t *)ls->p;
118 bcread_consume(ls, len); 109 ls->p += len;
119 ls->p = (char *)p + len; 110 lj_assertLS(ls->p <= ls->pe, "buffer read overflow");
120 return p; 111 return p;
121} 112}
122 113
@@ -129,25 +120,15 @@ static void bcread_block(LexState *ls, void *q, MSize len)
129/* Read byte from buffer. */ 120/* Read byte from buffer. */
130static LJ_AINLINE uint32_t bcread_byte(LexState *ls) 121static LJ_AINLINE uint32_t bcread_byte(LexState *ls)
131{ 122{
132 bcread_dec(ls); 123 lj_assertLS(ls->p < ls->pe, "buffer read overflow");
133 return (uint32_t)(uint8_t)*ls->p++; 124 return (uint32_t)(uint8_t)*ls->p++;
134} 125}
135 126
136/* Read ULEB128 value from buffer. */ 127/* Read ULEB128 value from buffer. */
137static uint32_t bcread_uleb128(LexState *ls) 128static LJ_AINLINE uint32_t bcread_uleb128(LexState *ls)
138{ 129{
139 const uint8_t *p = (const uint8_t *)ls->p; 130 uint32_t v = lj_buf_ruleb128(&ls->p);
140 uint32_t v = *p++; 131 lj_assertLS(ls->p <= ls->pe, "buffer read overflow");
141 if (LJ_UNLIKELY(v >= 0x80)) {
142 int sh = 0;
143 v &= 0x7f;
144 do {
145 v |= ((*p & 0x7f) << (sh += 7));
146 bcread_dec(ls);
147 } while (*p++ >= 0x80);
148 }
149 bcread_dec(ls);
150 ls->p = (char *)p;
151 return v; 132 return v;
152} 133}
153 134
@@ -161,11 +142,10 @@ static uint32_t bcread_uleb128_33(LexState *ls)
161 v &= 0x3f; 142 v &= 0x3f;
162 do { 143 do {
163 v |= ((*p & 0x7f) << (sh += 7)); 144 v |= ((*p & 0x7f) << (sh += 7));
164 bcread_dec(ls);
165 } while (*p++ >= 0x80); 145 } while (*p++ >= 0x80);
166 } 146 }
167 bcread_dec(ls);
168 ls->p = (char *)p; 147 ls->p = (char *)p;
148 lj_assertLS(ls->p <= ls->pe, "buffer read overflow");
169 return v; 149 return v;
170} 150}
171 151
@@ -212,8 +192,8 @@ static void bcread_ktabk(LexState *ls, TValue *o)
212 o->u32.lo = bcread_uleb128(ls); 192 o->u32.lo = bcread_uleb128(ls);
213 o->u32.hi = bcread_uleb128(ls); 193 o->u32.hi = bcread_uleb128(ls);
214 } else { 194 } else {
215 lua_assert(tp <= BCDUMP_KTAB_TRUE); 195 lj_assertLS(tp <= BCDUMP_KTAB_TRUE, "bad constant type %d", tp);
216 setitype(o, ~tp); 196 setpriV(o, ~tp);
217 } 197 }
218} 198}
219 199
@@ -234,7 +214,7 @@ static GCtab *bcread_ktab(LexState *ls)
234 for (i = 0; i < nhash; i++) { 214 for (i = 0; i < nhash; i++) {
235 TValue key; 215 TValue key;
236 bcread_ktabk(ls, &key); 216 bcread_ktabk(ls, &key);
237 lua_assert(!tvisnil(&key)); 217 lj_assertLS(!tvisnil(&key), "nil key");
238 bcread_ktabk(ls, lj_tab_set(ls->L, t, &key)); 218 bcread_ktabk(ls, lj_tab_set(ls->L, t, &key));
239 } 219 }
240 } 220 }
@@ -271,7 +251,7 @@ static void bcread_kgc(LexState *ls, GCproto *pt, MSize sizekgc)
271#endif 251#endif
272 } else { 252 } else {
273 lua_State *L = ls->L; 253 lua_State *L = ls->L;
274 lua_assert(tp == BCDUMP_KGC_CHILD); 254 lj_assertLS(tp == BCDUMP_KGC_CHILD, "bad constant type %d", tp);
275 if (L->top <= bcread_oldtop(L, ls)) /* Stack underflow? */ 255 if (L->top <= bcread_oldtop(L, ls)) /* Stack underflow? */
276 bcread_error(ls, LJ_ERR_BCBAD); 256 bcread_error(ls, LJ_ERR_BCBAD);
277 L->top--; 257 L->top--;
@@ -327,25 +307,13 @@ static void bcread_uv(LexState *ls, GCproto *pt, MSize sizeuv)
327} 307}
328 308
329/* Read a prototype. */ 309/* Read a prototype. */
330static GCproto *bcread_proto(LexState *ls) 310GCproto *lj_bcread_proto(LexState *ls)
331{ 311{
332 GCproto *pt; 312 GCproto *pt;
333 MSize framesize, numparams, flags, sizeuv, sizekgc, sizekn, sizebc, sizept; 313 MSize framesize, numparams, flags, sizeuv, sizekgc, sizekn, sizebc, sizept;
334 MSize ofsk, ofsuv, ofsdbg; 314 MSize ofsk, ofsuv, ofsdbg;
335 MSize sizedbg = 0; 315 MSize sizedbg = 0;
336 BCLine firstline = 0, numline = 0; 316 BCLine firstline = 0, numline = 0;
337 MSize len, startn;
338
339 /* Read length. */
340 if (ls->n > 0 && ls->p[0] == 0) { /* Shortcut EOF. */
341 ls->n--; ls->p++;
342 return NULL;
343 }
344 bcread_want(ls, 5);
345 len = bcread_uleb128(ls);
346 if (!len) return NULL; /* EOF */
347 bcread_need(ls, len);
348 startn = ls->n;
349 317
350 /* Read prototype header. */ 318 /* Read prototype header. */
351 flags = bcread_byte(ls); 319 flags = bcread_byte(ls);
@@ -414,9 +382,6 @@ static GCproto *bcread_proto(LexState *ls)
414 setmref(pt->uvinfo, NULL); 382 setmref(pt->uvinfo, NULL);
415 setmref(pt->varinfo, NULL); 383 setmref(pt->varinfo, NULL);
416 } 384 }
417
418 if (len != startn - ls->n)
419 bcread_error(ls, LJ_ERR_BCBAD);
420 return pt; 385 return pt;
421} 386}
422 387
@@ -430,14 +395,11 @@ static int bcread_header(LexState *ls)
430 bcread_byte(ls) != BCDUMP_VERSION) return 0; 395 bcread_byte(ls) != BCDUMP_VERSION) return 0;
431 bcread_flags(ls) = flags = bcread_uleb128(ls); 396 bcread_flags(ls) = flags = bcread_uleb128(ls);
432 if ((flags & ~(BCDUMP_F_KNOWN)) != 0) return 0; 397 if ((flags & ~(BCDUMP_F_KNOWN)) != 0) return 0;
398 if ((flags & BCDUMP_F_FR2) != LJ_FR2*BCDUMP_F_FR2) return 0;
433 if ((flags & BCDUMP_F_FFI)) { 399 if ((flags & BCDUMP_F_FFI)) {
434#if LJ_HASFFI 400#if LJ_HASFFI
435 lua_State *L = ls->L; 401 lua_State *L = ls->L;
436 if (!ctype_ctsG(G(L))) { 402 ctype_loadffi(L);
437 ptrdiff_t oldtop = savestack(L, L->top);
438 luaopen_ffi(L); /* Load FFI library on-demand. */
439 L->top = restorestack(L, oldtop);
440 }
441#else 403#else
442 return 0; 404 return 0;
443#endif 405#endif
@@ -456,19 +418,33 @@ static int bcread_header(LexState *ls)
456GCproto *lj_bcread(LexState *ls) 418GCproto *lj_bcread(LexState *ls)
457{ 419{
458 lua_State *L = ls->L; 420 lua_State *L = ls->L;
459 lua_assert(ls->current == BCDUMP_HEAD1); 421 lj_assertLS(ls->c == BCDUMP_HEAD1, "bad bytecode header");
460 bcread_savetop(L, ls, L->top); 422 bcread_savetop(L, ls, L->top);
461 lj_str_resetbuf(&ls->sb); 423 lj_buf_reset(&ls->sb);
462 /* Check for a valid bytecode dump header. */ 424 /* Check for a valid bytecode dump header. */
463 if (!bcread_header(ls)) 425 if (!bcread_header(ls))
464 bcread_error(ls, LJ_ERR_BCFMT); 426 bcread_error(ls, LJ_ERR_BCFMT);
465 for (;;) { /* Process all prototypes in the bytecode dump. */ 427 for (;;) { /* Process all prototypes in the bytecode dump. */
466 GCproto *pt = bcread_proto(ls); 428 GCproto *pt;
467 if (!pt) break; 429 MSize len;
430 const char *startp;
431 /* Read length. */
432 if (ls->p < ls->pe && ls->p[0] == 0) { /* Shortcut EOF. */
433 ls->p++;
434 break;
435 }
436 bcread_want(ls, 5);
437 len = bcread_uleb128(ls);
438 if (!len) break; /* EOF */
439 bcread_need(ls, len);
440 startp = ls->p;
441 pt = lj_bcread_proto(ls);
442 if (ls->p != startp + len)
443 bcread_error(ls, LJ_ERR_BCBAD);
468 setprotoV(L, L->top, pt); 444 setprotoV(L, L->top, pt);
469 incr_top(L); 445 incr_top(L);
470 } 446 }
471 if ((ls->n && !ls->endmark) || L->top-1 != bcread_oldtop(L, ls)) 447 if ((ls->pe != ls->p && !ls->endmark) || L->top-1 != bcread_oldtop(L, ls))
472 bcread_error(ls, LJ_ERR_BCBAD); 448 bcread_error(ls, LJ_ERR_BCBAD);
473 /* Pop off last prototype. */ 449 /* Pop off last prototype. */
474 L->top--; 450 L->top--;
diff --git a/src/lj_bcwrite.c b/src/lj_bcwrite.c
index 6282f767..dd969413 100644
--- a/src/lj_bcwrite.c
+++ b/src/lj_bcwrite.c
@@ -8,7 +8,7 @@
8 8
9#include "lj_obj.h" 9#include "lj_obj.h"
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_str.h" 11#include "lj_buf.h"
12#include "lj_bc.h" 12#include "lj_bc.h"
13#if LJ_HASFFI 13#if LJ_HASFFI
14#include "lj_ctype.h" 14#include "lj_ctype.h"
@@ -17,99 +17,67 @@
17#include "lj_dispatch.h" 17#include "lj_dispatch.h"
18#include "lj_jit.h" 18#include "lj_jit.h"
19#endif 19#endif
20#include "lj_strfmt.h"
20#include "lj_bcdump.h" 21#include "lj_bcdump.h"
21#include "lj_vm.h" 22#include "lj_vm.h"
22 23
23/* Context for bytecode writer. */ 24/* Context for bytecode writer. */
24typedef struct BCWriteCtx { 25typedef struct BCWriteCtx {
25 SBuf sb; /* Output buffer. */ 26 SBuf sb; /* Output buffer. */
26 lua_State *L; /* Lua state. */
27 GCproto *pt; /* Root prototype. */ 27 GCproto *pt; /* Root prototype. */
28 lua_Writer wfunc; /* Writer callback. */ 28 lua_Writer wfunc; /* Writer callback. */
29 void *wdata; /* Writer callback data. */ 29 void *wdata; /* Writer callback data. */
30 int strip; /* Strip debug info. */ 30 int strip; /* Strip debug info. */
31 int status; /* Status from writer callback. */ 31 int status; /* Status from writer callback. */
32#ifdef LUA_USE_ASSERT
33 global_State *g;
34#endif
32} BCWriteCtx; 35} BCWriteCtx;
33 36
34/* -- Output buffer handling ---------------------------------------------- */ 37#ifdef LUA_USE_ASSERT
35 38#define lj_assertBCW(c, ...) lj_assertG_(ctx->g, (c), __VA_ARGS__)
36/* Resize buffer if needed. */ 39#else
37static LJ_NOINLINE void bcwrite_resize(BCWriteCtx *ctx, MSize len) 40#define lj_assertBCW(c, ...) ((void)ctx)
38{ 41#endif
39 MSize sz = ctx->sb.sz * 2;
40 while (ctx->sb.n + len > sz) sz = sz * 2;
41 lj_str_resizebuf(ctx->L, &ctx->sb, sz);
42}
43
44/* Need a certain amount of buffer space. */
45static LJ_AINLINE void bcwrite_need(BCWriteCtx *ctx, MSize len)
46{
47 if (LJ_UNLIKELY(ctx->sb.n + len > ctx->sb.sz))
48 bcwrite_resize(ctx, len);
49}
50
51/* Add memory block to buffer. */
52static void bcwrite_block(BCWriteCtx *ctx, const void *p, MSize len)
53{
54 uint8_t *q = (uint8_t *)(ctx->sb.buf + ctx->sb.n);
55 MSize i;
56 ctx->sb.n += len;
57 for (i = 0; i < len; i++) q[i] = ((uint8_t *)p)[i];
58}
59
60/* Add byte to buffer. */
61static LJ_AINLINE void bcwrite_byte(BCWriteCtx *ctx, uint8_t b)
62{
63 ctx->sb.buf[ctx->sb.n++] = b;
64}
65
66/* Add ULEB128 value to buffer. */
67static void bcwrite_uleb128(BCWriteCtx *ctx, uint32_t v)
68{
69 MSize n = ctx->sb.n;
70 uint8_t *p = (uint8_t *)ctx->sb.buf;
71 for (; v >= 0x80; v >>= 7)
72 p[n++] = (uint8_t)((v & 0x7f) | 0x80);
73 p[n++] = (uint8_t)v;
74 ctx->sb.n = n;
75}
76 42
77/* -- Bytecode writer ----------------------------------------------------- */ 43/* -- Bytecode writer ----------------------------------------------------- */
78 44
79/* Write a single constant key/value of a template table. */ 45/* Write a single constant key/value of a template table. */
80static void bcwrite_ktabk(BCWriteCtx *ctx, cTValue *o, int narrow) 46static void bcwrite_ktabk(BCWriteCtx *ctx, cTValue *o, int narrow)
81{ 47{
82 bcwrite_need(ctx, 1+10); 48 char *p = lj_buf_more(&ctx->sb, 1+10);
83 if (tvisstr(o)) { 49 if (tvisstr(o)) {
84 const GCstr *str = strV(o); 50 const GCstr *str = strV(o);
85 MSize len = str->len; 51 MSize len = str->len;
86 bcwrite_need(ctx, 5+len); 52 p = lj_buf_more(&ctx->sb, 5+len);
87 bcwrite_uleb128(ctx, BCDUMP_KTAB_STR+len); 53 p = lj_strfmt_wuleb128(p, BCDUMP_KTAB_STR+len);
88 bcwrite_block(ctx, strdata(str), len); 54 p = lj_buf_wmem(p, strdata(str), len);
89 } else if (tvisint(o)) { 55 } else if (tvisint(o)) {
90 bcwrite_byte(ctx, BCDUMP_KTAB_INT); 56 *p++ = BCDUMP_KTAB_INT;
91 bcwrite_uleb128(ctx, intV(o)); 57 p = lj_strfmt_wuleb128(p, intV(o));
92 } else if (tvisnum(o)) { 58 } else if (tvisnum(o)) {
93 if (!LJ_DUALNUM && narrow) { /* Narrow number constants to integers. */ 59 if (!LJ_DUALNUM && narrow) { /* Narrow number constants to integers. */
94 lua_Number num = numV(o); 60 lua_Number num = numV(o);
95 int32_t k = lj_num2int(num); 61 int32_t k = lj_num2int(num);
96 if (num == (lua_Number)k) { /* -0 is never a constant. */ 62 if (num == (lua_Number)k) { /* -0 is never a constant. */
97 bcwrite_byte(ctx, BCDUMP_KTAB_INT); 63 *p++ = BCDUMP_KTAB_INT;
98 bcwrite_uleb128(ctx, k); 64 p = lj_strfmt_wuleb128(p, k);
65 ctx->sb.w = p;
99 return; 66 return;
100 } 67 }
101 } 68 }
102 bcwrite_byte(ctx, BCDUMP_KTAB_NUM); 69 *p++ = BCDUMP_KTAB_NUM;
103 bcwrite_uleb128(ctx, o->u32.lo); 70 p = lj_strfmt_wuleb128(p, o->u32.lo);
104 bcwrite_uleb128(ctx, o->u32.hi); 71 p = lj_strfmt_wuleb128(p, o->u32.hi);
105 } else { 72 } else {
106 lua_assert(tvispri(o)); 73 lj_assertBCW(tvispri(o), "unhandled type %d", itype(o));
107 bcwrite_byte(ctx, BCDUMP_KTAB_NIL+~itype(o)); 74 *p++ = BCDUMP_KTAB_NIL+~itype(o);
108 } 75 }
76 ctx->sb.w = p;
109} 77}
110 78
111/* Write a template table. */ 79/* Write a template table. */
112static void bcwrite_ktab(BCWriteCtx *ctx, const GCtab *t) 80static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t)
113{ 81{
114 MSize narray = 0, nhash = 0; 82 MSize narray = 0, nhash = 0;
115 if (t->asize > 0) { /* Determine max. length of array part. */ 83 if (t->asize > 0) { /* Determine max. length of array part. */
@@ -127,8 +95,9 @@ static void bcwrite_ktab(BCWriteCtx *ctx, const GCtab *t)
127 nhash += !tvisnil(&node[i].val); 95 nhash += !tvisnil(&node[i].val);
128 } 96 }
129 /* Write number of array slots and hash slots. */ 97 /* Write number of array slots and hash slots. */
130 bcwrite_uleb128(ctx, narray); 98 p = lj_strfmt_wuleb128(p, narray);
131 bcwrite_uleb128(ctx, nhash); 99 p = lj_strfmt_wuleb128(p, nhash);
100 ctx->sb.w = p;
132 if (narray) { /* Write array entries (may contain nil). */ 101 if (narray) { /* Write array entries (may contain nil). */
133 MSize i; 102 MSize i;
134 TValue *o = tvref(t->array); 103 TValue *o = tvref(t->array);
@@ -155,12 +124,13 @@ static void bcwrite_kgc(BCWriteCtx *ctx, GCproto *pt)
155 for (i = 0; i < sizekgc; i++, kr++) { 124 for (i = 0; i < sizekgc; i++, kr++) {
156 GCobj *o = gcref(*kr); 125 GCobj *o = gcref(*kr);
157 MSize tp, need = 1; 126 MSize tp, need = 1;
127 char *p;
158 /* Determine constant type and needed size. */ 128 /* Determine constant type and needed size. */
159 if (o->gch.gct == ~LJ_TSTR) { 129 if (o->gch.gct == ~LJ_TSTR) {
160 tp = BCDUMP_KGC_STR + gco2str(o)->len; 130 tp = BCDUMP_KGC_STR + gco2str(o)->len;
161 need = 5+gco2str(o)->len; 131 need = 5+gco2str(o)->len;
162 } else if (o->gch.gct == ~LJ_TPROTO) { 132 } else if (o->gch.gct == ~LJ_TPROTO) {
163 lua_assert((pt->flags & PROTO_CHILD)); 133 lj_assertBCW((pt->flags & PROTO_CHILD), "prototype has unexpected child");
164 tp = BCDUMP_KGC_CHILD; 134 tp = BCDUMP_KGC_CHILD;
165#if LJ_HASFFI 135#if LJ_HASFFI
166 } else if (o->gch.gct == ~LJ_TCDATA) { 136 } else if (o->gch.gct == ~LJ_TCDATA) {
@@ -171,34 +141,38 @@ static void bcwrite_kgc(BCWriteCtx *ctx, GCproto *pt)
171 } else if (id == CTID_UINT64) { 141 } else if (id == CTID_UINT64) {
172 tp = BCDUMP_KGC_U64; 142 tp = BCDUMP_KGC_U64;
173 } else { 143 } else {
174 lua_assert(id == CTID_COMPLEX_DOUBLE); 144 lj_assertBCW(id == CTID_COMPLEX_DOUBLE,
145 "bad cdata constant CTID %d", id);
175 tp = BCDUMP_KGC_COMPLEX; 146 tp = BCDUMP_KGC_COMPLEX;
176 } 147 }
177#endif 148#endif
178 } else { 149 } else {
179 lua_assert(o->gch.gct == ~LJ_TTAB); 150 lj_assertBCW(o->gch.gct == ~LJ_TTAB,
151 "bad constant GC type %d", o->gch.gct);
180 tp = BCDUMP_KGC_TAB; 152 tp = BCDUMP_KGC_TAB;
181 need = 1+2*5; 153 need = 1+2*5;
182 } 154 }
183 /* Write constant type. */ 155 /* Write constant type. */
184 bcwrite_need(ctx, need); 156 p = lj_buf_more(&ctx->sb, need);
185 bcwrite_uleb128(ctx, tp); 157 p = lj_strfmt_wuleb128(p, tp);
186 /* Write constant data (if any). */ 158 /* Write constant data (if any). */
187 if (tp >= BCDUMP_KGC_STR) { 159 if (tp >= BCDUMP_KGC_STR) {
188 bcwrite_block(ctx, strdata(gco2str(o)), gco2str(o)->len); 160 p = lj_buf_wmem(p, strdata(gco2str(o)), gco2str(o)->len);
189 } else if (tp == BCDUMP_KGC_TAB) { 161 } else if (tp == BCDUMP_KGC_TAB) {
190 bcwrite_ktab(ctx, gco2tab(o)); 162 bcwrite_ktab(ctx, p, gco2tab(o));
163 continue;
191#if LJ_HASFFI 164#if LJ_HASFFI
192 } else if (tp != BCDUMP_KGC_CHILD) { 165 } else if (tp != BCDUMP_KGC_CHILD) {
193 cTValue *p = (TValue *)cdataptr(gco2cd(o)); 166 cTValue *q = (TValue *)cdataptr(gco2cd(o));
194 bcwrite_uleb128(ctx, p[0].u32.lo); 167 p = lj_strfmt_wuleb128(p, q[0].u32.lo);
195 bcwrite_uleb128(ctx, p[0].u32.hi); 168 p = lj_strfmt_wuleb128(p, q[0].u32.hi);
196 if (tp == BCDUMP_KGC_COMPLEX) { 169 if (tp == BCDUMP_KGC_COMPLEX) {
197 bcwrite_uleb128(ctx, p[1].u32.lo); 170 p = lj_strfmt_wuleb128(p, q[1].u32.lo);
198 bcwrite_uleb128(ctx, p[1].u32.hi); 171 p = lj_strfmt_wuleb128(p, q[1].u32.hi);
199 } 172 }
200#endif 173#endif
201 } 174 }
175 ctx->sb.w = p;
202 } 176 }
203} 177}
204 178
@@ -207,7 +181,7 @@ static void bcwrite_knum(BCWriteCtx *ctx, GCproto *pt)
207{ 181{
208 MSize i, sizekn = pt->sizekn; 182 MSize i, sizekn = pt->sizekn;
209 cTValue *o = mref(pt->k, TValue); 183 cTValue *o = mref(pt->k, TValue);
210 bcwrite_need(ctx, 10*sizekn); 184 char *p = lj_buf_more(&ctx->sb, 10*sizekn);
211 for (i = 0; i < sizekn; i++, o++) { 185 for (i = 0; i < sizekn; i++, o++) {
212 int32_t k; 186 int32_t k;
213 if (tvisint(o)) { 187 if (tvisint(o)) {
@@ -215,60 +189,61 @@ static void bcwrite_knum(BCWriteCtx *ctx, GCproto *pt)
215 goto save_int; 189 goto save_int;
216 } else { 190 } else {
217 /* Write a 33 bit ULEB128 for the int (lsb=0) or loword (lsb=1). */ 191 /* Write a 33 bit ULEB128 for the int (lsb=0) or loword (lsb=1). */
218 if (!LJ_DUALNUM) { /* Narrow number constants to integers. */ 192 if (!LJ_DUALNUM && o->u32.hi != LJ_KEYINDEX) {
193 /* Narrow number constants to integers. */
219 lua_Number num = numV(o); 194 lua_Number num = numV(o);
220 k = lj_num2int(num); 195 k = lj_num2int(num);
221 if (num == (lua_Number)k) { /* -0 is never a constant. */ 196 if (num == (lua_Number)k) { /* -0 is never a constant. */
222 save_int: 197 save_int:
223 bcwrite_uleb128(ctx, 2*(uint32_t)k | ((uint32_t)k & 0x80000000u)); 198 p = lj_strfmt_wuleb128(p, 2*(uint32_t)k | ((uint32_t)k&0x80000000u));
224 if (k < 0) { 199 if (k < 0)
225 char *p = &ctx->sb.buf[ctx->sb.n-1]; 200 p[-1] = (p[-1] & 7) | ((k>>27) & 0x18);
226 *p = (*p & 7) | ((k>>27) & 0x18);
227 }
228 continue; 201 continue;
229 } 202 }
230 } 203 }
231 bcwrite_uleb128(ctx, 1+(2*o->u32.lo | (o->u32.lo & 0x80000000u))); 204 p = lj_strfmt_wuleb128(p, 1+(2*o->u32.lo | (o->u32.lo & 0x80000000u)));
232 if (o->u32.lo >= 0x80000000u) { 205 if (o->u32.lo >= 0x80000000u)
233 char *p = &ctx->sb.buf[ctx->sb.n-1]; 206 p[-1] = (p[-1] & 7) | ((o->u32.lo>>27) & 0x18);
234 *p = (*p & 7) | ((o->u32.lo>>27) & 0x18); 207 p = lj_strfmt_wuleb128(p, o->u32.hi);
235 }
236 bcwrite_uleb128(ctx, o->u32.hi);
237 } 208 }
238 } 209 }
210 ctx->sb.w = p;
239} 211}
240 212
241/* Write bytecode instructions. */ 213/* Write bytecode instructions. */
242static void bcwrite_bytecode(BCWriteCtx *ctx, GCproto *pt) 214static char *bcwrite_bytecode(BCWriteCtx *ctx, char *p, GCproto *pt)
243{ 215{
244 MSize nbc = pt->sizebc-1; /* Omit the [JI]FUNC* header. */ 216 MSize nbc = pt->sizebc-1; /* Omit the [JI]FUNC* header. */
245#if LJ_HASJIT 217#if LJ_HASJIT
246 uint8_t *p = (uint8_t *)&ctx->sb.buf[ctx->sb.n]; 218 uint8_t *q = (uint8_t *)p;
247#endif 219#endif
248 bcwrite_block(ctx, proto_bc(pt)+1, nbc*(MSize)sizeof(BCIns)); 220 p = lj_buf_wmem(p, proto_bc(pt)+1, nbc*(MSize)sizeof(BCIns));
221 UNUSED(ctx);
249#if LJ_HASJIT 222#if LJ_HASJIT
250 /* Unpatch modified bytecode containing ILOOP/JLOOP etc. */ 223 /* Unpatch modified bytecode containing ILOOP/JLOOP etc. */
251 if ((pt->flags & PROTO_ILOOP) || pt->trace) { 224 if ((pt->flags & PROTO_ILOOP) || pt->trace) {
252 jit_State *J = L2J(ctx->L); 225 jit_State *J = L2J(sbufL(&ctx->sb));
253 MSize i; 226 MSize i;
254 for (i = 0; i < nbc; i++, p += sizeof(BCIns)) { 227 for (i = 0; i < nbc; i++, q += sizeof(BCIns)) {
255 BCOp op = (BCOp)p[LJ_ENDIAN_SELECT(0, 3)]; 228 BCOp op = (BCOp)q[LJ_ENDIAN_SELECT(0, 3)];
256 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP || 229 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP ||
257 op == BC_JFORI) { 230 op == BC_JFORI) {
258 p[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_IFORL+BC_FORL); 231 q[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_IFORL+BC_FORL);
259 } else if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) { 232 } else if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
260 BCReg rd = p[LJ_ENDIAN_SELECT(2, 1)] + (p[LJ_ENDIAN_SELECT(3, 0)] << 8); 233 BCReg rd = q[LJ_ENDIAN_SELECT(2, 1)] + (q[LJ_ENDIAN_SELECT(3, 0)] << 8);
261 memcpy(p, &traceref(J, rd)->startins, 4); 234 memcpy(q, &traceref(J, rd)->startins, 4);
262 } 235 }
263 } 236 }
264 } 237 }
265#endif 238#endif
239 return p;
266} 240}
267 241
268/* Write prototype. */ 242/* Write prototype. */
269static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt) 243static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
270{ 244{
271 MSize sizedbg = 0; 245 MSize sizedbg = 0;
246 char *p;
272 247
273 /* Recursively write children of prototype. */ 248 /* Recursively write children of prototype. */
274 if ((pt->flags & PROTO_CHILD)) { 249 if ((pt->flags & PROTO_CHILD)) {
@@ -282,31 +257,32 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
282 } 257 }
283 258
284 /* Start writing the prototype info to a buffer. */ 259 /* Start writing the prototype info to a buffer. */
285 lj_str_resetbuf(&ctx->sb); 260 p = lj_buf_need(&ctx->sb,
286 ctx->sb.n = 5; /* Leave room for final size. */ 261 5+4+6*5+(pt->sizebc-1)*(MSize)sizeof(BCIns)+pt->sizeuv*2);
287 bcwrite_need(ctx, 4+6*5+(pt->sizebc-1)*(MSize)sizeof(BCIns)+pt->sizeuv*2); 262 p += 5; /* Leave room for final size. */
288 263
289 /* Write prototype header. */ 264 /* Write prototype header. */
290 bcwrite_byte(ctx, (pt->flags & (PROTO_CHILD|PROTO_VARARG|PROTO_FFI))); 265 *p++ = (pt->flags & (PROTO_CHILD|PROTO_VARARG|PROTO_FFI));
291 bcwrite_byte(ctx, pt->numparams); 266 *p++ = pt->numparams;
292 bcwrite_byte(ctx, pt->framesize); 267 *p++ = pt->framesize;
293 bcwrite_byte(ctx, pt->sizeuv); 268 *p++ = pt->sizeuv;
294 bcwrite_uleb128(ctx, pt->sizekgc); 269 p = lj_strfmt_wuleb128(p, pt->sizekgc);
295 bcwrite_uleb128(ctx, pt->sizekn); 270 p = lj_strfmt_wuleb128(p, pt->sizekn);
296 bcwrite_uleb128(ctx, pt->sizebc-1); 271 p = lj_strfmt_wuleb128(p, pt->sizebc-1);
297 if (!ctx->strip) { 272 if (!ctx->strip) {
298 if (proto_lineinfo(pt)) 273 if (proto_lineinfo(pt))
299 sizedbg = pt->sizept - (MSize)((char *)proto_lineinfo(pt) - (char *)pt); 274 sizedbg = pt->sizept - (MSize)((char *)proto_lineinfo(pt) - (char *)pt);
300 bcwrite_uleb128(ctx, sizedbg); 275 p = lj_strfmt_wuleb128(p, sizedbg);
301 if (sizedbg) { 276 if (sizedbg) {
302 bcwrite_uleb128(ctx, pt->firstline); 277 p = lj_strfmt_wuleb128(p, pt->firstline);
303 bcwrite_uleb128(ctx, pt->numline); 278 p = lj_strfmt_wuleb128(p, pt->numline);
304 } 279 }
305 } 280 }
306 281
307 /* Write bytecode instructions and upvalue refs. */ 282 /* Write bytecode instructions and upvalue refs. */
308 bcwrite_bytecode(ctx, pt); 283 p = bcwrite_bytecode(ctx, p, pt);
309 bcwrite_block(ctx, proto_uv(pt), pt->sizeuv*2); 284 p = lj_buf_wmem(p, proto_uv(pt), pt->sizeuv*2);
285 ctx->sb.w = p;
310 286
311 /* Write constants. */ 287 /* Write constants. */
312 bcwrite_kgc(ctx, pt); 288 bcwrite_kgc(ctx, pt);
@@ -314,18 +290,19 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
314 290
315 /* Write debug info, if not stripped. */ 291 /* Write debug info, if not stripped. */
316 if (sizedbg) { 292 if (sizedbg) {
317 bcwrite_need(ctx, sizedbg); 293 p = lj_buf_more(&ctx->sb, sizedbg);
318 bcwrite_block(ctx, proto_lineinfo(pt), sizedbg); 294 p = lj_buf_wmem(p, proto_lineinfo(pt), sizedbg);
295 ctx->sb.w = p;
319 } 296 }
320 297
321 /* Pass buffer to writer function. */ 298 /* Pass buffer to writer function. */
322 if (ctx->status == 0) { 299 if (ctx->status == 0) {
323 MSize n = ctx->sb.n - 5; 300 MSize n = sbuflen(&ctx->sb) - 5;
324 MSize nn = (lj_fls(n)+8)*9 >> 6; 301 MSize nn = (lj_fls(n)+8)*9 >> 6;
325 ctx->sb.n = 5 - nn; 302 char *q = ctx->sb.b + (5 - nn);
326 bcwrite_uleb128(ctx, n); /* Fill in final size. */ 303 p = lj_strfmt_wuleb128(q, n); /* Fill in final size. */
327 lua_assert(ctx->sb.n == 5); 304 lj_assertBCW(p == ctx->sb.b + 5, "bad ULEB128 write");
328 ctx->status = ctx->wfunc(ctx->L, ctx->sb.buf+5-nn, nn+n, ctx->wdata); 305 ctx->status = ctx->wfunc(sbufL(&ctx->sb), q, nn+n, ctx->wdata);
329 } 306 }
330} 307}
331 308
@@ -335,20 +312,21 @@ static void bcwrite_header(BCWriteCtx *ctx)
335 GCstr *chunkname = proto_chunkname(ctx->pt); 312 GCstr *chunkname = proto_chunkname(ctx->pt);
336 const char *name = strdata(chunkname); 313 const char *name = strdata(chunkname);
337 MSize len = chunkname->len; 314 MSize len = chunkname->len;
338 lj_str_resetbuf(&ctx->sb); 315 char *p = lj_buf_need(&ctx->sb, 5+5+len);
339 bcwrite_need(ctx, 5+5+len); 316 *p++ = BCDUMP_HEAD1;
340 bcwrite_byte(ctx, BCDUMP_HEAD1); 317 *p++ = BCDUMP_HEAD2;
341 bcwrite_byte(ctx, BCDUMP_HEAD2); 318 *p++ = BCDUMP_HEAD3;
342 bcwrite_byte(ctx, BCDUMP_HEAD3); 319 *p++ = BCDUMP_VERSION;
343 bcwrite_byte(ctx, BCDUMP_VERSION); 320 *p++ = (ctx->strip ? BCDUMP_F_STRIP : 0) +
344 bcwrite_byte(ctx, (ctx->strip ? BCDUMP_F_STRIP : 0) + 321 LJ_BE*BCDUMP_F_BE +
345 (LJ_BE ? BCDUMP_F_BE : 0) + 322 ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0) +
346 ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0)); 323 LJ_FR2*BCDUMP_F_FR2;
347 if (!ctx->strip) { 324 if (!ctx->strip) {
348 bcwrite_uleb128(ctx, len); 325 p = lj_strfmt_wuleb128(p, len);
349 bcwrite_block(ctx, name, len); 326 p = lj_buf_wmem(p, name, len);
350 } 327 }
351 ctx->status = ctx->wfunc(ctx->L, ctx->sb.buf, ctx->sb.n, ctx->wdata); 328 ctx->status = ctx->wfunc(sbufL(&ctx->sb), ctx->sb.b,
329 (MSize)(p - ctx->sb.b), ctx->wdata);
352} 330}
353 331
354/* Write footer of bytecode dump. */ 332/* Write footer of bytecode dump. */
@@ -356,7 +334,7 @@ static void bcwrite_footer(BCWriteCtx *ctx)
356{ 334{
357 if (ctx->status == 0) { 335 if (ctx->status == 0) {
358 uint8_t zero = 0; 336 uint8_t zero = 0;
359 ctx->status = ctx->wfunc(ctx->L, &zero, 1, ctx->wdata); 337 ctx->status = ctx->wfunc(sbufL(&ctx->sb), &zero, 1, ctx->wdata);
360 } 338 }
361} 339}
362 340
@@ -364,8 +342,8 @@ static void bcwrite_footer(BCWriteCtx *ctx)
364static TValue *cpwriter(lua_State *L, lua_CFunction dummy, void *ud) 342static TValue *cpwriter(lua_State *L, lua_CFunction dummy, void *ud)
365{ 343{
366 BCWriteCtx *ctx = (BCWriteCtx *)ud; 344 BCWriteCtx *ctx = (BCWriteCtx *)ud;
367 UNUSED(dummy); 345 UNUSED(L); UNUSED(dummy);
368 lj_str_resizebuf(L, &ctx->sb, 1024); /* Avoids resize for most prototypes. */ 346 lj_buf_need(&ctx->sb, 1024); /* Avoids resize for most prototypes. */
369 bcwrite_header(ctx); 347 bcwrite_header(ctx);
370 bcwrite_proto(ctx, ctx->pt); 348 bcwrite_proto(ctx, ctx->pt);
371 bcwrite_footer(ctx); 349 bcwrite_footer(ctx);
@@ -378,16 +356,18 @@ int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data,
378{ 356{
379 BCWriteCtx ctx; 357 BCWriteCtx ctx;
380 int status; 358 int status;
381 ctx.L = L;
382 ctx.pt = pt; 359 ctx.pt = pt;
383 ctx.wfunc = writer; 360 ctx.wfunc = writer;
384 ctx.wdata = data; 361 ctx.wdata = data;
385 ctx.strip = strip; 362 ctx.strip = strip;
386 ctx.status = 0; 363 ctx.status = 0;
387 lj_str_initbuf(&ctx.sb); 364#ifdef LUA_USE_ASSERT
365 ctx.g = G(L);
366#endif
367 lj_buf_init(L, &ctx.sb);
388 status = lj_vm_cpcall(L, NULL, &ctx, cpwriter); 368 status = lj_vm_cpcall(L, NULL, &ctx, cpwriter);
389 if (status == 0) status = ctx.status; 369 if (status == 0) status = ctx.status;
390 lj_str_freebuf(G(ctx.L), &ctx.sb); 370 lj_buf_free(G(sbufL(&ctx.sb)), &ctx.sb);
391 return status; 371 return status;
392} 372}
393 373
diff --git a/src/lj_buf.c b/src/lj_buf.c
new file mode 100644
index 00000000..ae2ccd82
--- /dev/null
+++ b/src/lj_buf.c
@@ -0,0 +1,305 @@
1/*
2** Buffer handling.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_buf_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10#include "lj_gc.h"
11#include "lj_err.h"
12#include "lj_buf.h"
13#include "lj_str.h"
14#include "lj_tab.h"
15#include "lj_strfmt.h"
16
17/* -- Buffer management --------------------------------------------------- */
18
19static void buf_grow(SBuf *sb, MSize sz)
20{
21 MSize osz = sbufsz(sb), len = sbuflen(sb), nsz = osz;
22 char *b;
23 GCSize flag;
24 if (nsz < LJ_MIN_SBUF) nsz = LJ_MIN_SBUF;
25 while (nsz < sz) nsz += nsz;
26 flag = sbufflag(sb);
27 if ((flag & SBUF_FLAG_COW)) { /* Copy-on-write semantics. */
28 lj_assertG_(G(sbufL(sb)), sb->w == sb->e, "bad SBuf COW");
29 b = (char *)lj_mem_new(sbufL(sb), nsz);
30 setsbufflag(sb, flag & ~(GCSize)SBUF_FLAG_COW);
31 setgcrefnull(sbufX(sb)->cowref);
32 memcpy(b, sb->b, osz);
33 } else {
34 b = (char *)lj_mem_realloc(sbufL(sb), sb->b, osz, nsz);
35 }
36 if ((flag & SBUF_FLAG_EXT)) {
37 sbufX(sb)->r = sbufX(sb)->r - sb->b + b; /* Adjust read pointer, too. */
38 }
39 /* Adjust buffer pointers. */
40 sb->b = b;
41 sb->w = b + len;
42 sb->e = b + nsz;
43 if ((flag & SBUF_FLAG_BORROW)) { /* Adjust borrowed buffer pointers. */
44 SBuf *bsb = mref(sbufX(sb)->bsb, SBuf);
45 bsb->b = b;
46 bsb->w = b + len;
47 bsb->e = b + nsz;
48 }
49}
50
51LJ_NOINLINE char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz)
52{
53 lj_assertG_(G(sbufL(sb)), sz > sbufsz(sb), "SBuf overflow");
54 if (LJ_UNLIKELY(sz > LJ_MAX_BUF))
55 lj_err_mem(sbufL(sb));
56 buf_grow(sb, sz);
57 return sb->b;
58}
59
60LJ_NOINLINE char *LJ_FASTCALL lj_buf_more2(SBuf *sb, MSize sz)
61{
62 if (sbufisext(sb)) {
63 SBufExt *sbx = (SBufExt *)sb;
64 MSize len = sbufxlen(sbx);
65 if (LJ_UNLIKELY(sz > LJ_MAX_BUF || len + sz > LJ_MAX_BUF))
66 lj_err_mem(sbufL(sbx));
67 if (len + sz > sbufsz(sbx)) { /* Must grow. */
68 buf_grow((SBuf *)sbx, len + sz);
69 } else if (sbufiscow(sb) || sbufxslack(sbx) < (sbufsz(sbx) >> 3)) {
70 /* Also grow to avoid excessive compactions, if slack < size/8. */
71 buf_grow((SBuf *)sbx, sbuflen(sbx) + sz); /* Not sbufxlen! */
72 return sbx->w;
73 }
74 if (sbx->r != sbx->b) { /* Compact by moving down. */
75 memmove(sbx->b, sbx->r, len);
76 sbx->r = sbx->b;
77 sbx->w = sbx->b + len;
78 lj_assertG_(G(sbufL(sbx)), len + sz <= sbufsz(sbx), "bad SBuf compact");
79 }
80 } else {
81 MSize len = sbuflen(sb);
82 lj_assertG_(G(sbufL(sb)), sz > sbufleft(sb), "SBuf overflow");
83 if (LJ_UNLIKELY(sz > LJ_MAX_BUF || len + sz > LJ_MAX_BUF))
84 lj_err_mem(sbufL(sb));
85 buf_grow(sb, len + sz);
86 }
87 return sb->w;
88}
89
90void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb)
91{
92 char *b = sb->b;
93 MSize osz = (MSize)(sb->e - b);
94 if (osz > 2*LJ_MIN_SBUF) {
95 MSize n = (MSize)(sb->w - b);
96 b = lj_mem_realloc(L, b, osz, (osz >> 1));
97 sb->b = b;
98 sb->w = b + n;
99 sb->e = b + (osz >> 1);
100 }
101 lj_assertG_(G(sbufL(sb)), !sbufisext(sb), "YAGNI shrink SBufExt");
102}
103
104char * LJ_FASTCALL lj_buf_tmp(lua_State *L, MSize sz)
105{
106 SBuf *sb = &G(L)->tmpbuf;
107 setsbufL(sb, L);
108 return lj_buf_need(sb, sz);
109}
110
111#if LJ_HASBUFFER && LJ_HASJIT
112void lj_bufx_set(SBufExt *sbx, const char *p, MSize len, GCobj *ref)
113{
114 lua_State *L = sbufL(sbx);
115 lj_bufx_free(L, sbx);
116 lj_bufx_set_cow(L, sbx, p, len);
117 setgcref(sbx->cowref, ref);
118 lj_gc_objbarrier(L, (GCudata *)sbx - 1, ref);
119}
120
121#if LJ_HASFFI
122MSize LJ_FASTCALL lj_bufx_more(SBufExt *sbx, MSize sz)
123{
124 lj_buf_more((SBuf *)sbx, sz);
125 return sbufleft(sbx);
126}
127#endif
128#endif
129
130/* -- Low-level buffer put operations ------------------------------------- */
131
132SBuf *lj_buf_putmem(SBuf *sb, const void *q, MSize len)
133{
134 char *w = lj_buf_more(sb, len);
135 w = lj_buf_wmem(w, q, len);
136 sb->w = w;
137 return sb;
138}
139
140#if LJ_HASJIT || LJ_HASFFI
141static LJ_NOINLINE SBuf * LJ_FASTCALL lj_buf_putchar2(SBuf *sb, int c)
142{
143 char *w = lj_buf_more2(sb, 1);
144 *w++ = (char)c;
145 sb->w = w;
146 return sb;
147}
148
149SBuf * LJ_FASTCALL lj_buf_putchar(SBuf *sb, int c)
150{
151 char *w = sb->w;
152 if (LJ_LIKELY(w < sb->e)) {
153 *w++ = (char)c;
154 sb->w = w;
155 return sb;
156 }
157 return lj_buf_putchar2(sb, c);
158}
159#endif
160
161SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s)
162{
163 MSize len = s->len;
164 char *w = lj_buf_more(sb, len);
165 w = lj_buf_wmem(w, strdata(s), len);
166 sb->w = w;
167 return sb;
168}
169
170/* -- High-level buffer put operations ------------------------------------ */
171
172SBuf * LJ_FASTCALL lj_buf_putstr_reverse(SBuf *sb, GCstr *s)
173{
174 MSize len = s->len;
175 char *w = lj_buf_more(sb, len), *e = w+len;
176 const char *q = strdata(s)+len-1;
177 while (w < e)
178 *w++ = *q--;
179 sb->w = w;
180 return sb;
181}
182
183SBuf * LJ_FASTCALL lj_buf_putstr_lower(SBuf *sb, GCstr *s)
184{
185 MSize len = s->len;
186 char *w = lj_buf_more(sb, len), *e = w+len;
187 const char *q = strdata(s);
188 for (; w < e; w++, q++) {
189 uint32_t c = *(unsigned char *)q;
190#if LJ_TARGET_PPC
191 *w = c + ((c >= 'A' && c <= 'Z') << 5);
192#else
193 if (c >= 'A' && c <= 'Z') c += 0x20;
194 *w = c;
195#endif
196 }
197 sb->w = w;
198 return sb;
199}
200
201SBuf * LJ_FASTCALL lj_buf_putstr_upper(SBuf *sb, GCstr *s)
202{
203 MSize len = s->len;
204 char *w = lj_buf_more(sb, len), *e = w+len;
205 const char *q = strdata(s);
206 for (; w < e; w++, q++) {
207 uint32_t c = *(unsigned char *)q;
208#if LJ_TARGET_PPC
209 *w = c - ((c >= 'a' && c <= 'z') << 5);
210#else
211 if (c >= 'a' && c <= 'z') c -= 0x20;
212 *w = c;
213#endif
214 }
215 sb->w = w;
216 return sb;
217}
218
219SBuf *lj_buf_putstr_rep(SBuf *sb, GCstr *s, int32_t rep)
220{
221 MSize len = s->len;
222 if (rep > 0 && len) {
223 uint64_t tlen = (uint64_t)rep * len;
224 char *w;
225 if (LJ_UNLIKELY(tlen > LJ_MAX_STR))
226 lj_err_mem(sbufL(sb));
227 w = lj_buf_more(sb, (MSize)tlen);
228 if (len == 1) { /* Optimize a common case. */
229 uint32_t c = strdata(s)[0];
230 do { *w++ = c; } while (--rep > 0);
231 } else {
232 const char *e = strdata(s) + len;
233 do {
234 const char *q = strdata(s);
235 do { *w++ = *q++; } while (q < e);
236 } while (--rep > 0);
237 }
238 sb->w = w;
239 }
240 return sb;
241}
242
243SBuf *lj_buf_puttab(SBuf *sb, GCtab *t, GCstr *sep, int32_t i, int32_t e)
244{
245 MSize seplen = sep ? sep->len : 0;
246 if (i <= e) {
247 for (;;) {
248 cTValue *o = lj_tab_getint(t, i);
249 char *w;
250 if (!o) {
251 badtype: /* Error: bad element type. */
252 sb->w = (char *)(intptr_t)i; /* Store failing index. */
253 return NULL;
254 } else if (tvisstr(o)) {
255 MSize len = strV(o)->len;
256 w = lj_buf_wmem(lj_buf_more(sb, len + seplen), strVdata(o), len);
257 } else if (tvisint(o)) {
258 w = lj_strfmt_wint(lj_buf_more(sb, STRFMT_MAXBUF_INT+seplen), intV(o));
259 } else if (tvisnum(o)) {
260 w = lj_buf_more(lj_strfmt_putfnum(sb, STRFMT_G14, numV(o)), seplen);
261 } else {
262 goto badtype;
263 }
264 if (i++ == e) {
265 sb->w = w;
266 break;
267 }
268 if (seplen) w = lj_buf_wmem(w, strdata(sep), seplen);
269 sb->w = w;
270 }
271 }
272 return sb;
273}
274
275/* -- Miscellaneous buffer operations ------------------------------------- */
276
277GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb)
278{
279 return lj_str_new(sbufL(sb), sb->b, sbuflen(sb));
280}
281
282/* Concatenate two strings. */
283GCstr *lj_buf_cat2str(lua_State *L, GCstr *s1, GCstr *s2)
284{
285 MSize len1 = s1->len, len2 = s2->len;
286 char *buf = lj_buf_tmp(L, len1 + len2);
287 memcpy(buf, strdata(s1), len1);
288 memcpy(buf+len1, strdata(s2), len2);
289 return lj_str_new(L, buf, len1 + len2);
290}
291
292/* Read ULEB128 from buffer. */
293uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp)
294{
295 const uint8_t *w = (const uint8_t *)*pp;
296 uint32_t v = *w++;
297 if (LJ_UNLIKELY(v >= 0x80)) {
298 int sh = 0;
299 v &= 0x7f;
300 do { v |= ((*w & 0x7f) << (sh += 7)); } while (*w++ >= 0x80);
301 }
302 *pp = (const char *)w;
303 return v;
304}
305
diff --git a/src/lj_buf.h b/src/lj_buf.h
new file mode 100644
index 00000000..744e5747
--- /dev/null
+++ b/src/lj_buf.h
@@ -0,0 +1,198 @@
1/*
2** Buffer handling.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_BUF_H
7#define _LJ_BUF_H
8
9#include "lj_obj.h"
10#include "lj_gc.h"
11#include "lj_str.h"
12
13/* Resizable string buffers. */
14
15/* The SBuf struct definition is in lj_obj.h:
16** char *w; Write pointer.
17** char *e; End pointer.
18** char *b; Base pointer.
19** MRef L; lua_State, used for buffer resizing. Extension bits in 3 LSB.
20*/
21
22/* Extended string buffer. */
23typedef struct SBufExt {
24 SBufHeader;
25 union {
26 GCRef cowref; /* Copy-on-write object reference. */
27 MRef bsb; /* Borrowed string buffer. */
28 };
29 char *r; /* Read pointer. */
30 GCRef dict_str; /* Serialization string dictionary table. */
31 GCRef dict_mt; /* Serialization metatable dictionary table. */
32 int depth; /* Remaining recursion depth. */
33} SBufExt;
34
35#define sbufsz(sb) ((MSize)((sb)->e - (sb)->b))
36#define sbuflen(sb) ((MSize)((sb)->w - (sb)->b))
37#define sbufleft(sb) ((MSize)((sb)->e - (sb)->w))
38#define sbufxlen(sbx) ((MSize)((sbx)->w - (sbx)->r))
39#define sbufxslack(sbx) ((MSize)((sbx)->r - (sbx)->b))
40
41#define SBUF_MASK_FLAG (7)
42#define SBUF_MASK_L (~(GCSize)SBUF_MASK_FLAG)
43#define SBUF_FLAG_EXT 1 /* Extended string buffer. */
44#define SBUF_FLAG_COW 2 /* Copy-on-write buffer. */
45#define SBUF_FLAG_BORROW 4 /* Borrowed string buffer. */
46
47#define sbufL(sb) \
48 ((lua_State *)(void *)(uintptr_t)(mrefu((sb)->L) & SBUF_MASK_L))
49#define setsbufL(sb, l) (setmref((sb)->L, (l)))
50#define setsbufXL(sb, l, flag) \
51 (setmrefu((sb)->L, (GCSize)(uintptr_t)(void *)(l) + (flag)))
52#define setsbufXL_(sb, l) \
53 (setmrefu((sb)->L, (GCSize)(uintptr_t)(void *)(l) | (mrefu((sb)->L) & SBUF_MASK_FLAG)))
54
55#define sbufflag(sb) (mrefu((sb)->L))
56#define sbufisext(sb) (sbufflag((sb)) & SBUF_FLAG_EXT)
57#define sbufiscow(sb) (sbufflag((sb)) & SBUF_FLAG_COW)
58#define sbufisborrow(sb) (sbufflag((sb)) & SBUF_FLAG_BORROW)
59#define sbufiscoworborrow(sb) (sbufflag((sb)) & (SBUF_FLAG_COW|SBUF_FLAG_BORROW))
60#define sbufX(sb) \
61 (lj_assertG_(G(sbufL(sb)), sbufisext(sb), "not an SBufExt"), (SBufExt *)(sb))
62#define setsbufflag(sb, flag) (setmrefu((sb)->L, (flag)))
63
64#define tvisbuf(o) \
65 (LJ_HASBUFFER && tvisudata(o) && udataV(o)->udtype == UDTYPE_BUFFER)
66#define bufV(o) check_exp(tvisbuf(o), ((SBufExt *)uddata(udataV(o))))
67
68/* Buffer management */
69LJ_FUNC char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz);
70LJ_FUNC char *LJ_FASTCALL lj_buf_more2(SBuf *sb, MSize sz);
71LJ_FUNC void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb);
72LJ_FUNC char * LJ_FASTCALL lj_buf_tmp(lua_State *L, MSize sz);
73
74static LJ_AINLINE void lj_buf_init(lua_State *L, SBuf *sb)
75{
76 setsbufL(sb, L);
77 sb->w = sb->e = sb->b = NULL;
78}
79
80static LJ_AINLINE void lj_buf_reset(SBuf *sb)
81{
82 sb->w = sb->b;
83}
84
85static LJ_AINLINE SBuf *lj_buf_tmp_(lua_State *L)
86{
87 SBuf *sb = &G(L)->tmpbuf;
88 setsbufL(sb, L);
89 lj_buf_reset(sb);
90 return sb;
91}
92
93static LJ_AINLINE void lj_buf_free(global_State *g, SBuf *sb)
94{
95 lj_assertG(!sbufisext(sb), "bad free of SBufExt");
96 lj_mem_free(g, sb->b, sbufsz(sb));
97}
98
99static LJ_AINLINE char *lj_buf_need(SBuf *sb, MSize sz)
100{
101 if (LJ_UNLIKELY(sz > sbufsz(sb)))
102 return lj_buf_need2(sb, sz);
103 return sb->b;
104}
105
106static LJ_AINLINE char *lj_buf_more(SBuf *sb, MSize sz)
107{
108 if (LJ_UNLIKELY(sz > sbufleft(sb)))
109 return lj_buf_more2(sb, sz);
110 return sb->w;
111}
112
113/* Extended buffer management */
114static LJ_AINLINE void lj_bufx_init(lua_State *L, SBufExt *sbx)
115{
116 memset(sbx, 0, sizeof(SBufExt));
117 setsbufXL(sbx, L, SBUF_FLAG_EXT);
118}
119
120static LJ_AINLINE void lj_bufx_set_borrow(lua_State *L, SBufExt *sbx, SBuf *sb)
121{
122 setsbufXL(sbx, L, SBUF_FLAG_EXT | SBUF_FLAG_BORROW);
123 setmref(sbx->bsb, sb);
124 sbx->r = sbx->w = sbx->b = sb->b;
125 sbx->e = sb->e;
126}
127
128static LJ_AINLINE void lj_bufx_set_cow(lua_State *L, SBufExt *sbx,
129 const char *p, MSize len)
130{
131 setsbufXL(sbx, L, SBUF_FLAG_EXT | SBUF_FLAG_COW);
132 sbx->r = sbx->b = (char *)p;
133 sbx->w = sbx->e = (char *)p + len;
134}
135
136static LJ_AINLINE void lj_bufx_reset(SBufExt *sbx)
137{
138 if (sbufiscow(sbx)) {
139 setmrefu(sbx->L, (mrefu(sbx->L) & ~(GCSize)SBUF_FLAG_COW));
140 setgcrefnull(sbx->cowref);
141 sbx->b = sbx->e = NULL;
142 }
143 sbx->r = sbx->w = sbx->b;
144}
145
146static LJ_AINLINE void lj_bufx_free(lua_State *L, SBufExt *sbx)
147{
148 if (!sbufiscoworborrow(sbx)) lj_mem_free(G(L), sbx->b, sbufsz(sbx));
149 setsbufXL(sbx, L, SBUF_FLAG_EXT);
150 setgcrefnull(sbx->cowref);
151 sbx->r = sbx->w = sbx->b = sbx->e = NULL;
152}
153
154#if LJ_HASBUFFER && LJ_HASJIT
155LJ_FUNC void lj_bufx_set(SBufExt *sbx, const char *p, MSize len, GCobj *o);
156#if LJ_HASFFI
157LJ_FUNC MSize LJ_FASTCALL lj_bufx_more(SBufExt *sbx, MSize sz);
158#endif
159#endif
160
161/* Low-level buffer put operations */
162LJ_FUNC SBuf *lj_buf_putmem(SBuf *sb, const void *q, MSize len);
163#if LJ_HASJIT || LJ_HASFFI
164LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putchar(SBuf *sb, int c);
165#endif
166LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s);
167
168static LJ_AINLINE char *lj_buf_wmem(char *p, const void *q, MSize len)
169{
170 return (char *)memcpy(p, q, len) + len;
171}
172
173static LJ_AINLINE void lj_buf_putb(SBuf *sb, int c)
174{
175 char *w = lj_buf_more(sb, 1);
176 *w++ = (char)c;
177 sb->w = w;
178}
179
180/* High-level buffer put operations */
181LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_reverse(SBuf *sb, GCstr *s);
182LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_lower(SBuf *sb, GCstr *s);
183LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_upper(SBuf *sb, GCstr *s);
184LJ_FUNC SBuf *lj_buf_putstr_rep(SBuf *sb, GCstr *s, int32_t rep);
185LJ_FUNC SBuf *lj_buf_puttab(SBuf *sb, GCtab *t, GCstr *sep,
186 int32_t i, int32_t e);
187
188/* Miscellaneous buffer operations */
189LJ_FUNCA GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb);
190LJ_FUNC GCstr *lj_buf_cat2str(lua_State *L, GCstr *s1, GCstr *s2);
191LJ_FUNC uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp);
192
193static LJ_AINLINE GCstr *lj_buf_str(lua_State *L, SBuf *sb)
194{
195 return lj_str_new(L, sb->b, sbuflen(sb));
196}
197
198#endif
diff --git a/src/lj_carith.c b/src/lj_carith.c
index 96384e87..df5f801e 100644
--- a/src/lj_carith.c
+++ b/src/lj_carith.c
@@ -11,10 +11,12 @@
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_tab.h" 12#include "lj_tab.h"
13#include "lj_meta.h" 13#include "lj_meta.h"
14#include "lj_ir.h"
14#include "lj_ctype.h" 15#include "lj_ctype.h"
15#include "lj_cconv.h" 16#include "lj_cconv.h"
16#include "lj_cdata.h" 17#include "lj_cdata.h"
17#include "lj_carith.h" 18#include "lj_carith.h"
19#include "lj_strscan.h"
18 20
19/* -- C data arithmetic --------------------------------------------------- */ 21/* -- C data arithmetic --------------------------------------------------- */
20 22
@@ -120,7 +122,7 @@ static int carith_ptr(lua_State *L, CTState *cts, CDArith *ca, MMS mm)
120 setboolV(L->top-1, ((uintptr_t)pp < (uintptr_t)pp2)); 122 setboolV(L->top-1, ((uintptr_t)pp < (uintptr_t)pp2));
121 return 1; 123 return 1;
122 } else { 124 } else {
123 lua_assert(mm == MM_le); 125 lj_assertL(mm == MM_le, "bad metamethod %d", mm);
124 setboolV(L->top-1, ((uintptr_t)pp <= (uintptr_t)pp2)); 126 setboolV(L->top-1, ((uintptr_t)pp <= (uintptr_t)pp2));
125 return 1; 127 return 1;
126 } 128 }
@@ -206,7 +208,9 @@ static int carith_int64(lua_State *L, CTState *cts, CDArith *ca, MMS mm)
206 *up = lj_carith_powu64(u0, u1); 208 *up = lj_carith_powu64(u0, u1);
207 break; 209 break;
208 case MM_unm: *up = ~u0+1u; break; 210 case MM_unm: *up = ~u0+1u; break;
209 default: lua_assert(0); break; 211 default:
212 lj_assertL(0, "bad metamethod %d", mm);
213 break;
210 } 214 }
211 lj_gc_check(L); 215 lj_gc_check(L);
212 return 1; 216 return 1;
@@ -272,6 +276,81 @@ int lj_carith_op(lua_State *L, MMS mm)
272 return lj_carith_meta(L, cts, &ca, mm); 276 return lj_carith_meta(L, cts, &ca, mm);
273} 277}
274 278
279/* -- 64 bit bit operations helpers --------------------------------------- */
280
281#if LJ_64
282#define B64DEF(name) \
283 static LJ_AINLINE uint64_t lj_carith_##name(uint64_t x, int32_t sh)
284#else
285/* Not inlined on 32 bit archs, since some of these are quite lengthy. */
286#define B64DEF(name) \
287 uint64_t LJ_NOINLINE lj_carith_##name(uint64_t x, int32_t sh)
288#endif
289
290B64DEF(shl64) { return x << (sh&63); }
291B64DEF(shr64) { return x >> (sh&63); }
292B64DEF(sar64) { return (uint64_t)((int64_t)x >> (sh&63)); }
293B64DEF(rol64) { return lj_rol(x, (sh&63)); }
294B64DEF(ror64) { return lj_ror(x, (sh&63)); }
295
296#undef B64DEF
297
298uint64_t lj_carith_shift64(uint64_t x, int32_t sh, int op)
299{
300 switch (op) {
301 case IR_BSHL-IR_BSHL: x = lj_carith_shl64(x, sh); break;
302 case IR_BSHR-IR_BSHL: x = lj_carith_shr64(x, sh); break;
303 case IR_BSAR-IR_BSHL: x = lj_carith_sar64(x, sh); break;
304 case IR_BROL-IR_BSHL: x = lj_carith_rol64(x, sh); break;
305 case IR_BROR-IR_BSHL: x = lj_carith_ror64(x, sh); break;
306 default:
307 lj_assertX(0, "bad shift op %d", op);
308 break;
309 }
310 return x;
311}
312
313/* Equivalent to lj_lib_checkbit(), but handles cdata. */
314uint64_t lj_carith_check64(lua_State *L, int narg, CTypeID *id)
315{
316 TValue *o = L->base + narg-1;
317 if (o >= L->top) {
318 err:
319 lj_err_argt(L, narg, LUA_TNUMBER);
320 } else if (LJ_LIKELY(tvisnumber(o))) {
321 /* Handled below. */
322 } else if (tviscdata(o)) {
323 CTState *cts = ctype_cts(L);
324 uint8_t *sp = (uint8_t *)cdataptr(cdataV(o));
325 CTypeID sid = cdataV(o)->ctypeid;
326 CType *s = ctype_get(cts, sid);
327 uint64_t x;
328 if (ctype_isref(s->info)) {
329 sp = *(void **)sp;
330 sid = ctype_cid(s->info);
331 }
332 s = ctype_raw(cts, sid);
333 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
334 if ((s->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
335 CTINFO(CT_NUM, CTF_UNSIGNED) && s->size == 8)
336 *id = CTID_UINT64; /* Use uint64_t, since it has the highest rank. */
337 else if (!*id)
338 *id = CTID_INT64; /* Use int64_t, unless already set. */
339 lj_cconv_ct_ct(cts, ctype_get(cts, *id), s,
340 (uint8_t *)&x, sp, CCF_ARG(narg));
341 return x;
342 } else if (!(tvisstr(o) && lj_strscan_number(strV(o), o))) {
343 goto err;
344 }
345 if (LJ_LIKELY(tvisint(o))) {
346 return (uint32_t)intV(o);
347 } else {
348 int32_t i = lj_num2bit(numV(o));
349 if (LJ_DUALNUM) setintV(o, i);
350 return (uint32_t)i;
351 }
352}
353
275/* -- 64 bit integer arithmetic helpers ----------------------------------- */ 354/* -- 64 bit integer arithmetic helpers ----------------------------------- */
276 355
277#if LJ_32 && LJ_HASJIT 356#if LJ_32 && LJ_HASJIT
diff --git a/src/lj_carith.h b/src/lj_carith.h
index d87c2d07..f124c27c 100644
--- a/src/lj_carith.h
+++ b/src/lj_carith.h
@@ -12,6 +12,16 @@
12 12
13LJ_FUNC int lj_carith_op(lua_State *L, MMS mm); 13LJ_FUNC int lj_carith_op(lua_State *L, MMS mm);
14 14
15#if LJ_32
16LJ_FUNC uint64_t lj_carith_shl64(uint64_t x, int32_t sh);
17LJ_FUNC uint64_t lj_carith_shr64(uint64_t x, int32_t sh);
18LJ_FUNC uint64_t lj_carith_sar64(uint64_t x, int32_t sh);
19LJ_FUNC uint64_t lj_carith_rol64(uint64_t x, int32_t sh);
20LJ_FUNC uint64_t lj_carith_ror64(uint64_t x, int32_t sh);
21#endif
22LJ_FUNC uint64_t lj_carith_shift64(uint64_t x, int32_t sh, int op);
23LJ_FUNC uint64_t lj_carith_check64(lua_State *L, int narg, CTypeID *id);
24
15#if LJ_32 && LJ_HASJIT 25#if LJ_32 && LJ_HASJIT
16LJ_FUNC int64_t lj_carith_mul64(int64_t x, int64_t k); 26LJ_FUNC int64_t lj_carith_mul64(int64_t x, int64_t k);
17#endif 27#endif
diff --git a/src/lj_ccall.c b/src/lj_ccall.c
index 79e726c6..04e306eb 100644
--- a/src/lj_ccall.c
+++ b/src/lj_ccall.c
@@ -9,7 +9,6 @@
9 9
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_str.h"
13#include "lj_tab.h" 12#include "lj_tab.h"
14#include "lj_ctype.h" 13#include "lj_ctype.h"
15#include "lj_cconv.h" 14#include "lj_cconv.h"
@@ -291,56 +290,85 @@
291#define CCALL_HANDLE_RET \ 290#define CCALL_HANDLE_RET \
292 if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0]; 291 if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
293 292
294#elif LJ_TARGET_PPC 293#elif LJ_TARGET_ARM64
295/* -- PPC calling conventions --------------------------------------------- */ 294/* -- ARM64 calling conventions ------------------------------------------- */
296 295
297#define CCALL_HANDLE_STRUCTRET \ 296#define CCALL_HANDLE_STRUCTRET \
298 cc->retref = 1; /* Return all structs by reference. */ \ 297 cc->retref = !ccall_classify_struct(cts, ctr); \
299 cc->gpr[ngpr++] = (GPRArg)dp; 298 if (cc->retref) cc->retp = dp;
299
300#define CCALL_HANDLE_STRUCTRET2 \
301 unsigned int cl = ccall_classify_struct(cts, ctr); \
302 if ((cl & 4)) { /* Combine float HFA from separate registers. */ \
303 CTSize i = (cl >> 8) - 1; \
304 do { ((uint32_t *)dp)[i] = cc->fpr[i].lo; } while (i--); \
305 } else { \
306 if (cl > 1) sp = (uint8_t *)&cc->fpr[0]; \
307 memcpy(dp, sp, ctr->size); \
308 }
300 309
301#define CCALL_HANDLE_COMPLEXRET \ 310#define CCALL_HANDLE_COMPLEXRET \
302 /* Complex values are returned in 2 or 4 GPRs. */ \ 311 /* Complex values are returned in one or two FPRs. */ \
303 cc->retref = 0; 312 cc->retref = 0;
304 313
305#define CCALL_HANDLE_COMPLEXRET2 \ 314#define CCALL_HANDLE_COMPLEXRET2 \
306 memcpy(dp, sp, ctr->size); /* Copy complex from GPRs. */ 315 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
316 ((float *)dp)[0] = cc->fpr[0].f; \
317 ((float *)dp)[1] = cc->fpr[1].f; \
318 } else { /* Copy complex double from FPRs. */ \
319 ((double *)dp)[0] = cc->fpr[0].d; \
320 ((double *)dp)[1] = cc->fpr[1].d; \
321 }
307 322
308#define CCALL_HANDLE_STRUCTARG \ 323#define CCALL_HANDLE_STRUCTARG \
309 rp = cdataptr(lj_cdata_new(cts, did, sz)); \ 324 unsigned int cl = ccall_classify_struct(cts, d); \
310 sz = CTSIZE_PTR; /* Pass all structs by reference. */ 325 if (cl == 0) { /* Pass struct by reference. */ \
326 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
327 sz = CTSIZE_PTR; \
328 } else if (cl > 1) { /* Pass struct in FPRs or on stack. */ \
329 isfp = (cl & 4) ? 2 : 1; \
330 } /* else: Pass struct in GPRs or on stack. */
311 331
312#define CCALL_HANDLE_COMPLEXARG \ 332#define CCALL_HANDLE_COMPLEXARG \
313 /* Pass complex by value in 2 or 4 GPRs. */ 333 /* Pass complex by value in separate (!) FPRs or on stack. */ \
334 isfp = sz == 2*sizeof(float) ? 2 : 1;
314 335
315#define CCALL_HANDLE_REGARG \ 336#define CCALL_HANDLE_REGARG \
316 if (isfp) { /* Try to pass argument in FPRs. */ \ 337 if (LJ_TARGET_OSX && isva) { \
317 if (nfpr + 1 <= CCALL_NARG_FPR) { \ 338 /* IOS: All variadic arguments are on the stack. */ \
339 } else if (isfp) { /* Try to pass argument in FPRs. */ \
340 int n2 = ctype_isvector(d->info) ? 1 : \
341 isfp == 1 ? n : (d->size >> (4-isfp)); \
342 if (nfpr + n2 <= CCALL_NARG_FPR) { \
318 dp = &cc->fpr[nfpr]; \ 343 dp = &cc->fpr[nfpr]; \
319 nfpr += 1; \ 344 nfpr += n2; \
320 d = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
321 goto done; \ 345 goto done; \
346 } else { \
347 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
348 if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \
322 } \ 349 } \
323 } else { /* Try to pass argument in GPRs. */ \ 350 } else { /* Try to pass argument in GPRs. */ \
324 if (n > 1) { \ 351 if (!LJ_TARGET_OSX && (d->info & CTF_ALIGN) > CTALIGN_PTR) \
325 lua_assert(n == 2 || n == 4); /* int64_t or complex (float). */ \ 352 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
326 if (ctype_isinteger(d->info)) \
327 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
328 else if (ngpr + n > maxgpr) \
329 ngpr = maxgpr; /* Prevent reordering. */ \
330 } \
331 if (ngpr + n <= maxgpr) { \ 353 if (ngpr + n <= maxgpr) { \
332 dp = &cc->gpr[ngpr]; \ 354 dp = &cc->gpr[ngpr]; \
333 ngpr += n; \ 355 ngpr += n; \
334 goto done; \ 356 goto done; \
357 } else { \
358 ngpr = maxgpr; /* Prevent reordering. */ \
359 if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \
335 } \ 360 } \
336 } 361 }
337 362
363#if LJ_BE
338#define CCALL_HANDLE_RET \ 364#define CCALL_HANDLE_RET \
339 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 365 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
340 ctr = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ 366 sp = (uint8_t *)&cc->fpr[0].f;
367#endif
341 368
342#elif LJ_TARGET_PPCSPE 369
343/* -- PPC/SPE calling conventions ----------------------------------------- */ 370#elif LJ_TARGET_PPC
371/* -- PPC calling conventions --------------------------------------------- */
344 372
345#define CCALL_HANDLE_STRUCTRET \ 373#define CCALL_HANDLE_STRUCTRET \
346 cc->retref = 1; /* Return all structs by reference. */ \ 374 cc->retref = 1; /* Return all structs by reference. */ \
@@ -360,12 +388,13 @@
360#define CCALL_HANDLE_COMPLEXARG \ 388#define CCALL_HANDLE_COMPLEXARG \
361 /* Pass complex by value in 2 or 4 GPRs. */ 389 /* Pass complex by value in 2 or 4 GPRs. */
362 390
363/* PPC/SPE has a softfp ABI. */ 391#define CCALL_HANDLE_GPR \
364#define CCALL_HANDLE_REGARG \ 392 /* Try to pass argument in GPRs. */ \
365 if (n > 1) { /* Doesn't fit in a single GPR? */ \ 393 if (n > 1) { \
366 lua_assert(n == 2 || n == 4); /* int64_t, double or complex (float). */ \ 394 /* int64_t or complex (float). */ \
367 if (n == 2) \ 395 lj_assertL(n == 2 || n == 4, "bad GPR size %d", n); \
368 ngpr = (ngpr + 1u) & ~1u; /* Only align 64 bit value to regpair. */ \ 396 if (ctype_isinteger(d->info) || ctype_isfp(d->info)) \
397 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
369 else if (ngpr + n > maxgpr) \ 398 else if (ngpr + n > maxgpr) \
370 ngpr = maxgpr; /* Prevent reordering. */ \ 399 ngpr = maxgpr; /* Prevent reordering. */ \
371 } \ 400 } \
@@ -373,10 +402,32 @@
373 dp = &cc->gpr[ngpr]; \ 402 dp = &cc->gpr[ngpr]; \
374 ngpr += n; \ 403 ngpr += n; \
375 goto done; \ 404 goto done; \
405 } \
406
407#if LJ_ABI_SOFTFP
408#define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
409#else
410#define CCALL_HANDLE_REGARG \
411 if (isfp) { /* Try to pass argument in FPRs. */ \
412 if (nfpr + 1 <= CCALL_NARG_FPR) { \
413 dp = &cc->fpr[nfpr]; \
414 nfpr += 1; \
415 d = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
416 goto done; \
417 } \
418 } else { \
419 CCALL_HANDLE_GPR \
376 } 420 }
421#endif
377 422
378#elif LJ_TARGET_MIPS 423#if !LJ_ABI_SOFTFP
379/* -- MIPS calling conventions -------------------------------------------- */ 424#define CCALL_HANDLE_RET \
425 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
426 ctr = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */
427#endif
428
429#elif LJ_TARGET_MIPS32
430/* -- MIPS o32 calling conventions ---------------------------------------- */
380 431
381#define CCALL_HANDLE_STRUCTRET \ 432#define CCALL_HANDLE_STRUCTRET \
382 cc->retref = 1; /* Return all structs by reference. */ \ 433 cc->retref = 1; /* Return all structs by reference. */ \
@@ -386,6 +437,18 @@
386 /* Complex values are returned in 1 or 2 FPRs. */ \ 437 /* Complex values are returned in 1 or 2 FPRs. */ \
387 cc->retref = 0; 438 cc->retref = 0;
388 439
440#if LJ_ABI_SOFTFP
441#define CCALL_HANDLE_COMPLEXRET2 \
442 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
443 ((intptr_t *)dp)[0] = cc->gpr[0]; \
444 ((intptr_t *)dp)[1] = cc->gpr[1]; \
445 } else { /* Copy complex double from GPRs. */ \
446 ((intptr_t *)dp)[0] = cc->gpr[0]; \
447 ((intptr_t *)dp)[1] = cc->gpr[1]; \
448 ((intptr_t *)dp)[2] = cc->gpr[2]; \
449 ((intptr_t *)dp)[3] = cc->gpr[3]; \
450 }
451#else
389#define CCALL_HANDLE_COMPLEXRET2 \ 452#define CCALL_HANDLE_COMPLEXRET2 \
390 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \ 453 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
391 ((float *)dp)[0] = cc->fpr[0].f; \ 454 ((float *)dp)[0] = cc->fpr[0].f; \
@@ -394,6 +457,7 @@
394 ((double *)dp)[0] = cc->fpr[0].d; \ 457 ((double *)dp)[0] = cc->fpr[0].d; \
395 ((double *)dp)[1] = cc->fpr[1].d; \ 458 ((double *)dp)[1] = cc->fpr[1].d; \
396 } 459 }
460#endif
397 461
398#define CCALL_HANDLE_STRUCTARG \ 462#define CCALL_HANDLE_STRUCTARG \
399 /* Pass all structs by value in registers and/or on the stack. */ 463 /* Pass all structs by value in registers and/or on the stack. */
@@ -401,6 +465,22 @@
401#define CCALL_HANDLE_COMPLEXARG \ 465#define CCALL_HANDLE_COMPLEXARG \
402 /* Pass complex by value in 2 or 4 GPRs. */ 466 /* Pass complex by value in 2 or 4 GPRs. */
403 467
468#define CCALL_HANDLE_GPR \
469 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
470 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
471 if (ngpr < maxgpr) { \
472 dp = &cc->gpr[ngpr]; \
473 if (ngpr + n > maxgpr) { \
474 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
475 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
476 ngpr = maxgpr; \
477 } else { \
478 ngpr += n; \
479 } \
480 goto done; \
481 }
482
483#if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
404#define CCALL_HANDLE_REGARG \ 484#define CCALL_HANDLE_REGARG \
405 if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \ 485 if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
406 /* Try to pass argument in FPRs. */ \ 486 /* Try to pass argument in FPRs. */ \
@@ -409,25 +489,91 @@
409 goto done; \ 489 goto done; \
410 } else { /* Try to pass argument in GPRs. */ \ 490 } else { /* Try to pass argument in GPRs. */ \
411 nfpr = CCALL_NARG_FPR; \ 491 nfpr = CCALL_NARG_FPR; \
412 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \ 492 CCALL_HANDLE_GPR \
413 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ 493 }
414 if (ngpr < maxgpr) { \ 494#else /* MIPS32 soft-float */
415 dp = &cc->gpr[ngpr]; \ 495#define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
416 if (ngpr + n > maxgpr) { \ 496#endif
417 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \ 497
418 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \ 498#if !LJ_ABI_SOFTFP
419 ngpr = maxgpr; \ 499/* On MIPS64 soft-float, position of float return values is endian-dependant. */
420 } else { \ 500#define CCALL_HANDLE_RET \
421 ngpr += n; \ 501 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
422 } \ 502 sp = (uint8_t *)&cc->fpr[0].f;
423 goto done; \ 503#endif
424 } \ 504
505#elif LJ_TARGET_MIPS64
506/* -- MIPS n64 calling conventions ---------------------------------------- */
507
508#define CCALL_HANDLE_STRUCTRET \
509 cc->retref = !(sz <= 16); \
510 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
511
512#define CCALL_HANDLE_STRUCTRET2 \
513 ccall_copy_struct(cc, ctr, dp, sp, ccall_classify_struct(cts, ctr, ct));
514
515#define CCALL_HANDLE_COMPLEXRET \
516 /* Complex values are returned in 1 or 2 FPRs. */ \
517 cc->retref = 0;
518
519#if LJ_ABI_SOFTFP /* MIPS64 soft-float */
520
521#define CCALL_HANDLE_COMPLEXRET2 \
522 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
523 ((intptr_t *)dp)[0] = cc->gpr[0]; \
524 } else { /* Copy complex double from GPRs. */ \
525 ((intptr_t *)dp)[0] = cc->gpr[0]; \
526 ((intptr_t *)dp)[1] = cc->gpr[1]; \
527 }
528
529#define CCALL_HANDLE_COMPLEXARG \
530 /* Pass complex by value in 2 or 4 GPRs. */
531
532/* Position of soft-float 'float' return value depends on endianess. */
533#define CCALL_HANDLE_RET \
534 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
535 sp = (uint8_t *)cc->gpr + LJ_ENDIAN_SELECT(0, 4);
536
537#else /* MIPS64 hard-float */
538
539#define CCALL_HANDLE_COMPLEXRET2 \
540 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
541 ((float *)dp)[0] = cc->fpr[0].f; \
542 ((float *)dp)[1] = cc->fpr[1].f; \
543 } else { /* Copy complex double from FPRs. */ \
544 ((double *)dp)[0] = cc->fpr[0].d; \
545 ((double *)dp)[1] = cc->fpr[1].d; \
546 }
547
548#define CCALL_HANDLE_COMPLEXARG \
549 if (sz == 2*sizeof(float)) { \
550 isfp = 2; \
551 if (ngpr < maxgpr) \
552 sz *= 2; \
425 } 553 }
426 554
427#define CCALL_HANDLE_RET \ 555#define CCALL_HANDLE_RET \
428 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 556 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
429 sp = (uint8_t *)&cc->fpr[0].f; 557 sp = (uint8_t *)&cc->fpr[0].f;
430 558
559#endif
560
561#define CCALL_HANDLE_STRUCTARG \
562 /* Pass all structs by value in registers and/or on the stack. */
563
564#define CCALL_HANDLE_REGARG \
565 if (ngpr < maxgpr) { \
566 dp = &cc->gpr[ngpr]; \
567 if (ngpr + n > maxgpr) { \
568 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
569 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
570 ngpr = maxgpr; \
571 } else { \
572 ngpr += n; \
573 } \
574 goto done; \
575 }
576
431#else 577#else
432#error "Missing calling convention definitions for this architecture" 578#error "Missing calling convention definitions for this architecture"
433#endif 579#endif
@@ -497,7 +643,8 @@ static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
497 ccall_classify_struct(cts, ct, rcl, ofs); 643 ccall_classify_struct(cts, ct, rcl, ofs);
498 } else { 644 } else {
499 int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT; 645 int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
500 lua_assert(ctype_hassize(ct->info)); 646 lj_assertCTS(ctype_hassize(ct->info),
647 "classify ctype %08x without size", ct->info);
501 if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM; /* Unaligned. */ 648 if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM; /* Unaligned. */
502 rcl[(ofs >= 8)] |= cl; 649 rcl[(ofs >= 8)] |= cl;
503 } 650 }
@@ -522,12 +669,13 @@ static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
522} 669}
523 670
524/* Try to split up a small struct into registers. */ 671/* Try to split up a small struct into registers. */
525static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl) 672static int ccall_struct_reg(CCallState *cc, CTState *cts, GPRArg *dp, int *rcl)
526{ 673{
527 MSize ngpr = cc->ngpr, nfpr = cc->nfpr; 674 MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
528 uint32_t i; 675 uint32_t i;
676 UNUSED(cts);
529 for (i = 0; i < 2; i++) { 677 for (i = 0; i < 2; i++) {
530 lua_assert(!(rcl[i] & CCALL_RCL_MEM)); 678 lj_assertCTS(!(rcl[i] & CCALL_RCL_MEM), "pass mem struct in reg");
531 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */ 679 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
532 if (ngpr >= CCALL_NARG_GPR) return 1; /* Register overflow. */ 680 if (ngpr >= CCALL_NARG_GPR) return 1; /* Register overflow. */
533 cc->gpr[ngpr++] = dp[i]; 681 cc->gpr[ngpr++] = dp[i];
@@ -548,7 +696,8 @@ static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
548 dp[0] = dp[1] = 0; 696 dp[0] = dp[1] = 0;
549 /* Convert to temp. struct. */ 697 /* Convert to temp. struct. */
550 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg)); 698 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
551 if (ccall_struct_reg(cc, dp, rcl)) { /* Register overflow? Pass on stack. */ 699 if (ccall_struct_reg(cc, cts, dp, rcl)) {
700 /* Register overflow? Pass on stack. */
552 MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1; 701 MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
553 if (nsp + n > CCALL_MAXSTACK) return 1; /* Too many arguments. */ 702 if (nsp + n > CCALL_MAXSTACK) return 1; /* Too many arguments. */
554 cc->nsp = nsp + n; 703 cc->nsp = nsp + n;
@@ -621,6 +770,125 @@ noth: /* Not a homogeneous float/double aggregate. */
621 770
622#endif 771#endif
623 772
773/* -- ARM64 ABI struct classification ------------------------------------- */
774
775#if LJ_TARGET_ARM64
776
777/* Classify a struct based on its fields. */
778static unsigned int ccall_classify_struct(CTState *cts, CType *ct)
779{
780 CTSize sz = ct->size;
781 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
782 while (ct->sib) {
783 CType *sct;
784 ct = ctype_get(cts, ct->sib);
785 if (ctype_isfield(ct->info)) {
786 sct = ctype_rawchild(cts, ct);
787 if (ctype_isfp(sct->info)) {
788 r |= sct->size;
789 if (!isu) n++; else if (n == 0) n = 1;
790 } else if (ctype_iscomplex(sct->info)) {
791 r |= (sct->size >> 1);
792 if (!isu) n += 2; else if (n < 2) n = 2;
793 } else if (ctype_isstruct(sct->info)) {
794 goto substruct;
795 } else {
796 goto noth;
797 }
798 } else if (ctype_isbitfield(ct->info)) {
799 goto noth;
800 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
801 sct = ctype_rawchild(cts, ct);
802 substruct:
803 if (sct->size > 0) {
804 unsigned int s = ccall_classify_struct(cts, sct);
805 if (s <= 1) goto noth;
806 r |= (s & 255);
807 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
808 }
809 }
810 }
811 if ((r == 4 || r == 8) && n <= 4)
812 return r + (n << 8);
813noth: /* Not a homogeneous float/double aggregate. */
814 return (sz <= 16); /* Return structs of size <= 16 in GPRs. */
815}
816
817#endif
818
819/* -- MIPS64 ABI struct classification ---------------------------- */
820
821#if LJ_TARGET_MIPS64
822
823#define FTYPE_FLOAT 1
824#define FTYPE_DOUBLE 2
825
826/* Classify FP fields (max. 2) and their types. */
827static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
828{
829 int n = 0, ft = 0;
830 if ((ctf->info & CTF_VARARG) || (ct->info & CTF_UNION))
831 goto noth;
832 while (ct->sib) {
833 CType *sct;
834 ct = ctype_get(cts, ct->sib);
835 if (n == 2) {
836 goto noth;
837 } else if (ctype_isfield(ct->info)) {
838 sct = ctype_rawchild(cts, ct);
839 if (ctype_isfp(sct->info)) {
840 ft |= (sct->size == 4 ? FTYPE_FLOAT : FTYPE_DOUBLE) << 2*n;
841 n++;
842 } else {
843 goto noth;
844 }
845 } else if (ctype_isbitfield(ct->info) ||
846 ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
847 goto noth;
848 }
849 }
850 if (n <= 2)
851 return ft;
852noth: /* Not a homogeneous float/double aggregate. */
853 return 0; /* Struct is in GPRs. */
854}
855
856static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
857 int ft)
858{
859 if (LJ_ABI_SOFTFP ? ft :
860 ((ft & 3) == FTYPE_FLOAT || (ft >> 2) == FTYPE_FLOAT)) {
861 int i, ofs = 0;
862 for (i = 0; ft != 0; i++, ft >>= 2) {
863 if ((ft & 3) == FTYPE_FLOAT) {
864#if LJ_ABI_SOFTFP
865 /* The 2nd FP struct result is in CARG1 (gpr[2]) and not CRET2. */
866 memcpy((uint8_t *)dp + ofs,
867 (uint8_t *)&cc->gpr[2*i] + LJ_ENDIAN_SELECT(0, 4), 4);
868#else
869 *(float *)((uint8_t *)dp + ofs) = cc->fpr[i].f;
870#endif
871 ofs += 4;
872 } else {
873 ofs = (ofs + 7) & ~7; /* 64 bit alignment. */
874#if LJ_ABI_SOFTFP
875 *(intptr_t *)((uint8_t *)dp + ofs) = cc->gpr[2*i];
876#else
877 *(double *)((uint8_t *)dp + ofs) = cc->fpr[i].d;
878#endif
879 ofs += 8;
880 }
881 }
882 } else {
883#if !LJ_ABI_SOFTFP
884 if (ft) sp = (uint8_t *)&cc->fpr[0];
885#endif
886 memcpy(dp, sp, ctr->size);
887 }
888}
889
890#endif
891
624/* -- Common C call handling ---------------------------------------------- */ 892/* -- Common C call handling ---------------------------------------------- */
625 893
626/* Infer the destination CTypeID for a vararg argument. */ 894/* Infer the destination CTypeID for a vararg argument. */
@@ -726,7 +994,7 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
726 if (fid) { /* Get argument type from field. */ 994 if (fid) { /* Get argument type from field. */
727 CType *ctf = ctype_get(cts, fid); 995 CType *ctf = ctype_get(cts, fid);
728 fid = ctf->sib; 996 fid = ctf->sib;
729 lua_assert(ctype_isfield(ctf->info)); 997 lj_assertL(ctype_isfield(ctf->info), "field expected");
730 did = ctype_cid(ctf->info); 998 did = ctype_cid(ctf->info);
731 } else { 999 } else {
732 if (!(ct->info & CTF_VARARG)) 1000 if (!(ct->info & CTF_VARARG))
@@ -788,6 +1056,19 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
788 *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp : 1056 *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
789 (int32_t)*(int16_t *)dp; 1057 (int32_t)*(int16_t *)dp;
790 } 1058 }
1059#if LJ_TARGET_ARM64 && LJ_BE
1060 if (isfp && d->size == sizeof(float))
1061 ((float *)dp)[1] = ((float *)dp)[0]; /* Floats occupy high slot. */
1062#endif
1063#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
1064 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
1065#if LJ_TARGET_MIPS64
1066 || (isfp && nsp == 0)
1067#endif
1068 ) && d->size <= 4) {
1069 *(int64_t *)dp = (int64_t)*(int32_t *)dp; /* Sign-extend to 64 bit. */
1070 }
1071#endif
791#if LJ_TARGET_X64 && LJ_ABI_WIN 1072#if LJ_TARGET_X64 && LJ_ABI_WIN
792 if (isva) { /* Windows/x64 mirrors varargs in both register sets. */ 1073 if (isva) { /* Windows/x64 mirrors varargs in both register sets. */
793 if (nfpr == ngpr) 1074 if (nfpr == ngpr)
@@ -803,13 +1084,19 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
803 cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1]; /* Split complex double. */ 1084 cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1]; /* Split complex double. */
804 cc->fpr[nfpr-2].d[1] = 0; 1085 cc->fpr[nfpr-2].d[1] = 0;
805 } 1086 }
1087#elif LJ_TARGET_ARM64 || (LJ_TARGET_MIPS64 && !LJ_ABI_SOFTFP)
1088 if (isfp == 2 && (uint8_t *)dp < (uint8_t *)cc->stack) {
1089 /* Split float HFA or complex float into separate registers. */
1090 CTSize i = (sz >> 2) - 1;
1091 do { ((uint64_t *)dp)[i] = ((uint32_t *)dp)[i]; } while (i--);
1092 }
806#else 1093#else
807 UNUSED(isfp); 1094 UNUSED(isfp);
808#endif 1095#endif
809 } 1096 }
810 if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too few arguments. */ 1097 if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too few arguments. */
811 1098
812#if LJ_TARGET_X64 || LJ_TARGET_PPC 1099#if LJ_TARGET_X64 || (LJ_TARGET_PPC && !LJ_ABI_SOFTFP)
813 cc->nfpr = nfpr; /* Required for vararg functions. */ 1100 cc->nfpr = nfpr; /* Required for vararg functions. */
814#endif 1101#endif
815 cc->nsp = nsp; 1102 cc->nsp = nsp;
@@ -844,7 +1131,8 @@ static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
844 CCALL_HANDLE_COMPLEXRET2 1131 CCALL_HANDLE_COMPLEXRET2
845 return 1; /* One GC step. */ 1132 return 1; /* One GC step. */
846 } 1133 }
847 if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR) 1134 if (LJ_BE && ctr->size < CTSIZE_PTR &&
1135 (ctype_isinteger_or_bool(ctr->info) || ctype_isenum(ctr->info)))
848 sp += (CTSIZE_PTR - ctr->size); 1136 sp += (CTSIZE_PTR - ctr->size);
849#if CCALL_NUM_FPR 1137#if CCALL_NUM_FPR
850 if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info)) 1138 if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
@@ -854,7 +1142,8 @@ static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
854 CCALL_HANDLE_RET 1142 CCALL_HANDLE_RET
855#endif 1143#endif
856 /* No reference types end up here, so there's no need for the CTypeID. */ 1144 /* No reference types end up here, so there's no need for the CTypeID. */
857 lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info))); 1145 lj_assertL(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)),
1146 "unexpected reference ctype");
858 return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp); 1147 return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
859} 1148}
860 1149
@@ -878,7 +1167,7 @@ int lj_ccall_func(lua_State *L, GCcdata *cd)
878 lj_vm_ffi_call(&cc); 1167 lj_vm_ffi_call(&cc);
879 if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */ 1168 if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */
880 TValue tv; 1169 TValue tv;
881 setlightudV(&tv, (void *)cc.func); 1170 tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
882 setboolV(lj_tab_set(L, cts->miscmap, &tv), 1); 1171 setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
883 } 1172 }
884 ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */ 1173 ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */
diff --git a/src/lj_ccall.h b/src/lj_ccall.h
index 2f4fa7a6..547415f7 100644
--- a/src/lj_ccall.h
+++ b/src/lj_ccall.h
@@ -68,35 +68,56 @@ typedef union FPRArg {
68 float f[2]; 68 float f[2];
69} FPRArg; 69} FPRArg;
70 70
71#elif LJ_TARGET_PPC 71#elif LJ_TARGET_ARM64
72 72
73#define CCALL_NARG_GPR 8 73#define CCALL_NARG_GPR 8
74#define CCALL_NRET_GPR 2
74#define CCALL_NARG_FPR 8 75#define CCALL_NARG_FPR 8
76#define CCALL_NRET_FPR 4
77#define CCALL_SPS_FREE 0
78
79typedef intptr_t GPRArg;
80typedef union FPRArg {
81 double d;
82 struct { LJ_ENDIAN_LOHI(float f; , float g;) };
83 struct { LJ_ENDIAN_LOHI(uint32_t lo; , uint32_t hi;) };
84} FPRArg;
85
86#elif LJ_TARGET_PPC
87
88#define CCALL_NARG_GPR 8
89#define CCALL_NARG_FPR (LJ_ABI_SOFTFP ? 0 : 8)
75#define CCALL_NRET_GPR 4 /* For complex double. */ 90#define CCALL_NRET_GPR 4 /* For complex double. */
76#define CCALL_NRET_FPR 1 91#define CCALL_NRET_FPR (LJ_ABI_SOFTFP ? 0 : 1)
77#define CCALL_SPS_EXTRA 4 92#define CCALL_SPS_EXTRA 4
78#define CCALL_SPS_FREE 0 93#define CCALL_SPS_FREE 0
79 94
80typedef intptr_t GPRArg; 95typedef intptr_t GPRArg;
81typedef double FPRArg; 96typedef double FPRArg;
82 97
83#elif LJ_TARGET_PPCSPE 98#elif LJ_TARGET_MIPS32
84 99
85#define CCALL_NARG_GPR 8 100#define CCALL_NARG_GPR 4
86#define CCALL_NARG_FPR 0 101#define CCALL_NARG_FPR (LJ_ABI_SOFTFP ? 0 : 2)
87#define CCALL_NRET_GPR 4 /* For softfp complex double. */ 102#define CCALL_NRET_GPR (LJ_ABI_SOFTFP ? 4 : 2)
88#define CCALL_NRET_FPR 0 103#define CCALL_NRET_FPR (LJ_ABI_SOFTFP ? 0 : 2)
89#define CCALL_SPS_FREE 0 /* NYI */ 104#define CCALL_SPS_EXTRA 7
105#define CCALL_SPS_FREE 1
90 106
91typedef intptr_t GPRArg; 107typedef intptr_t GPRArg;
108typedef union FPRArg {
109 double d;
110 struct { LJ_ENDIAN_LOHI(float f; , float g;) };
111} FPRArg;
92 112
93#elif LJ_TARGET_MIPS 113#elif LJ_TARGET_MIPS64
94 114
95#define CCALL_NARG_GPR 4 115/* FP args are positional and overlay the GPR array. */
96#define CCALL_NARG_FPR 2 116#define CCALL_NARG_GPR 8
117#define CCALL_NARG_FPR 0
97#define CCALL_NRET_GPR 2 118#define CCALL_NRET_GPR 2
98#define CCALL_NRET_FPR 2 119#define CCALL_NRET_FPR (LJ_ABI_SOFTFP ? 0 : 2)
99#define CCALL_SPS_EXTRA 7 120#define CCALL_SPS_EXTRA 3
100#define CCALL_SPS_FREE 1 121#define CCALL_SPS_FREE 1
101 122
102typedef intptr_t GPRArg; 123typedef intptr_t GPRArg;
@@ -145,6 +166,8 @@ typedef LJ_ALIGN(CCALL_ALIGN_CALLSTATE) struct CCallState {
145 uint8_t nfpr; /* Number of arguments in FPRs. */ 166 uint8_t nfpr; /* Number of arguments in FPRs. */
146#elif LJ_TARGET_X86 167#elif LJ_TARGET_X86
147 uint8_t resx87; /* Result on x87 stack: 1:float, 2:double. */ 168 uint8_t resx87; /* Result on x87 stack: 1:float, 2:double. */
169#elif LJ_TARGET_ARM64
170 void *retp; /* Aggregate return pointer in x8. */
148#elif LJ_TARGET_PPC 171#elif LJ_TARGET_PPC
149 uint8_t nfpr; /* Number of arguments in FPRs. */ 172 uint8_t nfpr; /* Number of arguments in FPRs. */
150#endif 173#endif
diff --git a/src/lj_ccallback.c b/src/lj_ccallback.c
index 363fef45..17d26b52 100644
--- a/src/lj_ccallback.c
+++ b/src/lj_ccallback.c
@@ -27,7 +27,7 @@
27 27
28#if LJ_OS_NOJIT 28#if LJ_OS_NOJIT
29 29
30/* Disabled callback support. */ 30/* Callbacks disabled. */
31#define CALLBACK_SLOT2OFS(slot) (0*(slot)) 31#define CALLBACK_SLOT2OFS(slot) (0*(slot))
32#define CALLBACK_OFS2SLOT(ofs) (0*(ofs)) 32#define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
33#define CALLBACK_MAX_SLOT 0 33#define CALLBACK_MAX_SLOT 0
@@ -35,7 +35,7 @@
35#elif LJ_TARGET_X86ORX64 35#elif LJ_TARGET_X86ORX64
36 36
37#define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0) 37#define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
38#define CALLBACK_MCODE_GROUP (-2+1+2+5+(LJ_64 ? 6 : 5)) 38#define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
39 39
40#define CALLBACK_SLOT2OFS(slot) \ 40#define CALLBACK_SLOT2OFS(slot) \
41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot)) 41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
@@ -54,23 +54,22 @@ static MSize CALLBACK_OFS2SLOT(MSize ofs)
54#elif LJ_TARGET_ARM 54#elif LJ_TARGET_ARM
55 55
56#define CALLBACK_MCODE_HEAD 32 56#define CALLBACK_MCODE_HEAD 32
57#define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot)) 57
58#define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8) 58#elif LJ_TARGET_ARM64
59#define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE)) 59
60#define CALLBACK_MCODE_HEAD 32
60 61
61#elif LJ_TARGET_PPC 62#elif LJ_TARGET_PPC
62 63
63#define CALLBACK_MCODE_HEAD 24 64#define CALLBACK_MCODE_HEAD 24
64#define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
65#define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
66#define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
67 65
68#elif LJ_TARGET_MIPS 66#elif LJ_TARGET_MIPS32
69 67
70#define CALLBACK_MCODE_HEAD 24 68#define CALLBACK_MCODE_HEAD 20
71#define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot)) 69
72#define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8) 70#elif LJ_TARGET_MIPS64
73#define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE)) 71
72#define CALLBACK_MCODE_HEAD 52
74 73
75#else 74#else
76 75
@@ -81,6 +80,12 @@ static MSize CALLBACK_OFS2SLOT(MSize ofs)
81 80
82#endif 81#endif
83 82
83#ifndef CALLBACK_SLOT2OFS
84#define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
85#define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
86#define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
87#endif
88
84/* Convert callback slot number to callback function pointer. */ 89/* Convert callback slot number to callback function pointer. */
85static void *callback_slot2ptr(CTState *cts, MSize slot) 90static void *callback_slot2ptr(CTState *cts, MSize slot)
86{ 91{
@@ -102,9 +107,9 @@ MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
102/* Initialize machine code for callback function pointers. */ 107/* Initialize machine code for callback function pointers. */
103#if LJ_OS_NOJIT 108#if LJ_OS_NOJIT
104/* Disabled callback support. */ 109/* Disabled callback support. */
105#define callback_mcode_init(g, p) UNUSED(p) 110#define callback_mcode_init(g, p) (p)
106#elif LJ_TARGET_X86ORX64 111#elif LJ_TARGET_X86ORX64
107static void callback_mcode_init(global_State *g, uint8_t *page) 112static void *callback_mcode_init(global_State *g, uint8_t *page)
108{ 113{
109 uint8_t *p = page; 114 uint8_t *p = page;
110 uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback; 115 uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
@@ -119,8 +124,13 @@ static void callback_mcode_init(global_State *g, uint8_t *page)
119 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */ 124 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
120 *p++ = XI_PUSH + RID_EBP; 125 *p++ = XI_PUSH + RID_EBP;
121 *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8); 126 *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
127#if LJ_GC64
128 *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
129 *(uint64_t *)p = (uint64_t)(g); p += 8;
130#else
122 *p++ = XI_MOVri | RID_EBP; 131 *p++ = XI_MOVri | RID_EBP;
123 *(int32_t *)p = i32ptr(g); p += 4; 132 *(int32_t *)p = i32ptr(g); p += 4;
133#endif
124#if LJ_64 134#if LJ_64
125 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */ 135 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
126 *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP; 136 *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
@@ -133,10 +143,10 @@ static void callback_mcode_init(global_State *g, uint8_t *page)
133 *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2); 143 *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
134 } 144 }
135 } 145 }
136 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 146 return p;
137} 147}
138#elif LJ_TARGET_ARM 148#elif LJ_TARGET_ARM
139static void callback_mcode_init(global_State *g, uint32_t *page) 149static void *callback_mcode_init(global_State *g, uint32_t *page)
140{ 150{
141 uint32_t *p = page; 151 uint32_t *p = page;
142 void *target = (void *)lj_vm_ffi_callback; 152 void *target = (void *)lj_vm_ffi_callback;
@@ -155,10 +165,30 @@ static void callback_mcode_init(global_State *g, uint32_t *page)
155 *p = ARMI_B | ((page-p-2) & 0x00ffffffu); 165 *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
156 p++; 166 p++;
157 } 167 }
158 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 168 return p;
169}
170#elif LJ_TARGET_ARM64
171static void *callback_mcode_init(global_State *g, uint32_t *page)
172{
173 uint32_t *p = page;
174 ASMFunction target = lj_vm_ffi_callback;
175 MSize slot;
176 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4));
177 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5));
178 *p++ = A64I_LE(A64I_BR_AUTH | A64F_N(RID_X11));
179 *p++ = A64I_LE(A64I_NOP);
180 ((ASMFunction *)p)[0] = target;
181 ((void **)p)[1] = g;
182 p += 4;
183 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
184 *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot));
185 *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu));
186 p++;
187 }
188 return p;
159} 189}
160#elif LJ_TARGET_PPC 190#elif LJ_TARGET_PPC
161static void callback_mcode_init(global_State *g, uint32_t *page) 191static void *callback_mcode_init(global_State *g, uint32_t *page)
162{ 192{
163 uint32_t *p = page; 193 uint32_t *p = page;
164 void *target = (void *)lj_vm_ffi_callback; 194 void *target = (void *)lj_vm_ffi_callback;
@@ -174,30 +204,43 @@ static void callback_mcode_init(global_State *g, uint32_t *page)
174 *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2); 204 *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
175 p++; 205 p++;
176 } 206 }
177 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 207 return p;
178} 208}
179#elif LJ_TARGET_MIPS 209#elif LJ_TARGET_MIPS
180static void callback_mcode_init(global_State *g, uint32_t *page) 210static void *callback_mcode_init(global_State *g, uint32_t *page)
181{ 211{
182 uint32_t *p = page; 212 uint32_t *p = page;
183 void *target = (void *)lj_vm_ffi_callback; 213 uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
214 uintptr_t ug = (uintptr_t)(void *)g;
184 MSize slot; 215 MSize slot;
185 *p++ = MIPSI_SW | MIPSF_T(RID_R1)|MIPSF_S(RID_SP) | 0; 216#if LJ_TARGET_MIPS32
186 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (u32ptr(target) >> 16); 217 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
187 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (u32ptr(g) >> 16); 218 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
188 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) |(u32ptr(target)&0xffff); 219#else
220 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 48);
221 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 48);
222 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
223 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
224 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
225 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
226 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
227 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
228 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
229 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
230#endif
231 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
189 *p++ = MIPSI_JR | MIPSF_S(RID_R3); 232 *p++ = MIPSI_JR | MIPSF_S(RID_R3);
190 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (u32ptr(g)&0xffff); 233 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
191 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 234 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
192 *p = MIPSI_B | ((page-p-1) & 0x0000ffffu); 235 *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
193 p++; 236 p++;
194 *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot; 237 *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
195 } 238 }
196 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 239 return p;
197} 240}
198#else 241#else
199/* Missing support for this architecture. */ 242/* Missing support for this architecture. */
200#define callback_mcode_init(g, p) UNUSED(p) 243#define callback_mcode_init(g, p) (p)
201#endif 244#endif
202 245
203/* -- Machine code management --------------------------------------------- */ 246/* -- Machine code management --------------------------------------------- */
@@ -213,6 +256,11 @@ static void callback_mcode_init(global_State *g, uint32_t *page)
213#ifndef MAP_ANONYMOUS 256#ifndef MAP_ANONYMOUS
214#define MAP_ANONYMOUS MAP_ANON 257#define MAP_ANONYMOUS MAP_ANON
215#endif 258#endif
259#ifdef PROT_MPROTECT
260#define CCPROT_CREATE (PROT_MPROTECT(PROT_EXEC))
261#else
262#define CCPROT_CREATE 0
263#endif
216 264
217#endif 265#endif
218 266
@@ -220,15 +268,15 @@ static void callback_mcode_init(global_State *g, uint32_t *page)
220static void callback_mcode_new(CTState *cts) 268static void callback_mcode_new(CTState *cts)
221{ 269{
222 size_t sz = (size_t)CALLBACK_MCODE_SIZE; 270 size_t sz = (size_t)CALLBACK_MCODE_SIZE;
223 void *p; 271 void *p, *pe;
224 if (CALLBACK_MAX_SLOT == 0) 272 if (CALLBACK_MAX_SLOT == 0)
225 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 273 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
226#if LJ_TARGET_WINDOWS 274#if LJ_TARGET_WINDOWS
227 p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); 275 p = LJ_WIN_VALLOC(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
228 if (!p) 276 if (!p)
229 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 277 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
230#elif LJ_TARGET_POSIX 278#elif LJ_TARGET_POSIX
231 p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS, 279 p = mmap(NULL, sz, (PROT_READ|PROT_WRITE|CCPROT_CREATE), MAP_PRIVATE|MAP_ANONYMOUS,
232 -1, 0); 280 -1, 0);
233 if (p == MAP_FAILED) 281 if (p == MAP_FAILED)
234 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 282 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
@@ -237,12 +285,15 @@ static void callback_mcode_new(CTState *cts)
237 p = lj_mem_new(cts->L, sz); 285 p = lj_mem_new(cts->L, sz);
238#endif 286#endif
239 cts->cb.mcode = p; 287 cts->cb.mcode = p;
240 callback_mcode_init(cts->g, p); 288 pe = callback_mcode_init(cts->g, p);
289 UNUSED(pe);
290 lj_assertCTS((size_t)((char *)pe - (char *)p) <= sz,
291 "miscalculated CALLBACK_MAX_SLOT");
241 lj_mcode_sync(p, (char *)p + sz); 292 lj_mcode_sync(p, (char *)p + sz);
242#if LJ_TARGET_WINDOWS 293#if LJ_TARGET_WINDOWS
243 { 294 {
244 DWORD oprot; 295 DWORD oprot;
245 VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot); 296 LJ_WIN_VPROTECT(p, sz, PAGE_EXECUTE_READ, &oprot);
246 } 297 }
247#elif LJ_TARGET_POSIX 298#elif LJ_TARGET_POSIX
248 mprotect(p, sz, (PROT_READ|PROT_EXEC)); 299 mprotect(p, sz, (PROT_READ|PROT_EXEC));
@@ -351,33 +402,78 @@ void lj_ccallback_mcode_free(CTState *cts)
351 goto done; \ 402 goto done; \
352 } CALLBACK_HANDLE_REGARG_FP2 403 } CALLBACK_HANDLE_REGARG_FP2
353 404
354#elif LJ_TARGET_PPC 405#elif LJ_TARGET_ARM64
355 406
356#define CALLBACK_HANDLE_REGARG \ 407#define CALLBACK_HANDLE_REGARG \
357 if (isfp) { \ 408 if (isfp) { \
358 if (nfpr + 1 <= CCALL_NARG_FPR) { \ 409 if (nfpr + n <= CCALL_NARG_FPR) { \
359 sp = &cts->cb.fpr[nfpr++]; \ 410 sp = &cts->cb.fpr[nfpr]; \
360 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \ 411 nfpr += n; \
361 goto done; \ 412 goto done; \
413 } else { \
414 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
362 } \ 415 } \
363 } else { /* Try to pass argument in GPRs. */ \ 416 } else { \
364 if (n > 1) { \ 417 if (!LJ_TARGET_OSX && n > 1) \
365 lua_assert(ctype_isinteger(cta->info) && n == 2); /* int64_t. */ \ 418 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
366 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
367 } \
368 if (ngpr + n <= maxgpr) { \ 419 if (ngpr + n <= maxgpr) { \
369 sp = &cts->cb.gpr[ngpr]; \ 420 sp = &cts->cb.gpr[ngpr]; \
370 ngpr += n; \ 421 ngpr += n; \
371 goto done; \ 422 goto done; \
423 } else { \
424 ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \
372 } \ 425 } \
373 } 426 }
374 427
428#elif LJ_TARGET_PPC
429
430#define CALLBACK_HANDLE_GPR \
431 if (n > 1) { \
432 lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) || /* double. */ \
433 ctype_isinteger(cta->info)) && n == 2, /* int64_t. */ \
434 "bad GPR type"); \
435 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
436 } \
437 if (ngpr + n <= maxgpr) { \
438 sp = &cts->cb.gpr[ngpr]; \
439 ngpr += n; \
440 goto done; \
441 }
442
443#if LJ_ABI_SOFTFP
444#define CALLBACK_HANDLE_REGARG \
445 CALLBACK_HANDLE_GPR \
446 UNUSED(isfp);
447#else
448#define CALLBACK_HANDLE_REGARG \
449 if (isfp) { \
450 if (nfpr + 1 <= CCALL_NARG_FPR) { \
451 sp = &cts->cb.fpr[nfpr++]; \
452 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
453 goto done; \
454 } \
455 } else { /* Try to pass argument in GPRs. */ \
456 CALLBACK_HANDLE_GPR \
457 }
458#endif
459
460#if !LJ_ABI_SOFTFP
375#define CALLBACK_HANDLE_RET \ 461#define CALLBACK_HANDLE_RET \
376 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 462 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
377 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */ 463 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */
464#endif
378 465
379#elif LJ_TARGET_MIPS 466#elif LJ_TARGET_MIPS32
380 467
468#define CALLBACK_HANDLE_GPR \
469 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
470 if (ngpr + n <= maxgpr) { \
471 sp = &cts->cb.gpr[ngpr]; \
472 ngpr += n; \
473 goto done; \
474 }
475
476#if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
381#define CALLBACK_HANDLE_REGARG \ 477#define CALLBACK_HANDLE_REGARG \
382 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \ 478 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \
383 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \ 479 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
@@ -385,13 +481,36 @@ void lj_ccallback_mcode_free(CTState *cts)
385 goto done; \ 481 goto done; \
386 } else { /* Try to pass argument in GPRs. */ \ 482 } else { /* Try to pass argument in GPRs. */ \
387 nfpr = CCALL_NARG_FPR; \ 483 nfpr = CCALL_NARG_FPR; \
388 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ 484 CALLBACK_HANDLE_GPR \
389 if (ngpr + n <= maxgpr) { \
390 sp = &cts->cb.gpr[ngpr]; \
391 ngpr += n; \
392 goto done; \
393 } \
394 } 485 }
486#else /* MIPS32 soft-float */
487#define CALLBACK_HANDLE_REGARG \
488 CALLBACK_HANDLE_GPR \
489 UNUSED(isfp);
490#endif
491
492#define CALLBACK_HANDLE_RET \
493 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
494 ((float *)dp)[1] = *(float *)dp;
495
496#elif LJ_TARGET_MIPS64
497
498#if !LJ_ABI_SOFTFP /* MIPS64 hard-float */
499#define CALLBACK_HANDLE_REGARG \
500 if (ngpr + n <= maxgpr) { \
501 sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
502 ngpr += n; \
503 goto done; \
504 }
505#else /* MIPS64 soft-float */
506#define CALLBACK_HANDLE_REGARG \
507 if (ngpr + n <= maxgpr) { \
508 UNUSED(isfp); \
509 sp = (void*) &cts->cb.gpr[ngpr]; \
510 ngpr += n; \
511 goto done; \
512 }
513#endif
395 514
396#define CALLBACK_HANDLE_RET \ 515#define CALLBACK_HANDLE_RET \
397 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 516 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
@@ -411,6 +530,7 @@ static void callback_conv_args(CTState *cts, lua_State *L)
411 int gcsteps = 0; 530 int gcsteps = 0;
412 CType *ct; 531 CType *ct;
413 GCfunc *fn; 532 GCfunc *fn;
533 int fntp;
414 MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR; 534 MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
415#if CCALL_NARG_FPR 535#if CCALL_NARG_FPR
416 MSize nfpr = 0; 536 MSize nfpr = 0;
@@ -421,18 +541,27 @@ static void callback_conv_args(CTState *cts, lua_State *L)
421 541
422 if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) { 542 if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
423 ct = ctype_get(cts, id); 543 ct = ctype_get(cts, id);
424 rid = ctype_cid(ct->info); 544 rid = ctype_cid(ct->info); /* Return type. x86: +(spadj<<16). */
425 fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot)); 545 fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
546 fntp = LJ_TFUNC;
426 } else { /* Must set up frame first, before throwing the error. */ 547 } else { /* Must set up frame first, before throwing the error. */
427 ct = NULL; 548 ct = NULL;
428 rid = 0; 549 rid = 0;
429 fn = (GCfunc *)L; 550 fn = (GCfunc *)L;
551 fntp = LJ_TTHREAD;
552 }
553 /* Continuation returns from callback. */
554 if (LJ_FR2) {
555 (o++)->u64 = LJ_CONT_FFI_CALLBACK;
556 (o++)->u64 = rid;
557 } else {
558 o->u32.lo = LJ_CONT_FFI_CALLBACK;
559 o->u32.hi = rid;
560 o++;
430 } 561 }
431 o->u32.lo = LJ_CONT_FFI_CALLBACK; /* Continuation returns from callback. */ 562 setframe_gc(o, obj2gco(fn), fntp);
432 o->u32.hi = rid; /* Return type. x86: +(spadj<<16). */ 563 if (LJ_FR2) o++;
433 o++; 564 setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
434 setframe_gc(o, obj2gco(fn));
435 setframe_ftsz(o, (int)((char *)(o+1) - (char *)L->base) + FRAME_CONT);
436 L->top = L->base = ++o; 565 L->top = L->base = ++o;
437 if (!ct) 566 if (!ct)
438 lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK); 567 lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
@@ -459,7 +588,7 @@ static void callback_conv_args(CTState *cts, lua_State *L)
459 CTSize sz; 588 CTSize sz;
460 int isfp; 589 int isfp;
461 MSize n; 590 MSize n;
462 lua_assert(ctype_isfield(ctf->info)); 591 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
463 cta = ctype_rawchild(cts, ctf); 592 cta = ctype_rawchild(cts, ctf);
464 isfp = ctype_isfp(cta->info); 593 isfp = ctype_isfp(cta->info);
465 sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1); 594 sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
@@ -474,7 +603,11 @@ static void callback_conv_args(CTState *cts, lua_State *L)
474 nsp += n; 603 nsp += n;
475 604
476 done: 605 done:
477 if (LJ_BE && cta->size < CTSIZE_PTR) 606 if (LJ_BE && cta->size < CTSIZE_PTR
607#if LJ_TARGET_MIPS64
608 && !(isfp && nsp)
609#endif
610 )
478 sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size); 611 sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
479 gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp); 612 gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
480 } 613 }
@@ -483,9 +616,14 @@ static void callback_conv_args(CTState *cts, lua_State *L)
483 L->top = o; 616 L->top = o;
484#if LJ_TARGET_X86 617#if LJ_TARGET_X86
485 /* Store stack adjustment for returns from non-cdecl callbacks. */ 618 /* Store stack adjustment for returns from non-cdecl callbacks. */
486 if (ctype_cconv(ct->info) != CTCC_CDECL) 619 if (ctype_cconv(ct->info) != CTCC_CDECL) {
620#if LJ_FR2
621 (L->base-3)->u64 |= (nsp << (16+2));
622#else
487 (L->base-2)->u32.hi |= (nsp << (16+2)); 623 (L->base-2)->u32.hi |= (nsp << (16+2));
488#endif 624#endif
625 }
626#endif
489 while (gcsteps-- > 0) 627 while (gcsteps-- > 0)
490 lj_gc_check(L); 628 lj_gc_check(L);
491} 629}
@@ -493,7 +631,11 @@ static void callback_conv_args(CTState *cts, lua_State *L)
493/* Convert Lua object to callback result. */ 631/* Convert Lua object to callback result. */
494static void callback_conv_result(CTState *cts, lua_State *L, TValue *o) 632static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
495{ 633{
634#if LJ_FR2
635 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
636#else
496 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi); 637 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
638#endif
497#if LJ_TARGET_X86 639#if LJ_TARGET_X86
498 cts->cb.gpr[2] = 0; 640 cts->cb.gpr[2] = 0;
499#endif 641#endif
@@ -503,6 +645,10 @@ static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
503 if (ctype_isfp(ctr->info)) 645 if (ctype_isfp(ctr->info))
504 dp = (uint8_t *)&cts->cb.fpr[0]; 646 dp = (uint8_t *)&cts->cb.fpr[0];
505#endif 647#endif
648#if LJ_TARGET_ARM64 && LJ_BE
649 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float))
650 dp = (uint8_t *)&cts->cb.fpr[0].f[1];
651#endif
506 lj_cconv_ct_tv(cts, ctr, dp, o, 0); 652 lj_cconv_ct_tv(cts, ctr, dp, o, 0);
507#ifdef CALLBACK_HANDLE_RET 653#ifdef CALLBACK_HANDLE_RET
508 CALLBACK_HANDLE_RET 654 CALLBACK_HANDLE_RET
@@ -516,6 +662,12 @@ static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
516 *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp : 662 *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
517 (int32_t)*(int16_t *)dp; 663 (int32_t)*(int16_t *)dp;
518 } 664 }
665#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
666 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
667 if (ctr->size <= 4 &&
668 (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
669 *(int64_t *)dp = (int64_t)*(int32_t *)dp;
670#endif
519#if LJ_TARGET_X86 671#if LJ_TARGET_X86
520 if (ctype_isfp(ctr->info)) 672 if (ctype_isfp(ctr->info))
521 cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2; 673 cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
@@ -528,8 +680,8 @@ lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
528{ 680{
529 lua_State *L = cts->L; 681 lua_State *L = cts->L;
530 global_State *g = cts->g; 682 global_State *g = cts->g;
531 lua_assert(L != NULL); 683 lj_assertG(L != NULL, "uninitialized cts->L in callback");
532 if (gcref(g->jit_L)) { 684 if (tvref(g->jit_base)) {
533 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK)); 685 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
534 if (g->panic) g->panic(L); 686 if (g->panic) g->panic(L);
535 exit(EXIT_FAILURE); 687 exit(EXIT_FAILURE);
@@ -562,9 +714,9 @@ void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
562 } 714 }
563 callback_conv_result(cts, L, o); 715 callback_conv_result(cts, L, o);
564 /* Finally drop C frame and continuation frame. */ 716 /* Finally drop C frame and continuation frame. */
565 L->cframe = cframe_prev(L->cframe); 717 L->top -= 2+2*LJ_FR2;
566 L->top -= 2;
567 L->base = obase; 718 L->base = obase;
719 L->cframe = cframe_prev(L->cframe);
568 cts->cb.slot = 0; /* Blacklist C function that called the callback. */ 720 cts->cb.slot = 0; /* Blacklist C function that called the callback. */
569} 721}
570 722
@@ -613,7 +765,7 @@ static CType *callback_checkfunc(CTState *cts, CType *ct)
613 CType *ctf = ctype_get(cts, fid); 765 CType *ctf = ctype_get(cts, fid);
614 if (!ctype_isattrib(ctf->info)) { 766 if (!ctype_isattrib(ctf->info)) {
615 CType *cta; 767 CType *cta;
616 lua_assert(ctype_isfield(ctf->info)); 768 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
617 cta = ctype_rawchild(cts, ctf); 769 cta = ctype_rawchild(cts, ctf);
618 if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) || 770 if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
619 (ctype_isnum(cta->info) && cta->size <= 8)) || 771 (ctype_isnum(cta->info) && cta->size <= 8)) ||
diff --git a/src/lj_cconv.c b/src/lj_cconv.c
index a5a4e69b..419a8f45 100644
--- a/src/lj_cconv.c
+++ b/src/lj_cconv.c
@@ -8,6 +8,7 @@
8#if LJ_HASFFI 8#if LJ_HASFFI
9 9
10#include "lj_err.h" 10#include "lj_err.h"
11#include "lj_buf.h"
11#include "lj_tab.h" 12#include "lj_tab.h"
12#include "lj_ctype.h" 13#include "lj_ctype.h"
13#include "lj_cdata.h" 14#include "lj_cdata.h"
@@ -122,19 +123,25 @@ void lj_cconv_ct_ct(CTState *cts, CType *d, CType *s,
122 CTInfo dinfo = d->info, sinfo = s->info; 123 CTInfo dinfo = d->info, sinfo = s->info;
123 void *tmpptr; 124 void *tmpptr;
124 125
125 lua_assert(!ctype_isenum(dinfo) && !ctype_isenum(sinfo)); 126 lj_assertCTS(!ctype_isenum(dinfo) && !ctype_isenum(sinfo),
126 lua_assert(!ctype_isattrib(dinfo) && !ctype_isattrib(sinfo)); 127 "unresolved enum");
128 lj_assertCTS(!ctype_isattrib(dinfo) && !ctype_isattrib(sinfo),
129 "unstripped attribute");
127 130
128 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT) 131 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
129 goto err_conv; 132 goto err_conv;
130 133
131 /* Some basic sanity checks. */ 134 /* Some basic sanity checks. */
132 lua_assert(!ctype_isnum(dinfo) || dsize > 0); 135 lj_assertCTS(!ctype_isnum(dinfo) || dsize > 0, "bad size for number type");
133 lua_assert(!ctype_isnum(sinfo) || ssize > 0); 136 lj_assertCTS(!ctype_isnum(sinfo) || ssize > 0, "bad size for number type");
134 lua_assert(!ctype_isbool(dinfo) || dsize == 1 || dsize == 4); 137 lj_assertCTS(!ctype_isbool(dinfo) || dsize == 1 || dsize == 4,
135 lua_assert(!ctype_isbool(sinfo) || ssize == 1 || ssize == 4); 138 "bad size for bool type");
136 lua_assert(!ctype_isinteger(dinfo) || (1u<<lj_fls(dsize)) == dsize); 139 lj_assertCTS(!ctype_isbool(sinfo) || ssize == 1 || ssize == 4,
137 lua_assert(!ctype_isinteger(sinfo) || (1u<<lj_fls(ssize)) == ssize); 140 "bad size for bool type");
141 lj_assertCTS(!ctype_isinteger(dinfo) || (1u<<lj_fls(dsize)) == dsize,
142 "bad size for integer type");
143 lj_assertCTS(!ctype_isinteger(sinfo) || (1u<<lj_fls(ssize)) == ssize,
144 "bad size for integer type");
138 145
139 switch (cconv_idx2(dinfo, sinfo)) { 146 switch (cconv_idx2(dinfo, sinfo)) {
140 /* Destination is a bool. */ 147 /* Destination is a bool. */
@@ -357,7 +364,7 @@ void lj_cconv_ct_ct(CTState *cts, CType *d, CType *s,
357 if ((flags & CCF_CAST) || (d->info & CTF_VLA) || d != s) 364 if ((flags & CCF_CAST) || (d->info & CTF_VLA) || d != s)
358 goto err_conv; /* Must be exact same type. */ 365 goto err_conv; /* Must be exact same type. */
359copyval: /* Copy value. */ 366copyval: /* Copy value. */
360 lua_assert(dsize == ssize); 367 lj_assertCTS(dsize == ssize, "value copy with different sizes");
361 memcpy(dp, sp, dsize); 368 memcpy(dp, sp, dsize);
362 break; 369 break;
363 370
@@ -389,7 +396,7 @@ int lj_cconv_tv_ct(CTState *cts, CType *s, CTypeID sid,
389 lj_cconv_ct_ct(cts, ctype_get(cts, CTID_DOUBLE), s, 396 lj_cconv_ct_ct(cts, ctype_get(cts, CTID_DOUBLE), s,
390 (uint8_t *)&o->n, sp, 0); 397 (uint8_t *)&o->n, sp, 0);
391 /* Numbers are NOT canonicalized here! Beware of uninitialized data. */ 398 /* Numbers are NOT canonicalized here! Beware of uninitialized data. */
392 lua_assert(tvisnum(o)); 399 lj_assertCTS(tvisnum(o), "non-canonical NaN passed");
393 } 400 }
394 } else { 401 } else {
395 uint32_t b = s->size == 1 ? (*sp != 0) : (*(int *)sp != 0); 402 uint32_t b = s->size == 1 ? (*sp != 0) : (*(int *)sp != 0);
@@ -406,7 +413,7 @@ int lj_cconv_tv_ct(CTState *cts, CType *s, CTypeID sid,
406 CTSize sz; 413 CTSize sz;
407 copyval: /* Copy value. */ 414 copyval: /* Copy value. */
408 sz = s->size; 415 sz = s->size;
409 lua_assert(sz != CTSIZE_INVALID); 416 lj_assertCTS(sz != CTSIZE_INVALID, "value copy with invalid size");
410 /* Attributes are stripped, qualifiers are kept (but mostly ignored). */ 417 /* Attributes are stripped, qualifiers are kept (but mostly ignored). */
411 cd = lj_cdata_new(cts, ctype_typeid(cts, s), sz); 418 cd = lj_cdata_new(cts, ctype_typeid(cts, s), sz);
412 setcdataV(cts->L, o, cd); 419 setcdataV(cts->L, o, cd);
@@ -421,19 +428,22 @@ int lj_cconv_tv_bf(CTState *cts, CType *s, TValue *o, uint8_t *sp)
421 CTInfo info = s->info; 428 CTInfo info = s->info;
422 CTSize pos, bsz; 429 CTSize pos, bsz;
423 uint32_t val; 430 uint32_t val;
424 lua_assert(ctype_isbitfield(info)); 431 lj_assertCTS(ctype_isbitfield(info), "bitfield expected");
425 /* NYI: packed bitfields may cause misaligned reads. */ 432 /* NYI: packed bitfields may cause misaligned reads. */
426 switch (ctype_bitcsz(info)) { 433 switch (ctype_bitcsz(info)) {
427 case 4: val = *(uint32_t *)sp; break; 434 case 4: val = *(uint32_t *)sp; break;
428 case 2: val = *(uint16_t *)sp; break; 435 case 2: val = *(uint16_t *)sp; break;
429 case 1: val = *(uint8_t *)sp; break; 436 case 1: val = *(uint8_t *)sp; break;
430 default: lua_assert(0); val = 0; break; 437 default:
438 lj_assertCTS(0, "bad bitfield container size %d", ctype_bitcsz(info));
439 val = 0;
440 break;
431 } 441 }
432 /* Check if a packed bitfield crosses a container boundary. */ 442 /* Check if a packed bitfield crosses a container boundary. */
433 pos = ctype_bitpos(info); 443 pos = ctype_bitpos(info);
434 bsz = ctype_bitbsz(info); 444 bsz = ctype_bitbsz(info);
435 lua_assert(pos < 8*ctype_bitcsz(info)); 445 lj_assertCTS(pos < 8*ctype_bitcsz(info), "bad bitfield position");
436 lua_assert(bsz > 0 && bsz <= 8*ctype_bitcsz(info)); 446 lj_assertCTS(bsz > 0 && bsz <= 8*ctype_bitcsz(info), "bad bitfield size");
437 if (pos + bsz > 8*ctype_bitcsz(info)) 447 if (pos + bsz > 8*ctype_bitcsz(info))
438 lj_err_caller(cts->L, LJ_ERR_FFI_NYIPACKBIT); 448 lj_err_caller(cts->L, LJ_ERR_FFI_NYIPACKBIT);
439 if (!(info & CTF_BOOL)) { 449 if (!(info & CTF_BOOL)) {
@@ -448,8 +458,10 @@ int lj_cconv_tv_bf(CTState *cts, CType *s, TValue *o, uint8_t *sp)
448 setintV(o, (int32_t)val); 458 setintV(o, (int32_t)val);
449 } 459 }
450 } else { 460 } else {
451 lua_assert(bsz == 1); 461 uint32_t b = (val >> pos) & 1;
452 setboolV(o, (val >> pos) & 1); 462 lj_assertCTS(bsz == 1, "bad bool bitfield size");
463 setboolV(o, b);
464 setboolV(&cts->g->tmptv2, b); /* Remember for trace recorder. */
453 } 465 }
454 return 0; /* No GC step needed. */ 466 return 0; /* No GC step needed. */
455} 467}
@@ -551,7 +563,7 @@ void lj_cconv_ct_tv(CTState *cts, CType *d,
551 sid = cdataV(o)->ctypeid; 563 sid = cdataV(o)->ctypeid;
552 s = ctype_get(cts, sid); 564 s = ctype_get(cts, sid);
553 if (ctype_isref(s->info)) { /* Resolve reference for value. */ 565 if (ctype_isref(s->info)) { /* Resolve reference for value. */
554 lua_assert(s->size == CTSIZE_PTR); 566 lj_assertCTS(s->size == CTSIZE_PTR, "ref is not pointer-sized");
555 sp = *(void **)sp; 567 sp = *(void **)sp;
556 sid = ctype_cid(s->info); 568 sid = ctype_cid(s->info);
557 } 569 }
@@ -571,7 +583,7 @@ void lj_cconv_ct_tv(CTState *cts, CType *d,
571 CType *cct = lj_ctype_getfield(cts, d, str, &ofs); 583 CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
572 if (!cct || !ctype_isconstval(cct->info)) 584 if (!cct || !ctype_isconstval(cct->info))
573 goto err_conv; 585 goto err_conv;
574 lua_assert(d->size == 4); 586 lj_assertCTS(d->size == 4, "only 32 bit enum supported"); /* NYI */
575 sp = (uint8_t *)&cct->size; 587 sp = (uint8_t *)&cct->size;
576 sid = ctype_cid(cct->info); 588 sid = ctype_cid(cct->info);
577 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */ 589 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */
@@ -610,8 +622,10 @@ void lj_cconv_ct_tv(CTState *cts, CType *d,
610 tmpptr = uddata(ud); 622 tmpptr = uddata(ud);
611 if (ud->udtype == UDTYPE_IO_FILE) 623 if (ud->udtype == UDTYPE_IO_FILE)
612 tmpptr = *(void **)tmpptr; 624 tmpptr = *(void **)tmpptr;
625 else if (ud->udtype == UDTYPE_BUFFER)
626 tmpptr = ((SBufExt *)tmpptr)->r;
613 } else if (tvislightud(o)) { 627 } else if (tvislightud(o)) {
614 tmpptr = lightudV(o); 628 tmpptr = lightudV(cts->g, o);
615 } else if (tvisfunc(o)) { 629 } else if (tvisfunc(o)) {
616 void *p = lj_ccallback_new(cts, d, funcV(o)); 630 void *p = lj_ccallback_new(cts, d, funcV(o));
617 if (p) { 631 if (p) {
@@ -635,10 +649,10 @@ void lj_cconv_bf_tv(CTState *cts, CType *d, uint8_t *dp, TValue *o)
635 CTInfo info = d->info; 649 CTInfo info = d->info;
636 CTSize pos, bsz; 650 CTSize pos, bsz;
637 uint32_t val, mask; 651 uint32_t val, mask;
638 lua_assert(ctype_isbitfield(info)); 652 lj_assertCTS(ctype_isbitfield(info), "bitfield expected");
639 if ((info & CTF_BOOL)) { 653 if ((info & CTF_BOOL)) {
640 uint8_t tmpbool; 654 uint8_t tmpbool;
641 lua_assert(ctype_bitbsz(info) == 1); 655 lj_assertCTS(ctype_bitbsz(info) == 1, "bad bool bitfield size");
642 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_BOOL), &tmpbool, o, 0); 656 lj_cconv_ct_tv(cts, ctype_get(cts, CTID_BOOL), &tmpbool, o, 0);
643 val = tmpbool; 657 val = tmpbool;
644 } else { 658 } else {
@@ -647,8 +661,8 @@ void lj_cconv_bf_tv(CTState *cts, CType *d, uint8_t *dp, TValue *o)
647 } 661 }
648 pos = ctype_bitpos(info); 662 pos = ctype_bitpos(info);
649 bsz = ctype_bitbsz(info); 663 bsz = ctype_bitbsz(info);
650 lua_assert(pos < 8*ctype_bitcsz(info)); 664 lj_assertCTS(pos < 8*ctype_bitcsz(info), "bad bitfield position");
651 lua_assert(bsz > 0 && bsz <= 8*ctype_bitcsz(info)); 665 lj_assertCTS(bsz > 0 && bsz <= 8*ctype_bitcsz(info), "bad bitfield size");
652 /* Check if a packed bitfield crosses a container boundary. */ 666 /* Check if a packed bitfield crosses a container boundary. */
653 if (pos + bsz > 8*ctype_bitcsz(info)) 667 if (pos + bsz > 8*ctype_bitcsz(info))
654 lj_err_caller(cts->L, LJ_ERR_FFI_NYIPACKBIT); 668 lj_err_caller(cts->L, LJ_ERR_FFI_NYIPACKBIT);
@@ -659,7 +673,9 @@ void lj_cconv_bf_tv(CTState *cts, CType *d, uint8_t *dp, TValue *o)
659 case 4: *(uint32_t *)dp = (*(uint32_t *)dp & ~mask) | (uint32_t)val; break; 673 case 4: *(uint32_t *)dp = (*(uint32_t *)dp & ~mask) | (uint32_t)val; break;
660 case 2: *(uint16_t *)dp = (*(uint16_t *)dp & ~mask) | (uint16_t)val; break; 674 case 2: *(uint16_t *)dp = (*(uint16_t *)dp & ~mask) | (uint16_t)val; break;
661 case 1: *(uint8_t *)dp = (*(uint8_t *)dp & ~mask) | (uint8_t)val; break; 675 case 1: *(uint8_t *)dp = (*(uint8_t *)dp & ~mask) | (uint8_t)val; break;
662 default: lua_assert(0); break; 676 default:
677 lj_assertCTS(0, "bad bitfield container size %d", ctype_bitcsz(info));
678 break;
663 } 679 }
664} 680}
665 681
diff --git a/src/lj_cconv.h b/src/lj_cconv.h
index 6199448d..9da2c33c 100644
--- a/src/lj_cconv.h
+++ b/src/lj_cconv.h
@@ -27,13 +27,14 @@ enum {
27static LJ_AINLINE uint32_t cconv_idx(CTInfo info) 27static LJ_AINLINE uint32_t cconv_idx(CTInfo info)
28{ 28{
29 uint32_t idx = ((info >> 26) & 15u); /* Dispatch bits. */ 29 uint32_t idx = ((info >> 26) & 15u); /* Dispatch bits. */
30 lua_assert(ctype_type(info) <= CT_MAYCONVERT); 30 lj_assertX(ctype_type(info) <= CT_MAYCONVERT,
31 "cannot convert ctype %08x", info);
31#if LJ_64 32#if LJ_64
32 idx = ((uint32_t)(U64x(f436fff5,fff7f021) >> 4*idx) & 15u); 33 idx = ((uint32_t)(U64x(f436fff5,fff7f021) >> 4*idx) & 15u);
33#else 34#else
34 idx = (((idx < 8 ? 0xfff7f021u : 0xf436fff5) >> 4*(idx & 7u)) & 15u); 35 idx = (((idx < 8 ? 0xfff7f021u : 0xf436fff5) >> 4*(idx & 7u)) & 15u);
35#endif 36#endif
36 lua_assert(idx < 8); 37 lj_assertX(idx < 8, "cannot convert ctype %08x", info);
37 return idx; 38 return idx;
38} 39}
39 40
diff --git a/src/lj_cdata.c b/src/lj_cdata.c
index 32c69829..77d9730f 100644
--- a/src/lj_cdata.c
+++ b/src/lj_cdata.c
@@ -9,7 +9,6 @@
9 9
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_str.h"
13#include "lj_tab.h" 12#include "lj_tab.h"
14#include "lj_ctype.h" 13#include "lj_ctype.h"
15#include "lj_cconv.h" 14#include "lj_cconv.h"
@@ -27,20 +26,20 @@ GCcdata *lj_cdata_newref(CTState *cts, const void *p, CTypeID id)
27} 26}
28 27
29/* Allocate variable-sized or specially aligned C data object. */ 28/* Allocate variable-sized or specially aligned C data object. */
30GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz, CTSize align) 29GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align)
31{ 30{
32 global_State *g; 31 global_State *g;
33 MSize extra = sizeof(GCcdataVar) + sizeof(GCcdata) + 32 MSize extra = sizeof(GCcdataVar) + sizeof(GCcdata) +
34 (align > CT_MEMALIGN ? (1u<<align) - (1u<<CT_MEMALIGN) : 0); 33 (align > CT_MEMALIGN ? (1u<<align) - (1u<<CT_MEMALIGN) : 0);
35 char *p = lj_mem_newt(cts->L, extra + sz, char); 34 char *p = lj_mem_newt(L, extra + sz, char);
36 uintptr_t adata = (uintptr_t)p + sizeof(GCcdataVar) + sizeof(GCcdata); 35 uintptr_t adata = (uintptr_t)p + sizeof(GCcdataVar) + sizeof(GCcdata);
37 uintptr_t almask = (1u << align) - 1u; 36 uintptr_t almask = (1u << align) - 1u;
38 GCcdata *cd = (GCcdata *)(((adata + almask) & ~almask) - sizeof(GCcdata)); 37 GCcdata *cd = (GCcdata *)(((adata + almask) & ~almask) - sizeof(GCcdata));
39 lua_assert((char *)cd - p < 65536); 38 lj_assertL((char *)cd - p < 65536, "excessive cdata alignment");
40 cdatav(cd)->offset = (uint16_t)((char *)cd - p); 39 cdatav(cd)->offset = (uint16_t)((char *)cd - p);
41 cdatav(cd)->extra = extra; 40 cdatav(cd)->extra = extra;
42 cdatav(cd)->len = sz; 41 cdatav(cd)->len = sz;
43 g = cts->g; 42 g = G(L);
44 setgcrefr(cd->nextgc, g->gc.root); 43 setgcrefr(cd->nextgc, g->gc.root);
45 setgcref(g->gc.root, obj2gco(cd)); 44 setgcref(g->gc.root, obj2gco(cd));
46 newwhite(g, obj2gco(cd)); 45 newwhite(g, obj2gco(cd));
@@ -50,6 +49,15 @@ GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz, CTSize align)
50 return cd; 49 return cd;
51} 50}
52 51
52/* Allocate arbitrary C data object. */
53GCcdata *lj_cdata_newx(CTState *cts, CTypeID id, CTSize sz, CTInfo info)
54{
55 if (!(info & CTF_VLA) && ctype_align(info) <= CT_MEMALIGN)
56 return lj_cdata_new(cts, id, sz);
57 else
58 return lj_cdata_newv(cts->L, id, sz, ctype_align(info));
59}
60
53/* Free a C data object. */ 61/* Free a C data object. */
54void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd) 62void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
55{ 63{
@@ -68,29 +76,30 @@ void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
68 } else if (LJ_LIKELY(!cdataisv(cd))) { 76 } else if (LJ_LIKELY(!cdataisv(cd))) {
69 CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid); 77 CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid);
70 CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR; 78 CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR;
71 lua_assert(ctype_hassize(ct->info) || ctype_isfunc(ct->info) || 79 lj_assertG(ctype_hassize(ct->info) || ctype_isfunc(ct->info) ||
72 ctype_isextern(ct->info)); 80 ctype_isextern(ct->info), "free of ctype without a size");
73 lj_mem_free(g, cd, sizeof(GCcdata) + sz); 81 lj_mem_free(g, cd, sizeof(GCcdata) + sz);
74 } else { 82 } else {
75 lj_mem_free(g, memcdatav(cd), sizecdatav(cd)); 83 lj_mem_free(g, memcdatav(cd), sizecdatav(cd));
76 } 84 }
77} 85}
78 86
79TValue * LJ_FASTCALL lj_cdata_setfin(lua_State *L, GCcdata *cd) 87void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it)
80{ 88{
81 global_State *g = G(L); 89 GCtab *t = ctype_ctsG(G(L))->finalizer;
82 GCtab *t = ctype_ctsG(g)->finalizer;
83 if (gcref(t->metatable)) { 90 if (gcref(t->metatable)) {
84 /* Add cdata to finalizer table, if still enabled. */ 91 /* Add cdata to finalizer table, if still enabled. */
85 TValue *tv, tmp; 92 TValue *tv, tmp;
86 setcdataV(L, &tmp, cd); 93 setcdataV(L, &tmp, cd);
87 lj_gc_anybarriert(L, t); 94 lj_gc_anybarriert(L, t);
88 tv = lj_tab_set(L, t, &tmp); 95 tv = lj_tab_set(L, t, &tmp);
89 cd->marked |= LJ_GC_CDATA_FIN; 96 if (it == LJ_TNIL) {
90 return tv; 97 setnilV(tv);
91 } else { 98 cd->marked &= ~LJ_GC_CDATA_FIN;
92 /* Otherwise return dummy TValue. */ 99 } else {
93 return &g->tmptv; 100 setgcV(L, tv, obj, it);
101 cd->marked |= LJ_GC_CDATA_FIN;
102 }
94 } 103 }
95} 104}
96 105
@@ -106,7 +115,7 @@ CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key, uint8_t **pp,
106 115
107 /* Resolve reference for cdata object. */ 116 /* Resolve reference for cdata object. */
108 if (ctype_isref(ct->info)) { 117 if (ctype_isref(ct->info)) {
109 lua_assert(ct->size == CTSIZE_PTR); 118 lj_assertCTS(ct->size == CTSIZE_PTR, "ref is not pointer-sized");
110 p = *(uint8_t **)p; 119 p = *(uint8_t **)p;
111 ct = ctype_child(cts, ct); 120 ct = ctype_child(cts, ct);
112 } 121 }
@@ -117,13 +126,19 @@ collect_attrib:
117 if (ctype_attrib(ct->info) == CTA_QUAL) *qual |= ct->size; 126 if (ctype_attrib(ct->info) == CTA_QUAL) *qual |= ct->size;
118 ct = ctype_child(cts, ct); 127 ct = ctype_child(cts, ct);
119 } 128 }
120 lua_assert(!ctype_isref(ct->info)); /* Interning rejects refs to refs. */ 129 /* Interning rejects refs to refs. */
130 lj_assertCTS(!ctype_isref(ct->info), "bad ref of ref");
121 131
122 if (tvisint(key)) { 132 if (tvisint(key)) {
123 idx = (ptrdiff_t)intV(key); 133 idx = (ptrdiff_t)intV(key);
124 goto integer_key; 134 goto integer_key;
125 } else if (tvisnum(key)) { /* Numeric key. */ 135 } else if (tvisnum(key)) { /* Numeric key. */
126 idx = LJ_64 ? (ptrdiff_t)numV(key) : (ptrdiff_t)lj_num2int(numV(key)); 136#ifdef _MSC_VER
137 /* Workaround for MSVC bug. */
138 volatile
139#endif
140 lua_Number n = numV(key);
141 idx = LJ_64 ? (ptrdiff_t)n : (ptrdiff_t)lj_num2int(n);
127 integer_key: 142 integer_key:
128 if (ctype_ispointer(ct->info)) { 143 if (ctype_ispointer(ct->info)) {
129 CTSize sz = lj_ctype_size(cts, ctype_cid(ct->info)); /* Element size. */ 144 CTSize sz = lj_ctype_size(cts, ctype_cid(ct->info)); /* Element size. */
@@ -198,7 +213,8 @@ collect_attrib:
198static void cdata_getconst(CTState *cts, TValue *o, CType *ct) 213static void cdata_getconst(CTState *cts, TValue *o, CType *ct)
199{ 214{
200 CType *ctt = ctype_child(cts, ct); 215 CType *ctt = ctype_child(cts, ct);
201 lua_assert(ctype_isinteger(ctt->info) && ctt->size <= 4); 216 lj_assertCTS(ctype_isinteger(ctt->info) && ctt->size <= 4,
217 "only 32 bit const supported"); /* NYI */
202 /* Constants are already zero-extended/sign-extended to 32 bits. */ 218 /* Constants are already zero-extended/sign-extended to 32 bits. */
203 if ((ctt->info & CTF_UNSIGNED) && (int32_t)ct->size < 0) 219 if ((ctt->info & CTF_UNSIGNED) && (int32_t)ct->size < 0)
204 setnumV(o, (lua_Number)(uint32_t)ct->size); 220 setnumV(o, (lua_Number)(uint32_t)ct->size);
@@ -219,13 +235,14 @@ int lj_cdata_get(CTState *cts, CType *s, TValue *o, uint8_t *sp)
219 } 235 }
220 236
221 /* Get child type of pointer/array/field. */ 237 /* Get child type of pointer/array/field. */
222 lua_assert(ctype_ispointer(s->info) || ctype_isfield(s->info)); 238 lj_assertCTS(ctype_ispointer(s->info) || ctype_isfield(s->info),
239 "pointer or field expected");
223 sid = ctype_cid(s->info); 240 sid = ctype_cid(s->info);
224 s = ctype_get(cts, sid); 241 s = ctype_get(cts, sid);
225 242
226 /* Resolve reference for field. */ 243 /* Resolve reference for field. */
227 if (ctype_isref(s->info)) { 244 if (ctype_isref(s->info)) {
228 lua_assert(s->size == CTSIZE_PTR); 245 lj_assertCTS(s->size == CTSIZE_PTR, "ref is not pointer-sized");
229 sp = *(uint8_t **)sp; 246 sp = *(uint8_t **)sp;
230 sid = ctype_cid(s->info); 247 sid = ctype_cid(s->info);
231 s = ctype_get(cts, sid); 248 s = ctype_get(cts, sid);
@@ -252,12 +269,13 @@ void lj_cdata_set(CTState *cts, CType *d, uint8_t *dp, TValue *o, CTInfo qual)
252 } 269 }
253 270
254 /* Get child type of pointer/array/field. */ 271 /* Get child type of pointer/array/field. */
255 lua_assert(ctype_ispointer(d->info) || ctype_isfield(d->info)); 272 lj_assertCTS(ctype_ispointer(d->info) || ctype_isfield(d->info),
273 "pointer or field expected");
256 d = ctype_child(cts, d); 274 d = ctype_child(cts, d);
257 275
258 /* Resolve reference for field. */ 276 /* Resolve reference for field. */
259 if (ctype_isref(d->info)) { 277 if (ctype_isref(d->info)) {
260 lua_assert(d->size == CTSIZE_PTR); 278 lj_assertCTS(d->size == CTSIZE_PTR, "ref is not pointer-sized");
261 dp = *(uint8_t **)dp; 279 dp = *(uint8_t **)dp;
262 d = ctype_child(cts, d); 280 d = ctype_child(cts, d);
263 } 281 }
@@ -272,7 +290,8 @@ void lj_cdata_set(CTState *cts, CType *d, uint8_t *dp, TValue *o, CTInfo qual)
272 d = ctype_child(cts, d); 290 d = ctype_child(cts, d);
273 } 291 }
274 292
275 lua_assert(ctype_hassize(d->info) && !ctype_isvoid(d->info)); 293 lj_assertCTS(ctype_hassize(d->info), "store to ctype without size");
294 lj_assertCTS(!ctype_isvoid(d->info), "store to void type");
276 295
277 if (((d->info|qual) & CTF_CONST)) { 296 if (((d->info|qual) & CTF_CONST)) {
278 err_const: 297 err_const:
diff --git a/src/lj_cdata.h b/src/lj_cdata.h
index 4de5969a..e6ab48ca 100644
--- a/src/lj_cdata.h
+++ b/src/lj_cdata.h
@@ -18,7 +18,7 @@ static LJ_AINLINE void *cdata_getptr(void *p, CTSize sz)
18 if (LJ_64 && sz == 4) { /* Support 32 bit pointers on 64 bit targets. */ 18 if (LJ_64 && sz == 4) { /* Support 32 bit pointers on 64 bit targets. */
19 return ((void *)(uintptr_t)*(uint32_t *)p); 19 return ((void *)(uintptr_t)*(uint32_t *)p);
20 } else { 20 } else {
21 lua_assert(sz == CTSIZE_PTR); 21 lj_assertX(sz == CTSIZE_PTR, "bad pointer size %d", sz);
22 return *(void **)p; 22 return *(void **)p;
23 } 23 }
24} 24}
@@ -29,7 +29,7 @@ static LJ_AINLINE void cdata_setptr(void *p, CTSize sz, const void *v)
29 if (LJ_64 && sz == 4) { /* Support 32 bit pointers on 64 bit targets. */ 29 if (LJ_64 && sz == 4) { /* Support 32 bit pointers on 64 bit targets. */
30 *(uint32_t *)p = (uint32_t)(uintptr_t)v; 30 *(uint32_t *)p = (uint32_t)(uintptr_t)v;
31 } else { 31 } else {
32 lua_assert(sz == CTSIZE_PTR); 32 lj_assertX(sz == CTSIZE_PTR, "bad pointer size %d", sz);
33 *(void **)p = (void *)v; 33 *(void **)p = (void *)v;
34 } 34 }
35} 35}
@@ -40,7 +40,8 @@ static LJ_AINLINE GCcdata *lj_cdata_new(CTState *cts, CTypeID id, CTSize sz)
40 GCcdata *cd; 40 GCcdata *cd;
41#ifdef LUA_USE_ASSERT 41#ifdef LUA_USE_ASSERT
42 CType *ct = ctype_raw(cts, id); 42 CType *ct = ctype_raw(cts, id);
43 lua_assert((ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR) == sz); 43 lj_assertCTS((ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR) == sz,
44 "inconsistent size of fixed-size cdata alloc");
44#endif 45#endif
45 cd = (GCcdata *)lj_mem_newgco(cts->L, sizeof(GCcdata) + sz); 46 cd = (GCcdata *)lj_mem_newgco(cts->L, sizeof(GCcdata) + sz);
46 cd->gct = ~LJ_TCDATA; 47 cd->gct = ~LJ_TCDATA;
@@ -58,11 +59,14 @@ static LJ_AINLINE GCcdata *lj_cdata_new_(lua_State *L, CTypeID id, CTSize sz)
58} 59}
59 60
60LJ_FUNC GCcdata *lj_cdata_newref(CTState *cts, const void *pp, CTypeID id); 61LJ_FUNC GCcdata *lj_cdata_newref(CTState *cts, const void *pp, CTypeID id);
61LJ_FUNC GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz, 62LJ_FUNC GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz,
62 CTSize align); 63 CTSize align);
64LJ_FUNC GCcdata *lj_cdata_newx(CTState *cts, CTypeID id, CTSize sz,
65 CTInfo info);
63 66
64LJ_FUNC void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd); 67LJ_FUNC void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd);
65LJ_FUNCA TValue * LJ_FASTCALL lj_cdata_setfin(lua_State *L, GCcdata *cd); 68LJ_FUNC void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj,
69 uint32_t it);
66 70
67LJ_FUNC CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key, 71LJ_FUNC CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key,
68 uint8_t **pp, CTInfo *qual); 72 uint8_t **pp, CTInfo *qual);
diff --git a/src/lj_clib.c b/src/lj_clib.c
index d8d879a0..513528ce 100644
--- a/src/lj_clib.c
+++ b/src/lj_clib.c
@@ -16,6 +16,7 @@
16#include "lj_cconv.h" 16#include "lj_cconv.h"
17#include "lj_cdata.h" 17#include "lj_cdata.h"
18#include "lj_clib.h" 18#include "lj_clib.h"
19#include "lj_strfmt.h"
19 20
20/* -- OS-specific functions ----------------------------------------------- */ 21/* -- OS-specific functions ----------------------------------------------- */
21 22
@@ -61,7 +62,7 @@ static const char *clib_extname(lua_State *L, const char *name)
61#endif 62#endif
62 ) { 63 ) {
63 if (!strchr(name, '.')) { 64 if (!strchr(name, '.')) {
64 name = lj_str_pushf(L, CLIB_SOEXT, name); 65 name = lj_strfmt_pushf(L, CLIB_SOEXT, name);
65 L->top--; 66 L->top--;
66#if LJ_TARGET_CYGWIN 67#if LJ_TARGET_CYGWIN
67 } else { 68 } else {
@@ -70,7 +71,7 @@ static const char *clib_extname(lua_State *L, const char *name)
70 } 71 }
71 if (!(name[0] == CLIB_SOPREFIX[0] && name[1] == CLIB_SOPREFIX[1] && 72 if (!(name[0] == CLIB_SOPREFIX[0] && name[1] == CLIB_SOPREFIX[1] &&
72 name[2] == CLIB_SOPREFIX[2])) { 73 name[2] == CLIB_SOPREFIX[2])) {
73 name = lj_str_pushf(L, CLIB_SOPREFIX "%s", name); 74 name = lj_strfmt_pushf(L, CLIB_SOPREFIX "%s", name);
74 L->top--; 75 L->top--;
75 } 76 }
76 } 77 }
@@ -158,11 +159,13 @@ BOOL WINAPI GetModuleHandleExA(DWORD, LPCSTR, HMODULE*);
158/* Default libraries. */ 159/* Default libraries. */
159enum { 160enum {
160 CLIB_HANDLE_EXE, 161 CLIB_HANDLE_EXE,
162#if !LJ_TARGET_UWP
161 CLIB_HANDLE_DLL, 163 CLIB_HANDLE_DLL,
162 CLIB_HANDLE_CRT, 164 CLIB_HANDLE_CRT,
163 CLIB_HANDLE_KERNEL32, 165 CLIB_HANDLE_KERNEL32,
164 CLIB_HANDLE_USER32, 166 CLIB_HANDLE_USER32,
165 CLIB_HANDLE_GDI32, 167 CLIB_HANDLE_GDI32,
168#endif
166 CLIB_HANDLE_MAX 169 CLIB_HANDLE_MAX
167}; 170};
168 171
@@ -172,11 +175,19 @@ LJ_NORET LJ_NOINLINE static void clib_error(lua_State *L, const char *fmt,
172 const char *name) 175 const char *name)
173{ 176{
174 DWORD err = GetLastError(); 177 DWORD err = GetLastError();
178#if LJ_TARGET_XBOXONE
179 wchar_t wbuf[128];
180 char buf[128*2];
181 if (!FormatMessageW(FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_FROM_SYSTEM,
182 NULL, err, 0, wbuf, sizeof(wbuf)/sizeof(wchar_t), NULL) ||
183 !WideCharToMultiByte(CP_ACP, 0, wbuf, 128, buf, 128*2, NULL, NULL))
184#else
175 char buf[128]; 185 char buf[128];
176 if (!FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_FROM_SYSTEM, 186 if (!FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_FROM_SYSTEM,
177 NULL, err, 0, buf, sizeof(buf), NULL)) 187 NULL, err, 0, buf, sizeof(buf), NULL))
188#endif
178 buf[0] = '\0'; 189 buf[0] = '\0';
179 lj_err_callermsg(L, lj_str_pushf(L, fmt, name, buf)); 190 lj_err_callermsg(L, lj_strfmt_pushf(L, fmt, name, buf));
180} 191}
181 192
182static int clib_needext(const char *s) 193static int clib_needext(const char *s)
@@ -191,7 +202,7 @@ static int clib_needext(const char *s)
191static const char *clib_extname(lua_State *L, const char *name) 202static const char *clib_extname(lua_State *L, const char *name)
192{ 203{
193 if (clib_needext(name)) { 204 if (clib_needext(name)) {
194 name = lj_str_pushf(L, "%s.dll", name); 205 name = lj_strfmt_pushf(L, "%s.dll", name);
195 L->top--; 206 L->top--;
196 } 207 }
197 return name; 208 return name;
@@ -200,7 +211,7 @@ static const char *clib_extname(lua_State *L, const char *name)
200static void *clib_loadlib(lua_State *L, const char *name, int global) 211static void *clib_loadlib(lua_State *L, const char *name, int global)
201{ 212{
202 DWORD oldwerr = GetLastError(); 213 DWORD oldwerr = GetLastError();
203 void *h = (void *)LoadLibraryA(clib_extname(L, name)); 214 void *h = LJ_WIN_LOADLIBA(clib_extname(L, name));
204 if (!h) clib_error(L, "cannot load module " LUA_QS ": %s", name); 215 if (!h) clib_error(L, "cannot load module " LUA_QS ": %s", name);
205 SetLastError(oldwerr); 216 SetLastError(oldwerr);
206 UNUSED(global); 217 UNUSED(global);
@@ -210,6 +221,7 @@ static void *clib_loadlib(lua_State *L, const char *name, int global)
210static void clib_unloadlib(CLibrary *cl) 221static void clib_unloadlib(CLibrary *cl)
211{ 222{
212 if (cl->handle == CLIB_DEFHANDLE) { 223 if (cl->handle == CLIB_DEFHANDLE) {
224#if !LJ_TARGET_UWP
213 MSize i; 225 MSize i;
214 for (i = CLIB_HANDLE_KERNEL32; i < CLIB_HANDLE_MAX; i++) { 226 for (i = CLIB_HANDLE_KERNEL32; i < CLIB_HANDLE_MAX; i++) {
215 void *h = clib_def_handle[i]; 227 void *h = clib_def_handle[i];
@@ -218,11 +230,16 @@ static void clib_unloadlib(CLibrary *cl)
218 FreeLibrary((HINSTANCE)h); 230 FreeLibrary((HINSTANCE)h);
219 } 231 }
220 } 232 }
233#endif
221 } else if (cl->handle) { 234 } else if (cl->handle) {
222 FreeLibrary((HINSTANCE)cl->handle); 235 FreeLibrary((HINSTANCE)cl->handle);
223 } 236 }
224} 237}
225 238
239#if LJ_TARGET_UWP
240EXTERN_C IMAGE_DOS_HEADER __ImageBase;
241#endif
242
226static void *clib_getsym(CLibrary *cl, const char *name) 243static void *clib_getsym(CLibrary *cl, const char *name)
227{ 244{
228 void *p = NULL; 245 void *p = NULL;
@@ -231,6 +248,9 @@ static void *clib_getsym(CLibrary *cl, const char *name)
231 for (i = 0; i < CLIB_HANDLE_MAX; i++) { 248 for (i = 0; i < CLIB_HANDLE_MAX; i++) {
232 HINSTANCE h = (HINSTANCE)clib_def_handle[i]; 249 HINSTANCE h = (HINSTANCE)clib_def_handle[i];
233 if (!(void *)h) { /* Resolve default library handles (once). */ 250 if (!(void *)h) { /* Resolve default library handles (once). */
251#if LJ_TARGET_UWP
252 h = (HINSTANCE)&__ImageBase;
253#else
234 switch (i) { 254 switch (i) {
235 case CLIB_HANDLE_EXE: GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, NULL, &h); break; 255 case CLIB_HANDLE_EXE: GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, NULL, &h); break;
236 case CLIB_HANDLE_DLL: 256 case CLIB_HANDLE_DLL:
@@ -241,11 +261,12 @@ static void *clib_getsym(CLibrary *cl, const char *name)
241 GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS|GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, 261 GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS|GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
242 (const char *)&_fmode, &h); 262 (const char *)&_fmode, &h);
243 break; 263 break;
244 case CLIB_HANDLE_KERNEL32: h = LoadLibraryA("kernel32.dll"); break; 264 case CLIB_HANDLE_KERNEL32: h = LJ_WIN_LOADLIBA("kernel32.dll"); break;
245 case CLIB_HANDLE_USER32: h = LoadLibraryA("user32.dll"); break; 265 case CLIB_HANDLE_USER32: h = LJ_WIN_LOADLIBA("user32.dll"); break;
246 case CLIB_HANDLE_GDI32: h = LoadLibraryA("gdi32.dll"); break; 266 case CLIB_HANDLE_GDI32: h = LJ_WIN_LOADLIBA("gdi32.dll"); break;
247 } 267 }
248 if (!h) continue; 268 if (!h) continue;
269#endif
249 clib_def_handle[i] = (void *)h; 270 clib_def_handle[i] = (void *)h;
250 } 271 }
251 p = (void *)GetProcAddress(h, name); 272 p = (void *)GetProcAddress(h, name);
@@ -264,7 +285,7 @@ static void *clib_getsym(CLibrary *cl, const char *name)
264LJ_NORET LJ_NOINLINE static void clib_error(lua_State *L, const char *fmt, 285LJ_NORET LJ_NOINLINE static void clib_error(lua_State *L, const char *fmt,
265 const char *name) 286 const char *name)
266{ 287{
267 lj_err_callermsg(L, lj_str_pushf(L, fmt, name, "no support for this OS")); 288 lj_err_callermsg(L, lj_strfmt_pushf(L, fmt, name, "no support for this OS"));
268} 289}
269 290
270static void *clib_loadlib(lua_State *L, const char *name, int global) 291static void *clib_loadlib(lua_State *L, const char *name, int global)
@@ -329,7 +350,8 @@ TValue *lj_clib_index(lua_State *L, CLibrary *cl, GCstr *name)
329 lj_err_callerv(L, LJ_ERR_FFI_NODECL, strdata(name)); 350 lj_err_callerv(L, LJ_ERR_FFI_NODECL, strdata(name));
330 if (ctype_isconstval(ct->info)) { 351 if (ctype_isconstval(ct->info)) {
331 CType *ctt = ctype_child(cts, ct); 352 CType *ctt = ctype_child(cts, ct);
332 lua_assert(ctype_isinteger(ctt->info) && ctt->size <= 4); 353 lj_assertCTS(ctype_isinteger(ctt->info) && ctt->size <= 4,
354 "only 32 bit const supported"); /* NYI */
333 if ((ctt->info & CTF_UNSIGNED) && (int32_t)ct->size < 0) 355 if ((ctt->info & CTF_UNSIGNED) && (int32_t)ct->size < 0)
334 setnumV(tv, (lua_Number)(uint32_t)ct->size); 356 setnumV(tv, (lua_Number)(uint32_t)ct->size);
335 else 357 else
@@ -341,14 +363,15 @@ TValue *lj_clib_index(lua_State *L, CLibrary *cl, GCstr *name)
341#endif 363#endif
342 void *p = clib_getsym(cl, sym); 364 void *p = clib_getsym(cl, sym);
343 GCcdata *cd; 365 GCcdata *cd;
344 lua_assert(ctype_isfunc(ct->info) || ctype_isextern(ct->info)); 366 lj_assertCTS(ctype_isfunc(ct->info) || ctype_isextern(ct->info),
367 "unexpected ctype %08x in clib", ct->info);
345#if LJ_TARGET_X86 && LJ_ABI_WIN 368#if LJ_TARGET_X86 && LJ_ABI_WIN
346 /* Retry with decorated name for fastcall/stdcall functions. */ 369 /* Retry with decorated name for fastcall/stdcall functions. */
347 if (!p && ctype_isfunc(ct->info)) { 370 if (!p && ctype_isfunc(ct->info)) {
348 CTInfo cconv = ctype_cconv(ct->info); 371 CTInfo cconv = ctype_cconv(ct->info);
349 if (cconv == CTCC_FASTCALL || cconv == CTCC_STDCALL) { 372 if (cconv == CTCC_FASTCALL || cconv == CTCC_STDCALL) {
350 CTSize sz = clib_func_argsize(cts, ct); 373 CTSize sz = clib_func_argsize(cts, ct);
351 const char *symd = lj_str_pushf(L, 374 const char *symd = lj_strfmt_pushf(L,
352 cconv == CTCC_FASTCALL ? "@%s@%d" : "_%s@%d", 375 cconv == CTCC_FASTCALL ? "@%s@%d" : "_%s@%d",
353 sym, sz); 376 sym, sz);
354 L->top--; 377 L->top--;
diff --git a/src/lj_cparse.c b/src/lj_cparse.c
index f807c5ae..6c3bb2f9 100644
--- a/src/lj_cparse.c
+++ b/src/lj_cparse.c
@@ -9,13 +9,14 @@
9 9
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_str.h" 12#include "lj_buf.h"
13#include "lj_ctype.h" 13#include "lj_ctype.h"
14#include "lj_cparse.h" 14#include "lj_cparse.h"
15#include "lj_frame.h" 15#include "lj_frame.h"
16#include "lj_vm.h" 16#include "lj_vm.h"
17#include "lj_char.h" 17#include "lj_char.h"
18#include "lj_strscan.h" 18#include "lj_strscan.h"
19#include "lj_strfmt.h"
19 20
20/* 21/*
21** Important note: this is NOT a validating C parser! This is a minimal 22** Important note: this is NOT a validating C parser! This is a minimal
@@ -27,6 +28,30 @@
27** If in doubt, please check the input against your favorite C compiler. 28** If in doubt, please check the input against your favorite C compiler.
28*/ 29*/
29 30
31#ifdef LUA_USE_ASSERT
32#define lj_assertCP(c, ...) (lj_assertG_(G(cp->L), (c), __VA_ARGS__))
33#else
34#define lj_assertCP(c, ...) ((void)cp)
35#endif
36
37/* -- Miscellaneous ------------------------------------------------------- */
38
39/* Match string against a C literal. */
40#define cp_str_is(str, k) \
41 ((str)->len == sizeof(k)-1 && !memcmp(strdata(str), k, sizeof(k)-1))
42
43/* Check string against a linear list of matches. */
44int lj_cparse_case(GCstr *str, const char *match)
45{
46 MSize len;
47 int n;
48 for (n = 0; (len = (MSize)*match++); n++, match += len) {
49 if (str->len == len && !memcmp(match, strdata(str), len))
50 return n;
51 }
52 return -1;
53}
54
30/* -- C lexer ------------------------------------------------------------- */ 55/* -- C lexer ------------------------------------------------------------- */
31 56
32/* C lexer token names. */ 57/* C lexer token names. */
@@ -42,13 +67,13 @@ LJ_NORET static void cp_err(CPState *cp, ErrMsg em);
42 67
43static const char *cp_tok2str(CPState *cp, CPToken tok) 68static const char *cp_tok2str(CPState *cp, CPToken tok)
44{ 69{
45 lua_assert(tok < CTOK_FIRSTDECL); 70 lj_assertCP(tok < CTOK_FIRSTDECL, "bad CPToken %d", tok);
46 if (tok > CTOK_OFS) 71 if (tok > CTOK_OFS)
47 return ctoknames[tok-CTOK_OFS-1]; 72 return ctoknames[tok-CTOK_OFS-1];
48 else if (!lj_char_iscntrl(tok)) 73 else if (!lj_char_iscntrl(tok))
49 return lj_str_pushf(cp->L, "%c", tok); 74 return lj_strfmt_pushf(cp->L, "%c", tok);
50 else 75 else
51 return lj_str_pushf(cp->L, "char(%d)", tok); 76 return lj_strfmt_pushf(cp->L, "char(%d)", tok);
52} 77}
53 78
54/* End-of-line? */ 79/* End-of-line? */
@@ -85,24 +110,10 @@ static LJ_NOINLINE CPChar cp_get_bs(CPState *cp)
85 return cp_get(cp); 110 return cp_get(cp);
86} 111}
87 112
88/* Grow save buffer. */
89static LJ_NOINLINE void cp_save_grow(CPState *cp, CPChar c)
90{
91 MSize newsize;
92 if (cp->sb.sz >= CPARSE_MAX_BUF/2)
93 cp_err(cp, LJ_ERR_XELEM);
94 newsize = cp->sb.sz * 2;
95 lj_str_resizebuf(cp->L, &cp->sb, newsize);
96 cp->sb.buf[cp->sb.n++] = (char)c;
97}
98
99/* Save character in buffer. */ 113/* Save character in buffer. */
100static LJ_AINLINE void cp_save(CPState *cp, CPChar c) 114static LJ_AINLINE void cp_save(CPState *cp, CPChar c)
101{ 115{
102 if (LJ_UNLIKELY(cp->sb.n + 1 > cp->sb.sz)) 116 lj_buf_putb(&cp->sb, c);
103 cp_save_grow(cp, c);
104 else
105 cp->sb.buf[cp->sb.n++] = (char)c;
106} 117}
107 118
108/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */ 119/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
@@ -122,20 +133,20 @@ LJ_NORET static void cp_errmsg(CPState *cp, CPToken tok, ErrMsg em, ...)
122 tokstr = NULL; 133 tokstr = NULL;
123 } else if (tok == CTOK_IDENT || tok == CTOK_INTEGER || tok == CTOK_STRING || 134 } else if (tok == CTOK_IDENT || tok == CTOK_INTEGER || tok == CTOK_STRING ||
124 tok >= CTOK_FIRSTDECL) { 135 tok >= CTOK_FIRSTDECL) {
125 if (cp->sb.n == 0) cp_save(cp, '$'); 136 if (cp->sb.w == cp->sb.b) cp_save(cp, '$');
126 cp_save(cp, '\0'); 137 cp_save(cp, '\0');
127 tokstr = cp->sb.buf; 138 tokstr = cp->sb.b;
128 } else { 139 } else {
129 tokstr = cp_tok2str(cp, tok); 140 tokstr = cp_tok2str(cp, tok);
130 } 141 }
131 L = cp->L; 142 L = cp->L;
132 va_start(argp, em); 143 va_start(argp, em);
133 msg = lj_str_pushvf(L, err2msg(em), argp); 144 msg = lj_strfmt_pushvf(L, err2msg(em), argp);
134 va_end(argp); 145 va_end(argp);
135 if (tokstr) 146 if (tokstr)
136 msg = lj_str_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tokstr); 147 msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tokstr);
137 if (cp->linenumber > 1) 148 if (cp->linenumber > 1)
138 msg = lj_str_pushf(L, "%s at line %d", msg, cp->linenumber); 149 msg = lj_strfmt_pushf(L, "%s at line %d", msg, cp->linenumber);
139 lj_err_callermsg(L, msg); 150 lj_err_callermsg(L, msg);
140} 151}
141 152
@@ -164,7 +175,8 @@ static CPToken cp_number(CPState *cp)
164 TValue o; 175 TValue o;
165 do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp))); 176 do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp)));
166 cp_save(cp, '\0'); 177 cp_save(cp, '\0');
167 fmt = lj_strscan_scan((const uint8_t *)cp->sb.buf, &o, STRSCAN_OPT_C); 178 fmt = lj_strscan_scan((const uint8_t *)(cp->sb.b), sbuflen(&cp->sb)-1,
179 &o, STRSCAN_OPT_C);
168 if (fmt == STRSCAN_INT) cp->val.id = CTID_INT32; 180 if (fmt == STRSCAN_INT) cp->val.id = CTID_INT32;
169 else if (fmt == STRSCAN_U32) cp->val.id = CTID_UINT32; 181 else if (fmt == STRSCAN_U32) cp->val.id = CTID_UINT32;
170 else if (!(cp->mode & CPARSE_MODE_SKIP)) 182 else if (!(cp->mode & CPARSE_MODE_SKIP))
@@ -177,7 +189,7 @@ static CPToken cp_number(CPState *cp)
177static CPToken cp_ident(CPState *cp) 189static CPToken cp_ident(CPState *cp)
178{ 190{
179 do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp))); 191 do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp)));
180 cp->str = lj_str_new(cp->L, cp->sb.buf, cp->sb.n); 192 cp->str = lj_buf_str(cp->L, &cp->sb);
181 cp->val.id = lj_ctype_getname(cp->cts, &cp->ct, cp->str, cp->tmask); 193 cp->val.id = lj_ctype_getname(cp->cts, &cp->ct, cp->str, cp->tmask);
182 if (ctype_type(cp->ct->info) == CT_KW) 194 if (ctype_type(cp->ct->info) == CT_KW)
183 return ctype_cid(cp->ct->info); 195 return ctype_cid(cp->ct->info);
@@ -263,11 +275,11 @@ static CPToken cp_string(CPState *cp)
263 } 275 }
264 cp_get(cp); 276 cp_get(cp);
265 if (delim == '"') { 277 if (delim == '"') {
266 cp->str = lj_str_new(cp->L, cp->sb.buf, cp->sb.n); 278 cp->str = lj_buf_str(cp->L, &cp->sb);
267 return CTOK_STRING; 279 return CTOK_STRING;
268 } else { 280 } else {
269 if (cp->sb.n != 1) cp_err_token(cp, '\''); 281 if (sbuflen(&cp->sb) != 1) cp_err_token(cp, '\'');
270 cp->val.i32 = (int32_t)(char)cp->sb.buf[0]; 282 cp->val.i32 = (int32_t)(char)*cp->sb.b;
271 cp->val.id = CTID_INT32; 283 cp->val.id = CTID_INT32;
272 return CTOK_INTEGER; 284 return CTOK_INTEGER;
273 } 285 }
@@ -296,7 +308,7 @@ static void cp_comment_cpp(CPState *cp)
296/* Lexical scanner for C. Only a minimal subset is implemented. */ 308/* Lexical scanner for C. Only a minimal subset is implemented. */
297static CPToken cp_next_(CPState *cp) 309static CPToken cp_next_(CPState *cp)
298{ 310{
299 lj_str_resetbuf(&cp->sb); 311 lj_buf_reset(&cp->sb);
300 for (;;) { 312 for (;;) {
301 if (lj_char_isident(cp->c)) 313 if (lj_char_isident(cp->c))
302 return lj_char_isdigit(cp->c) ? cp_number(cp) : cp_ident(cp); 314 return lj_char_isdigit(cp->c) ? cp_number(cp) : cp_ident(cp);
@@ -385,9 +397,8 @@ static void cp_init(CPState *cp)
385 cp->depth = 0; 397 cp->depth = 0;
386 cp->curpack = 0; 398 cp->curpack = 0;
387 cp->packstack[0] = 255; 399 cp->packstack[0] = 255;
388 lj_str_initbuf(&cp->sb); 400 lj_buf_init(cp->L, &cp->sb);
389 lj_str_resizebuf(cp->L, &cp->sb, LJ_MIN_SBUF); 401 lj_assertCP(cp->p != NULL, "uninitialized cp->p");
390 lua_assert(cp->p != NULL);
391 cp_get(cp); /* Read-ahead first char. */ 402 cp_get(cp); /* Read-ahead first char. */
392 cp->tok = 0; 403 cp->tok = 0;
393 cp->tmask = CPNS_DEFAULT; 404 cp->tmask = CPNS_DEFAULT;
@@ -398,7 +409,7 @@ static void cp_init(CPState *cp)
398static void cp_cleanup(CPState *cp) 409static void cp_cleanup(CPState *cp)
399{ 410{
400 global_State *g = G(cp->L); 411 global_State *g = G(cp->L);
401 lj_str_freebuf(g, &cp->sb); 412 lj_buf_free(g, &cp->sb);
402} 413}
403 414
404/* Check and consume optional token. */ 415/* Check and consume optional token. */
@@ -848,12 +859,13 @@ static CTypeID cp_decl_intern(CPState *cp, CPDecl *decl)
848 /* The cid is already part of info for copies of pointers/functions. */ 859 /* The cid is already part of info for copies of pointers/functions. */
849 idx = ct->next; 860 idx = ct->next;
850 if (ctype_istypedef(info)) { 861 if (ctype_istypedef(info)) {
851 lua_assert(id == 0); 862 lj_assertCP(id == 0, "typedef not at toplevel");
852 id = ctype_cid(info); 863 id = ctype_cid(info);
853 /* Always refetch info/size, since struct/enum may have been completed. */ 864 /* Always refetch info/size, since struct/enum may have been completed. */
854 cinfo = ctype_get(cp->cts, id)->info; 865 cinfo = ctype_get(cp->cts, id)->info;
855 csize = ctype_get(cp->cts, id)->size; 866 csize = ctype_get(cp->cts, id)->size;
856 lua_assert(ctype_isstruct(cinfo) || ctype_isenum(cinfo)); 867 lj_assertCP(ctype_isstruct(cinfo) || ctype_isenum(cinfo),
868 "typedef of bad type");
857 } else if (ctype_isfunc(info)) { /* Intern function. */ 869 } else if (ctype_isfunc(info)) { /* Intern function. */
858 CType *fct; 870 CType *fct;
859 CTypeID fid; 871 CTypeID fid;
@@ -886,7 +898,7 @@ static CTypeID cp_decl_intern(CPState *cp, CPDecl *decl)
886 /* Inherit csize/cinfo from original type. */ 898 /* Inherit csize/cinfo from original type. */
887 } else { 899 } else {
888 if (ctype_isnum(info)) { /* Handle mode/vector-size attributes. */ 900 if (ctype_isnum(info)) { /* Handle mode/vector-size attributes. */
889 lua_assert(id == 0); 901 lj_assertCP(id == 0, "number not at toplevel");
890 if (!(info & CTF_BOOL)) { 902 if (!(info & CTF_BOOL)) {
891 CTSize msize = ctype_msizeP(decl->attr); 903 CTSize msize = ctype_msizeP(decl->attr);
892 CTSize vsize = ctype_vsizeP(decl->attr); 904 CTSize vsize = ctype_vsizeP(decl->attr);
@@ -941,7 +953,7 @@ static CTypeID cp_decl_intern(CPState *cp, CPDecl *decl)
941 info = (info & ~CTF_ALIGN) | (cinfo & CTF_ALIGN); 953 info = (info & ~CTF_ALIGN) | (cinfo & CTF_ALIGN);
942 info |= (cinfo & CTF_QUAL); /* Inherit qual. */ 954 info |= (cinfo & CTF_QUAL); /* Inherit qual. */
943 } else { 955 } else {
944 lua_assert(ctype_isvoid(info)); 956 lj_assertCP(ctype_isvoid(info), "bad ctype %08x", info);
945 } 957 }
946 csize = size; 958 csize = size;
947 cinfo = info+id; 959 cinfo = info+id;
@@ -953,8 +965,6 @@ static CTypeID cp_decl_intern(CPState *cp, CPDecl *decl)
953 965
954/* -- C declaration parser ------------------------------------------------ */ 966/* -- C declaration parser ------------------------------------------------ */
955 967
956#define H_(le, be) LJ_ENDIAN_SELECT(0x##le, 0x##be)
957
958/* Reset declaration state to declaration specifier. */ 968/* Reset declaration state to declaration specifier. */
959static void cp_decl_reset(CPDecl *decl) 969static void cp_decl_reset(CPDecl *decl)
960{ 970{
@@ -1031,7 +1041,7 @@ static void cp_decl_asm(CPState *cp, CPDecl *decl)
1031 if (cp->tok == CTOK_STRING) { 1041 if (cp->tok == CTOK_STRING) {
1032 GCstr *str = cp->str; 1042 GCstr *str = cp->str;
1033 while (cp_next(cp) == CTOK_STRING) { 1043 while (cp_next(cp) == CTOK_STRING) {
1034 lj_str_pushf(cp->L, "%s%s", strdata(str), strdata(cp->str)); 1044 lj_strfmt_pushf(cp->L, "%s%s", strdata(str), strdata(cp->str));
1035 cp->L->top--; 1045 cp->L->top--;
1036 str = strV(cp->L->top); 1046 str = strV(cp->L->top);
1037 } 1047 }
@@ -1083,44 +1093,57 @@ static void cp_decl_gccattribute(CPState *cp, CPDecl *decl)
1083 if (cp->tok == CTOK_IDENT) { 1093 if (cp->tok == CTOK_IDENT) {
1084 GCstr *attrstr = cp->str; 1094 GCstr *attrstr = cp->str;
1085 cp_next(cp); 1095 cp_next(cp);
1086 switch (attrstr->hash) { 1096 switch (lj_cparse_case(attrstr,
1087 case H_(64a9208e,8ce14319): case H_(8e6331b2,95a282af): /* aligned */ 1097 "\007aligned" "\013__aligned__"
1098 "\006packed" "\012__packed__"
1099 "\004mode" "\010__mode__"
1100 "\013vector_size" "\017__vector_size__"
1101#if LJ_TARGET_X86
1102 "\007regparm" "\013__regparm__"
1103 "\005cdecl" "\011__cdecl__"
1104 "\010thiscall" "\014__thiscall__"
1105 "\010fastcall" "\014__fastcall__"
1106 "\007stdcall" "\013__stdcall__"
1107 "\012sseregparm" "\016__sseregparm__"
1108#endif
1109 )) {
1110 case 0: case 1: /* aligned */
1088 cp_decl_align(cp, decl); 1111 cp_decl_align(cp, decl);
1089 break; 1112 break;
1090 case H_(42eb47de,f0ede26c): case H_(29f48a09,cf383e0c): /* packed */ 1113 case 2: case 3: /* packed */
1091 decl->attr |= CTFP_PACKED; 1114 decl->attr |= CTFP_PACKED;
1092 break; 1115 break;
1093 case H_(0a84eef6,8dfab04c): case H_(995cf92c,d5696591): /* mode */ 1116 case 4: case 5: /* mode */
1094 cp_decl_mode(cp, decl); 1117 cp_decl_mode(cp, decl);
1095 break; 1118 break;
1096 case H_(0ab31997,2d5213fa): case H_(bf875611,200e9990): /* vector_size */ 1119 case 6: case 7: /* vector_size */
1097 { 1120 {
1098 CTSize vsize = cp_decl_sizeattr(cp); 1121 CTSize vsize = cp_decl_sizeattr(cp);
1099 if (vsize) CTF_INSERT(decl->attr, VSIZEP, lj_fls(vsize)); 1122 if (vsize) CTF_INSERT(decl->attr, VSIZEP, lj_fls(vsize));
1100 } 1123 }
1101 break; 1124 break;
1102#if LJ_TARGET_X86 1125#if LJ_TARGET_X86
1103 case H_(5ad22db8,c689b848): case H_(439150fa,65ea78cb): /* regparm */ 1126 case 8: case 9: /* regparm */
1104 CTF_INSERT(decl->fattr, REGPARM, cp_decl_sizeattr(cp)); 1127 CTF_INSERT(decl->fattr, REGPARM, cp_decl_sizeattr(cp));
1105 decl->fattr |= CTFP_CCONV; 1128 decl->fattr |= CTFP_CCONV;
1106 break; 1129 break;
1107 case H_(18fc0b98,7ff4c074): case H_(4e62abed,0a747424): /* cdecl */ 1130 case 10: case 11: /* cdecl */
1108 CTF_INSERT(decl->fattr, CCONV, CTCC_CDECL); 1131 CTF_INSERT(decl->fattr, CCONV, CTCC_CDECL);
1109 decl->fattr |= CTFP_CCONV; 1132 decl->fattr |= CTFP_CCONV;
1110 break; 1133 break;
1111 case H_(72b2e41b,494c5a44): case H_(f2356d59,f25fc9bd): /* thiscall */ 1134 case 12: case 13: /* thiscall */
1112 CTF_INSERT(decl->fattr, CCONV, CTCC_THISCALL); 1135 CTF_INSERT(decl->fattr, CCONV, CTCC_THISCALL);
1113 decl->fattr |= CTFP_CCONV; 1136 decl->fattr |= CTFP_CCONV;
1114 break; 1137 break;
1115 case H_(0d0ffc42,ab746f88): case H_(21c54ba1,7f0ca7e3): /* fastcall */ 1138 case 14: case 15: /* fastcall */
1116 CTF_INSERT(decl->fattr, CCONV, CTCC_FASTCALL); 1139 CTF_INSERT(decl->fattr, CCONV, CTCC_FASTCALL);
1117 decl->fattr |= CTFP_CCONV; 1140 decl->fattr |= CTFP_CCONV;
1118 break; 1141 break;
1119 case H_(ef76b040,9412e06a): case H_(de56697b,c750e6e1): /* stdcall */ 1142 case 16: case 17: /* stdcall */
1120 CTF_INSERT(decl->fattr, CCONV, CTCC_STDCALL); 1143 CTF_INSERT(decl->fattr, CCONV, CTCC_STDCALL);
1121 decl->fattr |= CTFP_CCONV; 1144 decl->fattr |= CTFP_CCONV;
1122 break; 1145 break;
1123 case H_(ea78b622,f234bd8e): case H_(252ffb06,8d50f34b): /* sseregparm */ 1146 case 18: case 19: /* sseregparm */
1124 decl->fattr |= CTF_SSEREGPARM; 1147 decl->fattr |= CTF_SSEREGPARM;
1125 decl->fattr |= CTFP_CCONV; 1148 decl->fattr |= CTFP_CCONV;
1126 break; 1149 break;
@@ -1152,16 +1175,13 @@ static void cp_decl_msvcattribute(CPState *cp, CPDecl *decl)
1152 while (cp->tok == CTOK_IDENT) { 1175 while (cp->tok == CTOK_IDENT) {
1153 GCstr *attrstr = cp->str; 1176 GCstr *attrstr = cp->str;
1154 cp_next(cp); 1177 cp_next(cp);
1155 switch (attrstr->hash) { 1178 if (cp_str_is(attrstr, "align")) {
1156 case H_(bc2395fa,98f267f8): /* align */
1157 cp_decl_align(cp, decl); 1179 cp_decl_align(cp, decl);
1158 break; 1180 } else { /* Ignore all other attributes. */
1159 default: /* Ignore all other attributes. */
1160 if (cp_opt(cp, '(')) { 1181 if (cp_opt(cp, '(')) {
1161 while (cp->tok != ')' && cp->tok != CTOK_EOF) cp_next(cp); 1182 while (cp->tok != ')' && cp->tok != CTOK_EOF) cp_next(cp);
1162 cp_check(cp, ')'); 1183 cp_check(cp, ')');
1163 } 1184 }
1164 break;
1165 } 1185 }
1166 } 1186 }
1167 cp_check(cp, ')'); 1187 cp_check(cp, ')');
@@ -1572,7 +1592,7 @@ end_decl:
1572 cp_errmsg(cp, cp->tok, LJ_ERR_FFI_DECLSPEC); 1592 cp_errmsg(cp, cp->tok, LJ_ERR_FFI_DECLSPEC);
1573 sz = sizeof(int); 1593 sz = sizeof(int);
1574 } 1594 }
1575 lua_assert(sz != 0); 1595 lj_assertCP(sz != 0, "basic ctype with zero size");
1576 info += CTALIGN(lj_fls(sz)); /* Use natural alignment. */ 1596 info += CTALIGN(lj_fls(sz)); /* Use natural alignment. */
1577 info += (decl->attr & CTF_QUAL); /* Merge qualifiers. */ 1597 info += (decl->attr & CTF_QUAL); /* Merge qualifiers. */
1578 cp_push(decl, info, sz); 1598 cp_push(decl, info, sz);
@@ -1741,17 +1761,16 @@ static CTypeID cp_decl_abstract(CPState *cp)
1741static void cp_pragma(CPState *cp, BCLine pragmaline) 1761static void cp_pragma(CPState *cp, BCLine pragmaline)
1742{ 1762{
1743 cp_next(cp); 1763 cp_next(cp);
1744 if (cp->tok == CTOK_IDENT && 1764 if (cp->tok == CTOK_IDENT && cp_str_is(cp->str, "pack")) {
1745 cp->str->hash == H_(e79b999f,42ca3e85)) { /* pack */
1746 cp_next(cp); 1765 cp_next(cp);
1747 cp_check(cp, '('); 1766 cp_check(cp, '(');
1748 if (cp->tok == CTOK_IDENT) { 1767 if (cp->tok == CTOK_IDENT) {
1749 if (cp->str->hash == H_(738e923c,a1b65954)) { /* push */ 1768 if (cp_str_is(cp->str, "push")) {
1750 if (cp->curpack < CPARSE_MAX_PACKSTACK) { 1769 if (cp->curpack < CPARSE_MAX_PACKSTACK) {
1751 cp->packstack[cp->curpack+1] = cp->packstack[cp->curpack]; 1770 cp->packstack[cp->curpack+1] = cp->packstack[cp->curpack];
1752 cp->curpack++; 1771 cp->curpack++;
1753 } 1772 }
1754 } else if (cp->str->hash == H_(6c71cf27,6c71cf27)) { /* pop */ 1773 } else if (cp_str_is(cp->str, "pop")) {
1755 if (cp->curpack > 0) cp->curpack--; 1774 if (cp->curpack > 0) cp->curpack--;
1756 } else { 1775 } else {
1757 cp_errmsg(cp, cp->tok, LJ_ERR_XSYMBOL); 1776 cp_errmsg(cp, cp->tok, LJ_ERR_XSYMBOL);
@@ -1773,6 +1792,16 @@ static void cp_pragma(CPState *cp, BCLine pragmaline)
1773 } 1792 }
1774} 1793}
1775 1794
1795/* Handle line number. */
1796static void cp_line(CPState *cp, BCLine hashline)
1797{
1798 BCLine newline = cp->val.u32;
1799 /* TODO: Handle file name and include it in error messages. */
1800 while (cp->tok != CTOK_EOF && cp->linenumber == hashline)
1801 cp_next(cp);
1802 cp->linenumber = newline;
1803}
1804
1776/* Parse multiple C declarations of types or extern identifiers. */ 1805/* Parse multiple C declarations of types or extern identifiers. */
1777static void cp_decl_multi(CPState *cp) 1806static void cp_decl_multi(CPState *cp)
1778{ 1807{
@@ -1785,12 +1814,21 @@ static void cp_decl_multi(CPState *cp)
1785 continue; 1814 continue;
1786 } 1815 }
1787 if (cp->tok == '#') { /* Workaround, since we have no preprocessor, yet. */ 1816 if (cp->tok == '#') { /* Workaround, since we have no preprocessor, yet. */
1788 BCLine pragmaline = cp->linenumber; 1817 BCLine hashline = cp->linenumber;
1789 if (!(cp_next(cp) == CTOK_IDENT && 1818 CPToken tok = cp_next(cp);
1790 cp->str->hash == H_(f5e6b4f8,1d509107))) /* pragma */ 1819 if (tok == CTOK_INTEGER) {
1820 cp_line(cp, hashline);
1821 continue;
1822 } else if (tok == CTOK_IDENT && cp_str_is(cp->str, "line")) {
1823 if (cp_next(cp) != CTOK_INTEGER) cp_err_token(cp, tok);
1824 cp_line(cp, hashline);
1825 continue;
1826 } else if (tok == CTOK_IDENT && cp_str_is(cp->str, "pragma")) {
1827 cp_pragma(cp, hashline);
1828 continue;
1829 } else {
1791 cp_errmsg(cp, cp->tok, LJ_ERR_XSYMBOL); 1830 cp_errmsg(cp, cp->tok, LJ_ERR_XSYMBOL);
1792 cp_pragma(cp, pragmaline); 1831 }
1793 continue;
1794 } 1832 }
1795 scl = cp_decl_spec(cp, &decl, CDF_TYPEDEF|CDF_EXTERN|CDF_STATIC); 1833 scl = cp_decl_spec(cp, &decl, CDF_TYPEDEF|CDF_EXTERN|CDF_STATIC);
1796 if ((cp->tok == ';' || cp->tok == CTOK_EOF) && 1834 if ((cp->tok == ';' || cp->tok == CTOK_EOF) &&
@@ -1814,7 +1852,7 @@ static void cp_decl_multi(CPState *cp)
1814 /* Treat both static and extern function declarations as extern. */ 1852 /* Treat both static and extern function declarations as extern. */
1815 ct = ctype_get(cp->cts, ctypeid); 1853 ct = ctype_get(cp->cts, ctypeid);
1816 /* We always get new anonymous functions (typedefs are copied). */ 1854 /* We always get new anonymous functions (typedefs are copied). */
1817 lua_assert(gcref(ct->name) == NULL); 1855 lj_assertCP(gcref(ct->name) == NULL, "unexpected named function");
1818 id = ctypeid; /* Just name it. */ 1856 id = ctypeid; /* Just name it. */
1819 } else if ((scl & CDF_STATIC)) { /* Accept static constants. */ 1857 } else if ((scl & CDF_STATIC)) { /* Accept static constants. */
1820 id = cp_decl_constinit(cp, &ct, ctypeid); 1858 id = cp_decl_constinit(cp, &ct, ctypeid);
@@ -1856,8 +1894,6 @@ static void cp_decl_single(CPState *cp)
1856 if (cp->tok != CTOK_EOF) cp_err_token(cp, CTOK_EOF); 1894 if (cp->tok != CTOK_EOF) cp_err_token(cp, CTOK_EOF);
1857} 1895}
1858 1896
1859#undef H_
1860
1861/* ------------------------------------------------------------------------ */ 1897/* ------------------------------------------------------------------------ */
1862 1898
1863/* Protected callback for C parser. */ 1899/* Protected callback for C parser. */
@@ -1873,7 +1909,7 @@ static TValue *cpcparser(lua_State *L, lua_CFunction dummy, void *ud)
1873 cp_decl_single(cp); 1909 cp_decl_single(cp);
1874 if (cp->param && cp->param != cp->L->top) 1910 if (cp->param && cp->param != cp->L->top)
1875 cp_err(cp, LJ_ERR_FFI_NUMPARAM); 1911 cp_err(cp, LJ_ERR_FFI_NUMPARAM);
1876 lua_assert(cp->depth == 0); 1912 lj_assertCP(cp->depth == 0, "unbalanced cparser declaration depth");
1877 return NULL; 1913 return NULL;
1878} 1914}
1879 1915
diff --git a/src/lj_cparse.h b/src/lj_cparse.h
index f5268693..42f827a0 100644
--- a/src/lj_cparse.h
+++ b/src/lj_cparse.h
@@ -60,6 +60,8 @@ typedef struct CPState {
60 60
61LJ_FUNC int lj_cparse(CPState *cp); 61LJ_FUNC int lj_cparse(CPState *cp);
62 62
63LJ_FUNC int lj_cparse_case(GCstr *str, const char *match);
64
63#endif 65#endif
64 66
65#endif 67#endif
diff --git a/src/lj_crecord.c b/src/lj_crecord.c
index 63a72aa4..04bc895d 100644
--- a/src/lj_crecord.c
+++ b/src/lj_crecord.c
@@ -11,13 +11,13 @@
11#if LJ_HASJIT && LJ_HASFFI 11#if LJ_HASJIT && LJ_HASFFI
12 12
13#include "lj_err.h" 13#include "lj_err.h"
14#include "lj_str.h"
15#include "lj_tab.h" 14#include "lj_tab.h"
16#include "lj_frame.h" 15#include "lj_frame.h"
17#include "lj_ctype.h" 16#include "lj_ctype.h"
18#include "lj_cdata.h" 17#include "lj_cdata.h"
19#include "lj_cparse.h" 18#include "lj_cparse.h"
20#include "lj_cconv.h" 19#include "lj_cconv.h"
20#include "lj_carith.h"
21#include "lj_clib.h" 21#include "lj_clib.h"
22#include "lj_ccall.h" 22#include "lj_ccall.h"
23#include "lj_ff.h" 23#include "lj_ff.h"
@@ -31,6 +31,7 @@
31#include "lj_snap.h" 31#include "lj_snap.h"
32#include "lj_crecord.h" 32#include "lj_crecord.h"
33#include "lj_dispatch.h" 33#include "lj_dispatch.h"
34#include "lj_strfmt.h"
34 35
35/* Some local macros to save typing. Undef'd at the end. */ 36/* Some local macros to save typing. Undef'd at the end. */
36#define IR(ref) (&J->cur.ir[(ref)]) 37#define IR(ref) (&J->cur.ir[(ref)])
@@ -60,7 +61,8 @@ static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
60static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr) 61static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
61{ 62{
62 CTypeID id; 63 CTypeID id;
63 lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID); 64 lj_assertJ(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID,
65 "expected CTypeID cdata");
64 id = *(CTypeID *)cdataptr(cd); 66 id = *(CTypeID *)cdataptr(cd);
65 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT); 67 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
66 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id)); 68 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
@@ -211,7 +213,7 @@ static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
211 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0); 213 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
212 ml[i].trofs = trofs; 214 ml[i].trofs = trofs;
213 i++; 215 i++;
214 rwin += (LJ_SOFTFP && ml[i].tp == IRT_NUM) ? 2 : 1; 216 rwin += (LJ_SOFTFP32 && ml[i].tp == IRT_NUM) ? 2 : 1;
215 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */ 217 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */
216 rwin = 0; 218 rwin = 0;
217 for ( ; j < i; j++) { 219 for ( ; j < i; j++) {
@@ -236,13 +238,14 @@ static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
236 if (len > CREC_COPY_MAXLEN) goto fallback; 238 if (len > CREC_COPY_MAXLEN) goto fallback;
237 if (ct) { 239 if (ct) {
238 CTState *cts = ctype_ctsG(J2G(J)); 240 CTState *cts = ctype_ctsG(J2G(J));
239 lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info)); 241 lj_assertJ(ctype_isarray(ct->info) || ctype_isstruct(ct->info),
242 "copy of non-aggregate");
240 if (ctype_isarray(ct->info)) { 243 if (ctype_isarray(ct->info)) {
241 CType *cct = ctype_rawchild(cts, ct); 244 CType *cct = ctype_rawchild(cts, ct);
242 tp = crec_ct2irt(cts, cct); 245 tp = crec_ct2irt(cts, cct);
243 if (tp == IRT_CDATA) goto rawcopy; 246 if (tp == IRT_CDATA) goto rawcopy;
244 step = lj_ir_type_size[tp]; 247 step = lj_ir_type_size[tp];
245 lua_assert((len & (step-1)) == 0); 248 lj_assertJ((len & (step-1)) == 0, "copy of fractional size");
246 } else if ((ct->info & CTF_UNION)) { 249 } else if ((ct->info & CTF_UNION)) {
247 step = (1u << ctype_align(ct->info)); 250 step = (1u << ctype_align(ct->info));
248 goto rawcopy; 251 goto rawcopy;
@@ -441,7 +444,7 @@ static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
441 /* fallthrough */ 444 /* fallthrough */
442 case CCX(I, F): 445 case CCX(I, F):
443 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi; 446 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
444 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY); 447 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
445 goto xstore; 448 goto xstore;
446 case CCX(I, P): 449 case CCX(I, P):
447 case CCX(I, A): 450 case CCX(I, A):
@@ -521,7 +524,7 @@ static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
521 if (st == IRT_CDATA) goto err_nyi; 524 if (st == IRT_CDATA) goto err_nyi;
522 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */ 525 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
523 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32, 526 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
524 st, IRCONV_TRUNC|IRCONV_ANY); 527 st, IRCONV_ANY);
525 goto xstore; 528 goto xstore;
526 529
527 /* Destination is an array. */ 530 /* Destination is an array. */
@@ -613,10 +616,12 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
613 sp = lj_ir_kptr(J, NULL); 616 sp = lj_ir_kptr(J, NULL);
614 } else if (tref_isudata(sp)) { 617 } else if (tref_isudata(sp)) {
615 GCudata *ud = udataV(sval); 618 GCudata *ud = udataV(sval);
616 if (ud->udtype == UDTYPE_IO_FILE) { 619 if (ud->udtype == UDTYPE_IO_FILE || ud->udtype == UDTYPE_BUFFER) {
617 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE); 620 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
618 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE)); 621 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, ud->udtype));
619 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE); 622 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp,
623 ud->udtype == UDTYPE_IO_FILE ? IRFL_UDATA_FILE :
624 IRFL_SBUF_R);
620 } else { 625 } else {
621 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata))); 626 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
622 } 627 }
@@ -628,7 +633,8 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
628 /* Specialize to the name of the enum constant. */ 633 /* Specialize to the name of the enum constant. */
629 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str)); 634 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
630 if (cct && ctype_isconstval(cct->info)) { 635 if (cct && ctype_isconstval(cct->info)) {
631 lua_assert(ctype_child(cts, cct)->size == 4); 636 lj_assertJ(ctype_child(cts, cct)->size == 4,
637 "only 32 bit const supported"); /* NYI */
632 svisnz = (void *)(intptr_t)(ofs != 0); 638 svisnz = (void *)(intptr_t)(ofs != 0);
633 sp = lj_ir_kint(J, (int32_t)ofs); 639 sp = lj_ir_kint(J, (int32_t)ofs);
634 sid = ctype_cid(cct->info); 640 sid = ctype_cid(cct->info);
@@ -640,12 +646,22 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
640 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr))); 646 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
641 sid = CTID_A_CCHAR; 647 sid = CTID_A_CCHAR;
642 } 648 }
643 } else { /* NYI: tref_istab(sp), tref_islightud(sp). */ 649 } else if (tref_islightud(sp)) {
650#if LJ_64
651 lj_trace_err(J, LJ_TRERR_NYICONV);
652#endif
653 } else { /* NYI: tref_istab(sp). */
644 IRType t; 654 IRType t;
645 sid = argv2cdata(J, sp, sval)->ctypeid; 655 sid = argv2cdata(J, sp, sval)->ctypeid;
646 s = ctype_raw(cts, sid); 656 s = ctype_raw(cts, sid);
647 svisnz = cdataptr(cdataV(sval)); 657 svisnz = cdataptr(cdataV(sval));
648 t = crec_ct2irt(cts, s); 658 if (ctype_isfunc(s->info)) {
659 sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
660 s = ctype_get(cts, sid);
661 t = IRT_PTR;
662 } else {
663 t = crec_ct2irt(cts, s);
664 }
649 if (ctype_isptr(s->info)) { 665 if (ctype_isptr(s->info)) {
650 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR); 666 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
651 if (ctype_isref(s->info)) { 667 if (ctype_isref(s->info)) {
@@ -700,6 +716,19 @@ static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
700 return tr; 716 return tr;
701} 717}
702 718
719/* Tailcall to function. */
720static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
721{
722 TRef kfunc = lj_ir_kfunc(J, funcV(tv));
723#if LJ_FR2
724 J->base[-2] = kfunc;
725 J->base[-1] = TREF_FRAME;
726#else
727 J->base[-1] = kfunc | TREF_FRAME;
728#endif
729 rd->nres = -1; /* Pending tailcall. */
730}
731
703/* Record ctype __index/__newindex metamethods. */ 732/* Record ctype __index/__newindex metamethods. */
704static void crec_index_meta(jit_State *J, CTState *cts, CType *ct, 733static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
705 RecordFFData *rd) 734 RecordFFData *rd)
@@ -709,8 +738,7 @@ static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
709 if (!tv) 738 if (!tv)
710 lj_trace_err(J, LJ_TRERR_BADTYPE); 739 lj_trace_err(J, LJ_TRERR_BADTYPE);
711 if (tvisfunc(tv)) { 740 if (tvisfunc(tv)) {
712 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME; 741 crec_tailcall(J, rd, tv);
713 rd->nres = -1; /* Pending tailcall. */
714 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) { 742 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
715 /* Specialize to result of __index lookup. */ 743 /* Specialize to result of __index lookup. */
716 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]); 744 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
@@ -727,6 +755,48 @@ static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
727 } 755 }
728} 756}
729 757
758/* Record bitfield load/store. */
759static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
760{
761 IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
762 TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
763 CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
764 lj_assertJ(t <= IRT_U32, "only 32 bit bitfields supported"); /* NYI */
765 if (rd->data == 0) { /* __index metamethod. */
766 if ((info & CTF_BOOL)) {
767 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
768 /* Assume not equal to zero. Fixup and emit pending guard later. */
769 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
770 J->postproc = LJ_POST_FIXGUARD;
771 tr = TREF_TRUE;
772 } else if (!(info & CTF_UNSIGNED)) {
773 tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
774 tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
775 } else {
776 lj_assertJ(bsz < 32, "unexpected full bitfield index");
777 tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
778 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
779 /* We can omit the U32 to NUM conversion, since bsz < 32. */
780 }
781 J->base[0] = tr;
782 } else { /* __newindex metamethod. */
783 CTState *cts = ctype_ctsG(J2G(J));
784 CType *ct = ctype_get(cts,
785 (info & CTF_BOOL) ? CTID_BOOL :
786 (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
787 int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
788 TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
789 sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
790 /* Use of the target type avoids forwarding conversions. */
791 sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
792 tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
793 tr = emitir(IRT(IR_BOR, t), tr, sp);
794 emitir(IRT(IR_XSTORE, t), ptr, tr);
795 rd->nres = 0;
796 J->needsnap = 1;
797 }
798}
799
730void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd) 800void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
731{ 801{
732 TRef idx, ptr = J->base[0]; 802 TRef idx, ptr = J->base[0];
@@ -801,6 +871,7 @@ again:
801 CType *fct; 871 CType *fct;
802 fct = lj_ctype_getfield(cts, ct, name, &fofs); 872 fct = lj_ctype_getfield(cts, ct, name, &fofs);
803 if (fct) { 873 if (fct) {
874 ofs += (ptrdiff_t)fofs;
804 /* Always specialize to the field name. */ 875 /* Always specialize to the field name. */
805 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name)); 876 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
806 if (ctype_isconstval(fct->info)) { 877 if (ctype_isconstval(fct->info)) {
@@ -812,12 +883,14 @@ again:
812 J->base[0] = lj_ir_kint(J, (int32_t)fct->size); 883 J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
813 return; /* Interpreter will throw for newindex. */ 884 return; /* Interpreter will throw for newindex. */
814 } else if (ctype_isbitfield(fct->info)) { 885 } else if (ctype_isbitfield(fct->info)) {
815 lj_trace_err(J, LJ_TRERR_NYICONV); 886 if (ofs)
887 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
888 crec_index_bf(J, rd, ptr, fct->info);
889 return;
816 } else { 890 } else {
817 lua_assert(ctype_isfield(fct->info)); 891 lj_assertJ(ctype_isfield(fct->info), "field expected");
818 sid = ctype_cid(fct->info); 892 sid = ctype_cid(fct->info);
819 } 893 }
820 ofs += (ptrdiff_t)fofs;
821 } 894 }
822 } else if (ctype_iscomplex(ct->info)) { 895 } else if (ctype_iscomplex(ct->info)) {
823 if (name->len == 2 && 896 if (name->len == 2 &&
@@ -867,21 +940,17 @@ again:
867} 940}
868 941
869/* Record setting a finalizer. */ 942/* Record setting a finalizer. */
870static void crec_finalizer(jit_State *J, TRef trcd, cTValue *fin) 943static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
871{ 944{
872 TRef trlo = lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd); 945 if (tvisgcv(fin)) {
873 TRef trhi = emitir(IRT(IR_ADD, IRT_P32), trlo, lj_ir_kint(J, 4)); 946 if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
874 if (LJ_BE) { TRef tmp = trlo; trlo = trhi; trhi = tmp; } 947 } else if (tvisnil(fin)) {
875 if (tvisfunc(fin)) { 948 trfin = lj_ir_kptr(J, NULL);
876 emitir(IRT(IR_XSTORE, IRT_P32), trlo, lj_ir_kfunc(J, funcV(fin)));
877 emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TFUNC));
878 } else if (tviscdata(fin)) {
879 emitir(IRT(IR_XSTORE, IRT_P32), trlo,
880 lj_ir_kgc(J, obj2gco(cdataV(fin)), IRT_CDATA));
881 emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TCDATA));
882 } else { 949 } else {
883 lj_trace_err(J, LJ_TRERR_BADTYPE); 950 lj_trace_err(J, LJ_TRERR_BADTYPE);
884 } 951 }
952 lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
953 trfin, lj_ir_kint(J, (int32_t)itype(fin)));
885 J->needsnap = 1; 954 J->needsnap = 1;
886} 955}
887 956
@@ -892,10 +961,8 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
892 CTSize sz; 961 CTSize sz;
893 CTInfo info = lj_ctype_info(cts, id, &sz); 962 CTInfo info = lj_ctype_info(cts, id, &sz);
894 CType *d = ctype_raw(cts, id); 963 CType *d = ctype_raw(cts, id);
895 TRef trid; 964 TRef trcd, trid = lj_ir_kint(J, id);
896 if (!sz || sz > 128 || (info & CTF_VLA) || ctype_align(info) > CT_MEMALIGN) 965 cTValue *fin;
897 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: large/special allocations. */
898 trid = lj_ir_kint(J, id);
899 /* Use special instruction to box pointer or 32/64 bit integer. */ 966 /* Use special instruction to box pointer or 32/64 bit integer. */
900 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) { 967 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
901 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) : 968 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
@@ -903,11 +970,36 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
903 sz == 4 ? lj_ir_kint(J, 0) : 970 sz == 4 ? lj_ir_kint(J, 0) :
904 (lj_needsplit(J), lj_ir_kint64(J, 0)); 971 (lj_needsplit(J), lj_ir_kint64(J, 0));
905 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp); 972 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
973 return;
906 } else { 974 } else {
907 TRef trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, TREF_NIL); 975 TRef trsz = TREF_NIL;
908 cTValue *fin; 976 if ((info & CTF_VLA)) { /* Calculate VLA/VLS size at runtime. */
909 J->base[0] = trcd; 977 CTSize sz0, sz1;
910 if (J->base[1] && !J->base[2] && 978 if (!J->base[1] || J->base[2])
979 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init VLA/VLS. */
980 trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
981 J->base[1], &rd->argv[1]);
982 sz0 = lj_ctype_vlsize(cts, d, 0);
983 sz1 = lj_ctype_vlsize(cts, d, 1);
984 trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
985 trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
986 J->base[1] = 0; /* Simplify logic below. */
987 } else if (ctype_align(info) > CT_MEMALIGN) {
988 trsz = lj_ir_kint(J, sz);
989 }
990 trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
991 if (sz > 128 || (info & CTF_VLA)) {
992 TRef dp;
993 CTSize align;
994 special: /* Only handle bulk zero-fill for large/VLA/VLS types. */
995 if (J->base[1])
996 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init large/VLA/VLS types. */
997 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
998 if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
999 align = ctype_align(info);
1000 if (align < CT_MEMALIGN) align = CT_MEMALIGN;
1001 crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
1002 } else if (J->base[1] && !J->base[2] &&
911 !lj_cconv_multi_init(cts, d, &rd->argv[1])) { 1003 !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
912 goto single_init; 1004 goto single_init;
913 } else if (ctype_isarray(d->info)) { 1005 } else if (ctype_isarray(d->info)) {
@@ -918,8 +1010,9 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
918 TValue *sval = &tv; 1010 TValue *sval = &tv;
919 MSize i; 1011 MSize i;
920 tv.u64 = 0; 1012 tv.u64 = 0;
921 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info))) 1013 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
922 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init array of aggregates. */ 1014 esize * CREC_FILL_MAXUNROLL < sz)
1015 goto special;
923 for (i = 1, ofs = 0; ofs < sz; ofs += esize) { 1016 for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
924 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, 1017 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
925 lj_ir_kintp(J, ofs + sizeof(GCcdata))); 1018 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
@@ -933,8 +1026,26 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
933 crec_ct_tv(J, dc, dp, sp, sval); 1026 crec_ct_tv(J, dc, dp, sp, sval);
934 } 1027 }
935 } else if (ctype_isstruct(d->info)) { 1028 } else if (ctype_isstruct(d->info)) {
936 CTypeID fid = d->sib; 1029 CTypeID fid;
937 MSize i = 1; 1030 MSize i = 1;
1031 if (!J->base[1]) { /* Handle zero-fill of struct-of-NYI. */
1032 fid = d->sib;
1033 while (fid) {
1034 CType *df = ctype_get(cts, fid);
1035 fid = df->sib;
1036 if (ctype_isfield(df->info)) {
1037 CType *dc;
1038 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1039 dc = ctype_rawchild(cts, df); /* Field type. */
1040 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1041 ctype_isenum(dc->info)))
1042 goto special;
1043 } else if (!ctype_isconstval(df->info)) {
1044 goto special;
1045 }
1046 }
1047 }
1048 fid = d->sib;
938 while (fid) { 1049 while (fid) {
939 CType *df = ctype_get(cts, fid); 1050 CType *df = ctype_get(cts, fid);
940 fid = df->sib; 1051 fid = df->sib;
@@ -981,11 +1092,12 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
981 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv); 1092 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
982 } 1093 }
983 } 1094 }
984 /* Handle __gc metamethod. */
985 fin = lj_ctype_meta(cts, id, MM_gc);
986 if (fin)
987 crec_finalizer(J, trcd, fin);
988 } 1095 }
1096 J->base[0] = trcd;
1097 /* Handle __gc metamethod. */
1098 fin = lj_ctype_meta(cts, id, MM_gc);
1099 if (fin)
1100 crec_finalizer(J, trcd, 0, fin);
989} 1101}
990 1102
991/* Record argument conversions. */ 1103/* Record argument conversions. */
@@ -1026,7 +1138,7 @@ static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1026 if (fid) { /* Get argument type from field. */ 1138 if (fid) { /* Get argument type from field. */
1027 CType *ctf = ctype_get(cts, fid); 1139 CType *ctf = ctype_get(cts, fid);
1028 fid = ctf->sib; 1140 fid = ctf->sib;
1029 lua_assert(ctype_isfield(ctf->info)); 1141 lj_assertJ(ctype_isfield(ctf->info), "field expected");
1030 did = ctype_cid(ctf->info); 1142 did = ctype_cid(ctf->info);
1031 } else { 1143 } else {
1032 if (!(ct->info & CTF_VARARG)) 1144 if (!(ct->info & CTF_VARARG))
@@ -1045,7 +1157,7 @@ static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1045 else 1157 else
1046 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT); 1158 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1047 } 1159 }
1048 } else if (LJ_SOFTFP && ctype_isfp(d->info) && d->size > 4) { 1160 } else if (LJ_SOFTFP32 && ctype_isfp(d->info) && d->size > 4) {
1049 lj_needsplit(J); 1161 lj_needsplit(J);
1050 } 1162 }
1051#if LJ_TARGET_X86 1163#if LJ_TARGET_X86
@@ -1091,20 +1203,20 @@ static void crec_snap_caller(jit_State *J)
1091 lua_State *L = J->L; 1203 lua_State *L = J->L;
1092 TValue *base = L->base, *top = L->top; 1204 TValue *base = L->base, *top = L->top;
1093 const BCIns *pc = J->pc; 1205 const BCIns *pc = J->pc;
1094 TRef ftr = J->base[-1]; 1206 TRef ftr = J->base[-1-LJ_FR2];
1095 ptrdiff_t delta; 1207 ptrdiff_t delta;
1096 if (!frame_islua(base-1) || J->framedepth <= 0) 1208 if (!frame_islua(base-1) || J->framedepth <= 0)
1097 lj_trace_err(J, LJ_TRERR_NYICALL); 1209 lj_trace_err(J, LJ_TRERR_NYICALL);
1098 J->pc = frame_pc(base-1); delta = 1+bc_a(J->pc[-1]); 1210 J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1099 L->top = base; L->base = base - delta; 1211 L->top = base; L->base = base - delta;
1100 J->base[-1] = TREF_FALSE; 1212 J->base[-1-LJ_FR2] = TREF_FALSE;
1101 J->base -= delta; J->baseslot -= (BCReg)delta; 1213 J->base -= delta; J->baseslot -= (BCReg)delta;
1102 J->maxslot = (BCReg)delta; J->framedepth--; 1214 J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1103 lj_snap_add(J); 1215 lj_snap_add(J);
1104 L->base = base; L->top = top; 1216 L->base = base; L->top = top;
1105 J->framedepth++; J->maxslot = 1; 1217 J->framedepth++; J->maxslot = 1;
1106 J->base += delta; J->baseslot += (BCReg)delta; 1218 J->base += delta; J->baseslot += (BCReg)delta;
1107 J->base[-1] = ftr; J->pc = pc; 1219 J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1108} 1220}
1109 1221
1110/* Record function call. */ 1222/* Record function call. */
@@ -1124,8 +1236,7 @@ static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1124 TRef tr; 1236 TRef tr;
1125 TValue tv; 1237 TValue tv;
1126 /* Check for blacklisted C functions that might call a callback. */ 1238 /* Check for blacklisted C functions that might call a callback. */
1127 setlightudV(&tv, 1239 tv.u64 = ((uintptr_t)cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4) >> 2) | U64x(800000000, 00000000);
1128 cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
1129 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv))) 1240 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1130 lj_trace_err(J, LJ_TRERR_BLACKL); 1241 lj_trace_err(J, LJ_TRERR_BLACKL);
1131 if (ctype_isvoid(ctr->info)) { 1242 if (ctype_isvoid(ctr->info)) {
@@ -1196,8 +1307,7 @@ void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1196 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm); 1307 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1197 if (tv) { 1308 if (tv) {
1198 if (tvisfunc(tv)) { 1309 if (tvisfunc(tv)) {
1199 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME; 1310 crec_tailcall(J, rd, tv);
1200 rd->nres = -1; /* Pending tailcall. */
1201 return; 1311 return;
1202 } 1312 }
1203 } else if (mm == MM_new) { 1313 } else if (mm == MM_new) {
@@ -1238,7 +1348,7 @@ static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1238 for (i = 0; i < 2; i++) { 1348 for (i = 0; i < 2; i++) {
1239 IRType st = tref_type(sp[i]); 1349 IRType st = tref_type(sp[i]);
1240 if (st == IRT_NUM || st == IRT_FLOAT) 1350 if (st == IRT_NUM || st == IRT_FLOAT)
1241 sp[i] = emitconv(sp[i], dt, st, IRCONV_TRUNC|IRCONV_ANY); 1351 sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1242 else if (!(st == IRT_I64 || st == IRT_U64)) 1352 else if (!(st == IRT_I64 || st == IRT_U64))
1243 sp[i] = emitconv(sp[i], dt, IRT_INT, 1353 sp[i] = emitconv(sp[i], dt, IRT_INT,
1244 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT); 1354 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
@@ -1307,15 +1417,14 @@ static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1307 CTypeID id; 1417 CTypeID id;
1308#if LJ_64 1418#if LJ_64
1309 if (t == IRT_NUM || t == IRT_FLOAT) 1419 if (t == IRT_NUM || t == IRT_FLOAT)
1310 tr = emitconv(tr, IRT_INTP, t, IRCONV_TRUNC|IRCONV_ANY); 1420 tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1311 else if (!(t == IRT_I64 || t == IRT_U64)) 1421 else if (!(t == IRT_I64 || t == IRT_U64))
1312 tr = emitconv(tr, IRT_INTP, IRT_INT, 1422 tr = emitconv(tr, IRT_INTP, IRT_INT,
1313 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT); 1423 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1314#else 1424#else
1315 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) { 1425 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1316 tr = emitconv(tr, IRT_INTP, t, 1426 tr = emitconv(tr, IRT_INTP, t,
1317 (t == IRT_NUM || t == IRT_FLOAT) ? 1427 (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1318 IRCONV_TRUNC|IRCONV_ANY : 0);
1319 } 1428 }
1320#endif 1429#endif
1321 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz)); 1430 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
@@ -1347,8 +1456,7 @@ static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1347 } 1456 }
1348 if (tv) { 1457 if (tv) {
1349 if (tvisfunc(tv)) { 1458 if (tvisfunc(tv)) {
1350 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME; 1459 crec_tailcall(J, rd, tv);
1351 rd->nres = -1; /* Pending tailcall. */
1352 return 0; 1460 return 0;
1353 } /* NYI: non-function metamethods. */ 1461 } /* NYI: non-function metamethods. */
1354 } else if ((MMS)rd->data == MM_eq) { /* Fallback cdata pointer comparison. */ 1462 } else if ((MMS)rd->data == MM_eq) { /* Fallback cdata pointer comparison. */
@@ -1464,8 +1572,7 @@ void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1464 !irt_isguard(J->guardemit)) { 1572 !irt_isguard(J->guardemit)) {
1465 const BCIns *pc = frame_contpc(J->L->base-1) - 1; 1573 const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1466 if (bc_op(*pc) <= BC_ISNEP) { 1574 if (bc_op(*pc) <= BC_ISNEP) {
1467 setframe_pc(&J2G(J)->tmptv, pc); 1575 J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1468 J2G(J)->tmptv.u32.lo = ((tref_istrue(tr) ^ bc_op(*pc)) & 1);
1469 J->postproc = LJ_POST_FIXCOMP; 1576 J->postproc = LJ_POST_FIXCOMP;
1470 } 1577 }
1471 } 1578 }
@@ -1654,7 +1761,140 @@ void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1654void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd) 1761void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1655{ 1762{
1656 argv2cdata(J, J->base[0], &rd->argv[0]); 1763 argv2cdata(J, J->base[0], &rd->argv[0]);
1657 crec_finalizer(J, J->base[0], &rd->argv[1]); 1764 if (!J->base[1])
1765 lj_trace_err(J, LJ_TRERR_BADTYPE);
1766 crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1767}
1768
1769/* -- 64 bit bit.* library functions -------------------------------------- */
1770
1771/* Determine bit operation type from argument type. */
1772static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1773{
1774 if (tviscdata(tv)) {
1775 CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1776 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1777 if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1778 CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1779 return CTID_UINT64; /* Use uint64_t, since it has the highest rank. */
1780 return CTID_INT64; /* Otherwise use int64_t. */
1781 }
1782 return 0; /* Use regular 32 bit ops. */
1783}
1784
1785void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1786{
1787 CTState *cts = ctype_ctsG(J2G(J));
1788 TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1789 J->base[0], &rd->argv[0]);
1790 if (!tref_isinteger(tr))
1791 tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1792 J->base[0] = tr;
1793}
1794
1795int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1796{
1797 CTState *cts = ctype_ctsG(J2G(J));
1798 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1799 if (id) {
1800 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1801 tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1802 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1803 return 1;
1804 }
1805 return 0;
1806}
1807
1808int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1809{
1810 CTState *cts = ctype_ctsG(J2G(J));
1811 CTypeID id = 0;
1812 MSize i;
1813 for (i = 0; J->base[i] != 0; i++) {
1814 CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1815 if (id < aid) id = aid; /* Determine highest type rank of all arguments. */
1816 }
1817 if (id) {
1818 CType *ct = ctype_get(cts, id);
1819 uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1820 TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
1821 for (i = 1; J->base[i] != 0; i++) {
1822 TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
1823 tr = emitir(ot, tr, tr2);
1824 }
1825 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1826 return 1;
1827 }
1828 return 0;
1829}
1830
1831int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1832{
1833 CTState *cts = ctype_ctsG(J2G(J));
1834 CTypeID id;
1835 TRef tsh = 0;
1836 if (J->base[0] && tref_iscdata(J->base[1])) {
1837 tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1838 J->base[1], &rd->argv[1]);
1839 if (!tref_isinteger(tsh))
1840 tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1841 J->base[1] = tsh;
1842 }
1843 id = crec_bit64_type(cts, &rd->argv[0]);
1844 if (id) {
1845 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1846 uint32_t op = rd->data;
1847 if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1848 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1849 !tref_isk(tsh))
1850 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
1851#ifdef LJ_TARGET_UNIFYROT
1852 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1853 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1854 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
1855 }
1856#endif
1857 tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1858 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1859 return 1;
1860 }
1861 return 0;
1862}
1863
1864TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1865{
1866 CTState *cts = ctype_ctsG(J2G(J));
1867 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1868 TRef tr, trsf = J->base[1];
1869 SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1870 int32_t n;
1871 if (trsf) {
1872 CTypeID id2 = 0;
1873 n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1874 if (id2)
1875 trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
1876 else
1877 trsf = lj_opt_narrow_tobit(J, trsf);
1878 emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n)); /* Specialize to n. */
1879 } else {
1880 n = id ? 16 : 8;
1881 }
1882 if (n < 0) { n = (int32_t)(~n+1u); sf |= STRFMT_F_UPPER; }
1883 if ((uint32_t)n > 254) n = 254;
1884 sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1885 if (id) {
1886 tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1887 if (n < 16)
1888 tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1889 lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1890 } else {
1891 tr = lj_opt_narrow_tobit(J, J->base[0]);
1892 if (n < 8)
1893 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1894 tr = emitconv(tr, IRT_U64, IRT_INT, 0); /* No sign-extension. */
1895 lj_needsplit(J);
1896 }
1897 return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1658} 1898}
1659 1899
1660/* -- Miscellaneous library functions ------------------------------------- */ 1900/* -- Miscellaneous library functions ------------------------------------- */
@@ -1678,6 +1918,30 @@ void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1678 } 1918 }
1679} 1919}
1680 1920
1921TRef lj_crecord_loadiu64(jit_State *J, TRef tr, cTValue *o)
1922{
1923 CTypeID id = argv2cdata(J, tr, o)->ctypeid;
1924 if (!(id == CTID_INT64 || id == CTID_UINT64))
1925 lj_trace_err(J, LJ_TRERR_BADTYPE);
1926 lj_needsplit(J);
1927 return emitir(IRT(IR_FLOAD, id == CTID_INT64 ? IRT_I64 : IRT_U64), tr,
1928 IRFL_CDATA_INT64);
1929}
1930
1931#if LJ_HASBUFFER
1932TRef lj_crecord_topcvoid(jit_State *J, TRef tr, cTValue *o)
1933{
1934 CTState *cts = ctype_ctsG(J2G(J));
1935 if (!tref_iscdata(tr)) lj_trace_err(J, LJ_TRERR_BADTYPE);
1936 return crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, o);
1937}
1938
1939TRef lj_crecord_topuint8(jit_State *J, TRef tr)
1940{
1941 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, CTID_P_UINT8), tr);
1942}
1943#endif
1944
1681#undef IR 1945#undef IR
1682#undef emitir 1946#undef emitir
1683#undef emitconv 1947#undef emitconv
diff --git a/src/lj_crecord.h b/src/lj_crecord.h
index e3ba70c1..32973cf4 100644
--- a/src/lj_crecord.h
+++ b/src/lj_crecord.h
@@ -25,7 +25,19 @@ LJ_FUNC void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd);
25LJ_FUNC void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd); 25LJ_FUNC void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd);
26LJ_FUNC void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd); 26LJ_FUNC void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd);
27LJ_FUNC void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd); 27LJ_FUNC void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd);
28
29LJ_FUNC void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd);
30LJ_FUNC int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd);
31LJ_FUNC int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd);
32LJ_FUNC int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd);
33LJ_FUNC TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr);
34
28LJ_FUNC void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd); 35LJ_FUNC void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd);
36LJ_FUNC TRef lj_crecord_loadiu64(jit_State *J, TRef tr, cTValue *o);
37#if LJ_HASBUFFER
38LJ_FUNC TRef lj_crecord_topcvoid(jit_State *J, TRef tr, cTValue *o);
39LJ_FUNC TRef lj_crecord_topuint8(jit_State *J, TRef tr);
40#endif
29#endif 41#endif
30 42
31#endif 43#endif
diff --git a/src/lj_ctype.c b/src/lj_ctype.c
index 872e8cda..8a4a55f8 100644
--- a/src/lj_ctype.c
+++ b/src/lj_ctype.c
@@ -11,8 +11,10 @@
11#include "lj_err.h" 11#include "lj_err.h"
12#include "lj_str.h" 12#include "lj_str.h"
13#include "lj_tab.h" 13#include "lj_tab.h"
14#include "lj_strfmt.h"
14#include "lj_ctype.h" 15#include "lj_ctype.h"
15#include "lj_ccallback.h" 16#include "lj_ccallback.h"
17#include "lj_buf.h"
16 18
17/* -- C type definitions -------------------------------------------------- */ 19/* -- C type definitions -------------------------------------------------- */
18 20
@@ -37,6 +39,8 @@
37 _("uint64_t", UINT64) \ 39 _("uint64_t", UINT64) \
38 _("intptr_t", INT_PSZ) \ 40 _("intptr_t", INT_PSZ) \
39 _("uintptr_t", UINT_PSZ) \ 41 _("uintptr_t", UINT_PSZ) \
42 /* From POSIX. */ \
43 _("ssize_t", INT_PSZ) \
40 /* End of typedef list. */ 44 /* End of typedef list. */
41 45
42/* Keywords (only the ones we actually care for). */ 46/* Keywords (only the ones we actually care for). */
@@ -149,7 +153,7 @@ CTypeID lj_ctype_new(CTState *cts, CType **ctp)
149{ 153{
150 CTypeID id = cts->top; 154 CTypeID id = cts->top;
151 CType *ct; 155 CType *ct;
152 lua_assert(cts->L); 156 lj_assertCTS(cts->L, "uninitialized cts->L");
153 if (LJ_UNLIKELY(id >= cts->sizetab)) { 157 if (LJ_UNLIKELY(id >= cts->sizetab)) {
154 if (id >= CTID_MAX) lj_err_msg(cts->L, LJ_ERR_TABOV); 158 if (id >= CTID_MAX) lj_err_msg(cts->L, LJ_ERR_TABOV);
155#ifdef LUAJIT_CTYPE_CHECK_ANCHOR 159#ifdef LUAJIT_CTYPE_CHECK_ANCHOR
@@ -178,7 +182,7 @@ CTypeID lj_ctype_intern(CTState *cts, CTInfo info, CTSize size)
178{ 182{
179 uint32_t h = ct_hashtype(info, size); 183 uint32_t h = ct_hashtype(info, size);
180 CTypeID id = cts->hash[h]; 184 CTypeID id = cts->hash[h];
181 lua_assert(cts->L); 185 lj_assertCTS(cts->L, "uninitialized cts->L");
182 while (id) { 186 while (id) {
183 CType *ct = ctype_get(cts, id); 187 CType *ct = ctype_get(cts, id);
184 if (ct->info == info && ct->size == size) 188 if (ct->info == info && ct->size == size)
@@ -306,9 +310,9 @@ CTSize lj_ctype_vlsize(CTState *cts, CType *ct, CTSize nelem)
306 } 310 }
307 ct = ctype_raw(cts, arrid); 311 ct = ctype_raw(cts, arrid);
308 } 312 }
309 lua_assert(ctype_isvlarray(ct->info)); /* Must be a VLA. */ 313 lj_assertCTS(ctype_isvlarray(ct->info), "VLA expected");
310 ct = ctype_rawchild(cts, ct); /* Get array element. */ 314 ct = ctype_rawchild(cts, ct); /* Get array element. */
311 lua_assert(ctype_hassize(ct->info)); 315 lj_assertCTS(ctype_hassize(ct->info), "bad VLA without size");
312 /* Calculate actual size of VLA and check for overflow. */ 316 /* Calculate actual size of VLA and check for overflow. */
313 xsz += (uint64_t)ct->size * nelem; 317 xsz += (uint64_t)ct->size * nelem;
314 return xsz < 0x80000000u ? (CTSize)xsz : CTSIZE_INVALID; 318 return xsz < 0x80000000u ? (CTSize)xsz : CTSIZE_INVALID;
@@ -331,7 +335,8 @@ CTInfo lj_ctype_info(CTState *cts, CTypeID id, CTSize *szp)
331 } else { 335 } else {
332 if (!(qual & CTFP_ALIGNED)) qual |= (info & CTF_ALIGN); 336 if (!(qual & CTFP_ALIGNED)) qual |= (info & CTF_ALIGN);
333 qual |= (info & ~(CTF_ALIGN|CTMASK_CID)); 337 qual |= (info & ~(CTF_ALIGN|CTMASK_CID));
334 lua_assert(ctype_hassize(info) || ctype_isfunc(info)); 338 lj_assertCTS(ctype_hassize(info) || ctype_isfunc(info),
339 "ctype without size");
335 *szp = ctype_isfunc(info) ? CTSIZE_INVALID : ct->size; 340 *szp = ctype_isfunc(info) ? CTSIZE_INVALID : ct->size;
336 break; 341 break;
337 } 342 }
@@ -544,7 +549,7 @@ static void ctype_repr(CTRepr *ctr, CTypeID id)
544 ctype_appc(ctr, ')'); 549 ctype_appc(ctr, ')');
545 break; 550 break;
546 default: 551 default:
547 lua_assert(0); 552 lj_assertG_(ctr->cts->g, 0, "bad ctype %08x", info);
548 break; 553 break;
549 } 554 }
550 ct = ctype_get(ctr->cts, ctype_cid(info)); 555 ct = ctype_get(ctr->cts, ctype_cid(info));
@@ -588,19 +593,18 @@ GCstr *lj_ctype_repr_int64(lua_State *L, uint64_t n, int isunsigned)
588/* Convert complex to string with 'i' or 'I' suffix. */ 593/* Convert complex to string with 'i' or 'I' suffix. */
589GCstr *lj_ctype_repr_complex(lua_State *L, void *sp, CTSize size) 594GCstr *lj_ctype_repr_complex(lua_State *L, void *sp, CTSize size)
590{ 595{
591 char buf[2*LJ_STR_NUMBUF+2+1]; 596 SBuf *sb = lj_buf_tmp_(L);
592 TValue re, im; 597 TValue re, im;
593 size_t len;
594 if (size == 2*sizeof(double)) { 598 if (size == 2*sizeof(double)) {
595 re.n = *(double *)sp; im.n = ((double *)sp)[1]; 599 re.n = *(double *)sp; im.n = ((double *)sp)[1];
596 } else { 600 } else {
597 re.n = (double)*(float *)sp; im.n = (double)((float *)sp)[1]; 601 re.n = (double)*(float *)sp; im.n = (double)((float *)sp)[1];
598 } 602 }
599 len = lj_str_bufnum(buf, &re); 603 lj_strfmt_putfnum(sb, STRFMT_G14, re.n);
600 if (!(im.u32.hi & 0x80000000u) || im.n != im.n) buf[len++] = '+'; 604 if (!(im.u32.hi & 0x80000000u) || im.n != im.n) lj_buf_putchar(sb, '+');
601 len += lj_str_bufnum(buf+len, &im); 605 lj_strfmt_putfnum(sb, STRFMT_G14, im.n);
602 buf[len] = buf[len-1] >= 'a' ? 'I' : 'i'; 606 lj_buf_putchar(sb, sb->w[-1] >= 'a' ? 'I' : 'i');
603 return lj_str_new(L, buf, len+1); 607 return lj_buf_str(L, sb);
604} 608}
605 609
606/* -- C type state -------------------------------------------------------- */ 610/* -- C type state -------------------------------------------------------- */
diff --git a/src/lj_ctype.h b/src/lj_ctype.h
index 0787d800..45e7234e 100644
--- a/src/lj_ctype.h
+++ b/src/lj_ctype.h
@@ -260,10 +260,16 @@ typedef struct CTState {
260 260
261#define CT_MEMALIGN 3 /* Alignment guaranteed by memory allocator. */ 261#define CT_MEMALIGN 3 /* Alignment guaranteed by memory allocator. */
262 262
263#ifdef LUA_USE_ASSERT
264#define lj_assertCTS(c, ...) (lj_assertG_(cts->g, (c), __VA_ARGS__))
265#else
266#define lj_assertCTS(c, ...) ((void)cts)
267#endif
268
263/* -- Predefined types ---------------------------------------------------- */ 269/* -- Predefined types ---------------------------------------------------- */
264 270
265/* Target-dependent types. */ 271/* Target-dependent types. */
266#if LJ_TARGET_PPC || LJ_TARGET_PPCSPE 272#if LJ_TARGET_PPC
267#define CTTYDEFP(_) \ 273#define CTTYDEFP(_) \
268 _(LINT32, 4, CT_NUM, CTF_LONG|CTALIGN(2)) 274 _(LINT32, 4, CT_NUM, CTF_LONG|CTALIGN(2))
269#else 275#else
@@ -292,6 +298,7 @@ typedef struct CTState {
292 _(P_VOID, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_VOID) \ 298 _(P_VOID, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_VOID) \
293 _(P_CVOID, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_CVOID) \ 299 _(P_CVOID, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_CVOID) \
294 _(P_CCHAR, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_CCHAR) \ 300 _(P_CCHAR, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_CCHAR) \
301 _(P_UINT8, CTSIZE_PTR, CT_PTR, CTALIGN_PTR|CTID_UINT8) \
295 _(A_CCHAR, -1, CT_ARRAY, CTF_CONST|CTALIGN(0)|CTID_CCHAR) \ 302 _(A_CCHAR, -1, CT_ARRAY, CTF_CONST|CTALIGN(0)|CTID_CCHAR) \
296 _(CTYPEID, 4, CT_ENUM, CTALIGN(2)|CTID_INT32) \ 303 _(CTYPEID, 4, CT_ENUM, CTALIGN(2)|CTID_INT32) \
297 CTTYDEFP(_) \ 304 CTTYDEFP(_) \
@@ -383,6 +390,16 @@ static LJ_AINLINE CTState *ctype_cts(lua_State *L)
383 return cts; 390 return cts;
384} 391}
385 392
393/* Load FFI library on-demand. */
394#define ctype_loadffi(L) \
395 do { \
396 if (!ctype_ctsG(G(L))) { \
397 ptrdiff_t oldtop = (char *)L->top - mref(L->stack, char); \
398 luaopen_ffi(L); \
399 L->top = (TValue *)(mref(L->stack, char) + oldtop); \
400 } \
401 } while (0)
402
386/* Save and restore state of C type table. */ 403/* Save and restore state of C type table. */
387#define LJ_CTYPE_SAVE(cts) CTState savects_ = *(cts) 404#define LJ_CTYPE_SAVE(cts) CTState savects_ = *(cts)
388#define LJ_CTYPE_RESTORE(cts) \ 405#define LJ_CTYPE_RESTORE(cts) \
@@ -392,7 +409,8 @@ static LJ_AINLINE CTState *ctype_cts(lua_State *L)
392/* Check C type ID for validity when assertions are enabled. */ 409/* Check C type ID for validity when assertions are enabled. */
393static LJ_AINLINE CTypeID ctype_check(CTState *cts, CTypeID id) 410static LJ_AINLINE CTypeID ctype_check(CTState *cts, CTypeID id)
394{ 411{
395 lua_assert(id > 0 && id < cts->top); UNUSED(cts); 412 UNUSED(cts);
413 lj_assertCTS(id > 0 && id < cts->top, "bad CTID %d", id);
396 return id; 414 return id;
397} 415}
398 416
@@ -408,8 +426,9 @@ static LJ_AINLINE CType *ctype_get(CTState *cts, CTypeID id)
408/* Get child C type. */ 426/* Get child C type. */
409static LJ_AINLINE CType *ctype_child(CTState *cts, CType *ct) 427static LJ_AINLINE CType *ctype_child(CTState *cts, CType *ct)
410{ 428{
411 lua_assert(!(ctype_isvoid(ct->info) || ctype_isstruct(ct->info) || 429 lj_assertCTS(!(ctype_isvoid(ct->info) || ctype_isstruct(ct->info) ||
412 ctype_isbitfield(ct->info))); /* These don't have children. */ 430 ctype_isbitfield(ct->info)),
431 "ctype %08x has no children", ct->info);
413 return ctype_get(cts, ctype_cid(ct->info)); 432 return ctype_get(cts, ctype_cid(ct->info));
414} 433}
415 434
diff --git a/src/lj_debug.c b/src/lj_debug.c
index e6a8be54..fa189b6e 100644
--- a/src/lj_debug.c
+++ b/src/lj_debug.c
@@ -9,12 +9,12 @@
9#include "lj_obj.h" 9#include "lj_obj.h"
10#include "lj_err.h" 10#include "lj_err.h"
11#include "lj_debug.h" 11#include "lj_debug.h"
12#include "lj_str.h" 12#include "lj_buf.h"
13#include "lj_tab.h" 13#include "lj_tab.h"
14#include "lj_state.h" 14#include "lj_state.h"
15#include "lj_frame.h" 15#include "lj_frame.h"
16#include "lj_bc.h" 16#include "lj_bc.h"
17#include "lj_vm.h" 17#include "lj_strfmt.h"
18#if LJ_HASJIT 18#if LJ_HASJIT
19#include "lj_jit.h" 19#include "lj_jit.h"
20#endif 20#endif
@@ -24,11 +24,11 @@
24/* Get frame corresponding to a level. */ 24/* Get frame corresponding to a level. */
25cTValue *lj_debug_frame(lua_State *L, int level, int *size) 25cTValue *lj_debug_frame(lua_State *L, int level, int *size)
26{ 26{
27 cTValue *frame, *nextframe, *bot = tvref(L->stack); 27 cTValue *frame, *nextframe, *bot = tvref(L->stack)+LJ_FR2;
28 /* Traverse frames backwards. */ 28 /* Traverse frames backwards. */
29 for (nextframe = frame = L->base-1; frame > bot; ) { 29 for (nextframe = frame = L->base-1; frame > bot; ) {
30 if (frame_gc(frame) == obj2gco(L)) 30 if (frame_gc(frame) == obj2gco(L))
31 level++; /* Skip dummy frames. See lj_meta_call(). */ 31 level++; /* Skip dummy frames. See lj_err_optype_call(). */
32 if (level-- == 0) { 32 if (level-- == 0) {
33 *size = (int)(nextframe - frame); 33 *size = (int)(nextframe - frame);
34 return frame; /* Level found. */ 34 return frame; /* Level found. */
@@ -55,7 +55,8 @@ static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
55 const BCIns *ins; 55 const BCIns *ins;
56 GCproto *pt; 56 GCproto *pt;
57 BCPos pos; 57 BCPos pos;
58 lua_assert(fn->c.gct == ~LJ_TFUNC || fn->c.gct == ~LJ_TTHREAD); 58 lj_assertL(fn->c.gct == ~LJ_TFUNC || fn->c.gct == ~LJ_TTHREAD,
59 "function or frame expected");
59 if (!isluafunc(fn)) { /* Cannot derive a PC for non-Lua functions. */ 60 if (!isluafunc(fn)) { /* Cannot derive a PC for non-Lua functions. */
60 return NO_BCPOS; 61 return NO_BCPOS;
61 } else if (nextframe == NULL) { /* Lua function on top. */ 62 } else if (nextframe == NULL) { /* Lua function on top. */
@@ -87,8 +88,7 @@ static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
87 if (frame_islua(f)) { 88 if (frame_islua(f)) {
88 f = frame_prevl(f); 89 f = frame_prevl(f);
89 } else { 90 } else {
90 if (frame_isc(f) || (LJ_HASFFI && frame_iscont(f) && 91 if (frame_isc(f) || (frame_iscont(f) && frame_iscont_fficb(f)))
91 (f-1)->u32.lo == LJ_CONT_FFI_CALLBACK))
92 cf = cframe_raw(cframe_prev(cf)); 92 cf = cframe_raw(cframe_prev(cf));
93 f = frame_prevd(f); 93 f = frame_prevd(f);
94 } 94 }
@@ -101,9 +101,12 @@ static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
101 pos = proto_bcpos(pt, ins) - 1; 101 pos = proto_bcpos(pt, ins) - 1;
102#if LJ_HASJIT 102#if LJ_HASJIT
103 if (pos > pt->sizebc) { /* Undo the effects of lj_trace_exit for JLOOP. */ 103 if (pos > pt->sizebc) { /* Undo the effects of lj_trace_exit for JLOOP. */
104 GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins)); 104 if (bc_isret(bc_op(ins[-1]))) {
105 lua_assert(bc_isret(bc_op(ins[-1]))); 105 GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
106 pos = proto_bcpos(pt, mref(T->startpc, const BCIns)); 106 pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
107 } else {
108 pos = NO_BCPOS; /* Punt in case of stack overflow for stitched trace. */
109 }
107 } 110 }
108#endif 111#endif
109 return pos; 112 return pos;
@@ -135,7 +138,7 @@ static BCLine debug_frameline(lua_State *L, GCfunc *fn, cTValue *nextframe)
135 BCPos pc = debug_framepc(L, fn, nextframe); 138 BCPos pc = debug_framepc(L, fn, nextframe);
136 if (pc != NO_BCPOS) { 139 if (pc != NO_BCPOS) {
137 GCproto *pt = funcproto(fn); 140 GCproto *pt = funcproto(fn);
138 lua_assert(pc <= pt->sizebc); 141 lj_assertL(pc <= pt->sizebc, "PC out of range");
139 return lj_debug_line(pt, pc); 142 return lj_debug_line(pt, pc);
140 } 143 }
141 return -1; 144 return -1;
@@ -143,38 +146,25 @@ static BCLine debug_frameline(lua_State *L, GCfunc *fn, cTValue *nextframe)
143 146
144/* -- Variable names ------------------------------------------------------ */ 147/* -- Variable names ------------------------------------------------------ */
145 148
146/* Read ULEB128 value. */
147static uint32_t debug_read_uleb128(const uint8_t **pp)
148{
149 const uint8_t *p = *pp;
150 uint32_t v = *p++;
151 if (LJ_UNLIKELY(v >= 0x80)) {
152 int sh = 0;
153 v &= 0x7f;
154 do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
155 }
156 *pp = p;
157 return v;
158}
159
160/* Get name of a local variable from slot number and PC. */ 149/* Get name of a local variable from slot number and PC. */
161static const char *debug_varname(const GCproto *pt, BCPos pc, BCReg slot) 150static const char *debug_varname(const GCproto *pt, BCPos pc, BCReg slot)
162{ 151{
163 const uint8_t *p = proto_varinfo(pt); 152 const char *p = (const char *)proto_varinfo(pt);
164 if (p) { 153 if (p) {
165 BCPos lastpc = 0; 154 BCPos lastpc = 0;
166 for (;;) { 155 for (;;) {
167 const char *name = (const char *)p; 156 const char *name = p;
168 uint32_t vn = *p++; 157 uint32_t vn = *(const uint8_t *)p;
169 BCPos startpc, endpc; 158 BCPos startpc, endpc;
170 if (vn < VARNAME__MAX) { 159 if (vn < VARNAME__MAX) {
171 if (vn == VARNAME_END) break; /* End of varinfo. */ 160 if (vn == VARNAME_END) break; /* End of varinfo. */
172 } else { 161 } else {
173 while (*p++) ; /* Skip over variable name string. */ 162 do { p++; } while (*(const uint8_t *)p); /* Skip over variable name. */
174 } 163 }
175 lastpc = startpc = lastpc + debug_read_uleb128(&p); 164 p++;
165 lastpc = startpc = lastpc + lj_buf_ruleb128(&p);
176 if (startpc > pc) break; 166 if (startpc > pc) break;
177 endpc = startpc + debug_read_uleb128(&p); 167 endpc = startpc + lj_buf_ruleb128(&p);
178 if (pc < endpc && slot-- == 0) { 168 if (pc < endpc && slot-- == 0) {
179 if (vn < VARNAME__MAX) { 169 if (vn < VARNAME__MAX) {
180#define VARNAMESTR(name, str) str "\0" 170#define VARNAMESTR(name, str) str "\0"
@@ -199,7 +189,7 @@ static TValue *debug_localname(lua_State *L, const lua_Debug *ar,
199 TValue *nextframe = size ? frame + size : NULL; 189 TValue *nextframe = size ? frame + size : NULL;
200 GCfunc *fn = frame_func(frame); 190 GCfunc *fn = frame_func(frame);
201 BCPos pc = debug_framepc(L, fn, nextframe); 191 BCPos pc = debug_framepc(L, fn, nextframe);
202 if (!nextframe) nextframe = L->top; 192 if (!nextframe) nextframe = L->top+LJ_FR2;
203 if ((int)slot1 < 0) { /* Negative slot number is for varargs. */ 193 if ((int)slot1 < 0) { /* Negative slot number is for varargs. */
204 if (pc != NO_BCPOS) { 194 if (pc != NO_BCPOS) {
205 GCproto *pt = funcproto(fn); 195 GCproto *pt = funcproto(fn);
@@ -209,7 +199,7 @@ static TValue *debug_localname(lua_State *L, const lua_Debug *ar,
209 nextframe = frame; 199 nextframe = frame;
210 frame = frame_prevd(frame); 200 frame = frame_prevd(frame);
211 } 201 }
212 if (frame + slot1 < nextframe) { 202 if (frame + slot1+LJ_FR2 < nextframe) {
213 *name = "(*vararg)"; 203 *name = "(*vararg)";
214 return frame+slot1; 204 return frame+slot1;
215 } 205 }
@@ -220,7 +210,7 @@ static TValue *debug_localname(lua_State *L, const lua_Debug *ar,
220 if (pc != NO_BCPOS && 210 if (pc != NO_BCPOS &&
221 (*name = debug_varname(funcproto(fn), pc, slot1-1)) != NULL) 211 (*name = debug_varname(funcproto(fn), pc, slot1-1)) != NULL)
222 ; 212 ;
223 else if (slot1 > 0 && frame + slot1 < nextframe) 213 else if (slot1 > 0 && frame + slot1+LJ_FR2 < nextframe)
224 *name = "(*temporary)"; 214 *name = "(*temporary)";
225 return frame+slot1; 215 return frame+slot1;
226} 216}
@@ -229,7 +219,7 @@ static TValue *debug_localname(lua_State *L, const lua_Debug *ar,
229const char *lj_debug_uvname(GCproto *pt, uint32_t idx) 219const char *lj_debug_uvname(GCproto *pt, uint32_t idx)
230{ 220{
231 const uint8_t *p = proto_uvinfo(pt); 221 const uint8_t *p = proto_uvinfo(pt);
232 lua_assert(idx < pt->sizeuv); 222 lj_assertX(idx < pt->sizeuv, "bad upvalue index");
233 if (!p) return ""; 223 if (!p) return "";
234 if (idx) while (*p++ || --idx) ; 224 if (idx) while (*p++ || --idx) ;
235 return (const char *)p; 225 return (const char *)p;
@@ -286,7 +276,7 @@ restart:
286 *name = strdata(gco2str(proto_kgc(pt, ~(ptrdiff_t)bc_c(ins)))); 276 *name = strdata(gco2str(proto_kgc(pt, ~(ptrdiff_t)bc_c(ins))));
287 if (ip > proto_bc(pt)) { 277 if (ip > proto_bc(pt)) {
288 BCIns insp = ip[-1]; 278 BCIns insp = ip[-1];
289 if (bc_op(insp) == BC_MOV && bc_a(insp) == ra+1 && 279 if (bc_op(insp) == BC_MOV && bc_a(insp) == ra+1+LJ_FR2 &&
290 bc_d(insp) == bc_b(ins)) 280 bc_d(insp) == bc_b(ins))
291 return "method"; 281 return "method";
292 } 282 }
@@ -303,12 +293,12 @@ restart:
303} 293}
304 294
305/* Deduce function name from caller of a frame. */ 295/* Deduce function name from caller of a frame. */
306const char *lj_debug_funcname(lua_State *L, TValue *frame, const char **name) 296const char *lj_debug_funcname(lua_State *L, cTValue *frame, const char **name)
307{ 297{
308 TValue *pframe; 298 cTValue *pframe;
309 GCfunc *fn; 299 GCfunc *fn;
310 BCPos pc; 300 BCPos pc;
311 if (frame <= tvref(L->stack)) 301 if (frame <= tvref(L->stack)+LJ_FR2)
312 return NULL; 302 return NULL;
313 if (frame_isvarg(frame)) 303 if (frame_isvarg(frame))
314 frame = frame_prevd(frame); 304 frame = frame_prevd(frame);
@@ -334,7 +324,7 @@ const char *lj_debug_funcname(lua_State *L, TValue *frame, const char **name)
334/* -- Source code locations ----------------------------------------------- */ 324/* -- Source code locations ----------------------------------------------- */
335 325
336/* Generate shortened source name. */ 326/* Generate shortened source name. */
337void lj_debug_shortname(char *out, GCstr *str) 327void lj_debug_shortname(char *out, GCstr *str, BCLine line)
338{ 328{
339 const char *src = strdata(str); 329 const char *src = strdata(str);
340 if (*src == '=') { 330 if (*src == '=') {
@@ -348,11 +338,11 @@ void lj_debug_shortname(char *out, GCstr *str)
348 *out++ = '.'; *out++ = '.'; *out++ = '.'; 338 *out++ = '.'; *out++ = '.'; *out++ = '.';
349 } 339 }
350 strcpy(out, src); 340 strcpy(out, src);
351 } else { /* Output [string "string"]. */ 341 } else { /* Output [string "string"] or [builtin:name]. */
352 size_t len; /* Length, up to first control char. */ 342 size_t len; /* Length, up to first control char. */
353 for (len = 0; len < LUA_IDSIZE-12; len++) 343 for (len = 0; len < LUA_IDSIZE-12; len++)
354 if (((const unsigned char *)src)[len] < ' ') break; 344 if (((const unsigned char *)src)[len] < ' ') break;
355 strcpy(out, "[string \""); out += 9; 345 strcpy(out, line == ~(BCLine)0 ? "[builtin:" : "[string \""); out += 9;
356 if (src[len] != '\0') { /* Must truncate? */ 346 if (src[len] != '\0') { /* Must truncate? */
357 if (len > LUA_IDSIZE-15) len = LUA_IDSIZE-15; 347 if (len > LUA_IDSIZE-15) len = LUA_IDSIZE-15;
358 strncpy(out, src, len); out += len; 348 strncpy(out, src, len); out += len;
@@ -360,7 +350,7 @@ void lj_debug_shortname(char *out, GCstr *str)
360 } else { 350 } else {
361 strcpy(out, src); out += len; 351 strcpy(out, src); out += len;
362 } 352 }
363 strcpy(out, "\"]"); 353 strcpy(out, line == ~(BCLine)0 ? "]" : "\"]");
364 } 354 }
365} 355}
366 356
@@ -373,14 +363,15 @@ void lj_debug_addloc(lua_State *L, const char *msg,
373 if (isluafunc(fn)) { 363 if (isluafunc(fn)) {
374 BCLine line = debug_frameline(L, fn, nextframe); 364 BCLine line = debug_frameline(L, fn, nextframe);
375 if (line >= 0) { 365 if (line >= 0) {
366 GCproto *pt = funcproto(fn);
376 char buf[LUA_IDSIZE]; 367 char buf[LUA_IDSIZE];
377 lj_debug_shortname(buf, proto_chunkname(funcproto(fn))); 368 lj_debug_shortname(buf, proto_chunkname(pt), pt->firstline);
378 lj_str_pushf(L, "%s:%d: %s", buf, line, msg); 369 lj_strfmt_pushf(L, "%s:%d: %s", buf, line, msg);
379 return; 370 return;
380 } 371 }
381 } 372 }
382 } 373 }
383 lj_str_pushf(L, "%s", msg); 374 lj_strfmt_pushf(L, "%s", msg);
384} 375}
385 376
386/* Push location string for a bytecode position to Lua stack. */ 377/* Push location string for a bytecode position to Lua stack. */
@@ -390,20 +381,22 @@ void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc)
390 const char *s = strdata(name); 381 const char *s = strdata(name);
391 MSize i, len = name->len; 382 MSize i, len = name->len;
392 BCLine line = lj_debug_line(pt, pc); 383 BCLine line = lj_debug_line(pt, pc);
393 if (*s == '@') { 384 if (pt->firstline == ~(BCLine)0) {
385 lj_strfmt_pushf(L, "builtin:%s", s);
386 } else if (*s == '@') {
394 s++; len--; 387 s++; len--;
395 for (i = len; i > 0; i--) 388 for (i = len; i > 0; i--)
396 if (s[i] == '/' || s[i] == '\\') { 389 if (s[i] == '/' || s[i] == '\\') {
397 s += i+1; 390 s += i+1;
398 break; 391 break;
399 } 392 }
400 lj_str_pushf(L, "%s:%d", s, line); 393 lj_strfmt_pushf(L, "%s:%d", s, line);
401 } else if (len > 40) { 394 } else if (len > 40) {
402 lj_str_pushf(L, "%p:%d", pt, line); 395 lj_strfmt_pushf(L, "%p:%d", pt, line);
403 } else if (*s == '=') { 396 } else if (*s == '=') {
404 lj_str_pushf(L, "%s:%d", s+1, line); 397 lj_strfmt_pushf(L, "%s:%d", s+1, line);
405 } else { 398 } else {
406 lj_str_pushf(L, "\"%s\":%d", s, line); 399 lj_strfmt_pushf(L, "\"%s\":%d", s, line);
407 } 400 }
408} 401}
409 402
@@ -451,13 +444,14 @@ int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, int ext)
451 } else { 444 } else {
452 uint32_t offset = (uint32_t)ar->i_ci & 0xffff; 445 uint32_t offset = (uint32_t)ar->i_ci & 0xffff;
453 uint32_t size = (uint32_t)ar->i_ci >> 16; 446 uint32_t size = (uint32_t)ar->i_ci >> 16;
454 lua_assert(offset != 0); 447 lj_assertL(offset != 0, "bad frame offset");
455 frame = tvref(L->stack) + offset; 448 frame = tvref(L->stack) + offset;
456 if (size) nextframe = frame + size; 449 if (size) nextframe = frame + size;
457 lua_assert(frame <= tvref(L->maxstack) && 450 lj_assertL(frame <= tvref(L->maxstack) &&
458 (!nextframe || nextframe <= tvref(L->maxstack))); 451 (!nextframe || nextframe <= tvref(L->maxstack)),
452 "broken frame chain");
459 fn = frame_func(frame); 453 fn = frame_func(frame);
460 lua_assert(fn->c.gct == ~LJ_TFUNC); 454 lj_assertL(fn->c.gct == ~LJ_TFUNC, "bad frame function");
461 } 455 }
462 for (; *what; what++) { 456 for (; *what; what++) {
463 if (*what == 'S') { 457 if (*what == 'S') {
@@ -466,7 +460,7 @@ int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, int ext)
466 BCLine firstline = pt->firstline; 460 BCLine firstline = pt->firstline;
467 GCstr *name = proto_chunkname(pt); 461 GCstr *name = proto_chunkname(pt);
468 ar->source = strdata(name); 462 ar->source = strdata(name);
469 lj_debug_shortname(ar->short_src, name); 463 lj_debug_shortname(ar->short_src, name, pt->firstline);
470 ar->linedefined = (int)firstline; 464 ar->linedefined = (int)firstline;
471 ar->lastlinedefined = (int)(firstline + pt->numline); 465 ar->lastlinedefined = (int)(firstline + pt->numline);
472 ar->what = (firstline || !pt->numline) ? "Lua" : "main"; 466 ar->what = (firstline || !pt->numline) ? "Lua" : "main";
@@ -556,6 +550,111 @@ LUA_API int lua_getstack(lua_State *L, int level, lua_Debug *ar)
556 } 550 }
557} 551}
558 552
553#if LJ_HASPROFILE
554/* Put the chunkname into a buffer. */
555static int debug_putchunkname(SBuf *sb, GCproto *pt, int pathstrip)
556{
557 GCstr *name = proto_chunkname(pt);
558 const char *p = strdata(name);
559 if (pt->firstline == ~(BCLine)0) {
560 lj_buf_putmem(sb, "[builtin:", 9);
561 lj_buf_putstr(sb, name);
562 lj_buf_putb(sb, ']');
563 return 0;
564 }
565 if (*p == '=' || *p == '@') {
566 MSize len = name->len-1;
567 p++;
568 if (pathstrip) {
569 int i;
570 for (i = len-1; i >= 0; i--)
571 if (p[i] == '/' || p[i] == '\\') {
572 len -= i+1;
573 p = p+i+1;
574 break;
575 }
576 }
577 lj_buf_putmem(sb, p, len);
578 } else {
579 lj_buf_putmem(sb, "[string]", 8);
580 }
581 return 1;
582}
583
584/* Put a compact stack dump into a buffer. */
585void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt, int depth)
586{
587 int level = 0, dir = 1, pathstrip = 1;
588 MSize lastlen = 0;
589 if (depth < 0) { level = ~depth; depth = dir = -1; } /* Reverse frames. */
590 while (level != depth) { /* Loop through all frame. */
591 int size;
592 cTValue *frame = lj_debug_frame(L, level, &size);
593 if (frame) {
594 cTValue *nextframe = size ? frame+size : NULL;
595 GCfunc *fn = frame_func(frame);
596 const uint8_t *p = (const uint8_t *)fmt;
597 int c;
598 while ((c = *p++)) {
599 switch (c) {
600 case 'p': /* Preserve full path. */
601 pathstrip = 0;
602 break;
603 case 'F': case 'f': { /* Dump function name. */
604 const char *name;
605 const char *what = lj_debug_funcname(L, frame, &name);
606 if (what) {
607 if (c == 'F' && isluafunc(fn)) { /* Dump module:name for 'F'. */
608 GCproto *pt = funcproto(fn);
609 if (pt->firstline != ~(BCLine)0) { /* Not a bytecode builtin. */
610 debug_putchunkname(sb, pt, pathstrip);
611 lj_buf_putb(sb, ':');
612 }
613 }
614 lj_buf_putmem(sb, name, (MSize)strlen(name));
615 break;
616 } /* else: can't derive a name, dump module:line. */
617 }
618 /* fallthrough */
619 case 'l': /* Dump module:line. */
620 if (isluafunc(fn)) {
621 GCproto *pt = funcproto(fn);
622 if (debug_putchunkname(sb, pt, pathstrip)) {
623 /* Regular Lua function. */
624 BCLine line = c == 'l' ? debug_frameline(L, fn, nextframe) :
625 pt->firstline;
626 lj_buf_putb(sb, ':');
627 lj_strfmt_putint(sb, line >= 0 ? line : pt->firstline);
628 }
629 } else if (isffunc(fn)) { /* Dump numbered builtins. */
630 lj_buf_putmem(sb, "[builtin#", 9);
631 lj_strfmt_putint(sb, fn->c.ffid);
632 lj_buf_putb(sb, ']');
633 } else { /* Dump C function address. */
634 lj_buf_putb(sb, '@');
635 lj_strfmt_putptr(sb, fn->c.f);
636 }
637 break;
638 case 'Z': /* Zap trailing separator. */
639 lastlen = sbuflen(sb);
640 break;
641 default:
642 lj_buf_putb(sb, c);
643 break;
644 }
645 }
646 } else if (dir == 1) {
647 break;
648 } else {
649 level -= size; /* Reverse frame order: quickly skip missing level. */
650 }
651 level += dir;
652 }
653 if (lastlen)
654 sb->w = sb->b + lastlen; /* Zap trailing separator. */
655}
656#endif
657
559/* Number of frames for the leading and trailing part of a traceback. */ 658/* Number of frames for the leading and trailing part of a traceback. */
560#define TRACEBACK_LEVELS1 12 659#define TRACEBACK_LEVELS1 12
561#define TRACEBACK_LEVELS2 10 660#define TRACEBACK_LEVELS2 10
diff --git a/src/lj_debug.h b/src/lj_debug.h
index 71f7c6a2..33ac29bc 100644
--- a/src/lj_debug.h
+++ b/src/lj_debug.h
@@ -33,14 +33,18 @@ LJ_FUNC const char *lj_debug_uvnamev(cTValue *o, uint32_t idx, TValue **tvp,
33 GCobj **op); 33 GCobj **op);
34LJ_FUNC const char *lj_debug_slotname(GCproto *pt, const BCIns *pc, 34LJ_FUNC const char *lj_debug_slotname(GCproto *pt, const BCIns *pc,
35 BCReg slot, const char **name); 35 BCReg slot, const char **name);
36LJ_FUNC const char *lj_debug_funcname(lua_State *L, TValue *frame, 36LJ_FUNC const char *lj_debug_funcname(lua_State *L, cTValue *frame,
37 const char **name); 37 const char **name);
38LJ_FUNC void lj_debug_shortname(char *out, GCstr *str); 38LJ_FUNC void lj_debug_shortname(char *out, GCstr *str, BCLine line);
39LJ_FUNC void lj_debug_addloc(lua_State *L, const char *msg, 39LJ_FUNC void lj_debug_addloc(lua_State *L, const char *msg,
40 cTValue *frame, cTValue *nextframe); 40 cTValue *frame, cTValue *nextframe);
41LJ_FUNC void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc); 41LJ_FUNC void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc);
42LJ_FUNC int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, 42LJ_FUNC int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar,
43 int ext); 43 int ext);
44#if LJ_HASPROFILE
45LJ_FUNC void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt,
46 int depth);
47#endif
44 48
45/* Fixed internal variable names. */ 49/* Fixed internal variable names. */
46#define VARNAMEDEF(_) \ 50#define VARNAMEDEF(_) \
diff --git a/src/lj_def.h b/src/lj_def.h
index a5e317cd..88bc6336 100644
--- a/src/lj_def.h
+++ b/src/lj_def.h
@@ -46,10 +46,14 @@ typedef unsigned int uintptr_t;
46#include <stdlib.h> 46#include <stdlib.h>
47 47
48/* Various VM limits. */ 48/* Various VM limits. */
49#define LJ_MAX_MEM 0x7fffff00 /* Max. total memory allocation. */ 49#define LJ_MAX_MEM32 0x7fffff00 /* Max. 32 bit memory allocation. */
50#define LJ_MAX_MEM64 ((uint64_t)1<<47) /* Max. 64 bit memory allocation. */
51/* Max. total memory allocation. */
52#define LJ_MAX_MEM (LJ_GC64 ? LJ_MAX_MEM64 : LJ_MAX_MEM32)
50#define LJ_MAX_ALLOC LJ_MAX_MEM /* Max. individual allocation length. */ 53#define LJ_MAX_ALLOC LJ_MAX_MEM /* Max. individual allocation length. */
51#define LJ_MAX_STR LJ_MAX_MEM /* Max. string length. */ 54#define LJ_MAX_STR LJ_MAX_MEM32 /* Max. string length. */
52#define LJ_MAX_UDATA LJ_MAX_MEM /* Max. userdata length. */ 55#define LJ_MAX_BUF LJ_MAX_MEM32 /* Max. buffer length. */
56#define LJ_MAX_UDATA LJ_MAX_MEM32 /* Max. userdata length. */
53 57
54#define LJ_MAX_STRTAB (1<<26) /* Max. string table size. */ 58#define LJ_MAX_STRTAB (1<<26) /* Max. string table size. */
55#define LJ_MAX_HBITS 26 /* Max. hash bits. */ 59#define LJ_MAX_HBITS 26 /* Max. hash bits. */
@@ -57,7 +61,7 @@ typedef unsigned int uintptr_t;
57#define LJ_MAX_ASIZE ((1<<(LJ_MAX_ABITS-1))+1) /* Max. array part size. */ 61#define LJ_MAX_ASIZE ((1<<(LJ_MAX_ABITS-1))+1) /* Max. array part size. */
58#define LJ_MAX_COLOSIZE 16 /* Max. elems for colocated array. */ 62#define LJ_MAX_COLOSIZE 16 /* Max. elems for colocated array. */
59 63
60#define LJ_MAX_LINE LJ_MAX_MEM /* Max. source code line number. */ 64#define LJ_MAX_LINE LJ_MAX_MEM32 /* Max. source code line number. */
61#define LJ_MAX_XLEVEL 200 /* Max. syntactic nesting level. */ 65#define LJ_MAX_XLEVEL 200 /* Max. syntactic nesting level. */
62#define LJ_MAX_BCINS (1<<26) /* Max. # of bytecode instructions. */ 66#define LJ_MAX_BCINS (1<<26) /* Max. # of bytecode instructions. */
63#define LJ_MAX_SLOTS 250 /* Max. # of slots in a Lua func. */ 67#define LJ_MAX_SLOTS 250 /* Max. # of slots in a Lua func. */
@@ -65,7 +69,7 @@ typedef unsigned int uintptr_t;
65#define LJ_MAX_UPVAL 60 /* Max. # of upvalues. */ 69#define LJ_MAX_UPVAL 60 /* Max. # of upvalues. */
66 70
67#define LJ_MAX_IDXCHAIN 100 /* __index/__newindex chain limit. */ 71#define LJ_MAX_IDXCHAIN 100 /* __index/__newindex chain limit. */
68#define LJ_STACK_EXTRA 5 /* Extra stack space (metamethods). */ 72#define LJ_STACK_EXTRA (5+2*LJ_FR2) /* Extra stack space (metamethods). */
69 73
70#define LJ_NUM_CBPAGE 1 /* Number of FFI callback pages. */ 74#define LJ_NUM_CBPAGE 1 /* Number of FFI callback pages. */
71 75
@@ -76,7 +80,6 @@ typedef unsigned int uintptr_t;
76#define LJ_MIN_SBUF 32 /* Min. string buffer length. */ 80#define LJ_MIN_SBUF 32 /* Min. string buffer length. */
77#define LJ_MIN_VECSZ 8 /* Min. size for growable vectors. */ 81#define LJ_MIN_VECSZ 8 /* Min. size for growable vectors. */
78#define LJ_MIN_IRSZ 32 /* Min. size for growable IR. */ 82#define LJ_MIN_IRSZ 32 /* Min. size for growable IR. */
79#define LJ_MIN_K64SZ 16 /* Min. size for chained K64Array. */
80 83
81/* JIT compiler limits. */ 84/* JIT compiler limits. */
82#define LJ_MAX_JSLOTS 250 /* Max. # of stack slots for a trace. */ 85#define LJ_MAX_JSLOTS 250 /* Max. # of stack slots for a trace. */
@@ -91,6 +94,9 @@ typedef unsigned int uintptr_t;
91#define U64x(hi, lo) (((uint64_t)0x##hi << 32) + (uint64_t)0x##lo) 94#define U64x(hi, lo) (((uint64_t)0x##hi << 32) + (uint64_t)0x##lo)
92#define i32ptr(p) ((int32_t)(intptr_t)(void *)(p)) 95#define i32ptr(p) ((int32_t)(intptr_t)(void *)(p))
93#define u32ptr(p) ((uint32_t)(intptr_t)(void *)(p)) 96#define u32ptr(p) ((uint32_t)(intptr_t)(void *)(p))
97#define i64ptr(p) ((int64_t)(intptr_t)(void *)(p))
98#define u64ptr(p) ((uint64_t)(intptr_t)(void *)(p))
99#define igcptr(p) (LJ_GC64 ? i64ptr(p) : i32ptr(p))
94 100
95#define checki8(x) ((x) == (int32_t)(int8_t)(x)) 101#define checki8(x) ((x) == (int32_t)(int8_t)(x))
96#define checku8(x) ((x) == (int32_t)(uint8_t)(x)) 102#define checku8(x) ((x) == (int32_t)(uint8_t)(x))
@@ -98,7 +104,10 @@ typedef unsigned int uintptr_t;
98#define checku16(x) ((x) == (int32_t)(uint16_t)(x)) 104#define checku16(x) ((x) == (int32_t)(uint16_t)(x))
99#define checki32(x) ((x) == (int32_t)(x)) 105#define checki32(x) ((x) == (int32_t)(x))
100#define checku32(x) ((x) == (uint32_t)(x)) 106#define checku32(x) ((x) == (uint32_t)(x))
107#define checkptr31(x) (((uint64_t)(uintptr_t)(x) >> 31) == 0)
101#define checkptr32(x) ((uintptr_t)(x) == (uint32_t)(uintptr_t)(x)) 108#define checkptr32(x) ((uintptr_t)(x) == (uint32_t)(uintptr_t)(x))
109#define checkptr47(x) (((uint64_t)(uintptr_t)(x) >> 47) == 0)
110#define checkptrGC(x) (LJ_GC64 ? checkptr47((x)) : LJ_64 ? checkptr31((x)) :1)
102 111
103/* Every half-decent C compiler transforms this into a rotate instruction. */ 112/* Every half-decent C compiler transforms this into a rotate instruction. */
104#define lj_rol(x, n) (((x)<<(n)) | ((x)>>(-(int)(n)&(8*sizeof(x)-1)))) 113#define lj_rol(x, n) (((x)<<(n)) | ((x)>>(-(int)(n)&(8*sizeof(x)-1))))
@@ -111,7 +120,7 @@ typedef uintptr_t BloomFilter;
111#define bloomset(b, x) ((b) |= bloombit((x))) 120#define bloomset(b, x) ((b) |= bloombit((x)))
112#define bloomtest(b, x) ((b) & bloombit((x))) 121#define bloomtest(b, x) ((b) & bloombit((x)))
113 122
114#if defined(__GNUC__) || defined(__psp2__) 123#if defined(__GNUC__) || defined(__clang__) || defined(__psp2__)
115 124
116#define LJ_NORET __attribute__((noreturn)) 125#define LJ_NORET __attribute__((noreturn))
117#define LJ_ALIGN(n) __attribute__((aligned(n))) 126#define LJ_ALIGN(n) __attribute__((aligned(n)))
@@ -173,7 +182,7 @@ static LJ_AINLINE uint64_t lj_bswap64(uint64_t x)
173{ 182{
174 return ((uint64_t)lj_bswap((uint32_t)x)<<32) | lj_bswap((uint32_t)(x>>32)); 183 return ((uint64_t)lj_bswap((uint32_t)x)<<32) | lj_bswap((uint32_t)(x>>32));
175} 184}
176#elif (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) 185#elif (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __clang__
177static LJ_AINLINE uint32_t lj_bswap(uint32_t x) 186static LJ_AINLINE uint32_t lj_bswap(uint32_t x)
178{ 187{
179 return (uint32_t)__builtin_bswap32((int32_t)x); 188 return (uint32_t)__builtin_bswap32((int32_t)x);
@@ -329,14 +338,28 @@ static LJ_AINLINE uint32_t lj_getu32(const void *v)
329#define LJ_FUNCA_NORET LJ_FUNCA LJ_NORET 338#define LJ_FUNCA_NORET LJ_FUNCA LJ_NORET
330#define LJ_ASMF_NORET LJ_ASMF LJ_NORET 339#define LJ_ASMF_NORET LJ_ASMF LJ_NORET
331 340
332/* Runtime assertions. */ 341/* Internal assertions. */
333#ifdef lua_assert 342#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
334#define check_exp(c, e) (lua_assert(c), (e)) 343#define lj_assert_check(g, c, ...) \
335#define api_check(l, e) lua_assert(e) 344 ((c) ? (void)0 : \
345 (lj_assert_fail((g), __FILE__, __LINE__, __func__, __VA_ARGS__), 0))
346#define lj_checkapi(c, ...) lj_assert_check(G(L), (c), __VA_ARGS__)
336#else 347#else
337#define lua_assert(c) ((void)0) 348#define lj_checkapi(c, ...) ((void)L)
349#endif
350
351#ifdef LUA_USE_ASSERT
352#define lj_assertG_(g, c, ...) lj_assert_check((g), (c), __VA_ARGS__)
353#define lj_assertG(c, ...) lj_assert_check(g, (c), __VA_ARGS__)
354#define lj_assertL(c, ...) lj_assert_check(G(L), (c), __VA_ARGS__)
355#define lj_assertX(c, ...) lj_assert_check(NULL, (c), __VA_ARGS__)
356#define check_exp(c, e) (lj_assertX((c), #c), (e))
357#else
358#define lj_assertG_(g, c, ...) ((void)0)
359#define lj_assertG(c, ...) ((void)g)
360#define lj_assertL(c, ...) ((void)L)
361#define lj_assertX(c, ...) ((void)0)
338#define check_exp(c, e) (e) 362#define check_exp(c, e) (e)
339#define api_check luai_apicheck
340#endif 363#endif
341 364
342/* Static assertions. */ 365/* Static assertions. */
@@ -350,4 +373,9 @@ static LJ_AINLINE uint32_t lj_getu32(const void *v)
350 extern void LJ_ASSERT_NAME(__LINE__)(int STATIC_ASSERTION_FAILED[(cond)?1:-1]) 373 extern void LJ_ASSERT_NAME(__LINE__)(int STATIC_ASSERTION_FAILED[(cond)?1:-1])
351#endif 374#endif
352 375
376/* PRNG state. Need this here, details in lj_prng.h. */
377typedef struct PRNGState {
378 uint64_t u[4];
379} PRNGState;
380
353#endif 381#endif
diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index 8009d289..7b66be7e 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -8,6 +8,7 @@
8 8
9#include "lj_obj.h" 9#include "lj_obj.h"
10#include "lj_err.h" 10#include "lj_err.h"
11#include "lj_buf.h"
11#include "lj_func.h" 12#include "lj_func.h"
12#include "lj_str.h" 13#include "lj_str.h"
13#include "lj_tab.h" 14#include "lj_tab.h"
@@ -17,6 +18,7 @@
17#include "lj_frame.h" 18#include "lj_frame.h"
18#include "lj_bc.h" 19#include "lj_bc.h"
19#include "lj_ff.h" 20#include "lj_ff.h"
21#include "lj_strfmt.h"
20#if LJ_HASJIT 22#if LJ_HASJIT
21#include "lj_jit.h" 23#include "lj_jit.h"
22#endif 24#endif
@@ -25,6 +27,9 @@
25#endif 27#endif
26#include "lj_trace.h" 28#include "lj_trace.h"
27#include "lj_dispatch.h" 29#include "lj_dispatch.h"
30#if LJ_HASPROFILE
31#include "lj_profile.h"
32#endif
28#include "lj_vm.h" 33#include "lj_vm.h"
29#include "luajit.h" 34#include "luajit.h"
30 35
@@ -37,6 +42,12 @@ LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
37#include <math.h> 42#include <math.h>
38LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L, 43LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
39 lua_State *co); 44 lua_State *co);
45#if !LJ_HASJIT
46#define lj_dispatch_stitch lj_dispatch_ins
47#endif
48#if !LJ_HASPROFILE
49#define lj_dispatch_profile lj_dispatch_ins
50#endif
40 51
41#define GOTFUNC(name) (ASMFunction)name, 52#define GOTFUNC(name) (ASMFunction)name,
42static const ASMFunction dispatch_got[] = { 53static const ASMFunction dispatch_got[] = {
@@ -57,6 +68,8 @@ void lj_dispatch_init(GG_State *GG)
57 /* The JIT engine is off by default. luaopen_jit() turns it on. */ 68 /* The JIT engine is off by default. luaopen_jit() turns it on. */
58 disp[BC_FORL] = disp[BC_IFORL]; 69 disp[BC_FORL] = disp[BC_IFORL];
59 disp[BC_ITERL] = disp[BC_IITERL]; 70 disp[BC_ITERL] = disp[BC_IITERL];
71 /* Workaround for stable v2.1 bytecode. TODO: Replace with BC_IITERN. */
72 disp[BC_ITERN] = &lj_vm_IITERN;
60 disp[BC_LOOP] = disp[BC_ILOOP]; 73 disp[BC_LOOP] = disp[BC_ILOOP];
61 disp[BC_FUNCF] = disp[BC_IFUNCF]; 74 disp[BC_FUNCF] = disp[BC_IFUNCF];
62 disp[BC_FUNCV] = disp[BC_IFUNCV]; 75 disp[BC_FUNCV] = disp[BC_IFUNCV];
@@ -64,7 +77,7 @@ void lj_dispatch_init(GG_State *GG)
64 for (i = 0; i < GG_NUM_ASMFF; i++) 77 for (i = 0; i < GG_NUM_ASMFF; i++)
65 GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0); 78 GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
66#if LJ_TARGET_MIPS 79#if LJ_TARGET_MIPS
67 memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4); 80 memcpy(GG->got, dispatch_got, LJ_GOT__MAX*sizeof(ASMFunction *));
68#endif 81#endif
69} 82}
70 83
@@ -82,11 +95,12 @@ void lj_dispatch_init_hotcount(global_State *g)
82#endif 95#endif
83 96
84/* Internal dispatch mode bits. */ 97/* Internal dispatch mode bits. */
85#define DISPMODE_JIT 0x01 /* JIT compiler on. */ 98#define DISPMODE_CALL 0x01 /* Override call dispatch. */
86#define DISPMODE_REC 0x02 /* Recording active. */ 99#define DISPMODE_RET 0x02 /* Override return dispatch. */
87#define DISPMODE_INS 0x04 /* Override instruction dispatch. */ 100#define DISPMODE_INS 0x04 /* Override instruction dispatch. */
88#define DISPMODE_CALL 0x08 /* Override call dispatch. */ 101#define DISPMODE_JIT 0x10 /* JIT compiler on. */
89#define DISPMODE_RET 0x10 /* Override return dispatch. */ 102#define DISPMODE_REC 0x20 /* Recording active. */
103#define DISPMODE_PROF 0x40 /* Profiling active. */
90 104
91/* Update dispatch table depending on various flags. */ 105/* Update dispatch table depending on various flags. */
92void lj_dispatch_update(global_State *g) 106void lj_dispatch_update(global_State *g)
@@ -98,24 +112,29 @@ void lj_dispatch_update(global_State *g)
98 mode |= G2J(g)->state != LJ_TRACE_IDLE ? 112 mode |= G2J(g)->state != LJ_TRACE_IDLE ?
99 (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0; 113 (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
100#endif 114#endif
115#if LJ_HASPROFILE
116 mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
117#endif
101 mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0; 118 mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
102 mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0; 119 mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
103 mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0; 120 mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
104 if (oldmode != mode) { /* Mode changed? */ 121 if (oldmode != mode) { /* Mode changed? */
105 ASMFunction *disp = G2GG(g)->dispatch; 122 ASMFunction *disp = G2GG(g)->dispatch;
106 ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv; 123 ASMFunction f_forl, f_iterl, f_itern, f_loop, f_funcf, f_funcv;
107 g->dispatchmode = mode; 124 g->dispatchmode = mode;
108 125
109 /* Hotcount if JIT is on, but not while recording. */ 126 /* Hotcount if JIT is on, but not while recording. */
110 if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) { 127 if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
111 f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]); 128 f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
112 f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]); 129 f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
130 f_itern = makeasmfunc(lj_bc_ofs[BC_ITERN]);
113 f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]); 131 f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
114 f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]); 132 f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
115 f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]); 133 f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
116 } else { /* Otherwise use the non-hotcounting instructions. */ 134 } else { /* Otherwise use the non-hotcounting instructions. */
117 f_forl = disp[GG_LEN_DDISP+BC_IFORL]; 135 f_forl = disp[GG_LEN_DDISP+BC_IFORL];
118 f_iterl = disp[GG_LEN_DDISP+BC_IITERL]; 136 f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
137 f_itern = &lj_vm_IITERN;
119 f_loop = disp[GG_LEN_DDISP+BC_ILOOP]; 138 f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
120 f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]); 139 f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
121 f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]); 140 f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
@@ -123,12 +142,13 @@ void lj_dispatch_update(global_State *g)
123 /* Init static counting instruction dispatch first (may be copied below). */ 142 /* Init static counting instruction dispatch first (may be copied below). */
124 disp[GG_LEN_DDISP+BC_FORL] = f_forl; 143 disp[GG_LEN_DDISP+BC_FORL] = f_forl;
125 disp[GG_LEN_DDISP+BC_ITERL] = f_iterl; 144 disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
145 disp[GG_LEN_DDISP+BC_ITERN] = f_itern;
126 disp[GG_LEN_DDISP+BC_LOOP] = f_loop; 146 disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
127 147
128 /* Set dynamic instruction dispatch. */ 148 /* Set dynamic instruction dispatch. */
129 if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) { 149 if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
130 /* Need to update the whole table. */ 150 /* Need to update the whole table. */
131 if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { /* No ins dispatch? */ 151 if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */
132 /* Copy static dispatch table to dynamic dispatch table. */ 152 /* Copy static dispatch table to dynamic dispatch table. */
133 memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction)); 153 memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
134 /* Overwrite with dynamic return dispatch. */ 154 /* Overwrite with dynamic return dispatch. */
@@ -140,15 +160,17 @@ void lj_dispatch_update(global_State *g)
140 } 160 }
141 } else { 161 } else {
142 /* The recording dispatch also checks for hooks. */ 162 /* The recording dispatch also checks for hooks. */
143 ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook; 163 ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
164 (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
144 uint32_t i; 165 uint32_t i;
145 for (i = 0; i < GG_LEN_SDISP; i++) 166 for (i = 0; i < GG_LEN_SDISP; i++)
146 disp[i] = f; 167 disp[i] = f;
147 } 168 }
148 } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { 169 } else if (!(mode & DISPMODE_INS)) {
149 /* Otherwise set dynamic counting ins. */ 170 /* Otherwise set dynamic counting ins. */
150 disp[BC_FORL] = f_forl; 171 disp[BC_FORL] = f_forl;
151 disp[BC_ITERL] = f_iterl; 172 disp[BC_ITERL] = f_iterl;
173 disp[BC_ITERN] = f_itern;
152 disp[BC_LOOP] = f_loop; 174 disp[BC_LOOP] = f_loop;
153 /* Set dynamic return dispatch. */ 175 /* Set dynamic return dispatch. */
154 if ((mode & DISPMODE_RET)) { 176 if ((mode & DISPMODE_RET)) {
@@ -236,22 +258,15 @@ int luaJIT_setmode(lua_State *L, int idx, int mode)
236 } else { 258 } else {
237 if (!(mode & LUAJIT_MODE_ON)) 259 if (!(mode & LUAJIT_MODE_ON))
238 G2J(g)->flags &= ~(uint32_t)JIT_F_ON; 260 G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
239#if LJ_TARGET_X86ORX64
240 else if ((G2J(g)->flags & JIT_F_SSE2))
241 G2J(g)->flags |= (uint32_t)JIT_F_ON;
242 else
243 return 0; /* Don't turn on JIT compiler without SSE2 support. */
244#else
245 else 261 else
246 G2J(g)->flags |= (uint32_t)JIT_F_ON; 262 G2J(g)->flags |= (uint32_t)JIT_F_ON;
247#endif
248 lj_dispatch_update(g); 263 lj_dispatch_update(g);
249 } 264 }
250 break; 265 break;
251 case LUAJIT_MODE_FUNC: 266 case LUAJIT_MODE_FUNC:
252 case LUAJIT_MODE_ALLFUNC: 267 case LUAJIT_MODE_ALLFUNC:
253 case LUAJIT_MODE_ALLSUBFUNC: { 268 case LUAJIT_MODE_ALLSUBFUNC: {
254 cTValue *tv = idx == 0 ? frame_prev(L->base-1) : 269 cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :
255 idx > 0 ? L->base + (idx-1) : L->top + idx; 270 idx > 0 ? L->base + (idx-1) : L->top + idx;
256 GCproto *pt; 271 GCproto *pt;
257 if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn)) 272 if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
@@ -286,7 +301,7 @@ int luaJIT_setmode(lua_State *L, int idx, int mode)
286 if (idx != 0) { 301 if (idx != 0) {
287 cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx; 302 cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
288 if (tvislightud(tv)) 303 if (tvislightud(tv))
289 g->wrapf = (lua_CFunction)lightudV(tv); 304 g->wrapf = (lua_CFunction)lightudV(g, tv);
290 else 305 else
291 return 0; /* Failed. */ 306 return 0; /* Failed. */
292 } else { 307 } else {
@@ -352,10 +367,19 @@ static void callhook(lua_State *L, int event, BCLine line)
352 /* Top frame, nextframe = NULL. */ 367 /* Top frame, nextframe = NULL. */
353 ar.i_ci = (int)((L->base-1) - tvref(L->stack)); 368 ar.i_ci = (int)((L->base-1) - tvref(L->stack));
354 lj_state_checkstack(L, 1+LUA_MINSTACK); 369 lj_state_checkstack(L, 1+LUA_MINSTACK);
370#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
371 lj_profile_hook_enter(g);
372#else
355 hook_enter(g); 373 hook_enter(g);
374#endif
356 hookf(L, &ar); 375 hookf(L, &ar);
357 lua_assert(hook_active(g)); 376 lj_assertG(hook_active(g), "active hook flag removed");
377 setgcref(g->cur_L, obj2gco(L));
378#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
379 lj_profile_hook_leave(g);
380#else
358 hook_leave(g); 381 hook_leave(g);
382#endif
359 } 383 }
360} 384}
361 385
@@ -368,7 +392,7 @@ static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
368 if (bc_op(ins) == BC_UCLO) 392 if (bc_op(ins) == BC_UCLO)
369 ins = pc[bc_j(ins)]; 393 ins = pc[bc_j(ins)];
370 switch (bc_op(ins)) { 394 switch (bc_op(ins)) {
371 case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1; 395 case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
372 case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1; 396 case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
373 case BC_TSETM: return bc_a(ins) + nres-1; 397 case BC_TSETM: return bc_a(ins) + nres-1;
374 default: return pt->framesize; 398 default: return pt->framesize;
@@ -397,7 +421,8 @@ void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
397#endif 421#endif
398 J->L = L; 422 J->L = L;
399 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */ 423 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
400 lua_assert(L->top - L->base == delta); 424 lj_assertG(L->top - L->base == delta,
425 "unbalanced stack after tracing of instruction");
401 } 426 }
402 } 427 }
403#endif 428#endif
@@ -457,7 +482,8 @@ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
457#endif 482#endif
458 pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1); 483 pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
459 lj_trace_hot(J, pc); 484 lj_trace_hot(J, pc);
460 lua_assert(L->top - L->base == delta); 485 lj_assertG(L->top - L->base == delta,
486 "unbalanced stack after hot call");
461 goto out; 487 goto out;
462 } else if (J->state != LJ_TRACE_IDLE && 488 } else if (J->state != LJ_TRACE_IDLE &&
463 !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) { 489 !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
@@ -466,7 +492,8 @@ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
466#endif 492#endif
467 /* Record the FUNC* bytecodes, too. */ 493 /* Record the FUNC* bytecodes, too. */
468 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */ 494 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
469 lua_assert(L->top - L->base == delta); 495 lj_assertG(L->top - L->base == delta,
496 "unbalanced stack after hot instruction");
470 } 497 }
471#endif 498#endif
472 if ((g->hookmask & LUA_MASKCALL)) { 499 if ((g->hookmask & LUA_MASKCALL)) {
@@ -492,3 +519,41 @@ out:
492 return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */ 519 return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
493} 520}
494 521
522#if LJ_HASJIT
523/* Stitch a new trace. */
524void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
525{
526 ERRNO_SAVE
527 lua_State *L = J->L;
528 void *cf = cframe_raw(L->cframe);
529 const BCIns *oldpc = cframe_pc(cf);
530 setcframe_pc(cf, pc);
531 /* Before dispatch, have to bias PC by 1. */
532 L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
533 lj_trace_stitch(J, pc-1); /* Point to the CALL instruction. */
534 setcframe_pc(cf, oldpc);
535 ERRNO_RESTORE
536}
537#endif
538
539#if LJ_HASPROFILE
540/* Profile dispatch. */
541void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
542{
543 ERRNO_SAVE
544 GCfunc *fn = curr_func(L);
545 GCproto *pt = funcproto(fn);
546 void *cf = cframe_raw(L->cframe);
547 const BCIns *oldpc = cframe_pc(cf);
548 global_State *g;
549 setcframe_pc(cf, pc);
550 L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
551 lj_profile_interpreter(L);
552 setcframe_pc(cf, oldpc);
553 g = G(L);
554 setgcref(g->cur_L, obj2gco(L));
555 setvmstate(g, INTERP);
556 ERRNO_RESTORE
557}
558#endif
559
diff --git a/src/lj_dispatch.h b/src/lj_dispatch.h
index b26c6b94..774bc4dc 100644
--- a/src/lj_dispatch.h
+++ b/src/lj_dispatch.h
@@ -14,8 +14,24 @@
14 14
15#if LJ_TARGET_MIPS 15#if LJ_TARGET_MIPS
16/* Need our own global offset table for the dreaded MIPS calling conventions. */ 16/* Need our own global offset table for the dreaded MIPS calling conventions. */
17
18#ifndef _LJ_VM_H
19LJ_ASMF int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b);
20#endif
21
22#if LJ_SOFTFP
23#ifndef _LJ_IRCALL_H
24extern double __adddf3(double a, double b);
25extern double __subdf3(double a, double b);
26extern double __muldf3(double a, double b);
27extern double __divdf3(double a, double b);
28#endif
29#define SFGOTDEF(_) _(sqrt) _(__adddf3) _(__subdf3) _(__muldf3) _(__divdf3)
30#else
31#define SFGOTDEF(_)
32#endif
17#if LJ_HASJIT 33#if LJ_HASJIT
18#define JITGOTDEF(_) _(lj_trace_exit) _(lj_trace_hot) 34#define JITGOTDEF(_) _(lj_err_trace) _(lj_trace_exit) _(lj_trace_hot)
19#else 35#else
20#define JITGOTDEF(_) 36#define JITGOTDEF(_)
21#endif 37#endif
@@ -28,16 +44,19 @@
28#define GOTDEF(_) \ 44#define GOTDEF(_) \
29 _(floor) _(ceil) _(trunc) _(log) _(log10) _(exp) _(sin) _(cos) _(tan) \ 45 _(floor) _(ceil) _(trunc) _(log) _(log10) _(exp) _(sin) _(cos) _(tan) \
30 _(asin) _(acos) _(atan) _(sinh) _(cosh) _(tanh) _(frexp) _(modf) _(atan2) \ 46 _(asin) _(acos) _(atan) _(sinh) _(cosh) _(tanh) _(frexp) _(modf) _(atan2) \
31 _(pow) _(fmod) _(ldexp) \ 47 _(pow) _(fmod) _(ldexp) _(lj_vm_modi) \
32 _(lj_dispatch_call) _(lj_dispatch_ins) _(lj_err_throw) _(lj_err_run) \ 48 _(lj_dispatch_call) _(lj_dispatch_ins) _(lj_dispatch_stitch) \
49 _(lj_dispatch_profile) _(lj_err_throw) \
33 _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \ 50 _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \
34 _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \ 51 _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \
35 _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \ 52 _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \
36 _(lj_meta_for) _(lj_meta_len) _(lj_meta_tget) _(lj_meta_tset) \ 53 _(lj_meta_for) _(lj_meta_istype) _(lj_meta_len) _(lj_meta_tget) \
37 _(lj_state_growstack) _(lj_str_fromnum) _(lj_str_fromnumber) _(lj_str_new) \ 54 _(lj_meta_tset) _(lj_state_growstack) _(lj_strfmt_number) \
38 _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) _(lj_tab_new) \ 55 _(lj_str_new) _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) \
39 _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \ 56 _(lj_tab_new) _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
40 JITGOTDEF(_) FFIGOTDEF(_) 57 _(lj_tab_setinth) _(lj_buf_putstr_reverse) _(lj_buf_putstr_lower) \
58 _(lj_buf_putstr_upper) _(lj_buf_tostr) \
59 JITGOTDEF(_) FFIGOTDEF(_) SFGOTDEF(_)
41 60
42enum { 61enum {
43#define GOTENUM(name) LJ_GOT_##name, 62#define GOTENUM(name) LJ_GOT_##name,
@@ -60,7 +79,7 @@ typedef uint16_t HotCount;
60#define HOTCOUNT_CALL 1 79#define HOTCOUNT_CALL 1
61 80
62/* This solves a circular dependency problem -- bump as needed. Sigh. */ 81/* This solves a circular dependency problem -- bump as needed. Sigh. */
63#define GG_NUM_ASMFF 62 82#define GG_NUM_ASMFF 57
64 83
65#define GG_LEN_DDISP (BC__MAX + GG_NUM_ASMFF) 84#define GG_LEN_DDISP (BC__MAX + GG_NUM_ASMFF)
66#define GG_LEN_SDISP BC_FUNCF 85#define GG_LEN_SDISP BC_FUNCF
@@ -70,7 +89,7 @@ typedef uint16_t HotCount;
70typedef struct GG_State { 89typedef struct GG_State {
71 lua_State L; /* Main thread. */ 90 lua_State L; /* Main thread. */
72 global_State g; /* Global state. */ 91 global_State g; /* Global state. */
73#if LJ_TARGET_ARM 92#if LJ_TARGET_ARM && !LJ_TARGET_NX
74 /* Make g reachable via K12 encoded DISPATCH-relative addressing. */ 93 /* Make g reachable via K12 encoded DISPATCH-relative addressing. */
75 uint8_t align1[(16-sizeof(global_State))&15]; 94 uint8_t align1[(16-sizeof(global_State))&15];
76#endif 95#endif
@@ -80,7 +99,7 @@ typedef struct GG_State {
80#if LJ_HASJIT 99#if LJ_HASJIT
81 jit_State J; /* JIT state. */ 100 jit_State J; /* JIT state. */
82 HotCount hotcount[HOTCOUNT_SIZE]; /* Hot counters. */ 101 HotCount hotcount[HOTCOUNT_SIZE]; /* Hot counters. */
83#if LJ_TARGET_ARM 102#if LJ_TARGET_ARM && !LJ_TARGET_NX
84 /* Ditto for J. */ 103 /* Ditto for J. */
85 uint8_t align2[(16-sizeof(jit_State)-sizeof(HotCount)*HOTCOUNT_SIZE)&15]; 104 uint8_t align2[(16-sizeof(jit_State)-sizeof(HotCount)*HOTCOUNT_SIZE)&15];
86#endif 105#endif
@@ -96,6 +115,7 @@ typedef struct GG_State {
96#define J2G(J) (&J2GG(J)->g) 115#define J2G(J) (&J2GG(J)->g)
97#define G2J(gl) (&G2GG(gl)->J) 116#define G2J(gl) (&G2GG(gl)->J)
98#define L2J(L) (&L2GG(L)->J) 117#define L2J(L) (&L2GG(L)->J)
118#define GG_G2J (GG_OFS(J) - GG_OFS(g))
99#define GG_G2DISP (GG_OFS(dispatch) - GG_OFS(g)) 119#define GG_G2DISP (GG_OFS(dispatch) - GG_OFS(g))
100#define GG_DISP2G (GG_OFS(g) - GG_OFS(dispatch)) 120#define GG_DISP2G (GG_OFS(g) - GG_OFS(dispatch))
101#define GG_DISP2J (GG_OFS(J) - GG_OFS(dispatch)) 121#define GG_DISP2J (GG_OFS(J) - GG_OFS(dispatch))
@@ -117,7 +137,12 @@ LJ_FUNC void lj_dispatch_update(global_State *g);
117/* Instruction dispatch callback for hooks or when recording. */ 137/* Instruction dispatch callback for hooks or when recording. */
118LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc); 138LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc);
119LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns*pc); 139LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns*pc);
120LJ_FUNCA void LJ_FASTCALL lj_dispatch_return(lua_State *L, const BCIns *pc); 140#if LJ_HASJIT
141LJ_FUNCA void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc);
142#endif
143#if LJ_HASPROFILE
144LJ_FUNCA void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc);
145#endif
121 146
122#if LJ_HASFFI && !defined(_BUILDVM_H) 147#if LJ_HASFFI && !defined(_BUILDVM_H)
123/* Save/restore errno and GetLastError() around hooks, exits and recording. */ 148/* Save/restore errno and GetLastError() around hooks, exits and recording. */
diff --git a/src/lj_emit_arm.h b/src/lj_emit_arm.h
index d38c8a38..5dcea839 100644
--- a/src/lj_emit_arm.h
+++ b/src/lj_emit_arm.h
@@ -81,7 +81,8 @@ static void emit_m(ASMState *as, ARMIns ai, Reg rm)
81 81
82static void emit_lsox(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs) 82static void emit_lsox(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs)
83{ 83{
84 lua_assert(ofs >= -255 && ofs <= 255); 84 lj_assertA(ofs >= -255 && ofs <= 255,
85 "load/store offset %d out of range", ofs);
85 if (ofs < 0) ofs = -ofs; else ai |= ARMI_LS_U; 86 if (ofs < 0) ofs = -ofs; else ai |= ARMI_LS_U;
86 *--as->mcp = ai | ARMI_LS_P | ARMI_LSX_I | ARMF_D(rd) | ARMF_N(rn) | 87 *--as->mcp = ai | ARMI_LS_P | ARMI_LSX_I | ARMF_D(rd) | ARMF_N(rn) |
87 ((ofs & 0xf0) << 4) | (ofs & 0x0f); 88 ((ofs & 0xf0) << 4) | (ofs & 0x0f);
@@ -89,7 +90,8 @@ static void emit_lsox(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs)
89 90
90static void emit_lso(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs) 91static void emit_lso(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs)
91{ 92{
92 lua_assert(ofs >= -4095 && ofs <= 4095); 93 lj_assertA(ofs >= -4095 && ofs <= 4095,
94 "load/store offset %d out of range", ofs);
93 /* Combine LDR/STR pairs to LDRD/STRD. */ 95 /* Combine LDR/STR pairs to LDRD/STRD. */
94 if (*as->mcp == (ai|ARMI_LS_P|ARMI_LS_U|ARMF_D(rd^1)|ARMF_N(rn)|(ofs^4)) && 96 if (*as->mcp == (ai|ARMI_LS_P|ARMI_LS_U|ARMF_D(rd^1)|ARMF_N(rn)|(ofs^4)) &&
95 (ai & ~(ARMI_LDR^ARMI_STR)) == ARMI_STR && rd != rn && 97 (ai & ~(ARMI_LDR^ARMI_STR)) == ARMI_STR && rd != rn &&
@@ -106,7 +108,8 @@ static void emit_lso(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs)
106#if !LJ_SOFTFP 108#if !LJ_SOFTFP
107static void emit_vlso(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs) 109static void emit_vlso(ASMState *as, ARMIns ai, Reg rd, Reg rn, int32_t ofs)
108{ 110{
109 lua_assert(ofs >= -1020 && ofs <= 1020 && (ofs&3) == 0); 111 lj_assertA(ofs >= -1020 && ofs <= 1020 && (ofs&3) == 0,
112 "load/store offset %d out of range", ofs);
110 if (ofs < 0) ofs = -ofs; else ai |= ARMI_LS_U; 113 if (ofs < 0) ofs = -ofs; else ai |= ARMI_LS_U;
111 *--as->mcp = ai | ARMI_LS_P | ARMF_D(rd & 15) | ARMF_N(rn) | (ofs >> 2); 114 *--as->mcp = ai | ARMI_LS_P | ARMF_D(rd & 15) | ARMF_N(rn) | (ofs >> 2);
112} 115}
@@ -124,7 +127,7 @@ static int emit_kdelta1(ASMState *as, Reg d, int32_t i)
124 while (work) { 127 while (work) {
125 Reg r = rset_picktop(work); 128 Reg r = rset_picktop(work);
126 IRRef ref = regcost_ref(as->cost[r]); 129 IRRef ref = regcost_ref(as->cost[r]);
127 lua_assert(r != d); 130 lj_assertA(r != d, "dest reg not free");
128 if (emit_canremat(ref)) { 131 if (emit_canremat(ref)) {
129 int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i); 132 int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i);
130 uint32_t k = emit_isk12(ARMI_ADD, delta); 133 uint32_t k = emit_isk12(ARMI_ADD, delta);
@@ -142,13 +145,13 @@ static int emit_kdelta1(ASMState *as, Reg d, int32_t i)
142} 145}
143 146
144/* Try to find a two step delta relative to another constant. */ 147/* Try to find a two step delta relative to another constant. */
145static int emit_kdelta2(ASMState *as, Reg d, int32_t i) 148static int emit_kdelta2(ASMState *as, Reg rd, int32_t i)
146{ 149{
147 RegSet work = ~as->freeset & RSET_GPR; 150 RegSet work = ~as->freeset & RSET_GPR;
148 while (work) { 151 while (work) {
149 Reg r = rset_picktop(work); 152 Reg r = rset_picktop(work);
150 IRRef ref = regcost_ref(as->cost[r]); 153 IRRef ref = regcost_ref(as->cost[r]);
151 lua_assert(r != d); 154 lj_assertA(r != rd, "dest reg %d not free", rd);
152 if (emit_canremat(ref)) { 155 if (emit_canremat(ref)) {
153 int32_t other = ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i; 156 int32_t other = ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i;
154 if (other) { 157 if (other) {
@@ -159,8 +162,8 @@ static int emit_kdelta2(ASMState *as, Reg d, int32_t i)
159 k2 = emit_isk12(0, delta & (255 << sh)); 162 k2 = emit_isk12(0, delta & (255 << sh));
160 k = emit_isk12(0, delta & ~(255 << sh)); 163 k = emit_isk12(0, delta & ~(255 << sh));
161 if (k) { 164 if (k) {
162 emit_dn(as, ARMI_ADD^k2^inv, d, d); 165 emit_dn(as, ARMI_ADD^k2^inv, rd, rd);
163 emit_dn(as, ARMI_ADD^k^inv, d, r); 166 emit_dn(as, ARMI_ADD^k^inv, rd, r);
164 return 1; 167 return 1;
165 } 168 }
166 } 169 }
@@ -171,23 +174,24 @@ static int emit_kdelta2(ASMState *as, Reg d, int32_t i)
171} 174}
172 175
173/* Load a 32 bit constant into a GPR. */ 176/* Load a 32 bit constant into a GPR. */
174static void emit_loadi(ASMState *as, Reg r, int32_t i) 177static void emit_loadi(ASMState *as, Reg rd, int32_t i)
175{ 178{
176 uint32_t k = emit_isk12(ARMI_MOV, i); 179 uint32_t k = emit_isk12(ARMI_MOV, i);
177 lua_assert(rset_test(as->freeset, r) || r == RID_TMP); 180 lj_assertA(rset_test(as->freeset, rd) || rd == RID_TMP,
181 "dest reg %d not free", rd);
178 if (k) { 182 if (k) {
179 /* Standard K12 constant. */ 183 /* Standard K12 constant. */
180 emit_d(as, ARMI_MOV^k, r); 184 emit_d(as, ARMI_MOV^k, rd);
181 } else if ((as->flags & JIT_F_ARMV6T2) && (uint32_t)i < 0x00010000u) { 185 } else if ((as->flags & JIT_F_ARMV6T2) && (uint32_t)i < 0x00010000u) {
182 /* 16 bit loword constant for ARMv6T2. */ 186 /* 16 bit loword constant for ARMv6T2. */
183 emit_d(as, ARMI_MOVW|(i & 0x0fff)|((i & 0xf000)<<4), r); 187 emit_d(as, ARMI_MOVW|(i & 0x0fff)|((i & 0xf000)<<4), rd);
184 } else if (emit_kdelta1(as, r, i)) { 188 } else if (emit_kdelta1(as, rd, i)) {
185 /* One step delta relative to another constant. */ 189 /* One step delta relative to another constant. */
186 } else if ((as->flags & JIT_F_ARMV6T2)) { 190 } else if ((as->flags & JIT_F_ARMV6T2)) {
187 /* 32 bit hiword/loword constant for ARMv6T2. */ 191 /* 32 bit hiword/loword constant for ARMv6T2. */
188 emit_d(as, ARMI_MOVT|((i>>16) & 0x0fff)|(((i>>16) & 0xf000)<<4), r); 192 emit_d(as, ARMI_MOVT|((i>>16) & 0x0fff)|(((i>>16) & 0xf000)<<4), rd);
189 emit_d(as, ARMI_MOVW|(i & 0x0fff)|((i & 0xf000)<<4), r); 193 emit_d(as, ARMI_MOVW|(i & 0x0fff)|((i & 0xf000)<<4), rd);
190 } else if (emit_kdelta2(as, r, i)) { 194 } else if (emit_kdelta2(as, rd, i)) {
191 /* Two step delta relative to another constant. */ 195 /* Two step delta relative to another constant. */
192 } else { 196 } else {
193 /* Otherwise construct the constant with up to 4 instructions. */ 197 /* Otherwise construct the constant with up to 4 instructions. */
@@ -197,17 +201,17 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
197 int32_t m = i & (255 << sh); 201 int32_t m = i & (255 << sh);
198 i &= ~(255 << sh); 202 i &= ~(255 << sh);
199 if (i == 0) { 203 if (i == 0) {
200 emit_d(as, ARMI_MOV ^ emit_isk12(0, m), r); 204 emit_d(as, ARMI_MOV ^ emit_isk12(0, m), rd);
201 break; 205 break;
202 } 206 }
203 emit_dn(as, ARMI_ORR ^ emit_isk12(0, m), r, r); 207 emit_dn(as, ARMI_ORR ^ emit_isk12(0, m), rd, rd);
204 } 208 }
205 } 209 }
206} 210}
207 211
208#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr))) 212#define emit_loada(as, rd, addr) emit_loadi(as, (rd), i32ptr((addr)))
209 213
210static Reg ra_allock(ASMState *as, int32_t k, RegSet allow); 214static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
211 215
212/* Get/set from constant pointer. */ 216/* Get/set from constant pointer. */
213static void emit_lsptr(ASMState *as, ARMIns ai, Reg r, void *p) 217static void emit_lsptr(ASMState *as, ARMIns ai, Reg r, void *p)
@@ -219,8 +223,9 @@ static void emit_lsptr(ASMState *as, ARMIns ai, Reg r, void *p)
219 223
220#if !LJ_SOFTFP 224#if !LJ_SOFTFP
221/* Load a number constant into an FPR. */ 225/* Load a number constant into an FPR. */
222static void emit_loadn(ASMState *as, Reg r, cTValue *tv) 226static void emit_loadk64(ASMState *as, Reg r, IRIns *ir)
223{ 227{
228 cTValue *tv = ir_knum(ir);
224 int32_t i; 229 int32_t i;
225 if ((as->flags & JIT_F_VFPV3) && !tv->u32.lo) { 230 if ((as->flags & JIT_F_VFPV3) && !tv->u32.lo) {
226 uint32_t hi = tv->u32.hi; 231 uint32_t hi = tv->u32.hi;
@@ -260,7 +265,7 @@ static void emit_branch(ASMState *as, ARMIns ai, MCode *target)
260{ 265{
261 MCode *p = as->mcp; 266 MCode *p = as->mcp;
262 ptrdiff_t delta = (target - p) - 1; 267 ptrdiff_t delta = (target - p) - 1;
263 lua_assert(((delta + 0x00800000) >> 24) == 0); 268 lj_assertA(((delta + 0x00800000) >> 24) == 0, "branch target out of range");
264 *--p = ai | ((uint32_t)delta & 0x00ffffffu); 269 *--p = ai | ((uint32_t)delta & 0x00ffffffu);
265 as->mcp = p; 270 as->mcp = p;
266} 271}
@@ -288,7 +293,7 @@ static void emit_call(ASMState *as, void *target)
288static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src) 293static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
289{ 294{
290#if LJ_SOFTFP 295#if LJ_SOFTFP
291 lua_assert(!irt_isnum(ir->t)); UNUSED(ir); 296 lj_assertA(!irt_isnum(ir->t), "unexpected FP op"); UNUSED(ir);
292#else 297#else
293 if (dst >= RID_MAX_GPR) { 298 if (dst >= RID_MAX_GPR) {
294 emit_dm(as, irt_isnum(ir->t) ? ARMI_VMOV_D : ARMI_VMOV_S, 299 emit_dm(as, irt_isnum(ir->t) ? ARMI_VMOV_D : ARMI_VMOV_S,
@@ -308,30 +313,30 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
308 emit_dm(as, ARMI_MOV, dst, src); 313 emit_dm(as, ARMI_MOV, dst, src);
309} 314}
310 315
311/* Generic load of register from stack slot. */ 316/* Generic load of register with base and (small) offset address. */
312static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 317static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
313{ 318{
314#if LJ_SOFTFP 319#if LJ_SOFTFP
315 lua_assert(!irt_isnum(ir->t)); UNUSED(ir); 320 lj_assertA(!irt_isnum(ir->t), "unexpected FP op"); UNUSED(ir);
316#else 321#else
317 if (r >= RID_MAX_GPR) 322 if (r >= RID_MAX_GPR)
318 emit_vlso(as, irt_isnum(ir->t) ? ARMI_VLDR_D : ARMI_VLDR_S, r, RID_SP, ofs); 323 emit_vlso(as, irt_isnum(ir->t) ? ARMI_VLDR_D : ARMI_VLDR_S, r, base, ofs);
319 else 324 else
320#endif 325#endif
321 emit_lso(as, ARMI_LDR, r, RID_SP, ofs); 326 emit_lso(as, ARMI_LDR, r, base, ofs);
322} 327}
323 328
324/* Generic store of register to stack slot. */ 329/* Generic store of register with base and (small) offset address. */
325static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 330static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
326{ 331{
327#if LJ_SOFTFP 332#if LJ_SOFTFP
328 lua_assert(!irt_isnum(ir->t)); UNUSED(ir); 333 lj_assertA(!irt_isnum(ir->t), "unexpected FP op"); UNUSED(ir);
329#else 334#else
330 if (r >= RID_MAX_GPR) 335 if (r >= RID_MAX_GPR)
331 emit_vlso(as, irt_isnum(ir->t) ? ARMI_VSTR_D : ARMI_VSTR_S, r, RID_SP, ofs); 336 emit_vlso(as, irt_isnum(ir->t) ? ARMI_VSTR_D : ARMI_VSTR_S, r, base, ofs);
332 else 337 else
333#endif 338#endif
334 emit_lso(as, ARMI_STR, r, RID_SP, ofs); 339 emit_lso(as, ARMI_STR, r, base, ofs);
335} 340}
336 341
337/* Emit an arithmetic/logic operation with a constant operand. */ 342/* Emit an arithmetic/logic operation with a constant operand. */
diff --git a/src/lj_emit_arm64.h b/src/lj_emit_arm64.h
new file mode 100644
index 00000000..52d010b8
--- /dev/null
+++ b/src/lj_emit_arm64.h
@@ -0,0 +1,431 @@
1/*
2** ARM64 instruction emitter.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4**
5** Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com.
6** Sponsored by Cisco Systems, Inc.
7*/
8
9/* -- Constant encoding --------------------------------------------------- */
10
11static uint64_t get_k64val(ASMState *as, IRRef ref)
12{
13 IRIns *ir = IR(ref);
14 if (ir->o == IR_KINT64) {
15 return ir_kint64(ir)->u64;
16 } else if (ir->o == IR_KGC) {
17 return (uint64_t)ir_kgc(ir);
18 } else if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
19 return (uint64_t)ir_kptr(ir);
20 } else {
21 lj_assertA(ir->o == IR_KINT || ir->o == IR_KNULL,
22 "bad 64 bit const IR op %d", ir->o);
23 return ir->i; /* Sign-extended. */
24 }
25}
26
27/* Encode constant in K12 format for data processing instructions. */
28static uint32_t emit_isk12(int64_t n)
29{
30 uint64_t k = n < 0 ? ~(uint64_t)n+1u : (uint64_t)n;
31 uint32_t m = n < 0 ? 0x40000000 : 0;
32 if (k < 0x1000) {
33 return A64I_K12|m|A64F_U12(k);
34 } else if ((k & 0xfff000) == k) {
35 return A64I_K12|m|0x400000|A64F_U12(k>>12);
36 }
37 return 0;
38}
39
40#define emit_clz64(n) __builtin_clzll(n)
41#define emit_ctz64(n) __builtin_ctzll(n)
42
43/* Encode constant in K13 format for logical data processing instructions. */
44static uint32_t emit_isk13(uint64_t n, int is64)
45{
46 int inv = 0, w = 128, lz, tz;
47 if (n & 1) { n = ~n; w = 64; inv = 1; } /* Avoid wrap-around of ones. */
48 if (!n) return 0; /* Neither all-zero nor all-ones are allowed. */
49 do { /* Find the repeat width. */
50 if (is64 && (uint32_t)(n^(n>>32))) break;
51 n = (uint32_t)n;
52 if (!n) return 0; /* Ditto when passing n=0xffffffff and is64=0. */
53 w = 32; if ((n^(n>>16)) & 0xffff) break;
54 n = n & 0xffff; w = 16; if ((n^(n>>8)) & 0xff) break;
55 n = n & 0xff; w = 8; if ((n^(n>>4)) & 0xf) break;
56 n = n & 0xf; w = 4; if ((n^(n>>2)) & 0x3) break;
57 n = n & 0x3; w = 2;
58 } while (0);
59 lz = emit_clz64(n);
60 tz = emit_ctz64(n);
61 if ((int64_t)(n << lz) >> (lz+tz) != -1ll) return 0; /* Non-contiguous? */
62 if (inv)
63 return A64I_K13 | (((lz-w) & 127) << 16) | (((lz+tz-w-1) & 63) << 10);
64 else
65 return A64I_K13 | ((w-tz) << 16) | (((63-lz-tz-w-w) & 63) << 10);
66}
67
68static uint32_t emit_isfpk64(uint64_t n)
69{
70 uint64_t etop9 = ((n >> 54) & 0x1ff);
71 if ((n << 16) == 0 && (etop9 == 0x100 || etop9 == 0x0ff)) {
72 return (uint32_t)(((n >> 48) & 0x7f) | ((n >> 56) & 0x80));
73 }
74 return ~0u;
75}
76
77/* -- Emit basic instructions --------------------------------------------- */
78
79static void emit_dnma(ASMState *as, A64Ins ai, Reg rd, Reg rn, Reg rm, Reg ra)
80{
81 *--as->mcp = ai | A64F_D(rd) | A64F_N(rn) | A64F_M(rm) | A64F_A(ra);
82}
83
84static void emit_dnm(ASMState *as, A64Ins ai, Reg rd, Reg rn, Reg rm)
85{
86 *--as->mcp = ai | A64F_D(rd) | A64F_N(rn) | A64F_M(rm);
87}
88
89static void emit_dm(ASMState *as, A64Ins ai, Reg rd, Reg rm)
90{
91 *--as->mcp = ai | A64F_D(rd) | A64F_M(rm);
92}
93
94static void emit_dn(ASMState *as, A64Ins ai, Reg rd, Reg rn)
95{
96 *--as->mcp = ai | A64F_D(rd) | A64F_N(rn);
97}
98
99static void emit_nm(ASMState *as, A64Ins ai, Reg rn, Reg rm)
100{
101 *--as->mcp = ai | A64F_N(rn) | A64F_M(rm);
102}
103
104static void emit_d(ASMState *as, A64Ins ai, Reg rd)
105{
106 *--as->mcp = ai | A64F_D(rd);
107}
108
109static void emit_n(ASMState *as, A64Ins ai, Reg rn)
110{
111 *--as->mcp = ai | A64F_N(rn);
112}
113
114static int emit_checkofs(A64Ins ai, int64_t ofs)
115{
116 int scale = (ai >> 30) & 3;
117 if (ofs < 0 || (ofs & ((1<<scale)-1))) {
118 return (ofs >= -256 && ofs <= 255) ? -1 : 0;
119 } else {
120 return (ofs < (4096<<scale)) ? 1 : 0;
121 }
122}
123
124static void emit_lso(ASMState *as, A64Ins ai, Reg rd, Reg rn, int64_t ofs)
125{
126 int ot = emit_checkofs(ai, ofs), sc = (ai >> 30) & 3;
127 lj_assertA(ot, "load/store offset %d out of range", ofs);
128 /* Combine LDR/STR pairs to LDP/STP. */
129 if ((sc == 2 || sc == 3) &&
130 (!(ai & 0x400000) || rd != rn) &&
131 as->mcp != as->mcloop) {
132 uint32_t prev = *as->mcp & ~A64F_D(31);
133 int ofsm = ofs - (1<<sc), ofsp = ofs + (1<<sc);
134 A64Ins aip;
135 if (prev == (ai | A64F_N(rn) | A64F_U12(ofsm>>sc)) ||
136 prev == ((ai^A64I_LS_U) | A64F_N(rn) | A64F_S9(ofsm&0x1ff))) {
137 aip = (A64F_A(rd) | A64F_D(*as->mcp & 31));
138 } else if (prev == (ai | A64F_N(rn) | A64F_U12(ofsp>>sc)) ||
139 prev == ((ai^A64I_LS_U) | A64F_N(rn) | A64F_S9(ofsp&0x1ff))) {
140 aip = (A64F_D(rd) | A64F_A(*as->mcp & 31));
141 ofsm = ofs;
142 } else {
143 goto nopair;
144 }
145 if (ofsm >= (int)((unsigned int)-64<<sc) && ofsm <= (63<<sc)) {
146 *as->mcp = aip | A64F_N(rn) | (((ofsm >> sc) & 0x7f) << 15) |
147 (ai ^ ((ai == A64I_LDRx || ai == A64I_STRx) ? 0x50000000 : 0x90000000));
148 return;
149 }
150 }
151nopair:
152 if (ot == 1)
153 *--as->mcp = ai | A64F_D(rd) | A64F_N(rn) | A64F_U12(ofs >> sc);
154 else
155 *--as->mcp = (ai^A64I_LS_U) | A64F_D(rd) | A64F_N(rn) | A64F_S9(ofs & 0x1ff);
156}
157
158/* -- Emit loads/stores --------------------------------------------------- */
159
160/* Prefer rematerialization of BASE/L from global_State over spills. */
161#define emit_canremat(ref) ((ref) <= ASMREF_L)
162
163/* Try to find an N-step delta relative to other consts with N < lim. */
164static int emit_kdelta(ASMState *as, Reg rd, uint64_t k, int lim)
165{
166 RegSet work = (~as->freeset & RSET_GPR) | RID2RSET(RID_GL);
167 if (lim <= 1) return 0; /* Can't beat that. */
168 while (work) {
169 Reg r = rset_picktop(work);
170 IRRef ref = regcost_ref(as->cost[r]);
171 lj_assertA(r != rd, "dest reg %d not free", rd);
172 if (ref < REF_TRUE) {
173 uint64_t kx = ra_iskref(ref) ? (uint64_t)ra_krefk(as, ref) :
174 get_k64val(as, ref);
175 int64_t delta = (int64_t)(k - kx);
176 if (delta == 0) {
177 emit_dm(as, A64I_MOVx, rd, r);
178 return 1;
179 } else {
180 uint32_t k12 = emit_isk12(delta < 0 ? (int64_t)(~(uint64_t)delta+1u) : delta);
181 if (k12) {
182 emit_dn(as, (delta < 0 ? A64I_SUBx : A64I_ADDx)^k12, rd, r);
183 return 1;
184 }
185 /* Do other ops or multi-step deltas pay off? Probably not.
186 ** E.g. XOR rarely helps with pointer consts.
187 */
188 }
189 }
190 rset_clear(work, r);
191 }
192 return 0; /* Failed. */
193}
194
195static void emit_loadk(ASMState *as, Reg rd, uint64_t u64, int is64)
196{
197 int i, zeros = 0, ones = 0, neg;
198 if (!is64) u64 = (int64_t)(int32_t)u64; /* Sign-extend. */
199 /* Count homogeneous 16 bit fragments. */
200 for (i = 0; i < 4; i++) {
201 uint64_t frag = (u64 >> i*16) & 0xffff;
202 zeros += (frag == 0);
203 ones += (frag == 0xffff);
204 }
205 neg = ones > zeros; /* Use MOVN if it pays off. */
206 if ((neg ? ones : zeros) < 3) { /* Need 2+ ins. Try shorter K13 encoding. */
207 uint32_t k13 = emit_isk13(u64, is64);
208 if (k13) {
209 emit_dn(as, (is64|A64I_ORRw)^k13, rd, RID_ZERO);
210 return;
211 }
212 }
213 if (!emit_kdelta(as, rd, u64, 4 - (neg ? ones : zeros))) {
214 int shift = 0, lshift = 0;
215 uint64_t n64 = neg ? ~u64 : u64;
216 if (n64 != 0) {
217 /* Find first/last fragment to be filled. */
218 shift = (63-emit_clz64(n64)) & ~15;
219 lshift = emit_ctz64(n64) & ~15;
220 }
221 /* MOVK requires the original value (u64). */
222 while (shift > lshift) {
223 uint32_t u16 = (u64 >> shift) & 0xffff;
224 /* Skip fragments that are correctly filled by MOVN/MOVZ. */
225 if (u16 != (neg ? 0xffff : 0))
226 emit_d(as, is64 | A64I_MOVKw | A64F_U16(u16) | A64F_LSL16(shift), rd);
227 shift -= 16;
228 }
229 /* But MOVN needs an inverted value (n64). */
230 emit_d(as, (neg ? A64I_MOVNx : A64I_MOVZx) |
231 A64F_U16((n64 >> lshift) & 0xffff) | A64F_LSL16(lshift), rd);
232 }
233}
234
235/* Load a 32 bit constant into a GPR. */
236#define emit_loadi(as, rd, i) emit_loadk(as, rd, i, 0)
237
238/* Load a 64 bit constant into a GPR. */
239#define emit_loadu64(as, rd, i) emit_loadk(as, rd, i, A64I_X)
240
241#define emit_loada(as, r, addr) emit_loadu64(as, (r), (uintptr_t)(addr))
242
243#define glofs(as, k) \
244 ((intptr_t)((uintptr_t)(k) - (uintptr_t)&J2GG(as->J)->g))
245#define mcpofs(as, k) \
246 ((intptr_t)((uintptr_t)(k) - (uintptr_t)(as->mcp - 1)))
247#define checkmcpofs(as, k) \
248 (A64F_S_OK(mcpofs(as, k)>>2, 19))
249
250static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
251
252/* Get/set from constant pointer. */
253static void emit_lsptr(ASMState *as, A64Ins ai, Reg r, void *p)
254{
255 /* First, check if ip + offset is in range. */
256 if ((ai & 0x00400000) && checkmcpofs(as, p)) {
257 emit_d(as, A64I_LDRLx | A64F_S19(mcpofs(as, p)>>2), r);
258 } else {
259 Reg base = RID_GL; /* Next, try GL + offset. */
260 int64_t ofs = glofs(as, p);
261 if (!emit_checkofs(ai, ofs)) { /* Else split up into base reg + offset. */
262 int64_t i64 = i64ptr(p);
263 base = ra_allock(as, (i64 & ~0x7fffull), rset_exclude(RSET_GPR, r));
264 ofs = i64 & 0x7fffull;
265 }
266 emit_lso(as, ai, r, base, ofs);
267 }
268}
269
270/* Load 64 bit IR constant into register. */
271static void emit_loadk64(ASMState *as, Reg r, IRIns *ir)
272{
273 const uint64_t *k = &ir_k64(ir)->u64;
274 int64_t ofs;
275 if (r >= RID_MAX_GPR) {
276 uint32_t fpk = emit_isfpk64(*k);
277 if (fpk != ~0u) {
278 emit_d(as, A64I_FMOV_DI | A64F_FP8(fpk), (r & 31));
279 return;
280 }
281 }
282 ofs = glofs(as, k);
283 if (emit_checkofs(A64I_LDRx, ofs)) {
284 emit_lso(as, r >= RID_MAX_GPR ? A64I_LDRd : A64I_LDRx,
285 (r & 31), RID_GL, ofs);
286 } else {
287 if (r >= RID_MAX_GPR) {
288 emit_dn(as, A64I_FMOV_D_R, (r & 31), RID_TMP);
289 r = RID_TMP;
290 }
291 if (checkmcpofs(as, k))
292 emit_d(as, A64I_LDRLx | A64F_S19(mcpofs(as, k)>>2), r);
293 else
294 emit_loadu64(as, r, *k);
295 }
296}
297
298/* Get/set global_State fields. */
299#define emit_getgl(as, r, field) \
300 emit_lsptr(as, A64I_LDRx, (r), (void *)&J2G(as->J)->field)
301#define emit_setgl(as, r, field) \
302 emit_lsptr(as, A64I_STRx, (r), (void *)&J2G(as->J)->field)
303
304/* Trace number is determined from pc of exit instruction. */
305#define emit_setvmstate(as, i) UNUSED(i)
306
307/* -- Emit control-flow instructions -------------------------------------- */
308
309/* Label for internal jumps. */
310typedef MCode *MCLabel;
311
312/* Return label pointing to current PC. */
313#define emit_label(as) ((as)->mcp)
314
315static void emit_cond_branch(ASMState *as, A64CC cond, MCode *target)
316{
317 MCode *p = --as->mcp;
318 ptrdiff_t delta = target - p;
319 lj_assertA(A64F_S_OK(delta, 19), "branch target out of range");
320 *p = A64I_BCC | A64F_S19(delta) | cond;
321}
322
323static void emit_branch(ASMState *as, A64Ins ai, MCode *target)
324{
325 MCode *p = --as->mcp;
326 ptrdiff_t delta = target - p;
327 lj_assertA(A64F_S_OK(delta, 26), "branch target out of range");
328 *p = ai | A64F_S26(delta);
329}
330
331static void emit_tnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit, MCode *target)
332{
333 MCode *p = --as->mcp;
334 ptrdiff_t delta = target - p;
335 lj_assertA(bit < 63, "bit number out of range");
336 lj_assertA(A64F_S_OK(delta, 14), "branch target out of range");
337 if (bit > 31) ai |= A64I_X;
338 *p = ai | A64F_BIT(bit & 31) | A64F_S14(delta) | r;
339}
340
341static void emit_cnb(ASMState *as, A64Ins ai, Reg r, MCode *target)
342{
343 MCode *p = --as->mcp;
344 ptrdiff_t delta = target - p;
345 lj_assertA(A64F_S_OK(delta, 19), "branch target out of range");
346 *p = ai | A64F_S19(delta) | r;
347}
348
349#define emit_jmp(as, target) emit_branch(as, A64I_B, (target))
350
351static void emit_call(ASMState *as, ASMFunction target)
352{
353 MCode *p = --as->mcp;
354#if LJ_ABI_PAUTH
355 char *targetp = ptrauth_auth_data((char *)target,
356 ptrauth_key_function_pointer, 0);
357#else
358 char *targetp = (char *)target;
359#endif
360 ptrdiff_t delta = targetp - (char *)p;
361 if (A64F_S_OK(delta>>2, 26)) {
362 *p = A64I_BL | A64F_S26(delta>>2);
363 } else { /* Target out of range: need indirect call. But don't use R0-R7. */
364 Reg r = ra_allock(as, i64ptr(target),
365 RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED);
366 *p = A64I_BLR_AUTH | A64F_N(r);
367 }
368}
369
370/* -- Emit generic operations --------------------------------------------- */
371
372/* Generic move between two regs. */
373static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
374{
375 if (dst >= RID_MAX_GPR) {
376 emit_dn(as, irt_isnum(ir->t) ? A64I_FMOV_D : A64I_FMOV_S,
377 (dst & 31), (src & 31));
378 return;
379 }
380 if (as->mcp != as->mcloop) { /* Swap early registers for loads/stores. */
381 MCode ins = *as->mcp, swp = (src^dst);
382 if ((ins & 0xbf800000) == 0xb9000000) {
383 if (!((ins ^ (dst << 5)) & 0x000003e0))
384 *as->mcp = ins ^ (swp << 5); /* Swap N in load/store. */
385 if (!(ins & 0x00400000) && !((ins ^ dst) & 0x0000001f))
386 *as->mcp = ins ^ swp; /* Swap D in store. */
387 }
388 }
389 emit_dm(as, A64I_MOVx, dst, src);
390}
391
392/* Generic load of register with base and (small) offset address. */
393static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
394{
395 if (r >= RID_MAX_GPR)
396 emit_lso(as, irt_isnum(ir->t) ? A64I_LDRd : A64I_LDRs, (r & 31), base, ofs);
397 else
398 emit_lso(as, irt_is64(ir->t) ? A64I_LDRx : A64I_LDRw, r, base, ofs);
399}
400
401/* Generic store of register with base and (small) offset address. */
402static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
403{
404 if (r >= RID_MAX_GPR)
405 emit_lso(as, irt_isnum(ir->t) ? A64I_STRd : A64I_STRs, (r & 31), base, ofs);
406 else
407 emit_lso(as, irt_is64(ir->t) ? A64I_STRx : A64I_STRw, r, base, ofs);
408}
409
410/* Emit an arithmetic operation with a constant operand. */
411static void emit_opk(ASMState *as, A64Ins ai, Reg dest, Reg src,
412 int32_t i, RegSet allow)
413{
414 uint32_t k = emit_isk12(i);
415 if (k)
416 emit_dn(as, ai^k, dest, src);
417 else
418 emit_dnm(as, ai, dest, src, ra_allock(as, i, allow));
419}
420
421/* Add offset to pointer. */
422static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
423{
424 if (ofs)
425 emit_opk(as, ofs < 0 ? A64I_SUBx : A64I_ADDx, r, r,
426 ofs < 0 ? (int32_t)(~(uint32_t)ofs+1u) : ofs,
427 rset_exclude(RSET_GPR, r));
428}
429
430#define emit_spsub(as, ofs) emit_addptr(as, RID_SP, -(ofs))
431
diff --git a/src/lj_emit_mips.h b/src/lj_emit_mips.h
index 57a7a7cd..dda9092d 100644
--- a/src/lj_emit_mips.h
+++ b/src/lj_emit_mips.h
@@ -3,6 +3,32 @@
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h 3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/ 4*/
5 5
6#if LJ_64
7static intptr_t get_k64val(ASMState *as, IRRef ref)
8{
9 IRIns *ir = IR(ref);
10 if (ir->o == IR_KINT64) {
11 return (intptr_t)ir_kint64(ir)->u64;
12 } else if (ir->o == IR_KGC) {
13 return (intptr_t)ir_kgc(ir);
14 } else if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
15 return (intptr_t)ir_kptr(ir);
16 } else if (LJ_SOFTFP && ir->o == IR_KNUM) {
17 return (intptr_t)ir_knum(ir)->u64;
18 } else {
19 lj_assertA(ir->o == IR_KINT || ir->o == IR_KNULL,
20 "bad 64 bit const IR op %d", ir->o);
21 return ir->i; /* Sign-extended. */
22 }
23}
24#endif
25
26#if LJ_64
27#define get_kval(as, ref) get_k64val(as, ref)
28#else
29#define get_kval(as, ref) (IR((ref))->i)
30#endif
31
6/* -- Emit basic instructions --------------------------------------------- */ 32/* -- Emit basic instructions --------------------------------------------- */
7 33
8static void emit_dst(ASMState *as, MIPSIns mi, Reg rd, Reg rs, Reg rt) 34static void emit_dst(ASMState *as, MIPSIns mi, Reg rd, Reg rs, Reg rt)
@@ -35,7 +61,7 @@ static void emit_fgh(ASMState *as, MIPSIns mi, Reg rf, Reg rg, Reg rh)
35 61
36static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift) 62static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift)
37{ 63{
38 if ((as->flags & JIT_F_MIPS32R2)) { 64 if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
39 emit_dta(as, MIPSI_ROTR, dest, src, shift); 65 emit_dta(as, MIPSI_ROTR, dest, src, shift);
40 } else { 66 } else {
41 emit_dst(as, MIPSI_OR, dest, dest, tmp); 67 emit_dst(as, MIPSI_OR, dest, dest, tmp);
@@ -44,23 +70,32 @@ static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift)
44 } 70 }
45} 71}
46 72
73#if LJ_64 || LJ_HASBUFFER
74static void emit_tsml(ASMState *as, MIPSIns mi, Reg rt, Reg rs, uint32_t msb,
75 uint32_t lsb)
76{
77 *--as->mcp = mi | MIPSF_T(rt) | MIPSF_S(rs) | MIPSF_M(msb) | MIPSF_L(lsb);
78}
79#endif
80
47/* -- Emit loads/stores --------------------------------------------------- */ 81/* -- Emit loads/stores --------------------------------------------------- */
48 82
49/* Prefer rematerialization of BASE/L from global_State over spills. */ 83/* Prefer rematerialization of BASE/L from global_State over spills. */
50#define emit_canremat(ref) ((ref) <= REF_BASE) 84#define emit_canremat(ref) ((ref) <= REF_BASE)
51 85
52/* Try to find a one step delta relative to another constant. */ 86/* Try to find a one step delta relative to another constant. */
53static int emit_kdelta1(ASMState *as, Reg t, int32_t i) 87static int emit_kdelta1(ASMState *as, Reg rd, intptr_t i)
54{ 88{
55 RegSet work = ~as->freeset & RSET_GPR; 89 RegSet work = ~as->freeset & RSET_GPR;
56 while (work) { 90 while (work) {
57 Reg r = rset_picktop(work); 91 Reg r = rset_picktop(work);
58 IRRef ref = regcost_ref(as->cost[r]); 92 IRRef ref = regcost_ref(as->cost[r]);
59 lua_assert(r != t); 93 lj_assertA(r != rd, "dest reg %d not free", rd);
60 if (ref < ASMREF_L) { 94 if (ref < ASMREF_L) {
61 int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i); 95 intptr_t delta = (intptr_t)((uintptr_t)i -
96 (uintptr_t)(ra_iskref(ref) ? ra_krefk(as, ref) : get_kval(as, ref)));
62 if (checki16(delta)) { 97 if (checki16(delta)) {
63 emit_tsi(as, MIPSI_ADDIU, t, r, delta); 98 emit_tsi(as, MIPSI_AADDIU, rd, r, delta);
64 return 1; 99 return 1;
65 } 100 }
66 } 101 }
@@ -76,8 +111,8 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
76 emit_ti(as, MIPSI_LI, r, i); 111 emit_ti(as, MIPSI_LI, r, i);
77 } else { 112 } else {
78 if ((i & 0xffff)) { 113 if ((i & 0xffff)) {
79 int32_t jgl = i32ptr(J2G(as->J)); 114 intptr_t jgl = (intptr_t)(void *)J2G(as->J);
80 if ((uint32_t)(i-jgl) < 65536) { 115 if ((uintptr_t)(i-jgl) < 65536) {
81 emit_tsi(as, MIPSI_ADDIU, r, RID_JGL, i-jgl-32768); 116 emit_tsi(as, MIPSI_ADDIU, r, RID_JGL, i-jgl-32768);
82 return; 117 return;
83 } else if (emit_kdelta1(as, r, i)) { 118 } else if (emit_kdelta1(as, r, i)) {
@@ -92,16 +127,49 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
92 } 127 }
93} 128}
94 129
130#if LJ_64
131/* Load a 64 bit constant into a GPR. */
132static void emit_loadu64(ASMState *as, Reg r, uint64_t u64)
133{
134 if (checki32((int64_t)u64)) {
135 emit_loadi(as, r, (int32_t)u64);
136 } else {
137 uint64_t delta = u64 - (uint64_t)(void *)J2G(as->J);
138 if (delta < 65536) {
139 emit_tsi(as, MIPSI_DADDIU, r, RID_JGL, (int32_t)(delta-32768));
140 } else if (emit_kdelta1(as, r, (intptr_t)u64)) {
141 return;
142 } else {
143 /* TODO MIPSR6: Use DAHI & DATI. Caveat: sign-extension. */
144 if ((u64 & 0xffff)) {
145 emit_tsi(as, MIPSI_ORI, r, r, u64 & 0xffff);
146 }
147 if (((u64 >> 16) & 0xffff)) {
148 emit_dta(as, MIPSI_DSLL, r, r, 16);
149 emit_tsi(as, MIPSI_ORI, r, r, (u64 >> 16) & 0xffff);
150 emit_dta(as, MIPSI_DSLL, r, r, 16);
151 } else {
152 emit_dta(as, MIPSI_DSLL32, r, r, 0);
153 }
154 emit_loadi(as, r, (int32_t)(u64 >> 32));
155 }
156 /* TODO: There are probably more optimization opportunities. */
157 }
158}
159
160#define emit_loada(as, r, addr) emit_loadu64(as, (r), u64ptr((addr)))
161#else
95#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr))) 162#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr)))
163#endif
96 164
97static Reg ra_allock(ASMState *as, int32_t k, RegSet allow); 165static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
98static void ra_allockreg(ASMState *as, int32_t k, Reg r); 166static void ra_allockreg(ASMState *as, intptr_t k, Reg r);
99 167
100/* Get/set from constant pointer. */ 168/* Get/set from constant pointer. */
101static void emit_lsptr(ASMState *as, MIPSIns mi, Reg r, void *p, RegSet allow) 169static void emit_lsptr(ASMState *as, MIPSIns mi, Reg r, void *p, RegSet allow)
102{ 170{
103 int32_t jgl = i32ptr(J2G(as->J)); 171 intptr_t jgl = (intptr_t)(J2G(as->J));
104 int32_t i = i32ptr(p); 172 intptr_t i = (intptr_t)(p);
105 Reg base; 173 Reg base;
106 if ((uint32_t)(i-jgl) < 65536) { 174 if ((uint32_t)(i-jgl) < 65536) {
107 i = i-jgl-32768; 175 i = i-jgl-32768;
@@ -112,8 +180,24 @@ static void emit_lsptr(ASMState *as, MIPSIns mi, Reg r, void *p, RegSet allow)
112 emit_tsi(as, mi, r, base, i); 180 emit_tsi(as, mi, r, base, i);
113} 181}
114 182
115#define emit_loadn(as, r, tv) \ 183#if LJ_64
116 emit_lsptr(as, MIPSI_LDC1, ((r) & 31), (void *)(tv), RSET_GPR) 184static void emit_loadk64(ASMState *as, Reg r, IRIns *ir)
185{
186 const uint64_t *k = &ir_k64(ir)->u64;
187 Reg r64 = r;
188 if (rset_test(RSET_FPR, r)) {
189 r64 = RID_TMP;
190 emit_tg(as, MIPSI_DMTC1, r64, r);
191 }
192 if ((uint32_t)((intptr_t)k-(intptr_t)J2G(as->J)) < 65536)
193 emit_lsptr(as, MIPSI_LD, r64, (void *)k, 0);
194 else
195 emit_loadu64(as, r64, *k);
196}
197#else
198#define emit_loadk64(as, r, ir) \
199 emit_lsptr(as, MIPSI_LDC1, ((r) & 31), (void *)&ir_knum((ir))->u64, RSET_GPR)
200#endif
117 201
118/* Get/set global_State fields. */ 202/* Get/set global_State fields. */
119static void emit_lsglptr(ASMState *as, MIPSIns mi, Reg r, int32_t ofs) 203static void emit_lsglptr(ASMState *as, MIPSIns mi, Reg r, int32_t ofs)
@@ -122,9 +206,9 @@ static void emit_lsglptr(ASMState *as, MIPSIns mi, Reg r, int32_t ofs)
122} 206}
123 207
124#define emit_getgl(as, r, field) \ 208#define emit_getgl(as, r, field) \
125 emit_lsglptr(as, MIPSI_LW, (r), (int32_t)offsetof(global_State, field)) 209 emit_lsglptr(as, MIPSI_AL, (r), (int32_t)offsetof(global_State, field))
126#define emit_setgl(as, r, field) \ 210#define emit_setgl(as, r, field) \
127 emit_lsglptr(as, MIPSI_SW, (r), (int32_t)offsetof(global_State, field)) 211 emit_lsglptr(as, MIPSI_AS, (r), (int32_t)offsetof(global_State, field))
128 212
129/* Trace number is determined from per-trace exit stubs. */ 213/* Trace number is determined from per-trace exit stubs. */
130#define emit_setvmstate(as, i) UNUSED(i) 214#define emit_setvmstate(as, i) UNUSED(i)
@@ -141,7 +225,7 @@ static void emit_branch(ASMState *as, MIPSIns mi, Reg rs, Reg rt, MCode *target)
141{ 225{
142 MCode *p = as->mcp; 226 MCode *p = as->mcp;
143 ptrdiff_t delta = target - p; 227 ptrdiff_t delta = target - p;
144 lua_assert(((delta + 0x8000) >> 16) == 0); 228 lj_assertA(((delta + 0x8000) >> 16) == 0, "branch target out of range");
145 *--p = mi | MIPSF_S(rs) | MIPSF_T(rt) | ((uint32_t)delta & 0xffffu); 229 *--p = mi | MIPSF_S(rs) | MIPSF_T(rt) | ((uint32_t)delta & 0xffffu);
146 as->mcp = p; 230 as->mcp = p;
147} 231}
@@ -152,16 +236,31 @@ static void emit_jmp(ASMState *as, MCode *target)
152 emit_branch(as, MIPSI_B, RID_ZERO, RID_ZERO, (target)); 236 emit_branch(as, MIPSI_B, RID_ZERO, RID_ZERO, (target));
153} 237}
154 238
155static void emit_call(ASMState *as, void *target) 239static void emit_call(ASMState *as, void *target, int needcfa)
156{ 240{
157 MCode *p = as->mcp; 241 MCode *p = as->mcp;
158 *--p = MIPSI_NOP; 242#if LJ_TARGET_MIPSR6
159 if ((((uintptr_t)target ^ (uintptr_t)p) >> 28) == 0) 243 ptrdiff_t delta = (char *)target - (char *)p;
244 if ((((delta>>2) + 0x02000000) >> 26) == 0) { /* Try compact call first. */
245 *--p = MIPSI_BALC | (((uintptr_t)delta >>2) & 0x03ffffffu);
246 as->mcp = p;
247 return;
248 }
249#endif
250 *--p = MIPSI_NOP; /* Delay slot. */
251 if ((((uintptr_t)target ^ (uintptr_t)p) >> 28) == 0) {
252#if !LJ_TARGET_MIPSR6
253 *--p = (((uintptr_t)target & 1) ? MIPSI_JALX : MIPSI_JAL) |
254 (((uintptr_t)target >>2) & 0x03ffffffu);
255#else
160 *--p = MIPSI_JAL | (((uintptr_t)target >>2) & 0x03ffffffu); 256 *--p = MIPSI_JAL | (((uintptr_t)target >>2) & 0x03ffffffu);
161 else /* Target out of range: need indirect call. */ 257#endif
258 } else { /* Target out of range: need indirect call. */
162 *--p = MIPSI_JALR | MIPSF_S(RID_CFUNCADDR); 259 *--p = MIPSI_JALR | MIPSF_S(RID_CFUNCADDR);
260 needcfa = 1;
261 }
163 as->mcp = p; 262 as->mcp = p;
164 ra_allockreg(as, i32ptr(target), RID_CFUNCADDR); 263 if (needcfa) ra_allockreg(as, (intptr_t)target, RID_CFUNCADDR);
165} 264}
166 265
167/* -- Emit generic operations --------------------------------------------- */ 266/* -- Emit generic operations --------------------------------------------- */
@@ -178,32 +277,32 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
178 emit_fg(as, irt_isnum(ir->t) ? MIPSI_MOV_D : MIPSI_MOV_S, dst, src); 277 emit_fg(as, irt_isnum(ir->t) ? MIPSI_MOV_D : MIPSI_MOV_S, dst, src);
179} 278}
180 279
181/* Generic load of register from stack slot. */ 280/* Generic load of register with base and (small) offset address. */
182static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 281static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
183{ 282{
184 if (r < RID_MAX_GPR) 283 if (r < RID_MAX_GPR)
185 emit_tsi(as, MIPSI_LW, r, RID_SP, ofs); 284 emit_tsi(as, irt_is64(ir->t) ? MIPSI_LD : MIPSI_LW, r, base, ofs);
186 else 285 else
187 emit_tsi(as, irt_isnum(ir->t) ? MIPSI_LDC1 : MIPSI_LWC1, 286 emit_tsi(as, irt_isnum(ir->t) ? MIPSI_LDC1 : MIPSI_LWC1,
188 (r & 31), RID_SP, ofs); 287 (r & 31), base, ofs);
189} 288}
190 289
191/* Generic store of register to stack slot. */ 290/* Generic store of register with base and (small) offset address. */
192static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 291static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
193{ 292{
194 if (r < RID_MAX_GPR) 293 if (r < RID_MAX_GPR)
195 emit_tsi(as, MIPSI_SW, r, RID_SP, ofs); 294 emit_tsi(as, irt_is64(ir->t) ? MIPSI_SD : MIPSI_SW, r, base, ofs);
196 else 295 else
197 emit_tsi(as, irt_isnum(ir->t) ? MIPSI_SDC1 : MIPSI_SWC1, 296 emit_tsi(as, irt_isnum(ir->t) ? MIPSI_SDC1 : MIPSI_SWC1,
198 (r&31), RID_SP, ofs); 297 (r&31), base, ofs);
199} 298}
200 299
201/* Add offset to pointer. */ 300/* Add offset to pointer. */
202static void emit_addptr(ASMState *as, Reg r, int32_t ofs) 301static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
203{ 302{
204 if (ofs) { 303 if (ofs) {
205 lua_assert(checki16(ofs)); 304 lj_assertA(checki16(ofs), "offset %d out of range", ofs);
206 emit_tsi(as, MIPSI_ADDIU, r, r, ofs); 305 emit_tsi(as, MIPSI_AADDIU, r, r, ofs);
207 } 306 }
208} 307}
209 308
diff --git a/src/lj_emit_ppc.h b/src/lj_emit_ppc.h
index 62462ade..c34dcbe2 100644
--- a/src/lj_emit_ppc.h
+++ b/src/lj_emit_ppc.h
@@ -41,13 +41,13 @@ static void emit_rot(ASMState *as, PPCIns pi, Reg ra, Reg rs,
41 41
42static void emit_slwi(ASMState *as, Reg ra, Reg rs, int32_t n) 42static void emit_slwi(ASMState *as, Reg ra, Reg rs, int32_t n)
43{ 43{
44 lua_assert(n >= 0 && n < 32); 44 lj_assertA(n >= 0 && n < 32, "shift out or range");
45 emit_rot(as, PPCI_RLWINM, ra, rs, n, 0, 31-n); 45 emit_rot(as, PPCI_RLWINM, ra, rs, n, 0, 31-n);
46} 46}
47 47
48static void emit_rotlwi(ASMState *as, Reg ra, Reg rs, int32_t n) 48static void emit_rotlwi(ASMState *as, Reg ra, Reg rs, int32_t n)
49{ 49{
50 lua_assert(n >= 0 && n < 32); 50 lj_assertA(n >= 0 && n < 32, "shift out or range");
51 emit_rot(as, PPCI_RLWINM, ra, rs, n, 0, 31); 51 emit_rot(as, PPCI_RLWINM, ra, rs, n, 0, 31);
52} 52}
53 53
@@ -57,17 +57,17 @@ static void emit_rotlwi(ASMState *as, Reg ra, Reg rs, int32_t n)
57#define emit_canremat(ref) ((ref) <= REF_BASE) 57#define emit_canremat(ref) ((ref) <= REF_BASE)
58 58
59/* Try to find a one step delta relative to another constant. */ 59/* Try to find a one step delta relative to another constant. */
60static int emit_kdelta1(ASMState *as, Reg t, int32_t i) 60static int emit_kdelta1(ASMState *as, Reg rd, int32_t i)
61{ 61{
62 RegSet work = ~as->freeset & RSET_GPR; 62 RegSet work = ~as->freeset & RSET_GPR;
63 while (work) { 63 while (work) {
64 Reg r = rset_picktop(work); 64 Reg r = rset_picktop(work);
65 IRRef ref = regcost_ref(as->cost[r]); 65 IRRef ref = regcost_ref(as->cost[r]);
66 lua_assert(r != t); 66 lj_assertA(r != rd, "dest reg %d not free", rd);
67 if (ref < ASMREF_L) { 67 if (ref < ASMREF_L) {
68 int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i); 68 int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i);
69 if (checki16(delta)) { 69 if (checki16(delta)) {
70 emit_tai(as, PPCI_ADDI, t, r, delta); 70 emit_tai(as, PPCI_ADDI, rd, r, delta);
71 return 1; 71 return 1;
72 } 72 }
73 } 73 }
@@ -98,7 +98,7 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
98 98
99#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr))) 99#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr)))
100 100
101static Reg ra_allock(ASMState *as, int32_t k, RegSet allow); 101static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
102 102
103/* Get/set from constant pointer. */ 103/* Get/set from constant pointer. */
104static void emit_lsptr(ASMState *as, PPCIns pi, Reg r, void *p, RegSet allow) 104static void emit_lsptr(ASMState *as, PPCIns pi, Reg r, void *p, RegSet allow)
@@ -115,8 +115,8 @@ static void emit_lsptr(ASMState *as, PPCIns pi, Reg r, void *p, RegSet allow)
115 emit_tai(as, pi, r, base, i); 115 emit_tai(as, pi, r, base, i);
116} 116}
117 117
118#define emit_loadn(as, r, tv) \ 118#define emit_loadk64(as, r, ir) \
119 emit_lsptr(as, PPCI_LFD, ((r) & 31), (void *)(tv), RSET_GPR) 119 emit_lsptr(as, PPCI_LFD, ((r) & 31), (void *)&ir_knum((ir))->u64, RSET_GPR)
120 120
121/* Get/set global_State fields. */ 121/* Get/set global_State fields. */
122static void emit_lsglptr(ASMState *as, PPCIns pi, Reg r, int32_t ofs) 122static void emit_lsglptr(ASMState *as, PPCIns pi, Reg r, int32_t ofs)
@@ -144,7 +144,7 @@ static void emit_condbranch(ASMState *as, PPCIns pi, PPCCC cc, MCode *target)
144{ 144{
145 MCode *p = --as->mcp; 145 MCode *p = --as->mcp;
146 ptrdiff_t delta = (char *)target - (char *)p; 146 ptrdiff_t delta = (char *)target - (char *)p;
147 lua_assert(((delta + 0x8000) >> 16) == 0); 147 lj_assertA(((delta + 0x8000) >> 16) == 0, "branch target out of range");
148 pi ^= (delta & 0x8000) * (PPCF_Y/0x8000); 148 pi ^= (delta & 0x8000) * (PPCF_Y/0x8000);
149 *p = pi | PPCF_CC(cc) | ((uint32_t)delta & 0xffffu); 149 *p = pi | PPCF_CC(cc) | ((uint32_t)delta & 0xffffu);
150} 150}
@@ -186,22 +186,22 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
186 emit_fb(as, PPCI_FMR, dst, src); 186 emit_fb(as, PPCI_FMR, dst, src);
187} 187}
188 188
189/* Generic load of register from stack slot. */ 189/* Generic load of register with base and (small) offset address. */
190static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 190static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
191{ 191{
192 if (r < RID_MAX_GPR) 192 if (r < RID_MAX_GPR)
193 emit_tai(as, PPCI_LWZ, r, RID_SP, ofs); 193 emit_tai(as, PPCI_LWZ, r, base, ofs);
194 else 194 else
195 emit_fai(as, irt_isnum(ir->t) ? PPCI_LFD : PPCI_LFS, r, RID_SP, ofs); 195 emit_fai(as, irt_isnum(ir->t) ? PPCI_LFD : PPCI_LFS, r, base, ofs);
196} 196}
197 197
198/* Generic store of register to stack slot. */ 198/* Generic store of register with base and (small) offset address. */
199static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 199static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
200{ 200{
201 if (r < RID_MAX_GPR) 201 if (r < RID_MAX_GPR)
202 emit_tai(as, PPCI_STW, r, RID_SP, ofs); 202 emit_tai(as, PPCI_STW, r, base, ofs);
203 else 203 else
204 emit_fai(as, irt_isnum(ir->t) ? PPCI_STFD : PPCI_STFS, r, RID_SP, ofs); 204 emit_fai(as, irt_isnum(ir->t) ? PPCI_STFD : PPCI_STFS, r, base, ofs);
205} 205}
206 206
207/* Emit a compare (for equality) with a constant operand. */ 207/* Emit a compare (for equality) with a constant operand. */
diff --git a/src/lj_emit_x86.h b/src/lj_emit_x86.h
index 0e26ad52..d215402c 100644
--- a/src/lj_emit_x86.h
+++ b/src/lj_emit_x86.h
@@ -13,10 +13,17 @@
13 if (rex != 0x40) *--(p) = rex; } 13 if (rex != 0x40) *--(p) = rex; }
14#define FORCE_REX 0x200 14#define FORCE_REX 0x200
15#define REX_64 (FORCE_REX|0x080000) 15#define REX_64 (FORCE_REX|0x080000)
16#define VEX_64 0x800000
16#else 17#else
17#define REXRB(p, rr, rb) ((void)0) 18#define REXRB(p, rr, rb) ((void)0)
18#define FORCE_REX 0 19#define FORCE_REX 0
19#define REX_64 0 20#define REX_64 0
21#define VEX_64 0
22#endif
23#if LJ_GC64
24#define REX_GC64 REX_64
25#else
26#define REX_GC64 0
20#endif 27#endif
21 28
22#define emit_i8(as, i) (*--as->mcp = (MCode)(i)) 29#define emit_i8(as, i) (*--as->mcp = (MCode)(i))
@@ -31,7 +38,14 @@ static LJ_AINLINE MCode *emit_op(x86Op xo, Reg rr, Reg rb, Reg rx,
31 MCode *p, int delta) 38 MCode *p, int delta)
32{ 39{
33 int n = (int8_t)xo; 40 int n = (int8_t)xo;
34#if defined(__GNUC__) 41 if (n == -60) { /* VEX-encoded instruction */
42#if LJ_64
43 xo ^= (((rr>>1)&4)+((rx>>2)&2)+((rb>>3)&1))<<13;
44#endif
45 *(uint32_t *)(p+delta-5) = (uint32_t)xo;
46 return p+delta-5;
47 }
48#if defined(__GNUC__) || defined(__clang__)
35 if (__builtin_constant_p(xo) && n == -2) 49 if (__builtin_constant_p(xo) && n == -2)
36 p[delta-2] = (MCode)(xo >> 24); 50 p[delta-2] = (MCode)(xo >> 24);
37 else if (__builtin_constant_p(xo) && n == -3) 51 else if (__builtin_constant_p(xo) && n == -3)
@@ -78,33 +92,24 @@ static void emit_rr(ASMState *as, x86Op xo, Reg r1, Reg r2)
78/* [addr] is sign-extended in x64 and must be in lower 2G (not 4G). */ 92/* [addr] is sign-extended in x64 and must be in lower 2G (not 4G). */
79static int32_t ptr2addr(const void *p) 93static int32_t ptr2addr(const void *p)
80{ 94{
81 lua_assert((uintptr_t)p < (uintptr_t)0x80000000); 95 lj_assertX((uintptr_t)p < (uintptr_t)0x80000000, "pointer outside 2G range");
82 return i32ptr(p); 96 return i32ptr(p);
83} 97}
84#else 98#else
85#define ptr2addr(p) (i32ptr((p))) 99#define ptr2addr(p) (i32ptr((p)))
86#endif 100#endif
87 101
88/* op r, [addr] */
89static void emit_rma(ASMState *as, x86Op xo, Reg rr, const void *addr)
90{
91 MCode *p = as->mcp;
92 *(int32_t *)(p-4) = ptr2addr(addr);
93#if LJ_64
94 p[-5] = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
95 as->mcp = emit_opm(xo, XM_OFS0, rr, RID_ESP, p, -5);
96#else
97 as->mcp = emit_opm(xo, XM_OFS0, rr, RID_EBP, p, -4);
98#endif
99}
100
101/* op r, [base+ofs] */ 102/* op r, [base+ofs] */
102static void emit_rmro(ASMState *as, x86Op xo, Reg rr, Reg rb, int32_t ofs) 103static void emit_rmro(ASMState *as, x86Op xo, Reg rr, Reg rb, int32_t ofs)
103{ 104{
104 MCode *p = as->mcp; 105 MCode *p = as->mcp;
105 x86Mode mode; 106 x86Mode mode;
106 if (ra_hasreg(rb)) { 107 if (ra_hasreg(rb)) {
107 if (ofs == 0 && (rb&7) != RID_EBP) { 108 if (LJ_GC64 && rb == RID_RIP) {
109 mode = XM_OFS0;
110 p -= 4;
111 *(int32_t *)p = ofs;
112 } else if (ofs == 0 && (rb&7) != RID_EBP) {
108 mode = XM_OFS0; 113 mode = XM_OFS0;
109 } else if (checki8(ofs)) { 114 } else if (checki8(ofs)) {
110 *--p = (MCode)ofs; 115 *--p = (MCode)ofs;
@@ -202,6 +207,11 @@ static void emit_mrm(ASMState *as, x86Op xo, Reg rr, Reg rb)
202 *--p = MODRM(XM_SCALE1, RID_ESP, RID_EBP); 207 *--p = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
203 rb = RID_ESP; 208 rb = RID_ESP;
204#endif 209#endif
210 } else if (LJ_GC64 && rb == RID_RIP) {
211 lj_assertA(as->mrm.idx == RID_NONE, "RIP-rel mrm cannot have index");
212 mode = XM_OFS0;
213 p -= 4;
214 *(int32_t *)p = as->mrm.ofs;
205 } else { 215 } else {
206 if (as->mrm.ofs == 0 && (rb&7) != RID_EBP) { 216 if (as->mrm.ofs == 0 && (rb&7) != RID_EBP) {
207 mode = XM_OFS0; 217 mode = XM_OFS0;
@@ -241,10 +251,6 @@ static void emit_gmrmi(ASMState *as, x86Group xg, Reg rb, int32_t i)
241 251
242/* -- Emit loads/stores --------------------------------------------------- */ 252/* -- Emit loads/stores --------------------------------------------------- */
243 253
244/* Instruction selection for XMM moves. */
245#define XMM_MOVRR(as) ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVSD : XO_MOVAPS)
246#define XMM_MOVRM(as) ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVLPD : XO_MOVSD)
247
248/* mov [base+ofs], i */ 254/* mov [base+ofs], i */
249static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i) 255static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i)
250{ 256{
@@ -259,8 +265,8 @@ static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i)
259/* Get/set global_State fields. */ 265/* Get/set global_State fields. */
260#define emit_opgl(as, xo, r, field) \ 266#define emit_opgl(as, xo, r, field) \
261 emit_rma(as, (xo), (r), (void *)&J2G(as->J)->field) 267 emit_rma(as, (xo), (r), (void *)&J2G(as->J)->field)
262#define emit_getgl(as, r, field) emit_opgl(as, XO_MOV, (r), field) 268#define emit_getgl(as, r, field) emit_opgl(as, XO_MOV, (r)|REX_GC64, field)
263#define emit_setgl(as, r, field) emit_opgl(as, XO_MOVto, (r), field) 269#define emit_setgl(as, r, field) emit_opgl(as, XO_MOVto, (r)|REX_GC64, field)
264 270
265#define emit_setvmstate(as, i) \ 271#define emit_setvmstate(as, i) \
266 (emit_i32(as, i), emit_opgl(as, XO_MOVmi, 0, vmstate)) 272 (emit_i32(as, i), emit_opgl(as, XO_MOVmi, 0, vmstate))
@@ -285,9 +291,21 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
285 } 291 }
286} 292}
287 293
294#if LJ_GC64
295#define dispofs(as, k) \
296 ((intptr_t)((uintptr_t)(k) - (uintptr_t)J2GG(as->J)->dispatch))
297#define mcpofs(as, k) \
298 ((intptr_t)((uintptr_t)(k) - (uintptr_t)as->mcp))
299#define mctopofs(as, k) \
300 ((intptr_t)((uintptr_t)(k) - (uintptr_t)as->mctop))
301/* mov r, addr */
302#define emit_loada(as, r, addr) \
303 emit_loadu64(as, (r), (uintptr_t)(addr))
304#else
288/* mov r, addr */ 305/* mov r, addr */
289#define emit_loada(as, r, addr) \ 306#define emit_loada(as, r, addr) \
290 emit_loadi(as, (r), ptr2addr((addr))) 307 emit_loadi(as, (r), ptr2addr((addr)))
308#endif
291 309
292#if LJ_64 310#if LJ_64
293/* mov r, imm64 or shorter 32 bit extended load. */ 311/* mov r, imm64 or shorter 32 bit extended load. */
@@ -299,6 +317,15 @@ static void emit_loadu64(ASMState *as, Reg r, uint64_t u64)
299 MCode *p = as->mcp; 317 MCode *p = as->mcp;
300 *(int32_t *)(p-4) = (int32_t)u64; 318 *(int32_t *)(p-4) = (int32_t)u64;
301 as->mcp = emit_opm(XO_MOVmi, XM_REG, REX_64, r, p, -4); 319 as->mcp = emit_opm(XO_MOVmi, XM_REG, REX_64, r, p, -4);
320#if LJ_GC64
321 } else if (checki32(dispofs(as, u64))) {
322 emit_rmro(as, XO_LEA, r|REX_64, RID_DISPATCH, (int32_t)dispofs(as, u64));
323 } else if (checki32(mcpofs(as, u64)) && checki32(mctopofs(as, u64))) {
324 /* Since as->realign assumes the code size doesn't change, check
325 ** RIP-relative addressing reachability for both as->mcp and as->mctop.
326 */
327 emit_rmro(as, XO_LEA, r|REX_64, RID_RIP, (int32_t)mcpofs(as, u64));
328#endif
302 } else { /* Full-size 64 bit load. */ 329 } else { /* Full-size 64 bit load. */
303 MCode *p = as->mcp; 330 MCode *p = as->mcp;
304 *(uint64_t *)(p-8) = u64; 331 *(uint64_t *)(p-8) = u64;
@@ -310,13 +337,90 @@ static void emit_loadu64(ASMState *as, Reg r, uint64_t u64)
310} 337}
311#endif 338#endif
312 339
313/* movsd r, [&tv->n] / xorps r, r */ 340/* op r, [addr] */
314static void emit_loadn(ASMState *as, Reg r, cTValue *tv) 341static void emit_rma(ASMState *as, x86Op xo, Reg rr, const void *addr)
315{ 342{
316 if (tvispzero(tv)) /* Use xor only for +0. */ 343#if LJ_GC64
317 emit_rr(as, XO_XORPS, r, r); 344 if (checki32(dispofs(as, addr))) {
318 else 345 emit_rmro(as, xo, rr, RID_DISPATCH, (int32_t)dispofs(as, addr));
319 emit_rma(as, XMM_MOVRM(as), r, &tv->n); 346 } else if (checki32(mcpofs(as, addr)) && checki32(mctopofs(as, addr))) {
347 emit_rmro(as, xo, rr, RID_RIP, (int32_t)mcpofs(as, addr));
348 } else if (!checki32((intptr_t)addr)) {
349 Reg ra = (rr & 15);
350 if (xo != XO_MOV) {
351 /* We can't allocate a register here. Use and restore DISPATCH. Ugly. */
352 uint64_t dispaddr = (uintptr_t)J2GG(as->J)->dispatch;
353 uint8_t i8 = xo == XO_GROUP3b ? *as->mcp++ : 0;
354 ra = RID_DISPATCH;
355 if (checku32(dispaddr)) {
356 emit_loadi(as, ra, (int32_t)dispaddr);
357 } else { /* Full-size 64 bit load. */
358 MCode *p = as->mcp;
359 *(uint64_t *)(p-8) = dispaddr;
360 p[-9] = (MCode)(XI_MOVri+(ra&7));
361 p[-10] = 0x48 + ((ra>>3)&1);
362 p -= 10;
363 as->mcp = p;
364 }
365 if (xo == XO_GROUP3b) emit_i8(as, i8);
366 }
367 emit_rmro(as, xo, rr, ra, 0);
368 emit_loadu64(as, ra, (uintptr_t)addr);
369 } else
370#endif
371 {
372 MCode *p = as->mcp;
373 *(int32_t *)(p-4) = ptr2addr(addr);
374#if LJ_64
375 p[-5] = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
376 as->mcp = emit_opm(xo, XM_OFS0, rr, RID_ESP, p, -5);
377#else
378 as->mcp = emit_opm(xo, XM_OFS0, rr, RID_EBP, p, -4);
379#endif
380 }
381}
382
383/* Load 64 bit IR constant into register. */
384static void emit_loadk64(ASMState *as, Reg r, IRIns *ir)
385{
386 Reg r64;
387 x86Op xo;
388 const uint64_t *k = &ir_k64(ir)->u64;
389 if (rset_test(RSET_FPR, r)) {
390 r64 = r;
391 xo = XO_MOVSD;
392 } else {
393 r64 = r | REX_64;
394 xo = XO_MOV;
395 }
396 if (*k == 0) {
397 emit_rr(as, rset_test(RSET_FPR, r) ? XO_XORPS : XO_ARITH(XOg_XOR), r, r);
398#if LJ_GC64
399 } else if (checki32((intptr_t)k) || checki32(dispofs(as, k)) ||
400 (checki32(mcpofs(as, k)) && checki32(mctopofs(as, k)))) {
401 emit_rma(as, xo, r64, k);
402 } else {
403 if (ir->i) {
404 lj_assertA(*k == *(uint64_t*)(as->mctop - ir->i),
405 "bad interned 64 bit constant");
406 } else if (as->curins <= as->stopins && rset_test(RSET_GPR, r)) {
407 emit_loadu64(as, r, *k);
408 return;
409 } else {
410 /* If all else fails, add the FP constant at the MCode area bottom. */
411 while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3;
412 *(uint64_t *)as->mcbot = *k;
413 ir->i = (int32_t)(as->mctop - as->mcbot);
414 as->mcbot += 8;
415 as->mclim = as->mcbot + MCLIM_REDZONE;
416 lj_mcode_commitbot(as->J, as->mcbot);
417 }
418 emit_rmro(as, xo, r64, RID_RIP, (int32_t)mcpofs(as, as->mctop - ir->i));
419#else
420 } else {
421 emit_rma(as, xo, r64, k);
422#endif
423 }
320} 424}
321 425
322/* -- Emit control-flow instructions -------------------------------------- */ 426/* -- Emit control-flow instructions -------------------------------------- */
@@ -330,7 +434,7 @@ static void emit_sjmp(ASMState *as, MCLabel target)
330{ 434{
331 MCode *p = as->mcp; 435 MCode *p = as->mcp;
332 ptrdiff_t delta = target - p; 436 ptrdiff_t delta = target - p;
333 lua_assert(delta == (int8_t)delta); 437 lj_assertA(delta == (int8_t)delta, "short jump target out of range");
334 p[-1] = (MCode)(int8_t)delta; 438 p[-1] = (MCode)(int8_t)delta;
335 p[-2] = XI_JMPs; 439 p[-2] = XI_JMPs;
336 as->mcp = p - 2; 440 as->mcp = p - 2;
@@ -342,7 +446,7 @@ static void emit_sjcc(ASMState *as, int cc, MCLabel target)
342{ 446{
343 MCode *p = as->mcp; 447 MCode *p = as->mcp;
344 ptrdiff_t delta = target - p; 448 ptrdiff_t delta = target - p;
345 lua_assert(delta == (int8_t)delta); 449 lj_assertA(delta == (int8_t)delta, "short jump target out of range");
346 p[-1] = (MCode)(int8_t)delta; 450 p[-1] = (MCode)(int8_t)delta;
347 p[-2] = (MCode)(XI_JCCs+(cc&15)); 451 p[-2] = (MCode)(XI_JCCs+(cc&15));
348 as->mcp = p - 2; 452 as->mcp = p - 2;
@@ -368,10 +472,11 @@ static void emit_sfixup(ASMState *as, MCLabel source)
368#define emit_label(as) ((as)->mcp) 472#define emit_label(as) ((as)->mcp)
369 473
370/* Compute relative 32 bit offset for jump and call instructions. */ 474/* Compute relative 32 bit offset for jump and call instructions. */
371static LJ_AINLINE int32_t jmprel(MCode *p, MCode *target) 475static LJ_AINLINE int32_t jmprel(jit_State *J, MCode *p, MCode *target)
372{ 476{
373 ptrdiff_t delta = target - p; 477 ptrdiff_t delta = target - p;
374 lua_assert(delta == (int32_t)delta); 478 UNUSED(J);
479 lj_assertJ(delta == (int32_t)delta, "jump target out of range");
375 return (int32_t)delta; 480 return (int32_t)delta;
376} 481}
377 482
@@ -379,7 +484,7 @@ static LJ_AINLINE int32_t jmprel(MCode *p, MCode *target)
379static void emit_jcc(ASMState *as, int cc, MCode *target) 484static void emit_jcc(ASMState *as, int cc, MCode *target)
380{ 485{
381 MCode *p = as->mcp; 486 MCode *p = as->mcp;
382 *(int32_t *)(p-4) = jmprel(p, target); 487 *(int32_t *)(p-4) = jmprel(as->J, p, target);
383 p[-5] = (MCode)(XI_JCCn+(cc&15)); 488 p[-5] = (MCode)(XI_JCCn+(cc&15));
384 p[-6] = 0x0f; 489 p[-6] = 0x0f;
385 as->mcp = p - 6; 490 as->mcp = p - 6;
@@ -389,7 +494,7 @@ static void emit_jcc(ASMState *as, int cc, MCode *target)
389static void emit_jmp(ASMState *as, MCode *target) 494static void emit_jmp(ASMState *as, MCode *target)
390{ 495{
391 MCode *p = as->mcp; 496 MCode *p = as->mcp;
392 *(int32_t *)(p-4) = jmprel(p, target); 497 *(int32_t *)(p-4) = jmprel(as->J, p, target);
393 p[-5] = XI_JMP; 498 p[-5] = XI_JMP;
394 as->mcp = p - 5; 499 as->mcp = p - 5;
395} 500}
@@ -406,7 +511,7 @@ static void emit_call_(ASMState *as, MCode *target)
406 return; 511 return;
407 } 512 }
408#endif 513#endif
409 *(int32_t *)(p-4) = jmprel(p, target); 514 *(int32_t *)(p-4) = jmprel(as->J, p, target);
410 p[-5] = XI_CALL; 515 p[-5] = XI_CALL;
411 as->mcp = p - 5; 516 as->mcp = p - 5;
412} 517}
@@ -418,8 +523,10 @@ static void emit_call_(ASMState *as, MCode *target)
418/* Use 64 bit operations to handle 64 bit IR types. */ 523/* Use 64 bit operations to handle 64 bit IR types. */
419#if LJ_64 524#if LJ_64
420#define REX_64IR(ir, r) ((r) + (irt_is64((ir)->t) ? REX_64 : 0)) 525#define REX_64IR(ir, r) ((r) + (irt_is64((ir)->t) ? REX_64 : 0))
526#define VEX_64IR(ir, r) ((r) + (irt_is64((ir)->t) ? VEX_64 : 0))
421#else 527#else
422#define REX_64IR(ir, r) (r) 528#define REX_64IR(ir, r) (r)
529#define VEX_64IR(ir, r) (r)
423#endif 530#endif
424 531
425/* Generic move between two regs. */ 532/* Generic move between two regs. */
@@ -429,35 +536,32 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
429 if (dst < RID_MAX_GPR) 536 if (dst < RID_MAX_GPR)
430 emit_rr(as, XO_MOV, REX_64IR(ir, dst), src); 537 emit_rr(as, XO_MOV, REX_64IR(ir, dst), src);
431 else 538 else
432 emit_rr(as, XMM_MOVRR(as), dst, src); 539 emit_rr(as, XO_MOVAPS, dst, src);
433} 540}
434 541
435/* Generic load of register from stack slot. */ 542/* Generic load of register with base and (small) offset address. */
436static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 543static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
437{ 544{
438 if (r < RID_MAX_GPR) 545 if (r < RID_MAX_GPR)
439 emit_rmro(as, XO_MOV, REX_64IR(ir, r), RID_ESP, ofs); 546 emit_rmro(as, XO_MOV, REX_64IR(ir, r), base, ofs);
440 else 547 else
441 emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS, r, RID_ESP, ofs); 548 emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS, r, base, ofs);
442} 549}
443 550
444/* Generic store of register to stack slot. */ 551/* Generic store of register with base and (small) offset address. */
445static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs) 552static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
446{ 553{
447 if (r < RID_MAX_GPR) 554 if (r < RID_MAX_GPR)
448 emit_rmro(as, XO_MOVto, REX_64IR(ir, r), RID_ESP, ofs); 555 emit_rmro(as, XO_MOVto, REX_64IR(ir, r), base, ofs);
449 else 556 else
450 emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto, r, RID_ESP, ofs); 557 emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto, r, base, ofs);
451} 558}
452 559
453/* Add offset to pointer. */ 560/* Add offset to pointer. */
454static void emit_addptr(ASMState *as, Reg r, int32_t ofs) 561static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
455{ 562{
456 if (ofs) { 563 if (ofs) {
457 if ((as->flags & JIT_F_LEA_AGU)) 564 emit_gri(as, XG_ARITHi(XOg_ADD), r|REX_GC64, ofs);
458 emit_rmro(as, XO_LEA, r, r, ofs);
459 else
460 emit_gri(as, XG_ARITHi(XOg_ADD), r, ofs);
461 } 565 }
462} 566}
463 567
diff --git a/src/lj_err.c b/src/lj_err.c
index 4a2d6bbd..6e50cbee 100644
--- a/src/lj_err.c
+++ b/src/lj_err.c
@@ -16,6 +16,7 @@
16#include "lj_ff.h" 16#include "lj_ff.h"
17#include "lj_trace.h" 17#include "lj_trace.h"
18#include "lj_vm.h" 18#include "lj_vm.h"
19#include "lj_strfmt.h"
19 20
20/* 21/*
21** LuaJIT can either use internal or external frame unwinding: 22** LuaJIT can either use internal or external frame unwinding:
@@ -28,12 +29,18 @@
28** Pros and Cons: 29** Pros and Cons:
29** 30**
30** - EXT requires unwind tables for *all* functions on the C stack between 31** - EXT requires unwind tables for *all* functions on the C stack between
31** the pcall/catch and the error/throw. This is the default on x64, 32** the pcall/catch and the error/throw. C modules used by Lua code can
32** but needs to be manually enabled on x86/PPC for non-C++ code. 33** throw errors, so these need to have unwind tables, too. Transitively
34** this applies to all system libraries used by C modules -- at least
35** when they have callbacks which may throw an error.
33** 36**
34** - INT is faster when actually throwing errors (but this happens rarely). 37** - INT is faster when actually throwing errors, but this happens rarely.
35** Setting up error handlers is zero-cost in any case. 38** Setting up error handlers is zero-cost in any case.
36** 39**
40** - INT needs to save *all* callee-saved registers when entering the
41** interpreter. EXT only needs to save those actually used inside the
42** interpreter. JIT-compiled code may need to save some more.
43**
37** - EXT provides full interoperability with C++ exceptions. You can throw 44** - EXT provides full interoperability with C++ exceptions. You can throw
38** Lua errors or C++ exceptions through a mix of Lua frames and C++ frames. 45** Lua errors or C++ exceptions through a mix of Lua frames and C++ frames.
39** C++ destructors are called as needed. C++ exceptions caught by pcall 46** C++ destructors are called as needed. C++ exceptions caught by pcall
@@ -45,27 +52,38 @@
45** the wrapper function feature. Lua errors thrown through C++ frames 52** the wrapper function feature. Lua errors thrown through C++ frames
46** cannot be caught by C++ code and C++ destructors are not run. 53** cannot be caught by C++ code and C++ destructors are not run.
47** 54**
48** EXT is the default on x64 systems, INT is the default on all other systems. 55** - EXT can handle errors from internal helper functions that are called
56** from JIT-compiled code (except for Windows/x86 and 32 bit ARM).
57** INT has no choice but to call the panic handler, if this happens.
58** Note: this is mainly relevant for out-of-memory errors.
59**
60** EXT is the default on all systems where the toolchain produces unwind
61** tables by default (*). This is hard-coded and/or detected in src/Makefile.
62** You can thwart the detection with: TARGET_XCFLAGS=-DLUAJIT_UNWIND_INTERNAL
63**
64** INT is the default on all other systems.
65**
66** EXT can be manually enabled for toolchains that are able to produce
67** conforming unwind tables:
68** "TARGET_XCFLAGS=-funwind-tables -DLUAJIT_UNWIND_EXTERNAL"
69** As explained above, *all* C code used directly or indirectly by LuaJIT
70** must be compiled with -funwind-tables (or -fexceptions). C++ code must
71** *not* be compiled with -fno-exceptions.
72**
73** If you're unsure whether error handling inside the VM works correctly,
74** try running this and check whether it prints "OK":
49** 75**
50** EXT can be manually enabled on POSIX systems using GCC and DWARF2 stack 76** luajit -e "print(select(2, load('OK')):match('OK'))"
51** unwinding with -DLUAJIT_UNWIND_EXTERNAL. *All* C code must be compiled
52** with -funwind-tables (or -fexceptions). This includes LuaJIT itself (set
53** TARGET_CFLAGS), all of your C/Lua binding code, all loadable C modules
54** and all C libraries that have callbacks which may be used to call back
55** into Lua. C++ code must *not* be compiled with -fno-exceptions.
56** 77**
57** EXT cannot be enabled on WIN32 since system exceptions use code-driven SEH. 78** (*) Originally, toolchains only generated unwind tables for C++ code. For
58** EXT is mandatory on WIN64 since the calling convention has an abundance 79** interoperability reasons, this can be manually enabled for plain C code,
59** of callee-saved registers (rbx, rbp, rsi, rdi, r12-r15, xmm6-xmm15). 80** too (with -funwind-tables). With the introduction of the x64 architecture,
60** The POSIX/x64 interpreter only saves r12/r13 for INT (e.g. PS4). 81** the corresponding POSIX and Windows ABIs mandated unwind tables for all
82** code. Over the following years most desktop and server platforms have
83** enabled unwind tables by default on all architectures. OTOH mobile and
84** embedded platforms do not consistently mandate unwind tables.
61*/ 85*/
62 86
63#if defined(__GNUC__) && (LJ_TARGET_X64 || defined(LUAJIT_UNWIND_EXTERNAL)) && !LJ_NO_UNWIND
64#define LJ_UNWIND_EXT 1
65#elif LJ_TARGET_X64 && LJ_TARGET_WINDOWS
66#define LJ_UNWIND_EXT 1
67#endif
68
69/* -- Error messages ------------------------------------------------------ */ 87/* -- Error messages ------------------------------------------------------ */
70 88
71/* Error message strings. */ 89/* Error message strings. */
@@ -98,14 +116,14 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
98 TValue *top = restorestack(L, -nres); 116 TValue *top = restorestack(L, -nres);
99 if (frame < top) { /* Frame reached? */ 117 if (frame < top) { /* Frame reached? */
100 if (errcode) { 118 if (errcode) {
101 L->cframe = cframe_prev(cf);
102 L->base = frame+1; 119 L->base = frame+1;
120 L->cframe = cframe_prev(cf);
103 unwindstack(L, top); 121 unwindstack(L, top);
104 } 122 }
105 return cf; 123 return cf;
106 } 124 }
107 } 125 }
108 if (frame <= tvref(L->stack)) 126 if (frame <= tvref(L->stack)+LJ_FR2)
109 break; 127 break;
110 switch (frame_typep(frame)) { 128 switch (frame_typep(frame)) {
111 case FRAME_LUA: /* Lua frame. */ 129 case FRAME_LUA: /* Lua frame. */
@@ -113,14 +131,12 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
113 frame = frame_prevl(frame); 131 frame = frame_prevl(frame);
114 break; 132 break;
115 case FRAME_C: /* C frame. */ 133 case FRAME_C: /* C frame. */
116#if LJ_HASFFI
117 unwind_c: 134 unwind_c:
118#endif
119#if LJ_UNWIND_EXT 135#if LJ_UNWIND_EXT
120 if (errcode) { 136 if (errcode) {
121 L->cframe = cframe_prev(cf);
122 L->base = frame_prevd(frame) + 1; 137 L->base = frame_prevd(frame) + 1;
123 unwindstack(L, frame); 138 L->cframe = cframe_prev(cf);
139 unwindstack(L, frame - LJ_FR2);
124 } else if (cf != stopcf) { 140 } else if (cf != stopcf) {
125 cf = cframe_prev(cf); 141 cf = cframe_prev(cf);
126 frame = frame_prevd(frame); 142 frame = frame_prevd(frame);
@@ -143,16 +159,14 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
143 return cf; 159 return cf;
144 } 160 }
145 if (errcode) { 161 if (errcode) {
146 L->cframe = cframe_prev(cf);
147 L->base = frame_prevd(frame) + 1; 162 L->base = frame_prevd(frame) + 1;
148 unwindstack(L, frame); 163 L->cframe = cframe_prev(cf);
164 unwindstack(L, frame - LJ_FR2);
149 } 165 }
150 return cf; 166 return cf;
151 case FRAME_CONT: /* Continuation frame. */ 167 case FRAME_CONT: /* Continuation frame. */
152#if LJ_HASFFI 168 if (frame_iscont_fficb(frame))
153 if ((frame-1)->u32.lo == LJ_CONT_FFI_CALLBACK)
154 goto unwind_c; 169 goto unwind_c;
155#endif
156 /* fallthrough */ 170 /* fallthrough */
157 case FRAME_VARG: /* Vararg frame. */ 171 case FRAME_VARG: /* Vararg frame. */
158 frame = frame_prevd(frame); 172 frame = frame_prevd(frame);
@@ -166,8 +180,8 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
166 } 180 }
167 if (frame_typep(frame) == FRAME_PCALL) 181 if (frame_typep(frame) == FRAME_PCALL)
168 hook_leave(G(L)); 182 hook_leave(G(L));
169 L->cframe = cf;
170 L->base = frame_prevd(frame) + 1; 183 L->base = frame_prevd(frame) + 1;
184 L->cframe = cf;
171 unwindstack(L, L->base); 185 unwindstack(L, L->base);
172 } 186 }
173 return (void *)((intptr_t)cf | CFRAME_UNWIND_FF); 187 return (void *)((intptr_t)cf | CFRAME_UNWIND_FF);
@@ -175,8 +189,8 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
175 } 189 }
176 /* No C frame. */ 190 /* No C frame. */
177 if (errcode) { 191 if (errcode) {
192 L->base = tvref(L->stack)+1+LJ_FR2;
178 L->cframe = NULL; 193 L->cframe = NULL;
179 L->base = tvref(L->stack)+1;
180 unwindstack(L, L->base); 194 unwindstack(L, L->base);
181 if (G(L)->panic) 195 if (G(L)->panic)
182 G(L)->panic(L); 196 G(L)->panic(L);
@@ -187,33 +201,206 @@ static void *err_unwind(lua_State *L, void *stopcf, int errcode)
187 201
188/* -- External frame unwinding -------------------------------------------- */ 202/* -- External frame unwinding -------------------------------------------- */
189 203
190#if defined(__GNUC__) && !LJ_NO_UNWIND && !LJ_ABI_WIN 204#if LJ_ABI_WIN
191 205
192/* 206/*
193** We have to use our own definitions instead of the mandatory (!) unwind.h, 207** Someone in Redmond owes me several days of my life. A lot of this is
194** since various OS, distros and compilers mess up the header installation. 208** undocumented or just plain wrong on MSDN. Some of it can be gathered
209** from 3rd party docs or must be found by trial-and-error. They really
210** don't want you to write your own language-specific exception handler
211** or to interact gracefully with MSVC. :-(
212**
213** Apparently MSVC doesn't call C++ destructors for foreign exceptions
214** unless you compile your C++ code with /EHa. Unfortunately this means
215** catch (...) also catches things like access violations. The use of
216** _set_se_translator doesn't really help, because it requires /EHa, too.
195*/ 217*/
196 218
197typedef struct _Unwind_Exception 219#define WIN32_LEAN_AND_MEAN
220#include <windows.h>
221
222#if LJ_TARGET_X86
223typedef void *UndocumentedDispatcherContext; /* Unused on x86. */
224#else
225/* Taken from: http://www.nynaeve.net/?p=99 */
226typedef struct UndocumentedDispatcherContext {
227 ULONG64 ControlPc;
228 ULONG64 ImageBase;
229 PRUNTIME_FUNCTION FunctionEntry;
230 ULONG64 EstablisherFrame;
231 ULONG64 TargetIp;
232 PCONTEXT ContextRecord;
233 void (*LanguageHandler)(void);
234 PVOID HandlerData;
235 PUNWIND_HISTORY_TABLE HistoryTable;
236 ULONG ScopeIndex;
237 ULONG Fill0;
238} UndocumentedDispatcherContext;
239#endif
240
241/* Another wild guess. */
242extern void __DestructExceptionObject(EXCEPTION_RECORD *rec, int nothrow);
243
244#if LJ_TARGET_X64 && defined(MINGW_SDK_INIT)
245/* Workaround for broken MinGW64 declaration. */
246VOID RtlUnwindEx_FIXED(PVOID,PVOID,PVOID,PVOID,PVOID,PVOID) asm("RtlUnwindEx");
247#define RtlUnwindEx RtlUnwindEx_FIXED
248#endif
249
250#define LJ_MSVC_EXCODE ((DWORD)0xe06d7363)
251#define LJ_GCC_EXCODE ((DWORD)0x20474343)
252
253#define LJ_EXCODE ((DWORD)0xe24c4a00)
254#define LJ_EXCODE_MAKE(c) (LJ_EXCODE | (DWORD)(c))
255#define LJ_EXCODE_CHECK(cl) (((cl) ^ LJ_EXCODE) <= 0xff)
256#define LJ_EXCODE_ERRCODE(cl) ((int)((cl) & 0xff))
257
258/* Windows exception handler for interpreter frame. */
259LJ_FUNCA int lj_err_unwind_win(EXCEPTION_RECORD *rec,
260 void *f, CONTEXT *ctx, UndocumentedDispatcherContext *dispatch)
198{ 261{
199 uint64_t exclass; 262#if LJ_TARGET_X86
200 void (*excleanup)(int, struct _Unwind_Exception *); 263 void *cf = (char *)f - CFRAME_OFS_SEH;
201 uintptr_t p1, p2; 264#else
202} __attribute__((__aligned__)) _Unwind_Exception; 265 void *cf = f;
266#endif
267 lua_State *L = cframe_L(cf);
268 int errcode = LJ_EXCODE_CHECK(rec->ExceptionCode) ?
269 LJ_EXCODE_ERRCODE(rec->ExceptionCode) : LUA_ERRRUN;
270 if ((rec->ExceptionFlags & 6)) { /* EH_UNWINDING|EH_EXIT_UNWIND */
271 /* Unwind internal frames. */
272 err_unwind(L, cf, errcode);
273 } else {
274 void *cf2 = err_unwind(L, cf, 0);
275 if (cf2) { /* We catch it, so start unwinding the upper frames. */
276 if (rec->ExceptionCode == LJ_MSVC_EXCODE ||
277 rec->ExceptionCode == LJ_GCC_EXCODE) {
278#if !LJ_TARGET_CYGWIN
279 __DestructExceptionObject(rec, 1);
280#endif
281 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
282 } else if (!LJ_EXCODE_CHECK(rec->ExceptionCode)) {
283 /* Don't catch access violations etc. */
284 return 1; /* ExceptionContinueSearch */
285 }
286#if LJ_TARGET_X86
287 UNUSED(ctx);
288 UNUSED(dispatch);
289 /* Call all handlers for all lower C frames (including ourselves) again
290 ** with EH_UNWINDING set. Then call the specified function, passing cf
291 ** and errcode.
292 */
293 lj_vm_rtlunwind(cf, (void *)rec,
294 (cframe_unwind_ff(cf2) && errcode != LUA_YIELD) ?
295 (void *)lj_vm_unwind_ff : (void *)lj_vm_unwind_c, errcode);
296 /* lj_vm_rtlunwind does not return. */
297#else
298 /* Unwind the stack and call all handlers for all lower C frames
299 ** (including ourselves) again with EH_UNWINDING set. Then set
300 ** stack pointer = cf, result = errcode and jump to the specified target.
301 */
302 RtlUnwindEx(cf, (void *)((cframe_unwind_ff(cf2) && errcode != LUA_YIELD) ?
303 lj_vm_unwind_ff_eh :
304 lj_vm_unwind_c_eh),
305 rec, (void *)(uintptr_t)errcode, ctx, dispatch->HistoryTable);
306 /* RtlUnwindEx should never return. */
307#endif
308 }
309 }
310 return 1; /* ExceptionContinueSearch */
311}
312
313#if LJ_UNWIND_JIT
314
315#if LJ_TARGET_X64
316#define CONTEXT_REG_PC Rip
317#elif LJ_TARGET_ARM64
318#define CONTEXT_REG_PC Pc
319#else
320#error "NYI: Windows arch-specific unwinder for JIT-compiled code"
321#endif
322
323/* Windows unwinder for JIT-compiled code. */
324static void err_unwind_win_jit(global_State *g, int errcode)
325{
326 CONTEXT ctx;
327 UNWIND_HISTORY_TABLE hist;
328
329 memset(&hist, 0, sizeof(hist));
330 RtlCaptureContext(&ctx);
331 while (1) {
332 DWORD64 frame, base, addr = ctx.CONTEXT_REG_PC;
333 void *hdata;
334 PRUNTIME_FUNCTION func = RtlLookupFunctionEntry(addr, &base, &hist);
335 if (!func) { /* Found frame without .pdata: must be JIT-compiled code. */
336 ExitNo exitno;
337 uintptr_t stub = lj_trace_unwind(G2J(g), (uintptr_t)(addr - sizeof(MCode)), &exitno);
338 if (stub) { /* Jump to side exit to unwind the trace. */
339 ctx.CONTEXT_REG_PC = stub;
340 G2J(g)->exitcode = errcode;
341 RtlRestoreContext(&ctx, NULL); /* Does not return. */
342 }
343 break;
344 }
345 RtlVirtualUnwind(UNW_FLAG_NHANDLER, base, addr, func,
346 &ctx, &hdata, &frame, NULL);
347 if (!addr) break;
348 }
349 /* Unwinding failed, if we end up here. */
350}
351#endif
352
353/* Raise Windows exception. */
354static void err_raise_ext(global_State *g, int errcode)
355{
356#if LJ_UNWIND_JIT
357 if (tvref(g->jit_base)) {
358 err_unwind_win_jit(g, errcode);
359 return; /* Unwinding failed. */
360 }
361#elif LJ_HASJIT
362 /* Cannot catch on-trace errors for Windows/x86 SEH. Unwind to interpreter. */
363 setmref(g->jit_base, NULL);
364#endif
365 UNUSED(g);
366 RaiseException(LJ_EXCODE_MAKE(errcode), 1 /* EH_NONCONTINUABLE */, 0, NULL);
367}
368
369#elif !LJ_NO_UNWIND && (defined(__GNUC__) || defined(__clang__))
370
371/*
372** We have to use our own definitions instead of the mandatory (!) unwind.h,
373** since various OS, distros and compilers mess up the header installation.
374*/
203 375
204typedef struct _Unwind_Context _Unwind_Context; 376typedef struct _Unwind_Context _Unwind_Context;
205 377
206#define _URC_OK 0 378#define _URC_OK 0
379#define _URC_FATAL_PHASE2_ERROR 2
207#define _URC_FATAL_PHASE1_ERROR 3 380#define _URC_FATAL_PHASE1_ERROR 3
208#define _URC_HANDLER_FOUND 6 381#define _URC_HANDLER_FOUND 6
209#define _URC_INSTALL_CONTEXT 7 382#define _URC_INSTALL_CONTEXT 7
210#define _URC_CONTINUE_UNWIND 8 383#define _URC_CONTINUE_UNWIND 8
211#define _URC_FAILURE 9 384#define _URC_FAILURE 9
212 385
386#define LJ_UEXCLASS 0x4c55414a49543200ULL /* LUAJIT2\0 */
387#define LJ_UEXCLASS_MAKE(c) (LJ_UEXCLASS | (uint64_t)(c))
388#define LJ_UEXCLASS_CHECK(cl) (((cl) ^ LJ_UEXCLASS) <= 0xff)
389#define LJ_UEXCLASS_ERRCODE(cl) ((int)((cl) & 0xff))
390
213#if !LJ_TARGET_ARM 391#if !LJ_TARGET_ARM
214 392
393typedef struct _Unwind_Exception
394{
395 uint64_t exclass;
396 void (*excleanup)(int, struct _Unwind_Exception *);
397 uintptr_t p1, p2;
398} __attribute__((__aligned__)) _Unwind_Exception;
399#define UNWIND_EXCEPTION_TYPE _Unwind_Exception
400
215extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 401extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
216extern void _Unwind_SetGR(_Unwind_Context *, int, uintptr_t); 402extern void _Unwind_SetGR(_Unwind_Context *, int, uintptr_t);
403extern uintptr_t _Unwind_GetIP(_Unwind_Context *);
217extern void _Unwind_SetIP(_Unwind_Context *, uintptr_t); 404extern void _Unwind_SetIP(_Unwind_Context *, uintptr_t);
218extern void _Unwind_DeleteException(_Unwind_Exception *); 405extern void _Unwind_DeleteException(_Unwind_Exception *);
219extern int _Unwind_RaiseException(_Unwind_Exception *); 406extern int _Unwind_RaiseException(_Unwind_Exception *);
@@ -223,11 +410,6 @@ extern int _Unwind_RaiseException(_Unwind_Exception *);
223#define _UA_HANDLER_FRAME 4 410#define _UA_HANDLER_FRAME 4
224#define _UA_FORCE_UNWIND 8 411#define _UA_FORCE_UNWIND 8
225 412
226#define LJ_UEXCLASS 0x4c55414a49543200ULL /* LUAJIT2\0 */
227#define LJ_UEXCLASS_MAKE(c) (LJ_UEXCLASS | (uint64_t)(c))
228#define LJ_UEXCLASS_CHECK(cl) (((cl) ^ LJ_UEXCLASS) <= 0xff)
229#define LJ_UEXCLASS_ERRCODE(cl) ((int)((cl) & 0xff))
230
231/* DWARF2 personality handler referenced from interpreter .eh_frame. */ 413/* DWARF2 personality handler referenced from interpreter .eh_frame. */
232LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions, 414LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions,
233 uint64_t uexclass, _Unwind_Exception *uex, _Unwind_Context *ctx) 415 uint64_t uexclass, _Unwind_Exception *uex, _Unwind_Context *ctx)
@@ -236,7 +418,6 @@ LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions,
236 lua_State *L; 418 lua_State *L;
237 if (version != 1) 419 if (version != 1)
238 return _URC_FATAL_PHASE1_ERROR; 420 return _URC_FATAL_PHASE1_ERROR;
239 UNUSED(uexclass);
240 cf = (void *)_Unwind_GetCFA(ctx); 421 cf = (void *)_Unwind_GetCFA(ctx);
241 L = cframe_L(cf); 422 L = cframe_L(cf);
242 if ((actions & _UA_SEARCH_PHASE)) { 423 if ((actions & _UA_SEARCH_PHASE)) {
@@ -263,10 +444,10 @@ LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions,
263 if ((actions & _UA_FORCE_UNWIND)) { 444 if ((actions & _UA_FORCE_UNWIND)) {
264 return _URC_CONTINUE_UNWIND; 445 return _URC_CONTINUE_UNWIND;
265 } else if (cf) { 446 } else if (cf) {
447 ASMFunction ip;
266 _Unwind_SetGR(ctx, LJ_TARGET_EHRETREG, errcode); 448 _Unwind_SetGR(ctx, LJ_TARGET_EHRETREG, errcode);
267 _Unwind_SetIP(ctx, (uintptr_t)(cframe_unwind_ff(cf) ? 449 ip = cframe_unwind_ff(cf) ? lj_vm_unwind_ff_eh : lj_vm_unwind_c_eh;
268 lj_vm_unwind_ff_eh : 450 _Unwind_SetIP(ctx, (uintptr_t)lj_ptr_strip(ip));
269 lj_vm_unwind_c_eh));
270 return _URC_INSTALL_CONTEXT; 451 return _URC_INSTALL_CONTEXT;
271 } 452 }
272#if LJ_TARGET_X86ORX64 453#if LJ_TARGET_X86ORX64
@@ -284,27 +465,170 @@ LJ_FUNCA int lj_err_unwind_dwarf(int version, int actions,
284 ** it on non-x64 because the interpreter restores all callee-saved regs. 465 ** it on non-x64 because the interpreter restores all callee-saved regs.
285 */ 466 */
286 lj_err_throw(L, errcode); 467 lj_err_throw(L, errcode);
468#if LJ_TARGET_X64
469#error "Broken build system -- only use the provided Makefiles!"
470#endif
287#endif 471#endif
288 } 472 }
289 return _URC_CONTINUE_UNWIND; 473 return _URC_CONTINUE_UNWIND;
290} 474}
291 475
292#if LJ_UNWIND_EXT 476#if LJ_UNWIND_EXT && defined(LUA_USE_ASSERT)
293static __thread _Unwind_Exception static_uex; 477struct dwarf_eh_bases { void *tbase, *dbase, *func; };
478extern const void *_Unwind_Find_FDE(void *pc, struct dwarf_eh_bases *bases);
294 479
295/* Raise DWARF2 exception. */ 480/* Verify that external error handling actually has a chance to work. */
296static void err_raise_ext(int errcode) 481void lj_err_verify(void)
297{ 482{
298 static_uex.exclass = LJ_UEXCLASS_MAKE(errcode); 483#if !LJ_TARGET_OSX
299 static_uex.excleanup = NULL; 484 /* Check disabled on MacOS due to brilliant software engineering at Apple. */
300 _Unwind_RaiseException(&static_uex); 485 struct dwarf_eh_bases ehb;
486 lj_assertX(_Unwind_Find_FDE((void *)lj_err_throw, &ehb), "broken build: external frame unwinding enabled, but missing -funwind-tables");
487#endif
488 /* Check disabled, because of broken Fedora/ARM64. See #722.
489 lj_assertX(_Unwind_Find_FDE((void *)_Unwind_RaiseException, &ehb), "broken build: external frame unwinding enabled, but system libraries have no unwind tables");
490 */
301} 491}
302#endif 492#endif
303 493
494#if LJ_UNWIND_JIT
495/* DWARF2 personality handler for JIT-compiled code. */
496static int err_unwind_jit(int version, int actions,
497 uint64_t uexclass, _Unwind_Exception *uex, _Unwind_Context *ctx)
498{
499 /* NYI: FFI C++ exception interoperability. */
500 if (version != 1 || !LJ_UEXCLASS_CHECK(uexclass))
501 return _URC_FATAL_PHASE1_ERROR;
502 if ((actions & _UA_SEARCH_PHASE)) {
503 return _URC_HANDLER_FOUND;
504 }
505 if ((actions & _UA_CLEANUP_PHASE)) {
506 global_State *g = *(global_State **)(uex+1);
507 ExitNo exitno;
508 uintptr_t addr = _Unwind_GetIP(ctx); /* Return address _after_ call. */
509 uintptr_t stub = lj_trace_unwind(G2J(g), addr - sizeof(MCode), &exitno);
510 lj_assertG(tvref(g->jit_base), "unexpected throw across mcode frame");
511 if (stub) { /* Jump to side exit to unwind the trace. */
512 G2J(g)->exitcode = LJ_UEXCLASS_ERRCODE(uexclass);
513#ifdef LJ_TARGET_MIPS
514 _Unwind_SetGR(ctx, 4, stub);
515 _Unwind_SetGR(ctx, 5, exitno);
516 _Unwind_SetIP(ctx, (uintptr_t)(void *)lj_vm_unwind_stub);
304#else 517#else
518 _Unwind_SetIP(ctx, stub);
519#endif
520 return _URC_INSTALL_CONTEXT;
521 }
522 return _URC_FATAL_PHASE2_ERROR;
523 }
524 return _URC_FATAL_PHASE1_ERROR;
525}
305 526
306extern void _Unwind_DeleteException(void *); 527/* DWARF2 template frame info for JIT-compiled code.
307extern int __gnu_unwind_frame (void *, _Unwind_Context *); 528**
529** After copying the template to the start of the mcode segment,
530** the frame handler function and the code size is patched.
531** The frame handler always installs a new context to jump to the exit,
532** so don't bother to add any unwind opcodes.
533*/
534static const uint8_t err_frame_jit_template[] = {
535#if LJ_BE
536 0,0,0,
537#endif
538 LJ_64 ? 0x1c : 0x14, /* CIE length. */
539#if LJ_LE
540 0,0,0,
541#endif
542 0,0,0,0, 1, 'z','P','R',0, /* CIE mark, CIE version, augmentation. */
543 1, LJ_64 ? 0x78 : 0x7c, LJ_TARGET_EHRAREG, /* Code/data align, RA. */
544#if LJ_64
545 10, 0, 0,0,0,0,0,0,0,0, 0x1b, /* Aug. data ABS handler, PCREL|SDATA4 code. */
546 0,0,0,0,0, /* Alignment. */
547#else
548 6, 0, 0,0,0,0, 0x1b, /* Aug. data ABS handler, PCREL|SDATA4 code. */
549 0, /* Alignment. */
550#endif
551#if LJ_BE
552 0,0,0,
553#endif
554 LJ_64 ? 0x14 : 0x10, /* FDE length. */
555 0,0,0,
556 LJ_64 ? 0x24 : 0x1c, /* CIE offset. */
557 0,0,0,
558 LJ_64 ? 0x14 : 0x10, /* Code offset. After Final FDE. */
559#if LJ_LE
560 0,0,0,
561#endif
562 0,0,0,0, 0, 0,0,0, /* Code size, augmentation length, alignment. */
563#if LJ_64
564 0,0,0,0, /* Alignment. */
565#endif
566 0,0,0,0 /* Final FDE. */
567};
568
569#define ERR_FRAME_JIT_OFS_HANDLER 0x12
570#define ERR_FRAME_JIT_OFS_FDE (LJ_64 ? 0x20 : 0x18)
571#define ERR_FRAME_JIT_OFS_CODE_SIZE (LJ_64 ? 0x2c : 0x24)
572#if LJ_TARGET_OSX
573#define ERR_FRAME_JIT_OFS_REGISTER ERR_FRAME_JIT_OFS_FDE
574#else
575#define ERR_FRAME_JIT_OFS_REGISTER 0
576#endif
577
578extern void __register_frame(const void *);
579extern void __deregister_frame(const void *);
580
581uint8_t *lj_err_register_mcode(void *base, size_t sz, uint8_t *info)
582{
583 ASMFunction handler = (ASMFunction)err_unwind_jit;
584 memcpy(info, err_frame_jit_template, sizeof(err_frame_jit_template));
585#if LJ_ABI_PAUTH
586#if LJ_TARGET_ARM64
587 handler = ptrauth_auth_and_resign(handler,
588 ptrauth_key_function_pointer, 0,
589 ptrauth_key_process_independent_code, info + ERR_FRAME_JIT_OFS_HANDLER);
590#else
591#error "missing pointer authentication support for this architecture"
592#endif
593#endif
594 memcpy(info + ERR_FRAME_JIT_OFS_HANDLER, &handler, sizeof(handler));
595 *(uint32_t *)(info + ERR_FRAME_JIT_OFS_CODE_SIZE) =
596 (uint32_t)(sz - sizeof(err_frame_jit_template) - (info - (uint8_t *)base));
597 __register_frame(info + ERR_FRAME_JIT_OFS_REGISTER);
598#ifdef LUA_USE_ASSERT
599 {
600 struct dwarf_eh_bases ehb;
601 lj_assertX(_Unwind_Find_FDE(info + sizeof(err_frame_jit_template)+1, &ehb),
602 "bad JIT unwind table registration");
603 }
604#endif
605 return info + sizeof(err_frame_jit_template);
606}
607
608void lj_err_deregister_mcode(void *base, size_t sz, uint8_t *info)
609{
610 UNUSED(base); UNUSED(sz);
611 __deregister_frame(info + ERR_FRAME_JIT_OFS_REGISTER);
612}
613#endif
614
615#else /* LJ_TARGET_ARM */
616
617#define _US_VIRTUAL_UNWIND_FRAME 0
618#define _US_UNWIND_FRAME_STARTING 1
619#define _US_ACTION_MASK 3
620#define _US_FORCE_UNWIND 8
621
622typedef struct _Unwind_Control_Block _Unwind_Control_Block;
623#define UNWIND_EXCEPTION_TYPE _Unwind_Control_Block
624
625struct _Unwind_Control_Block {
626 uint64_t exclass;
627 uint32_t misc[20];
628};
629
630extern int _Unwind_RaiseException(_Unwind_Control_Block *);
631extern int __gnu_unwind_frame(_Unwind_Control_Block *, _Unwind_Context *);
308extern int _Unwind_VRS_Set(_Unwind_Context *, int, uint32_t, int, void *); 632extern int _Unwind_VRS_Set(_Unwind_Context *, int, uint32_t, int, void *);
309extern int _Unwind_VRS_Get(_Unwind_Context *, int, uint32_t, int, void *); 633extern int _Unwind_VRS_Get(_Unwind_Context *, int, uint32_t, int, void *);
310 634
@@ -320,126 +644,98 @@ static inline void _Unwind_SetGR(_Unwind_Context *ctx, int r, uint32_t v)
320 _Unwind_VRS_Set(ctx, 0, r, 0, &v); 644 _Unwind_VRS_Set(ctx, 0, r, 0, &v);
321} 645}
322 646
323#define _US_VIRTUAL_UNWIND_FRAME 0 647extern void lj_vm_unwind_ext(void);
324#define _US_UNWIND_FRAME_STARTING 1
325#define _US_ACTION_MASK 3
326#define _US_FORCE_UNWIND 8
327 648
328/* ARM unwinder personality handler referenced from interpreter .ARM.extab. */ 649/* ARM unwinder personality handler referenced from interpreter .ARM.extab. */
329LJ_FUNCA int lj_err_unwind_arm(int state, void *ucb, _Unwind_Context *ctx) 650LJ_FUNCA int lj_err_unwind_arm(int state, _Unwind_Control_Block *ucb,
651 _Unwind_Context *ctx)
330{ 652{
331 void *cf = (void *)_Unwind_GetGR(ctx, 13); 653 void *cf = (void *)_Unwind_GetGR(ctx, 13);
332 lua_State *L = cframe_L(cf); 654 lua_State *L = cframe_L(cf);
333 if ((state & _US_ACTION_MASK) == _US_VIRTUAL_UNWIND_FRAME) { 655 int errcode;
334 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP)); 656
657 switch ((state & _US_ACTION_MASK)) {
658 case _US_VIRTUAL_UNWIND_FRAME:
659 if ((state & _US_FORCE_UNWIND)) break;
335 return _URC_HANDLER_FOUND; 660 return _URC_HANDLER_FOUND;
336 } 661 case _US_UNWIND_FRAME_STARTING:
337 if ((state&(_US_ACTION_MASK|_US_FORCE_UNWIND)) == _US_UNWIND_FRAME_STARTING) { 662 if (LJ_UEXCLASS_CHECK(ucb->exclass)) {
338 _Unwind_DeleteException(ucb); 663 errcode = LJ_UEXCLASS_ERRCODE(ucb->exclass);
339 _Unwind_SetGR(ctx, 15, (uint32_t)(void *)lj_err_throw); 664 } else {
340 _Unwind_SetGR(ctx, 0, (uint32_t)L); 665 errcode = LUA_ERRRUN;
341 _Unwind_SetGR(ctx, 1, (uint32_t)LUA_ERRRUN); 666 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
667 }
668 cf = err_unwind(L, cf, errcode);
669 if ((state & _US_FORCE_UNWIND) || cf == NULL) break;
670 _Unwind_SetGR(ctx, 15, (uint32_t)lj_vm_unwind_ext);
671 _Unwind_SetGR(ctx, 0, (uint32_t)ucb);
672 _Unwind_SetGR(ctx, 1, (uint32_t)errcode);
673 _Unwind_SetGR(ctx, 2, cframe_unwind_ff(cf) ?
674 (uint32_t)lj_vm_unwind_ff_eh :
675 (uint32_t)lj_vm_unwind_c_eh);
342 return _URC_INSTALL_CONTEXT; 676 return _URC_INSTALL_CONTEXT;
677 default:
678 return _URC_FAILURE;
343 } 679 }
344 if (__gnu_unwind_frame(ucb, ctx) != _URC_OK) 680 if (__gnu_unwind_frame(ucb, ctx) != _URC_OK)
345 return _URC_FAILURE; 681 return _URC_FAILURE;
682#ifdef LUA_USE_ASSERT
683 /* We should never get here unless this is a forced unwind aka backtrace. */
684 if (_Unwind_GetGR(ctx, 0) == 0xff33aa77) {
685 _Unwind_SetGR(ctx, 0, 0xff33aa88);
686 }
687#endif
346 return _URC_CONTINUE_UNWIND; 688 return _URC_CONTINUE_UNWIND;
347} 689}
348 690
349#endif 691#if LJ_UNWIND_EXT && defined(LUA_USE_ASSERT)
692typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
693extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
350 694
351#elif LJ_TARGET_X64 && LJ_ABI_WIN 695static int err_verify_bt(_Unwind_Context *ctx, int *got)
696{
697 if (_Unwind_GetGR(ctx, 0) == 0xff33aa88) { *got = 2; }
698 else if (*got == 0) { *got = 1; _Unwind_SetGR(ctx, 0, 0xff33aa77); }
699 return _URC_OK;
700}
701
702/* Verify that external error handling actually has a chance to work. */
703void lj_err_verify(void)
704{
705 int got = 0;
706 _Unwind_Backtrace((_Unwind_Trace_Fn)err_verify_bt, &got);
707 lj_assertX(got == 2, "broken build: external frame unwinding enabled, but missing -funwind-tables");
708}
709#endif
352 710
353/* 711/*
354** Someone in Redmond owes me several days of my life. A lot of this is 712** Note: LJ_UNWIND_JIT is not implemented for 32 bit ARM.
355** undocumented or just plain wrong on MSDN. Some of it can be gathered
356** from 3rd party docs or must be found by trial-and-error. They really
357** don't want you to write your own language-specific exception handler
358** or to interact gracefully with MSVC. :-(
359** 713**
360** Apparently MSVC doesn't call C++ destructors for foreign exceptions 714** The quirky ARM unwind API doesn't have __register_frame().
361** unless you compile your C++ code with /EHa. Unfortunately this means 715** A potential workaround might involve _Unwind_Backtrace.
362** catch (...) also catches things like access violations. The use of 716** But most 32 bit ARM targets don't qualify for LJ_UNWIND_EXT, anyway,
363** _set_se_translator doesn't really help, because it requires /EHa, too. 717** since they are built without unwind tables by default.
364*/ 718*/
365 719
366#define WIN32_LEAN_AND_MEAN 720#endif /* LJ_TARGET_ARM */
367#include <windows.h>
368 721
369/* Taken from: http://www.nynaeve.net/?p=99 */
370typedef struct UndocumentedDispatcherContext {
371 ULONG64 ControlPc;
372 ULONG64 ImageBase;
373 PRUNTIME_FUNCTION FunctionEntry;
374 ULONG64 EstablisherFrame;
375 ULONG64 TargetIp;
376 PCONTEXT ContextRecord;
377 void (*LanguageHandler)(void);
378 PVOID HandlerData;
379 PUNWIND_HISTORY_TABLE HistoryTable;
380 ULONG ScopeIndex;
381 ULONG Fill0;
382} UndocumentedDispatcherContext;
383
384/* Another wild guess. */
385extern void __DestructExceptionObject(EXCEPTION_RECORD *rec, int nothrow);
386
387#ifdef MINGW_SDK_INIT
388/* Workaround for broken MinGW64 declaration. */
389VOID RtlUnwindEx_FIXED(PVOID,PVOID,PVOID,PVOID,PVOID,PVOID) asm("RtlUnwindEx");
390#define RtlUnwindEx RtlUnwindEx_FIXED
391#endif
392
393#define LJ_MSVC_EXCODE ((DWORD)0xe06d7363)
394#define LJ_GCC_EXCODE ((DWORD)0x20474343)
395 722
396#define LJ_EXCODE ((DWORD)0xe24c4a00) 723#if LJ_UNWIND_EXT
397#define LJ_EXCODE_MAKE(c) (LJ_EXCODE | (DWORD)(c)) 724static __thread struct {
398#define LJ_EXCODE_CHECK(cl) (((cl) ^ LJ_EXCODE) <= 0xff) 725 UNWIND_EXCEPTION_TYPE ex;
399#define LJ_EXCODE_ERRCODE(cl) ((int)((cl) & 0xff)) 726 global_State *g;
727} static_uex;
400 728
401/* Win64 exception handler for interpreter frame. */ 729/* Raise external exception. */
402LJ_FUNCA EXCEPTION_DISPOSITION lj_err_unwind_win64(EXCEPTION_RECORD *rec, 730static void err_raise_ext(global_State *g, int errcode)
403 void *cf, CONTEXT *ctx, UndocumentedDispatcherContext *dispatch)
404{ 731{
405 lua_State *L = cframe_L(cf); 732 memset(&static_uex, 0, sizeof(static_uex));
406 int errcode = LJ_EXCODE_CHECK(rec->ExceptionCode) ? 733 static_uex.ex.exclass = LJ_UEXCLASS_MAKE(errcode);
407 LJ_EXCODE_ERRCODE(rec->ExceptionCode) : LUA_ERRRUN; 734 static_uex.g = g;
408 if ((rec->ExceptionFlags & 6)) { /* EH_UNWINDING|EH_EXIT_UNWIND */ 735 _Unwind_RaiseException(&static_uex.ex);
409 /* Unwind internal frames. */
410 err_unwind(L, cf, errcode);
411 } else {
412 void *cf2 = err_unwind(L, cf, 0);
413 if (cf2) { /* We catch it, so start unwinding the upper frames. */
414 if (rec->ExceptionCode == LJ_MSVC_EXCODE ||
415 rec->ExceptionCode == LJ_GCC_EXCODE) {
416#if LJ_TARGET_WINDOWS
417 __DestructExceptionObject(rec, 1);
418#endif
419 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRCPP));
420 } else if (!LJ_EXCODE_CHECK(rec->ExceptionCode)) {
421 /* Don't catch access violations etc. */
422 return ExceptionContinueSearch;
423 }
424 /* Unwind the stack and call all handlers for all lower C frames
425 ** (including ourselves) again with EH_UNWINDING set. Then set
426 ** rsp = cf, rax = errcode and jump to the specified target.
427 */
428 RtlUnwindEx(cf, (void *)((cframe_unwind_ff(cf2) && errcode != LUA_YIELD) ?
429 lj_vm_unwind_ff_eh :
430 lj_vm_unwind_c_eh),
431 rec, (void *)(uintptr_t)errcode, ctx, dispatch->HistoryTable);
432 /* RtlUnwindEx should never return. */
433 }
434 }
435 return ExceptionContinueSearch;
436} 736}
437 737
438/* Raise Windows exception. */ 738#endif
439static void err_raise_ext(int errcode)
440{
441 RaiseException(LJ_EXCODE_MAKE(errcode), 1 /* EH_NONCONTINUABLE */, 0, NULL);
442}
443 739
444#endif 740#endif
445 741
@@ -450,22 +746,23 @@ LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode)
450{ 746{
451 global_State *g = G(L); 747 global_State *g = G(L);
452 lj_trace_abort(g); 748 lj_trace_abort(g);
453 setgcrefnull(g->jit_L); 749 L->status = LUA_OK;
454 L->status = 0;
455#if LJ_UNWIND_EXT 750#if LJ_UNWIND_EXT
456 err_raise_ext(errcode); 751 err_raise_ext(g, errcode);
457 /* 752 /*
458 ** A return from this function signals a corrupt C stack that cannot be 753 ** A return from this function signals a corrupt C stack that cannot be
459 ** unwound. We have no choice but to call the panic function and exit. 754 ** unwound. We have no choice but to call the panic function and exit.
460 ** 755 **
461 ** Usually this is caused by a C function without unwind information. 756 ** Usually this is caused by a C function without unwind information.
462 ** This should never happen on x64, but may happen if you've manually 757 ** This may happen if you've manually enabled LUAJIT_UNWIND_EXTERNAL
463 ** enabled LUAJIT_UNWIND_EXTERNAL and forgot to recompile *every* 758 ** and forgot to recompile *every* non-C++ file with -funwind-tables.
464 ** non-C++ file with -funwind-tables.
465 */ 759 */
466 if (G(L)->panic) 760 if (G(L)->panic)
467 G(L)->panic(L); 761 G(L)->panic(L);
468#else 762#else
763#if LJ_HASJIT
764 setmref(g->jit_base, NULL);
765#endif
469 { 766 {
470 void *cf = err_unwind(L, NULL, errcode); 767 void *cf = err_unwind(L, NULL, errcode);
471 if (cframe_unwind_ff(cf)) 768 if (cframe_unwind_ff(cf))
@@ -488,6 +785,10 @@ LJ_NOINLINE void lj_err_mem(lua_State *L)
488{ 785{
489 if (L->status == LUA_ERRERR+1) /* Don't touch the stack during lua_open. */ 786 if (L->status == LUA_ERRERR+1) /* Don't touch the stack during lua_open. */
490 lj_vm_unwind_c(L->cframe, LUA_ERRMEM); 787 lj_vm_unwind_c(L->cframe, LUA_ERRMEM);
788 if (LJ_HASJIT) {
789 TValue *base = tvref(G(L)->jit_base);
790 if (base) L->base = base;
791 }
491 if (curr_funcisL(L)) L->top = curr_topL(L); 792 if (curr_funcisL(L)) L->top = curr_topL(L);
492 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRMEM)); 793 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_ERRMEM));
493 lj_err_throw(L, LUA_ERRMEM); 794 lj_err_throw(L, LUA_ERRMEM);
@@ -496,7 +797,7 @@ LJ_NOINLINE void lj_err_mem(lua_State *L)
496/* Find error function for runtime errors. Requires an extra stack traversal. */ 797/* Find error function for runtime errors. Requires an extra stack traversal. */
497static ptrdiff_t finderrfunc(lua_State *L) 798static ptrdiff_t finderrfunc(lua_State *L)
498{ 799{
499 cTValue *frame = L->base-1, *bot = tvref(L->stack); 800 cTValue *frame = L->base-1, *bot = tvref(L->stack)+LJ_FR2;
500 void *cf = L->cframe; 801 void *cf = L->cframe;
501 while (frame > bot && cf) { 802 while (frame > bot && cf) {
502 while (cframe_nres(cframe_raw(cf)) < 0) { /* cframe without frame? */ 803 while (cframe_nres(cframe_raw(cf)) < 0) { /* cframe without frame? */
@@ -520,10 +821,8 @@ static ptrdiff_t finderrfunc(lua_State *L)
520 frame = frame_prevd(frame); 821 frame = frame_prevd(frame);
521 break; 822 break;
522 case FRAME_CONT: 823 case FRAME_CONT:
523#if LJ_HASFFI 824 if (frame_iscont_fficb(frame))
524 if ((frame-1)->u32.lo == LJ_CONT_FFI_CALLBACK)
525 cf = cframe_prev(cf); 825 cf = cframe_prev(cf);
526#endif
527 frame = frame_prevd(frame); 826 frame = frame_prevd(frame);
528 break; 827 break;
529 case FRAME_CP: 828 case FRAME_CP:
@@ -535,11 +834,11 @@ static ptrdiff_t finderrfunc(lua_State *L)
535 break; 834 break;
536 case FRAME_PCALL: 835 case FRAME_PCALL:
537 case FRAME_PCALLH: 836 case FRAME_PCALLH:
538 if (frame_ftsz(frame) >= (ptrdiff_t)(2*sizeof(TValue))) /* xpcall? */ 837 if (frame_func(frame_prevd(frame))->c.ffid == FF_xpcall)
539 return savestack(L, frame-1); /* Point to xpcall's errorfunc. */ 838 return savestack(L, frame_prevd(frame)+1); /* xpcall's errorfunc. */
540 return 0; 839 return 0;
541 default: 840 default:
542 lua_assert(0); 841 lj_assertL(0, "bad frame type");
543 return 0; 842 return 0;
544 } 843 }
545 } 844 }
@@ -549,7 +848,7 @@ static ptrdiff_t finderrfunc(lua_State *L)
549/* Runtime error. */ 848/* Runtime error. */
550LJ_NOINLINE void LJ_FASTCALL lj_err_run(lua_State *L) 849LJ_NOINLINE void LJ_FASTCALL lj_err_run(lua_State *L)
551{ 850{
552 ptrdiff_t ef = finderrfunc(L); 851 ptrdiff_t ef = (LJ_HASJIT && tvref(G(L)->jit_base)) ? 0 : finderrfunc(L);
553 if (ef) { 852 if (ef) {
554 TValue *errfunc = restorestack(L, ef); 853 TValue *errfunc = restorestack(L, ef);
555 TValue *top = L->top; 854 TValue *top = L->top;
@@ -559,22 +858,37 @@ LJ_NOINLINE void LJ_FASTCALL lj_err_run(lua_State *L)
559 lj_err_throw(L, LUA_ERRERR); 858 lj_err_throw(L, LUA_ERRERR);
560 } 859 }
561 L->status = LUA_ERRERR; 860 L->status = LUA_ERRERR;
562 copyTV(L, top, top-1); 861 copyTV(L, top+LJ_FR2, top-1);
563 copyTV(L, top-1, errfunc); 862 copyTV(L, top-1, errfunc);
863 if (LJ_FR2) setnilV(top++);
564 L->top = top+1; 864 L->top = top+1;
565 lj_vm_call(L, top, 1+1); /* Stack: |errfunc|msg| -> |msg| */ 865 lj_vm_call(L, top, 1+1); /* Stack: |errfunc|msg| -> |msg| */
566 } 866 }
567 lj_err_throw(L, LUA_ERRRUN); 867 lj_err_throw(L, LUA_ERRRUN);
568} 868}
569 869
870#if LJ_HASJIT
871LJ_NOINLINE void LJ_FASTCALL lj_err_trace(lua_State *L, int errcode)
872{
873 if (errcode == LUA_ERRRUN)
874 lj_err_run(L);
875 else
876 lj_err_throw(L, errcode);
877}
878#endif
879
570/* Formatted runtime error message. */ 880/* Formatted runtime error message. */
571LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...) 881LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...)
572{ 882{
573 const char *msg; 883 const char *msg;
574 va_list argp; 884 va_list argp;
575 va_start(argp, em); 885 va_start(argp, em);
886 if (LJ_HASJIT) {
887 TValue *base = tvref(G(L)->jit_base);
888 if (base) L->base = base;
889 }
576 if (curr_funcisL(L)) L->top = curr_topL(L); 890 if (curr_funcisL(L)) L->top = curr_topL(L);
577 msg = lj_str_pushvf(L, err2msg(em), argp); 891 msg = lj_strfmt_pushvf(L, err2msg(em), argp);
578 va_end(argp); 892 va_end(argp);
579 lj_debug_addloc(L, msg, L->base-1, NULL); 893 lj_debug_addloc(L, msg, L->base-1, NULL);
580 lj_err_run(L); 894 lj_err_run(L);
@@ -592,11 +906,11 @@ LJ_NOINLINE void lj_err_lex(lua_State *L, GCstr *src, const char *tok,
592{ 906{
593 char buff[LUA_IDSIZE]; 907 char buff[LUA_IDSIZE];
594 const char *msg; 908 const char *msg;
595 lj_debug_shortname(buff, src); 909 lj_debug_shortname(buff, src, line);
596 msg = lj_str_pushvf(L, err2msg(em), argp); 910 msg = lj_strfmt_pushvf(L, err2msg(em), argp);
597 msg = lj_str_pushf(L, "%s:%d: %s", buff, line, msg); 911 msg = lj_strfmt_pushf(L, "%s:%d: %s", buff, line, msg);
598 if (tok) 912 if (tok)
599 lj_str_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tok); 913 lj_strfmt_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tok);
600 lj_err_throw(L, LUA_ERRSYNTAX); 914 lj_err_throw(L, LUA_ERRSYNTAX);
601} 915}
602 916
@@ -635,8 +949,9 @@ LJ_NOINLINE void lj_err_optype_call(lua_State *L, TValue *o)
635 const BCIns *pc = cframe_Lpc(L); 949 const BCIns *pc = cframe_Lpc(L);
636 if (((ptrdiff_t)pc & FRAME_TYPE) != FRAME_LUA) { 950 if (((ptrdiff_t)pc & FRAME_TYPE) != FRAME_LUA) {
637 const char *tname = lj_typename(o); 951 const char *tname = lj_typename(o);
952 setframe_gc(o, obj2gco(L), LJ_TTHREAD);
953 if (LJ_FR2) o++;
638 setframe_pc(o, pc); 954 setframe_pc(o, pc);
639 setframe_gc(o, obj2gco(L));
640 L->top = L->base = o+1; 955 L->top = L->base = o+1;
641 err_msgv(L, LJ_ERR_BADCALL, tname); 956 err_msgv(L, LJ_ERR_BADCALL, tname);
642 } 957 }
@@ -646,28 +961,27 @@ LJ_NOINLINE void lj_err_optype_call(lua_State *L, TValue *o)
646/* Error in context of caller. */ 961/* Error in context of caller. */
647LJ_NOINLINE void lj_err_callermsg(lua_State *L, const char *msg) 962LJ_NOINLINE void lj_err_callermsg(lua_State *L, const char *msg)
648{ 963{
649 TValue *frame = L->base-1; 964 TValue *frame = NULL, *pframe = NULL;
650 TValue *pframe = NULL; 965 if (!(LJ_HASJIT && tvref(G(L)->jit_base))) {
651 if (frame_islua(frame)) { 966 frame = L->base-1;
652 pframe = frame_prevl(frame); 967 if (frame_islua(frame)) {
653 } else if (frame_iscont(frame)) { 968 pframe = frame_prevl(frame);
969 } else if (frame_iscont(frame)) {
970 if (frame_iscont_fficb(frame)) {
971 pframe = frame;
972 frame = NULL;
973 } else {
974 pframe = frame_prevd(frame);
654#if LJ_HASFFI 975#if LJ_HASFFI
655 if ((frame-1)->u32.lo == LJ_CONT_FFI_CALLBACK) { 976 /* Remove frame for FFI metamethods. */
656 pframe = frame; 977 if (frame_func(frame)->c.ffid >= FF_ffi_meta___index &&
657 frame = NULL; 978 frame_func(frame)->c.ffid <= FF_ffi_meta___tostring) {
658 } else 979 L->base = pframe+1;
980 L->top = frame;
981 setcframe_pc(cframe_raw(L->cframe), frame_contpc(frame));
982 }
659#endif 983#endif
660 {
661 pframe = frame_prevd(frame);
662#if LJ_HASFFI
663 /* Remove frame for FFI metamethods. */
664 if (frame_func(frame)->c.ffid >= FF_ffi_meta___index &&
665 frame_func(frame)->c.ffid <= FF_ffi_meta___tostring) {
666 L->base = pframe+1;
667 L->top = frame;
668 setcframe_pc(cframe_raw(L->cframe), frame_contpc(frame));
669 } 984 }
670#endif
671 } 985 }
672 } 986 }
673 lj_debug_addloc(L, msg, pframe, frame); 987 lj_debug_addloc(L, msg, pframe, frame);
@@ -680,7 +994,7 @@ LJ_NOINLINE void lj_err_callerv(lua_State *L, ErrMsg em, ...)
680 const char *msg; 994 const char *msg;
681 va_list argp; 995 va_list argp;
682 va_start(argp, em); 996 va_start(argp, em);
683 msg = lj_str_pushvf(L, err2msg(em), argp); 997 msg = lj_strfmt_pushvf(L, err2msg(em), argp);
684 va_end(argp); 998 va_end(argp);
685 lj_err_callermsg(L, msg); 999 lj_err_callermsg(L, msg);
686} 1000}
@@ -700,9 +1014,9 @@ LJ_NORET LJ_NOINLINE static void err_argmsg(lua_State *L, int narg,
700 if (narg < 0 && narg > LUA_REGISTRYINDEX) 1014 if (narg < 0 && narg > LUA_REGISTRYINDEX)
701 narg = (int)(L->top - L->base) + narg + 1; 1015 narg = (int)(L->top - L->base) + narg + 1;
702 if (ftype && ftype[3] == 'h' && --narg == 0) /* Check for "method". */ 1016 if (ftype && ftype[3] == 'h' && --narg == 0) /* Check for "method". */
703 msg = lj_str_pushf(L, err2msg(LJ_ERR_BADSELF), fname, msg); 1017 msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADSELF), fname, msg);
704 else 1018 else
705 msg = lj_str_pushf(L, err2msg(LJ_ERR_BADARG), narg, fname, msg); 1019 msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADARG), narg, fname, msg);
706 lj_err_callermsg(L, msg); 1020 lj_err_callermsg(L, msg);
707} 1021}
708 1022
@@ -712,7 +1026,7 @@ LJ_NOINLINE void lj_err_argv(lua_State *L, int narg, ErrMsg em, ...)
712 const char *msg; 1026 const char *msg;
713 va_list argp; 1027 va_list argp;
714 va_start(argp, em); 1028 va_start(argp, em);
715 msg = lj_str_pushvf(L, err2msg(em), argp); 1029 msg = lj_strfmt_pushvf(L, err2msg(em), argp);
716 va_end(argp); 1030 va_end(argp);
717 err_argmsg(L, narg, msg); 1031 err_argmsg(L, narg, msg);
718} 1032}
@@ -742,7 +1056,7 @@ LJ_NOINLINE void lj_err_argtype(lua_State *L, int narg, const char *xname)
742 TValue *o = narg < 0 ? L->top + narg : L->base + narg-1; 1056 TValue *o = narg < 0 ? L->top + narg : L->base + narg-1;
743 tname = o < L->top ? lj_typename(o) : lj_obj_typename[0]; 1057 tname = o < L->top ? lj_typename(o) : lj_obj_typename[0];
744 } 1058 }
745 msg = lj_str_pushf(L, err2msg(LJ_ERR_BADTYPE), xname, tname); 1059 msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADTYPE), xname, tname);
746 err_argmsg(L, narg, msg); 1060 err_argmsg(L, narg, msg);
747} 1061}
748 1062
@@ -792,7 +1106,7 @@ LUALIB_API int luaL_error(lua_State *L, const char *fmt, ...)
792 const char *msg; 1106 const char *msg;
793 va_list argp; 1107 va_list argp;
794 va_start(argp, fmt); 1108 va_start(argp, fmt);
795 msg = lj_str_pushvf(L, fmt, argp); 1109 msg = lj_strfmt_pushvf(L, fmt, argp);
796 va_end(argp); 1110 va_end(argp);
797 lj_err_callermsg(L, msg); 1111 lj_err_callermsg(L, msg);
798 return 0; /* unreachable */ 1112 return 0; /* unreachable */
diff --git a/src/lj_err.h b/src/lj_err.h
index 321719a9..8768fefd 100644
--- a/src/lj_err.h
+++ b/src/lj_err.h
@@ -23,7 +23,10 @@ LJ_DATA const char *lj_err_allmsg;
23LJ_FUNC GCstr *lj_err_str(lua_State *L, ErrMsg em); 23LJ_FUNC GCstr *lj_err_str(lua_State *L, ErrMsg em);
24LJ_FUNCA_NORET void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode); 24LJ_FUNCA_NORET void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode);
25LJ_FUNC_NORET void lj_err_mem(lua_State *L); 25LJ_FUNC_NORET void lj_err_mem(lua_State *L);
26LJ_FUNCA_NORET void LJ_FASTCALL lj_err_run(lua_State *L); 26LJ_FUNC_NORET void LJ_FASTCALL lj_err_run(lua_State *L);
27#if LJ_HASJIT
28LJ_FUNCA_NORET void LJ_FASTCALL lj_err_trace(lua_State *L, int errcode);
29#endif
27LJ_FUNC_NORET void lj_err_msg(lua_State *L, ErrMsg em); 30LJ_FUNC_NORET void lj_err_msg(lua_State *L, ErrMsg em);
28LJ_FUNC_NORET void lj_err_lex(lua_State *L, GCstr *src, const char *tok, 31LJ_FUNC_NORET void lj_err_lex(lua_State *L, GCstr *src, const char *tok,
29 BCLine line, ErrMsg em, va_list argp); 32 BCLine line, ErrMsg em, va_list argp);
@@ -38,4 +41,18 @@ LJ_FUNC_NORET void lj_err_argv(lua_State *L, int narg, ErrMsg em, ...);
38LJ_FUNC_NORET void lj_err_argtype(lua_State *L, int narg, const char *xname); 41LJ_FUNC_NORET void lj_err_argtype(lua_State *L, int narg, const char *xname);
39LJ_FUNC_NORET void lj_err_argt(lua_State *L, int narg, int tt); 42LJ_FUNC_NORET void lj_err_argt(lua_State *L, int narg, int tt);
40 43
44#if LJ_UNWIND_JIT && !LJ_ABI_WIN
45LJ_FUNC uint8_t *lj_err_register_mcode(void *base, size_t sz, uint8_t *info);
46LJ_FUNC void lj_err_deregister_mcode(void *base, size_t sz, uint8_t *info);
47#else
48#define lj_err_register_mcode(base, sz, info) (info)
49#define lj_err_deregister_mcode(base, sz, info) UNUSED(base)
50#endif
51
52#if LJ_UNWIND_EXT && !LJ_ABI_WIN && defined(LUA_USE_ASSERT)
53LJ_FUNC void lj_err_verify(void);
54#else
55#define lj_err_verify() ((void)0)
56#endif
57
41#endif 58#endif
diff --git a/src/lj_errmsg.h b/src/lj_errmsg.h
index 9c695ce5..127c06da 100644
--- a/src/lj_errmsg.h
+++ b/src/lj_errmsg.h
@@ -67,6 +67,7 @@ ERRDEF(PROTMT, "cannot change a protected metatable")
67ERRDEF(UNPACK, "too many results to unpack") 67ERRDEF(UNPACK, "too many results to unpack")
68ERRDEF(RDRSTR, "reader function must return a string") 68ERRDEF(RDRSTR, "reader function must return a string")
69ERRDEF(PRTOSTR, LUA_QL("tostring") " must return a string to " LUA_QL("print")) 69ERRDEF(PRTOSTR, LUA_QL("tostring") " must return a string to " LUA_QL("print"))
70ERRDEF(NUMRNG, "number out of range")
70ERRDEF(IDXRNG, "index out of range") 71ERRDEF(IDXRNG, "index out of range")
71ERRDEF(BASERNG, "base out of range") 72ERRDEF(BASERNG, "base out of range")
72ERRDEF(LVLRNG, "level out of range") 73ERRDEF(LVLRNG, "level out of range")
@@ -96,18 +97,12 @@ ERRDEF(STRPATX, "pattern too complex")
96ERRDEF(STRCAPI, "invalid capture index") 97ERRDEF(STRCAPI, "invalid capture index")
97ERRDEF(STRCAPN, "too many captures") 98ERRDEF(STRCAPN, "too many captures")
98ERRDEF(STRCAPU, "unfinished capture") 99ERRDEF(STRCAPU, "unfinished capture")
99ERRDEF(STRFMTO, "invalid option " LUA_QL("%%%c") " to " LUA_QL("format")) 100ERRDEF(STRFMT, "invalid option " LUA_QS " to " LUA_QL("format"))
100ERRDEF(STRFMTR, "invalid format (repeated flags)")
101ERRDEF(STRFMTW, "invalid format (width or precision too long)")
102ERRDEF(STRGSRV, "invalid replacement value (a %s)") 101ERRDEF(STRGSRV, "invalid replacement value (a %s)")
103ERRDEF(BADMODN, "name conflict for module " LUA_QS) 102ERRDEF(BADMODN, "name conflict for module " LUA_QS)
104#if LJ_HASJIT 103#if LJ_HASJIT
105ERRDEF(JITPROT, "runtime code generation failed, restricted kernel?") 104ERRDEF(JITPROT, "runtime code generation failed, restricted kernel?")
106#if LJ_TARGET_X86ORX64
107ERRDEF(NOJIT, "JIT compiler disabled, CPU does not support SSE2")
108#else
109ERRDEF(NOJIT, "JIT compiler disabled") 105ERRDEF(NOJIT, "JIT compiler disabled")
110#endif
111#elif defined(LJ_ARCH_NOJIT) 106#elif defined(LJ_ARCH_NOJIT)
112ERRDEF(NOJIT, "no JIT compiler for this architecture (yet)") 107ERRDEF(NOJIT, "no JIT compiler for this architecture (yet)")
113#else 108#else
@@ -118,7 +113,6 @@ ERRDEF(JITOPT, "unknown or malformed optimization flag " LUA_QS)
118/* Lexer/parser errors. */ 113/* Lexer/parser errors. */
119ERRDEF(XMODE, "attempt to load chunk with wrong mode") 114ERRDEF(XMODE, "attempt to load chunk with wrong mode")
120ERRDEF(XNEAR, "%s near " LUA_QS) 115ERRDEF(XNEAR, "%s near " LUA_QS)
121ERRDEF(XELEM, "lexical element too long")
122ERRDEF(XLINES, "chunk has too many lines") 116ERRDEF(XLINES, "chunk has too many lines")
123ERRDEF(XLEVELS, "chunk has too many syntax levels") 117ERRDEF(XLEVELS, "chunk has too many syntax levels")
124ERRDEF(XNUMBER, "malformed number") 118ERRDEF(XNUMBER, "malformed number")
@@ -186,6 +180,19 @@ ERRDEF(FFI_NYIPACKBIT, "NYI: packed bit fields")
186ERRDEF(FFI_NYICALL, "NYI: cannot call this C function (yet)") 180ERRDEF(FFI_NYICALL, "NYI: cannot call this C function (yet)")
187#endif 181#endif
188 182
183#if LJ_HASBUFFER
184/* String buffer errors. */
185ERRDEF(BUFFER_SELF, "cannot put buffer into itself")
186ERRDEF(BUFFER_BADOPT, "bad options table")
187ERRDEF(BUFFER_BADENC, "cannot serialize " LUA_QS)
188ERRDEF(BUFFER_BADDEC, "cannot deserialize tag 0x%02x")
189ERRDEF(BUFFER_BADDICTX, "cannot deserialize dictionary index %d")
190ERRDEF(BUFFER_DEPTH, "too deep to serialize")
191ERRDEF(BUFFER_DUPKEY, "duplicate table key")
192ERRDEF(BUFFER_EOB, "unexpected end of buffer")
193ERRDEF(BUFFER_LEFTOV, "left-over data in buffer")
194#endif
195
189#undef ERRDEF 196#undef ERRDEF
190 197
191/* Detecting unused error messages: 198/* Detecting unused error messages:
diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c
index 4127fe81..8ebf4165 100644
--- a/src/lj_ffrecord.c
+++ b/src/lj_ffrecord.c
@@ -11,6 +11,7 @@
11#if LJ_HASJIT 11#if LJ_HASJIT
12 12
13#include "lj_err.h" 13#include "lj_err.h"
14#include "lj_buf.h"
14#include "lj_str.h" 15#include "lj_str.h"
15#include "lj_tab.h" 16#include "lj_tab.h"
16#include "lj_frame.h" 17#include "lj_frame.h"
@@ -27,6 +28,8 @@
27#include "lj_dispatch.h" 28#include "lj_dispatch.h"
28#include "lj_vm.h" 29#include "lj_vm.h"
29#include "lj_strscan.h" 30#include "lj_strscan.h"
31#include "lj_strfmt.h"
32#include "lj_serialize.h"
30 33
31/* Some local macros to save typing. Undef'd at the end. */ 34/* Some local macros to save typing. Undef'd at the end. */
32#define IR(ref) (&J->cur.ir[(ref)]) 35#define IR(ref) (&J->cur.ir[(ref)])
@@ -79,10 +82,7 @@ static GCstr *argv2str(jit_State *J, TValue *o)
79 GCstr *s; 82 GCstr *s;
80 if (!tvisnumber(o)) 83 if (!tvisnumber(o))
81 lj_trace_err(J, LJ_TRERR_BADTYPE); 84 lj_trace_err(J, LJ_TRERR_BADTYPE);
82 if (tvisint(o)) 85 s = lj_strfmt_number(J->L, o);
83 s = lj_str_fromint(J->L, intV(o));
84 else
85 s = lj_str_fromnum(J->L, &o->n);
86 setstrV(J->L, o, s); 86 setstrV(J->L, o, s);
87 return s; 87 return s;
88 } 88 }
@@ -98,27 +98,102 @@ static ptrdiff_t results_wanted(jit_State *J)
98 return -1; 98 return -1;
99} 99}
100 100
101/* Throw error for unsupported variant of fast function. */ 101/* Trace stitching: add continuation below frame to start a new trace. */
102LJ_NORET static void recff_nyiu(jit_State *J) 102static void recff_stitch(jit_State *J)
103{ 103{
104 setfuncV(J->L, &J->errinfo, J->fn); 104 ASMFunction cont = lj_cont_stitch;
105 lj_trace_err_info(J, LJ_TRERR_NYIFFU); 105 lua_State *L = J->L;
106 TValue *base = L->base;
107 BCReg nslot = J->maxslot + 1 + LJ_FR2;
108 TValue *nframe = base + 1 + LJ_FR2;
109 const BCIns *pc = frame_pc(base-1);
110 TValue *pframe = frame_prevl(base-1);
111
112 /* Check for this now. Throwing in lj_record_stop messes up the stack. */
113 if (J->cur.nsnap >= (MSize)J->param[JIT_P_maxsnap])
114 lj_trace_err(J, LJ_TRERR_SNAPOV);
115
116 /* Move func + args up in Lua stack and insert continuation. */
117 memmove(&base[1], &base[-1-LJ_FR2], sizeof(TValue)*nslot);
118 setframe_ftsz(nframe, ((char *)nframe - (char *)pframe) + FRAME_CONT);
119 setcont(base-LJ_FR2, cont);
120 setframe_pc(base, pc);
121 setnilV(base-1-LJ_FR2); /* Incorrect, but rec_check_slots() won't run anymore. */
122 L->base += 2 + LJ_FR2;
123 L->top += 2 + LJ_FR2;
124
125 /* Ditto for the IR. */
126 memmove(&J->base[1], &J->base[-1-LJ_FR2], sizeof(TRef)*nslot);
127#if LJ_FR2
128 J->base[2] = TREF_FRAME;
129 J->base[-1] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
130 J->base[0] = lj_ir_k64(J, IR_KNUM, u64ptr(pc)) | TREF_CONT;
131#else
132 J->base[0] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
133#endif
134 J->ktrace = tref_ref((J->base[-1-LJ_FR2] = lj_ir_ktrace(J)));
135 J->base += 2 + LJ_FR2;
136 J->baseslot += 2 + LJ_FR2;
137 J->framedepth++;
138
139 lj_record_stop(J, LJ_TRLINK_STITCH, 0);
140
141 /* Undo Lua stack changes. */
142 memmove(&base[-1-LJ_FR2], &base[1], sizeof(TValue)*nslot);
143 setframe_pc(base-1, pc);
144 L->base -= 2 + LJ_FR2;
145 L->top -= 2 + LJ_FR2;
106} 146}
107 147
108/* Fallback handler for all fast functions that are not recorded (yet). */ 148/* Fallback handler for fast functions that are not recorded (yet). */
109static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd) 149static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd)
110{ 150{
111 setfuncV(J->L, &J->errinfo, J->fn); 151 if (J->cur.nins < (IRRef)J->param[JIT_P_minstitch] + REF_BASE) {
112 lj_trace_err_info(J, LJ_TRERR_NYIFF); 152 lj_trace_err_info(J, LJ_TRERR_TRACEUV);
113 UNUSED(rd); 153 } else {
154 /* Can only stitch from Lua call. */
155 if (J->framedepth && frame_islua(J->L->base-1)) {
156 BCOp op = bc_op(*frame_pc(J->L->base-1));
157 /* Stitched trace cannot start with *M op with variable # of args. */
158 if (!(op == BC_CALLM || op == BC_CALLMT ||
159 op == BC_RETM || op == BC_TSETM)) {
160 switch (J->fn->c.ffid) {
161 case FF_error:
162 case FF_debug_sethook:
163 case FF_jit_flush:
164 break; /* Don't stitch across special builtins. */
165 default:
166 recff_stitch(J); /* Use trace stitching. */
167 rd->nres = -1;
168 return;
169 }
170 }
171 }
172 /* Otherwise stop trace and return to interpreter. */
173 lj_record_stop(J, LJ_TRLINK_RETURN, 0);
174 rd->nres = -1;
175 }
114} 176}
115 177
116/* C functions can have arbitrary side-effects and are not recorded (yet). */ 178/* Fallback handler for unsupported variants of fast functions. */
117static void LJ_FASTCALL recff_c(jit_State *J, RecordFFData *rd) 179#define recff_nyiu recff_nyi
180
181/* Must stop the trace for classic C functions with arbitrary side-effects. */
182#define recff_c recff_nyi
183
184/* Emit BUFHDR for the global temporary buffer. */
185static TRef recff_bufhdr(jit_State *J)
118{ 186{
119 setfuncV(J->L, &J->errinfo, J->fn); 187 return emitir(IRT(IR_BUFHDR, IRT_PGC),
120 lj_trace_err_info(J, LJ_TRERR_NYICF); 188 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
121 UNUSED(rd); 189}
190
191/* Emit TMPREF. */
192static TRef recff_tmpref(jit_State *J, TRef tr, int mode)
193{
194 if (!LJ_DUALNUM && tref_isinteger(tr))
195 tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
196 return emitir(IRT(IR_TMPREF, IRT_PGC), tr, mode);
122} 197}
123 198
124/* -- Base library fast functions ----------------------------------------- */ 199/* -- Base library fast functions ----------------------------------------- */
@@ -135,7 +210,7 @@ static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
135 uint32_t t; 210 uint32_t t;
136 if (tvisnumber(&rd->argv[0])) 211 if (tvisnumber(&rd->argv[0]))
137 t = ~LJ_TNUMX; 212 t = ~LJ_TNUMX;
138 else if (LJ_64 && tvislightud(&rd->argv[0])) 213 else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0]))
139 t = ~LJ_TLIGHTUD; 214 t = ~LJ_TLIGHTUD;
140 else 215 else
141 t = ~itype(&rd->argv[0]); 216 t = ~itype(&rd->argv[0]);
@@ -167,7 +242,7 @@ static void LJ_FASTCALL recff_setmetatable(jit_State *J, RecordFFData *rd)
167 ix.tab = tr; 242 ix.tab = tr;
168 copyTV(J->L, &ix.tabv, &rd->argv[0]); 243 copyTV(J->L, &ix.tabv, &rd->argv[0]);
169 lj_record_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable. */ 244 lj_record_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable. */
170 fref = emitir(IRT(IR_FREF, IRT_P32), tr, IRFL_TAB_META); 245 fref = emitir(IRT(IR_FREF, IRT_PGC), tr, IRFL_TAB_META);
171 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt; 246 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt;
172 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref); 247 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref);
173 if (!tref_isnil(mt)) 248 if (!tref_isnil(mt))
@@ -220,7 +295,7 @@ static void LJ_FASTCALL recff_rawlen(jit_State *J, RecordFFData *rd)
220 if (tref_isstr(tr)) 295 if (tref_isstr(tr))
221 J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN); 296 J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN);
222 else if (tref_istab(tr)) 297 else if (tref_istab(tr))
223 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, tr); 298 J->base[0] = emitir(IRTI(IR_ALEN), tr, TREF_NIL);
224 /* else: Interpreter will throw. */ 299 /* else: Interpreter will throw. */
225 UNUSED(rd); 300 UNUSED(rd);
226} 301}
@@ -233,9 +308,9 @@ int32_t lj_ffrecord_select_mode(jit_State *J, TRef tr, TValue *tv)
233 if (strV(tv)->len == 1) { 308 if (strV(tv)->len == 1) {
234 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv))); 309 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv)));
235 } else { 310 } else {
236 TRef trptr = emitir(IRT(IR_STRREF, IRT_P32), tr, lj_ir_kint(J, 0)); 311 TRef trptr = emitir(IRT(IR_STRREF, IRT_PGC), tr, lj_ir_kint(J, 0));
237 TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY); 312 TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY);
238 emitir(IRTG(IR_EQ, IRT_INT), trchar, lj_ir_kint(J, '#')); 313 emitir(IRTGI(IR_EQ), trchar, lj_ir_kint(J, '#'));
239 } 314 }
240 return 0; 315 return 0;
241 } else { /* select(n, ...) */ 316 } else { /* select(n, ...) */
@@ -263,7 +338,8 @@ static void LJ_FASTCALL recff_select(jit_State *J, RecordFFData *rd)
263 J->base[i] = J->base[start+i]; 338 J->base[i] = J->base[start+i];
264 } /* else: Interpreter will throw. */ 339 } /* else: Interpreter will throw. */
265 } else { 340 } else {
266 recff_nyiu(J); 341 recff_nyiu(J, rd);
342 return;
267 } 343 }
268 } /* else: Interpreter will throw. */ 344 } /* else: Interpreter will throw. */
269} 345}
@@ -274,14 +350,18 @@ static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd)
274 TRef base = J->base[1]; 350 TRef base = J->base[1];
275 if (tr && !tref_isnil(base)) { 351 if (tr && !tref_isnil(base)) {
276 base = lj_opt_narrow_toint(J, base); 352 base = lj_opt_narrow_toint(J, base);
277 if (!tref_isk(base) || IR(tref_ref(base))->i != 10) 353 if (!tref_isk(base) || IR(tref_ref(base))->i != 10) {
278 recff_nyiu(J); 354 recff_nyiu(J, rd);
355 return;
356 }
279 } 357 }
280 if (tref_isnumber_str(tr)) { 358 if (tref_isnumber_str(tr)) {
281 if (tref_isstr(tr)) { 359 if (tref_isstr(tr)) {
282 TValue tmp; 360 TValue tmp;
283 if (!lj_strscan_num(strV(&rd->argv[0]), &tmp)) 361 if (!lj_strscan_num(strV(&rd->argv[0]), &tmp)) {
284 recff_nyiu(J); /* Would need an inverted STRTO for this case. */ 362 recff_nyiu(J, rd); /* Would need an inverted STRTO for this case. */
363 return;
364 }
285 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0); 365 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
286 } 366 }
287#if LJ_HASFFI 367#if LJ_HASFFI
@@ -313,10 +393,10 @@ static int recff_metacall(jit_State *J, RecordFFData *rd, MMS mm)
313 int errcode; 393 int errcode;
314 TValue argv0; 394 TValue argv0;
315 /* Temporarily insert metamethod below object. */ 395 /* Temporarily insert metamethod below object. */
316 J->base[1] = J->base[0]; 396 J->base[1+LJ_FR2] = J->base[0];
317 J->base[0] = ix.mobj; 397 J->base[0] = ix.mobj;
318 copyTV(J->L, &argv0, &rd->argv[0]); 398 copyTV(J->L, &argv0, &rd->argv[0]);
319 copyTV(J->L, &rd->argv[1], &rd->argv[0]); 399 copyTV(J->L, &rd->argv[1+LJ_FR2], &rd->argv[0]);
320 copyTV(J->L, &rd->argv[0], &ix.mobjv); 400 copyTV(J->L, &rd->argv[0], &ix.mobjv);
321 /* Need to protect lj_record_tailcall because it may throw. */ 401 /* Need to protect lj_record_tailcall because it may throw. */
322 errcode = lj_vm_cpcall(J->L, NULL, J, recff_metacall_cp); 402 errcode = lj_vm_cpcall(J->L, NULL, J, recff_metacall_cp);
@@ -336,13 +416,15 @@ static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd)
336 if (tref_isstr(tr)) { 416 if (tref_isstr(tr)) {
337 /* Ignore __tostring in the string base metatable. */ 417 /* Ignore __tostring in the string base metatable. */
338 /* Pass on result in J->base[0]. */ 418 /* Pass on result in J->base[0]. */
339 } else if (!recff_metacall(J, rd, MM_tostring)) { 419 } else if (tr && !recff_metacall(J, rd, MM_tostring)) {
340 if (tref_isnumber(tr)) { 420 if (tref_isnumber(tr)) {
341 J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0); 421 J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr,
422 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
342 } else if (tref_ispri(tr)) { 423 } else if (tref_ispri(tr)) {
343 J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[tref_type(tr)])); 424 J->base[0] = lj_ir_kstr(J, lj_strfmt_obj(J->L, &rd->argv[0]));
344 } else { 425 } else {
345 recff_nyiu(J); 426 recff_nyiu(J, rd);
427 return;
346 } 428 }
347 } 429 }
348} 430}
@@ -364,15 +446,15 @@ static void LJ_FASTCALL recff_ipairs_aux(jit_State *J, RecordFFData *rd)
364 } /* else: Interpreter will throw. */ 446 } /* else: Interpreter will throw. */
365} 447}
366 448
367static void LJ_FASTCALL recff_ipairs(jit_State *J, RecordFFData *rd) 449static void LJ_FASTCALL recff_xpairs(jit_State *J, RecordFFData *rd)
368{ 450{
369 TRef tr = J->base[0]; 451 TRef tr = J->base[0];
370 if (!((LJ_52 || (LJ_HASFFI && tref_iscdata(tr))) && 452 if (!((LJ_52 || (LJ_HASFFI && tref_iscdata(tr))) &&
371 recff_metacall(J, rd, MM_ipairs))) { 453 recff_metacall(J, rd, MM_pairs + rd->data))) {
372 if (tref_istab(tr)) { 454 if (tref_istab(tr)) {
373 J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0])); 455 J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0]));
374 J->base[1] = tr; 456 J->base[1] = tr;
375 J->base[2] = lj_ir_kint(J, 0); 457 J->base[2] = rd->data ? lj_ir_kint(J, 0) : TREF_NIL;
376 rd->nres = 3; 458 rd->nres = 3;
377 } /* else: Interpreter will throw. */ 459 } /* else: Interpreter will throw. */
378 } 460 }
@@ -381,8 +463,13 @@ static void LJ_FASTCALL recff_ipairs(jit_State *J, RecordFFData *rd)
381static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd) 463static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd)
382{ 464{
383 if (J->maxslot >= 1) { 465 if (J->maxslot >= 1) {
466#if LJ_FR2
467 /* Shift function arguments up. */
468 memmove(J->base + 1, J->base, sizeof(TRef) * J->maxslot);
469#endif
384 lj_record_call(J, 0, J->maxslot - 1); 470 lj_record_call(J, 0, J->maxslot - 1);
385 rd->nres = -1; /* Pending call. */ 471 rd->nres = -1; /* Pending call. */
472 J->needsnap = 1; /* Start catching on-trace errors. */
386 } /* else: Interpreter will throw. */ 473 } /* else: Interpreter will throw. */
387} 474}
388 475
@@ -406,6 +493,10 @@ static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd)
406 copyTV(J->L, &argv1, &rd->argv[1]); 493 copyTV(J->L, &argv1, &rd->argv[1]);
407 copyTV(J->L, &rd->argv[0], &argv1); 494 copyTV(J->L, &rd->argv[0], &argv1);
408 copyTV(J->L, &rd->argv[1], &argv0); 495 copyTV(J->L, &rd->argv[1], &argv0);
496#if LJ_FR2
497 /* Shift function arguments up. */
498 memmove(J->base + 2, J->base + 1, sizeof(TRef) * (J->maxslot-1));
499#endif
409 /* Need to protect lj_record_call because it may throw. */ 500 /* Need to protect lj_record_call because it may throw. */
410 errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp); 501 errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp);
411 /* Always undo Lua stack swap to avoid confusing the interpreter. */ 502 /* Always undo Lua stack swap to avoid confusing the interpreter. */
@@ -414,15 +505,62 @@ static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd)
414 if (errcode) 505 if (errcode)
415 lj_err_throw(J->L, errcode); /* Propagate errors. */ 506 lj_err_throw(J->L, errcode); /* Propagate errors. */
416 rd->nres = -1; /* Pending call. */ 507 rd->nres = -1; /* Pending call. */
508 J->needsnap = 1; /* Start catching on-trace errors. */
417 } /* else: Interpreter will throw. */ 509 } /* else: Interpreter will throw. */
418} 510}
419 511
512static void LJ_FASTCALL recff_getfenv(jit_State *J, RecordFFData *rd)
513{
514 TRef tr = J->base[0];
515 /* Only support getfenv(0) for now. */
516 if (tref_isint(tr) && tref_isk(tr) && IR(tref_ref(tr))->i == 0) {
517 TRef trl = emitir(IRT(IR_LREF, IRT_THREAD), 0, 0);
518 J->base[0] = emitir(IRT(IR_FLOAD, IRT_TAB), trl, IRFL_THREAD_ENV);
519 return;
520 }
521 recff_nyiu(J, rd);
522}
523
524static void LJ_FASTCALL recff_next(jit_State *J, RecordFFData *rd)
525{
526#if LJ_BE
527 /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
528 ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
529 */
530 recff_nyi(J, rd);
531#else
532 TRef tab = J->base[0];
533 if (tref_istab(tab)) {
534 RecordIndex ix;
535 cTValue *keyv;
536 ix.tab = tab;
537 if (tref_isnil(J->base[1])) { /* Shortcut for start of traversal. */
538 ix.key = lj_ir_kint(J, 0);
539 keyv = niltvg(J2G(J));
540 } else {
541 TRef tmp = recff_tmpref(J, J->base[1], IRTMPREF_IN1);
542 ix.key = lj_ir_call(J, IRCALL_lj_tab_keyindex, tab, tmp);
543 keyv = &rd->argv[1];
544 }
545 copyTV(J->L, &ix.tabv, &rd->argv[0]);
546 ix.keyv.u32.lo = lj_tab_keyindex(tabV(&ix.tabv), keyv);
547 /* Omit the value, if not used by the caller. */
548 ix.idxchain = (J->framedepth && frame_islua(J->L->base-1) &&
549 bc_b(frame_pc(J->L->base-1)[-1])-1 < 2);
550 ix.mobj = 0; /* We don't need the next index. */
551 rd->nres = lj_record_next(J, &ix);
552 J->base[0] = ix.key;
553 J->base[1] = ix.val;
554 } /* else: Interpreter will throw. */
555#endif
556}
557
420/* -- Math library fast functions ----------------------------------------- */ 558/* -- Math library fast functions ----------------------------------------- */
421 559
422static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd) 560static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd)
423{ 561{
424 TRef tr = lj_ir_tonum(J, J->base[0]); 562 TRef tr = lj_ir_tonum(J, J->base[0]);
425 J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J)); 563 J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_ksimd(J, LJ_KSIMD_ABS));
426 UNUSED(rd); 564 UNUSED(rd);
427} 565}
428 566
@@ -475,7 +613,7 @@ static void LJ_FASTCALL recff_math_atan2(jit_State *J, RecordFFData *rd)
475{ 613{
476 TRef tr = lj_ir_tonum(J, J->base[0]); 614 TRef tr = lj_ir_tonum(J, J->base[0]);
477 TRef tr2 = lj_ir_tonum(J, J->base[1]); 615 TRef tr2 = lj_ir_tonum(J, J->base[1]);
478 J->base[0] = emitir(IRTN(IR_ATAN2), tr, tr2); 616 J->base[0] = lj_ir_call(J, IRCALL_atan2, tr, tr2);
479 UNUSED(rd); 617 UNUSED(rd);
480} 618}
481 619
@@ -492,55 +630,16 @@ static void LJ_FASTCALL recff_math_ldexp(jit_State *J, RecordFFData *rd)
492 UNUSED(rd); 630 UNUSED(rd);
493} 631}
494 632
495/* Record math.asin, math.acos, math.atan. */ 633static void LJ_FASTCALL recff_math_call(jit_State *J, RecordFFData *rd)
496static void LJ_FASTCALL recff_math_atrig(jit_State *J, RecordFFData *rd)
497{
498 TRef y = lj_ir_tonum(J, J->base[0]);
499 TRef x = lj_ir_knum_one(J);
500 uint32_t ffid = rd->data;
501 if (ffid != FF_math_atan) {
502 TRef tmp = emitir(IRTN(IR_MUL), y, y);
503 tmp = emitir(IRTN(IR_SUB), x, tmp);
504 tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT);
505 if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; }
506 }
507 J->base[0] = emitir(IRTN(IR_ATAN2), y, x);
508}
509
510static void LJ_FASTCALL recff_math_htrig(jit_State *J, RecordFFData *rd)
511{ 634{
512 TRef tr = lj_ir_tonum(J, J->base[0]); 635 TRef tr = lj_ir_tonum(J, J->base[0]);
513 J->base[0] = emitir(IRTN(IR_CALLN), tr, rd->data); 636 J->base[0] = emitir(IRTN(IR_CALLN), tr, rd->data);
514} 637}
515 638
516static void LJ_FASTCALL recff_math_modf(jit_State *J, RecordFFData *rd)
517{
518 TRef tr = J->base[0];
519 if (tref_isinteger(tr)) {
520 J->base[0] = tr;
521 J->base[1] = lj_ir_kint(J, 0);
522 } else {
523 TRef trt;
524 tr = lj_ir_tonum(J, tr);
525 trt = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC);
526 J->base[0] = trt;
527 J->base[1] = emitir(IRTN(IR_SUB), tr, trt);
528 }
529 rd->nres = 2;
530}
531
532static void LJ_FASTCALL recff_math_degrad(jit_State *J, RecordFFData *rd)
533{
534 TRef tr = lj_ir_tonum(J, J->base[0]);
535 TRef trm = lj_ir_knum(J, numV(&J->fn->c.upvalue[0]));
536 J->base[0] = emitir(IRTN(IR_MUL), tr, trm);
537 UNUSED(rd);
538}
539
540static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd) 639static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd)
541{ 640{
542 J->base[0] = lj_opt_narrow_pow(J, J->base[0], J->base[1], 641 J->base[0] = lj_opt_narrow_arith(J, J->base[0], J->base[1],
543 &rd->argv[0], &rd->argv[1]); 642 &rd->argv[0], &rd->argv[1], IR_POW);
544 UNUSED(rd); 643 UNUSED(rd);
545} 644}
546 645
@@ -567,7 +666,7 @@ static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
567 GCudata *ud = udataV(&J->fn->c.upvalue[0]); 666 GCudata *ud = udataV(&J->fn->c.upvalue[0]);
568 TRef tr, one; 667 TRef tr, one;
569 lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */ 668 lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */
570 tr = lj_ir_call(J, IRCALL_lj_math_random_step, lj_ir_kptr(J, uddata(ud))); 669 tr = lj_ir_call(J, IRCALL_lj_prng_u64d, lj_ir_kptr(J, uddata(ud)));
571 one = lj_ir_knum_one(J); 670 one = lj_ir_knum_one(J);
572 tr = emitir(IRTN(IR_SUB), tr, one); 671 tr = emitir(IRTN(IR_SUB), tr, one);
573 if (J->base[0]) { 672 if (J->base[0]) {
@@ -591,48 +690,105 @@ static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
591 690
592/* -- Bit library fast functions ------------------------------------------ */ 691/* -- Bit library fast functions ------------------------------------------ */
593 692
594/* Record unary bit.tobit, bit.bnot, bit.bswap. */ 693/* Record bit.tobit. */
694static void LJ_FASTCALL recff_bit_tobit(jit_State *J, RecordFFData *rd)
695{
696 TRef tr = J->base[0];
697#if LJ_HASFFI
698 if (tref_iscdata(tr)) { recff_bit64_tobit(J, rd); return; }
699#endif
700 J->base[0] = lj_opt_narrow_tobit(J, tr);
701 UNUSED(rd);
702}
703
704/* Record unary bit.bnot, bit.bswap. */
595static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd) 705static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd)
596{ 706{
597 TRef tr = lj_opt_narrow_tobit(J, J->base[0]); 707#if LJ_HASFFI
598 J->base[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0); 708 if (recff_bit64_unary(J, rd))
709 return;
710#endif
711 J->base[0] = emitir(IRTI(rd->data), lj_opt_narrow_tobit(J, J->base[0]), 0);
599} 712}
600 713
601/* Record N-ary bit.band, bit.bor, bit.bxor. */ 714/* Record N-ary bit.band, bit.bor, bit.bxor. */
602static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd) 715static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd)
603{ 716{
604 TRef tr = lj_opt_narrow_tobit(J, J->base[0]); 717#if LJ_HASFFI
605 uint32_t op = rd->data; 718 if (recff_bit64_nary(J, rd))
606 BCReg i; 719 return;
607 for (i = 1; J->base[i] != 0; i++) 720#endif
608 tr = emitir(IRTI(op), tr, lj_opt_narrow_tobit(J, J->base[i])); 721 {
609 J->base[0] = tr; 722 TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
723 uint32_t ot = IRTI(rd->data);
724 BCReg i;
725 for (i = 1; J->base[i] != 0; i++)
726 tr = emitir(ot, tr, lj_opt_narrow_tobit(J, J->base[i]));
727 J->base[0] = tr;
728 }
610} 729}
611 730
612/* Record bit shifts. */ 731/* Record bit shifts. */
613static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd) 732static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd)
614{ 733{
615 TRef tr = lj_opt_narrow_tobit(J, J->base[0]); 734#if LJ_HASFFI
616 TRef tsh = lj_opt_narrow_tobit(J, J->base[1]); 735 if (recff_bit64_shift(J, rd))
617 IROp op = (IROp)rd->data; 736 return;
618 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) && 737#endif
619 !tref_isk(tsh)) 738 {
620 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31)); 739 TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
740 TRef tsh = lj_opt_narrow_tobit(J, J->base[1]);
741 IROp op = (IROp)rd->data;
742 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
743 !tref_isk(tsh))
744 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
621#ifdef LJ_TARGET_UNIFYROT 745#ifdef LJ_TARGET_UNIFYROT
622 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) { 746 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
623 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR; 747 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
624 tsh = emitir(IRTI(IR_NEG), tsh, tsh); 748 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
749 }
750#endif
751 J->base[0] = emitir(IRTI(op), tr, tsh);
625 } 752 }
753}
754
755static void LJ_FASTCALL recff_bit_tohex(jit_State *J, RecordFFData *rd)
756{
757#if LJ_HASFFI
758 TRef hdr = recff_bufhdr(J);
759 TRef tr = recff_bit64_tohex(J, rd, hdr);
760 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
761#else
762 recff_nyiu(J, rd); /* Don't bother working around this NYI. */
626#endif 763#endif
627 J->base[0] = emitir(IRTI(op), tr, tsh);
628} 764}
629 765
630/* -- String library fast functions --------------------------------------- */ 766/* -- String library fast functions --------------------------------------- */
631 767
632static void LJ_FASTCALL recff_string_len(jit_State *J, RecordFFData *rd) 768/* Specialize to relative starting position for string. */
769static TRef recff_string_start(jit_State *J, GCstr *s, int32_t *st, TRef tr,
770 TRef trlen, TRef tr0)
633{ 771{
634 J->base[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, J->base[0]), IRFL_STR_LEN); 772 int32_t start = *st;
635 UNUSED(rd); 773 if (start < 0) {
774 emitir(IRTGI(IR_LT), tr, tr0);
775 tr = emitir(IRTI(IR_ADD), trlen, tr);
776 start = start + (int32_t)s->len;
777 emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), tr, tr0);
778 if (start < 0) {
779 tr = tr0;
780 start = 0;
781 }
782 } else if (start == 0) {
783 emitir(IRTGI(IR_EQ), tr, tr0);
784 tr = tr0;
785 } else {
786 tr = emitir(IRTI(IR_ADD), tr, lj_ir_kint(J, -1));
787 emitir(IRTGI(IR_GE), tr, tr0);
788 start--;
789 }
790 *st = start;
791 return tr;
636} 792}
637 793
638/* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */ 794/* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
@@ -679,39 +835,21 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
679 } else if ((MSize)end <= str->len) { 835 } else if ((MSize)end <= str->len) {
680 emitir(IRTGI(IR_ULE), trend, trlen); 836 emitir(IRTGI(IR_ULE), trend, trlen);
681 } else { 837 } else {
682 emitir(IRTGI(IR_GT), trend, trlen); 838 emitir(IRTGI(IR_UGT), trend, trlen);
683 end = (int32_t)str->len; 839 end = (int32_t)str->len;
684 trend = trlen; 840 trend = trlen;
685 } 841 }
686 if (start < 0) { 842 trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
687 emitir(IRTGI(IR_LT), trstart, tr0);
688 trstart = emitir(IRTI(IR_ADD), trlen, trstart);
689 start = start+(int32_t)str->len;
690 emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), trstart, tr0);
691 if (start < 0) {
692 trstart = tr0;
693 start = 0;
694 }
695 } else {
696 if (start == 0) {
697 emitir(IRTGI(IR_EQ), trstart, tr0);
698 trstart = tr0;
699 } else {
700 trstart = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, -1));
701 emitir(IRTGI(IR_GE), trstart, tr0);
702 start--;
703 }
704 }
705 if (rd->data) { /* Return string.sub result. */ 843 if (rd->data) { /* Return string.sub result. */
706 if (end - start >= 0) { 844 if (end - start >= 0) {
707 /* Also handle empty range here, to avoid extra traces. */ 845 /* Also handle empty range here, to avoid extra traces. */
708 TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart); 846 TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart);
709 emitir(IRTGI(IR_GE), trslen, tr0); 847 emitir(IRTGI(IR_GE), trslen, tr0);
710 trptr = emitir(IRT(IR_STRREF, IRT_P32), trstr, trstart); 848 trptr = emitir(IRT(IR_STRREF, IRT_PGC), trstr, trstart);
711 J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen); 849 J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
712 } else { /* Range underflow: return empty string. */ 850 } else { /* Range underflow: return empty string. */
713 emitir(IRTGI(IR_LT), trend, trstart); 851 emitir(IRTGI(IR_LT), trend, trstart);
714 J->base[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0)); 852 J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
715 } 853 }
716 } else { /* Return string.byte result(s). */ 854 } else { /* Return string.byte result(s). */
717 ptrdiff_t i, len = end - start; 855 ptrdiff_t i, len = end - start;
@@ -723,7 +861,7 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
723 rd->nres = len; 861 rd->nres = len;
724 for (i = 0; i < len; i++) { 862 for (i = 0; i < len; i++) {
725 TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, (int32_t)i)); 863 TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, (int32_t)i));
726 tmp = emitir(IRT(IR_STRREF, IRT_P32), trstr, tmp); 864 tmp = emitir(IRT(IR_STRREF, IRT_PGC), trstr, tmp);
727 J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY); 865 J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY);
728 } 866 }
729 } else { /* Empty range or range underflow: return no results. */ 867 } else { /* Empty range or range underflow: return no results. */
@@ -733,48 +871,535 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
733 } 871 }
734} 872}
735 873
736/* -- Table library fast functions ---------------------------------------- */ 874static void LJ_FASTCALL recff_string_char(jit_State *J, RecordFFData *rd)
737
738static void LJ_FASTCALL recff_table_getn(jit_State *J, RecordFFData *rd)
739{ 875{
740 if (tref_istab(J->base[0])) 876 TRef k255 = lj_ir_kint(J, 255);
741 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, J->base[0]); 877 BCReg i;
742 /* else: Interpreter will throw. */ 878 for (i = 0; J->base[i] != 0; i++) { /* Convert char values to strings. */
879 TRef tr = lj_opt_narrow_toint(J, J->base[i]);
880 emitir(IRTGI(IR_ULE), tr, k255);
881 J->base[i] = emitir(IRT(IR_TOSTR, IRT_STR), tr, IRTOSTR_CHAR);
882 }
883 if (i > 1) { /* Concatenate the strings, if there's more than one. */
884 TRef hdr = recff_bufhdr(J), tr = hdr;
885 for (i = 0; J->base[i] != 0; i++)
886 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, J->base[i]);
887 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
888 } else if (i == 0) {
889 J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
890 }
743 UNUSED(rd); 891 UNUSED(rd);
744} 892}
745 893
746static void LJ_FASTCALL recff_table_remove(jit_State *J, RecordFFData *rd) 894static void LJ_FASTCALL recff_string_rep(jit_State *J, RecordFFData *rd)
747{ 895{
748 TRef tab = J->base[0]; 896 TRef str = lj_ir_tostr(J, J->base[0]);
749 rd->nres = 0; 897 TRef rep = lj_opt_narrow_toint(J, J->base[1]);
750 if (tref_istab(tab)) { 898 TRef hdr, tr, str2 = 0;
751 if (tref_isnil(J->base[1])) { /* Simple pop: t[#t] = nil */ 899 if (!tref_isnil(J->base[2])) {
752 TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, tab); 900 TRef sep = lj_ir_tostr(J, J->base[2]);
753 GCtab *t = tabV(&rd->argv[0]); 901 int32_t vrep = argv2int(J, &rd->argv[1]);
754 MSize len = lj_tab_len(t); 902 emitir(IRTGI(vrep > 1 ? IR_GT : IR_LE), rep, lj_ir_kint(J, 1));
755 emitir(IRTGI(len ? IR_NE : IR_EQ), trlen, lj_ir_kint(J, 0)); 903 if (vrep > 1) {
756 if (len) { 904 TRef hdr2 = recff_bufhdr(J);
757 RecordIndex ix; 905 TRef tr2 = emitir(IRTG(IR_BUFPUT, IRT_PGC), hdr2, sep);
758 ix.tab = tab; 906 tr2 = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr2, str);
759 ix.key = trlen; 907 str2 = emitir(IRTG(IR_BUFSTR, IRT_STR), tr2, hdr2);
760 settabV(J->L, &ix.tabv, t); 908 }
761 setintV(&ix.keyv, len); 909 }
762 ix.idxchain = 0; 910 tr = hdr = recff_bufhdr(J);
763 if (results_wanted(J) != 0) { /* Specialize load only if needed. */ 911 if (str2) {
764 ix.val = 0; 912 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, str);
765 J->base[0] = lj_record_idx(J, &ix); /* Load previous value. */ 913 str = str2;
766 rd->nres = 1; 914 rep = emitir(IRTI(IR_ADD), rep, lj_ir_kint(J, -1));
767 /* Assumes ix.key/ix.tab is not modified for raw lj_record_idx(). */ 915 }
916 tr = lj_ir_call(J, IRCALL_lj_buf_putstr_rep, tr, str, rep);
917 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
918}
919
920static void LJ_FASTCALL recff_string_op(jit_State *J, RecordFFData *rd)
921{
922 TRef str = lj_ir_tostr(J, J->base[0]);
923 TRef hdr = recff_bufhdr(J);
924 TRef tr = lj_ir_call(J, rd->data, hdr, str);
925 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
926}
927
928static void LJ_FASTCALL recff_string_find(jit_State *J, RecordFFData *rd)
929{
930 TRef trstr = lj_ir_tostr(J, J->base[0]);
931 TRef trpat = lj_ir_tostr(J, J->base[1]);
932 TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
933 TRef tr0 = lj_ir_kint(J, 0);
934 TRef trstart;
935 GCstr *str = argv2str(J, &rd->argv[0]);
936 GCstr *pat = argv2str(J, &rd->argv[1]);
937 int32_t start;
938 J->needsnap = 1;
939 if (tref_isnil(J->base[2])) {
940 trstart = lj_ir_kint(J, 1);
941 start = 1;
942 } else {
943 trstart = lj_opt_narrow_toint(J, J->base[2]);
944 start = argv2int(J, &rd->argv[2]);
945 }
946 trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
947 if ((MSize)start <= str->len) {
948 emitir(IRTGI(IR_ULE), trstart, trlen);
949 } else {
950 emitir(IRTGI(IR_UGT), trstart, trlen);
951#if LJ_52
952 J->base[0] = TREF_NIL;
953 return;
954#else
955 trstart = trlen;
956 start = str->len;
957#endif
958 }
959 /* Fixed arg or no pattern matching chars? (Specialized to pattern string.) */
960 if ((J->base[2] && tref_istruecond(J->base[3])) ||
961 (emitir(IRTG(IR_EQ, IRT_STR), trpat, lj_ir_kstr(J, pat)),
962 !lj_str_haspattern(pat))) { /* Search for fixed string. */
963 TRef trsptr = emitir(IRT(IR_STRREF, IRT_PGC), trstr, trstart);
964 TRef trpptr = emitir(IRT(IR_STRREF, IRT_PGC), trpat, tr0);
965 TRef trslen = emitir(IRTI(IR_SUB), trlen, trstart);
966 TRef trplen = emitir(IRTI(IR_FLOAD), trpat, IRFL_STR_LEN);
967 TRef tr = lj_ir_call(J, IRCALL_lj_str_find, trsptr, trpptr, trslen, trplen);
968 TRef trp0 = lj_ir_kkptr(J, NULL);
969 if (lj_str_find(strdata(str)+(MSize)start, strdata(pat),
970 str->len-(MSize)start, pat->len)) {
971 TRef pos;
972 emitir(IRTG(IR_NE, IRT_PGC), tr, trp0);
973 /* Recompute offset. trsptr may not point into trstr after folding. */
974 pos = emitir(IRTI(IR_ADD), emitir(IRTI(IR_SUB), tr, trsptr), trstart);
975 J->base[0] = emitir(IRTI(IR_ADD), pos, lj_ir_kint(J, 1));
976 J->base[1] = emitir(IRTI(IR_ADD), pos, trplen);
977 rd->nres = 2;
978 } else {
979 emitir(IRTG(IR_EQ, IRT_PGC), tr, trp0);
980 J->base[0] = TREF_NIL;
981 }
982 } else { /* Search for pattern. */
983 recff_nyiu(J, rd);
984 return;
985 }
986}
987
988static void recff_format(jit_State *J, RecordFFData *rd, TRef hdr, int sbufx)
989{
990 ptrdiff_t arg = sbufx;
991 TRef tr = hdr, trfmt = lj_ir_tostr(J, J->base[arg]);
992 GCstr *fmt = argv2str(J, &rd->argv[arg]);
993 FormatState fs;
994 SFormat sf;
995 /* Specialize to the format string. */
996 emitir(IRTG(IR_EQ, IRT_STR), trfmt, lj_ir_kstr(J, fmt));
997 lj_strfmt_init(&fs, strdata(fmt), fmt->len);
998 while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) { /* Parse format. */
999 TRef tra = sf == STRFMT_LIT ? 0 : J->base[++arg];
1000 TRef trsf = lj_ir_kint(J, (int32_t)sf);
1001 IRCallID id;
1002 switch (STRFMT_TYPE(sf)) {
1003 case STRFMT_LIT:
1004 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1005 lj_ir_kstr(J, lj_str_new(J->L, fs.str, fs.len)));
1006 break;
1007 case STRFMT_INT:
1008 id = IRCALL_lj_strfmt_putfnum_int;
1009 handle_int:
1010 if (!tref_isinteger(tra)) {
1011#if LJ_HASFFI
1012 if (tref_iscdata(tra)) {
1013 tra = lj_crecord_loadiu64(J, tra, &rd->argv[arg]);
1014 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra);
1015 break;
768 } 1016 }
769 ix.val = TREF_NIL; 1017#endif
770 lj_record_idx(J, &ix); /* Remove value. */ 1018 goto handle_num;
1019 }
1020 if (sf == STRFMT_INT) { /* Shortcut for plain %d. */
1021 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1022 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_INT));
1023 } else {
1024#if LJ_HASFFI
1025 tra = emitir(IRT(IR_CONV, IRT_U64), tra,
1026 (IRT_INT|(IRT_U64<<5)|IRCONV_SEXT));
1027 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra);
1028 lj_needsplit(J);
1029#else
1030 recff_nyiu(J, rd); /* Don't bother working around this NYI. */
1031 return;
1032#endif
771 } 1033 }
772 } else { /* Complex case: remove in the middle. */ 1034 break;
773 recff_nyiu(J); 1035 case STRFMT_UINT:
1036 id = IRCALL_lj_strfmt_putfnum_uint;
1037 goto handle_int;
1038 case STRFMT_NUM:
1039 id = IRCALL_lj_strfmt_putfnum;
1040 handle_num:
1041 tra = lj_ir_tonum(J, tra);
1042 tr = lj_ir_call(J, id, tr, trsf, tra);
1043 if (LJ_SOFTFP32) lj_needsplit(J);
1044 break;
1045 case STRFMT_STR:
1046 if (!tref_isstr(tra)) {
1047 recff_nyiu(J, rd); /* NYI: __tostring and non-string types for %s. */
1048 /* NYI: also buffers. */
1049 return;
1050 }
1051 if (sf == STRFMT_STR) /* Shortcut for plain %s. */
1052 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, tra);
1053 else if ((sf & STRFMT_T_QUOTED))
1054 tr = lj_ir_call(J, IRCALL_lj_strfmt_putquoted, tr, tra);
1055 else
1056 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfstr, tr, trsf, tra);
1057 break;
1058 case STRFMT_CHAR:
1059 tra = lj_opt_narrow_toint(J, tra);
1060 if (sf == STRFMT_CHAR) /* Shortcut for plain %c. */
1061 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1062 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_CHAR));
1063 else
1064 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfchar, tr, trsf, tra);
1065 break;
1066 case STRFMT_PTR: /* NYI */
1067 case STRFMT_ERR:
1068 default:
1069 recff_nyiu(J, rd);
1070 return;
1071 }
1072 }
1073 if (sbufx) {
1074 emitir(IRT(IR_USE, IRT_NIL), tr, 0);
1075 } else {
1076 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
1077 }
1078}
1079
1080static void LJ_FASTCALL recff_string_format(jit_State *J, RecordFFData *rd)
1081{
1082 recff_format(J, rd, recff_bufhdr(J), 0);
1083}
1084
1085/* -- Buffer library fast functions --------------------------------------- */
1086
1087#if LJ_HASBUFFER
1088
1089static LJ_AINLINE TRef recff_sbufx_get_L(jit_State *J, TRef ud)
1090{
1091 return emitir(IRT(IR_FLOAD, IRT_PGC), ud, IRFL_SBUF_L);
1092}
1093
1094static LJ_AINLINE void recff_sbufx_set_L(jit_State *J, TRef ud, TRef val)
1095{
1096 TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ud, IRFL_SBUF_L);
1097 emitir(IRT(IR_FSTORE, IRT_PGC), fref, val);
1098}
1099
1100static LJ_AINLINE TRef recff_sbufx_get_ptr(jit_State *J, TRef ud, IRFieldID fl)
1101{
1102 return emitir(IRT(IR_FLOAD, IRT_PTR), ud, fl);
1103}
1104
1105static LJ_AINLINE void recff_sbufx_set_ptr(jit_State *J, TRef ud, IRFieldID fl, TRef val)
1106{
1107 TRef fref = emitir(IRT(IR_FREF, IRT_PTR), ud, fl);
1108 emitir(IRT(IR_FSTORE, IRT_PTR), fref, val);
1109}
1110
1111static LJ_AINLINE TRef recff_sbufx_len(jit_State *J, TRef trr, TRef trw)
1112{
1113 TRef len = emitir(IRT(IR_SUB, IRT_INTP), trw, trr);
1114 if (LJ_64)
1115 len = emitir(IRTI(IR_CONV), len, (IRT_INT<<5)|IRT_INTP|IRCONV_NONE);
1116 return len;
1117}
1118
1119/* Emit typecheck for string buffer. */
1120static TRef recff_sbufx_check(jit_State *J, RecordFFData *rd, ptrdiff_t arg)
1121{
1122 TRef trtype, ud = J->base[arg];
1123 if (!tvisbuf(&rd->argv[arg])) lj_trace_err(J, LJ_TRERR_BADTYPE);
1124 trtype = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
1125 emitir(IRTGI(IR_EQ), trtype, lj_ir_kint(J, UDTYPE_BUFFER));
1126 J->needsnap = 1;
1127 return ud;
1128}
1129
1130/* Emit BUFHDR for write to extended string buffer. */
1131static TRef recff_sbufx_write(jit_State *J, TRef ud)
1132{
1133 TRef trbuf = emitir(IRT(IR_ADD, IRT_PGC), ud, lj_ir_kint(J, sizeof(GCudata)));
1134 return emitir(IRT(IR_BUFHDR, IRT_PGC), trbuf, IRBUFHDR_WRITE);
1135}
1136
1137/* Check for integer in range for the buffer API. */
1138static TRef recff_sbufx_checkint(jit_State *J, RecordFFData *rd, ptrdiff_t arg)
1139{
1140 TRef tr = J->base[arg];
1141 TRef trlim = lj_ir_kint(J, LJ_MAX_BUF);
1142 if (tref_isinteger(tr)) {
1143 emitir(IRTGI(IR_ULE), tr, trlim);
1144 } else if (tref_isnum(tr)) {
1145 tr = emitir(IRTI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_ANY);
1146 emitir(IRTGI(IR_ULE), tr, trlim);
1147#if LJ_HASFFI
1148 } else if (tref_iscdata(tr)) {
1149 tr = lj_crecord_loadiu64(J, tr, &rd->argv[arg]);
1150 emitir(IRTG(IR_ULE, IRT_U64), tr, lj_ir_kint64(J, LJ_MAX_BUF));
1151 tr = emitir(IRTI(IR_CONV), tr, (IRT_INT<<5)|IRT_I64|IRCONV_NONE);
1152#else
1153 UNUSED(rd);
1154#endif
1155 } else {
1156 lj_trace_err(J, LJ_TRERR_BADTYPE);
1157 }
1158 return tr;
1159}
1160
1161static void LJ_FASTCALL recff_buffer_method_reset(jit_State *J, RecordFFData *rd)
1162{
1163 TRef ud = recff_sbufx_check(J, rd, 0);
1164 SBufExt *sbx = bufV(&rd->argv[0]);
1165 int iscow = (int)sbufiscow(sbx);
1166 TRef trl = recff_sbufx_get_L(J, ud);
1167 TRef trcow = emitir(IRT(IR_BAND, IRT_IGC), trl, lj_ir_kint(J, SBUF_FLAG_COW));
1168 TRef zero = lj_ir_kint(J, 0);
1169 emitir(IRTG(iscow ? IR_NE : IR_EQ, IRT_IGC), trcow, zero);
1170 if (iscow) {
1171 trl = emitir(IRT(IR_BXOR, IRT_IGC), trl,
1172 LJ_GC64 ? lj_ir_kint64(J, SBUF_FLAG_COW) :
1173 lj_ir_kint(J, SBUF_FLAG_COW));
1174 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, zero);
1175 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_E, zero);
1176 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_B, zero);
1177 recff_sbufx_set_L(J, ud, trl);
1178 emitir(IRT(IR_FSTORE, IRT_PGC),
1179 emitir(IRT(IR_FREF, IRT_PGC), ud, IRFL_SBUF_REF), zero);
1180 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, zero);
1181 } else {
1182 TRef trb = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_B);
1183 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, trb);
1184 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trb);
1185 }
1186}
1187
1188static void LJ_FASTCALL recff_buffer_method_skip(jit_State *J, RecordFFData *rd)
1189{
1190 TRef ud = recff_sbufx_check(J, rd, 0);
1191 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1192 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1193 TRef len = recff_sbufx_len(J, trr, trw);
1194 TRef trn = recff_sbufx_checkint(J, rd, 1);
1195 len = emitir(IRTI(IR_MIN), len, trn);
1196 trr = emitir(IRT(IR_ADD, IRT_PTR), trr, len);
1197 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1198}
1199
1200static void LJ_FASTCALL recff_buffer_method_set(jit_State *J, RecordFFData *rd)
1201{
1202 TRef ud = recff_sbufx_check(J, rd, 0);
1203 TRef trbuf = recff_sbufx_write(J, ud);
1204 TRef tr = J->base[1];
1205 if (tref_isstr(tr)) {
1206 TRef trp = emitir(IRT(IR_STRREF, IRT_PGC), tr, lj_ir_kint(J, 0));
1207 TRef len = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN);
1208 lj_ir_call(J, IRCALL_lj_bufx_set, trbuf, trp, len, tr);
1209#if LJ_HASFFI
1210 } else if (tref_iscdata(tr)) {
1211 TRef trp = lj_crecord_topcvoid(J, tr, &rd->argv[1]);
1212 TRef len = recff_sbufx_checkint(J, rd, 2);
1213 lj_ir_call(J, IRCALL_lj_bufx_set, trbuf, trp, len, tr);
1214#endif
1215 } /* else: Interpreter will throw. */
1216}
1217
1218static void LJ_FASTCALL recff_buffer_method_put(jit_State *J, RecordFFData *rd)
1219{
1220 TRef ud = recff_sbufx_check(J, rd, 0);
1221 TRef trbuf = recff_sbufx_write(J, ud);
1222 TRef tr;
1223 ptrdiff_t arg;
1224 if (!J->base[1]) return;
1225 for (arg = 1; (tr = J->base[arg]); arg++) {
1226 if (tref_isudata(tr)) {
1227 TRef ud2 = recff_sbufx_check(J, rd, arg);
1228 emitir(IRTG(IR_NE, IRT_PGC), ud, ud2);
1229 }
1230 }
1231 for (arg = 1; (tr = J->base[arg]); arg++) {
1232 if (tref_isstr(tr)) {
1233 trbuf = emitir(IRTG(IR_BUFPUT, IRT_PGC), trbuf, tr);
1234 } else if (tref_isnumber(tr)) {
1235 trbuf = emitir(IRTG(IR_BUFPUT, IRT_PGC), trbuf,
1236 emitir(IRT(IR_TOSTR, IRT_STR), tr,
1237 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT));
1238 } else if (tref_isudata(tr)) {
1239 TRef trr = recff_sbufx_get_ptr(J, tr, IRFL_SBUF_R);
1240 TRef trw = recff_sbufx_get_ptr(J, tr, IRFL_SBUF_W);
1241 TRef len = recff_sbufx_len(J, trr, trw);
1242 trbuf = lj_ir_call(J, IRCALL_lj_buf_putmem, trbuf, trr, len);
1243 } else {
1244 recff_nyiu(J, rd);
1245 }
1246 }
1247 emitir(IRT(IR_USE, IRT_NIL), trbuf, 0);
1248}
1249
1250static void LJ_FASTCALL recff_buffer_method_putf(jit_State *J, RecordFFData *rd)
1251{
1252 TRef ud = recff_sbufx_check(J, rd, 0);
1253 TRef trbuf = recff_sbufx_write(J, ud);
1254 recff_format(J, rd, trbuf, 1);
1255}
1256
1257static void LJ_FASTCALL recff_buffer_method_get(jit_State *J, RecordFFData *rd)
1258{
1259 TRef ud = recff_sbufx_check(J, rd, 0);
1260 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1261 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1262 TRef tr;
1263 ptrdiff_t arg;
1264 if (!J->base[1]) { J->base[1] = TREF_NIL; J->base[2] = 0; }
1265 for (arg = 0; (tr = J->base[arg+1]); arg++) {
1266 if (!tref_isnil(tr)) {
1267 J->base[arg+1] = recff_sbufx_checkint(J, rd, arg+1);
774 } 1268 }
1269 }
1270 for (arg = 0; (tr = J->base[arg+1]); arg++) {
1271 TRef len = recff_sbufx_len(J, trr, trw);
1272 if (tref_isnil(tr)) {
1273 J->base[arg] = emitir(IRT(IR_XSNEW, IRT_STR), trr, len);
1274 trr = trw;
1275 } else {
1276 TRef tru;
1277 len = emitir(IRTI(IR_MIN), len, tr);
1278 tru = emitir(IRT(IR_ADD, IRT_PTR), trr, len);
1279 J->base[arg] = emitir(IRT(IR_XSNEW, IRT_STR), trr, len);
1280 trr = tru; /* Doing the ADD before the SNEW generates better code. */
1281 }
1282 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1283 }
1284 rd->nres = arg;
1285}
1286
1287static void LJ_FASTCALL recff_buffer_method___tostring(jit_State *J, RecordFFData *rd)
1288{
1289 TRef ud = recff_sbufx_check(J, rd, 0);
1290 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1291 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1292 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), trr, recff_sbufx_len(J, trr, trw));
1293}
1294
1295static void LJ_FASTCALL recff_buffer_method___len(jit_State *J, RecordFFData *rd)
1296{
1297 TRef ud = recff_sbufx_check(J, rd, 0);
1298 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1299 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1300 J->base[0] = recff_sbufx_len(J, trr, trw);
1301}
1302
1303#if LJ_HASFFI
1304static void LJ_FASTCALL recff_buffer_method_putcdata(jit_State *J, RecordFFData *rd)
1305{
1306 TRef ud = recff_sbufx_check(J, rd, 0);
1307 TRef trbuf = recff_sbufx_write(J, ud);
1308 TRef tr = lj_crecord_topcvoid(J, J->base[1], &rd->argv[1]);
1309 TRef len = recff_sbufx_checkint(J, rd, 2);
1310 trbuf = lj_ir_call(J, IRCALL_lj_buf_putmem, trbuf, tr, len);
1311 emitir(IRT(IR_USE, IRT_NIL), trbuf, 0);
1312}
1313
1314static void LJ_FASTCALL recff_buffer_method_reserve(jit_State *J, RecordFFData *rd)
1315{
1316 TRef ud = recff_sbufx_check(J, rd, 0);
1317 TRef trbuf = recff_sbufx_write(J, ud);
1318 TRef trsz = recff_sbufx_checkint(J, rd, 1);
1319 J->base[1] = lj_ir_call(J, IRCALL_lj_bufx_more, trbuf, trsz);
1320 J->base[0] = lj_crecord_topuint8(J, recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W));
1321 rd->nres = 2;
1322}
1323
1324static void LJ_FASTCALL recff_buffer_method_commit(jit_State *J, RecordFFData *rd)
1325{
1326 TRef ud = recff_sbufx_check(J, rd, 0);
1327 TRef len = recff_sbufx_checkint(J, rd, 1);
1328 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1329 TRef tre = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_E);
1330 TRef left = emitir(IRT(IR_SUB, IRT_INTP), tre, trw);
1331 if (LJ_64)
1332 left = emitir(IRTI(IR_CONV), left, (IRT_INT<<5)|IRT_INTP|IRCONV_NONE);
1333 emitir(IRTGI(IR_ULE), len, left);
1334 trw = emitir(IRT(IR_ADD, IRT_PTR), trw, len);
1335 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, trw);
1336}
1337
1338static void LJ_FASTCALL recff_buffer_method_ref(jit_State *J, RecordFFData *rd)
1339{
1340 TRef ud = recff_sbufx_check(J, rd, 0);
1341 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1342 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1343 J->base[0] = lj_crecord_topuint8(J, trr);
1344 J->base[1] = recff_sbufx_len(J, trr, trw);
1345 rd->nres = 2;
1346}
1347#endif
1348
1349static void LJ_FASTCALL recff_buffer_method_encode(jit_State *J, RecordFFData *rd)
1350{
1351 TRef ud = recff_sbufx_check(J, rd, 0);
1352 TRef trbuf = recff_sbufx_write(J, ud);
1353 TRef tmp = recff_tmpref(J, J->base[1], IRTMPREF_IN1);
1354 lj_ir_call(J, IRCALL_lj_serialize_put, trbuf, tmp);
1355 /* No IR_USE needed, since the call is a store. */
1356}
1357
1358static void LJ_FASTCALL recff_buffer_method_decode(jit_State *J, RecordFFData *rd)
1359{
1360 TRef ud = recff_sbufx_check(J, rd, 0);
1361 TRef trbuf = recff_sbufx_write(J, ud);
1362 TRef tmp = recff_tmpref(J, TREF_NIL, IRTMPREF_OUT1);
1363 TRef trr = lj_ir_call(J, IRCALL_lj_serialize_get, trbuf, tmp);
1364 IRType t = (IRType)lj_serialize_peektype(bufV(&rd->argv[0]));
1365 /* No IR_USE needed, since the call is a store. */
1366 J->base[0] = lj_record_vload(J, tmp, 0, t);
1367 /* The sbx->r store must be after the VLOAD type check, in case it fails. */
1368 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1369}
1370
1371static void LJ_FASTCALL recff_buffer_encode(jit_State *J, RecordFFData *rd)
1372{
1373 TRef tmp = recff_tmpref(J, J->base[0], IRTMPREF_IN1);
1374 J->base[0] = lj_ir_call(J, IRCALL_lj_serialize_encode, tmp);
1375 /* IR_USE needed for IR_CALLA, because the encoder may throw non-OOM. */
1376 emitir(IRT(IR_USE, IRT_NIL), J->base[0], 0);
1377 UNUSED(rd);
1378}
1379
1380static void LJ_FASTCALL recff_buffer_decode(jit_State *J, RecordFFData *rd)
1381{
1382 if (tvisstr(&rd->argv[0])) {
1383 GCstr *str = strV(&rd->argv[0]);
1384 SBufExt sbx;
1385 IRType t;
1386 TRef tmp = recff_tmpref(J, TREF_NIL, IRTMPREF_OUT1);
1387 TRef tr = lj_ir_call(J, IRCALL_lj_serialize_decode, tmp, J->base[0]);
1388 /* IR_USE needed for IR_CALLA, because the decoder may throw non-OOM.
1389 ** That's why IRCALL_lj_serialize_decode needs a fake INT result.
1390 */
1391 emitir(IRT(IR_USE, IRT_NIL), tr, 0);
1392 memset(&sbx, 0, sizeof(SBufExt));
1393 lj_bufx_set_cow(J->L, &sbx, strdata(str), str->len);
1394 t = (IRType)lj_serialize_peektype(&sbx);
1395 J->base[0] = lj_record_vload(J, tmp, 0, t);
775 } /* else: Interpreter will throw. */ 1396 } /* else: Interpreter will throw. */
776} 1397}
777 1398
1399#endif
1400
1401/* -- Table library fast functions ---------------------------------------- */
1402
778static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd) 1403static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
779{ 1404{
780 RecordIndex ix; 1405 RecordIndex ix;
@@ -783,7 +1408,7 @@ static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
783 rd->nres = 0; 1408 rd->nres = 0;
784 if (tref_istab(ix.tab) && ix.val) { 1409 if (tref_istab(ix.tab) && ix.val) {
785 if (!J->base[2]) { /* Simple push: t[#t+1] = v */ 1410 if (!J->base[2]) { /* Simple push: t[#t+1] = v */
786 TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, ix.tab); 1411 TRef trlen = emitir(IRTI(IR_ALEN), ix.tab, TREF_NIL);
787 GCtab *t = tabV(&rd->argv[0]); 1412 GCtab *t = tabV(&rd->argv[0]);
788 ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1)); 1413 ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
789 settabV(J->L, &ix.tabv, t); 1414 settabV(J->L, &ix.tabv, t);
@@ -791,11 +1416,49 @@ static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
791 ix.idxchain = 0; 1416 ix.idxchain = 0;
792 lj_record_idx(J, &ix); /* Set new value. */ 1417 lj_record_idx(J, &ix); /* Set new value. */
793 } else { /* Complex case: insert in the middle. */ 1418 } else { /* Complex case: insert in the middle. */
794 recff_nyiu(J); 1419 recff_nyiu(J, rd);
1420 return;
795 } 1421 }
796 } /* else: Interpreter will throw. */ 1422 } /* else: Interpreter will throw. */
797} 1423}
798 1424
1425static void LJ_FASTCALL recff_table_concat(jit_State *J, RecordFFData *rd)
1426{
1427 TRef tab = J->base[0];
1428 if (tref_istab(tab)) {
1429 TRef sep = !tref_isnil(J->base[1]) ?
1430 lj_ir_tostr(J, J->base[1]) : lj_ir_knull(J, IRT_STR);
1431 TRef tri = (J->base[1] && !tref_isnil(J->base[2])) ?
1432 lj_opt_narrow_toint(J, J->base[2]) : lj_ir_kint(J, 1);
1433 TRef tre = (J->base[1] && J->base[2] && !tref_isnil(J->base[3])) ?
1434 lj_opt_narrow_toint(J, J->base[3]) :
1435 emitir(IRTI(IR_ALEN), tab, TREF_NIL);
1436 TRef hdr = recff_bufhdr(J);
1437 TRef tr = lj_ir_call(J, IRCALL_lj_buf_puttab, hdr, tab, sep, tri, tre);
1438 emitir(IRTG(IR_NE, IRT_PTR), tr, lj_ir_kptr(J, NULL));
1439 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
1440 } /* else: Interpreter will throw. */
1441 UNUSED(rd);
1442}
1443
1444static void LJ_FASTCALL recff_table_new(jit_State *J, RecordFFData *rd)
1445{
1446 TRef tra = lj_opt_narrow_toint(J, J->base[0]);
1447 TRef trh = lj_opt_narrow_toint(J, J->base[1]);
1448 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_new_ah, tra, trh);
1449 UNUSED(rd);
1450}
1451
1452static void LJ_FASTCALL recff_table_clear(jit_State *J, RecordFFData *rd)
1453{
1454 TRef tr = J->base[0];
1455 if (tref_istab(tr)) {
1456 rd->nres = 0;
1457 lj_ir_call(J, IRCALL_lj_tab_clear, tr);
1458 J->needsnap = 1;
1459 } /* else: Interpreter will throw. */
1460}
1461
799/* -- I/O library fast functions ------------------------------------------ */ 1462/* -- I/O library fast functions ------------------------------------------ */
800 1463
801/* Get FILE* for I/O function. Any I/O error aborts recording, so there's 1464/* Get FILE* for I/O function. Any I/O error aborts recording, so there's
@@ -805,8 +1468,7 @@ static TRef recff_io_fp(jit_State *J, TRef *udp, int32_t id)
805{ 1468{
806 TRef tr, ud, fp; 1469 TRef tr, ud, fp;
807 if (id) { /* io.func() */ 1470 if (id) { /* io.func() */
808 tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]); 1471 ud = lj_ir_ggfload(J, IRT_UDATA, GG_OFS(g.gcroot[id]));
809 ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0);
810 } else { /* fp:method() */ 1472 } else { /* fp:method() */
811 ud = J->base[0]; 1473 ud = J->base[0];
812 if (!tref_isudata(ud)) 1474 if (!tref_isudata(ud))
@@ -828,10 +1490,13 @@ static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd)
828 ptrdiff_t i = rd->data == 0 ? 1 : 0; 1490 ptrdiff_t i = rd->data == 0 ? 1 : 0;
829 for (; J->base[i]; i++) { 1491 for (; J->base[i]; i++) {
830 TRef str = lj_ir_tostr(J, J->base[i]); 1492 TRef str = lj_ir_tostr(J, J->base[i]);
831 TRef buf = emitir(IRT(IR_STRREF, IRT_P32), str, zero); 1493 TRef buf = emitir(IRT(IR_STRREF, IRT_PGC), str, zero);
832 TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN); 1494 TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN);
833 if (tref_isk(len) && IR(tref_ref(len))->i == 1) { 1495 if (tref_isk(len) && IR(tref_ref(len))->i == 1) {
834 TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY); 1496 IRIns *irs = IR(tref_ref(str));
1497 TRef tr = (irs->o == IR_TOSTR && irs->op2 == IRTOSTR_CHAR) ?
1498 irs->op1 :
1499 emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
835 tr = lj_ir_call(J, IRCALL_fputc, tr, fp); 1500 tr = lj_ir_call(J, IRCALL_fputc, tr, fp);
836 if (results_wanted(J) != 0) /* Check result only if not ignored. */ 1501 if (results_wanted(J) != 0) /* Check result only if not ignored. */
837 emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1)); 1502 emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1));
@@ -853,6 +1518,28 @@ static void LJ_FASTCALL recff_io_flush(jit_State *J, RecordFFData *rd)
853 J->base[0] = TREF_TRUE; 1518 J->base[0] = TREF_TRUE;
854} 1519}
855 1520
1521/* -- Debug library fast functions ---------------------------------------- */
1522
1523static void LJ_FASTCALL recff_debug_getmetatable(jit_State *J, RecordFFData *rd)
1524{
1525 GCtab *mt;
1526 TRef mtref;
1527 TRef tr = J->base[0];
1528 if (tref_istab(tr)) {
1529 mt = tabref(tabV(&rd->argv[0])->metatable);
1530 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_TAB_META);
1531 } else if (tref_isudata(tr)) {
1532 mt = tabref(udataV(&rd->argv[0])->metatable);
1533 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_UDATA_META);
1534 } else {
1535 mt = tabref(basemt_obj(J2G(J), &rd->argv[0]));
1536 J->base[0] = mt ? lj_ir_ktab(J, mt) : TREF_NIL;
1537 return;
1538 }
1539 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1540 J->base[0] = mt ? mtref : TREF_NIL;
1541}
1542
856/* -- Record calls to fast functions -------------------------------------- */ 1543/* -- Record calls to fast functions -------------------------------------- */
857 1544
858#include "lj_recdef.h" 1545#include "lj_recdef.h"
diff --git a/src/lj_frame.h b/src/lj_frame.h
index 35cbb4ea..bb7dda63 100644
--- a/src/lj_frame.h
+++ b/src/lj_frame.h
@@ -11,7 +11,16 @@
11 11
12/* -- Lua stack frame ----------------------------------------------------- */ 12/* -- Lua stack frame ----------------------------------------------------- */
13 13
14/* Frame type markers in callee function slot (callee base-1). */ 14/* Frame type markers in LSB of PC (4-byte aligned) or delta (8-byte aligned:
15**
16** PC 00 Lua frame
17** delta 001 C frame
18** delta 010 Continuation frame
19** delta 011 Lua vararg frame
20** delta 101 cpcall() frame
21** delta 110 ff pcall() frame
22** delta 111 ff pcall() frame with active hook
23*/
15enum { 24enum {
16 FRAME_LUA, FRAME_C, FRAME_CONT, FRAME_VARG, 25 FRAME_LUA, FRAME_C, FRAME_CONT, FRAME_VARG,
17 FRAME_LUAP, FRAME_CP, FRAME_PCALL, FRAME_PCALLH 26 FRAME_LUAP, FRAME_CP, FRAME_PCALL, FRAME_PCALLH
@@ -21,9 +30,47 @@ enum {
21#define FRAME_TYPEP (FRAME_TYPE|FRAME_P) 30#define FRAME_TYPEP (FRAME_TYPE|FRAME_P)
22 31
23/* Macros to access and modify Lua frames. */ 32/* Macros to access and modify Lua frames. */
33#if LJ_FR2
34/* Two-slot frame info, required for 64 bit PC/GCRef:
35**
36** base-2 base-1 | base base+1 ...
37** [func PC/delta/ft] | [slots ...]
38** ^-- frame | ^-- base ^-- top
39**
40** Continuation frames:
41**
42** base-4 base-3 base-2 base-1 | base base+1 ...
43** [cont PC ] [func PC/delta/ft] | [slots ...]
44** ^-- frame | ^-- base ^-- top
45*/
46#define frame_gc(f) (gcval((f)-1))
47#define frame_ftsz(f) ((ptrdiff_t)(f)->ftsz)
48#define frame_pc(f) ((const BCIns *)frame_ftsz(f))
49#define setframe_gc(f, p, tp) (setgcVraw((f), (p), (tp)))
50#define setframe_ftsz(f, sz) ((f)->ftsz = (sz))
51#define setframe_pc(f, pc) ((f)->ftsz = (int64_t)(intptr_t)(pc))
52#else
53/* One-slot frame info, sufficient for 32 bit PC/GCRef:
54**
55** base-1 | base base+1 ...
56** lo hi |
57** [func | PC/delta/ft] | [slots ...]
58** ^-- frame | ^-- base ^-- top
59**
60** Continuation frames:
61**
62** base-2 base-1 | base base+1 ...
63** lo hi lo hi |
64** [cont | PC] [func | PC/delta/ft] | [slots ...]
65** ^-- frame | ^-- base ^-- top
66*/
24#define frame_gc(f) (gcref((f)->fr.func)) 67#define frame_gc(f) (gcref((f)->fr.func))
25#define frame_func(f) (&frame_gc(f)->fn) 68#define frame_ftsz(f) ((ptrdiff_t)(f)->fr.tp.ftsz)
26#define frame_ftsz(f) ((f)->fr.tp.ftsz) 69#define frame_pc(f) (mref((f)->fr.tp.pcr, const BCIns))
70#define setframe_gc(f, p, tp) (setgcref((f)->fr.func, (p)), UNUSED(tp))
71#define setframe_ftsz(f, sz) ((f)->fr.tp.ftsz = (int32_t)(sz))
72#define setframe_pc(f, pc) (setmref((f)->fr.tp.pcr, (pc)))
73#endif
27 74
28#define frame_type(f) (frame_ftsz(f) & FRAME_TYPE) 75#define frame_type(f) (frame_ftsz(f) & FRAME_TYPE)
29#define frame_typep(f) (frame_ftsz(f) & FRAME_TYPEP) 76#define frame_typep(f) (frame_ftsz(f) & FRAME_TYPEP)
@@ -33,33 +80,53 @@ enum {
33#define frame_isvarg(f) (frame_typep(f) == FRAME_VARG) 80#define frame_isvarg(f) (frame_typep(f) == FRAME_VARG)
34#define frame_ispcall(f) ((frame_ftsz(f) & 6) == FRAME_PCALL) 81#define frame_ispcall(f) ((frame_ftsz(f) & 6) == FRAME_PCALL)
35 82
36#define frame_pc(f) (mref((f)->fr.tp.pcr, const BCIns)) 83#define frame_func(f) (&frame_gc(f)->fn)
84#define frame_delta(f) (frame_ftsz(f) >> 3)
85#define frame_sized(f) (frame_ftsz(f) & ~FRAME_TYPEP)
86
87enum { LJ_CONT_TAILCALL, LJ_CONT_FFI_CALLBACK }; /* Special continuations. */
88
89#if LJ_FR2
90#define frame_contpc(f) (frame_pc((f)-2))
91#define frame_contv(f) (((f)-3)->u64)
92#else
37#define frame_contpc(f) (frame_pc((f)-1)) 93#define frame_contpc(f) (frame_pc((f)-1))
38#if LJ_64 94#define frame_contv(f) (((f)-1)->u32.lo)
95#endif
96#if LJ_FR2
97#define frame_contf(f) ((ASMFunction)(uintptr_t)((f)-3)->u64)
98#elif LJ_64
39#define frame_contf(f) \ 99#define frame_contf(f) \
40 ((ASMFunction)(void *)((intptr_t)lj_vm_asm_begin + \ 100 ((ASMFunction)(void *)((intptr_t)lj_vm_asm_begin + \
41 (intptr_t)(int32_t)((f)-1)->u32.lo)) 101 (intptr_t)(int32_t)((f)-1)->u32.lo))
42#else 102#else
43#define frame_contf(f) ((ASMFunction)gcrefp(((f)-1)->gcr, void)) 103#define frame_contf(f) ((ASMFunction)gcrefp(((f)-1)->gcr, void))
44#endif 104#endif
45#define frame_delta(f) (frame_ftsz(f) >> 3) 105#define frame_iscont_fficb(f) \
46#define frame_sized(f) (frame_ftsz(f) & ~FRAME_TYPEP) 106 (LJ_HASFFI && frame_contv(f) == LJ_CONT_FFI_CALLBACK)
47 107
48#define frame_prevl(f) ((f) - (1+bc_a(frame_pc(f)[-1]))) 108#define frame_prevl(f) ((f) - (1+LJ_FR2+bc_a(frame_pc(f)[-1])))
49#define frame_prevd(f) ((TValue *)((char *)(f) - frame_sized(f))) 109#define frame_prevd(f) ((TValue *)((char *)(f) - frame_sized(f)))
50#define frame_prev(f) (frame_islua(f)?frame_prevl(f):frame_prevd(f)) 110#define frame_prev(f) (frame_islua(f)?frame_prevl(f):frame_prevd(f))
51/* Note: this macro does not skip over FRAME_VARG. */ 111/* Note: this macro does not skip over FRAME_VARG. */
52 112
53#define setframe_pc(f, pc) (setmref((f)->fr.tp.pcr, (pc)))
54#define setframe_ftsz(f, sz) ((f)->fr.tp.ftsz = (sz))
55#define setframe_gc(f, p) (setgcref((f)->fr.func, (p)))
56
57/* -- C stack frame ------------------------------------------------------- */ 113/* -- C stack frame ------------------------------------------------------- */
58 114
59/* Macros to access and modify the C stack frame chain. */ 115/* Macros to access and modify the C stack frame chain. */
60 116
61/* These definitions must match with the arch-specific *.dasc files. */ 117/* These definitions must match with the arch-specific *.dasc files. */
62#if LJ_TARGET_X86 118#if LJ_TARGET_X86
119#if LJ_ABI_WIN
120#define CFRAME_OFS_ERRF (19*4)
121#define CFRAME_OFS_NRES (18*4)
122#define CFRAME_OFS_PREV (17*4)
123#define CFRAME_OFS_L (16*4)
124#define CFRAME_OFS_SEH (9*4)
125#define CFRAME_OFS_PC (6*4)
126#define CFRAME_OFS_MULTRES (5*4)
127#define CFRAME_SIZE (16*4)
128#define CFRAME_SHIFT_MULTRES 0
129#else
63#define CFRAME_OFS_ERRF (15*4) 130#define CFRAME_OFS_ERRF (15*4)
64#define CFRAME_OFS_NRES (14*4) 131#define CFRAME_OFS_NRES (14*4)
65#define CFRAME_OFS_PREV (13*4) 132#define CFRAME_OFS_PREV (13*4)
@@ -68,24 +135,41 @@ enum {
68#define CFRAME_OFS_MULTRES (5*4) 135#define CFRAME_OFS_MULTRES (5*4)
69#define CFRAME_SIZE (12*4) 136#define CFRAME_SIZE (12*4)
70#define CFRAME_SHIFT_MULTRES 0 137#define CFRAME_SHIFT_MULTRES 0
138#endif
71#elif LJ_TARGET_X64 139#elif LJ_TARGET_X64
72#if LJ_ABI_WIN 140#if LJ_ABI_WIN
73#define CFRAME_OFS_PREV (13*8) 141#define CFRAME_OFS_PREV (13*8)
142#if LJ_GC64
143#define CFRAME_OFS_PC (12*8)
144#define CFRAME_OFS_L (11*8)
145#define CFRAME_OFS_ERRF (21*4)
146#define CFRAME_OFS_NRES (20*4)
147#define CFRAME_OFS_MULTRES (8*4)
148#else
74#define CFRAME_OFS_PC (25*4) 149#define CFRAME_OFS_PC (25*4)
75#define CFRAME_OFS_L (24*4) 150#define CFRAME_OFS_L (24*4)
76#define CFRAME_OFS_ERRF (23*4) 151#define CFRAME_OFS_ERRF (23*4)
77#define CFRAME_OFS_NRES (22*4) 152#define CFRAME_OFS_NRES (22*4)
78#define CFRAME_OFS_MULTRES (21*4) 153#define CFRAME_OFS_MULTRES (21*4)
154#endif
79#define CFRAME_SIZE (10*8) 155#define CFRAME_SIZE (10*8)
80#define CFRAME_SIZE_JIT (CFRAME_SIZE + 9*16 + 4*8) 156#define CFRAME_SIZE_JIT (CFRAME_SIZE + 9*16 + 4*8)
81#define CFRAME_SHIFT_MULTRES 0 157#define CFRAME_SHIFT_MULTRES 0
82#else 158#else
83#define CFRAME_OFS_PREV (4*8) 159#define CFRAME_OFS_PREV (4*8)
160#if LJ_GC64
161#define CFRAME_OFS_PC (3*8)
162#define CFRAME_OFS_L (2*8)
163#define CFRAME_OFS_ERRF (3*4)
164#define CFRAME_OFS_NRES (2*4)
165#define CFRAME_OFS_MULTRES (0*4)
166#else
84#define CFRAME_OFS_PC (7*4) 167#define CFRAME_OFS_PC (7*4)
85#define CFRAME_OFS_L (6*4) 168#define CFRAME_OFS_L (6*4)
86#define CFRAME_OFS_ERRF (5*4) 169#define CFRAME_OFS_ERRF (5*4)
87#define CFRAME_OFS_NRES (4*4) 170#define CFRAME_OFS_NRES (4*4)
88#define CFRAME_OFS_MULTRES (1*4) 171#define CFRAME_OFS_MULTRES (1*4)
172#endif
89#if LJ_NO_UNWIND 173#if LJ_NO_UNWIND
90#define CFRAME_SIZE (12*8) 174#define CFRAME_SIZE (12*8)
91#else 175#else
@@ -107,6 +191,15 @@ enum {
107#define CFRAME_SIZE 64 191#define CFRAME_SIZE 64
108#endif 192#endif
109#define CFRAME_SHIFT_MULTRES 3 193#define CFRAME_SHIFT_MULTRES 3
194#elif LJ_TARGET_ARM64
195#define CFRAME_OFS_ERRF 36
196#define CFRAME_OFS_NRES 40
197#define CFRAME_OFS_PREV 0
198#define CFRAME_OFS_L 16
199#define CFRAME_OFS_PC 8
200#define CFRAME_OFS_MULTRES 32
201#define CFRAME_SIZE 208
202#define CFRAME_SHIFT_MULTRES 3
110#elif LJ_TARGET_PPC 203#elif LJ_TARGET_PPC
111#if LJ_TARGET_XBOX360 204#if LJ_TARGET_XBOX360
112#define CFRAME_OFS_ERRF 424 205#define CFRAME_OFS_ERRF 424
@@ -117,7 +210,7 @@ enum {
117#define CFRAME_OFS_MULTRES 408 210#define CFRAME_OFS_MULTRES 408
118#define CFRAME_SIZE 384 211#define CFRAME_SIZE 384
119#define CFRAME_SHIFT_MULTRES 3 212#define CFRAME_SHIFT_MULTRES 3
120#elif LJ_ARCH_PPC64 213#elif LJ_ARCH_PPC32ON64
121#define CFRAME_OFS_ERRF 472 214#define CFRAME_OFS_ERRF 472
122#define CFRAME_OFS_NRES 468 215#define CFRAME_OFS_NRES 468
123#define CFRAME_OFS_PREV 448 216#define CFRAME_OFS_PREV 448
@@ -133,26 +226,43 @@ enum {
133#define CFRAME_OFS_L 36 226#define CFRAME_OFS_L 36
134#define CFRAME_OFS_PC 32 227#define CFRAME_OFS_PC 32
135#define CFRAME_OFS_MULTRES 28 228#define CFRAME_OFS_MULTRES 28
136#define CFRAME_SIZE 272 229#define CFRAME_SIZE (LJ_ARCH_HASFPU ? 272 : 128)
137#define CFRAME_SHIFT_MULTRES 3 230#define CFRAME_SHIFT_MULTRES 3
138#endif 231#endif
139#elif LJ_TARGET_PPCSPE 232#elif LJ_TARGET_MIPS32
140#define CFRAME_OFS_ERRF 28 233#if LJ_ARCH_HASFPU
141#define CFRAME_OFS_NRES 24
142#define CFRAME_OFS_PREV 20
143#define CFRAME_OFS_L 16
144#define CFRAME_OFS_PC 12
145#define CFRAME_OFS_MULTRES 8
146#define CFRAME_SIZE 184
147#define CFRAME_SHIFT_MULTRES 3
148#elif LJ_TARGET_MIPS
149#define CFRAME_OFS_ERRF 124 234#define CFRAME_OFS_ERRF 124
150#define CFRAME_OFS_NRES 120 235#define CFRAME_OFS_NRES 120
151#define CFRAME_OFS_PREV 116 236#define CFRAME_OFS_PREV 116
152#define CFRAME_OFS_L 112 237#define CFRAME_OFS_L 112
238#define CFRAME_SIZE 112
239#else
240#define CFRAME_OFS_ERRF 76
241#define CFRAME_OFS_NRES 72
242#define CFRAME_OFS_PREV 68
243#define CFRAME_OFS_L 64
244#define CFRAME_SIZE 64
245#endif
153#define CFRAME_OFS_PC 20 246#define CFRAME_OFS_PC 20
154#define CFRAME_OFS_MULTRES 16 247#define CFRAME_OFS_MULTRES 16
155#define CFRAME_SIZE 112 248#define CFRAME_SHIFT_MULTRES 3
249#elif LJ_TARGET_MIPS64
250#if LJ_ARCH_HASFPU
251#define CFRAME_OFS_ERRF 188
252#define CFRAME_OFS_NRES 184
253#define CFRAME_OFS_PREV 176
254#define CFRAME_OFS_L 168
255#define CFRAME_OFS_PC 160
256#define CFRAME_SIZE 192
257#else
258#define CFRAME_OFS_ERRF 124
259#define CFRAME_OFS_NRES 120
260#define CFRAME_OFS_PREV 112
261#define CFRAME_OFS_L 104
262#define CFRAME_OFS_PC 96
263#define CFRAME_SIZE 128
264#endif
265#define CFRAME_OFS_MULTRES 0
156#define CFRAME_SHIFT_MULTRES 3 266#define CFRAME_SHIFT_MULTRES 3
157#else 267#else
158#error "Missing CFRAME_* definitions for this architecture" 268#error "Missing CFRAME_* definitions for this architecture"
diff --git a/src/lj_func.c b/src/lj_func.c
index 4ac47875..6a537649 100644
--- a/src/lj_func.c
+++ b/src/lj_func.c
@@ -24,9 +24,11 @@ void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt)
24 24
25/* -- Upvalues ------------------------------------------------------------ */ 25/* -- Upvalues ------------------------------------------------------------ */
26 26
27static void unlinkuv(GCupval *uv) 27static void unlinkuv(global_State *g, GCupval *uv)
28{ 28{
29 lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv); 29 UNUSED(g);
30 lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv,
31 "broken upvalue chain");
30 setgcrefr(uvnext(uv)->prev, uv->prev); 32 setgcrefr(uvnext(uv)->prev, uv->prev);
31 setgcrefr(uvprev(uv)->next, uv->next); 33 setgcrefr(uvprev(uv)->next, uv->next);
32} 34}
@@ -40,7 +42,7 @@ static GCupval *func_finduv(lua_State *L, TValue *slot)
40 GCupval *uv; 42 GCupval *uv;
41 /* Search the sorted list of open upvalues. */ 43 /* Search the sorted list of open upvalues. */
42 while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) { 44 while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) {
43 lua_assert(!p->closed && uvval(p) != &p->tv); 45 lj_assertG(!p->closed && uvval(p) != &p->tv, "closed upvalue in chain");
44 if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */ 46 if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */
45 if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */ 47 if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */
46 flipwhite(obj2gco(p)); 48 flipwhite(obj2gco(p));
@@ -61,7 +63,8 @@ static GCupval *func_finduv(lua_State *L, TValue *slot)
61 setgcrefr(uv->next, g->uvhead.next); 63 setgcrefr(uv->next, g->uvhead.next);
62 setgcref(uvnext(uv)->prev, obj2gco(uv)); 64 setgcref(uvnext(uv)->prev, obj2gco(uv));
63 setgcref(g->uvhead.next, obj2gco(uv)); 65 setgcref(g->uvhead.next, obj2gco(uv));
64 lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv); 66 lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv,
67 "broken upvalue chain");
65 return uv; 68 return uv;
66} 69}
67 70
@@ -84,12 +87,13 @@ void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level)
84 while (gcref(L->openupval) != NULL && 87 while (gcref(L->openupval) != NULL &&
85 uvval((uv = gco2uv(gcref(L->openupval)))) >= level) { 88 uvval((uv = gco2uv(gcref(L->openupval)))) >= level) {
86 GCobj *o = obj2gco(uv); 89 GCobj *o = obj2gco(uv);
87 lua_assert(!isblack(o) && !uv->closed && uvval(uv) != &uv->tv); 90 lj_assertG(!isblack(o), "bad black upvalue");
91 lj_assertG(!uv->closed && uvval(uv) != &uv->tv, "closed upvalue in chain");
88 setgcrefr(L->openupval, uv->nextgc); /* No longer in open list. */ 92 setgcrefr(L->openupval, uv->nextgc); /* No longer in open list. */
89 if (isdead(g, o)) { 93 if (isdead(g, o)) {
90 lj_func_freeuv(g, uv); 94 lj_func_freeuv(g, uv);
91 } else { 95 } else {
92 unlinkuv(uv); 96 unlinkuv(g, uv);
93 lj_gc_closeuv(g, uv); 97 lj_gc_closeuv(g, uv);
94 } 98 }
95 } 99 }
@@ -98,7 +102,7 @@ void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level)
98void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv) 102void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv)
99{ 103{
100 if (!uv->closed) 104 if (!uv->closed)
101 unlinkuv(uv); 105 unlinkuv(g, uv);
102 lj_mem_freet(g, uv); 106 lj_mem_freet(g, uv);
103} 107}
104 108
diff --git a/src/lj_gc.c b/src/lj_gc.c
index 06484f6f..c3a0c258 100644
--- a/src/lj_gc.c
+++ b/src/lj_gc.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#include "lj_tab.h" 17#include "lj_tab.h"
17#include "lj_func.h" 18#include "lj_func.h"
@@ -24,7 +25,9 @@
24#include "lj_cdata.h" 25#include "lj_cdata.h"
25#endif 26#endif
26#include "lj_trace.h" 27#include "lj_trace.h"
28#include "lj_dispatch.h"
27#include "lj_vm.h" 29#include "lj_vm.h"
30#include "lj_vmevent.h"
28 31
29#define GCSTEPSIZE 1024u 32#define GCSTEPSIZE 1024u
30#define GCSWEEPMAX 40 33#define GCSWEEPMAX 40
@@ -40,7 +43,8 @@
40 43
41/* Mark a TValue (if needed). */ 44/* Mark a TValue (if needed). */
42#define gc_marktv(g, tv) \ 45#define gc_marktv(g, tv) \
43 { lua_assert(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct)); \ 46 { lj_assertG(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct), \
47 "TValue and GC type mismatch"); \
44 if (tviswhite(tv)) gc_mark(g, gcV(tv)); } 48 if (tviswhite(tv)) gc_mark(g, gcV(tv)); }
45 49
46/* Mark a GCobj (if needed). */ 50/* Mark a GCobj (if needed). */
@@ -54,21 +58,32 @@
54static void gc_mark(global_State *g, GCobj *o) 58static void gc_mark(global_State *g, GCobj *o)
55{ 59{
56 int gct = o->gch.gct; 60 int gct = o->gch.gct;
57 lua_assert(iswhite(o) && !isdead(g, o)); 61 lj_assertG(iswhite(o), "mark of non-white object");
62 lj_assertG(!isdead(g, o), "mark of dead object");
58 white2gray(o); 63 white2gray(o);
59 if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) { 64 if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) {
60 GCtab *mt = tabref(gco2ud(o)->metatable); 65 GCtab *mt = tabref(gco2ud(o)->metatable);
61 gray2black(o); /* Userdata are never gray. */ 66 gray2black(o); /* Userdata are never gray. */
62 if (mt) gc_markobj(g, mt); 67 if (mt) gc_markobj(g, mt);
63 gc_markobj(g, tabref(gco2ud(o)->env)); 68 gc_markobj(g, tabref(gco2ud(o)->env));
69 if (LJ_HASBUFFER && gco2ud(o)->udtype == UDTYPE_BUFFER) {
70 SBufExt *sbx = (SBufExt *)uddata(gco2ud(o));
71 if (sbufiscow(sbx) && gcref(sbx->cowref))
72 gc_markobj(g, gcref(sbx->cowref));
73 if (gcref(sbx->dict_str))
74 gc_markobj(g, gcref(sbx->dict_str));
75 if (gcref(sbx->dict_mt))
76 gc_markobj(g, gcref(sbx->dict_mt));
77 }
64 } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) { 78 } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) {
65 GCupval *uv = gco2uv(o); 79 GCupval *uv = gco2uv(o);
66 gc_marktv(g, uvval(uv)); 80 gc_marktv(g, uvval(uv));
67 if (uv->closed) 81 if (uv->closed)
68 gray2black(o); /* Closed upvalues are never gray. */ 82 gray2black(o); /* Closed upvalues are never gray. */
69 } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) { 83 } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) {
70 lua_assert(gct == ~LJ_TFUNC || gct == ~LJ_TTAB || 84 lj_assertG(gct == ~LJ_TFUNC || gct == ~LJ_TTAB ||
71 gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO); 85 gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO || gct == ~LJ_TTRACE,
86 "bad GC type %d", gct);
72 setgcrefr(o->gch.gclist, g->gc.gray); 87 setgcrefr(o->gch.gclist, g->gc.gray);
73 setgcref(g->gc.gray, o); 88 setgcref(g->gc.gray, o);
74 } 89 }
@@ -101,7 +116,8 @@ static void gc_mark_uv(global_State *g)
101{ 116{
102 GCupval *uv; 117 GCupval *uv;
103 for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) { 118 for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) {
104 lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv); 119 lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv,
120 "broken upvalue chain");
105 if (isgray(obj2gco(uv))) 121 if (isgray(obj2gco(uv)))
106 gc_marktv(g, uvval(uv)); 122 gc_marktv(g, uvval(uv));
107 } 123 }
@@ -196,7 +212,7 @@ static int gc_traverse_tab(global_State *g, GCtab *t)
196 for (i = 0; i <= hmask; i++) { 212 for (i = 0; i <= hmask; i++) {
197 Node *n = &node[i]; 213 Node *n = &node[i];
198 if (!tvisnil(&n->val)) { /* Mark non-empty slot. */ 214 if (!tvisnil(&n->val)) { /* Mark non-empty slot. */
199 lua_assert(!tvisnil(&n->key)); 215 lj_assertG(!tvisnil(&n->key), "mark of nil key in non-empty slot");
200 if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key); 216 if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key);
201 if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val); 217 if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val);
202 } 218 }
@@ -211,7 +227,8 @@ static void gc_traverse_func(global_State *g, GCfunc *fn)
211 gc_markobj(g, tabref(fn->c.env)); 227 gc_markobj(g, tabref(fn->c.env));
212 if (isluafunc(fn)) { 228 if (isluafunc(fn)) {
213 uint32_t i; 229 uint32_t i;
214 lua_assert(fn->l.nupvalues <= funcproto(fn)->sizeuv); 230 lj_assertG(fn->l.nupvalues <= funcproto(fn)->sizeuv,
231 "function upvalues out of range");
215 gc_markobj(g, funcproto(fn)); 232 gc_markobj(g, funcproto(fn));
216 for (i = 0; i < fn->l.nupvalues; i++) /* Mark Lua function upvalues. */ 233 for (i = 0; i < fn->l.nupvalues; i++) /* Mark Lua function upvalues. */
217 gc_markobj(g, &gcref(fn->l.uvptr[i])->uv); 234 gc_markobj(g, &gcref(fn->l.uvptr[i])->uv);
@@ -227,7 +244,7 @@ static void gc_traverse_func(global_State *g, GCfunc *fn)
227static void gc_marktrace(global_State *g, TraceNo traceno) 244static void gc_marktrace(global_State *g, TraceNo traceno)
228{ 245{
229 GCobj *o = obj2gco(traceref(G2J(g), traceno)); 246 GCobj *o = obj2gco(traceref(G2J(g), traceno));
230 lua_assert(traceno != G2J(g)->cur.traceno); 247 lj_assertG(traceno != G2J(g)->cur.traceno, "active trace escaped");
231 if (iswhite(o)) { 248 if (iswhite(o)) {
232 white2gray(o); 249 white2gray(o);
233 setgcrefr(o->gch.gclist, g->gc.gray); 250 setgcrefr(o->gch.gclist, g->gc.gray);
@@ -244,6 +261,8 @@ static void gc_traverse_trace(global_State *g, GCtrace *T)
244 IRIns *ir = &T->ir[ref]; 261 IRIns *ir = &T->ir[ref];
245 if (ir->o == IR_KGC) 262 if (ir->o == IR_KGC)
246 gc_markobj(g, ir_kgc(ir)); 263 gc_markobj(g, ir_kgc(ir));
264 if (irt_is64(ir->t) && ir->o != IR_KNULL)
265 ref++;
247 } 266 }
248 if (T->link) gc_marktrace(g, T->link); 267 if (T->link) gc_marktrace(g, T->link);
249 if (T->nextroot) gc_marktrace(g, T->nextroot); 268 if (T->nextroot) gc_marktrace(g, T->nextroot);
@@ -274,12 +293,12 @@ static MSize gc_traverse_frames(global_State *g, lua_State *th)
274{ 293{
275 TValue *frame, *top = th->top-1, *bot = tvref(th->stack); 294 TValue *frame, *top = th->top-1, *bot = tvref(th->stack);
276 /* Note: extra vararg frame not skipped, marks function twice (harmless). */ 295 /* Note: extra vararg frame not skipped, marks function twice (harmless). */
277 for (frame = th->base-1; frame > bot; frame = frame_prev(frame)) { 296 for (frame = th->base-1; frame > bot+LJ_FR2; frame = frame_prev(frame)) {
278 GCfunc *fn = frame_func(frame); 297 GCfunc *fn = frame_func(frame);
279 TValue *ftop = frame; 298 TValue *ftop = frame;
280 if (isluafunc(fn)) ftop += funcproto(fn)->framesize; 299 if (isluafunc(fn)) ftop += funcproto(fn)->framesize;
281 if (ftop > top) top = ftop; 300 if (ftop > top) top = ftop;
282 gc_markobj(g, fn); /* Need to mark hidden function (or L). */ 301 if (!LJ_FR2) gc_markobj(g, fn); /* Need to mark hidden function (or L). */
283 } 302 }
284 top++; /* Correct bias of -1 (frame == base-1). */ 303 top++; /* Correct bias of -1 (frame == base-1). */
285 if (top > tvref(th->maxstack)) top = tvref(th->maxstack); 304 if (top > tvref(th->maxstack)) top = tvref(th->maxstack);
@@ -290,7 +309,7 @@ static MSize gc_traverse_frames(global_State *g, lua_State *th)
290static void gc_traverse_thread(global_State *g, lua_State *th) 309static void gc_traverse_thread(global_State *g, lua_State *th)
291{ 310{
292 TValue *o, *top = th->top; 311 TValue *o, *top = th->top;
293 for (o = tvref(th->stack)+1; o < top; o++) 312 for (o = tvref(th->stack)+1+LJ_FR2; o < top; o++)
294 gc_marktv(g, o); 313 gc_marktv(g, o);
295 if (g->gc.state == GCSatomic) { 314 if (g->gc.state == GCSatomic) {
296 top = tvref(th->stack) + th->stacksize; 315 top = tvref(th->stack) + th->stacksize;
@@ -306,7 +325,7 @@ static size_t propagatemark(global_State *g)
306{ 325{
307 GCobj *o = gcref(g->gc.gray); 326 GCobj *o = gcref(g->gc.gray);
308 int gct = o->gch.gct; 327 int gct = o->gch.gct;
309 lua_assert(isgray(o)); 328 lj_assertG(isgray(o), "propagation of non-gray object");
310 gray2black(o); 329 gray2black(o);
311 setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */ 330 setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */
312 if (LJ_LIKELY(gct == ~LJ_TTAB)) { 331 if (LJ_LIKELY(gct == ~LJ_TTAB)) {
@@ -338,7 +357,7 @@ static size_t propagatemark(global_State *g)
338 return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + 357 return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
339 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry); 358 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry);
340#else 359#else
341 lua_assert(0); 360 lj_assertG(0, "bad GC type %d", gct);
342 return 0; 361 return 0;
343#endif 362#endif
344 } 363 }
@@ -355,15 +374,6 @@ static size_t gc_propagate_gray(global_State *g)
355 374
356/* -- Sweep phase --------------------------------------------------------- */ 375/* -- Sweep phase --------------------------------------------------------- */
357 376
358/* Try to shrink some common data structures. */
359static void gc_shrink(global_State *g, lua_State *L)
360{
361 if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1)
362 lj_str_resize(L, g->strmask >> 1); /* Shrink string table. */
363 if (g->tmpbuf.sz > LJ_MIN_SBUF*2)
364 lj_str_resizebuf(L, &g->tmpbuf, g->tmpbuf.sz >> 1); /* Shrink temp buf. */
365}
366
367/* Type of GC free functions. */ 377/* Type of GC free functions. */
368typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o); 378typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o);
369 379
@@ -389,7 +399,7 @@ static const GCFreeFunc gc_freefunc[] = {
389}; 399};
390 400
391/* Full sweep of a GC list. */ 401/* Full sweep of a GC list. */
392#define gc_fullsweep(g, p) gc_sweep(g, (p), LJ_MAX_MEM) 402#define gc_fullsweep(g, p) gc_sweep(g, (p), ~(uint32_t)0)
393 403
394/* Partial sweep of a GC list. */ 404/* Partial sweep of a GC list. */
395static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim) 405static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
@@ -401,11 +411,13 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
401 if (o->gch.gct == ~LJ_TTHREAD) /* Need to sweep open upvalues, too. */ 411 if (o->gch.gct == ~LJ_TTHREAD) /* Need to sweep open upvalues, too. */
402 gc_fullsweep(g, &gco2th(o)->openupval); 412 gc_fullsweep(g, &gco2th(o)->openupval);
403 if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */ 413 if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */
404 lua_assert(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED)); 414 lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED),
415 "sweep of undead object");
405 makewhite(g, o); /* Value is alive, change to the current white. */ 416 makewhite(g, o); /* Value is alive, change to the current white. */
406 p = &o->gch.nextgc; 417 p = &o->gch.nextgc;
407 } else { /* Otherwise value is dead, free it. */ 418 } else { /* Otherwise value is dead, free it. */
408 lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED); 419 lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED,
420 "sweep of unlive object");
409 setgcrefr(*p, o->gch.nextgc); 421 setgcrefr(*p, o->gch.nextgc);
410 if (o == gcref(g->gc.root)) 422 if (o == gcref(g->gc.root))
411 setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */ 423 setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */
@@ -415,6 +427,32 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
415 return p; 427 return p;
416} 428}
417 429
430/* Sweep one string interning table chain. Preserves hashalg bit. */
431static void gc_sweepstr(global_State *g, GCRef *chain)
432{
433 /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */
434 int ow = otherwhite(g);
435 uintptr_t u = gcrefu(*chain);
436 GCRef q;
437 GCRef *p = &q;
438 GCobj *o;
439 setgcrefp(q, (u & ~(uintptr_t)1));
440 while ((o = gcref(*p)) != NULL) {
441 if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */
442 lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED),
443 "sweep of undead string");
444 makewhite(g, o); /* String is alive, change to the current white. */
445 p = &o->gch.nextgc;
446 } else { /* Otherwise string is dead, free it. */
447 lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED,
448 "sweep of unlive string");
449 setgcrefr(*p, o->gch.nextgc);
450 lj_str_free(g, gco2str(o));
451 }
452 }
453 setgcrefp(*chain, (gcrefu(q) | (u & 1)));
454}
455
418/* Check whether we can clear a key or a value slot from a table. */ 456/* Check whether we can clear a key or a value slot from a table. */
419static int gc_mayclear(cTValue *o, int val) 457static int gc_mayclear(cTValue *o, int val)
420{ 458{
@@ -432,11 +470,12 @@ static int gc_mayclear(cTValue *o, int val)
432} 470}
433 471
434/* Clear collected entries from weak tables. */ 472/* Clear collected entries from weak tables. */
435static void gc_clearweak(GCobj *o) 473static void gc_clearweak(global_State *g, GCobj *o)
436{ 474{
475 UNUSED(g);
437 while (o) { 476 while (o) {
438 GCtab *t = gco2tab(o); 477 GCtab *t = gco2tab(o);
439 lua_assert((t->marked & LJ_GC_WEAK)); 478 lj_assertG((t->marked & LJ_GC_WEAK), "clear of non-weak table");
440 if ((t->marked & LJ_GC_WEAKVAL)) { 479 if ((t->marked & LJ_GC_WEAKVAL)) {
441 MSize i, asize = t->asize; 480 MSize i, asize = t->asize;
442 for (i = 0; i < asize; i++) { 481 for (i = 0; i < asize; i++) {
@@ -467,21 +506,29 @@ static void gc_call_finalizer(global_State *g, lua_State *L,
467{ 506{
468 /* Save and restore lots of state around the __gc callback. */ 507 /* Save and restore lots of state around the __gc callback. */
469 uint8_t oldh = hook_save(g); 508 uint8_t oldh = hook_save(g);
470 MSize oldt = g->gc.threshold; 509 GCSize oldt = g->gc.threshold;
471 int errcode; 510 int errcode;
472 TValue *top; 511 TValue *top;
473 lj_trace_abort(g); 512 lj_trace_abort(g);
474 top = L->top;
475 L->top = top+2;
476 hook_entergc(g); /* Disable hooks and new traces during __gc. */ 513 hook_entergc(g); /* Disable hooks and new traces during __gc. */
514 if (LJ_HASPROFILE && (oldh & HOOK_PROFILE)) lj_dispatch_update(g);
477 g->gc.threshold = LJ_MAX_MEM; /* Prevent GC steps. */ 515 g->gc.threshold = LJ_MAX_MEM; /* Prevent GC steps. */
478 copyTV(L, top, mo); 516 top = L->top;
479 setgcV(L, top+1, o, ~o->gch.gct); 517 copyTV(L, top++, mo);
480 errcode = lj_vm_pcall(L, top+1, 1+0, -1); /* Stack: |mo|o| -> | */ 518 if (LJ_FR2) setnilV(top++);
519 setgcV(L, top, o, ~o->gch.gct);
520 L->top = top+1;
521 errcode = lj_vm_pcall(L, top, 1+0, -1); /* Stack: |mo|o| -> | */
481 hook_restore(g, oldh); 522 hook_restore(g, oldh);
523 if (LJ_HASPROFILE && (oldh & HOOK_PROFILE)) lj_dispatch_update(g);
482 g->gc.threshold = oldt; /* Restore GC threshold. */ 524 g->gc.threshold = oldt; /* Restore GC threshold. */
483 if (errcode) 525 if (errcode) {
484 lj_err_throw(L, errcode); /* Propagate errors. */ 526 ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */
527 lj_vmevent_send(L, ERRFIN,
528 copyTV(L, L->top++, restorestack(L, errobj));
529 );
530 L->top--;
531 }
485} 532}
486 533
487/* Finalize one userdata or cdata object from the mmudata list. */ 534/* Finalize one userdata or cdata object from the mmudata list. */
@@ -490,7 +537,7 @@ static void gc_finalize(lua_State *L)
490 global_State *g = G(L); 537 global_State *g = G(L);
491 GCobj *o = gcnext(gcref(g->gc.mmudata)); 538 GCobj *o = gcnext(gcref(g->gc.mmudata));
492 cTValue *mo; 539 cTValue *mo;
493 lua_assert(gcref(g->jit_L) == NULL); /* Must not be called on trace. */ 540 lj_assertG(tvref(g->jit_base) == NULL, "finalizer called on trace");
494 /* Unchain from list of userdata to be finalized. */ 541 /* Unchain from list of userdata to be finalized. */
495 if (o == gcref(g->gc.mmudata)) 542 if (o == gcref(g->gc.mmudata))
496 setgcrefnull(g->gc.mmudata); 543 setgcrefnull(g->gc.mmudata);
@@ -565,9 +612,9 @@ void lj_gc_freeall(global_State *g)
565 /* Free everything, except super-fixed objects (the main thread). */ 612 /* Free everything, except super-fixed objects (the main thread). */
566 g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED; 613 g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED;
567 gc_fullsweep(g, &g->gc.root); 614 gc_fullsweep(g, &g->gc.root);
568 strmask = g->strmask; 615 strmask = g->str.mask;
569 for (i = 0; i <= strmask; i++) /* Free all string hash chains. */ 616 for (i = 0; i <= strmask; i++) /* Free all string hash chains. */
570 gc_fullsweep(g, &g->strhash[i]); 617 gc_sweepstr(g, &g->str.tab[i]);
571} 618}
572 619
573/* -- Collector ----------------------------------------------------------- */ 620/* -- Collector ----------------------------------------------------------- */
@@ -582,7 +629,7 @@ static void atomic(global_State *g, lua_State *L)
582 629
583 setgcrefr(g->gc.gray, g->gc.weak); /* Empty the list of weak tables. */ 630 setgcrefr(g->gc.gray, g->gc.weak); /* Empty the list of weak tables. */
584 setgcrefnull(g->gc.weak); 631 setgcrefnull(g->gc.weak);
585 lua_assert(!iswhite(obj2gco(mainthread(g)))); 632 lj_assertG(!iswhite(obj2gco(mainthread(g))), "main thread turned white");
586 gc_markobj(g, L); /* Mark running thread. */ 633 gc_markobj(g, L); /* Mark running thread. */
587 gc_traverse_curtrace(g); /* Traverse current trace. */ 634 gc_traverse_curtrace(g); /* Traverse current trace. */
588 gc_mark_gcroot(g); /* Mark GC roots (again). */ 635 gc_mark_gcroot(g); /* Mark GC roots (again). */
@@ -597,13 +644,15 @@ static void atomic(global_State *g, lua_State *L)
597 udsize += gc_propagate_gray(g); /* And propagate the marks. */ 644 udsize += gc_propagate_gray(g); /* And propagate the marks. */
598 645
599 /* All marking done, clear weak tables. */ 646 /* All marking done, clear weak tables. */
600 gc_clearweak(gcref(g->gc.weak)); 647 gc_clearweak(g, gcref(g->gc.weak));
648
649 lj_buf_shrink(L, &g->tmpbuf); /* Shrink temp buffer. */
601 650
602 /* Prepare for sweep phase. */ 651 /* Prepare for sweep phase. */
603 g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */ 652 g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */
604 g->strempty.marked = g->gc.currentwhite; 653 g->strempty.marked = g->gc.currentwhite;
605 setmref(g->gc.sweep, &g->gc.root); 654 setmref(g->gc.sweep, &g->gc.root);
606 g->gc.estimate = g->gc.total - (MSize)udsize; /* Initial estimate. */ 655 g->gc.estimate = g->gc.total - (GCSize)udsize; /* Initial estimate. */
607} 656}
608 657
609/* GC state machine. Returns a cost estimate for each step performed. */ 658/* GC state machine. Returns a cost estimate for each step performed. */
@@ -620,28 +669,29 @@ static size_t gc_onestep(lua_State *L)
620 g->gc.state = GCSatomic; /* End of mark phase. */ 669 g->gc.state = GCSatomic; /* End of mark phase. */
621 return 0; 670 return 0;
622 case GCSatomic: 671 case GCSatomic:
623 if (gcref(g->jit_L)) /* Don't run atomic phase on trace. */ 672 if (tvref(g->jit_base)) /* Don't run atomic phase on trace. */
624 return LJ_MAX_MEM; 673 return LJ_MAX_MEM;
625 atomic(g, L); 674 atomic(g, L);
626 g->gc.state = GCSsweepstring; /* Start of sweep phase. */ 675 g->gc.state = GCSsweepstring; /* Start of sweep phase. */
627 g->gc.sweepstr = 0; 676 g->gc.sweepstr = 0;
628 return 0; 677 return 0;
629 case GCSsweepstring: { 678 case GCSsweepstring: {
630 MSize old = g->gc.total; 679 GCSize old = g->gc.total;
631 gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]); /* Sweep one chain. */ 680 gc_sweepstr(g, &g->str.tab[g->gc.sweepstr++]); /* Sweep one chain. */
632 if (g->gc.sweepstr > g->strmask) 681 if (g->gc.sweepstr > g->str.mask)
633 g->gc.state = GCSsweep; /* All string hash chains sweeped. */ 682 g->gc.state = GCSsweep; /* All string hash chains sweeped. */
634 lua_assert(old >= g->gc.total); 683 lj_assertG(old >= g->gc.total, "sweep increased memory");
635 g->gc.estimate -= old - g->gc.total; 684 g->gc.estimate -= old - g->gc.total;
636 return GCSWEEPCOST; 685 return GCSWEEPCOST;
637 } 686 }
638 case GCSsweep: { 687 case GCSsweep: {
639 MSize old = g->gc.total; 688 GCSize old = g->gc.total;
640 setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX)); 689 setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX));
641 lua_assert(old >= g->gc.total); 690 lj_assertG(old >= g->gc.total, "sweep increased memory");
642 g->gc.estimate -= old - g->gc.total; 691 g->gc.estimate -= old - g->gc.total;
643 if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) { 692 if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) {
644 gc_shrink(g, L); 693 if (g->str.num <= (g->str.mask >> 2) && g->str.mask > LJ_MIN_STRTAB*2-1)
694 lj_str_resize(L, g->str.mask >> 1); /* Shrink string table. */
645 if (gcref(g->gc.mmudata)) { /* Need any finalizations? */ 695 if (gcref(g->gc.mmudata)) { /* Need any finalizations? */
646 g->gc.state = GCSfinalize; 696 g->gc.state = GCSfinalize;
647#if LJ_HASFFI 697#if LJ_HASFFI
@@ -656,9 +706,12 @@ static size_t gc_onestep(lua_State *L)
656 } 706 }
657 case GCSfinalize: 707 case GCSfinalize:
658 if (gcref(g->gc.mmudata) != NULL) { 708 if (gcref(g->gc.mmudata) != NULL) {
659 if (gcref(g->jit_L)) /* Don't call finalizers on trace. */ 709 GCSize old = g->gc.total;
710 if (tvref(g->jit_base)) /* Don't call finalizers on trace. */
660 return LJ_MAX_MEM; 711 return LJ_MAX_MEM;
661 gc_finalize(L); /* Finalize one userdata object. */ 712 gc_finalize(L); /* Finalize one userdata object. */
713 if (old >= g->gc.total && g->gc.estimate > old - g->gc.total)
714 g->gc.estimate -= old - g->gc.total;
662 if (g->gc.estimate > GCFINALIZECOST) 715 if (g->gc.estimate > GCFINALIZECOST)
663 g->gc.estimate -= GCFINALIZECOST; 716 g->gc.estimate -= GCFINALIZECOST;
664 return GCFINALIZECOST; 717 return GCFINALIZECOST;
@@ -670,7 +723,7 @@ static size_t gc_onestep(lua_State *L)
670 g->gc.debt = 0; 723 g->gc.debt = 0;
671 return 0; 724 return 0;
672 default: 725 default:
673 lua_assert(0); 726 lj_assertG(0, "bad GC state");
674 return 0; 727 return 0;
675 } 728 }
676} 729}
@@ -679,7 +732,7 @@ static size_t gc_onestep(lua_State *L)
679int LJ_FASTCALL lj_gc_step(lua_State *L) 732int LJ_FASTCALL lj_gc_step(lua_State *L)
680{ 733{
681 global_State *g = G(L); 734 global_State *g = G(L);
682 MSize lim; 735 GCSize lim;
683 int32_t ostate = g->vmstate; 736 int32_t ostate = g->vmstate;
684 setvmstate(g, GC); 737 setvmstate(g, GC);
685 lim = (GCSTEPSIZE/100) * g->gc.stepmul; 738 lim = (GCSTEPSIZE/100) * g->gc.stepmul;
@@ -688,13 +741,13 @@ int LJ_FASTCALL lj_gc_step(lua_State *L)
688 if (g->gc.total > g->gc.threshold) 741 if (g->gc.total > g->gc.threshold)
689 g->gc.debt += g->gc.total - g->gc.threshold; 742 g->gc.debt += g->gc.total - g->gc.threshold;
690 do { 743 do {
691 lim -= (MSize)gc_onestep(L); 744 lim -= (GCSize)gc_onestep(L);
692 if (g->gc.state == GCSpause) { 745 if (g->gc.state == GCSpause) {
693 g->gc.threshold = (g->gc.estimate/100) * g->gc.pause; 746 g->gc.threshold = (g->gc.estimate/100) * g->gc.pause;
694 g->vmstate = ostate; 747 g->vmstate = ostate;
695 return 1; /* Finished a GC cycle. */ 748 return 1; /* Finished a GC cycle. */
696 } 749 }
697 } while ((int32_t)lim > 0); 750 } while (sizeof(lim) == 8 ? ((int64_t)lim > 0) : ((int32_t)lim > 0));
698 if (g->gc.debt < GCSTEPSIZE) { 751 if (g->gc.debt < GCSTEPSIZE) {
699 g->gc.threshold = g->gc.total + GCSTEPSIZE; 752 g->gc.threshold = g->gc.total + GCSTEPSIZE;
700 g->vmstate = ostate; 753 g->vmstate = ostate;
@@ -718,8 +771,8 @@ void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L)
718/* Perform multiple GC steps. Called from JIT-compiled code. */ 771/* Perform multiple GC steps. Called from JIT-compiled code. */
719int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps) 772int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps)
720{ 773{
721 lua_State *L = gco2th(gcref(g->jit_L)); 774 lua_State *L = gco2th(gcref(g->cur_L));
722 L->base = mref(G(L)->jit_base, TValue); 775 L->base = tvref(G(L)->jit_base);
723 L->top = curr_topL(L); 776 L->top = curr_topL(L);
724 while (steps-- > 0 && lj_gc_step(L) == 0) 777 while (steps-- > 0 && lj_gc_step(L) == 0)
725 ; 778 ;
@@ -744,7 +797,8 @@ void lj_gc_fullgc(lua_State *L)
744 } 797 }
745 while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep) 798 while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep)
746 gc_onestep(L); /* Finish sweep. */ 799 gc_onestep(L); /* Finish sweep. */
747 lua_assert(g->gc.state == GCSfinalize || g->gc.state == GCSpause); 800 lj_assertG(g->gc.state == GCSfinalize || g->gc.state == GCSpause,
801 "bad GC state");
748 /* Now perform a full GC. */ 802 /* Now perform a full GC. */
749 g->gc.state = GCSpause; 803 g->gc.state = GCSpause;
750 do { gc_onestep(L); } while (g->gc.state != GCSpause); 804 do { gc_onestep(L); } while (g->gc.state != GCSpause);
@@ -757,9 +811,11 @@ void lj_gc_fullgc(lua_State *L)
757/* Move the GC propagation frontier forward. */ 811/* Move the GC propagation frontier forward. */
758void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v) 812void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v)
759{ 813{
760 lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o)); 814 lj_assertG(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o),
761 lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause); 815 "bad object states for forward barrier");
762 lua_assert(o->gch.gct != ~LJ_TTAB); 816 lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause,
817 "bad GC state");
818 lj_assertG(o->gch.gct != ~LJ_TTAB, "barrier object is not a table");
763 /* Preserve invariant during propagation. Otherwise it doesn't matter. */ 819 /* Preserve invariant during propagation. Otherwise it doesn't matter. */
764 if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) 820 if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
765 gc_mark(g, v); /* Move frontier forward. */ 821 gc_mark(g, v); /* Move frontier forward. */
@@ -796,7 +852,8 @@ void lj_gc_closeuv(global_State *g, GCupval *uv)
796 lj_gc_barrierf(g, o, gcV(&uv->tv)); 852 lj_gc_barrierf(g, o, gcV(&uv->tv));
797 } else { 853 } else {
798 makewhite(g, o); /* Make it white, i.e. sweep the upvalue. */ 854 makewhite(g, o); /* Make it white, i.e. sweep the upvalue. */
799 lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause); 855 lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause,
856 "bad GC state");
800 } 857 }
801 } 858 }
802} 859}
@@ -813,27 +870,29 @@ void lj_gc_barriertrace(global_State *g, uint32_t traceno)
813/* -- Allocator ----------------------------------------------------------- */ 870/* -- Allocator ----------------------------------------------------------- */
814 871
815/* Call pluggable memory allocator to allocate or resize a fragment. */ 872/* Call pluggable memory allocator to allocate or resize a fragment. */
816void *lj_mem_realloc(lua_State *L, void *p, MSize osz, MSize nsz) 873void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz)
817{ 874{
818 global_State *g = G(L); 875 global_State *g = G(L);
819 lua_assert((osz == 0) == (p == NULL)); 876 lj_assertG((osz == 0) == (p == NULL), "realloc API violation");
820 p = g->allocf(g->allocd, p, osz, nsz); 877 p = g->allocf(g->allocd, p, osz, nsz);
821 if (p == NULL && nsz > 0) 878 if (p == NULL && nsz > 0)
822 lj_err_mem(L); 879 lj_err_mem(L);
823 lua_assert((nsz == 0) == (p == NULL)); 880 lj_assertG((nsz == 0) == (p == NULL), "allocf API violation");
824 lua_assert(checkptr32(p)); 881 lj_assertG(checkptrGC(p),
882 "allocated memory address %p outside required range", p);
825 g->gc.total = (g->gc.total - osz) + nsz; 883 g->gc.total = (g->gc.total - osz) + nsz;
826 return p; 884 return p;
827} 885}
828 886
829/* Allocate new GC object and link it to the root set. */ 887/* Allocate new GC object and link it to the root set. */
830void * LJ_FASTCALL lj_mem_newgco(lua_State *L, MSize size) 888void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size)
831{ 889{
832 global_State *g = G(L); 890 global_State *g = G(L);
833 GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size); 891 GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size);
834 if (o == NULL) 892 if (o == NULL)
835 lj_err_mem(L); 893 lj_err_mem(L);
836 lua_assert(checkptr32(o)); 894 lj_assertG(checkptrGC(o),
895 "allocated memory address %p outside required range", o);
837 g->gc.total += size; 896 g->gc.total += size;
838 setgcrefr(o->gch.nextgc, g->gc.root); 897 setgcrefr(o->gch.nextgc, g->gc.root);
839 setgcref(g->gc.root, o); 898 setgcref(g->gc.root, o);
diff --git a/src/lj_gc.h b/src/lj_gc.h
index 8c8ed336..3ed790ca 100644
--- a/src/lj_gc.h
+++ b/src/lj_gc.h
@@ -81,8 +81,10 @@ LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno);
81static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t) 81static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t)
82{ 82{
83 GCobj *o = obj2gco(t); 83 GCobj *o = obj2gco(t);
84 lua_assert(isblack(o) && !isdead(g, o)); 84 lj_assertG(isblack(o) && !isdead(g, o),
85 lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause); 85 "bad object states for backward barrier");
86 lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause,
87 "bad GC state");
86 black2gray(o); 88 black2gray(o);
87 setgcrefr(t->gclist, g->gc.grayagain); 89 setgcrefr(t->gclist, g->gc.grayagain);
88 setgcref(g->gc.grayagain, o); 90 setgcref(g->gc.grayagain, o);
@@ -107,8 +109,8 @@ static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t)
107 lj_gc_barrierf(G(L), obj2gco(p), obj2gco(o)); } 109 lj_gc_barrierf(G(L), obj2gco(p), obj2gco(o)); }
108 110
109/* Allocator. */ 111/* Allocator. */
110LJ_FUNC void *lj_mem_realloc(lua_State *L, void *p, MSize osz, MSize nsz); 112LJ_FUNC void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz);
111LJ_FUNC void * LJ_FASTCALL lj_mem_newgco(lua_State *L, MSize size); 113LJ_FUNC void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size);
112LJ_FUNC void *lj_mem_grow(lua_State *L, void *p, 114LJ_FUNC void *lj_mem_grow(lua_State *L, void *p,
113 MSize *szp, MSize lim, MSize esz); 115 MSize *szp, MSize lim, MSize esz);
114 116
@@ -116,13 +118,13 @@ LJ_FUNC void *lj_mem_grow(lua_State *L, void *p,
116 118
117static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize) 119static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize)
118{ 120{
119 g->gc.total -= (MSize)osize; 121 g->gc.total -= (GCSize)osize;
120 g->allocf(g->allocd, p, osize, 0); 122 g->allocf(g->allocd, p, osize, 0);
121} 123}
122 124
123#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (MSize)((n)*sizeof(t)))) 125#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n)*sizeof(t))))
124#define lj_mem_reallocvec(L, p, on, n, t) \ 126#define lj_mem_reallocvec(L, p, on, n, t) \
125 ((p) = (t *)lj_mem_realloc(L, p, (on)*sizeof(t), (MSize)((n)*sizeof(t)))) 127 ((p) = (t *)lj_mem_realloc(L, p, (on)*sizeof(t), (GCSize)((n)*sizeof(t))))
126#define lj_mem_growvec(L, p, n, m, t) \ 128#define lj_mem_growvec(L, p, n, m, t) \
127 ((p) = (t *)lj_mem_grow(L, (p), &(n), (m), (MSize)sizeof(t))) 129 ((p) = (t *)lj_mem_grow(L, (p), &(n), (m), (MSize)sizeof(t)))
128#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n)*sizeof(t)) 130#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n)*sizeof(t))
diff --git a/src/lj_gdbjit.c b/src/lj_gdbjit.c
index 01f51ba7..e8a66635 100644
--- a/src/lj_gdbjit.c
+++ b/src/lj_gdbjit.c
@@ -14,6 +14,8 @@
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_debug.h" 15#include "lj_debug.h"
16#include "lj_frame.h" 16#include "lj_frame.h"
17#include "lj_buf.h"
18#include "lj_strfmt.h"
17#include "lj_jit.h" 19#include "lj_jit.h"
18#include "lj_dispatch.h" 20#include "lj_dispatch.h"
19 21
@@ -294,6 +296,9 @@ enum {
294#elif LJ_TARGET_ARM 296#elif LJ_TARGET_ARM
295 DW_REG_SP = 13, 297 DW_REG_SP = 13,
296 DW_REG_RA = 14, 298 DW_REG_RA = 14,
299#elif LJ_TARGET_ARM64
300 DW_REG_SP = 31,
301 DW_REG_RA = 30,
297#elif LJ_TARGET_PPC 302#elif LJ_TARGET_PPC
298 DW_REG_SP = 1, 303 DW_REG_SP = 1,
299 DW_REG_RA = 65, 304 DW_REG_RA = 65,
@@ -358,7 +363,7 @@ static const ELFheader elfhdr_template = {
358 .eosabi = 12, 363 .eosabi = 12,
359#elif defined(__DragonFly__) 364#elif defined(__DragonFly__)
360 .eosabi = 0, 365 .eosabi = 0,
361#elif (defined(__sun__) && defined(__svr4__)) 366#elif LJ_TARGET_SOLARIS
362 .eosabi = 6, 367 .eosabi = 6,
363#else 368#else
364 .eosabi = 0, 369 .eosabi = 0,
@@ -372,6 +377,8 @@ static const ELFheader elfhdr_template = {
372 .machine = 62, 377 .machine = 62,
373#elif LJ_TARGET_ARM 378#elif LJ_TARGET_ARM
374 .machine = 40, 379 .machine = 40,
380#elif LJ_TARGET_ARM64
381 .machine = 183,
375#elif LJ_TARGET_PPC 382#elif LJ_TARGET_PPC
376 .machine = 20, 383 .machine = 20,
377#elif LJ_TARGET_MIPS 384#elif LJ_TARGET_MIPS
@@ -428,16 +435,6 @@ static void gdbjit_catnum(GDBJITctx *ctx, uint32_t n)
428 *ctx->p++ = '0' + n; 435 *ctx->p++ = '0' + n;
429} 436}
430 437
431/* Add a ULEB128 value. */
432static void gdbjit_uleb128(GDBJITctx *ctx, uint32_t v)
433{
434 uint8_t *p = ctx->p;
435 for (; v >= 0x80; v >>= 7)
436 *p++ = (uint8_t)((v & 0x7f) | 0x80);
437 *p++ = (uint8_t)v;
438 ctx->p = p;
439}
440
441/* Add a SLEB128 value. */ 438/* Add a SLEB128 value. */
442static void gdbjit_sleb128(GDBJITctx *ctx, int32_t v) 439static void gdbjit_sleb128(GDBJITctx *ctx, int32_t v)
443{ 440{
@@ -454,7 +451,7 @@ static void gdbjit_sleb128(GDBJITctx *ctx, int32_t v)
454#define DU16(x) (*(uint16_t *)p = (x), p += 2) 451#define DU16(x) (*(uint16_t *)p = (x), p += 2)
455#define DU32(x) (*(uint32_t *)p = (x), p += 4) 452#define DU32(x) (*(uint32_t *)p = (x), p += 4)
456#define DADDR(x) (*(uintptr_t *)p = (x), p += sizeof(uintptr_t)) 453#define DADDR(x) (*(uintptr_t *)p = (x), p += sizeof(uintptr_t))
457#define DUV(x) (ctx->p = p, gdbjit_uleb128(ctx, (x)), p = ctx->p) 454#define DUV(x) (p = (uint8_t *)lj_strfmt_wuleb128((char *)p, (x)))
458#define DSV(x) (ctx->p = p, gdbjit_sleb128(ctx, (x)), p = ctx->p) 455#define DSV(x) (ctx->p = p, gdbjit_sleb128(ctx, (x)), p = ctx->p)
459#define DSTR(str) (ctx->p = p, gdbjit_strz(ctx, (str)), p = ctx->p) 456#define DSTR(str) (ctx->p = p, gdbjit_strz(ctx, (str)), p = ctx->p)
460#define DALIGNNOP(s) while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop 457#define DALIGNNOP(s) while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop
@@ -564,13 +561,20 @@ static void LJ_FASTCALL gdbjit_ehframe(GDBJITctx *ctx)
564 DB(DW_CFA_offset|DW_REG_15); DUV(4); 561 DB(DW_CFA_offset|DW_REG_15); DUV(4);
565 DB(DW_CFA_offset|DW_REG_14); DUV(5); 562 DB(DW_CFA_offset|DW_REG_14); DUV(5);
566 /* Extra registers saved for JIT-compiled code. */ 563 /* Extra registers saved for JIT-compiled code. */
567 DB(DW_CFA_offset|DW_REG_13); DUV(9); 564 DB(DW_CFA_offset|DW_REG_13); DUV(LJ_GC64 ? 10 : 9);
568 DB(DW_CFA_offset|DW_REG_12); DUV(10); 565 DB(DW_CFA_offset|DW_REG_12); DUV(LJ_GC64 ? 11 : 10);
569#elif LJ_TARGET_ARM 566#elif LJ_TARGET_ARM
570 { 567 {
571 int i; 568 int i;
572 for (i = 11; i >= 4; i--) { DB(DW_CFA_offset|i); DUV(2+(11-i)); } 569 for (i = 11; i >= 4; i--) { DB(DW_CFA_offset|i); DUV(2+(11-i)); }
573 } 570 }
571#elif LJ_TARGET_ARM64
572 {
573 int i;
574 DB(DW_CFA_offset|31); DUV(2);
575 for (i = 28; i >= 19; i--) { DB(DW_CFA_offset|i); DUV(3+(28-i)); }
576 for (i = 15; i >= 8; i--) { DB(DW_CFA_offset|32|i); DUV(28-i); }
577 }
574#elif LJ_TARGET_PPC 578#elif LJ_TARGET_PPC
575 { 579 {
576 int i; 580 int i;
@@ -720,13 +724,27 @@ static void gdbjit_buildobj(GDBJITctx *ctx)
720 SECTALIGN(ctx->p, sizeof(uintptr_t)); 724 SECTALIGN(ctx->p, sizeof(uintptr_t));
721 gdbjit_initsect(ctx, GDBJIT_SECT_eh_frame, gdbjit_ehframe); 725 gdbjit_initsect(ctx, GDBJIT_SECT_eh_frame, gdbjit_ehframe);
722 ctx->objsize = (size_t)((char *)ctx->p - (char *)obj); 726 ctx->objsize = (size_t)((char *)ctx->p - (char *)obj);
723 lua_assert(ctx->objsize < sizeof(GDBJITobj)); 727 lj_assertX(ctx->objsize < sizeof(GDBJITobj), "GDBJITobj overflow");
724} 728}
725 729
726#undef SECTALIGN 730#undef SECTALIGN
727 731
728/* -- Interface to GDB JIT API -------------------------------------------- */ 732/* -- Interface to GDB JIT API -------------------------------------------- */
729 733
734static int gdbjit_lock;
735
736static void gdbjit_lock_acquire()
737{
738 while (__sync_lock_test_and_set(&gdbjit_lock, 1)) {
739 /* Just spin; futexes or pthreads aren't worth the portability cost. */
740 }
741}
742
743static void gdbjit_lock_release()
744{
745 __sync_lock_release(&gdbjit_lock);
746}
747
730/* Add new entry to GDB JIT symbol chain. */ 748/* Add new entry to GDB JIT symbol chain. */
731static void gdbjit_newentry(lua_State *L, GDBJITctx *ctx) 749static void gdbjit_newentry(lua_State *L, GDBJITctx *ctx)
732{ 750{
@@ -738,6 +756,7 @@ static void gdbjit_newentry(lua_State *L, GDBJITctx *ctx)
738 ctx->T->gdbjit_entry = (void *)eo; 756 ctx->T->gdbjit_entry = (void *)eo;
739 /* Link new entry to chain and register it. */ 757 /* Link new entry to chain and register it. */
740 eo->entry.prev_entry = NULL; 758 eo->entry.prev_entry = NULL;
759 gdbjit_lock_acquire();
741 eo->entry.next_entry = __jit_debug_descriptor.first_entry; 760 eo->entry.next_entry = __jit_debug_descriptor.first_entry;
742 if (eo->entry.next_entry) 761 if (eo->entry.next_entry)
743 eo->entry.next_entry->prev_entry = &eo->entry; 762 eo->entry.next_entry->prev_entry = &eo->entry;
@@ -747,6 +766,7 @@ static void gdbjit_newentry(lua_State *L, GDBJITctx *ctx)
747 __jit_debug_descriptor.relevant_entry = &eo->entry; 766 __jit_debug_descriptor.relevant_entry = &eo->entry;
748 __jit_debug_descriptor.action_flag = GDBJIT_REGISTER; 767 __jit_debug_descriptor.action_flag = GDBJIT_REGISTER;
749 __jit_debug_register_code(); 768 __jit_debug_register_code();
769 gdbjit_lock_release();
750} 770}
751 771
752/* Add debug info for newly compiled trace and notify GDB. */ 772/* Add debug info for newly compiled trace and notify GDB. */
@@ -762,7 +782,8 @@ void lj_gdbjit_addtrace(jit_State *J, GCtrace *T)
762 ctx.spadjp = CFRAME_SIZE_JIT + 782 ctx.spadjp = CFRAME_SIZE_JIT +
763 (MSize)(parent ? traceref(J, parent)->spadjust : 0); 783 (MSize)(parent ? traceref(J, parent)->spadjust : 0);
764 ctx.spadj = CFRAME_SIZE_JIT + T->spadjust; 784 ctx.spadj = CFRAME_SIZE_JIT + T->spadjust;
765 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc); 785 lj_assertJ(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc,
786 "start PC out of range");
766 ctx.lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); 787 ctx.lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
767 ctx.filename = proto_chunknamestr(pt); 788 ctx.filename = proto_chunknamestr(pt);
768 if (*ctx.filename == '@' || *ctx.filename == '=') 789 if (*ctx.filename == '@' || *ctx.filename == '=')
@@ -778,6 +799,7 @@ void lj_gdbjit_deltrace(jit_State *J, GCtrace *T)
778{ 799{
779 GDBJITentryobj *eo = (GDBJITentryobj *)T->gdbjit_entry; 800 GDBJITentryobj *eo = (GDBJITentryobj *)T->gdbjit_entry;
780 if (eo) { 801 if (eo) {
802 gdbjit_lock_acquire();
781 if (eo->entry.prev_entry) 803 if (eo->entry.prev_entry)
782 eo->entry.prev_entry->next_entry = eo->entry.next_entry; 804 eo->entry.prev_entry->next_entry = eo->entry.next_entry;
783 else 805 else
@@ -787,6 +809,7 @@ void lj_gdbjit_deltrace(jit_State *J, GCtrace *T)
787 __jit_debug_descriptor.relevant_entry = &eo->entry; 809 __jit_debug_descriptor.relevant_entry = &eo->entry;
788 __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER; 810 __jit_debug_descriptor.action_flag = GDBJIT_UNREGISTER;
789 __jit_debug_register_code(); 811 __jit_debug_register_code();
812 gdbjit_lock_release();
790 lj_mem_free(J2G(J), eo, eo->sz); 813 lj_mem_free(J2G(J), eo, eo->sz);
791 } 814 }
792} 815}
diff --git a/src/lj_ir.c b/src/lj_ir.c
index cad40292..631b7cbc 100644
--- a/src/lj_ir.c
+++ b/src/lj_ir.c
@@ -15,6 +15,7 @@
15#if LJ_HASJIT 15#if LJ_HASJIT
16 16
17#include "lj_gc.h" 17#include "lj_gc.h"
18#include "lj_buf.h"
18#include "lj_str.h" 19#include "lj_str.h"
19#include "lj_tab.h" 20#include "lj_tab.h"
20#include "lj_ir.h" 21#include "lj_ir.h"
@@ -29,14 +30,16 @@
29#endif 30#endif
30#include "lj_vm.h" 31#include "lj_vm.h"
31#include "lj_strscan.h" 32#include "lj_strscan.h"
32#include "lj_lib.h" 33#include "lj_serialize.h"
34#include "lj_strfmt.h"
35#include "lj_prng.h"
33 36
34/* Some local macros to save typing. Undef'd at the end. */ 37/* Some local macros to save typing. Undef'd at the end. */
35#define IR(ref) (&J->cur.ir[(ref)]) 38#define IR(ref) (&J->cur.ir[(ref)])
36#define fins (&J->fold.ins) 39#define fins (&J->fold.ins)
37 40
38/* Pass IR on to next optimization in chain (FOLD). */ 41/* Pass IR on to next optimization in chain (FOLD). */
39#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) 42#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
40 43
41/* -- IR tables ----------------------------------------------------------- */ 44/* -- IR tables ----------------------------------------------------------- */
42 45
@@ -88,8 +91,9 @@ static void lj_ir_growbot(jit_State *J)
88{ 91{
89 IRIns *baseir = J->irbuf + J->irbotlim; 92 IRIns *baseir = J->irbuf + J->irbotlim;
90 MSize szins = J->irtoplim - J->irbotlim; 93 MSize szins = J->irtoplim - J->irbotlim;
91 lua_assert(szins != 0); 94 lj_assertJ(szins != 0, "zero IR size");
92 lua_assert(J->cur.nk == J->irbotlim); 95 lj_assertJ(J->cur.nk == J->irbotlim || J->cur.nk-1 == J->irbotlim,
96 "unexpected IR growth");
93 if (J->cur.nins + (szins >> 1) < J->irtoplim) { 97 if (J->cur.nins + (szins >> 1) < J->irtoplim) {
94 /* More than half of the buffer is free on top: shift up by a quarter. */ 98 /* More than half of the buffer is free on top: shift up by a quarter. */
95 MSize ofs = szins >> 2; 99 MSize ofs = szins >> 2;
@@ -143,6 +147,17 @@ TRef lj_ir_call(jit_State *J, IRCallID id, ...)
143 return emitir(CCI_OPTYPE(ci), tr, id); 147 return emitir(CCI_OPTYPE(ci), tr, id);
144} 148}
145 149
150/* Load field of type t from GG_State + offset. Must be 32 bit aligned. */
151TRef lj_ir_ggfload(jit_State *J, IRType t, uintptr_t ofs)
152{
153 lj_assertJ((ofs & 3) == 0, "unaligned GG_State field offset");
154 ofs >>= 2;
155 lj_assertJ(ofs >= IRFL__MAX && ofs <= 0x3ff,
156 "GG_State field offset breaks 10 bit FOLD key limit");
157 lj_ir_set(J, IRT(IR_FLOAD, t), REF_NIL, ofs);
158 return lj_opt_fold(J);
159}
160
146/* -- Interning of constants ---------------------------------------------- */ 161/* -- Interning of constants ---------------------------------------------- */
147 162
148/* 163/*
@@ -163,6 +178,24 @@ static LJ_AINLINE IRRef ir_nextk(jit_State *J)
163 return ref; 178 return ref;
164} 179}
165 180
181/* Get ref of next 64 bit IR constant and optionally grow IR.
182** Note: this may invalidate all IRIns *!
183*/
184static LJ_AINLINE IRRef ir_nextk64(jit_State *J)
185{
186 IRRef ref = J->cur.nk - 2;
187 lj_assertJ(J->state != LJ_TRACE_ASM, "bad JIT state");
188 if (LJ_UNLIKELY(ref < J->irbotlim)) lj_ir_growbot(J);
189 J->cur.nk = ref;
190 return ref;
191}
192
193#if LJ_GC64
194#define ir_nextkgc ir_nextk64
195#else
196#define ir_nextkgc ir_nextk
197#endif
198
166/* Intern int32_t constant. */ 199/* Intern int32_t constant. */
167TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k) 200TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k)
168{ 201{
@@ -182,79 +215,21 @@ found:
182 return TREF(ref, IRT_INT); 215 return TREF(ref, IRT_INT);
183} 216}
184 217
185/* The MRef inside the KNUM/KINT64 IR instructions holds the address of the 218/* Intern 64 bit constant, given by its 64 bit pattern. */
186** 64 bit constant. The constants themselves are stored in a chained array 219TRef lj_ir_k64(jit_State *J, IROp op, uint64_t u64)
187** and shared across traces.
188**
189** Rationale for choosing this data structure:
190** - The address of the constants is embedded in the generated machine code
191** and must never move. A resizable array or hash table wouldn't work.
192** - Most apps need very few non-32 bit integer constants (less than a dozen).
193** - Linear search is hard to beat in terms of speed and low complexity.
194*/
195typedef struct K64Array {
196 MRef next; /* Pointer to next list. */
197 MSize numk; /* Number of used elements in this array. */
198 TValue k[LJ_MIN_K64SZ]; /* Array of constants. */
199} K64Array;
200
201/* Free all chained arrays. */
202void lj_ir_k64_freeall(jit_State *J)
203{
204 K64Array *k;
205 for (k = mref(J->k64, K64Array); k; ) {
206 K64Array *next = mref(k->next, K64Array);
207 lj_mem_free(J2G(J), k, sizeof(K64Array));
208 k = next;
209 }
210}
211
212/* Find 64 bit constant in chained array or add it. */
213cTValue *lj_ir_k64_find(jit_State *J, uint64_t u64)
214{
215 K64Array *k, *kp = NULL;
216 TValue *ntv;
217 MSize idx;
218 /* Search for the constant in the whole chain of arrays. */
219 for (k = mref(J->k64, K64Array); k; k = mref(k->next, K64Array)) {
220 kp = k; /* Remember previous element in list. */
221 for (idx = 0; idx < k->numk; idx++) { /* Search one array. */
222 TValue *tv = &k->k[idx];
223 if (tv->u64 == u64) /* Needed for +-0/NaN/absmask. */
224 return tv;
225 }
226 }
227 /* Constant was not found, need to add it. */
228 if (!(kp && kp->numk < LJ_MIN_K64SZ)) { /* Allocate a new array. */
229 K64Array *kn = lj_mem_newt(J->L, sizeof(K64Array), K64Array);
230 setmref(kn->next, NULL);
231 kn->numk = 0;
232 if (kp)
233 setmref(kp->next, kn); /* Chain to the end of the list. */
234 else
235 setmref(J->k64, kn); /* Link first array. */
236 kp = kn;
237 }
238 ntv = &kp->k[kp->numk++]; /* Add to current array. */
239 ntv->u64 = u64;
240 return ntv;
241}
242
243/* Intern 64 bit constant, given by its address. */
244TRef lj_ir_k64(jit_State *J, IROp op, cTValue *tv)
245{ 220{
246 IRIns *ir, *cir = J->cur.ir; 221 IRIns *ir, *cir = J->cur.ir;
247 IRRef ref; 222 IRRef ref;
248 IRType t = op == IR_KNUM ? IRT_NUM : IRT_I64; 223 IRType t = op == IR_KNUM ? IRT_NUM : IRT_I64;
249 for (ref = J->chain[op]; ref; ref = cir[ref].prev) 224 for (ref = J->chain[op]; ref; ref = cir[ref].prev)
250 if (ir_k64(&cir[ref]) == tv) 225 if (ir_k64(&cir[ref])->u64 == u64)
251 goto found; 226 goto found;
252 ref = ir_nextk(J); 227 ref = ir_nextk64(J);
253 ir = IR(ref); 228 ir = IR(ref);
254 lua_assert(checkptr32(tv)); 229 ir[1].tv.u64 = u64;
255 setmref(ir->ptr, tv);
256 ir->t.irt = t; 230 ir->t.irt = t;
257 ir->o = op; 231 ir->o = op;
232 ir->op12 = 0;
258 ir->prev = J->chain[op]; 233 ir->prev = J->chain[op];
259 J->chain[op] = (IRRef1)ref; 234 J->chain[op] = (IRRef1)ref;
260found: 235found:
@@ -264,13 +239,13 @@ found:
264/* Intern FP constant, given by its 64 bit pattern. */ 239/* Intern FP constant, given by its 64 bit pattern. */
265TRef lj_ir_knum_u64(jit_State *J, uint64_t u64) 240TRef lj_ir_knum_u64(jit_State *J, uint64_t u64)
266{ 241{
267 return lj_ir_k64(J, IR_KNUM, lj_ir_k64_find(J, u64)); 242 return lj_ir_k64(J, IR_KNUM, u64);
268} 243}
269 244
270/* Intern 64 bit integer constant. */ 245/* Intern 64 bit integer constant. */
271TRef lj_ir_kint64(jit_State *J, uint64_t u64) 246TRef lj_ir_kint64(jit_State *J, uint64_t u64)
272{ 247{
273 return lj_ir_k64(J, IR_KINT64, lj_ir_k64_find(J, u64)); 248 return lj_ir_k64(J, IR_KINT64, u64);
274} 249}
275 250
276/* Check whether a number is int and return it. -0 is NOT considered an int. */ 251/* Check whether a number is int and return it. -0 is NOT considered an int. */
@@ -305,14 +280,15 @@ TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t)
305{ 280{
306 IRIns *ir, *cir = J->cur.ir; 281 IRIns *ir, *cir = J->cur.ir;
307 IRRef ref; 282 IRRef ref;
308 lua_assert(!isdead(J2G(J), o)); 283 lj_assertJ(!isdead(J2G(J), o), "interning of dead GC object");
309 for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev) 284 for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)
310 if (ir_kgc(&cir[ref]) == o) 285 if (ir_kgc(&cir[ref]) == o)
311 goto found; 286 goto found;
312 ref = ir_nextk(J); 287 ref = ir_nextkgc(J);
313 ir = IR(ref); 288 ir = IR(ref);
314 /* NOBARRIER: Current trace is a GC root. */ 289 /* NOBARRIER: Current trace is a GC root. */
315 setgcref(ir->gcr, o); 290 ir->op12 = 0;
291 setgcref(ir[LJ_GC64].gcr, o);
316 ir->t.irt = (uint8_t)t; 292 ir->t.irt = (uint8_t)t;
317 ir->o = IR_KGC; 293 ir->o = IR_KGC;
318 ir->prev = J->chain[IR_KGC]; 294 ir->prev = J->chain[IR_KGC];
@@ -321,24 +297,44 @@ found:
321 return TREF(ref, t); 297 return TREF(ref, t);
322} 298}
323 299
324/* Intern 32 bit pointer constant. */ 300/* Allocate GCtrace constant placeholder (no interning). */
301TRef lj_ir_ktrace(jit_State *J)
302{
303 IRRef ref = ir_nextkgc(J);
304 IRIns *ir = IR(ref);
305 lj_assertJ(irt_toitype_(IRT_P64) == LJ_TTRACE, "mismatched type mapping");
306 ir->t.irt = IRT_P64;
307 ir->o = LJ_GC64 ? IR_KNUM : IR_KNULL; /* Not IR_KGC yet, but same size. */
308 ir->op12 = 0;
309 ir->prev = 0;
310 return TREF(ref, IRT_P64);
311}
312
313/* Intern pointer constant. */
325TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr) 314TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr)
326{ 315{
327 IRIns *ir, *cir = J->cur.ir; 316 IRIns *ir, *cir = J->cur.ir;
328 IRRef ref; 317 IRRef ref;
329 lua_assert((void *)(intptr_t)i32ptr(ptr) == ptr); 318#if LJ_64 && !LJ_GC64
319 lj_assertJ((void *)(uintptr_t)u32ptr(ptr) == ptr, "out-of-range GC pointer");
320#endif
330 for (ref = J->chain[op]; ref; ref = cir[ref].prev) 321 for (ref = J->chain[op]; ref; ref = cir[ref].prev)
331 if (mref(cir[ref].ptr, void) == ptr) 322 if (ir_kptr(&cir[ref]) == ptr)
332 goto found; 323 goto found;
324#if LJ_GC64
325 ref = ir_nextk64(J);
326#else
333 ref = ir_nextk(J); 327 ref = ir_nextk(J);
328#endif
334 ir = IR(ref); 329 ir = IR(ref);
335 setmref(ir->ptr, ptr); 330 ir->op12 = 0;
336 ir->t.irt = IRT_P32; 331 setmref(ir[LJ_GC64].ptr, ptr);
332 ir->t.irt = IRT_PGC;
337 ir->o = op; 333 ir->o = op;
338 ir->prev = J->chain[op]; 334 ir->prev = J->chain[op];
339 J->chain[op] = (IRRef1)ref; 335 J->chain[op] = (IRRef1)ref;
340found: 336found:
341 return TREF(ref, IRT_P32); 337 return TREF(ref, IRT_PGC);
342} 338}
343 339
344/* Intern typed NULL constant. */ 340/* Intern typed NULL constant. */
@@ -367,7 +363,8 @@ TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot)
367 IRRef2 op12 = IRREF2((IRRef1)key, (IRRef1)slot); 363 IRRef2 op12 = IRREF2((IRRef1)key, (IRRef1)slot);
368 IRRef ref; 364 IRRef ref;
369 /* Const part is not touched by CSE/DCE, so 0-65535 is ok for IRMlit here. */ 365 /* Const part is not touched by CSE/DCE, so 0-65535 is ok for IRMlit here. */
370 lua_assert(tref_isk(key) && slot == (IRRef)(IRRef1)slot); 366 lj_assertJ(tref_isk(key) && slot == (IRRef)(IRRef1)slot,
367 "out-of-range key/slot");
371 for (ref = J->chain[IR_KSLOT]; ref; ref = cir[ref].prev) 368 for (ref = J->chain[IR_KSLOT]; ref; ref = cir[ref].prev)
372 if (cir[ref].op12 == op12) 369 if (cir[ref].op12 == op12)
373 goto found; 370 goto found;
@@ -388,14 +385,15 @@ found:
388void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir) 385void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir)
389{ 386{
390 UNUSED(L); 387 UNUSED(L);
391 lua_assert(ir->o != IR_KSLOT); /* Common mistake. */ 388 lj_assertL(ir->o != IR_KSLOT, "unexpected KSLOT"); /* Common mistake. */
392 switch (ir->o) { 389 switch (ir->o) {
393 case IR_KPRI: setitype(tv, irt_toitype(ir->t)); break; 390 case IR_KPRI: setpriV(tv, irt_toitype(ir->t)); break;
394 case IR_KINT: setintV(tv, ir->i); break; 391 case IR_KINT: setintV(tv, ir->i); break;
395 case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break; 392 case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break;
396 case IR_KPTR: case IR_KKPTR: case IR_KNULL: 393 case IR_KPTR: case IR_KKPTR:
397 setlightudV(tv, mref(ir->ptr, void)); 394 setnumV(tv, (lua_Number)(uintptr_t)ir_kptr(ir));
398 break; 395 break;
396 case IR_KNULL: setintV(tv, 0); break;
399 case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break; 397 case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break;
400#if LJ_HASFFI 398#if LJ_HASFFI
401 case IR_KINT64: { 399 case IR_KINT64: {
@@ -405,7 +403,7 @@ void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir)
405 break; 403 break;
406 } 404 }
407#endif 405#endif
408 default: lua_assert(0); break; 406 default: lj_assertL(0, "bad IR constant op %d", ir->o); break;
409 } 407 }
410} 408}
411 409
@@ -443,7 +441,8 @@ TRef LJ_FASTCALL lj_ir_tostr(jit_State *J, TRef tr)
443 if (!tref_isstr(tr)) { 441 if (!tref_isstr(tr)) {
444 if (!tref_isnumber(tr)) 442 if (!tref_isnumber(tr))
445 lj_trace_err(J, LJ_TRERR_BADTYPE); 443 lj_trace_err(J, LJ_TRERR_BADTYPE);
446 tr = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0); 444 tr = emitir(IRT(IR_TOSTR, IRT_STR), tr,
445 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
447 } 446 }
448 return tr; 447 return tr;
449} 448}
@@ -464,7 +463,7 @@ int lj_ir_numcmp(lua_Number a, lua_Number b, IROp op)
464 case IR_UGE: return !(a < b); 463 case IR_UGE: return !(a < b);
465 case IR_ULE: return !(a > b); 464 case IR_ULE: return !(a > b);
466 case IR_UGT: return !(a <= b); 465 case IR_UGT: return !(a <= b);
467 default: lua_assert(0); return 0; 466 default: lj_assertX(0, "bad IR op %d", op); return 0;
468 } 467 }
469} 468}
470 469
@@ -477,7 +476,7 @@ int lj_ir_strcmp(GCstr *a, GCstr *b, IROp op)
477 case IR_GE: return (res >= 0); 476 case IR_GE: return (res >= 0);
478 case IR_LE: return (res <= 0); 477 case IR_LE: return (res <= 0);
479 case IR_GT: return (res > 0); 478 case IR_GT: return (res > 0);
480 default: lua_assert(0); return 0; 479 default: lj_assertX(0, "bad IR op %d", op); return 0;
481 } 480 }
482} 481}
483 482
diff --git a/src/lj_ir.h b/src/lj_ir.h
index 9fd4e275..b32bd095 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -40,6 +40,7 @@
40 _(USE, S , ref, ___) \ 40 _(USE, S , ref, ___) \
41 _(PHI, S , ref, ref) \ 41 _(PHI, S , ref, ref) \
42 _(RENAME, S , ref, lit) \ 42 _(RENAME, S , ref, lit) \
43 _(PROF, S , ___, ___) \
43 \ 44 \
44 /* Constants. */ \ 45 /* Constants. */ \
45 _(KPRI, N , ___, ___) \ 46 _(KPRI, N , ___, ___) \
@@ -74,7 +75,6 @@
74 _(NEG, N , ref, ref) \ 75 _(NEG, N , ref, ref) \
75 \ 76 \
76 _(ABS, N , ref, ref) \ 77 _(ABS, N , ref, ref) \
77 _(ATAN2, N , ref, ref) \
78 _(LDEXP, N , ref, ref) \ 78 _(LDEXP, N , ref, ref) \
79 _(MIN, C , ref, ref) \ 79 _(MIN, C , ref, ref) \
80 _(MAX, C , ref, ref) \ 80 _(MAX, C , ref, ref) \
@@ -95,7 +95,9 @@
95 _(UREFO, LW, ref, lit) \ 95 _(UREFO, LW, ref, lit) \
96 _(UREFC, LW, ref, lit) \ 96 _(UREFC, LW, ref, lit) \
97 _(FREF, R , ref, lit) \ 97 _(FREF, R , ref, lit) \
98 _(TMPREF, S , ref, lit) \
98 _(STRREF, N , ref, ref) \ 99 _(STRREF, N , ref, ref) \
100 _(LREF, L , ___, ___) \
99 \ 101 \
100 /* Loads and Stores. These must be in the same order. */ \ 102 /* Loads and Stores. These must be in the same order. */ \
101 _(ALOAD, L , ref, ___) \ 103 _(ALOAD, L , ref, ___) \
@@ -104,7 +106,8 @@
104 _(FLOAD, L , ref, lit) \ 106 _(FLOAD, L , ref, lit) \
105 _(XLOAD, L , ref, lit) \ 107 _(XLOAD, L , ref, lit) \
106 _(SLOAD, L , lit, lit) \ 108 _(SLOAD, L , lit, lit) \
107 _(VLOAD, L , ref, ___) \ 109 _(VLOAD, L , ref, lit) \
110 _(ALEN, L , ref, ref) \
108 \ 111 \
109 _(ASTORE, S , ref, ref) \ 112 _(ASTORE, S , ref, ref) \
110 _(HSTORE, S , ref, ref) \ 113 _(HSTORE, S , ref, ref) \
@@ -120,6 +123,11 @@
120 _(CNEW, AW, ref, ref) \ 123 _(CNEW, AW, ref, ref) \
121 _(CNEWI, NW, ref, ref) /* CSE is ok, not marked as A. */ \ 124 _(CNEWI, NW, ref, ref) /* CSE is ok, not marked as A. */ \
122 \ 125 \
126 /* Buffer operations. */ \
127 _(BUFHDR, L , ref, lit) \
128 _(BUFPUT, LW, ref, ref) \
129 _(BUFSTR, AW, ref, ref) \
130 \
123 /* Barriers. */ \ 131 /* Barriers. */ \
124 _(TBAR, S , ref, ___) \ 132 _(TBAR, S , ref, ___) \
125 _(OBAR, S , ref, ref) \ 133 _(OBAR, S , ref, ref) \
@@ -128,12 +136,13 @@
128 /* Type conversions. */ \ 136 /* Type conversions. */ \
129 _(CONV, N , ref, lit) \ 137 _(CONV, N , ref, lit) \
130 _(TOBIT, N , ref, ref) \ 138 _(TOBIT, N , ref, ref) \
131 _(TOSTR, N , ref, ___) \ 139 _(TOSTR, N , ref, lit) \
132 _(STRTO, N , ref, ___) \ 140 _(STRTO, N , ref, ___) \
133 \ 141 \
134 /* Calls. */ \ 142 /* Calls. */ \
135 _(CALLN, N , ref, lit) \ 143 _(CALLN, NW, ref, lit) \
136 _(CALLL, L , ref, lit) \ 144 _(CALLA, AW, ref, lit) \
145 _(CALLL, LW, ref, lit) \
137 _(CALLS, S , ref, lit) \ 146 _(CALLS, S , ref, lit) \
138 _(CALLXS, S , ref, ref) \ 147 _(CALLXS, S , ref, ref) \
139 _(CARG, N , ref, ref) \ 148 _(CARG, N , ref, ref) \
@@ -170,8 +179,7 @@ LJ_STATIC_ASSERT((int)IR_XLOAD + IRDELTA_L2S == (int)IR_XSTORE);
170/* FPMATH sub-functions. ORDER FPM. */ 179/* FPMATH sub-functions. ORDER FPM. */
171#define IRFPMDEF(_) \ 180#define IRFPMDEF(_) \
172 _(FLOOR) _(CEIL) _(TRUNC) /* Must be first and in this order. */ \ 181 _(FLOOR) _(CEIL) _(TRUNC) /* Must be first and in this order. */ \
173 _(SQRT) _(EXP) _(EXP2) _(LOG) _(LOG2) _(LOG10) \ 182 _(SQRT) _(LOG) _(LOG2) \
174 _(SIN) _(COS) _(TAN) \
175 _(OTHER) 183 _(OTHER)
176 184
177typedef enum { 185typedef enum {
@@ -186,6 +194,8 @@ IRFPMDEF(FPMENUM)
186 _(STR_LEN, offsetof(GCstr, len)) \ 194 _(STR_LEN, offsetof(GCstr, len)) \
187 _(FUNC_ENV, offsetof(GCfunc, l.env)) \ 195 _(FUNC_ENV, offsetof(GCfunc, l.env)) \
188 _(FUNC_PC, offsetof(GCfunc, l.pc)) \ 196 _(FUNC_PC, offsetof(GCfunc, l.pc)) \
197 _(FUNC_FFID, offsetof(GCfunc, l.ffid)) \
198 _(THREAD_ENV, offsetof(lua_State, env)) \
189 _(TAB_META, offsetof(GCtab, metatable)) \ 199 _(TAB_META, offsetof(GCtab, metatable)) \
190 _(TAB_ARRAY, offsetof(GCtab, array)) \ 200 _(TAB_ARRAY, offsetof(GCtab, array)) \
191 _(TAB_NODE, offsetof(GCtab, node)) \ 201 _(TAB_NODE, offsetof(GCtab, node)) \
@@ -195,9 +205,15 @@ IRFPMDEF(FPMENUM)
195 _(UDATA_META, offsetof(GCudata, metatable)) \ 205 _(UDATA_META, offsetof(GCudata, metatable)) \
196 _(UDATA_UDTYPE, offsetof(GCudata, udtype)) \ 206 _(UDATA_UDTYPE, offsetof(GCudata, udtype)) \
197 _(UDATA_FILE, sizeof(GCudata)) \ 207 _(UDATA_FILE, sizeof(GCudata)) \
208 _(SBUF_W, sizeof(GCudata) + offsetof(SBufExt, w)) \
209 _(SBUF_E, sizeof(GCudata) + offsetof(SBufExt, e)) \
210 _(SBUF_B, sizeof(GCudata) + offsetof(SBufExt, b)) \
211 _(SBUF_L, sizeof(GCudata) + offsetof(SBufExt, L)) \
212 _(SBUF_REF, sizeof(GCudata) + offsetof(SBufExt, cowref)) \
213 _(SBUF_R, sizeof(GCudata) + offsetof(SBufExt, r)) \
198 _(CDATA_CTYPEID, offsetof(GCcdata, ctypeid)) \ 214 _(CDATA_CTYPEID, offsetof(GCcdata, ctypeid)) \
199 _(CDATA_PTR, sizeof(GCcdata)) \ 215 _(CDATA_PTR, sizeof(GCcdata)) \
200 _(CDATA_INT, sizeof(GCcdata)) \ 216 _(CDATA_INT, sizeof(GCcdata)) \
201 _(CDATA_INT64, sizeof(GCcdata)) \ 217 _(CDATA_INT64, sizeof(GCcdata)) \
202 _(CDATA_INT64_4, sizeof(GCcdata) + 4) 218 _(CDATA_INT64_4, sizeof(GCcdata) + 4)
203 219
@@ -208,18 +224,29 @@ IRFLDEF(FLENUM)
208 IRFL__MAX 224 IRFL__MAX
209} IRFieldID; 225} IRFieldID;
210 226
227/* TMPREF mode bits, stored in op2. */
228#define IRTMPREF_IN1 0x01 /* First input value. */
229#define IRTMPREF_OUT1 0x02 /* First output value. */
230#define IRTMPREF_OUT2 0x04 /* Second output value. */
231
211/* SLOAD mode bits, stored in op2. */ 232/* SLOAD mode bits, stored in op2. */
212#define IRSLOAD_PARENT 0x01 /* Coalesce with parent trace. */ 233#define IRSLOAD_PARENT 0x01 /* Coalesce with parent trace. */
213#define IRSLOAD_FRAME 0x02 /* Load hiword of frame. */ 234#define IRSLOAD_FRAME 0x02 /* Load 32 bits of ftsz. */
214#define IRSLOAD_TYPECHECK 0x04 /* Needs type check. */ 235#define IRSLOAD_TYPECHECK 0x04 /* Needs type check. */
215#define IRSLOAD_CONVERT 0x08 /* Number to integer conversion. */ 236#define IRSLOAD_CONVERT 0x08 /* Number to integer conversion. */
216#define IRSLOAD_READONLY 0x10 /* Read-only, omit slot store. */ 237#define IRSLOAD_READONLY 0x10 /* Read-only, omit slot store. */
217#define IRSLOAD_INHERIT 0x20 /* Inherited by exits/side traces. */ 238#define IRSLOAD_INHERIT 0x20 /* Inherited by exits/side traces. */
239#define IRSLOAD_KEYINDEX 0x40 /* Table traversal key index. */
218 240
219/* XLOAD mode, stored in op2. */ 241/* XLOAD mode bits, stored in op2. */
220#define IRXLOAD_READONLY 1 /* Load from read-only data. */ 242#define IRXLOAD_READONLY 0x01 /* Load from read-only data. */
221#define IRXLOAD_VOLATILE 2 /* Load from volatile data. */ 243#define IRXLOAD_VOLATILE 0x02 /* Load from volatile data. */
222#define IRXLOAD_UNALIGNED 4 /* Unaligned load. */ 244#define IRXLOAD_UNALIGNED 0x04 /* Unaligned load. */
245
246/* BUFHDR mode, stored in op2. */
247#define IRBUFHDR_RESET 0 /* Reset buffer. */
248#define IRBUFHDR_APPEND 1 /* Append to buffer. */
249#define IRBUFHDR_WRITE 2 /* Write to string buffer. */
223 250
224/* CONV mode, stored in op2. */ 251/* CONV mode, stored in op2. */
225#define IRCONV_SRCMASK 0x001f /* Source IRType. */ 252#define IRCONV_SRCMASK 0x001f /* Source IRType. */
@@ -227,7 +254,6 @@ IRFLDEF(FLENUM)
227#define IRCONV_DSH 5 254#define IRCONV_DSH 5
228#define IRCONV_NUM_INT ((IRT_NUM<<IRCONV_DSH)|IRT_INT) 255#define IRCONV_NUM_INT ((IRT_NUM<<IRCONV_DSH)|IRT_INT)
229#define IRCONV_INT_NUM ((IRT_INT<<IRCONV_DSH)|IRT_NUM) 256#define IRCONV_INT_NUM ((IRT_INT<<IRCONV_DSH)|IRT_NUM)
230#define IRCONV_TRUNC 0x0400 /* Truncate number to integer. */
231#define IRCONV_SEXT 0x0800 /* Sign-extend integer to integer. */ 257#define IRCONV_SEXT 0x0800 /* Sign-extend integer to integer. */
232#define IRCONV_MODEMASK 0x0fff 258#define IRCONV_MODEMASK 0x0fff
233#define IRCONV_CONVMASK 0xf000 259#define IRCONV_CONVMASK 0xf000
@@ -237,6 +263,12 @@ IRFLDEF(FLENUM)
237#define IRCONV_ANY (1<<IRCONV_CSH) /* Any FP number is ok. */ 263#define IRCONV_ANY (1<<IRCONV_CSH) /* Any FP number is ok. */
238#define IRCONV_INDEX (2<<IRCONV_CSH) /* Check + special backprop rules. */ 264#define IRCONV_INDEX (2<<IRCONV_CSH) /* Check + special backprop rules. */
239#define IRCONV_CHECK (3<<IRCONV_CSH) /* Number checked for integerness. */ 265#define IRCONV_CHECK (3<<IRCONV_CSH) /* Number checked for integerness. */
266#define IRCONV_NONE IRCONV_ANY /* INT|*64 no conv, but change type. */
267
268/* TOSTR mode, stored in op2. */
269#define IRTOSTR_INT 0 /* Convert integer to string. */
270#define IRTOSTR_NUM 1 /* Convert number to string. */
271#define IRTOSTR_CHAR 2 /* Convert char value to string. */
240 272
241/* -- IR operands --------------------------------------------------------- */ 273/* -- IR operands --------------------------------------------------------- */
242 274
@@ -276,7 +308,9 @@ LJ_DATA const uint8_t lj_ir_mode[IR__MAX+1];
276 308
277/* -- IR instruction types ------------------------------------------------ */ 309/* -- IR instruction types ------------------------------------------------ */
278 310
279/* Map of itypes to non-negative numbers. ORDER LJ_T. 311#define IRTSIZE_PGC (LJ_GC64 ? 8 : 4)
312
313/* Map of itypes to non-negative numbers and their sizes. ORDER LJ_T.
280** LJ_TUPVAL/LJ_TTRACE never appear in a TValue. Use these itypes for 314** LJ_TUPVAL/LJ_TTRACE never appear in a TValue. Use these itypes for
281** IRT_P32 and IRT_P64, which never escape the IR. 315** IRT_P32 and IRT_P64, which never escape the IR.
282** The various integers are only used in the IR and can only escape to 316** The various integers are only used in the IR and can only escape to
@@ -284,12 +318,13 @@ LJ_DATA const uint8_t lj_ir_mode[IR__MAX+1];
284** contiguous and next to IRT_NUM (see the typerange macros below). 318** contiguous and next to IRT_NUM (see the typerange macros below).
285*/ 319*/
286#define IRTDEF(_) \ 320#define IRTDEF(_) \
287 _(NIL, 4) _(FALSE, 4) _(TRUE, 4) _(LIGHTUD, LJ_64 ? 8 : 4) _(STR, 4) \ 321 _(NIL, 4) _(FALSE, 4) _(TRUE, 4) _(LIGHTUD, LJ_64 ? 8 : 4) \
288 _(P32, 4) _(THREAD, 4) _(PROTO, 4) _(FUNC, 4) _(P64, 8) _(CDATA, 4) \ 322 _(STR, IRTSIZE_PGC) _(P32, 4) _(THREAD, IRTSIZE_PGC) _(PROTO, IRTSIZE_PGC) \
289 _(TAB, 4) _(UDATA, 4) \ 323 _(FUNC, IRTSIZE_PGC) _(P64, 8) _(CDATA, IRTSIZE_PGC) _(TAB, IRTSIZE_PGC) \
324 _(UDATA, IRTSIZE_PGC) \
290 _(FLOAT, 4) _(NUM, 8) _(I8, 1) _(U8, 1) _(I16, 2) _(U16, 2) \ 325 _(FLOAT, 4) _(NUM, 8) _(I8, 1) _(U8, 1) _(I16, 2) _(U16, 2) \
291 _(INT, 4) _(U32, 4) _(I64, 8) _(U64, 8) \ 326 _(INT, 4) _(U32, 4) _(I64, 8) _(U64, 8) \
292 _(SOFTFP, 4) /* There is room for 9 more types. */ 327 _(SOFTFP, 4) /* There is room for 8 more types. */
293 328
294/* IR result type and flags (8 bit). */ 329/* IR result type and flags (8 bit). */
295typedef enum { 330typedef enum {
@@ -300,6 +335,8 @@ IRTDEF(IRTENUM)
300 335
301 /* Native pointer type and the corresponding integer type. */ 336 /* Native pointer type and the corresponding integer type. */
302 IRT_PTR = LJ_64 ? IRT_P64 : IRT_P32, 337 IRT_PTR = LJ_64 ? IRT_P64 : IRT_P32,
338 IRT_PGC = LJ_GC64 ? IRT_P64 : IRT_P32,
339 IRT_IGC = LJ_GC64 ? IRT_I64 : IRT_INT,
303 IRT_INTP = LJ_64 ? IRT_I64 : IRT_INT, 340 IRT_INTP = LJ_64 ? IRT_I64 : IRT_INT,
304 IRT_UINTP = LJ_64 ? IRT_U64 : IRT_U32, 341 IRT_UINTP = LJ_64 ? IRT_U64 : IRT_U32,
305 342
@@ -353,7 +390,14 @@ typedef struct IRType1 { uint8_t irt; } IRType1;
353#define irt_isaddr(t) (irt_typerange((t), IRT_LIGHTUD, IRT_UDATA)) 390#define irt_isaddr(t) (irt_typerange((t), IRT_LIGHTUD, IRT_UDATA))
354#define irt_isint64(t) (irt_typerange((t), IRT_I64, IRT_U64)) 391#define irt_isint64(t) (irt_typerange((t), IRT_I64, IRT_U64))
355 392
356#if LJ_64 393#if LJ_GC64
394/* Include IRT_NIL, so IR(ASMREF_L) (aka REF_NIL) is considered 64 bit. */
395#define IRT_IS64 \
396 ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|\
397 (1u<<IRT_LIGHTUD)|(1u<<IRT_STR)|(1u<<IRT_THREAD)|(1u<<IRT_PROTO)|\
398 (1u<<IRT_FUNC)|(1u<<IRT_CDATA)|(1u<<IRT_TAB)|(1u<<IRT_UDATA)|\
399 (1u<<IRT_NIL))
400#elif LJ_64
357#define IRT_IS64 \ 401#define IRT_IS64 \
358 ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|(1u<<IRT_LIGHTUD)) 402 ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|(1u<<IRT_LIGHTUD))
359#else 403#else
@@ -374,7 +418,7 @@ static LJ_AINLINE IRType itype2irt(const TValue *tv)
374 return IRT_INT; 418 return IRT_INT;
375 else if (tvisnum(tv)) 419 else if (tvisnum(tv))
376 return IRT_NUM; 420 return IRT_NUM;
377#if LJ_64 421#if LJ_64 && !LJ_GC64
378 else if (tvislightud(tv)) 422 else if (tvislightud(tv))
379 return IRT_LIGHTUD; 423 return IRT_LIGHTUD;
380#endif 424#endif
@@ -384,11 +428,12 @@ static LJ_AINLINE IRType itype2irt(const TValue *tv)
384 428
385static LJ_AINLINE uint32_t irt_toitype_(IRType t) 429static LJ_AINLINE uint32_t irt_toitype_(IRType t)
386{ 430{
387 lua_assert(!LJ_64 || t != IRT_LIGHTUD); 431 lj_assertX(!LJ_64 || LJ_GC64 || t != IRT_LIGHTUD,
432 "no plain type tag for lightuserdata");
388 if (LJ_DUALNUM && t > IRT_NUM) { 433 if (LJ_DUALNUM && t > IRT_NUM) {
389 return LJ_TISNUM; 434 return LJ_TISNUM;
390 } else { 435 } else {
391 lua_assert(t <= IRT_NUM); 436 lj_assertX(t <= IRT_NUM, "no plain type tag for IR type %d", t);
392 return ~(uint32_t)t; 437 return ~(uint32_t)t;
393 } 438 }
394} 439}
@@ -451,6 +496,7 @@ typedef uint32_t TRef;
451#define TREF_REFMASK 0x0000ffff 496#define TREF_REFMASK 0x0000ffff
452#define TREF_FRAME 0x00010000 497#define TREF_FRAME 0x00010000
453#define TREF_CONT 0x00020000 498#define TREF_CONT 0x00020000
499#define TREF_KEYINDEX 0x00100000
454 500
455#define TREF(ref, t) ((TRef)((ref) + ((t)<<24))) 501#define TREF(ref, t) ((TRef)((ref) + ((t)<<24)))
456 502
@@ -464,6 +510,7 @@ typedef uint32_t TRef;
464#define tref_isnil(tr) (tref_istype((tr), IRT_NIL)) 510#define tref_isnil(tr) (tref_istype((tr), IRT_NIL))
465#define tref_isfalse(tr) (tref_istype((tr), IRT_FALSE)) 511#define tref_isfalse(tr) (tref_istype((tr), IRT_FALSE))
466#define tref_istrue(tr) (tref_istype((tr), IRT_TRUE)) 512#define tref_istrue(tr) (tref_istype((tr), IRT_TRUE))
513#define tref_islightud(tr) (tref_istype((tr), IRT_LIGHTUD))
467#define tref_isstr(tr) (tref_istype((tr), IRT_STR)) 514#define tref_isstr(tr) (tref_istype((tr), IRT_STR))
468#define tref_isfunc(tr) (tref_istype((tr), IRT_FUNC)) 515#define tref_isfunc(tr) (tref_istype((tr), IRT_FUNC))
469#define tref_iscdata(tr) (tref_istype((tr), IRT_CDATA)) 516#define tref_iscdata(tr) (tref_istype((tr), IRT_CDATA))
@@ -496,7 +543,9 @@ typedef uint32_t TRef;
496** +-------+-------+---+---+---+---+ 543** +-------+-------+---+---+---+---+
497** | op1 | op2 | t | o | r | s | 544** | op1 | op2 | t | o | r | s |
498** +-------+-------+---+---+---+---+ 545** +-------+-------+---+---+---+---+
499** | op12/i/gco | ot | prev | (alternative fields in union) 546** | op12/i/gco32 | ot | prev | (alternative fields in union)
547** +-------+-------+---+---+---+---+
548** | TValue/gco64 | (2nd IR slot for 64 bit constants)
500** +---------------+-------+-------+ 549** +---------------+-------+-------+
501** 32 16 16 550** 32 16 16
502** 551**
@@ -524,21 +573,27 @@ typedef union IRIns {
524 ) 573 )
525 }; 574 };
526 int32_t i; /* 32 bit signed integer literal (overlaps op12). */ 575 int32_t i; /* 32 bit signed integer literal (overlaps op12). */
527 GCRef gcr; /* GCobj constant (overlaps op12). */ 576 GCRef gcr; /* GCobj constant (overlaps op12 or entire slot). */
528 MRef ptr; /* Pointer constant (overlaps op12). */ 577 MRef ptr; /* Pointer constant (overlaps op12 or entire slot). */
578 TValue tv; /* TValue constant (overlaps entire slot). */
529} IRIns; 579} IRIns;
530 580
531#define ir_kgc(ir) check_exp((ir)->o == IR_KGC, gcref((ir)->gcr)) 581#define ir_isk64(ir) \
582 ((ir)->o == IR_KNUM || (ir)->o == IR_KINT64 || \
583 (LJ_GC64 && \
584 ((ir)->o == IR_KGC || (ir)->o == IR_KPTR || (ir)->o == IR_KKPTR)))
585
586#define ir_kgc(ir) check_exp((ir)->o == IR_KGC, gcref((ir)[LJ_GC64].gcr))
532#define ir_kstr(ir) (gco2str(ir_kgc((ir)))) 587#define ir_kstr(ir) (gco2str(ir_kgc((ir))))
533#define ir_ktab(ir) (gco2tab(ir_kgc((ir)))) 588#define ir_ktab(ir) (gco2tab(ir_kgc((ir))))
534#define ir_kfunc(ir) (gco2func(ir_kgc((ir)))) 589#define ir_kfunc(ir) (gco2func(ir_kgc((ir))))
535#define ir_kcdata(ir) (gco2cd(ir_kgc((ir)))) 590#define ir_kcdata(ir) (gco2cd(ir_kgc((ir))))
536#define ir_knum(ir) check_exp((ir)->o == IR_KNUM, mref((ir)->ptr, cTValue)) 591#define ir_knum(ir) check_exp((ir)->o == IR_KNUM, &(ir)[1].tv)
537#define ir_kint64(ir) check_exp((ir)->o == IR_KINT64, mref((ir)->ptr,cTValue)) 592#define ir_kint64(ir) check_exp((ir)->o == IR_KINT64, &(ir)[1].tv)
538#define ir_k64(ir) \ 593#define ir_k64(ir) check_exp(ir_isk64(ir), &(ir)[1].tv)
539 check_exp((ir)->o == IR_KNUM || (ir)->o == IR_KINT64, mref((ir)->ptr,cTValue))
540#define ir_kptr(ir) \ 594#define ir_kptr(ir) \
541 check_exp((ir)->o == IR_KPTR || (ir)->o == IR_KKPTR, mref((ir)->ptr, void)) 595 check_exp((ir)->o == IR_KPTR || (ir)->o == IR_KKPTR, \
596 mref((ir)[LJ_GC64].ptr, void))
542 597
543/* A store or any other op with a non-weak guard has a side-effect. */ 598/* A store or any other op with a non-weak guard has a side-effect. */
544static LJ_AINLINE int ir_sideeff(IRIns *ir) 599static LJ_AINLINE int ir_sideeff(IRIns *ir)
diff --git a/src/lj_ircall.h b/src/lj_ircall.h
index 4e99b8b5..569134e9 100644
--- a/src/lj_ircall.h
+++ b/src/lj_ircall.h
@@ -16,22 +16,26 @@ typedef struct CCallInfo {
16 uint32_t flags; /* Number of arguments and flags. */ 16 uint32_t flags; /* Number of arguments and flags. */
17} CCallInfo; 17} CCallInfo;
18 18
19#define CCI_NARGS(ci) ((ci)->flags & 0xff) /* Extract # of args. */ 19#define CCI_NARGS(ci) ((ci)->flags & 0xff) /* # of args. */
20#define CCI_NARGS_MAX 32 /* Max. # of args. */ 20#define CCI_NARGS_MAX 32 /* Max. # of args. */
21 21
22#define CCI_OTSHIFT 16 22#define CCI_OTSHIFT 16
23#define CCI_OPTYPE(ci) ((ci)->flags >> CCI_OTSHIFT) /* Get op/type. */ 23#define CCI_OPTYPE(ci) ((ci)->flags >> CCI_OTSHIFT) /* Get op/type. */
24#define CCI_TYPE(ci) (((ci)->flags>>CCI_OTSHIFT) & IRT_TYPE)
24#define CCI_OPSHIFT 24 25#define CCI_OPSHIFT 24
25#define CCI_OP(ci) ((ci)->flags >> CCI_OPSHIFT) /* Get op. */ 26#define CCI_OP(ci) ((ci)->flags >> CCI_OPSHIFT) /* Get op. */
26 27
27#define CCI_CALL_N (IR_CALLN << CCI_OPSHIFT) 28#define CCI_CALL_N (IR_CALLN << CCI_OPSHIFT)
29#define CCI_CALL_A (IR_CALLA << CCI_OPSHIFT)
28#define CCI_CALL_L (IR_CALLL << CCI_OPSHIFT) 30#define CCI_CALL_L (IR_CALLL << CCI_OPSHIFT)
29#define CCI_CALL_S (IR_CALLS << CCI_OPSHIFT) 31#define CCI_CALL_S (IR_CALLS << CCI_OPSHIFT)
30#define CCI_CALL_FN (CCI_CALL_N|CCI_CC_FASTCALL) 32#define CCI_CALL_FN (CCI_CALL_N|CCI_CC_FASTCALL)
33#define CCI_CALL_FA (CCI_CALL_A|CCI_CC_FASTCALL)
31#define CCI_CALL_FL (CCI_CALL_L|CCI_CC_FASTCALL) 34#define CCI_CALL_FL (CCI_CALL_L|CCI_CC_FASTCALL)
32#define CCI_CALL_FS (CCI_CALL_S|CCI_CC_FASTCALL) 35#define CCI_CALL_FS (CCI_CALL_S|CCI_CC_FASTCALL)
33 36
34/* C call info flags. */ 37/* C call info flags. */
38#define CCI_T (IRT_GUARD << CCI_OTSHIFT) /* May throw. */
35#define CCI_L 0x0100 /* Implicit L arg. */ 39#define CCI_L 0x0100 /* Implicit L arg. */
36#define CCI_CASTU64 0x0200 /* Cast u64 result to number. */ 40#define CCI_CASTU64 0x0200 /* Cast u64 result to number. */
37#define CCI_NOFPRCLOBBER 0x0400 /* Does not clobber any FPRs. */ 41#define CCI_NOFPRCLOBBER 0x0400 /* Does not clobber any FPRs. */
@@ -45,6 +49,17 @@ typedef struct CCallInfo {
45#define CCI_CC_FASTCALL 0x2000 /* Fastcall calling convention. */ 49#define CCI_CC_FASTCALL 0x2000 /* Fastcall calling convention. */
46#define CCI_CC_STDCALL 0x3000 /* Stdcall calling convention. */ 50#define CCI_CC_STDCALL 0x3000 /* Stdcall calling convention. */
47 51
52/* Extra args for SOFTFP, SPLIT 64 bit. */
53#define CCI_XARGS_SHIFT 14
54#define CCI_XARGS(ci) (((ci)->flags >> CCI_XARGS_SHIFT) & 3)
55#define CCI_XA (1u << CCI_XARGS_SHIFT)
56
57#if LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)
58#define CCI_XNARGS(ci) (CCI_NARGS((ci)) + CCI_XARGS((ci)))
59#else
60#define CCI_XNARGS(ci) CCI_NARGS((ci))
61#endif
62
48/* Helpers for conditional function definitions. */ 63/* Helpers for conditional function definitions. */
49#define IRCALLCOND_ANY(x) x 64#define IRCALLCOND_ANY(x) x
50 65
@@ -66,6 +81,18 @@ typedef struct CCallInfo {
66#define IRCALLCOND_SOFTFP_FFI(x) NULL 81#define IRCALLCOND_SOFTFP_FFI(x) NULL
67#endif 82#endif
68 83
84#if LJ_SOFTFP && LJ_TARGET_MIPS
85#define IRCALLCOND_SOFTFP_MIPS(x) x
86#else
87#define IRCALLCOND_SOFTFP_MIPS(x) NULL
88#endif
89
90#if LJ_SOFTFP && LJ_TARGET_MIPS64
91#define IRCALLCOND_SOFTFP_MIPS64(x) x
92#else
93#define IRCALLCOND_SOFTFP_MIPS64(x) NULL
94#endif
95
69#define LJ_NEED_FP64 (LJ_TARGET_ARM || LJ_TARGET_PPC || LJ_TARGET_MIPS) 96#define LJ_NEED_FP64 (LJ_TARGET_ARM || LJ_TARGET_PPC || LJ_TARGET_MIPS)
70 97
71#if LJ_HASFFI && (LJ_SOFTFP || LJ_NEED_FP64) 98#if LJ_HASFFI && (LJ_SOFTFP || LJ_NEED_FP64)
@@ -86,93 +113,158 @@ typedef struct CCallInfo {
86#define IRCALLCOND_FFI32(x) NULL 113#define IRCALLCOND_FFI32(x) NULL
87#endif 114#endif
88 115
116#if LJ_HASBUFFER
117#define IRCALLCOND_BUFFER(x) x
118#else
119#define IRCALLCOND_BUFFER(x) NULL
120#endif
121
122#if LJ_HASBUFFER && LJ_HASFFI
123#define IRCALLCOND_BUFFFI(x) x
124#else
125#define IRCALLCOND_BUFFFI(x) NULL
126#endif
127
89#if LJ_SOFTFP 128#if LJ_SOFTFP
90#define ARG1_FP 2 /* Treat as 2 32 bit arguments. */ 129#define XA_FP CCI_XA
130#define XA2_FP (CCI_XA+CCI_XA)
91#else 131#else
92#define ARG1_FP 1 132#define XA_FP 0
133#define XA2_FP 0
134#endif
135
136#if LJ_SOFTFP32
137#define XA_FP32 CCI_XA
138#define XA2_FP32 (CCI_XA+CCI_XA)
139#else
140#define XA_FP32 0
141#define XA2_FP32 0
93#endif 142#endif
94 143
95#if LJ_32 144#if LJ_32
96#define ARG2_64 4 /* Treat as 4 32 bit arguments. */ 145#define XA_64 CCI_XA
146#define XA2_64 (CCI_XA+CCI_XA)
97#else 147#else
98#define ARG2_64 2 148#define XA_64 0
149#define XA2_64 0
99#endif 150#endif
100 151
101/* Function definitions for CALL* instructions. */ 152/* Function definitions for CALL* instructions. */
102#define IRCALLDEF(_) \ 153#define IRCALLDEF(_) \
103 _(ANY, lj_str_cmp, 2, FN, INT, CCI_NOFPRCLOBBER) \ 154 _(ANY, lj_str_cmp, 2, FN, INT, CCI_NOFPRCLOBBER) \
104 _(ANY, lj_str_new, 3, S, STR, CCI_L) \ 155 _(ANY, lj_str_find, 4, N, PGC, 0) \
156 _(ANY, lj_str_new, 3, S, STR, CCI_L|CCI_T) \
105 _(ANY, lj_strscan_num, 2, FN, INT, 0) \ 157 _(ANY, lj_strscan_num, 2, FN, INT, 0) \
106 _(ANY, lj_str_fromint, 2, FN, STR, CCI_L) \ 158 _(ANY, lj_strfmt_int, 2, FN, STR, CCI_L|CCI_T) \
107 _(ANY, lj_str_fromnum, 2, FN, STR, CCI_L) \ 159 _(ANY, lj_strfmt_num, 2, FN, STR, CCI_L|CCI_T) \
108 _(ANY, lj_tab_new1, 2, FS, TAB, CCI_L) \ 160 _(ANY, lj_strfmt_char, 2, FN, STR, CCI_L|CCI_T) \
109 _(ANY, lj_tab_dup, 2, FS, TAB, CCI_L) \ 161 _(ANY, lj_strfmt_putint, 2, FL, PGC, CCI_T) \
110 _(ANY, lj_tab_newkey, 3, S, P32, CCI_L) \ 162 _(ANY, lj_strfmt_putnum, 2, FL, PGC, CCI_T) \
163 _(ANY, lj_strfmt_putquoted, 2, FL, PGC, CCI_T) \
164 _(ANY, lj_strfmt_putfxint, 3, L, PGC, XA_64|CCI_T) \
165 _(ANY, lj_strfmt_putfnum_int, 3, L, PGC, XA_FP|CCI_T) \
166 _(ANY, lj_strfmt_putfnum_uint, 3, L, PGC, XA_FP|CCI_T) \
167 _(ANY, lj_strfmt_putfnum, 3, L, PGC, XA_FP|CCI_T) \
168 _(ANY, lj_strfmt_putfstr, 3, L, PGC, CCI_T) \
169 _(ANY, lj_strfmt_putfchar, 3, L, PGC, CCI_T) \
170 _(ANY, lj_buf_putmem, 3, S, PGC, CCI_T) \
171 _(ANY, lj_buf_putstr, 2, FL, PGC, CCI_T) \
172 _(ANY, lj_buf_putchar, 2, FL, PGC, CCI_T) \
173 _(ANY, lj_buf_putstr_reverse, 2, FL, PGC, CCI_T) \
174 _(ANY, lj_buf_putstr_lower, 2, FL, PGC, CCI_T) \
175 _(ANY, lj_buf_putstr_upper, 2, FL, PGC, CCI_T) \
176 _(ANY, lj_buf_putstr_rep, 3, L, PGC, CCI_T) \
177 _(ANY, lj_buf_puttab, 5, L, PGC, CCI_T) \
178 _(BUFFER, lj_bufx_set, 4, S, NIL, 0) \
179 _(BUFFFI, lj_bufx_more, 2, FS, INT, CCI_T) \
180 _(BUFFER, lj_serialize_put, 2, FS, PGC, CCI_T) \
181 _(BUFFER, lj_serialize_get, 2, FS, PTR, CCI_T) \
182 _(BUFFER, lj_serialize_encode, 2, FA, STR, CCI_L|CCI_T) \
183 _(BUFFER, lj_serialize_decode, 3, A, INT, CCI_L|CCI_T) \
184 _(ANY, lj_buf_tostr, 1, FL, STR, CCI_T) \
185 _(ANY, lj_tab_new_ah, 3, A, TAB, CCI_L|CCI_T) \
186 _(ANY, lj_tab_new1, 2, FA, TAB, CCI_L|CCI_T) \
187 _(ANY, lj_tab_dup, 2, FA, TAB, CCI_L|CCI_T) \
188 _(ANY, lj_tab_clear, 1, FS, NIL, 0) \
189 _(ANY, lj_tab_newkey, 3, S, PGC, CCI_L|CCI_T) \
190 _(ANY, lj_tab_keyindex, 2, FL, INT, 0) \
191 _(ANY, lj_vm_next, 2, FL, PTR, 0) \
111 _(ANY, lj_tab_len, 1, FL, INT, 0) \ 192 _(ANY, lj_tab_len, 1, FL, INT, 0) \
193 _(ANY, lj_tab_len_hint, 2, FL, INT, 0) \
112 _(ANY, lj_gc_step_jit, 2, FS, NIL, CCI_L) \ 194 _(ANY, lj_gc_step_jit, 2, FS, NIL, CCI_L) \
113 _(ANY, lj_gc_barrieruv, 2, FS, NIL, 0) \ 195 _(ANY, lj_gc_barrieruv, 2, FS, NIL, 0) \
114 _(ANY, lj_mem_newgco, 2, FS, P32, CCI_L) \ 196 _(ANY, lj_mem_newgco, 2, FA, PGC, CCI_L|CCI_T) \
115 _(ANY, lj_math_random_step, 1, FS, NUM, CCI_CASTU64) \ 197 _(ANY, lj_prng_u64d, 1, FS, NUM, CCI_CASTU64) \
116 _(ANY, lj_vm_modi, 2, FN, INT, 0) \ 198 _(ANY, lj_vm_modi, 2, FN, INT, 0) \
117 _(ANY, sinh, ARG1_FP, N, NUM, 0) \ 199 _(ANY, log10, 1, N, NUM, XA_FP) \
118 _(ANY, cosh, ARG1_FP, N, NUM, 0) \ 200 _(ANY, exp, 1, N, NUM, XA_FP) \
119 _(ANY, tanh, ARG1_FP, N, NUM, 0) \ 201 _(ANY, sin, 1, N, NUM, XA_FP) \
120 _(ANY, fputc, 2, S, INT, 0) \ 202 _(ANY, cos, 1, N, NUM, XA_FP) \
121 _(ANY, fwrite, 4, S, INT, 0) \ 203 _(ANY, tan, 1, N, NUM, XA_FP) \
122 _(ANY, fflush, 1, S, INT, 0) \ 204 _(ANY, asin, 1, N, NUM, XA_FP) \
205 _(ANY, acos, 1, N, NUM, XA_FP) \
206 _(ANY, atan, 1, N, NUM, XA_FP) \
207 _(ANY, sinh, 1, N, NUM, XA_FP) \
208 _(ANY, cosh, 1, N, NUM, XA_FP) \
209 _(ANY, tanh, 1, N, NUM, XA_FP) \
210 _(ANY, fputc, 2, S, INT, 0) \
211 _(ANY, fwrite, 4, S, INT, 0) \
212 _(ANY, fflush, 1, S, INT, 0) \
123 /* ORDER FPM */ \ 213 /* ORDER FPM */ \
124 _(FPMATH, lj_vm_floor, ARG1_FP, N, NUM, 0) \ 214 _(FPMATH, lj_vm_floor, 1, N, NUM, XA_FP) \
125 _(FPMATH, lj_vm_ceil, ARG1_FP, N, NUM, 0) \ 215 _(FPMATH, lj_vm_ceil, 1, N, NUM, XA_FP) \
126 _(FPMATH, lj_vm_trunc, ARG1_FP, N, NUM, 0) \ 216 _(FPMATH, lj_vm_trunc, 1, N, NUM, XA_FP) \
127 _(FPMATH, sqrt, ARG1_FP, N, NUM, 0) \ 217 _(FPMATH, sqrt, 1, N, NUM, XA_FP) \
128 _(FPMATH, exp, ARG1_FP, N, NUM, 0) \ 218 _(ANY, log, 1, N, NUM, XA_FP) \
129 _(FPMATH, lj_vm_exp2, ARG1_FP, N, NUM, 0) \ 219 _(ANY, lj_vm_log2, 1, N, NUM, XA_FP) \
130 _(FPMATH, log, ARG1_FP, N, NUM, 0) \ 220 _(ANY, pow, 2, N, NUM, XA2_FP) \
131 _(FPMATH, lj_vm_log2, ARG1_FP, N, NUM, 0) \ 221 _(ANY, atan2, 2, N, NUM, XA2_FP) \
132 _(FPMATH, log10, ARG1_FP, N, NUM, 0) \ 222 _(ANY, ldexp, 2, N, NUM, XA_FP) \
133 _(FPMATH, sin, ARG1_FP, N, NUM, 0) \ 223 _(SOFTFP, lj_vm_tobit, 1, N, INT, XA_FP32) \
134 _(FPMATH, cos, ARG1_FP, N, NUM, 0) \ 224 _(SOFTFP, softfp_add, 2, N, NUM, XA2_FP32) \
135 _(FPMATH, tan, ARG1_FP, N, NUM, 0) \ 225 _(SOFTFP, softfp_sub, 2, N, NUM, XA2_FP32) \
136 _(FPMATH, lj_vm_powi, ARG1_FP+1, N, NUM, 0) \ 226 _(SOFTFP, softfp_mul, 2, N, NUM, XA2_FP32) \
137 _(FPMATH, pow, ARG1_FP*2, N, NUM, 0) \ 227 _(SOFTFP, softfp_div, 2, N, NUM, XA2_FP32) \
138 _(FPMATH, atan2, ARG1_FP*2, N, NUM, 0) \ 228 _(SOFTFP, softfp_cmp, 2, N, NIL, XA2_FP32) \
139 _(FPMATH, ldexp, ARG1_FP+1, N, NUM, 0) \
140 _(SOFTFP, lj_vm_tobit, 2, N, INT, 0) \
141 _(SOFTFP, softfp_add, 4, N, NUM, 0) \
142 _(SOFTFP, softfp_sub, 4, N, NUM, 0) \
143 _(SOFTFP, softfp_mul, 4, N, NUM, 0) \
144 _(SOFTFP, softfp_div, 4, N, NUM, 0) \
145 _(SOFTFP, softfp_cmp, 4, N, NIL, 0) \
146 _(SOFTFP, softfp_i2d, 1, N, NUM, 0) \ 229 _(SOFTFP, softfp_i2d, 1, N, NUM, 0) \
147 _(SOFTFP, softfp_d2i, 2, N, INT, 0) \ 230 _(SOFTFP, softfp_d2i, 1, N, INT, XA_FP32) \
231 _(SOFTFP_MIPS, lj_vm_sfmin, 2, N, NUM, XA2_FP32) \
232 _(SOFTFP_MIPS, lj_vm_sfmax, 2, N, NUM, XA2_FP32) \
233 _(SOFTFP_MIPS64, lj_vm_tointg, 1, N, INT, 0) \
148 _(SOFTFP_FFI, softfp_ui2d, 1, N, NUM, 0) \ 234 _(SOFTFP_FFI, softfp_ui2d, 1, N, NUM, 0) \
149 _(SOFTFP_FFI, softfp_f2d, 1, N, NUM, 0) \ 235 _(SOFTFP_FFI, softfp_f2d, 1, N, NUM, 0) \
150 _(SOFTFP_FFI, softfp_d2ui, 2, N, INT, 0) \ 236 _(SOFTFP_FFI, softfp_d2ui, 1, N, INT, XA_FP32) \
151 _(SOFTFP_FFI, softfp_d2f, 2, N, FLOAT, 0) \ 237 _(SOFTFP_FFI, softfp_d2f, 1, N, FLOAT, XA_FP32) \
152 _(SOFTFP_FFI, softfp_i2f, 1, N, FLOAT, 0) \ 238 _(SOFTFP_FFI, softfp_i2f, 1, N, FLOAT, 0) \
153 _(SOFTFP_FFI, softfp_ui2f, 1, N, FLOAT, 0) \ 239 _(SOFTFP_FFI, softfp_ui2f, 1, N, FLOAT, 0) \
154 _(SOFTFP_FFI, softfp_f2i, 1, N, INT, 0) \ 240 _(SOFTFP_FFI, softfp_f2i, 1, N, INT, 0) \
155 _(SOFTFP_FFI, softfp_f2ui, 1, N, INT, 0) \ 241 _(SOFTFP_FFI, softfp_f2ui, 1, N, INT, 0) \
156 _(FP64_FFI, fp64_l2d, 2, N, NUM, 0) \ 242 _(FP64_FFI, fp64_l2d, 1, N, NUM, XA_64) \
157 _(FP64_FFI, fp64_ul2d, 2, N, NUM, 0) \ 243 _(FP64_FFI, fp64_ul2d, 1, N, NUM, XA_64) \
158 _(FP64_FFI, fp64_l2f, 2, N, FLOAT, 0) \ 244 _(FP64_FFI, fp64_l2f, 1, N, FLOAT, XA_64) \
159 _(FP64_FFI, fp64_ul2f, 2, N, FLOAT, 0) \ 245 _(FP64_FFI, fp64_ul2f, 1, N, FLOAT, XA_64) \
160 _(FP64_FFI, fp64_d2l, ARG1_FP, N, I64, 0) \ 246 _(FP64_FFI, fp64_d2l, 1, N, I64, XA_FP) \
161 _(FP64_FFI, fp64_d2ul, ARG1_FP, N, U64, 0) \ 247 _(FP64_FFI, fp64_d2ul, 1, N, U64, XA_FP) \
162 _(FP64_FFI, fp64_f2l, 1, N, I64, 0) \ 248 _(FP64_FFI, fp64_f2l, 1, N, I64, 0) \
163 _(FP64_FFI, fp64_f2ul, 1, N, U64, 0) \ 249 _(FP64_FFI, fp64_f2ul, 1, N, U64, 0) \
164 _(FFI, lj_carith_divi64, ARG2_64, N, I64, CCI_NOFPRCLOBBER) \ 250 _(FFI, lj_carith_divi64, 2, N, I64, XA2_64|CCI_NOFPRCLOBBER) \
165 _(FFI, lj_carith_divu64, ARG2_64, N, U64, CCI_NOFPRCLOBBER) \ 251 _(FFI, lj_carith_divu64, 2, N, U64, XA2_64|CCI_NOFPRCLOBBER) \
166 _(FFI, lj_carith_modi64, ARG2_64, N, I64, CCI_NOFPRCLOBBER) \ 252 _(FFI, lj_carith_modi64, 2, N, I64, XA2_64|CCI_NOFPRCLOBBER) \
167 _(FFI, lj_carith_modu64, ARG2_64, N, U64, CCI_NOFPRCLOBBER) \ 253 _(FFI, lj_carith_modu64, 2, N, U64, XA2_64|CCI_NOFPRCLOBBER) \
168 _(FFI, lj_carith_powi64, ARG2_64, N, I64, CCI_NOFPRCLOBBER) \ 254 _(FFI, lj_carith_powi64, 2, N, I64, XA2_64|CCI_NOFPRCLOBBER) \
169 _(FFI, lj_carith_powu64, ARG2_64, N, U64, CCI_NOFPRCLOBBER) \ 255 _(FFI, lj_carith_powu64, 2, N, U64, XA2_64|CCI_NOFPRCLOBBER) \
170 _(FFI, lj_cdata_setfin, 2, FN, P32, CCI_L) \ 256 _(FFI, lj_cdata_newv, 4, S, CDATA, CCI_L) \
171 _(FFI, strlen, 1, L, INTP, 0) \ 257 _(FFI, lj_cdata_setfin, 4, S, NIL, CCI_L) \
172 _(FFI, memcpy, 3, S, PTR, 0) \ 258 _(FFI, strlen, 1, L, INTP, 0) \
173 _(FFI, memset, 3, S, PTR, 0) \ 259 _(FFI, memcpy, 3, S, PTR, 0) \
174 _(FFI, lj_vm_errno, 0, S, INT, CCI_NOFPRCLOBBER) \ 260 _(FFI, memset, 3, S, PTR, 0) \
175 _(FFI32, lj_carith_mul64, ARG2_64, N, I64, CCI_NOFPRCLOBBER) 261 _(FFI, lj_vm_errno, 0, S, INT, CCI_NOFPRCLOBBER) \
262 _(FFI32, lj_carith_mul64, 2, N, I64, XA2_64|CCI_NOFPRCLOBBER) \
263 _(FFI32, lj_carith_shl64, 2, N, U64, XA_64|CCI_NOFPRCLOBBER) \
264 _(FFI32, lj_carith_shr64, 2, N, U64, XA_64|CCI_NOFPRCLOBBER) \
265 _(FFI32, lj_carith_sar64, 2, N, U64, XA_64|CCI_NOFPRCLOBBER) \
266 _(FFI32, lj_carith_rol64, 2, N, U64, XA_64|CCI_NOFPRCLOBBER) \
267 _(FFI32, lj_carith_ror64, 2, N, U64, XA_64|CCI_NOFPRCLOBBER) \
176 \ 268 \
177 /* End of list. */ 269 /* End of list. */
178 270
@@ -220,6 +312,22 @@ LJ_DATA const CCallInfo lj_ir_callinfo[IRCALL__MAX+1];
220#define fp64_f2l __aeabi_f2lz 312#define fp64_f2l __aeabi_f2lz
221#define fp64_f2ul __aeabi_f2ulz 313#define fp64_f2ul __aeabi_f2ulz
222#endif 314#endif
315#elif LJ_TARGET_MIPS || LJ_TARGET_PPC
316#define softfp_add __adddf3
317#define softfp_sub __subdf3
318#define softfp_mul __muldf3
319#define softfp_div __divdf3
320#define softfp_cmp __ledf2
321#define softfp_i2d __floatsidf
322#define softfp_d2i __fixdfsi
323#define softfp_ui2d __floatunsidf
324#define softfp_f2d __extendsfdf2
325#define softfp_d2ui __fixunsdfsi
326#define softfp_d2f __truncdfsf2
327#define softfp_i2f __floatsisf
328#define softfp_ui2f __floatunsisf
329#define softfp_f2i __fixsfsi
330#define softfp_f2ui __fixunssfsi
223#else 331#else
224#error "Missing soft-float definitions for target architecture" 332#error "Missing soft-float definitions for target architecture"
225#endif 333#endif
@@ -240,10 +348,14 @@ extern float softfp_ui2f(uint32_t a);
240extern int32_t softfp_f2i(float a); 348extern int32_t softfp_f2i(float a);
241extern uint32_t softfp_f2ui(float a); 349extern uint32_t softfp_f2ui(float a);
242#endif 350#endif
351#if LJ_TARGET_MIPS
352extern double lj_vm_sfmin(double a, double b);
353extern double lj_vm_sfmax(double a, double b);
354#endif
243#endif 355#endif
244 356
245#if LJ_HASFFI && LJ_NEED_FP64 && !(LJ_TARGET_ARM && LJ_SOFTFP) 357#if LJ_HASFFI && LJ_NEED_FP64 && !(LJ_TARGET_ARM && LJ_SOFTFP)
246#ifdef __GNUC__ 358#if defined(__GNUC__) || defined(__clang__)
247#define fp64_l2d __floatdidf 359#define fp64_l2d __floatdidf
248#define fp64_ul2d __floatundidf 360#define fp64_ul2d __floatundidf
249#define fp64_l2f __floatdisf 361#define fp64_l2f __floatdisf
diff --git a/src/lj_iropt.h b/src/lj_iropt.h
index 9aa03abc..458a5511 100644
--- a/src/lj_iropt.h
+++ b/src/lj_iropt.h
@@ -36,11 +36,11 @@ static LJ_AINLINE IRRef lj_ir_nextins(jit_State *J)
36 return ref; 36 return ref;
37} 37}
38 38
39LJ_FUNC TRef lj_ir_ggfload(jit_State *J, IRType t, uintptr_t ofs);
40
39/* Interning of constants. */ 41/* Interning of constants. */
40LJ_FUNC TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k); 42LJ_FUNC TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k);
41LJ_FUNC void lj_ir_k64_freeall(jit_State *J); 43LJ_FUNC TRef lj_ir_k64(jit_State *J, IROp op, uint64_t u64);
42LJ_FUNC TRef lj_ir_k64(jit_State *J, IROp op, cTValue *tv);
43LJ_FUNC cTValue *lj_ir_k64_find(jit_State *J, uint64_t u64);
44LJ_FUNC TRef lj_ir_knum_u64(jit_State *J, uint64_t u64); 44LJ_FUNC TRef lj_ir_knum_u64(jit_State *J, uint64_t u64);
45LJ_FUNC TRef lj_ir_knumint(jit_State *J, lua_Number n); 45LJ_FUNC TRef lj_ir_knumint(jit_State *J, lua_Number n);
46LJ_FUNC TRef lj_ir_kint64(jit_State *J, uint64_t u64); 46LJ_FUNC TRef lj_ir_kint64(jit_State *J, uint64_t u64);
@@ -48,6 +48,7 @@ LJ_FUNC TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t);
48LJ_FUNC TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr); 48LJ_FUNC TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr);
49LJ_FUNC TRef lj_ir_knull(jit_State *J, IRType t); 49LJ_FUNC TRef lj_ir_knull(jit_State *J, IRType t);
50LJ_FUNC TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot); 50LJ_FUNC TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot);
51LJ_FUNC TRef lj_ir_ktrace(jit_State *J);
51 52
52#if LJ_64 53#if LJ_64
53#define lj_ir_kintp(J, k) lj_ir_kint64(J, (uint64_t)(k)) 54#define lj_ir_kintp(J, k) lj_ir_kint64(J, (uint64_t)(k))
@@ -74,8 +75,8 @@ static LJ_AINLINE TRef lj_ir_knum(jit_State *J, lua_Number n)
74#define lj_ir_knum_tobit(J) lj_ir_knum_u64(J, U64x(43380000,00000000)) 75#define lj_ir_knum_tobit(J) lj_ir_knum_u64(J, U64x(43380000,00000000))
75 76
76/* Special 128 bit SIMD constants. */ 77/* Special 128 bit SIMD constants. */
77#define lj_ir_knum_abs(J) lj_ir_k64(J, IR_KNUM, LJ_KSIMD(J, LJ_KSIMD_ABS)) 78#define lj_ir_ksimd(J, idx) \
78#define lj_ir_knum_neg(J) lj_ir_k64(J, IR_KNUM, LJ_KSIMD(J, LJ_KSIMD_NEG)) 79 lj_ir_ggfload(J, IRT_NUM, (uintptr_t)LJ_KSIMD(J, idx) - (uintptr_t)J2GG(J))
79 80
80/* Access to constants. */ 81/* Access to constants. */
81LJ_FUNC void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir); 82LJ_FUNC void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir);
@@ -119,10 +120,11 @@ LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_hload(jit_State *J);
119LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_uload(jit_State *J); 120LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_uload(jit_State *J);
120LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_fload(jit_State *J); 121LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_fload(jit_State *J);
121LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_xload(jit_State *J); 122LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_xload(jit_State *J);
122LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_tab_len(jit_State *J); 123LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_alen(jit_State *J);
123LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_hrefk(jit_State *J); 124LJ_FUNC TRef LJ_FASTCALL lj_opt_fwd_hrefk(jit_State *J);
124LJ_FUNC int LJ_FASTCALL lj_opt_fwd_href_nokey(jit_State *J); 125LJ_FUNC int LJ_FASTCALL lj_opt_fwd_href_nokey(jit_State *J);
125LJ_FUNC int LJ_FASTCALL lj_opt_fwd_tptr(jit_State *J, IRRef lim); 126LJ_FUNC int LJ_FASTCALL lj_opt_fwd_tptr(jit_State *J, IRRef lim);
127LJ_FUNC int LJ_FASTCALL lj_opt_fwd_sbuf(jit_State *J, IRRef lim);
126LJ_FUNC int lj_opt_fwd_wasnonnil(jit_State *J, IROpT loadop, IRRef xref); 128LJ_FUNC int lj_opt_fwd_wasnonnil(jit_State *J, IROpT loadop, IRRef xref);
127 129
128/* Dead-store elimination. */ 130/* Dead-store elimination. */
@@ -143,13 +145,12 @@ LJ_FUNC TRef lj_opt_narrow_arith(jit_State *J, TRef rb, TRef rc,
143 TValue *vb, TValue *vc, IROp op); 145 TValue *vb, TValue *vc, IROp op);
144LJ_FUNC TRef lj_opt_narrow_unm(jit_State *J, TRef rc, TValue *vc); 146LJ_FUNC TRef lj_opt_narrow_unm(jit_State *J, TRef rc, TValue *vc);
145LJ_FUNC TRef lj_opt_narrow_mod(jit_State *J, TRef rb, TRef rc, TValue *vb, TValue *vc); 147LJ_FUNC TRef lj_opt_narrow_mod(jit_State *J, TRef rb, TRef rc, TValue *vb, TValue *vc);
146LJ_FUNC TRef lj_opt_narrow_pow(jit_State *J, TRef rb, TRef rc, TValue *vb, TValue *vc);
147LJ_FUNC IRType lj_opt_narrow_forl(jit_State *J, cTValue *forbase); 148LJ_FUNC IRType lj_opt_narrow_forl(jit_State *J, cTValue *forbase);
148 149
149/* Optimization passes. */ 150/* Optimization passes. */
150LJ_FUNC void lj_opt_dce(jit_State *J); 151LJ_FUNC void lj_opt_dce(jit_State *J);
151LJ_FUNC int lj_opt_loop(jit_State *J); 152LJ_FUNC int lj_opt_loop(jit_State *J);
152#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI) 153#if LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)
153LJ_FUNC void lj_opt_split(jit_State *J); 154LJ_FUNC void lj_opt_split(jit_State *J);
154#else 155#else
155#define lj_opt_split(J) UNUSED(J) 156#define lj_opt_split(J) UNUSED(J)
diff --git a/src/lj_jit.h b/src/lj_jit.h
index 66aa9aad..19810639 100644
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -7,75 +7,91 @@
7#define _LJ_JIT_H 7#define _LJ_JIT_H
8 8
9#include "lj_obj.h" 9#include "lj_obj.h"
10#if LJ_HASJIT
10#include "lj_ir.h" 11#include "lj_ir.h"
11 12
12/* JIT engine flags. */ 13/* -- JIT engine flags ---------------------------------------------------- */
14
15/* General JIT engine flags. 4 bits. */
13#define JIT_F_ON 0x00000001 16#define JIT_F_ON 0x00000001
14 17
15/* CPU-specific JIT engine flags. */ 18/* CPU-specific JIT engine flags. 12 bits. Flags and strings must match. */
19#define JIT_F_CPU 0x00000010
20
16#if LJ_TARGET_X86ORX64 21#if LJ_TARGET_X86ORX64
17#define JIT_F_CMOV 0x00000010 22
18#define JIT_F_SSE2 0x00000020 23#define JIT_F_SSE3 (JIT_F_CPU << 0)
19#define JIT_F_SSE3 0x00000040 24#define JIT_F_SSE4_1 (JIT_F_CPU << 1)
20#define JIT_F_SSE4_1 0x00000080 25#define JIT_F_BMI2 (JIT_F_CPU << 2)
21#define JIT_F_P4 0x00000100 26
22#define JIT_F_PREFER_IMUL 0x00000200 27
23#define JIT_F_SPLIT_XMM 0x00000400 28#define JIT_F_CPUSTRING "\4SSE3\6SSE4.1\4BMI2"
24#define JIT_F_LEA_AGU 0x00000800 29
25
26/* Names for the CPU-specific flags. Must match the order above. */
27#define JIT_F_CPU_FIRST JIT_F_CMOV
28#define JIT_F_CPUSTRING "\4CMOV\4SSE2\4SSE3\6SSE4.1\2P4\3AMD\2K8\4ATOM"
29#elif LJ_TARGET_ARM 30#elif LJ_TARGET_ARM
30#define JIT_F_ARMV6_ 0x00000010 31
31#define JIT_F_ARMV6T2_ 0x00000020 32#define JIT_F_ARMV6_ (JIT_F_CPU << 0)
32#define JIT_F_ARMV7 0x00000040 33#define JIT_F_ARMV6T2_ (JIT_F_CPU << 1)
33#define JIT_F_VFPV2 0x00000080 34#define JIT_F_ARMV7 (JIT_F_CPU << 2)
34#define JIT_F_VFPV3 0x00000100 35#define JIT_F_ARMV8 (JIT_F_CPU << 3)
35 36#define JIT_F_VFPV2 (JIT_F_CPU << 4)
36#define JIT_F_ARMV6 (JIT_F_ARMV6_|JIT_F_ARMV6T2_|JIT_F_ARMV7) 37#define JIT_F_VFPV3 (JIT_F_CPU << 5)
37#define JIT_F_ARMV6T2 (JIT_F_ARMV6T2_|JIT_F_ARMV7) 38
39#define JIT_F_ARMV6 (JIT_F_ARMV6_|JIT_F_ARMV6T2_|JIT_F_ARMV7|JIT_F_ARMV8)
40#define JIT_F_ARMV6T2 (JIT_F_ARMV6T2_|JIT_F_ARMV7|JIT_F_ARMV8)
38#define JIT_F_VFP (JIT_F_VFPV2|JIT_F_VFPV3) 41#define JIT_F_VFP (JIT_F_VFPV2|JIT_F_VFPV3)
39 42
40/* Names for the CPU-specific flags. Must match the order above. */ 43#define JIT_F_CPUSTRING "\5ARMv6\7ARMv6T2\5ARMv7\5ARMv8\5VFPv2\5VFPv3"
41#define JIT_F_CPU_FIRST JIT_F_ARMV6_ 44
42#define JIT_F_CPUSTRING "\5ARMv6\7ARMv6T2\5ARMv7\5VFPv2\5VFPv3"
43#elif LJ_TARGET_PPC 45#elif LJ_TARGET_PPC
44#define JIT_F_SQRT 0x00000010
45#define JIT_F_ROUND 0x00000020
46 46
47/* Names for the CPU-specific flags. Must match the order above. */ 47#define JIT_F_SQRT (JIT_F_CPU << 0)
48#define JIT_F_CPU_FIRST JIT_F_SQRT 48#define JIT_F_ROUND (JIT_F_CPU << 1)
49
49#define JIT_F_CPUSTRING "\4SQRT\5ROUND" 50#define JIT_F_CPUSTRING "\4SQRT\5ROUND"
51
50#elif LJ_TARGET_MIPS 52#elif LJ_TARGET_MIPS
51#define JIT_F_MIPS32R2 0x00000010
52 53
53/* Names for the CPU-specific flags. Must match the order above. */ 54#define JIT_F_MIPSXXR2 (JIT_F_CPU << 0)
54#define JIT_F_CPU_FIRST JIT_F_MIPS32R2 55
56#if LJ_TARGET_MIPS32
57#if LJ_TARGET_MIPSR6
58#define JIT_F_CPUSTRING "\010MIPS32R6"
59#else
55#define JIT_F_CPUSTRING "\010MIPS32R2" 60#define JIT_F_CPUSTRING "\010MIPS32R2"
61#endif
56#else 62#else
57#define JIT_F_CPU_FIRST 0 63#if LJ_TARGET_MIPSR6
64#define JIT_F_CPUSTRING "\010MIPS64R6"
65#else
66#define JIT_F_CPUSTRING "\010MIPS64R2"
67#endif
68#endif
69
70#else
71
58#define JIT_F_CPUSTRING "" 72#define JIT_F_CPUSTRING ""
73
59#endif 74#endif
60 75
61/* Optimization flags. */ 76/* Optimization flags. 12 bits. */
77#define JIT_F_OPT 0x00010000
62#define JIT_F_OPT_MASK 0x0fff0000 78#define JIT_F_OPT_MASK 0x0fff0000
63 79
64#define JIT_F_OPT_FOLD 0x00010000 80#define JIT_F_OPT_FOLD (JIT_F_OPT << 0)
65#define JIT_F_OPT_CSE 0x00020000 81#define JIT_F_OPT_CSE (JIT_F_OPT << 1)
66#define JIT_F_OPT_DCE 0x00040000 82#define JIT_F_OPT_DCE (JIT_F_OPT << 2)
67#define JIT_F_OPT_FWD 0x00080000 83#define JIT_F_OPT_FWD (JIT_F_OPT << 3)
68#define JIT_F_OPT_DSE 0x00100000 84#define JIT_F_OPT_DSE (JIT_F_OPT << 4)
69#define JIT_F_OPT_NARROW 0x00200000 85#define JIT_F_OPT_NARROW (JIT_F_OPT << 5)
70#define JIT_F_OPT_LOOP 0x00400000 86#define JIT_F_OPT_LOOP (JIT_F_OPT << 6)
71#define JIT_F_OPT_ABC 0x00800000 87#define JIT_F_OPT_ABC (JIT_F_OPT << 7)
72#define JIT_F_OPT_SINK 0x01000000 88#define JIT_F_OPT_SINK (JIT_F_OPT << 8)
73#define JIT_F_OPT_FUSE 0x02000000 89#define JIT_F_OPT_FUSE (JIT_F_OPT << 9)
90#define JIT_F_OPT_FMA (JIT_F_OPT << 10)
74 91
75/* Optimizations names for -O. Must match the order above. */ 92/* Optimizations names for -O. Must match the order above. */
76#define JIT_F_OPT_FIRST JIT_F_OPT_FOLD
77#define JIT_F_OPTSTRING \ 93#define JIT_F_OPTSTRING \
78 "\4fold\3cse\3dce\3fwd\3dse\6narrow\4loop\3abc\4sink\4fuse" 94 "\4fold\3cse\3dce\3fwd\3dse\6narrow\4loop\3abc\4sink\4fuse\3fma"
79 95
80/* Optimization levels set a fixed combination of flags. */ 96/* Optimization levels set a fixed combination of flags. */
81#define JIT_F_OPT_0 0 97#define JIT_F_OPT_0 0
@@ -84,6 +100,9 @@
84#define JIT_F_OPT_3 (JIT_F_OPT_2|\ 100#define JIT_F_OPT_3 (JIT_F_OPT_2|\
85 JIT_F_OPT_FWD|JIT_F_OPT_DSE|JIT_F_OPT_ABC|JIT_F_OPT_SINK|JIT_F_OPT_FUSE) 101 JIT_F_OPT_FWD|JIT_F_OPT_DSE|JIT_F_OPT_ABC|JIT_F_OPT_SINK|JIT_F_OPT_FUSE)
86#define JIT_F_OPT_DEFAULT JIT_F_OPT_3 102#define JIT_F_OPT_DEFAULT JIT_F_OPT_3
103/* Note: FMA is not set by default. */
104
105/* -- JIT engine parameters ----------------------------------------------- */
87 106
88#if LJ_TARGET_WINDOWS || LJ_64 107#if LJ_TARGET_WINDOWS || LJ_64
89/* See: http://blogs.msdn.com/oldnewthing/archive/2003/10/08/55239.aspx */ 108/* See: http://blogs.msdn.com/oldnewthing/archive/2003/10/08/55239.aspx */
@@ -100,6 +119,7 @@
100 _(\012, maxirconst, 500) /* Max. # of IR constants of a trace. */ \ 119 _(\012, maxirconst, 500) /* Max. # of IR constants of a trace. */ \
101 _(\007, maxside, 100) /* Max. # of side traces of a root trace. */ \ 120 _(\007, maxside, 100) /* Max. # of side traces of a root trace. */ \
102 _(\007, maxsnap, 500) /* Max. # of snapshots for a trace. */ \ 121 _(\007, maxsnap, 500) /* Max. # of snapshots for a trace. */ \
122 _(\011, minstitch, 0) /* Min. # of IR ins for a stitched trace. */ \
103 \ 123 \
104 _(\007, hotloop, 56) /* # of iter. to detect a hot loop/call. */ \ 124 _(\007, hotloop, 56) /* # of iter. to detect a hot loop/call. */ \
105 _(\007, hotexit, 10) /* # of taken exits to start a side trace. */ \ 125 _(\007, hotexit, 10) /* # of taken exits to start a side trace. */ \
@@ -126,11 +146,14 @@ JIT_PARAMDEF(JIT_PARAMENUM)
126#define JIT_PARAMSTR(len, name, value) #len #name 146#define JIT_PARAMSTR(len, name, value) #len #name
127#define JIT_P_STRING JIT_PARAMDEF(JIT_PARAMSTR) 147#define JIT_P_STRING JIT_PARAMDEF(JIT_PARAMSTR)
128 148
149/* -- JIT engine data structures ------------------------------------------ */
150
129/* Trace compiler state. */ 151/* Trace compiler state. */
130typedef enum { 152typedef enum {
131 LJ_TRACE_IDLE, /* Trace compiler idle. */ 153 LJ_TRACE_IDLE, /* Trace compiler idle. */
132 LJ_TRACE_ACTIVE = 0x10, 154 LJ_TRACE_ACTIVE = 0x10,
133 LJ_TRACE_RECORD, /* Bytecode recording active. */ 155 LJ_TRACE_RECORD, /* Bytecode recording active. */
156 LJ_TRACE_RECORD_1ST, /* Record 1st instruction, too. */
134 LJ_TRACE_START, /* New trace started. */ 157 LJ_TRACE_START, /* New trace started. */
135 LJ_TRACE_END, /* End of trace. */ 158 LJ_TRACE_END, /* End of trace. */
136 LJ_TRACE_ASM, /* Assemble trace. */ 159 LJ_TRACE_ASM, /* Assemble trace. */
@@ -165,6 +188,7 @@ typedef struct MCLink {
165typedef struct SnapShot { 188typedef struct SnapShot {
166 uint32_t mapofs; /* Offset into snapshot map. */ 189 uint32_t mapofs; /* Offset into snapshot map. */
167 IRRef1 ref; /* First IR ref for this snapshot. */ 190 IRRef1 ref; /* First IR ref for this snapshot. */
191 uint16_t mcofs; /* Offset into machine code in MCode units. */
168 uint8_t nslots; /* Number of valid slots. */ 192 uint8_t nslots; /* Number of valid slots. */
169 uint8_t topslot; /* Maximum frame extent. */ 193 uint8_t topslot; /* Maximum frame extent. */
170 uint8_t nent; /* Number of compressed entries. */ 194 uint8_t nent; /* Number of compressed entries. */
@@ -180,20 +204,35 @@ typedef uint32_t SnapEntry;
180#define SNAP_CONT 0x020000 /* Continuation slot. */ 204#define SNAP_CONT 0x020000 /* Continuation slot. */
181#define SNAP_NORESTORE 0x040000 /* No need to restore slot. */ 205#define SNAP_NORESTORE 0x040000 /* No need to restore slot. */
182#define SNAP_SOFTFPNUM 0x080000 /* Soft-float number. */ 206#define SNAP_SOFTFPNUM 0x080000 /* Soft-float number. */
207#define SNAP_KEYINDEX 0x100000 /* Traversal key index. */
183LJ_STATIC_ASSERT(SNAP_FRAME == TREF_FRAME); 208LJ_STATIC_ASSERT(SNAP_FRAME == TREF_FRAME);
184LJ_STATIC_ASSERT(SNAP_CONT == TREF_CONT); 209LJ_STATIC_ASSERT(SNAP_CONT == TREF_CONT);
210LJ_STATIC_ASSERT(SNAP_KEYINDEX == TREF_KEYINDEX);
185 211
186#define SNAP(slot, flags, ref) (((SnapEntry)(slot) << 24) + (flags) + (ref)) 212#define SNAP(slot, flags, ref) (((SnapEntry)(slot) << 24) + (flags) + (ref))
187#define SNAP_TR(slot, tr) \ 213#define SNAP_TR(slot, tr) \
188 (((SnapEntry)(slot) << 24) + ((tr) & (TREF_CONT|TREF_FRAME|TREF_REFMASK))) 214 (((SnapEntry)(slot) << 24) + \
215 ((tr) & (TREF_KEYINDEX|TREF_CONT|TREF_FRAME|TREF_REFMASK)))
216#if !LJ_FR2
189#define SNAP_MKPC(pc) ((SnapEntry)u32ptr(pc)) 217#define SNAP_MKPC(pc) ((SnapEntry)u32ptr(pc))
218#endif
190#define SNAP_MKFTSZ(ftsz) ((SnapEntry)(ftsz)) 219#define SNAP_MKFTSZ(ftsz) ((SnapEntry)(ftsz))
191#define snap_ref(sn) ((sn) & 0xffff) 220#define snap_ref(sn) ((sn) & 0xffff)
192#define snap_slot(sn) ((BCReg)((sn) >> 24)) 221#define snap_slot(sn) ((BCReg)((sn) >> 24))
193#define snap_isframe(sn) ((sn) & SNAP_FRAME) 222#define snap_isframe(sn) ((sn) & SNAP_FRAME)
194#define snap_pc(sn) ((const BCIns *)(uintptr_t)(sn))
195#define snap_setref(sn, ref) (((sn) & (0xffff0000&~SNAP_NORESTORE)) | (ref)) 223#define snap_setref(sn, ref) (((sn) & (0xffff0000&~SNAP_NORESTORE)) | (ref))
196 224
225static LJ_AINLINE const BCIns *snap_pc(SnapEntry *sn)
226{
227#if LJ_FR2
228 uint64_t pcbase;
229 memcpy(&pcbase, sn, sizeof(uint64_t));
230 return (const BCIns *)(pcbase >> 8);
231#else
232 return (const BCIns *)(uintptr_t)*sn;
233#endif
234}
235
197/* Snapshot and exit numbers. */ 236/* Snapshot and exit numbers. */
198typedef uint32_t SnapNo; 237typedef uint32_t SnapNo;
199typedef uint32_t ExitNo; 238typedef uint32_t ExitNo;
@@ -211,7 +250,8 @@ typedef enum {
211 LJ_TRLINK_UPREC, /* Up-recursion. */ 250 LJ_TRLINK_UPREC, /* Up-recursion. */
212 LJ_TRLINK_DOWNREC, /* Down-recursion. */ 251 LJ_TRLINK_DOWNREC, /* Down-recursion. */
213 LJ_TRLINK_INTERP, /* Fallback to interpreter. */ 252 LJ_TRLINK_INTERP, /* Fallback to interpreter. */
214 LJ_TRLINK_RETURN /* Return to interpreter. */ 253 LJ_TRLINK_RETURN, /* Return to interpreter. */
254 LJ_TRLINK_STITCH /* Trace stitching. */
215} TraceLink; 255} TraceLink;
216 256
217/* Trace object. */ 257/* Trace object. */
@@ -219,6 +259,9 @@ typedef struct GCtrace {
219 GCHeader; 259 GCHeader;
220 uint16_t nsnap; /* Number of snapshots. */ 260 uint16_t nsnap; /* Number of snapshots. */
221 IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */ 261 IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */
262#if LJ_GC64
263 uint32_t unused_gc64;
264#endif
222 GCRef gclist; 265 GCRef gclist;
223 IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */ 266 IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
224 IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */ 267 IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */
@@ -230,6 +273,9 @@ typedef struct GCtrace {
230 BCIns startins; /* Original bytecode of starting instruction. */ 273 BCIns startins; /* Original bytecode of starting instruction. */
231 MSize szmcode; /* Size of machine code. */ 274 MSize szmcode; /* Size of machine code. */
232 MCode *mcode; /* Start of machine code. */ 275 MCode *mcode; /* Start of machine code. */
276#if LJ_ABI_PAUTH
277 ASMFunction mcauth; /* Start of machine code, with ptr auth applied. */
278#endif
233 MSize mcloop; /* Offset of loop start in machine code. */ 279 MSize mcloop; /* Offset of loop start in machine code. */
234 uint16_t nchild; /* Number of child traces (root trace only). */ 280 uint16_t nchild; /* Number of child traces (root trace only). */
235 uint16_t spadjust; /* Stack pointer adjustment (offset in bytes). */ 281 uint16_t spadjust; /* Stack pointer adjustment (offset in bytes). */
@@ -294,6 +340,16 @@ typedef struct ScEvEntry {
294 uint8_t dir; /* Direction. 1: +, 0: -. */ 340 uint8_t dir; /* Direction. 1: +, 0: -. */
295} ScEvEntry; 341} ScEvEntry;
296 342
343/* Reverse bytecode map (IRRef -> PC). Only for selected instructions. */
344typedef struct RBCHashEntry {
345 MRef pc; /* Bytecode PC. */
346 GCRef pt; /* Prototype. */
347 IRRef ref; /* IR reference. */
348} RBCHashEntry;
349
350/* Number of slots in the reverse bytecode hash table. Must be a power of 2. */
351#define RBCHASH_SLOTS 8
352
297/* 128 bit SIMD constants. */ 353/* 128 bit SIMD constants. */
298enum { 354enum {
299 LJ_KSIMD_ABS, 355 LJ_KSIMD_ABS,
@@ -301,12 +357,53 @@ enum {
301 LJ_KSIMD__MAX 357 LJ_KSIMD__MAX
302}; 358};
303 359
360enum {
361#if LJ_TARGET_X86ORX64
362 LJ_K64_TOBIT, /* 2^52 + 2^51 */
363 LJ_K64_2P64, /* 2^64 */
364 LJ_K64_M2P64, /* -2^64 */
365#if LJ_32
366 LJ_K64_M2P64_31, /* -2^64 or -2^31 */
367#else
368 LJ_K64_M2P64_31 = LJ_K64_M2P64,
369#endif
370#endif
371#if LJ_TARGET_MIPS
372 LJ_K64_2P31, /* 2^31 */
373#if LJ_64
374 LJ_K64_2P63, /* 2^63 */
375 LJ_K64_M2P64, /* -2^64 */
376#endif
377#endif
378 LJ_K64__MAX,
379};
380#define LJ_K64__USED (LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS)
381
382enum {
383#if LJ_TARGET_X86ORX64
384 LJ_K32_M2P64_31, /* -2^64 or -2^31 */
385#endif
386#if LJ_TARGET_PPC
387 LJ_K32_2P52_2P31, /* 2^52 + 2^31 */
388 LJ_K32_2P52, /* 2^52 */
389#endif
390#if LJ_TARGET_PPC || LJ_TARGET_MIPS
391 LJ_K32_2P31, /* 2^31 */
392#endif
393#if LJ_TARGET_MIPS64
394 LJ_K32_2P63, /* 2^63 */
395 LJ_K32_M2P64, /* -2^64 */
396#endif
397 LJ_K32__MAX
398};
399#define LJ_K32__USED (LJ_TARGET_X86ORX64 || LJ_TARGET_PPC || LJ_TARGET_MIPS)
400
304/* Get 16 byte aligned pointer to SIMD constant. */ 401/* Get 16 byte aligned pointer to SIMD constant. */
305#define LJ_KSIMD(J, n) \ 402#define LJ_KSIMD(J, n) \
306 ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15)) 403 ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15))
307 404
308/* Set/reset flag to activate the SPLIT pass for the current trace. */ 405/* Set/reset flag to activate the SPLIT pass for the current trace. */
309#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI) 406#if LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)
310#define lj_needsplit(J) (J->needsplit = 1) 407#define lj_needsplit(J) (J->needsplit = 1)
311#define lj_resetsplit(J) (J->needsplit = 0) 408#define lj_resetsplit(J) (J->needsplit = 0)
312#else 409#else
@@ -317,13 +414,14 @@ enum {
317/* Fold state is used to fold instructions on-the-fly. */ 414/* Fold state is used to fold instructions on-the-fly. */
318typedef struct FoldState { 415typedef struct FoldState {
319 IRIns ins; /* Currently emitted instruction. */ 416 IRIns ins; /* Currently emitted instruction. */
320 IRIns left; /* Instruction referenced by left operand. */ 417 IRIns left[2]; /* Instruction referenced by left operand. */
321 IRIns right; /* Instruction referenced by right operand. */ 418 IRIns right[2]; /* Instruction referenced by right operand. */
322} FoldState; 419} FoldState;
323 420
324/* JIT compiler state. */ 421/* JIT compiler state. */
325typedef struct jit_State { 422typedef struct jit_State {
326 GCtrace cur; /* Current trace. */ 423 GCtrace cur; /* Current trace. */
424 GCtrace *curfinal; /* Final address of current trace (set during asm). */
327 425
328 lua_State *L; /* Current Lua state. */ 426 lua_State *L; /* Current Lua state. */
329 const BCIns *pc; /* Current PC. */ 427 const BCIns *pc; /* Current PC. */
@@ -353,8 +451,13 @@ typedef struct jit_State {
353 int32_t framedepth; /* Current frame depth. */ 451 int32_t framedepth; /* Current frame depth. */
354 int32_t retdepth; /* Return frame depth (count of RETF). */ 452 int32_t retdepth; /* Return frame depth (count of RETF). */
355 453
356 MRef k64; /* Pointer to chained array of 64 bit constants. */ 454#if LJ_K32__USED
455 uint32_t k32[LJ_K32__MAX]; /* Common 4 byte constants used by backends. */
456#endif
357 TValue ksimd[LJ_KSIMD__MAX*2+1]; /* 16 byte aligned SIMD constants. */ 457 TValue ksimd[LJ_KSIMD__MAX*2+1]; /* 16 byte aligned SIMD constants. */
458#if LJ_K64__USED
459 TValue k64[LJ_K64__MAX]; /* Common 8 byte constants. */
460#endif
358 461
359 IRIns *irbuf; /* Temp. IR instruction buffer. Biased with REF_BIAS. */ 462 IRIns *irbuf; /* Temp. IR instruction buffer. Biased with REF_BIAS. */
360 IRRef irtoplim; /* Upper limit of instuction buffer (biased). */ 463 IRRef irtoplim; /* Upper limit of instuction buffer (biased). */
@@ -367,13 +470,15 @@ typedef struct jit_State {
367 MSize sizesnapmap; /* Size of temp. snapshot map buffer. */ 470 MSize sizesnapmap; /* Size of temp. snapshot map buffer. */
368 471
369 PostProc postproc; /* Required post-processing after execution. */ 472 PostProc postproc; /* Required post-processing after execution. */
370#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI) 473#if LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)
371 int needsplit; /* Need SPLIT pass. */ 474 uint8_t needsplit; /* Need SPLIT pass. */
372#endif 475#endif
476 uint8_t retryrec; /* Retry recording. */
373 477
374 GCRef *trace; /* Array of traces. */ 478 GCRef *trace; /* Array of traces. */
375 TraceNo freetrace; /* Start of scan for next free trace. */ 479 TraceNo freetrace; /* Start of scan for next free trace. */
376 MSize sizetrace; /* Size of trace array. */ 480 MSize sizetrace; /* Size of trace array. */
481 IRRef1 ktrace; /* Reference to KGC with GCtrace. */
377 482
378 IRRef1 chain[IR__MAX]; /* IR instruction skip-list chain anchors. */ 483 IRRef1 chain[IR__MAX]; /* IR instruction skip-list chain anchors. */
379 TRef slot[LJ_MAX_JSLOTS+LJ_STACK_EXTRA]; /* Stack slot map. */ 484 TRef slot[LJ_MAX_JSLOTS+LJ_STACK_EXTRA]; /* Stack slot map. */
@@ -384,7 +489,10 @@ typedef struct jit_State {
384 489
385 HotPenalty penalty[PENALTY_SLOTS]; /* Penalty slots. */ 490 HotPenalty penalty[PENALTY_SLOTS]; /* Penalty slots. */
386 uint32_t penaltyslot; /* Round-robin index into penalty slots. */ 491 uint32_t penaltyslot; /* Round-robin index into penalty slots. */
387 uint32_t prngstate; /* PRNG state. */ 492
493#ifdef LUAJIT_ENABLE_TABLE_BUMP
494 RBCHashEntry rbchash[RBCHASH_SLOTS]; /* Reverse bytecode map. */
495#endif
388 496
389 BPropEntry bpropcache[BPROP_SLOTS]; /* Backpropagation cache slots. */ 497 BPropEntry bpropcache[BPROP_SLOTS]; /* Backpropagation cache slots. */
390 uint32_t bpropslot; /* Round-robin index into bpropcache slots. */ 498 uint32_t bpropslot; /* Round-robin index into bpropcache slots. */
@@ -394,6 +502,7 @@ typedef struct jit_State {
394 const BCIns *startpc; /* Bytecode PC of starting instruction. */ 502 const BCIns *startpc; /* Bytecode PC of starting instruction. */
395 TraceNo parent; /* Parent of current side trace (0 for root traces). */ 503 TraceNo parent; /* Parent of current side trace (0 for root traces). */
396 ExitNo exitno; /* Exit number in parent of current side trace. */ 504 ExitNo exitno; /* Exit number in parent of current side trace. */
505 int exitcode; /* Exit code from unwound trace. */
397 506
398 BCIns *patchpc; /* PC for pending re-patch. */ 507 BCIns *patchpc; /* PC for pending re-patch. */
399 BCIns patchins; /* Instruction for pending re-patch. */ 508 BCIns patchins; /* Instruction for pending re-patch. */
@@ -406,14 +515,19 @@ typedef struct jit_State {
406 size_t szallmcarea; /* Total size of all allocated mcode areas. */ 515 size_t szallmcarea; /* Total size of all allocated mcode areas. */
407 516
408 TValue errinfo; /* Additional info element for trace errors. */ 517 TValue errinfo; /* Additional info element for trace errors. */
518
519#if LJ_HASPROFILE
520 GCproto *prev_pt; /* Previous prototype. */
521 BCLine prev_line; /* Previous line. */
522 int prof_mode; /* Profiling mode: 0, 'f', 'l'. */
523#endif
409} jit_State; 524} jit_State;
410 525
411/* Trivial PRNG e.g. used for penalty randomization. */ 526#ifdef LUA_USE_ASSERT
412static LJ_AINLINE uint32_t LJ_PRNG_BITS(jit_State *J, int bits) 527#define lj_assertJ(c, ...) lj_assertG_(J2G(J), (c), __VA_ARGS__)
413{ 528#else
414 /* Yes, this LCG is very weak, but that doesn't matter for our use case. */ 529#define lj_assertJ(c, ...) ((void)J)
415 J->prngstate = J->prngstate * 1103515245 + 12345; 530#endif
416 return J->prngstate >> (32-bits); 531#endif
417}
418 532
419#endif 533#endif
diff --git a/src/lj_lex.c b/src/lj_lex.c
index 87601597..61b04c4b 100644
--- a/src/lj_lex.c
+++ b/src/lj_lex.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#if LJ_HASFFI 17#if LJ_HASFFI
17#include "lj_tab.h" 18#include "lj_tab.h"
@@ -24,6 +25,7 @@
24#include "lj_parse.h" 25#include "lj_parse.h"
25#include "lj_char.h" 26#include "lj_char.h"
26#include "lj_strscan.h" 27#include "lj_strscan.h"
28#include "lj_strfmt.h"
27 29
28/* Lua lexer token names. */ 30/* Lua lexer token names. */
29static const char *const tokennames[] = { 31static const char *const tokennames[] = {
@@ -37,54 +39,54 @@ TKDEF(TKSTR1, TKSTR2)
37 39
38/* -- Buffer handling ----------------------------------------------------- */ 40/* -- Buffer handling ----------------------------------------------------- */
39 41
40#define char2int(c) ((int)(uint8_t)(c)) 42#define LEX_EOF (-1)
41#define next(ls) \ 43#define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
42 (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
43#define save_and_next(ls) (save(ls, ls->current), next(ls))
44#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
45#define END_OF_STREAM (-1)
46 44
47static int fillbuf(LexState *ls) 45/* Get more input from reader. */
46static LJ_NOINLINE LexChar lex_more(LexState *ls)
48{ 47{
49 size_t sz; 48 size_t sz;
50 const char *buf = ls->rfunc(ls->L, ls->rdata, &sz); 49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
51 if (buf == NULL || sz == 0) return END_OF_STREAM; 50 if (p == NULL || sz == 0) return LEX_EOF;
52 if (sz >= LJ_MAX_MEM) { 51 if (sz >= LJ_MAX_BUF) {
53 if (sz != ~(size_t)0) lj_err_mem(ls->L); 52 if (sz != ~(size_t)0) lj_err_mem(ls->L);
53 sz = ~(uintptr_t)0 - (uintptr_t)p;
54 if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1;
54 ls->endmark = 1; 55 ls->endmark = 1;
55 } 56 }
56 ls->n = (MSize)sz - 1; 57 ls->pe = p + sz;
57 ls->p = buf; 58 ls->p = p + 1;
58 return char2int(*(ls->p++)); 59 return (LexChar)(uint8_t)p[0];
59} 60}
60 61
61static LJ_NOINLINE void save_grow(LexState *ls, int c) 62/* Get next character. */
63static LJ_AINLINE LexChar lex_next(LexState *ls)
62{ 64{
63 MSize newsize; 65 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
64 if (ls->sb.sz >= LJ_MAX_STR/2)
65 lj_lex_error(ls, 0, LJ_ERR_XELEM);
66 newsize = ls->sb.sz * 2;
67 lj_str_resizebuf(ls->L, &ls->sb, newsize);
68 ls->sb.buf[ls->sb.n++] = (char)c;
69} 66}
70 67
71static LJ_AINLINE void save(LexState *ls, int c) 68/* Save character. */
69static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
72{ 70{
73 if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz)) 71 lj_buf_putb(&ls->sb, c);
74 save_grow(ls, c); 72}
75 else 73
76 ls->sb.buf[ls->sb.n++] = (char)c; 74/* Save previous character and get next character. */
75static LJ_AINLINE LexChar lex_savenext(LexState *ls)
76{
77 lex_save(ls, ls->c);
78 return lex_next(ls);
77} 79}
78 80
79static void inclinenumber(LexState *ls) 81/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
82static void lex_newline(LexState *ls)
80{ 83{
81 int old = ls->current; 84 LexChar old = ls->c;
82 lua_assert(currIsNewline(ls)); 85 lj_assertLS(lex_iseol(ls), "bad usage");
83 next(ls); /* skip `\n' or `\r' */ 86 lex_next(ls); /* Skip "\n" or "\r". */
84 if (currIsNewline(ls) && ls->current != old) 87 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
85 next(ls); /* skip `\n\r' or `\r\n' */
86 if (++ls->linenumber >= LJ_MAX_LINE) 88 if (++ls->linenumber >= LJ_MAX_LINE)
87 lj_lex_error(ls, ls->token, LJ_ERR_XLINES); 89 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
88} 90}
89 91
90/* -- Scanner for terminals ----------------------------------------------- */ 92/* -- Scanner for terminals ----------------------------------------------- */
@@ -93,19 +95,17 @@ static void inclinenumber(LexState *ls)
93static void lex_number(LexState *ls, TValue *tv) 95static void lex_number(LexState *ls, TValue *tv)
94{ 96{
95 StrScanFmt fmt; 97 StrScanFmt fmt;
96 int c, xp = 'e'; 98 LexChar c, xp = 'e';
97 lua_assert(lj_char_isdigit(ls->current)); 99 lj_assertLS(lj_char_isdigit(ls->c), "bad usage");
98 if ((c = ls->current) == '0') { 100 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
99 save_and_next(ls); 101 xp = 'p';
100 if ((ls->current | 0x20) == 'x') xp = 'p'; 102 while (lj_char_isident(ls->c) || ls->c == '.' ||
101 } 103 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
102 while (lj_char_isident(ls->current) || ls->current == '.' || 104 c = ls->c;
103 ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) { 105 lex_savenext(ls);
104 c = ls->current;
105 save_and_next(ls);
106 } 106 }
107 save(ls, '\0'); 107 lex_save(ls, '\0');
108 fmt = lj_strscan_scan((const uint8_t *)ls->sb.buf, tv, 108 fmt = lj_strscan_scan((const uint8_t *)ls->sb.b, sbuflen(&ls->sb)-1, tv,
109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) | 109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0)); 110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
111 if (LJ_DUALNUM && fmt == STRSCAN_INT) { 111 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
@@ -116,12 +116,9 @@ static void lex_number(LexState *ls, TValue *tv)
116 } else if (fmt != STRSCAN_ERROR) { 116 } else if (fmt != STRSCAN_ERROR) {
117 lua_State *L = ls->L; 117 lua_State *L = ls->L;
118 GCcdata *cd; 118 GCcdata *cd;
119 lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG); 119 lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG,
120 if (!ctype_ctsG(G(L))) { 120 "unexpected number format %d", fmt);
121 ptrdiff_t oldtop = savestack(L, L->top); 121 ctype_loadffi(L);
122 luaopen_ffi(L); /* Load FFI library on-demand. */
123 L->top = restorestack(L, oldtop);
124 }
125 if (fmt == STRSCAN_IMAG) { 122 if (fmt == STRSCAN_IMAG) {
126 cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double)); 123 cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
127 ((double *)cdataptr(cd))[0] = 0; 124 ((double *)cdataptr(cd))[0] = 0;
@@ -133,65 +130,66 @@ static void lex_number(LexState *ls, TValue *tv)
133 lj_parse_keepcdata(ls, tv, cd); 130 lj_parse_keepcdata(ls, tv, cd);
134#endif 131#endif
135 } else { 132 } else {
136 lua_assert(fmt == STRSCAN_ERROR); 133 lj_assertLS(fmt == STRSCAN_ERROR,
134 "unexpected number format %d", fmt);
137 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); 135 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
138 } 136 }
139} 137}
140 138
141static int skip_sep(LexState *ls) 139/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
140static int lex_skipeq(LexState *ls)
142{ 141{
143 int count = 0; 142 int count = 0;
144 int s = ls->current; 143 LexChar s = ls->c;
145 lua_assert(s == '[' || s == ']'); 144 lj_assertLS(s == '[' || s == ']', "bad usage");
146 save_and_next(ls); 145 while (lex_savenext(ls) == '=' && count < 0x20000000)
147 while (ls->current == '=' && count < 0x20000000) {
148 save_and_next(ls);
149 count++; 146 count++;
150 } 147 return (ls->c == s) ? count : (-count) - 1;
151 return (ls->current == s) ? count : (-count) - 1;
152} 148}
153 149
154static void read_long_string(LexState *ls, TValue *tv, int sep) 150/* Parse a long string or long comment (tv set to NULL). */
151static void lex_longstring(LexState *ls, TValue *tv, int sep)
155{ 152{
156 save_and_next(ls); /* skip 2nd `[' */ 153 lex_savenext(ls); /* Skip second '['. */
157 if (currIsNewline(ls)) /* string starts with a newline? */ 154 if (lex_iseol(ls)) /* Skip initial newline. */
158 inclinenumber(ls); /* skip it */ 155 lex_newline(ls);
159 for (;;) { 156 for (;;) {
160 switch (ls->current) { 157 switch (ls->c) {
161 case END_OF_STREAM: 158 case LEX_EOF:
162 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); 159 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
163 break; 160 break;
164 case ']': 161 case ']':
165 if (skip_sep(ls) == sep) { 162 if (lex_skipeq(ls) == sep) {
166 save_and_next(ls); /* skip 2nd `]' */ 163 lex_savenext(ls); /* Skip second ']'. */
167 goto endloop; 164 goto endloop;
168 } 165 }
169 break; 166 break;
170 case '\n': 167 case '\n':
171 case '\r': 168 case '\r':
172 save(ls, '\n'); 169 lex_save(ls, '\n');
173 inclinenumber(ls); 170 lex_newline(ls);
174 if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */ 171 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
175 break; 172 break;
176 default: 173 default:
177 if (tv) save_and_next(ls); 174 lex_savenext(ls);
178 else next(ls);
179 break; 175 break;
180 } 176 }
181 } endloop: 177 } endloop:
182 if (tv) { 178 if (tv) {
183 GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep), 179 GCstr *str = lj_parse_keepstr(ls, ls->sb.b + (2 + (MSize)sep),
184 ls->sb.n - 2*(2 + (MSize)sep)); 180 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
185 setstrV(ls->L, tv, str); 181 setstrV(ls->L, tv, str);
186 } 182 }
187} 183}
188 184
189static void read_string(LexState *ls, int delim, TValue *tv) 185/* Parse a string. */
186static void lex_string(LexState *ls, TValue *tv)
190{ 187{
191 save_and_next(ls); 188 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
192 while (ls->current != delim) { 189 lex_savenext(ls);
193 switch (ls->current) { 190 while (ls->c != delim) {
194 case END_OF_STREAM: 191 switch (ls->c) {
192 case LEX_EOF:
195 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); 193 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
196 continue; 194 continue;
197 case '\n': 195 case '\n':
@@ -199,7 +197,7 @@ static void read_string(LexState *ls, int delim, TValue *tv)
199 lj_lex_error(ls, TK_string, LJ_ERR_XSTR); 197 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
200 continue; 198 continue;
201 case '\\': { 199 case '\\': {
202 int c = next(ls); /* Skip the '\\'. */ 200 LexChar c = lex_next(ls); /* Skip the '\\'. */
203 switch (c) { 201 switch (c) {
204 case 'a': c = '\a'; break; 202 case 'a': c = '\a'; break;
205 case 'b': c = '\b'; break; 203 case 'b': c = '\b'; break;
@@ -209,111 +207,139 @@ static void read_string(LexState *ls, int delim, TValue *tv)
209 case 't': c = '\t'; break; 207 case 't': c = '\t'; break;
210 case 'v': c = '\v'; break; 208 case 'v': c = '\v'; break;
211 case 'x': /* Hexadecimal escape '\xXX'. */ 209 case 'x': /* Hexadecimal escape '\xXX'. */
212 c = (next(ls) & 15u) << 4; 210 c = (lex_next(ls) & 15u) << 4;
213 if (!lj_char_isdigit(ls->current)) { 211 if (!lj_char_isdigit(ls->c)) {
214 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 212 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
215 c += 9 << 4; 213 c += 9 << 4;
216 } 214 }
217 c += (next(ls) & 15u); 215 c += (lex_next(ls) & 15u);
218 if (!lj_char_isdigit(ls->current)) { 216 if (!lj_char_isdigit(ls->c)) {
219 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 217 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
220 c += 9; 218 c += 9;
221 } 219 }
222 break; 220 break;
221 case 'u': /* Unicode escape '\u{XX...}'. */
222 if (lex_next(ls) != '{') goto err_xesc;
223 lex_next(ls);
224 c = 0;
225 do {
226 c = (c << 4) | (ls->c & 15u);
227 if (!lj_char_isdigit(ls->c)) {
228 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
229 c += 9;
230 }
231 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
232 } while (lex_next(ls) != '}');
233 if (c < 0x800) {
234 if (c < 0x80) break;
235 lex_save(ls, 0xc0 | (c >> 6));
236 } else {
237 if (c >= 0x10000) {
238 lex_save(ls, 0xf0 | (c >> 18));
239 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
240 } else {
241 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
242 lex_save(ls, 0xe0 | (c >> 12));
243 }
244 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
245 }
246 c = 0x80 | (c & 0x3f);
247 break;
223 case 'z': /* Skip whitespace. */ 248 case 'z': /* Skip whitespace. */
224 next(ls); 249 lex_next(ls);
225 while (lj_char_isspace(ls->current)) 250 while (lj_char_isspace(ls->c))
226 if (currIsNewline(ls)) inclinenumber(ls); else next(ls); 251 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
227 continue; 252 continue;
228 case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue; 253 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
229 case '\\': case '\"': case '\'': break; 254 case '\\': case '\"': case '\'': break;
230 case END_OF_STREAM: continue; 255 case LEX_EOF: continue;
231 default: 256 default:
232 if (!lj_char_isdigit(c)) 257 if (!lj_char_isdigit(c))
233 goto err_xesc; 258 goto err_xesc;
234 c -= '0'; /* Decimal escape '\ddd'. */ 259 c -= '0'; /* Decimal escape '\ddd'. */
235 if (lj_char_isdigit(next(ls))) { 260 if (lj_char_isdigit(lex_next(ls))) {
236 c = c*10 + (ls->current - '0'); 261 c = c*10 + (ls->c - '0');
237 if (lj_char_isdigit(next(ls))) { 262 if (lj_char_isdigit(lex_next(ls))) {
238 c = c*10 + (ls->current - '0'); 263 c = c*10 + (ls->c - '0');
239 if (c > 255) { 264 if (c > 255) {
240 err_xesc: 265 err_xesc:
241 lj_lex_error(ls, TK_string, LJ_ERR_XESC); 266 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
242 } 267 }
243 next(ls); 268 lex_next(ls);
244 } 269 }
245 } 270 }
246 save(ls, c); 271 lex_save(ls, c);
247 continue; 272 continue;
248 } 273 }
249 save(ls, c); 274 lex_save(ls, c);
250 next(ls); 275 lex_next(ls);
251 continue; 276 continue;
252 } 277 }
253 default: 278 default:
254 save_and_next(ls); 279 lex_savenext(ls);
255 break; 280 break;
256 } 281 }
257 } 282 }
258 save_and_next(ls); /* skip delimiter */ 283 lex_savenext(ls); /* Skip trailing delimiter. */
259 setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2)); 284 setstrV(ls->L, tv,
285 lj_parse_keepstr(ls, ls->sb.b+1, sbuflen(&ls->sb)-2));
260} 286}
261 287
262/* -- Main lexical scanner ------------------------------------------------ */ 288/* -- Main lexical scanner ------------------------------------------------ */
263 289
264static int llex(LexState *ls, TValue *tv) 290/* Get next lexical token. */
291static LexToken lex_scan(LexState *ls, TValue *tv)
265{ 292{
266 lj_str_resetbuf(&ls->sb); 293 lj_buf_reset(&ls->sb);
267 for (;;) { 294 for (;;) {
268 if (lj_char_isident(ls->current)) { 295 if (lj_char_isident(ls->c)) {
269 GCstr *s; 296 GCstr *s;
270 if (lj_char_isdigit(ls->current)) { /* Numeric literal. */ 297 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
271 lex_number(ls, tv); 298 lex_number(ls, tv);
272 return TK_number; 299 return TK_number;
273 } 300 }
274 /* Identifier or reserved word. */ 301 /* Identifier or reserved word. */
275 do { 302 do {
276 save_and_next(ls); 303 lex_savenext(ls);
277 } while (lj_char_isident(ls->current)); 304 } while (lj_char_isident(ls->c));
278 s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); 305 s = lj_parse_keepstr(ls, ls->sb.b, sbuflen(&ls->sb));
279 setstrV(ls->L, tv, s); 306 setstrV(ls->L, tv, s);
280 if (s->reserved > 0) /* Reserved word? */ 307 if (s->reserved > 0) /* Reserved word? */
281 return TK_OFS + s->reserved; 308 return TK_OFS + s->reserved;
282 return TK_name; 309 return TK_name;
283 } 310 }
284 switch (ls->current) { 311 switch (ls->c) {
285 case '\n': 312 case '\n':
286 case '\r': 313 case '\r':
287 inclinenumber(ls); 314 lex_newline(ls);
288 continue; 315 continue;
289 case ' ': 316 case ' ':
290 case '\t': 317 case '\t':
291 case '\v': 318 case '\v':
292 case '\f': 319 case '\f':
293 next(ls); 320 lex_next(ls);
294 continue; 321 continue;
295 case '-': 322 case '-':
296 next(ls); 323 lex_next(ls);
297 if (ls->current != '-') return '-'; 324 if (ls->c != '-') return '-';
298 /* else is a comment */ 325 lex_next(ls);
299 next(ls); 326 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
300 if (ls->current == '[') { 327 int sep = lex_skipeq(ls);
301 int sep = skip_sep(ls); 328 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
302 lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */
303 if (sep >= 0) { 329 if (sep >= 0) {
304 read_long_string(ls, NULL, sep); /* long comment */ 330 lex_longstring(ls, NULL, sep);
305 lj_str_resetbuf(&ls->sb); 331 lj_buf_reset(&ls->sb);
306 continue; 332 continue;
307 } 333 }
308 } 334 }
309 /* else short comment */ 335 /* Short comment "--.*\n". */
310 while (!currIsNewline(ls) && ls->current != END_OF_STREAM) 336 while (!lex_iseol(ls) && ls->c != LEX_EOF)
311 next(ls); 337 lex_next(ls);
312 continue; 338 continue;
313 case '[': { 339 case '[': {
314 int sep = skip_sep(ls); 340 int sep = lex_skipeq(ls);
315 if (sep >= 0) { 341 if (sep >= 0) {
316 read_long_string(ls, tv, sep); 342 lex_longstring(ls, tv, sep);
317 return TK_string; 343 return TK_string;
318 } else if (sep == -1) { 344 } else if (sep == -1) {
319 return '['; 345 return '[';
@@ -323,44 +349,43 @@ static int llex(LexState *ls, TValue *tv)
323 } 349 }
324 } 350 }
325 case '=': 351 case '=':
326 next(ls); 352 lex_next(ls);
327 if (ls->current != '=') return '='; else { next(ls); return TK_eq; } 353 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
328 case '<': 354 case '<':
329 next(ls); 355 lex_next(ls);
330 if (ls->current != '=') return '<'; else { next(ls); return TK_le; } 356 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
331 case '>': 357 case '>':
332 next(ls); 358 lex_next(ls);
333 if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } 359 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
334 case '~': 360 case '~':
335 next(ls); 361 lex_next(ls);
336 if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } 362 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
337 case ':': 363 case ':':
338 next(ls); 364 lex_next(ls);
339 if (ls->current != ':') return ':'; else { next(ls); return TK_label; } 365 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
340 case '"': 366 case '"':
341 case '\'': 367 case '\'':
342 read_string(ls, ls->current, tv); 368 lex_string(ls, tv);
343 return TK_string; 369 return TK_string;
344 case '.': 370 case '.':
345 save_and_next(ls); 371 if (lex_savenext(ls) == '.') {
346 if (ls->current == '.') { 372 lex_next(ls);
347 next(ls); 373 if (ls->c == '.') {
348 if (ls->current == '.') { 374 lex_next(ls);
349 next(ls);
350 return TK_dots; /* ... */ 375 return TK_dots; /* ... */
351 } 376 }
352 return TK_concat; /* .. */ 377 return TK_concat; /* .. */
353 } else if (!lj_char_isdigit(ls->current)) { 378 } else if (!lj_char_isdigit(ls->c)) {
354 return '.'; 379 return '.';
355 } else { 380 } else {
356 lex_number(ls, tv); 381 lex_number(ls, tv);
357 return TK_number; 382 return TK_number;
358 } 383 }
359 case END_OF_STREAM: 384 case LEX_EOF:
360 return TK_eof; 385 return TK_eof;
361 default: { 386 default: {
362 int c = ls->current; 387 LexChar c = ls->c;
363 next(ls); 388 lex_next(ls);
364 return c; /* Single-char tokens (+ - / ...). */ 389 return c; /* Single-char tokens (+ - / ...). */
365 } 390 }
366 } 391 }
@@ -375,36 +400,33 @@ int lj_lex_setup(lua_State *L, LexState *ls)
375 int header = 0; 400 int header = 0;
376 ls->L = L; 401 ls->L = L;
377 ls->fs = NULL; 402 ls->fs = NULL;
378 ls->n = 0; 403 ls->pe = ls->p = NULL;
379 ls->p = NULL;
380 ls->vstack = NULL; 404 ls->vstack = NULL;
381 ls->sizevstack = 0; 405 ls->sizevstack = 0;
382 ls->vtop = 0; 406 ls->vtop = 0;
383 ls->bcstack = NULL; 407 ls->bcstack = NULL;
384 ls->sizebcstack = 0; 408 ls->sizebcstack = 0;
385 ls->token = 0; 409 ls->tok = 0;
386 ls->lookahead = TK_eof; /* No look-ahead token. */ 410 ls->lookahead = TK_eof; /* No look-ahead token. */
387 ls->linenumber = 1; 411 ls->linenumber = 1;
388 ls->lastline = 1; 412 ls->lastline = 1;
389 ls->endmark = 0; 413 ls->endmark = 0;
390 lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF); 414 lex_next(ls); /* Read-ahead first char. */
391 next(ls); /* Read-ahead first char. */ 415 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
392 if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb && 416 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
393 char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
394 ls->n -= 2;
395 ls->p += 2; 417 ls->p += 2;
396 next(ls); 418 lex_next(ls);
397 header = 1; 419 header = 1;
398 } 420 }
399 if (ls->current == '#') { /* Skip POSIX #! header line. */ 421 if (ls->c == '#') { /* Skip POSIX #! header line. */
400 do { 422 do {
401 next(ls); 423 lex_next(ls);
402 if (ls->current == END_OF_STREAM) return 0; 424 if (ls->c == LEX_EOF) return 0;
403 } while (!currIsNewline(ls)); 425 } while (!lex_iseol(ls));
404 inclinenumber(ls); 426 lex_newline(ls);
405 header = 1; 427 header = 1;
406 } 428 }
407 if (ls->current == LUA_SIGNATURE[0]) { /* Bytecode dump. */ 429 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
408 if (header) { 430 if (header) {
409 /* 431 /*
410 ** Loading bytecode with an extra header is disabled for security 432 ** Loading bytecode with an extra header is disabled for security
@@ -426,55 +448,60 @@ void lj_lex_cleanup(lua_State *L, LexState *ls)
426 global_State *g = G(L); 448 global_State *g = G(L);
427 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); 449 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
428 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); 450 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
429 lj_str_freebuf(g, &ls->sb); 451 lj_buf_free(g, &ls->sb);
430} 452}
431 453
454/* Return next lexical token. */
432void lj_lex_next(LexState *ls) 455void lj_lex_next(LexState *ls)
433{ 456{
434 ls->lastline = ls->linenumber; 457 ls->lastline = ls->linenumber;
435 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ 458 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
436 ls->token = llex(ls, &ls->tokenval); /* Get next token. */ 459 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
437 } else { /* Otherwise return lookahead token. */ 460 } else { /* Otherwise return lookahead token. */
438 ls->token = ls->lookahead; 461 ls->tok = ls->lookahead;
439 ls->lookahead = TK_eof; 462 ls->lookahead = TK_eof;
440 ls->tokenval = ls->lookaheadval; 463 ls->tokval = ls->lookaheadval;
441 } 464 }
442} 465}
443 466
467/* Look ahead for the next token. */
444LexToken lj_lex_lookahead(LexState *ls) 468LexToken lj_lex_lookahead(LexState *ls)
445{ 469{
446 lua_assert(ls->lookahead == TK_eof); 470 lj_assertLS(ls->lookahead == TK_eof, "double lookahead");
447 ls->lookahead = llex(ls, &ls->lookaheadval); 471 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
448 return ls->lookahead; 472 return ls->lookahead;
449} 473}
450 474
451const char *lj_lex_token2str(LexState *ls, LexToken token) 475/* Convert token to string. */
476const char *lj_lex_token2str(LexState *ls, LexToken tok)
452{ 477{
453 if (token > TK_OFS) 478 if (tok > TK_OFS)
454 return tokennames[token-TK_OFS-1]; 479 return tokennames[tok-TK_OFS-1];
455 else if (!lj_char_iscntrl(token)) 480 else if (!lj_char_iscntrl(tok))
456 return lj_str_pushf(ls->L, "%c", token); 481 return lj_strfmt_pushf(ls->L, "%c", tok);
457 else 482 else
458 return lj_str_pushf(ls->L, "char(%d)", token); 483 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
459} 484}
460 485
461void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...) 486/* Lexer error. */
487void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
462{ 488{
463 const char *tok; 489 const char *tokstr;
464 va_list argp; 490 va_list argp;
465 if (token == 0) { 491 if (tok == 0) {
466 tok = NULL; 492 tokstr = NULL;
467 } else if (token == TK_name || token == TK_string || token == TK_number) { 493 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
468 save(ls, '\0'); 494 lex_save(ls, '\0');
469 tok = ls->sb.buf; 495 tokstr = ls->sb.b;
470 } else { 496 } else {
471 tok = lj_lex_token2str(ls, token); 497 tokstr = lj_lex_token2str(ls, tok);
472 } 498 }
473 va_start(argp, em); 499 va_start(argp, em);
474 lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp); 500 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
475 va_end(argp); 501 va_end(argp);
476} 502}
477 503
504/* Initialize strings for reserved words. */
478void lj_lex_init(lua_State *L) 505void lj_lex_init(lua_State *L)
479{ 506{
480 uint32_t i; 507 uint32_t i;
diff --git a/src/lj_lex.h b/src/lj_lex.h
index e244806e..e46fbd89 100644
--- a/src/lj_lex.h
+++ b/src/lj_lex.h
@@ -30,7 +30,8 @@ TKDEF(TKENUM1, TKENUM2)
30 TK_RESERVED = TK_while - TK_OFS 30 TK_RESERVED = TK_while - TK_OFS
31}; 31};
32 32
33typedef int LexToken; 33typedef int LexChar; /* Lexical character. Unsigned ext. from char. */
34typedef int LexToken; /* Lexical token. */
34 35
35/* Combined bytecode ins/line. Only used during bytecode generation. */ 36/* Combined bytecode ins/line. Only used during bytecode generation. */
36typedef struct BCInsLine { 37typedef struct BCInsLine {
@@ -51,13 +52,13 @@ typedef struct VarInfo {
51typedef struct LexState { 52typedef struct LexState {
52 struct FuncState *fs; /* Current FuncState. Defined in lj_parse.c. */ 53 struct FuncState *fs; /* Current FuncState. Defined in lj_parse.c. */
53 struct lua_State *L; /* Lua state. */ 54 struct lua_State *L; /* Lua state. */
54 TValue tokenval; /* Current token value. */ 55 TValue tokval; /* Current token value. */
55 TValue lookaheadval; /* Lookahead token value. */ 56 TValue lookaheadval; /* Lookahead token value. */
56 int current; /* Current character (charint). */
57 LexToken token; /* Current token. */
58 LexToken lookahead; /* Lookahead token. */
59 MSize n; /* Bytes left in input buffer. */
60 const char *p; /* Current position in input buffer. */ 57 const char *p; /* Current position in input buffer. */
58 const char *pe; /* End of input buffer. */
59 LexChar c; /* Current character. */
60 LexToken tok; /* Current token. */
61 LexToken lookahead; /* Lookahead token. */
61 SBuf sb; /* String buffer for tokens. */ 62 SBuf sb; /* String buffer for tokens. */
62 lua_Reader rfunc; /* Reader callback. */ 63 lua_Reader rfunc; /* Reader callback. */
63 void *rdata; /* Reader callback data. */ 64 void *rdata; /* Reader callback data. */
@@ -79,8 +80,14 @@ LJ_FUNC int lj_lex_setup(lua_State *L, LexState *ls);
79LJ_FUNC void lj_lex_cleanup(lua_State *L, LexState *ls); 80LJ_FUNC void lj_lex_cleanup(lua_State *L, LexState *ls);
80LJ_FUNC void lj_lex_next(LexState *ls); 81LJ_FUNC void lj_lex_next(LexState *ls);
81LJ_FUNC LexToken lj_lex_lookahead(LexState *ls); 82LJ_FUNC LexToken lj_lex_lookahead(LexState *ls);
82LJ_FUNC const char *lj_lex_token2str(LexState *ls, LexToken token); 83LJ_FUNC const char *lj_lex_token2str(LexState *ls, LexToken tok);
83LJ_FUNC_NORET void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...); 84LJ_FUNC_NORET void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...);
84LJ_FUNC void lj_lex_init(lua_State *L); 85LJ_FUNC void lj_lex_init(lua_State *L);
85 86
87#ifdef LUA_USE_ASSERT
88#define lj_assertLS(c, ...) (lj_assertG_(G(ls->L), (c), __VA_ARGS__))
89#else
90#define lj_assertLS(c, ...) ((void)ls)
91#endif
92
86#endif 93#endif
diff --git a/src/lj_lib.c b/src/lj_lib.c
index 67ecb5a3..ebe0dc78 100644
--- a/src/lj_lib.c
+++ b/src/lj_lib.c
@@ -16,8 +16,14 @@
16#include "lj_func.h" 16#include "lj_func.h"
17#include "lj_bc.h" 17#include "lj_bc.h"
18#include "lj_dispatch.h" 18#include "lj_dispatch.h"
19#if LJ_HASFFI
20#include "lj_ctype.h"
21#endif
19#include "lj_vm.h" 22#include "lj_vm.h"
20#include "lj_strscan.h" 23#include "lj_strscan.h"
24#include "lj_strfmt.h"
25#include "lj_lex.h"
26#include "lj_bcdump.h"
21#include "lj_lib.h" 27#include "lj_lib.h"
22 28
23/* -- Library initialization ---------------------------------------------- */ 29/* -- Library initialization ---------------------------------------------- */
@@ -43,6 +49,28 @@ static GCtab *lib_create_table(lua_State *L, const char *libname, int hsize)
43 return tabV(L->top-1); 49 return tabV(L->top-1);
44} 50}
45 51
52static const uint8_t *lib_read_lfunc(lua_State *L, const uint8_t *p, GCtab *tab)
53{
54 int len = *p++;
55 GCstr *name = lj_str_new(L, (const char *)p, len);
56 LexState ls;
57 GCproto *pt;
58 GCfunc *fn;
59 memset(&ls, 0, sizeof(ls));
60 ls.L = L;
61 ls.p = (const char *)(p+len);
62 ls.pe = (const char *)~(uintptr_t)0;
63 ls.c = -1;
64 ls.level = (BCDUMP_F_STRIP|(LJ_BE*BCDUMP_F_BE));
65 ls.chunkname = name;
66 pt = lj_bcread_proto(&ls);
67 pt->firstline = ~(BCLine)0;
68 fn = lj_func_newL_empty(L, pt, tabref(L->env));
69 /* NOBARRIER: See below for common barrier. */
70 setfuncV(L, lj_tab_setstr(L, tab, name), fn);
71 return (const uint8_t *)ls.p;
72}
73
46void lj_lib_register(lua_State *L, const char *libname, 74void lj_lib_register(lua_State *L, const char *libname,
47 const uint8_t *p, const lua_CFunction *cf) 75 const uint8_t *p, const lua_CFunction *cf)
48{ 76{
@@ -87,6 +115,9 @@ void lj_lib_register(lua_State *L, const char *libname,
87 ofn = fn; 115 ofn = fn;
88 } else { 116 } else {
89 switch (tag | len) { 117 switch (tag | len) {
118 case LIBINIT_LUA:
119 p = lib_read_lfunc(L, p, tab);
120 break;
90 case LIBINIT_SET: 121 case LIBINIT_SET:
91 L->top -= 2; 122 L->top -= 2;
92 if (tvisstr(L->top+1) && strV(L->top+1)->len == 0) 123 if (tvisstr(L->top+1) && strV(L->top+1)->len == 0)
@@ -120,6 +151,37 @@ void lj_lib_register(lua_State *L, const char *libname,
120 } 151 }
121} 152}
122 153
154/* Push internal function on the stack. */
155GCfunc *lj_lib_pushcc(lua_State *L, lua_CFunction f, int id, int n)
156{
157 GCfunc *fn;
158 lua_pushcclosure(L, f, n);
159 fn = funcV(L->top-1);
160 fn->c.ffid = (uint8_t)id;
161 setmref(fn->c.pc, &G(L)->bc_cfunc_int);
162 return fn;
163}
164
165void lj_lib_prereg(lua_State *L, const char *name, lua_CFunction f, GCtab *env)
166{
167 luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOAD", 4);
168 lua_pushcfunction(L, f);
169 /* NOBARRIER: The function is new (marked white). */
170 setgcref(funcV(L->top-1)->c.env, obj2gco(env));
171 lua_setfield(L, -2, name);
172 L->top--;
173}
174
175int lj_lib_postreg(lua_State *L, lua_CFunction cf, int id, const char *name)
176{
177 GCfunc *fn = lj_lib_pushcf(L, cf, id);
178 GCtab *t = tabref(curr_func(L)->c.env); /* Reference to parent table. */
179 setfuncV(L, lj_tab_setstr(L, t, lj_str_newz(L, name)), fn);
180 lj_gc_anybarriert(L, t);
181 setfuncV(L, L->top++, fn);
182 return 1;
183}
184
123/* -- Type checks --------------------------------------------------------- */ 185/* -- Type checks --------------------------------------------------------- */
124 186
125TValue *lj_lib_checkany(lua_State *L, int narg) 187TValue *lj_lib_checkany(lua_State *L, int narg)
@@ -137,7 +199,7 @@ GCstr *lj_lib_checkstr(lua_State *L, int narg)
137 if (LJ_LIKELY(tvisstr(o))) { 199 if (LJ_LIKELY(tvisstr(o))) {
138 return strV(o); 200 return strV(o);
139 } else if (tvisnumber(o)) { 201 } else if (tvisnumber(o)) {
140 GCstr *s = lj_str_fromnumber(L, o); 202 GCstr *s = lj_strfmt_number(L, o);
141 setstrV(L, o, s); 203 setstrV(L, o, s);
142 return s; 204 return s;
143 } 205 }
@@ -196,20 +258,6 @@ int32_t lj_lib_optint(lua_State *L, int narg, int32_t def)
196 return (o < L->top && !tvisnil(o)) ? lj_lib_checkint(L, narg) : def; 258 return (o < L->top && !tvisnil(o)) ? lj_lib_checkint(L, narg) : def;
197} 259}
198 260
199int32_t lj_lib_checkbit(lua_State *L, int narg)
200{
201 TValue *o = L->base + narg-1;
202 if (!(o < L->top && lj_strscan_numberobj(o)))
203 lj_err_argt(L, narg, LUA_TNUMBER);
204 if (LJ_LIKELY(tvisint(o))) {
205 return intV(o);
206 } else {
207 int32_t i = lj_num2bit(numV(o));
208 if (LJ_DUALNUM) setintV(o, i);
209 return i;
210 }
211}
212
213GCfunc *lj_lib_checkfunc(lua_State *L, int narg) 261GCfunc *lj_lib_checkfunc(lua_State *L, int narg)
214{ 262{
215 TValue *o = L->base + narg-1; 263 TValue *o = L->base + narg-1;
@@ -256,3 +304,56 @@ int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst)
256 return def; 304 return def;
257} 305}
258 306
307/* -- Strict type checks -------------------------------------------------- */
308
309/* The following type checks do not coerce between strings and numbers.
310** And they handle plain int64_t/uint64_t FFI numbers, too.
311*/
312
313#if LJ_HASBUFFER
314GCstr *lj_lib_checkstrx(lua_State *L, int narg)
315{
316 TValue *o = L->base + narg-1;
317 if (!(o < L->top && tvisstr(o))) lj_err_argt(L, narg, LUA_TSTRING);
318 return strV(o);
319}
320
321int32_t lj_lib_checkintrange(lua_State *L, int narg, int32_t a, int32_t b)
322{
323 TValue *o = L->base + narg-1;
324 lj_assertL(b >= 0, "expected range must be non-negative");
325 if (o < L->top) {
326 if (LJ_LIKELY(tvisint(o))) {
327 int32_t i = intV(o);
328 if (i >= a && i <= b) return i;
329 } else if (LJ_LIKELY(tvisnum(o))) {
330 /* For performance reasons, this doesn't check for integerness or
331 ** integer overflow. Overflow detection still works, since all FPUs
332 ** return either MININT or MAXINT, which is then out of range.
333 */
334 int32_t i = (int32_t)numV(o);
335 if (i >= a && i <= b) return i;
336#if LJ_HASFFI
337 } else if (tviscdata(o)) {
338 GCcdata *cd = cdataV(o);
339 if (cd->ctypeid == CTID_INT64) {
340 int64_t i = *(int64_t *)cdataptr(cd);
341 if (i >= (int64_t)a && i <= (int64_t)b) return (int32_t)i;
342 } else if (cd->ctypeid == CTID_UINT64) {
343 uint64_t i = *(uint64_t *)cdataptr(cd);
344 if ((a < 0 || i >= (uint64_t)a) && i <= (uint64_t)b) return (int32_t)i;
345 } else {
346 goto badtype;
347 }
348#endif
349 } else {
350 goto badtype;
351 }
352 lj_err_arg(L, narg, LJ_ERR_NUMRNG);
353 }
354badtype:
355 lj_err_argt(L, narg, LUA_TNUMBER);
356 return 0; /* unreachable */
357}
358#endif
359
diff --git a/src/lj_lib.h b/src/lj_lib.h
index d8e93925..6c3a1c83 100644
--- a/src/lj_lib.h
+++ b/src/lj_lib.h
@@ -41,15 +41,28 @@ LJ_FUNC void lj_lib_checknumber(lua_State *L, int narg);
41LJ_FUNC lua_Number lj_lib_checknum(lua_State *L, int narg); 41LJ_FUNC lua_Number lj_lib_checknum(lua_State *L, int narg);
42LJ_FUNC int32_t lj_lib_checkint(lua_State *L, int narg); 42LJ_FUNC int32_t lj_lib_checkint(lua_State *L, int narg);
43LJ_FUNC int32_t lj_lib_optint(lua_State *L, int narg, int32_t def); 43LJ_FUNC int32_t lj_lib_optint(lua_State *L, int narg, int32_t def);
44LJ_FUNC int32_t lj_lib_checkbit(lua_State *L, int narg);
45LJ_FUNC GCfunc *lj_lib_checkfunc(lua_State *L, int narg); 44LJ_FUNC GCfunc *lj_lib_checkfunc(lua_State *L, int narg);
46LJ_FUNC GCtab *lj_lib_checktab(lua_State *L, int narg); 45LJ_FUNC GCtab *lj_lib_checktab(lua_State *L, int narg);
47LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg); 46LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg);
48LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst); 47LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst);
49 48
49#if LJ_HASBUFFER
50LJ_FUNC GCstr *lj_lib_checkstrx(lua_State *L, int narg);
51LJ_FUNC int32_t lj_lib_checkintrange(lua_State *L, int narg,
52 int32_t a, int32_t b);
53#endif
54
50/* Avoid including lj_frame.h. */ 55/* Avoid including lj_frame.h. */
56#if LJ_GC64
57#define lj_lib_upvalue(L, n) \
58 (&gcval(L->base-2)->fn.c.upvalue[(n)-1])
59#elif LJ_FR2
60#define lj_lib_upvalue(L, n) \
61 (&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1])
62#else
51#define lj_lib_upvalue(L, n) \ 63#define lj_lib_upvalue(L, n) \
52 (&gcref((L->base-1)->fr.func)->fn.c.upvalue[(n)-1]) 64 (&gcref((L->base-1)->fr.func)->fn.c.upvalue[(n)-1])
65#endif
53 66
54#if LJ_TARGET_WINDOWS 67#if LJ_TARGET_WINDOWS
55#define lj_lib_checkfpu(L) \ 68#define lj_lib_checkfpu(L) \
@@ -60,23 +73,14 @@ LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst);
60#define lj_lib_checkfpu(L) UNUSED(L) 73#define lj_lib_checkfpu(L) UNUSED(L)
61#endif 74#endif
62 75
63/* Push internal function on the stack. */ 76LJ_FUNC GCfunc *lj_lib_pushcc(lua_State *L, lua_CFunction f, int id, int n);
64static LJ_AINLINE void lj_lib_pushcc(lua_State *L, lua_CFunction f,
65 int id, int n)
66{
67 GCfunc *fn;
68 lua_pushcclosure(L, f, n);
69 fn = funcV(L->top-1);
70 fn->c.ffid = (uint8_t)id;
71 setmref(fn->c.pc, &G(L)->bc_cfunc_int);
72}
73
74#define lj_lib_pushcf(L, fn, id) (lj_lib_pushcc(L, (fn), (id), 0)) 77#define lj_lib_pushcf(L, fn, id) (lj_lib_pushcc(L, (fn), (id), 0))
75 78
76/* Library function declarations. Scanned by buildvm. */ 79/* Library function declarations. Scanned by buildvm. */
77#define LJLIB_CF(name) static int lj_cf_##name(lua_State *L) 80#define LJLIB_CF(name) static int lj_cf_##name(lua_State *L)
78#define LJLIB_ASM(name) static int lj_ffh_##name(lua_State *L) 81#define LJLIB_ASM(name) static int lj_ffh_##name(lua_State *L)
79#define LJLIB_ASM_(name) 82#define LJLIB_ASM_(name)
83#define LJLIB_LUA(name)
80#define LJLIB_SET(name) 84#define LJLIB_SET(name)
81#define LJLIB_PUSH(arg) 85#define LJLIB_PUSH(arg)
82#define LJLIB_REC(handler) 86#define LJLIB_REC(handler)
@@ -88,6 +92,10 @@ static LJ_AINLINE void lj_lib_pushcc(lua_State *L, lua_CFunction f,
88 92
89LJ_FUNC void lj_lib_register(lua_State *L, const char *libname, 93LJ_FUNC void lj_lib_register(lua_State *L, const char *libname,
90 const uint8_t *init, const lua_CFunction *cf); 94 const uint8_t *init, const lua_CFunction *cf);
95LJ_FUNC void lj_lib_prereg(lua_State *L, const char *name, lua_CFunction f,
96 GCtab *env);
97LJ_FUNC int lj_lib_postreg(lua_State *L, lua_CFunction cf, int id,
98 const char *name);
91 99
92/* Library init data tags. */ 100/* Library init data tags. */
93#define LIBINIT_LENMASK 0x3f 101#define LIBINIT_LENMASK 0x3f
@@ -96,7 +104,8 @@ LJ_FUNC void lj_lib_register(lua_State *L, const char *libname,
96#define LIBINIT_ASM 0x40 104#define LIBINIT_ASM 0x40
97#define LIBINIT_ASM_ 0x80 105#define LIBINIT_ASM_ 0x80
98#define LIBINIT_STRING 0xc0 106#define LIBINIT_STRING 0xc0
99#define LIBINIT_MAXSTR 0x39 107#define LIBINIT_MAXSTR 0x38
108#define LIBINIT_LUA 0xf9
100#define LIBINIT_SET 0xfa 109#define LIBINIT_SET 0xfa
101#define LIBINIT_NUMBER 0xfb 110#define LIBINIT_NUMBER 0xfb
102#define LIBINIT_COPY 0xfc 111#define LIBINIT_COPY 0xfc
@@ -104,9 +113,4 @@ LJ_FUNC void lj_lib_register(lua_State *L, const char *libname,
104#define LIBINIT_FFID 0xfe 113#define LIBINIT_FFID 0xfe
105#define LIBINIT_END 0xff 114#define LIBINIT_END 0xff
106 115
107/* Exported library functions. */
108
109typedef struct RandomState RandomState;
110LJ_FUNC uint64_t LJ_FASTCALL lj_math_random_step(RandomState *rs);
111
112#endif 116#endif
diff --git a/src/lj_load.c b/src/lj_load.c
index dab037b4..07304487 100644
--- a/src/lj_load.c
+++ b/src/lj_load.c
@@ -15,7 +15,7 @@
15#include "lj_obj.h" 15#include "lj_obj.h"
16#include "lj_gc.h" 16#include "lj_gc.h"
17#include "lj_err.h" 17#include "lj_err.h"
18#include "lj_str.h" 18#include "lj_buf.h"
19#include "lj_func.h" 19#include "lj_func.h"
20#include "lj_frame.h" 20#include "lj_frame.h"
21#include "lj_vm.h" 21#include "lj_vm.h"
@@ -54,7 +54,7 @@ LUA_API int lua_loadx(lua_State *L, lua_Reader reader, void *data,
54 ls.rdata = data; 54 ls.rdata = data;
55 ls.chunkarg = chunkname ? chunkname : "?"; 55 ls.chunkarg = chunkname ? chunkname : "?";
56 ls.mode = mode; 56 ls.mode = mode;
57 lj_str_initbuf(&ls.sb); 57 lj_buf_init(L, &ls.sb);
58 status = lj_vm_cpcall(L, NULL, &ls, cpparser); 58 status = lj_vm_cpcall(L, NULL, &ls, cpparser);
59 lj_lex_cleanup(L, &ls); 59 lj_lex_cleanup(L, &ls);
60 lj_gc_check(L); 60 lj_gc_check(L);
@@ -159,7 +159,7 @@ LUALIB_API int luaL_loadstring(lua_State *L, const char *s)
159LUA_API int lua_dump(lua_State *L, lua_Writer writer, void *data) 159LUA_API int lua_dump(lua_State *L, lua_Writer writer, void *data)
160{ 160{
161 cTValue *o = L->top-1; 161 cTValue *o = L->top-1;
162 api_check(L, L->top > L->base); 162 lj_checkapi(L->top > L->base, "top slot empty");
163 if (tvisfunc(o) && isluafunc(funcV(o))) 163 if (tvisfunc(o) && isluafunc(funcV(o)))
164 return lj_bcwrite(L, funcproto(funcV(o)), writer, data, 0); 164 return lj_bcwrite(L, funcproto(funcV(o)), writer, data, 0);
165 else 165 else
diff --git a/src/lj_mcode.c b/src/lj_mcode.c
index 7013cd7d..94767937 100644
--- a/src/lj_mcode.c
+++ b/src/lj_mcode.c
@@ -14,6 +14,7 @@
14#include "lj_mcode.h" 14#include "lj_mcode.h"
15#include "lj_trace.h" 15#include "lj_trace.h"
16#include "lj_dispatch.h" 16#include "lj_dispatch.h"
17#include "lj_prng.h"
17#endif 18#endif
18#if LJ_HASJIT || LJ_HASFFI 19#if LJ_HASJIT || LJ_HASFFI
19#include "lj_vm.h" 20#include "lj_vm.h"
@@ -44,7 +45,7 @@ void lj_mcode_sync(void *start, void *end)
44 sys_icache_invalidate(start, (char *)end-(char *)start); 45 sys_icache_invalidate(start, (char *)end-(char *)start);
45#elif LJ_TARGET_PPC 46#elif LJ_TARGET_PPC
46 lj_vm_cachesync(start, end); 47 lj_vm_cachesync(start, end);
47#elif defined(__GNUC__) 48#elif defined(__GNUC__) || defined(__clang__)
48 __clear_cache(start, end); 49 __clear_cache(start, end);
49#else 50#else
50#error "Missing builtin to flush instruction cache" 51#error "Missing builtin to flush instruction cache"
@@ -66,8 +67,8 @@ void lj_mcode_sync(void *start, void *end)
66 67
67static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, DWORD prot) 68static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, DWORD prot)
68{ 69{
69 void *p = VirtualAlloc((void *)hint, sz, 70 void *p = LJ_WIN_VALLOC((void *)hint, sz,
70 MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot); 71 MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot);
71 if (!p && !hint) 72 if (!p && !hint)
72 lj_trace_err(J, LJ_TRERR_MCODEAL); 73 lj_trace_err(J, LJ_TRERR_MCODEAL);
73 return p; 74 return p;
@@ -82,7 +83,7 @@ static void mcode_free(jit_State *J, void *p, size_t sz)
82static int mcode_setprot(void *p, size_t sz, DWORD prot) 83static int mcode_setprot(void *p, size_t sz, DWORD prot)
83{ 84{
84 DWORD oprot; 85 DWORD oprot;
85 return !VirtualProtect(p, sz, prot, &oprot); 86 return !LJ_WIN_VPROTECT(p, sz, prot, &oprot);
86} 87}
87 88
88#elif LJ_TARGET_POSIX 89#elif LJ_TARGET_POSIX
@@ -96,10 +97,15 @@ static int mcode_setprot(void *p, size_t sz, DWORD prot)
96#define MCPROT_RW (PROT_READ|PROT_WRITE) 97#define MCPROT_RW (PROT_READ|PROT_WRITE)
97#define MCPROT_RX (PROT_READ|PROT_EXEC) 98#define MCPROT_RX (PROT_READ|PROT_EXEC)
98#define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC) 99#define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC)
100#ifdef PROT_MPROTECT
101#define MCPROT_CREATE (PROT_MPROTECT(MCPROT_RWX))
102#else
103#define MCPROT_CREATE 0
104#endif
99 105
100static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) 106static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
101{ 107{
102 void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); 108 void *p = mmap((void *)hint, sz, prot|MCPROT_CREATE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
103 if (p == MAP_FAILED) { 109 if (p == MAP_FAILED) {
104 if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL); 110 if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL);
105 p = NULL; 111 p = NULL;
@@ -118,52 +124,34 @@ static int mcode_setprot(void *p, size_t sz, int prot)
118 return mprotect(p, sz, prot); 124 return mprotect(p, sz, prot);
119} 125}
120 126
121#elif LJ_64
122
123#error "Missing OS support for explicit placement of executable memory"
124
125#else 127#else
126 128
127/* Fallback allocator. This will fail if memory is not executable by default. */ 129#error "Missing OS support for explicit placement of executable memory"
128#define LUAJIT_UNPROTECT_MCODE
129#define MCPROT_RW 0
130#define MCPROT_RX 0
131#define MCPROT_RWX 0
132
133static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
134{
135 UNUSED(hint); UNUSED(prot);
136 return lj_mem_new(J->L, sz);
137}
138
139static void mcode_free(jit_State *J, void *p, size_t sz)
140{
141 lj_mem_free(J2G(J), p, sz);
142}
143 130
144#endif 131#endif
145 132
146/* -- MCode area protection ----------------------------------------------- */ 133/* -- MCode area protection ----------------------------------------------- */
147 134
148/* Define this ONLY if page protection twiddling becomes a bottleneck. */ 135#if LUAJIT_SECURITY_MCODE == 0
149#ifdef LUAJIT_UNPROTECT_MCODE
150 136
151/* It's generally considered to be a potential security risk to have 137/* Define this ONLY if page protection twiddling becomes a bottleneck.
138**
139** It's generally considered to be a potential security risk to have
152** pages with simultaneous write *and* execute access in a process. 140** pages with simultaneous write *and* execute access in a process.
153** 141**
154** Do not even think about using this mode for server processes or 142** Do not even think about using this mode for server processes or
155** apps handling untrusted external data (such as a browser). 143** apps handling untrusted external data.
156** 144**
157** The security risk is not in LuaJIT itself -- but if an adversary finds 145** The security risk is not in LuaJIT itself -- but if an adversary finds
158** any *other* flaw in your C application logic, then any RWX memory page 146** any *other* flaw in your C application logic, then any RWX memory pages
159** simplifies writing an exploit considerably. 147** simplify writing an exploit considerably.
160*/ 148*/
161#define MCPROT_GEN MCPROT_RWX 149#define MCPROT_GEN MCPROT_RWX
162#define MCPROT_RUN MCPROT_RWX 150#define MCPROT_RUN MCPROT_RWX
163 151
164static void mcode_protect(jit_State *J, int prot) 152static void mcode_protect(jit_State *J, int prot)
165{ 153{
166 UNUSED(J); UNUSED(prot); 154 UNUSED(J); UNUSED(prot); UNUSED(mcode_setprot);
167} 155}
168 156
169#else 157#else
@@ -222,8 +210,8 @@ static void *mcode_alloc(jit_State *J, size_t sz)
222 */ 210 */
223#if LJ_TARGET_MIPS 211#if LJ_TARGET_MIPS
224 /* Use the middle of the 256MB-aligned region. */ 212 /* Use the middle of the 256MB-aligned region. */
225 uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler & 0xf0000000u) + 213 uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler &
226 0x08000000u; 214 ~(uintptr_t)0x0fffffffu) + 0x08000000u;
227#else 215#else
228 uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff; 216 uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff;
229#endif 217#endif
@@ -243,7 +231,7 @@ static void *mcode_alloc(jit_State *J, size_t sz)
243 } 231 }
244 /* Next try probing 64K-aligned pseudo-random addresses. */ 232 /* Next try probing 64K-aligned pseudo-random addresses. */
245 do { 233 do {
246 hint = LJ_PRNG_BITS(J, LJ_TARGET_JUMPRANGE-16) << 16; 234 hint = lj_prng_u64(&J2G(J)->prng) & ((1u<<LJ_TARGET_JUMPRANGE)-0x10000);
247 } while (!(hint + sz < range+range)); 235 } while (!(hint + sz < range+range));
248 hint = target + hint - range; 236 hint = target + hint - range;
249 } 237 }
@@ -256,7 +244,7 @@ static void *mcode_alloc(jit_State *J, size_t sz)
256/* All memory addresses are reachable by relative jumps. */ 244/* All memory addresses are reachable by relative jumps. */
257static void *mcode_alloc(jit_State *J, size_t sz) 245static void *mcode_alloc(jit_State *J, size_t sz)
258{ 246{
259#ifdef __OpenBSD__ 247#if defined(__OpenBSD__) || defined(__NetBSD__) || LJ_TARGET_UWP
260 /* Allow better executable memory allocation for OpenBSD W^X mode. */ 248 /* Allow better executable memory allocation for OpenBSD W^X mode. */
261 void *p = mcode_alloc_at(J, 0, sz, MCPROT_RUN); 249 void *p = mcode_alloc_at(J, 0, sz, MCPROT_RUN);
262 if (p && mcode_setprot(p, sz, MCPROT_GEN)) { 250 if (p && mcode_setprot(p, sz, MCPROT_GEN)) {
@@ -287,6 +275,7 @@ static void mcode_allocarea(jit_State *J)
287 ((MCLink *)J->mcarea)->next = oldarea; 275 ((MCLink *)J->mcarea)->next = oldarea;
288 ((MCLink *)J->mcarea)->size = sz; 276 ((MCLink *)J->mcarea)->size = sz;
289 J->szallmcarea += sz; 277 J->szallmcarea += sz;
278 J->mcbot = (MCode *)lj_err_register_mcode(J->mcarea, sz, (uint8_t *)J->mcbot);
290} 279}
291 280
292/* Free all MCode areas. */ 281/* Free all MCode areas. */
@@ -297,7 +286,9 @@ void lj_mcode_free(jit_State *J)
297 J->szallmcarea = 0; 286 J->szallmcarea = 0;
298 while (mc) { 287 while (mc) {
299 MCode *next = ((MCLink *)mc)->next; 288 MCode *next = ((MCLink *)mc)->next;
300 mcode_free(J, mc, ((MCLink *)mc)->size); 289 size_t sz = ((MCLink *)mc)->size;
290 lj_err_deregister_mcode(mc, sz, (uint8_t *)mc + sizeof(MCLink));
291 mcode_free(J, mc, sz);
301 mc = next; 292 mc = next;
302 } 293 }
303} 294}
@@ -332,35 +323,36 @@ void lj_mcode_abort(jit_State *J)
332/* Set/reset protection to allow patching of MCode areas. */ 323/* Set/reset protection to allow patching of MCode areas. */
333MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish) 324MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
334{ 325{
335#ifdef LUAJIT_UNPROTECT_MCODE
336 UNUSED(J); UNUSED(ptr); UNUSED(finish);
337 return NULL;
338#else
339 if (finish) { 326 if (finish) {
327#if LUAJIT_SECURITY_MCODE
340 if (J->mcarea == ptr) 328 if (J->mcarea == ptr)
341 mcode_protect(J, MCPROT_RUN); 329 mcode_protect(J, MCPROT_RUN);
342 else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN))) 330 else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN)))
343 mcode_protfail(J); 331 mcode_protfail(J);
332#endif
344 return NULL; 333 return NULL;
345 } else { 334 } else {
346 MCode *mc = J->mcarea; 335 MCode *mc = J->mcarea;
347 /* Try current area first to use the protection cache. */ 336 /* Try current area first to use the protection cache. */
348 if (ptr >= mc && ptr < (MCode *)((char *)mc + J->szmcarea)) { 337 if (ptr >= mc && ptr < (MCode *)((char *)mc + J->szmcarea)) {
338#if LUAJIT_SECURITY_MCODE
349 mcode_protect(J, MCPROT_GEN); 339 mcode_protect(J, MCPROT_GEN);
340#endif
350 return mc; 341 return mc;
351 } 342 }
352 /* Otherwise search through the list of MCode areas. */ 343 /* Otherwise search through the list of MCode areas. */
353 for (;;) { 344 for (;;) {
354 mc = ((MCLink *)mc)->next; 345 mc = ((MCLink *)mc)->next;
355 lua_assert(mc != NULL); 346 lj_assertJ(mc != NULL, "broken MCode area chain");
356 if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) { 347 if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) {
348#if LUAJIT_SECURITY_MCODE
357 if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN))) 349 if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN)))
358 mcode_protfail(J); 350 mcode_protfail(J);
351#endif
359 return mc; 352 return mc;
360 } 353 }
361 } 354 }
362 } 355 }
363#endif
364} 356}
365 357
366/* Limit of MCode reservation reached. */ 358/* Limit of MCode reservation reached. */
diff --git a/src/lj_meta.c b/src/lj_meta.c
index 15c13aa2..5940097e 100644
--- a/src/lj_meta.c
+++ b/src/lj_meta.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#include "lj_tab.h" 17#include "lj_tab.h"
17#include "lj_meta.h" 18#include "lj_meta.h"
@@ -19,6 +20,8 @@
19#include "lj_bc.h" 20#include "lj_bc.h"
20#include "lj_vm.h" 21#include "lj_vm.h"
21#include "lj_strscan.h" 22#include "lj_strscan.h"
23#include "lj_strfmt.h"
24#include "lj_lib.h"
22 25
23/* -- Metamethod handling ------------------------------------------------- */ 26/* -- Metamethod handling ------------------------------------------------- */
24 27
@@ -44,7 +47,7 @@ void lj_meta_init(lua_State *L)
44cTValue *lj_meta_cache(GCtab *mt, MMS mm, GCstr *name) 47cTValue *lj_meta_cache(GCtab *mt, MMS mm, GCstr *name)
45{ 48{
46 cTValue *mo = lj_tab_getstr(mt, name); 49 cTValue *mo = lj_tab_getstr(mt, name);
47 lua_assert(mm <= MM_FAST); 50 lj_assertX(mm <= MM_FAST, "bad metamethod %d", mm);
48 if (!mo || tvisnil(mo)) { /* No metamethod? */ 51 if (!mo || tvisnil(mo)) { /* No metamethod? */
49 mt->nomm |= (uint8_t)(1u<<mm); /* Set negative cache flag. */ 52 mt->nomm |= (uint8_t)(1u<<mm); /* Set negative cache flag. */
50 return NULL; 53 return NULL;
@@ -77,12 +80,16 @@ int lj_meta_tailcall(lua_State *L, cTValue *tv)
77 TValue *base = L->base; 80 TValue *base = L->base;
78 TValue *top = L->top; 81 TValue *top = L->top;
79 const BCIns *pc = frame_pc(base-1); /* Preserve old PC from frame. */ 82 const BCIns *pc = frame_pc(base-1); /* Preserve old PC from frame. */
80 copyTV(L, base-1, tv); /* Replace frame with new object. */ 83 copyTV(L, base-1-LJ_FR2, tv); /* Replace frame with new object. */
81 top->u32.lo = LJ_CONT_TAILCALL; 84 if (LJ_FR2)
82 setframe_pc(top, pc); 85 (top++)->u64 = LJ_CONT_TAILCALL;
83 setframe_gc(top+1, obj2gco(L)); /* Dummy frame object. */ 86 else
84 setframe_ftsz(top+1, (int)((char *)(top+2) - (char *)base) + FRAME_CONT); 87 top->u32.lo = LJ_CONT_TAILCALL;
85 L->base = L->top = top+2; 88 setframe_pc(top++, pc);
89 setframe_gc(top, obj2gco(L), LJ_TTHREAD); /* Dummy frame object. */
90 if (LJ_FR2) top++;
91 setframe_ftsz(top, ((char *)(top+1) - (char *)base) + FRAME_CONT);
92 L->base = L->top = top+1;
86 /* 93 /*
87 ** before: [old_mo|PC] [... ...] 94 ** before: [old_mo|PC] [... ...]
88 ** ^base ^top 95 ** ^base ^top
@@ -113,11 +120,13 @@ static TValue *mmcall(lua_State *L, ASMFunction cont, cTValue *mo,
113 */ 120 */
114 TValue *top = L->top; 121 TValue *top = L->top;
115 if (curr_funcisL(L)) top = curr_topL(L); 122 if (curr_funcisL(L)) top = curr_topL(L);
116 setcont(top, cont); /* Assembler VM stores PC in upper word. */ 123 setcont(top++, cont); /* Assembler VM stores PC in upper word or FR2. */
117 copyTV(L, top+1, mo); /* Store metamethod and two arguments. */ 124 if (LJ_FR2) setnilV(top++);
118 copyTV(L, top+2, a); 125 copyTV(L, top++, mo); /* Store metamethod and two arguments. */
119 copyTV(L, top+3, b); 126 if (LJ_FR2) setnilV(top++);
120 return top+2; /* Return new base. */ 127 copyTV(L, top, a);
128 copyTV(L, top+1, b);
129 return top; /* Return new base. */
121} 130}
122 131
123/* -- C helpers for some instructions, called from assembler VM ----------- */ 132/* -- C helpers for some instructions, called from assembler VM ----------- */
@@ -225,27 +234,14 @@ TValue *lj_meta_arith(lua_State *L, TValue *ra, cTValue *rb, cTValue *rc,
225 } 234 }
226} 235}
227 236
228/* In-place coercion of a number to a string. */
229static LJ_AINLINE int tostring(lua_State *L, TValue *o)
230{
231 if (tvisstr(o)) {
232 return 1;
233 } else if (tvisnumber(o)) {
234 setstrV(L, o, lj_str_fromnumber(L, o));
235 return 1;
236 } else {
237 return 0;
238 }
239}
240
241/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */ 237/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */
242TValue *lj_meta_cat(lua_State *L, TValue *top, int left) 238TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
243{ 239{
244 int fromc = 0; 240 int fromc = 0;
245 if (left < 0) { left = -left; fromc = 1; } 241 if (left < 0) { left = -left; fromc = 1; }
246 do { 242 do {
247 int n = 1; 243 if (!(tvisstr(top) || tvisnumber(top) || tvisbuf(top)) ||
248 if (!(tvisstr(top-1) || tvisnumber(top-1)) || !tostring(L, top)) { 244 !(tvisstr(top-1) || tvisnumber(top-1) || tvisbuf(top-1))) {
249 cTValue *mo = lj_meta_lookup(L, top-1, MM_concat); 245 cTValue *mo = lj_meta_lookup(L, top-1, MM_concat);
250 if (tvisnil(mo)) { 246 if (tvisnil(mo)) {
251 mo = lj_meta_lookup(L, top, MM_concat); 247 mo = lj_meta_lookup(L, top, MM_concat);
@@ -266,13 +262,12 @@ TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
266 ** after mm: [...][CAT stack ...] <--push-- [result] 262 ** after mm: [...][CAT stack ...] <--push-- [result]
267 ** next step: [...][CAT stack .............] 263 ** next step: [...][CAT stack .............]
268 */ 264 */
269 copyTV(L, top+2, top); /* Careful with the order of stack copies! */ 265 copyTV(L, top+2*LJ_FR2+2, top); /* Carefully ordered stack copies! */
270 copyTV(L, top+1, top-1); 266 copyTV(L, top+2*LJ_FR2+1, top-1);
271 copyTV(L, top, mo); 267 copyTV(L, top+LJ_FR2, mo);
272 setcont(top-1, lj_cont_cat); 268 setcont(top-1, lj_cont_cat);
269 if (LJ_FR2) { setnilV(top); setnilV(top+2); top += 2; }
273 return top+1; /* Trigger metamethod call. */ 270 return top+1; /* Trigger metamethod call. */
274 } else if (strV(top)->len == 0) { /* Shortcut. */
275 (void)tostring(L, top-1);
276 } else { 271 } else {
277 /* Pick as many strings as possible from the top and concatenate them: 272 /* Pick as many strings as possible from the top and concatenate them:
278 ** 273 **
@@ -281,27 +276,33 @@ TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
281 ** concat: [...][CAT stack ...] [result] 276 ** concat: [...][CAT stack ...] [result]
282 ** next step: [...][CAT stack ............] 277 ** next step: [...][CAT stack ............]
283 */ 278 */
284 MSize tlen = strV(top)->len; 279 TValue *e, *o = top;
285 char *buffer; 280 uint64_t tlen = tvisstr(o) ? strV(o)->len :
286 int i; 281 tvisbuf(o) ? sbufxlen(bufV(o)) : STRFMT_MAXBUF_NUM;
287 for (n = 1; n <= left && tostring(L, top-n); n++) { 282 SBuf *sb;
288 MSize len = strV(top-n)->len; 283 do {
289 if (len >= LJ_MAX_STR - tlen) 284 o--; tlen += tvisstr(o) ? strV(o)->len :
290 lj_err_msg(L, LJ_ERR_STROV); 285 tvisbuf(o) ? sbufxlen(bufV(o)) : STRFMT_MAXBUF_NUM;
291 tlen += len; 286 } while (--left > 0 && (tvisstr(o-1) || tvisnumber(o-1)));
292 } 287 if (tlen >= LJ_MAX_STR) lj_err_msg(L, LJ_ERR_STROV);
293 buffer = lj_str_needbuf(L, &G(L)->tmpbuf, tlen); 288 sb = lj_buf_tmp_(L);
294 n--; 289 lj_buf_more(sb, (MSize)tlen);
295 tlen = 0; 290 for (e = top, top = o; o <= e; o++) {
296 for (i = n; i >= 0; i--) { 291 if (tvisstr(o)) {
297 MSize len = strV(top-i)->len; 292 GCstr *s = strV(o);
298 memcpy(buffer + tlen, strVdata(top-i), len); 293 MSize len = s->len;
299 tlen += len; 294 lj_buf_putmem(sb, strdata(s), len);
295 } else if (tvisbuf(o)) {
296 SBufExt *sbx = bufV(o);
297 lj_buf_putmem(sb, sbx->r, sbufxlen(sbx));
298 } else if (tvisint(o)) {
299 lj_strfmt_putint(sb, intV(o));
300 } else {
301 lj_strfmt_putfnum(sb, STRFMT_G14, numV(o));
302 }
300 } 303 }
301 setstrV(L, top-n, lj_str_new(L, buffer, tlen)); 304 setstrV(L, top, lj_buf_str(L, sb));
302 } 305 }
303 left -= n;
304 top -= n;
305 } while (left >= 1); 306 } while (left >= 1);
306 if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) { 307 if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) {
307 if (!fromc) L->top = curr_topL(L); 308 if (!fromc) L->top = curr_topL(L);
@@ -338,12 +339,14 @@ TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne)
338 return (TValue *)(intptr_t)ne; 339 return (TValue *)(intptr_t)ne;
339 } 340 }
340 top = curr_top(L); 341 top = curr_top(L);
341 setcont(top, ne ? lj_cont_condf : lj_cont_condt); 342 setcont(top++, ne ? lj_cont_condf : lj_cont_condt);
342 copyTV(L, top+1, mo); 343 if (LJ_FR2) setnilV(top++);
344 copyTV(L, top++, mo);
345 if (LJ_FR2) setnilV(top++);
343 it = ~(uint32_t)o1->gch.gct; 346 it = ~(uint32_t)o1->gch.gct;
344 setgcV(L, top+2, o1, it); 347 setgcV(L, top, o1, it);
345 setgcV(L, top+3, o2, it); 348 setgcV(L, top+1, o2, it);
346 return top+2; /* Trigger metamethod call. */ 349 return top; /* Trigger metamethod call. */
347 } 350 }
348 return (TValue *)(intptr_t)ne; 351 return (TValue *)(intptr_t)ne;
349} 352}
@@ -365,8 +368,8 @@ TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins)
365 } else if (op == BC_ISEQN) { 368 } else if (op == BC_ISEQN) {
366 o2 = &mref(curr_proto(L)->k, cTValue)[bc_d(ins)]; 369 o2 = &mref(curr_proto(L)->k, cTValue)[bc_d(ins)];
367 } else { 370 } else {
368 lua_assert(op == BC_ISEQP); 371 lj_assertL(op == BC_ISEQP, "bad bytecode op %d", op);
369 setitype(&tv, ~bc_d(ins)); 372 setpriV(&tv, ~bc_d(ins));
370 o2 = &tv; 373 o2 = &tv;
371 } 374 }
372 mo = lj_meta_lookup(L, o1mm, MM_eq); 375 mo = lj_meta_lookup(L, o1mm, MM_eq);
@@ -423,6 +426,18 @@ TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
423 } 426 }
424} 427}
425 428
429/* Helper for ISTYPE and ISNUM. Implicit coercion or error. */
430void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp)
431{
432 L->top = curr_topL(L);
433 ra++; tp--;
434 lj_assertL(LJ_DUALNUM || tp != ~LJ_TNUMX, "bad type for ISTYPE");
435 if (LJ_DUALNUM && tp == ~LJ_TNUMX) lj_lib_checkint(L, ra);
436 else if (tp == ~LJ_TNUMX+1) lj_lib_checknum(L, ra);
437 else if (tp == ~LJ_TSTR) lj_lib_checkstr(L, ra);
438 else lj_err_argtype(L, ra, lj_obj_itypename[tp]);
439}
440
426/* Helper for calls. __call metamethod. */ 441/* Helper for calls. __call metamethod. */
427void lj_meta_call(lua_State *L, TValue *func, TValue *top) 442void lj_meta_call(lua_State *L, TValue *func, TValue *top)
428{ 443{
@@ -430,7 +445,8 @@ void lj_meta_call(lua_State *L, TValue *func, TValue *top)
430 TValue *p; 445 TValue *p;
431 if (!tvisfunc(mo)) 446 if (!tvisfunc(mo))
432 lj_err_optype_call(L, func); 447 lj_err_optype_call(L, func);
433 for (p = top; p > func; p--) copyTV(L, p, p-1); 448 for (p = top; p > func+2*LJ_FR2; p--) copyTV(L, p, p-1);
449 if (LJ_FR2) copyTV(L, func+2, func);
434 copyTV(L, func, mo); 450 copyTV(L, func, mo);
435} 451}
436 452
diff --git a/src/lj_meta.h b/src/lj_meta.h
index 53ec9c0a..f067cf59 100644
--- a/src/lj_meta.h
+++ b/src/lj_meta.h
@@ -31,6 +31,7 @@ LJ_FUNCA TValue * LJ_FASTCALL lj_meta_len(lua_State *L, cTValue *o);
31LJ_FUNCA TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne); 31LJ_FUNCA TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne);
32LJ_FUNCA TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins); 32LJ_FUNCA TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins);
33LJ_FUNCA TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op); 33LJ_FUNCA TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op);
34LJ_FUNCA void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp);
34LJ_FUNCA void lj_meta_call(lua_State *L, TValue *func, TValue *top); 35LJ_FUNCA void lj_meta_call(lua_State *L, TValue *func, TValue *top);
35LJ_FUNCA void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o); 36LJ_FUNCA void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o);
36 37
diff --git a/src/lj_obj.c b/src/lj_obj.c
index ce837134..cd4ca50a 100644
--- a/src/lj_obj.c
+++ b/src/lj_obj.c
@@ -20,7 +20,7 @@ LJ_DATADEF const char *const lj_obj_itypename[] = { /* ORDER LJ_T */
20}; 20};
21 21
22/* Compare two objects without calling metamethods. */ 22/* Compare two objects without calling metamethods. */
23int lj_obj_equal(cTValue *o1, cTValue *o2) 23int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2)
24{ 24{
25 if (itype(o1) == itype(o2)) { 25 if (itype(o1) == itype(o2)) {
26 if (tvispri(o1)) 26 if (tvispri(o1))
@@ -33,3 +33,19 @@ int lj_obj_equal(cTValue *o1, cTValue *o2)
33 return numberVnum(o1) == numberVnum(o2); 33 return numberVnum(o1) == numberVnum(o2);
34} 34}
35 35
36/* Return pointer to object or its object data. */
37const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o)
38{
39 UNUSED(g);
40 if (tvisudata(o))
41 return uddata(udataV(o));
42 else if (tvislightud(o))
43 return lightudV(g, o);
44 else if (LJ_HASFFI && tviscdata(o))
45 return cdataptr(cdataV(o));
46 else if (tvisgcv(o))
47 return gcV(o);
48 else
49 return NULL;
50}
51
diff --git a/src/lj_obj.h b/src/lj_obj.h
index 6cb93f89..2d4386e1 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -13,44 +13,81 @@
13#include "lj_def.h" 13#include "lj_def.h"
14#include "lj_arch.h" 14#include "lj_arch.h"
15 15
16/* -- Memory references (32 bit address space) ---------------------------- */ 16/* -- Memory references --------------------------------------------------- */
17 17
18/* Memory size. */ 18/* Memory and GC object sizes. */
19typedef uint32_t MSize; 19typedef uint32_t MSize;
20#if LJ_GC64
21typedef uint64_t GCSize;
22#else
23typedef uint32_t GCSize;
24#endif
20 25
21/* Memory reference */ 26/* Memory reference */
22typedef struct MRef { 27typedef struct MRef {
28#if LJ_GC64
29 uint64_t ptr64; /* True 64 bit pointer. */
30#else
23 uint32_t ptr32; /* Pseudo 32 bit pointer. */ 31 uint32_t ptr32; /* Pseudo 32 bit pointer. */
32#endif
24} MRef; 33} MRef;
25 34
35#if LJ_GC64
36#define mref(r, t) ((t *)(void *)(r).ptr64)
37#define mrefu(r) ((r).ptr64)
38
39#define setmref(r, p) ((r).ptr64 = (uint64_t)(void *)(p))
40#define setmrefu(r, u) ((r).ptr64 = (uint64_t)(u))
41#define setmrefr(r, v) ((r).ptr64 = (v).ptr64)
42#else
26#define mref(r, t) ((t *)(void *)(uintptr_t)(r).ptr32) 43#define mref(r, t) ((t *)(void *)(uintptr_t)(r).ptr32)
44#define mrefu(r) ((r).ptr32)
27 45
28#define setmref(r, p) ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p)) 46#define setmref(r, p) ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
47#define setmrefu(r, u) ((r).ptr32 = (uint32_t)(u))
29#define setmrefr(r, v) ((r).ptr32 = (v).ptr32) 48#define setmrefr(r, v) ((r).ptr32 = (v).ptr32)
49#endif
30 50
31/* -- GC object references (32 bit address space) ------------------------- */ 51/* -- GC object references ------------------------------------------------ */
32 52
33/* GCobj reference */ 53/* GCobj reference */
34typedef struct GCRef { 54typedef struct GCRef {
55#if LJ_GC64
56 uint64_t gcptr64; /* True 64 bit pointer. */
57#else
35 uint32_t gcptr32; /* Pseudo 32 bit pointer. */ 58 uint32_t gcptr32; /* Pseudo 32 bit pointer. */
59#endif
36} GCRef; 60} GCRef;
37 61
38/* Common GC header for all collectable objects. */ 62/* Common GC header for all collectable objects. */
39#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct 63#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct
40/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */ 64/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
41 65
66#if LJ_GC64
67#define gcref(r) ((GCobj *)(r).gcptr64)
68#define gcrefp(r, t) ((t *)(void *)(r).gcptr64)
69#define gcrefu(r) ((r).gcptr64)
70#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
71
72#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
73#define setgcreft(r, gc, it) \
74 (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
75#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
76#define setgcrefnull(r) ((r).gcptr64 = 0)
77#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
78#else
42#define gcref(r) ((GCobj *)(uintptr_t)(r).gcptr32) 79#define gcref(r) ((GCobj *)(uintptr_t)(r).gcptr32)
43#define gcrefp(r, t) ((t *)(void *)(uintptr_t)(r).gcptr32) 80#define gcrefp(r, t) ((t *)(void *)(uintptr_t)(r).gcptr32)
44#define gcrefu(r) ((r).gcptr32) 81#define gcrefu(r) ((r).gcptr32)
45#define gcrefi(r) ((int32_t)(r).gcptr32)
46#define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32) 82#define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
47#define gcnext(gc) (gcref((gc)->gch.nextgc))
48 83
49#define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch) 84#define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
50#define setgcrefi(r, i) ((r).gcptr32 = (uint32_t)(i))
51#define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p)) 85#define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
52#define setgcrefnull(r) ((r).gcptr32 = 0) 86#define setgcrefnull(r) ((r).gcptr32 = 0)
53#define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32) 87#define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
88#endif
89
90#define gcnext(gc) (gcref((gc)->gch.nextgc))
54 91
55/* IMPORTANT NOTE: 92/* IMPORTANT NOTE:
56** 93**
@@ -119,11 +156,10 @@ typedef int32_t BCLine; /* Bytecode line number. */
119/* Internal assembler functions. Never call these directly from C. */ 156/* Internal assembler functions. Never call these directly from C. */
120typedef void (*ASMFunction)(void); 157typedef void (*ASMFunction)(void);
121 158
122/* Resizable string buffer. Need this here, details in lj_str.h. */ 159/* Resizable string buffer. Need this here, details in lj_buf.h. */
160#define SBufHeader char *w, *e, *b; MRef L
123typedef struct SBuf { 161typedef struct SBuf {
124 char *buf; /* String buffer base. */ 162 SBufHeader;
125 MSize n; /* String buffer length. */
126 MSize sz; /* String buffer size. */
127} SBuf; 163} SBuf;
128 164
129/* -- Tags and values ----------------------------------------------------- */ 165/* -- Tags and values ----------------------------------------------------- */
@@ -131,13 +167,23 @@ typedef struct SBuf {
131/* Frame link. */ 167/* Frame link. */
132typedef union { 168typedef union {
133 int32_t ftsz; /* Frame type and size of previous frame. */ 169 int32_t ftsz; /* Frame type and size of previous frame. */
134 MRef pcr; /* Overlaps PC for Lua frames. */ 170 MRef pcr; /* Or PC for Lua frames. */
135} FrameLink; 171} FrameLink;
136 172
137/* Tagged value. */ 173/* Tagged value. */
138typedef LJ_ALIGN(8) union TValue { 174typedef LJ_ALIGN(8) union TValue {
139 uint64_t u64; /* 64 bit pattern overlaps number. */ 175 uint64_t u64; /* 64 bit pattern overlaps number. */
140 lua_Number n; /* Number object overlaps split tag/value object. */ 176 lua_Number n; /* Number object overlaps split tag/value object. */
177#if LJ_GC64
178 GCRef gcr; /* GCobj reference with tag. */
179 int64_t it64;
180 struct {
181 LJ_ENDIAN_LOHI(
182 int32_t i; /* Integer value. */
183 , uint32_t it; /* Internal object tag. Must overlap MSW of number. */
184 )
185 };
186#else
141 struct { 187 struct {
142 LJ_ENDIAN_LOHI( 188 LJ_ENDIAN_LOHI(
143 union { 189 union {
@@ -147,12 +193,17 @@ typedef LJ_ALIGN(8) union TValue {
147 , uint32_t it; /* Internal object tag. Must overlap MSW of number. */ 193 , uint32_t it; /* Internal object tag. Must overlap MSW of number. */
148 ) 194 )
149 }; 195 };
196#endif
197#if LJ_FR2
198 int64_t ftsz; /* Frame type and size of previous frame, or PC. */
199#else
150 struct { 200 struct {
151 LJ_ENDIAN_LOHI( 201 LJ_ENDIAN_LOHI(
152 GCRef func; /* Function for next frame (or dummy L). */ 202 GCRef func; /* Function for next frame (or dummy L). */
153 , FrameLink tp; /* Link to previous frame. */ 203 , FrameLink tp; /* Link to previous frame. */
154 ) 204 )
155 } fr; 205 } fr;
206#endif
156 struct { 207 struct {
157 LJ_ENDIAN_LOHI( 208 LJ_ENDIAN_LOHI(
158 uint32_t lo; /* Lower 32 bits of number. */ 209 uint32_t lo; /* Lower 32 bits of number. */
@@ -172,6 +223,8 @@ typedef const TValue cTValue;
172 223
173/* Internal object tags. 224/* Internal object tags.
174** 225**
226** Format for 32 bit GC references (!LJ_GC64):
227**
175** Internal tags overlap the MSW of a number object (must be a double). 228** Internal tags overlap the MSW of a number object (must be a double).
176** Interpreted as a double these are special NaNs. The FPU only generates 229** Interpreted as a double these are special NaNs. The FPU only generates
177** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available 230** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
@@ -181,11 +234,24 @@ typedef const TValue cTValue;
181** ---MSW---.---LSW--- 234** ---MSW---.---LSW---
182** primitive types | itype | | 235** primitive types | itype | |
183** lightuserdata | itype | void * | (32 bit platforms) 236** lightuserdata | itype | void * | (32 bit platforms)
184** lightuserdata |ffff| void * | (64 bit platforms, 47 bit pointers) 237** lightuserdata |ffff|seg| ofs | (64 bit platforms)
185** GC objects | itype | GCRef | 238** GC objects | itype | GCRef |
186** int (LJ_DUALNUM)| itype | int | 239** int (LJ_DUALNUM)| itype | int |
187** number -------double------ 240** number -------double------
188** 241**
242** Format for 64 bit GC references (LJ_GC64):
243**
244** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
245** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
246** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
247**
248** ------MSW------.------LSW------
249** primitive types |1..1|itype|1..................1|
250** GC objects |1..1|itype|-------GCRef--------|
251** lightuserdata |1..1|itype|seg|------ofs-------|
252** int (LJ_DUALNUM) |1..1|itype|0..0|-----int-------|
253** number ------------double-------------
254**
189** ORDER LJ_T 255** ORDER LJ_T
190** Primitive types nil/false/true must be first, lightuserdata next. 256** Primitive types nil/false/true must be first, lightuserdata next.
191** GC objects are at the end, table/userdata must be lowest. 257** GC objects are at the end, table/userdata must be lowest.
@@ -208,7 +274,7 @@ typedef const TValue cTValue;
208#define LJ_TNUMX (~13u) 274#define LJ_TNUMX (~13u)
209 275
210/* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */ 276/* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
211#if LJ_64 277#if LJ_64 && !LJ_GC64
212#define LJ_TISNUM 0xfffeffffu 278#define LJ_TISNUM 0xfffeffffu
213#else 279#else
214#define LJ_TISNUM LJ_TNUMX 280#define LJ_TISNUM LJ_TNUMX
@@ -218,14 +284,31 @@ typedef const TValue cTValue;
218#define LJ_TISGCV (LJ_TSTR+1) 284#define LJ_TISGCV (LJ_TSTR+1)
219#define LJ_TISTABUD LJ_TTAB 285#define LJ_TISTABUD LJ_TTAB
220 286
287/* Type marker for slot holding a traversal index. Must be lightuserdata. */
288#define LJ_KEYINDEX 0xfffe7fffu
289
290#if LJ_GC64
291#define LJ_GCVMASK (((uint64_t)1 << 47) - 1)
292#endif
293
294#if LJ_64
295/* To stay within 47 bits, lightuserdata is segmented. */
296#define LJ_LIGHTUD_BITS_SEG 8
297#define LJ_LIGHTUD_BITS_LO (47 - LJ_LIGHTUD_BITS_SEG)
298#endif
299
221/* -- String object ------------------------------------------------------- */ 300/* -- String object ------------------------------------------------------- */
222 301
302typedef uint32_t StrHash; /* String hash value. */
303typedef uint32_t StrID; /* String ID. */
304
223/* String object header. String payload follows. */ 305/* String object header. String payload follows. */
224typedef struct GCstr { 306typedef struct GCstr {
225 GCHeader; 307 GCHeader;
226 uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */ 308 uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */
227 uint8_t unused; 309 uint8_t hashalg; /* Hash algorithm. */
228 MSize hash; /* Hash of string. */ 310 StrID sid; /* Interned string ID. */
311 StrHash hash; /* Hash of string. */
229 MSize len; /* Size of string. */ 312 MSize len; /* Size of string. */
230} GCstr; 313} GCstr;
231 314
@@ -233,7 +316,6 @@ typedef struct GCstr {
233#define strdata(s) ((const char *)((s)+1)) 316#define strdata(s) ((const char *)((s)+1))
234#define strdatawr(s) ((char *)((s)+1)) 317#define strdatawr(s) ((char *)((s)+1))
235#define strVdata(o) strdata(strV(o)) 318#define strVdata(o) strdata(strV(o))
236#define sizestring(s) (sizeof(struct GCstr)+(s)->len+1)
237 319
238/* -- Userdata object ----------------------------------------------------- */ 320/* -- Userdata object ----------------------------------------------------- */
239 321
@@ -253,6 +335,7 @@ enum {
253 UDTYPE_USERDATA, /* Regular userdata. */ 335 UDTYPE_USERDATA, /* Regular userdata. */
254 UDTYPE_IO_FILE, /* I/O library FILE. */ 336 UDTYPE_IO_FILE, /* I/O library FILE. */
255 UDTYPE_FFI_CLIB, /* FFI C library namespace. */ 337 UDTYPE_FFI_CLIB, /* FFI C library namespace. */
338 UDTYPE_BUFFER, /* String buffer. */
256 UDTYPE__MAX 339 UDTYPE__MAX
257}; 340};
258 341
@@ -291,6 +374,9 @@ typedef struct GCproto {
291 uint8_t numparams; /* Number of parameters. */ 374 uint8_t numparams; /* Number of parameters. */
292 uint8_t framesize; /* Fixed frame size. */ 375 uint8_t framesize; /* Fixed frame size. */
293 MSize sizebc; /* Number of bytecode instructions. */ 376 MSize sizebc; /* Number of bytecode instructions. */
377#if LJ_GC64
378 uint32_t unused_gc64;
379#endif
294 GCRef gclist; 380 GCRef gclist;
295 MRef k; /* Split constant array (points to the middle). */ 381 MRef k; /* Split constant array (points to the middle). */
296 MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */ 382 MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */
@@ -402,7 +488,9 @@ typedef struct Node {
402 TValue val; /* Value object. Must be first field. */ 488 TValue val; /* Value object. Must be first field. */
403 TValue key; /* Key object. */ 489 TValue key; /* Key object. */
404 MRef next; /* Hash chain. */ 490 MRef next; /* Hash chain. */
491#if !LJ_GC64
405 MRef freetop; /* Top of free elements (stored in t->node[0]). */ 492 MRef freetop; /* Top of free elements (stored in t->node[0]). */
493#endif
406} Node; 494} Node;
407 495
408LJ_STATIC_ASSERT(offsetof(Node, val) == 0); 496LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
@@ -417,12 +505,22 @@ typedef struct GCtab {
417 MRef node; /* Hash part. */ 505 MRef node; /* Hash part. */
418 uint32_t asize; /* Size of array part (keys [0, asize-1]). */ 506 uint32_t asize; /* Size of array part (keys [0, asize-1]). */
419 uint32_t hmask; /* Hash part mask (size of hash part - 1). */ 507 uint32_t hmask; /* Hash part mask (size of hash part - 1). */
508#if LJ_GC64
509 MRef freetop; /* Top of free elements. */
510#endif
420} GCtab; 511} GCtab;
421 512
422#define sizetabcolo(n) ((n)*sizeof(TValue) + sizeof(GCtab)) 513#define sizetabcolo(n) ((n)*sizeof(TValue) + sizeof(GCtab))
423#define tabref(r) ((GCtab *)gcref((r))) 514#define tabref(r) ((GCtab *)gcref((r)))
424#define noderef(r) (mref((r), Node)) 515#define noderef(r) (mref((r), Node))
425#define nextnode(n) (mref((n)->next, Node)) 516#define nextnode(n) (mref((n)->next, Node))
517#if LJ_GC64
518#define getfreetop(t, n) (noderef((t)->freetop))
519#define setfreetop(t, n, v) (setmref((t)->freetop, (v)))
520#else
521#define getfreetop(t, n) (noderef((n)->freetop))
522#define setfreetop(t, n, v) (setmref((n)->freetop, (v)))
523#endif
426 524
427/* -- State objects ------------------------------------------------------- */ 525/* -- State objects ------------------------------------------------------- */
428 526
@@ -488,13 +586,18 @@ typedef enum {
488#define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)]) 586#define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
489#define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)])) 587#define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
490 588
589/* Garbage collector state. */
491typedef struct GCState { 590typedef struct GCState {
492 MSize total; /* Memory currently allocated. */ 591 GCSize total; /* Memory currently allocated. */
493 MSize threshold; /* Memory threshold. */ 592 GCSize threshold; /* Memory threshold. */
494 uint8_t currentwhite; /* Current white color. */ 593 uint8_t currentwhite; /* Current white color. */
495 uint8_t state; /* GC state. */ 594 uint8_t state; /* GC state. */
496 uint8_t nocdatafin; /* No cdata finalizer called. */ 595 uint8_t nocdatafin; /* No cdata finalizer called. */
497 uint8_t unused2; 596#if LJ_64
597 uint8_t lightudnum; /* Number of lightuserdata segments - 1. */
598#else
599 uint8_t unused1;
600#endif
498 MSize sweepstr; /* Sweep position in string table. */ 601 MSize sweepstr; /* Sweep position in string table. */
499 GCRef root; /* List of all collectable objects. */ 602 GCRef root; /* List of all collectable objects. */
500 MRef sweep; /* Sweep position in root list. */ 603 MRef sweep; /* Sweep position in root list. */
@@ -502,42 +605,57 @@ typedef struct GCState {
502 GCRef grayagain; /* List of objects for atomic traversal. */ 605 GCRef grayagain; /* List of objects for atomic traversal. */
503 GCRef weak; /* List of weak tables (to be cleared). */ 606 GCRef weak; /* List of weak tables (to be cleared). */
504 GCRef mmudata; /* List of userdata (to be finalized). */ 607 GCRef mmudata; /* List of userdata (to be finalized). */
608 GCSize debt; /* Debt (how much GC is behind schedule). */
609 GCSize estimate; /* Estimate of memory actually in use. */
505 MSize stepmul; /* Incremental GC step granularity. */ 610 MSize stepmul; /* Incremental GC step granularity. */
506 MSize debt; /* Debt (how much GC is behind schedule). */
507 MSize estimate; /* Estimate of memory actually in use. */
508 MSize pause; /* Pause between successive GC cycles. */ 611 MSize pause; /* Pause between successive GC cycles. */
612#if LJ_64
613 MRef lightudseg; /* Upper bits of lightuserdata segments. */
614#endif
509} GCState; 615} GCState;
510 616
617/* String interning state. */
618typedef struct StrInternState {
619 GCRef *tab; /* String hash table anchors. */
620 MSize mask; /* String hash mask (size of hash table - 1). */
621 MSize num; /* Number of strings in hash table. */
622 StrID id; /* Next string ID. */
623 uint8_t idreseed; /* String ID reseed counter. */
624 uint8_t second; /* String interning table uses secondary hashing. */
625 uint8_t unused1;
626 uint8_t unused2;
627 LJ_ALIGN(8) uint64_t seed; /* Random string seed. */
628} StrInternState;
629
511/* Global state, shared by all threads of a Lua universe. */ 630/* Global state, shared by all threads of a Lua universe. */
512typedef struct global_State { 631typedef struct global_State {
513 GCRef *strhash; /* String hash table (hash chain anchors). */
514 MSize strmask; /* String hash mask (size of hash table - 1). */
515 MSize strnum; /* Number of strings in hash table. */
516 lua_Alloc allocf; /* Memory allocator. */ 632 lua_Alloc allocf; /* Memory allocator. */
517 void *allocd; /* Memory allocator data. */ 633 void *allocd; /* Memory allocator data. */
518 GCState gc; /* Garbage collector. */ 634 GCState gc; /* Garbage collector. */
519 SBuf tmpbuf; /* Temporary buffer for string concatenation. */
520 Node nilnode; /* Fallback 1-element hash part (nil key and value). */
521 GCstr strempty; /* Empty string. */ 635 GCstr strempty; /* Empty string. */
522 uint8_t stremptyz; /* Zero terminator of empty string. */ 636 uint8_t stremptyz; /* Zero terminator of empty string. */
523 uint8_t hookmask; /* Hook mask. */ 637 uint8_t hookmask; /* Hook mask. */
524 uint8_t dispatchmode; /* Dispatch mode. */ 638 uint8_t dispatchmode; /* Dispatch mode. */
525 uint8_t vmevmask; /* VM event mask. */ 639 uint8_t vmevmask; /* VM event mask. */
640 StrInternState str; /* String interning. */
641 volatile int32_t vmstate; /* VM state or current JIT code trace number. */
526 GCRef mainthref; /* Link to main thread. */ 642 GCRef mainthref; /* Link to main thread. */
527 TValue registrytv; /* Anchor for registry. */ 643 SBuf tmpbuf; /* Temporary string buffer. */
528 TValue tmptv, tmptv2; /* Temporary TValues. */ 644 TValue tmptv, tmptv2; /* Temporary TValues. */
645 Node nilnode; /* Fallback 1-element hash part (nil key and value). */
646 TValue registrytv; /* Anchor for registry. */
529 GCupval uvhead; /* Head of double-linked list of all open upvalues. */ 647 GCupval uvhead; /* Head of double-linked list of all open upvalues. */
530 int32_t hookcount; /* Instruction hook countdown. */ 648 int32_t hookcount; /* Instruction hook countdown. */
531 int32_t hookcstart; /* Start count for instruction hook counter. */ 649 int32_t hookcstart; /* Start count for instruction hook counter. */
532 lua_Hook hookf; /* Hook function. */ 650 lua_Hook hookf; /* Hook function. */
533 lua_CFunction wrapf; /* Wrapper for C function calls. */ 651 lua_CFunction wrapf; /* Wrapper for C function calls. */
534 lua_CFunction panic; /* Called as a last resort for errors. */ 652 lua_CFunction panic; /* Called as a last resort for errors. */
535 volatile int32_t vmstate; /* VM state or current JIT code trace number. */
536 BCIns bc_cfunc_int; /* Bytecode for internal C function calls. */ 653 BCIns bc_cfunc_int; /* Bytecode for internal C function calls. */
537 BCIns bc_cfunc_ext; /* Bytecode for external C function calls. */ 654 BCIns bc_cfunc_ext; /* Bytecode for external C function calls. */
538 GCRef jit_L; /* Current JIT code lua_State or NULL. */ 655 GCRef cur_L; /* Currently executing lua_State. */
539 MRef jit_base; /* Current JIT code L->base. */ 656 MRef jit_base; /* Current JIT code L->base or NULL. */
540 MRef ctype_state; /* Pointer to C type state. */ 657 MRef ctype_state; /* Pointer to C type state. */
658 PRNGState prng; /* Global PRNG state. */
541 GCRef gcroot[GCROOT_MAX]; /* GC roots. */ 659 GCRef gcroot[GCROOT_MAX]; /* GC roots. */
542} global_State; 660} global_State;
543 661
@@ -553,9 +671,11 @@ typedef struct global_State {
553#define HOOK_ACTIVE_SHIFT 4 671#define HOOK_ACTIVE_SHIFT 4
554#define HOOK_VMEVENT 0x20 672#define HOOK_VMEVENT 0x20
555#define HOOK_GC 0x40 673#define HOOK_GC 0x40
674#define HOOK_PROFILE 0x80
556#define hook_active(g) ((g)->hookmask & HOOK_ACTIVE) 675#define hook_active(g) ((g)->hookmask & HOOK_ACTIVE)
557#define hook_enter(g) ((g)->hookmask |= HOOK_ACTIVE) 676#define hook_enter(g) ((g)->hookmask |= HOOK_ACTIVE)
558#define hook_entergc(g) ((g)->hookmask |= (HOOK_ACTIVE|HOOK_GC)) 677#define hook_entergc(g) \
678 ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
559#define hook_vmevent(g) ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT)) 679#define hook_vmevent(g) ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
560#define hook_leave(g) ((g)->hookmask &= ~HOOK_ACTIVE) 680#define hook_leave(g) ((g)->hookmask &= ~HOOK_ACTIVE)
561#define hook_save(g) ((g)->hookmask & ~HOOK_EVENTMASK) 681#define hook_save(g) ((g)->hookmask & ~HOOK_EVENTMASK)
@@ -583,12 +703,23 @@ struct lua_State {
583#define registry(L) (&G(L)->registrytv) 703#define registry(L) (&G(L)->registrytv)
584 704
585/* Macros to access the currently executing (Lua) function. */ 705/* Macros to access the currently executing (Lua) function. */
706#if LJ_GC64
707#define curr_func(L) (&gcval(L->base-2)->fn)
708#elif LJ_FR2
709#define curr_func(L) (&gcref((L->base-2)->gcr)->fn)
710#else
586#define curr_func(L) (&gcref((L->base-1)->fr.func)->fn) 711#define curr_func(L) (&gcref((L->base-1)->fr.func)->fn)
712#endif
587#define curr_funcisL(L) (isluafunc(curr_func(L))) 713#define curr_funcisL(L) (isluafunc(curr_func(L)))
588#define curr_proto(L) (funcproto(curr_func(L))) 714#define curr_proto(L) (funcproto(curr_func(L)))
589#define curr_topL(L) (L->base + curr_proto(L)->framesize) 715#define curr_topL(L) (L->base + curr_proto(L)->framesize)
590#define curr_top(L) (curr_funcisL(L) ? curr_topL(L) : L->top) 716#define curr_top(L) (curr_funcisL(L) ? curr_topL(L) : L->top)
591 717
718#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
719LJ_FUNC_NORET void lj_assert_fail(global_State *g, const char *file, int line,
720 const char *func, const char *fmt, ...);
721#endif
722
592/* -- GC object definition and conversions -------------------------------- */ 723/* -- GC object definition and conversions -------------------------------- */
593 724
594/* GC header for generic access to common fields of GC objects. */ 725/* GC header for generic access to common fields of GC objects. */
@@ -642,17 +773,18 @@ typedef union GCobj {
642 773
643/* -- TValue getters/setters ---------------------------------------------- */ 774/* -- TValue getters/setters ---------------------------------------------- */
644 775
645#ifdef LUA_USE_ASSERT
646#include "lj_gc.h"
647#endif
648
649/* Macros to test types. */ 776/* Macros to test types. */
777#if LJ_GC64
778#define itype(o) ((uint32_t)((o)->it64 >> 47))
779#define tvisnil(o) ((o)->it64 == -1)
780#else
650#define itype(o) ((o)->it) 781#define itype(o) ((o)->it)
651#define tvisnil(o) (itype(o) == LJ_TNIL) 782#define tvisnil(o) (itype(o) == LJ_TNIL)
783#endif
652#define tvisfalse(o) (itype(o) == LJ_TFALSE) 784#define tvisfalse(o) (itype(o) == LJ_TFALSE)
653#define tvistrue(o) (itype(o) == LJ_TTRUE) 785#define tvistrue(o) (itype(o) == LJ_TTRUE)
654#define tvisbool(o) (tvisfalse(o) || tvistrue(o)) 786#define tvisbool(o) (tvisfalse(o) || tvistrue(o))
655#if LJ_64 787#if LJ_64 && !LJ_GC64
656#define tvislightud(o) (((int32_t)itype(o) >> 15) == -2) 788#define tvislightud(o) (((int32_t)itype(o) >> 15) == -2)
657#else 789#else
658#define tvislightud(o) (itype(o) == LJ_TLIGHTUD) 790#define tvislightud(o) (itype(o) == LJ_TLIGHTUD)
@@ -686,7 +818,7 @@ typedef union GCobj {
686#define rawnumequal(o1, o2) ((o1)->u64 == (o2)->u64) 818#define rawnumequal(o1, o2) ((o1)->u64 == (o2)->u64)
687 819
688/* Macros to convert type ids. */ 820/* Macros to convert type ids. */
689#if LJ_64 821#if LJ_64 && !LJ_GC64
690#define itypemap(o) \ 822#define itypemap(o) \
691 (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o)) 823 (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
692#else 824#else
@@ -694,13 +826,31 @@ typedef union GCobj {
694#endif 826#endif
695 827
696/* Macros to get tagged values. */ 828/* Macros to get tagged values. */
829#if LJ_GC64
830#define gcval(o) ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
831#else
697#define gcval(o) (gcref((o)->gcr)) 832#define gcval(o) (gcref((o)->gcr))
698#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - (o)->it)) 833#endif
834#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
699#if LJ_64 835#if LJ_64
700#define lightudV(o) \ 836#define lightudseg(u) \
701 check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff))) 837 (((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
838#define lightudlo(u) \
839 ((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
840#define lightudup(p) \
841 ((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
842static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
843{
844 uint64_t u = o->u64;
845 uint64_t seg = lightudseg(u);
846 uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
847 lj_assertG(tvislightud(o), "lightuserdata expected");
848 if (seg == (1 << LJ_LIGHTUD_BITS_SEG)-1) return NULL;
849 lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
850 return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
851}
702#else 852#else
703#define lightudV(o) check_exp(tvislightud(o), gcrefp((o)->gcr, void)) 853#define lightudV(g, o) check_exp(tvislightud(o), gcrefp((o)->gcr, void))
704#endif 854#endif
705#define gcV(o) check_exp(tvisgcv(o), gcval(o)) 855#define gcV(o) check_exp(tvisgcv(o), gcval(o))
706#define strV(o) check_exp(tvisstr(o), &gcval(o)->str) 856#define strV(o) check_exp(tvisstr(o), &gcval(o)->str)
@@ -714,40 +864,70 @@ typedef union GCobj {
714#define intV(o) check_exp(tvisint(o), (int32_t)(o)->i) 864#define intV(o) check_exp(tvisint(o), (int32_t)(o)->i)
715 865
716/* Macros to set tagged values. */ 866/* Macros to set tagged values. */
867#if LJ_GC64
868#define setitype(o, i) ((o)->it = ((i) << 15))
869#define setnilV(o) ((o)->it64 = -1)
870#define setpriV(o, x) ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
871#define setboolV(o, x) ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
872#else
717#define setitype(o, i) ((o)->it = (i)) 873#define setitype(o, i) ((o)->it = (i))
718#define setnilV(o) ((o)->it = LJ_TNIL) 874#define setnilV(o) ((o)->it = LJ_TNIL)
719#define setboolV(o, x) ((o)->it = LJ_TFALSE-(uint32_t)(x)) 875#define setboolV(o, x) ((o)->it = LJ_TFALSE-(uint32_t)(x))
876#define setpriV(o, i) (setitype((o), (i)))
877#endif
720 878
721static LJ_AINLINE void setlightudV(TValue *o, void *p) 879static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
722{ 880{
723#if LJ_64 881#if LJ_GC64
882 o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
883#elif LJ_64
724 o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48); 884 o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
725#else 885#else
726 setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD); 886 setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
727#endif 887#endif
728} 888}
729 889
730#if LJ_64 890#if LJ_FR2 || LJ_32
731#define checklightudptr(L, p) \ 891#define contptr(f) ((void *)(f))
732 (((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p)) 892#define setcont(o, f) ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
893#else
894#define contptr(f) \
895 ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
733#define setcont(o, f) \ 896#define setcont(o, f) \
734 ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin) 897 ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
735#else
736#define checklightudptr(L, p) (p)
737#define setcont(o, f) setlightudV((o), (void *)(f))
738#endif 898#endif
739 899
740#define tvchecklive(L, o) \ 900static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
741 UNUSED(L), lua_assert(!tvisgcv(o) || \ 901{
742 ((~itype(o) == gcval(o)->gch.gct) && !isdead(G(L), gcval(o)))) 902 UNUSED(L); UNUSED(o); UNUSED(msg);
903#if LUA_USE_ASSERT
904 if (tvisgcv(o)) {
905 lj_assertL(~itype(o) == gcval(o)->gch.gct,
906 "mismatch of TValue type %d vs GC type %d",
907 ~itype(o), gcval(o)->gch.gct);
908 /* Copy of isdead check from lj_gc.h to avoid circular include. */
909 lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
910 }
911#endif
912}
913
914static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
915{
916#if LJ_GC64
917 setgcreft(o->gcr, v, itype);
918#else
919 setgcref(o->gcr, v); setitype(o, itype);
920#endif
921}
743 922
744static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t itype) 923static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
745{ 924{
746 setgcref(o->gcr, v); setitype(o, itype); tvchecklive(L, o); 925 setgcVraw(o, v, it);
926 checklivetv(L, o, "store to dead GC object");
747} 927}
748 928
749#define define_setV(name, type, tag) \ 929#define define_setV(name, type, tag) \
750static LJ_AINLINE void name(lua_State *L, TValue *o, type *v) \ 930static LJ_AINLINE void name(lua_State *L, TValue *o, const type *v) \
751{ \ 931{ \
752 setgcV(L, o, obj2gco(v), tag); \ 932 setgcV(L, o, obj2gco(v), tag); \
753} 933}
@@ -790,13 +970,17 @@ static LJ_AINLINE void setint64V(TValue *o, int64_t i)
790/* Copy tagged values. */ 970/* Copy tagged values. */
791static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2) 971static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
792{ 972{
793 *o1 = *o2; tvchecklive(L, o1); 973 *o1 = *o2;
974 checklivetv(L, o1, "copy of dead GC object");
794} 975}
795 976
796/* -- Number to integer conversion ---------------------------------------- */ 977/* -- Number to integer conversion ---------------------------------------- */
797 978
798#if LJ_SOFTFP 979#if LJ_SOFTFP
799LJ_ASMF int32_t lj_vm_tobit(double x); 980LJ_ASMF int32_t lj_vm_tobit(double x);
981#if LJ_TARGET_MIPS64
982LJ_ASMF int32_t lj_vm_tointg(double x);
983#endif
800#endif 984#endif
801 985
802static LJ_AINLINE int32_t lj_num2bit(lua_Number n) 986static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
@@ -810,11 +994,7 @@ static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
810#endif 994#endif
811} 995}
812 996
813#if LJ_TARGET_X86 && !defined(__SSE2__)
814#define lj_num2int(n) lj_num2bit((n))
815#else
816#define lj_num2int(n) ((int32_t)(n)) 997#define lj_num2int(n) ((int32_t)(n))
817#endif
818 998
819/* 999/*
820** This must match the JIT backend behavior. In particular for archs 1000** This must match the JIT backend behavior. In particular for archs
@@ -859,6 +1039,21 @@ LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
859#define lj_typename(o) (lj_obj_itypename[itypemap(o)]) 1039#define lj_typename(o) (lj_obj_itypename[itypemap(o)])
860 1040
861/* Compare two objects without calling metamethods. */ 1041/* Compare two objects without calling metamethods. */
862LJ_FUNC int lj_obj_equal(cTValue *o1, cTValue *o2); 1042LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
1043LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
1044
1045#if LJ_ABI_PAUTH
1046#if LJ_TARGET_ARM64
1047#include <ptrauth.h>
1048#define lj_ptr_sign(ptr, ctx) \
1049 ptrauth_sign_unauthenticated((ptr), ptrauth_key_function_pointer, (ctx))
1050#define lj_ptr_strip(ptr) ptrauth_strip((ptr), ptrauth_key_function_pointer)
1051#else
1052#error "No support for pointer authentication for this architecture"
1053#endif
1054#else
1055#define lj_ptr_sign(ptr, ctx) (ptr)
1056#define lj_ptr_strip(ptr) (ptr)
1057#endif
863 1058
864#endif 1059#endif
diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c
index ab158143..48effb8a 100644
--- a/src/lj_opt_fold.c
+++ b/src/lj_opt_fold.c
@@ -14,18 +14,21 @@
14 14
15#if LJ_HASJIT 15#if LJ_HASJIT
16 16
17#include "lj_buf.h"
17#include "lj_str.h" 18#include "lj_str.h"
18#include "lj_tab.h" 19#include "lj_tab.h"
19#include "lj_ir.h" 20#include "lj_ir.h"
20#include "lj_jit.h" 21#include "lj_jit.h"
22#include "lj_ircall.h"
21#include "lj_iropt.h" 23#include "lj_iropt.h"
22#include "lj_trace.h" 24#include "lj_trace.h"
23#if LJ_HASFFI 25#if LJ_HASFFI
24#include "lj_ctype.h" 26#include "lj_ctype.h"
25#endif
26#include "lj_carith.h" 27#include "lj_carith.h"
28#endif
27#include "lj_vm.h" 29#include "lj_vm.h"
28#include "lj_strscan.h" 30#include "lj_strscan.h"
31#include "lj_strfmt.h"
29 32
30/* Here's a short description how the FOLD engine processes instructions: 33/* Here's a short description how the FOLD engine processes instructions:
31** 34**
@@ -133,8 +136,8 @@
133/* Some local macros to save typing. Undef'd at the end. */ 136/* Some local macros to save typing. Undef'd at the end. */
134#define IR(ref) (&J->cur.ir[(ref)]) 137#define IR(ref) (&J->cur.ir[(ref)])
135#define fins (&J->fold.ins) 138#define fins (&J->fold.ins)
136#define fleft (&J->fold.left) 139#define fleft (J->fold.left)
137#define fright (&J->fold.right) 140#define fright (J->fold.right)
138#define knumleft (ir_knum(fleft)->n) 141#define knumleft (ir_knum(fleft)->n)
139#define knumright (ir_knum(fright)->n) 142#define knumright (ir_knum(fright)->n)
140 143
@@ -155,13 +158,14 @@ typedef IRRef (LJ_FASTCALL *FoldFunc)(jit_State *J);
155 158
156/* Barrier to prevent folding across a GC step. 159/* Barrier to prevent folding across a GC step.
157** GC steps can only happen at the head of a trace and at LOOP. 160** GC steps can only happen at the head of a trace and at LOOP.
158** And the GC is only driven forward if there is at least one allocation. 161** And the GC is only driven forward if there's at least one allocation.
159*/ 162*/
160#define gcstep_barrier(J, ref) \ 163#define gcstep_barrier(J, ref) \
161 ((ref) < J->chain[IR_LOOP] && \ 164 ((ref) < J->chain[IR_LOOP] && \
162 (J->chain[IR_SNEW] || J->chain[IR_XSNEW] || \ 165 (J->chain[IR_SNEW] || J->chain[IR_XSNEW] || \
163 J->chain[IR_TNEW] || J->chain[IR_TDUP] || \ 166 J->chain[IR_TNEW] || J->chain[IR_TDUP] || \
164 J->chain[IR_CNEW] || J->chain[IR_CNEWI] || J->chain[IR_TOSTR])) 167 J->chain[IR_CNEW] || J->chain[IR_CNEWI] || \
168 J->chain[IR_BUFSTR] || J->chain[IR_TOSTR] || J->chain[IR_CALLA]))
165 169
166/* -- Constant folding for FP numbers ------------------------------------- */ 170/* -- Constant folding for FP numbers ------------------------------------- */
167 171
@@ -169,9 +173,6 @@ LJFOLD(ADD KNUM KNUM)
169LJFOLD(SUB KNUM KNUM) 173LJFOLD(SUB KNUM KNUM)
170LJFOLD(MUL KNUM KNUM) 174LJFOLD(MUL KNUM KNUM)
171LJFOLD(DIV KNUM KNUM) 175LJFOLD(DIV KNUM KNUM)
172LJFOLD(NEG KNUM KNUM)
173LJFOLD(ABS KNUM KNUM)
174LJFOLD(ATAN2 KNUM KNUM)
175LJFOLD(LDEXP KNUM KNUM) 176LJFOLD(LDEXP KNUM KNUM)
176LJFOLD(MIN KNUM KNUM) 177LJFOLD(MIN KNUM KNUM)
177LJFOLD(MAX KNUM KNUM) 178LJFOLD(MAX KNUM KNUM)
@@ -183,6 +184,15 @@ LJFOLDF(kfold_numarith)
183 return lj_ir_knum(J, y); 184 return lj_ir_knum(J, y);
184} 185}
185 186
187LJFOLD(NEG KNUM FLOAD)
188LJFOLD(ABS KNUM FLOAD)
189LJFOLDF(kfold_numabsneg)
190{
191 lua_Number a = knumleft;
192 lua_Number y = lj_vm_foldarith(a, a, fins->o - IR_ADD);
193 return lj_ir_knum(J, y);
194}
195
186LJFOLD(LDEXP KNUM KINT) 196LJFOLD(LDEXP KNUM KINT)
187LJFOLDF(kfold_ldexp) 197LJFOLDF(kfold_ldexp)
188{ 198{
@@ -202,13 +212,34 @@ LJFOLDF(kfold_fpmath)
202 return lj_ir_knum(J, y); 212 return lj_ir_knum(J, y);
203} 213}
204 214
205LJFOLD(POW KNUM KINT) 215LJFOLD(CALLN KNUM any)
216LJFOLDF(kfold_fpcall1)
217{
218 const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
219 if (CCI_TYPE(ci) == IRT_NUM) {
220 double y = ((double (*)(double))ci->func)(knumleft);
221 return lj_ir_knum(J, y);
222 }
223 return NEXTFOLD;
224}
225
226LJFOLD(CALLN CARG IRCALL_atan2)
227LJFOLDF(kfold_fpcall2)
228{
229 if (irref_isk(fleft->op1) && irref_isk(fleft->op2)) {
230 const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
231 double a = ir_knum(IR(fleft->op1))->n;
232 double b = ir_knum(IR(fleft->op2))->n;
233 double y = ((double (*)(double, double))ci->func)(a, b);
234 return lj_ir_knum(J, y);
235 }
236 return NEXTFOLD;
237}
238
239LJFOLD(POW KNUM KNUM)
206LJFOLDF(kfold_numpow) 240LJFOLDF(kfold_numpow)
207{ 241{
208 lua_Number a = knumleft; 242 return lj_ir_knum(J, lj_vm_foldarith(knumleft, knumright, IR_POW - IR_ADD));
209 lua_Number b = (lua_Number)fright->i;
210 lua_Number y = lj_vm_foldarith(a, b, IR_POW - IR_ADD);
211 return lj_ir_knum(J, y);
212} 243}
213 244
214/* Must not use kfold_kref for numbers (could be NaN). */ 245/* Must not use kfold_kref for numbers (could be NaN). */
@@ -247,7 +278,7 @@ static int32_t kfold_intop(int32_t k1, int32_t k2, IROp op)
247 case IR_BROR: k1 = (int32_t)lj_ror((uint32_t)k1, (k2 & 31)); break; 278 case IR_BROR: k1 = (int32_t)lj_ror((uint32_t)k1, (k2 & 31)); break;
248 case IR_MIN: k1 = k1 < k2 ? k1 : k2; break; 279 case IR_MIN: k1 = k1 < k2 ? k1 : k2; break;
249 case IR_MAX: k1 = k1 > k2 ? k1 : k2; break; 280 case IR_MAX: k1 = k1 > k2 ? k1 : k2; break;
250 default: lua_assert(0); break; 281 default: lj_assertX(0, "bad IR op %d", op); break;
251 } 282 }
252 return k1; 283 return k1;
253} 284}
@@ -319,7 +350,7 @@ LJFOLDF(kfold_intcomp)
319 case IR_ULE: return CONDFOLD((uint32_t)a <= (uint32_t)b); 350 case IR_ULE: return CONDFOLD((uint32_t)a <= (uint32_t)b);
320 case IR_ABC: 351 case IR_ABC:
321 case IR_UGT: return CONDFOLD((uint32_t)a > (uint32_t)b); 352 case IR_UGT: return CONDFOLD((uint32_t)a > (uint32_t)b);
322 default: lua_assert(0); return FAILFOLD; 353 default: lj_assertJ(0, "bad IR op %d", fins->o); return FAILFOLD;
323 } 354 }
324} 355}
325 356
@@ -333,21 +364,29 @@ LJFOLDF(kfold_intcomp0)
333 364
334/* -- Constant folding for 64 bit integers -------------------------------- */ 365/* -- Constant folding for 64 bit integers -------------------------------- */
335 366
336static uint64_t kfold_int64arith(uint64_t k1, uint64_t k2, IROp op) 367static uint64_t kfold_int64arith(jit_State *J, uint64_t k1, uint64_t k2,
368 IROp op)
337{ 369{
370 UNUSED(J);
371#if LJ_HASFFI
338 switch (op) { 372 switch (op) {
339#if LJ_64 || LJ_HASFFI
340 case IR_ADD: k1 += k2; break; 373 case IR_ADD: k1 += k2; break;
341 case IR_SUB: k1 -= k2; break; 374 case IR_SUB: k1 -= k2; break;
342#endif
343#if LJ_HASFFI
344 case IR_MUL: k1 *= k2; break; 375 case IR_MUL: k1 *= k2; break;
345 case IR_BAND: k1 &= k2; break; 376 case IR_BAND: k1 &= k2; break;
346 case IR_BOR: k1 |= k2; break; 377 case IR_BOR: k1 |= k2; break;
347 case IR_BXOR: k1 ^= k2; break; 378 case IR_BXOR: k1 ^= k2; break;
348#endif 379 case IR_BSHL: k1 <<= (k2 & 63); break;
349 default: UNUSED(k2); lua_assert(0); break; 380 case IR_BSHR: k1 = (int32_t)((uint32_t)k1 >> (k2 & 63)); break;
381 case IR_BSAR: k1 >>= (k2 & 63); break;
382 case IR_BROL: k1 = (int32_t)lj_rol((uint32_t)k1, (k2 & 63)); break;
383 case IR_BROR: k1 = (int32_t)lj_ror((uint32_t)k1, (k2 & 63)); break;
384 default: lj_assertJ(0, "bad IR op %d", op); break;
350 } 385 }
386#else
387 UNUSED(k2); UNUSED(op);
388 lj_assertJ(0, "FFI IR op without FFI");
389#endif
351 return k1; 390 return k1;
352} 391}
353 392
@@ -359,7 +398,7 @@ LJFOLD(BOR KINT64 KINT64)
359LJFOLD(BXOR KINT64 KINT64) 398LJFOLD(BXOR KINT64 KINT64)
360LJFOLDF(kfold_int64arith) 399LJFOLDF(kfold_int64arith)
361{ 400{
362 return INT64FOLD(kfold_int64arith(ir_k64(fleft)->u64, 401 return INT64FOLD(kfold_int64arith(J, ir_k64(fleft)->u64,
363 ir_k64(fright)->u64, (IROp)fins->o)); 402 ir_k64(fright)->u64, (IROp)fins->o));
364} 403}
365 404
@@ -381,7 +420,7 @@ LJFOLDF(kfold_int64arith2)
381 } 420 }
382 return INT64FOLD(k1); 421 return INT64FOLD(k1);
383#else 422#else
384 UNUSED(J); lua_assert(0); return FAILFOLD; 423 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
385#endif 424#endif
386} 425}
387 426
@@ -392,22 +431,12 @@ LJFOLD(BROL KINT64 KINT)
392LJFOLD(BROR KINT64 KINT) 431LJFOLD(BROR KINT64 KINT)
393LJFOLDF(kfold_int64shift) 432LJFOLDF(kfold_int64shift)
394{ 433{
395#if LJ_HASFFI || LJ_64 434#if LJ_HASFFI
396 uint64_t k = ir_k64(fleft)->u64; 435 uint64_t k = ir_k64(fleft)->u64;
397 int32_t sh = (fright->i & 63); 436 int32_t sh = (fright->i & 63);
398 switch ((IROp)fins->o) { 437 return INT64FOLD(lj_carith_shift64(k, sh, fins->o - IR_BSHL));
399 case IR_BSHL: k <<= sh; break;
400#if LJ_HASFFI
401 case IR_BSHR: k >>= sh; break;
402 case IR_BSAR: k = (uint64_t)((int64_t)k >> sh); break;
403 case IR_BROL: k = lj_rol(k, sh); break;
404 case IR_BROR: k = lj_ror(k, sh); break;
405#endif
406 default: lua_assert(0); break;
407 }
408 return INT64FOLD(k);
409#else 438#else
410 UNUSED(J); lua_assert(0); return FAILFOLD; 439 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
411#endif 440#endif
412} 441}
413 442
@@ -417,7 +446,7 @@ LJFOLDF(kfold_bnot64)
417#if LJ_HASFFI 446#if LJ_HASFFI
418 return INT64FOLD(~ir_k64(fleft)->u64); 447 return INT64FOLD(~ir_k64(fleft)->u64);
419#else 448#else
420 UNUSED(J); lua_assert(0); return FAILFOLD; 449 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
421#endif 450#endif
422} 451}
423 452
@@ -427,7 +456,7 @@ LJFOLDF(kfold_bswap64)
427#if LJ_HASFFI 456#if LJ_HASFFI
428 return INT64FOLD(lj_bswap64(ir_k64(fleft)->u64)); 457 return INT64FOLD(lj_bswap64(ir_k64(fleft)->u64));
429#else 458#else
430 UNUSED(J); lua_assert(0); return FAILFOLD; 459 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
431#endif 460#endif
432} 461}
433 462
@@ -452,10 +481,10 @@ LJFOLDF(kfold_int64comp)
452 case IR_UGE: return CONDFOLD(a >= b); 481 case IR_UGE: return CONDFOLD(a >= b);
453 case IR_ULE: return CONDFOLD(a <= b); 482 case IR_ULE: return CONDFOLD(a <= b);
454 case IR_UGT: return CONDFOLD(a > b); 483 case IR_UGT: return CONDFOLD(a > b);
455 default: lua_assert(0); return FAILFOLD; 484 default: lj_assertJ(0, "bad IR op %d", fins->o); return FAILFOLD;
456 } 485 }
457#else 486#else
458 UNUSED(J); lua_assert(0); return FAILFOLD; 487 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
459#endif 488#endif
460} 489}
461 490
@@ -467,7 +496,7 @@ LJFOLDF(kfold_int64comp0)
467 return DROPFOLD; 496 return DROPFOLD;
468 return NEXTFOLD; 497 return NEXTFOLD;
469#else 498#else
470 UNUSED(J); lua_assert(0); return FAILFOLD; 499 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
471#endif 500#endif
472} 501}
473 502
@@ -481,6 +510,7 @@ LJFOLDF(kfold_snew_kptr)
481} 510}
482 511
483LJFOLD(SNEW any KINT) 512LJFOLD(SNEW any KINT)
513LJFOLD(XSNEW any KINT)
484LJFOLDF(kfold_snew_empty) 514LJFOLDF(kfold_snew_empty)
485{ 515{
486 if (fright->i == 0) 516 if (fright->i == 0)
@@ -492,7 +522,7 @@ LJFOLD(STRREF KGC KINT)
492LJFOLDF(kfold_strref) 522LJFOLDF(kfold_strref)
493{ 523{
494 GCstr *str = ir_kstr(fleft); 524 GCstr *str = ir_kstr(fleft);
495 lua_assert((MSize)fright->i <= str->len); 525 lj_assertJ((MSize)fright->i <= str->len, "bad string ref");
496 return lj_ir_kkptr(J, (char *)strdata(str) + fright->i); 526 return lj_ir_kkptr(J, (char *)strdata(str) + fright->i);
497} 527}
498 528
@@ -510,7 +540,7 @@ LJFOLDF(kfold_strref_snew)
510 PHIBARRIER(ir); 540 PHIBARRIER(ir);
511 fins->op2 = emitir(IRTI(IR_ADD), ir->op2, fins->op2); /* Clobbers fins! */ 541 fins->op2 = emitir(IRTI(IR_ADD), ir->op2, fins->op2); /* Clobbers fins! */
512 fins->op1 = str; 542 fins->op1 = str;
513 fins->ot = IRT(IR_STRREF, IRT_P32); 543 fins->ot = IRT(IR_STRREF, IRT_PGC);
514 return RETRYFOLD; 544 return RETRYFOLD;
515 } 545 }
516 } 546 }
@@ -528,6 +558,211 @@ LJFOLDF(kfold_strcmp)
528 return NEXTFOLD; 558 return NEXTFOLD;
529} 559}
530 560
561/* -- Constant folding and forwarding for buffers ------------------------- */
562
563/*
564** Buffer ops perform stores, but their effect is limited to the buffer
565** itself. Also, buffer ops are chained: a use of an op implies a use of
566** all other ops up the chain. Conversely, if an op is unused, all ops
567** up the chain can go unsed. This largely eliminates the need to treat
568** them as stores.
569**
570** Alas, treating them as normal (IRM_N) ops doesn't work, because they
571** cannot be CSEd in isolation. CSE for IRM_N is implicitly done in LOOP
572** or if FOLD is disabled.
573**
574** The compromise is to declare them as loads, emit them like stores and
575** CSE whole chains manually when the BUFSTR is to be emitted. Any chain
576** fragments left over from CSE are eliminated by DCE.
577**
578** The string buffer methods emit a USE instead of a BUFSTR to keep the
579** chain alive.
580*/
581
582LJFOLD(BUFHDR any any)
583LJFOLDF(bufhdr_merge)
584{
585 return fins->op2 == IRBUFHDR_WRITE ? CSEFOLD : EMITFOLD;
586}
587
588LJFOLD(BUFPUT any BUFSTR)
589LJFOLDF(bufput_bufstr)
590{
591 if ((J->flags & JIT_F_OPT_FWD)) {
592 IRRef hdr = fright->op2;
593 /* New buffer, no other buffer op inbetween and same buffer? */
594 if (fleft->o == IR_BUFHDR && fleft->op2 == IRBUFHDR_RESET &&
595 fleft->prev == hdr &&
596 fleft->op1 == IR(hdr)->op1 &&
597 !(irt_isphi(fright->t) && IR(hdr)->prev) &&
598 (!LJ_HASBUFFER || J->chain[IR_CALLA] < hdr)) {
599 IRRef ref = fins->op1;
600 IR(ref)->op2 = IRBUFHDR_APPEND; /* Modify BUFHDR. */
601 IR(ref)->op1 = fright->op1;
602 return ref;
603 }
604 /* Replay puts to global temporary buffer. */
605 if (IR(hdr)->op2 == IRBUFHDR_RESET && !irt_isphi(fright->t)) {
606 IRIns *ir = IR(fright->op1);
607 /* For now only handle single string.reverse .lower .upper .rep. */
608 if (ir->o == IR_CALLL &&
609 ir->op2 >= IRCALL_lj_buf_putstr_reverse &&
610 ir->op2 <= IRCALL_lj_buf_putstr_rep) {
611 IRIns *carg1 = IR(ir->op1);
612 if (ir->op2 == IRCALL_lj_buf_putstr_rep) {
613 IRIns *carg2 = IR(carg1->op1);
614 if (carg2->op1 == hdr) {
615 return lj_ir_call(J, ir->op2, fins->op1, carg2->op2, carg1->op2);
616 }
617 } else if (carg1->op1 == hdr) {
618 return lj_ir_call(J, ir->op2, fins->op1, carg1->op2);
619 }
620 }
621 }
622 }
623 return EMITFOLD; /* Always emit, CSE later. */
624}
625
626LJFOLD(BUFPUT any any)
627LJFOLDF(bufput_kgc)
628{
629 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && fright->o == IR_KGC) {
630 GCstr *s2 = ir_kstr(fright);
631 if (s2->len == 0) { /* Empty string? */
632 return LEFTFOLD;
633 } else {
634 if (fleft->o == IR_BUFPUT && irref_isk(fleft->op2) &&
635 !irt_isphi(fleft->t)) { /* Join two constant string puts in a row. */
636 GCstr *s1 = ir_kstr(IR(fleft->op2));
637 IRRef kref = lj_ir_kstr(J, lj_buf_cat2str(J->L, s1, s2));
638 /* lj_ir_kstr() may realloc the IR and invalidates any IRIns *. */
639 IR(fins->op1)->op2 = kref; /* Modify previous BUFPUT. */
640 return fins->op1;
641 }
642 }
643 }
644 return EMITFOLD; /* Always emit, CSE later. */
645}
646
647LJFOLD(BUFSTR any any)
648LJFOLDF(bufstr_kfold_cse)
649{
650 lj_assertJ(fleft->o == IR_BUFHDR || fleft->o == IR_BUFPUT ||
651 fleft->o == IR_CALLL,
652 "bad buffer constructor IR op %d", fleft->o);
653 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD)) {
654 if (fleft->o == IR_BUFHDR) { /* No put operations? */
655 if (fleft->op2 == IRBUFHDR_RESET) /* Empty buffer? */
656 return lj_ir_kstr(J, &J2G(J)->strempty);
657 fins->op1 = fleft->op1;
658 fins->op2 = fleft->prev; /* Relies on checks in bufput_append. */
659 return CSEFOLD;
660 } else if (fleft->o == IR_BUFPUT) {
661 IRIns *irb = IR(fleft->op1);
662 if (irb->o == IR_BUFHDR && irb->op2 == IRBUFHDR_RESET)
663 return fleft->op2; /* Shortcut for a single put operation. */
664 }
665 }
666 /* Try to CSE the whole chain. */
667 if (LJ_LIKELY(J->flags & JIT_F_OPT_CSE)) {
668 IRRef ref = J->chain[IR_BUFSTR];
669 while (ref) {
670 IRIns *irs = IR(ref), *ira = fleft, *irb = IR(irs->op1);
671 while (ira->o == irb->o && ira->op2 == irb->op2) {
672 lj_assertJ(ira->o == IR_BUFHDR || ira->o == IR_BUFPUT ||
673 ira->o == IR_CALLL || ira->o == IR_CARG,
674 "bad buffer constructor IR op %d", ira->o);
675 if (ira->o == IR_BUFHDR && ira->op2 == IRBUFHDR_RESET)
676 return ref; /* CSE succeeded. */
677 if (ira->o == IR_CALLL && ira->op2 == IRCALL_lj_buf_puttab)
678 break;
679 ira = IR(ira->op1);
680 irb = IR(irb->op1);
681 }
682 ref = irs->prev;
683 }
684 }
685 return EMITFOLD; /* No CSE possible. */
686}
687
688LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_reverse)
689LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_upper)
690LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_lower)
691LJFOLD(CALLL CARG IRCALL_lj_strfmt_putquoted)
692LJFOLDF(bufput_kfold_op)
693{
694 if (irref_isk(fleft->op2)) {
695 const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
696 SBuf *sb = lj_buf_tmp_(J->L);
697 sb = ((SBuf * (LJ_FASTCALL *)(SBuf *, GCstr *))ci->func)(sb,
698 ir_kstr(IR(fleft->op2)));
699 fins->o = IR_BUFPUT;
700 fins->op1 = fleft->op1;
701 fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
702 return RETRYFOLD;
703 }
704 return EMITFOLD; /* Always emit, CSE later. */
705}
706
707LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_rep)
708LJFOLDF(bufput_kfold_rep)
709{
710 if (irref_isk(fleft->op2)) {
711 IRIns *irc = IR(fleft->op1);
712 if (irref_isk(irc->op2)) {
713 SBuf *sb = lj_buf_tmp_(J->L);
714 sb = lj_buf_putstr_rep(sb, ir_kstr(IR(irc->op2)), IR(fleft->op2)->i);
715 fins->o = IR_BUFPUT;
716 fins->op1 = irc->op1;
717 fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
718 return RETRYFOLD;
719 }
720 }
721 return EMITFOLD; /* Always emit, CSE later. */
722}
723
724LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfxint)
725LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum_int)
726LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum_uint)
727LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum)
728LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfstr)
729LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfchar)
730LJFOLDF(bufput_kfold_fmt)
731{
732 IRIns *irc = IR(fleft->op1);
733 lj_assertJ(irref_isk(irc->op2), "SFormat must be const");
734 if (irref_isk(fleft->op2)) {
735 SFormat sf = (SFormat)IR(irc->op2)->i;
736 IRIns *ira = IR(fleft->op2);
737 SBuf *sb = lj_buf_tmp_(J->L);
738 switch (fins->op2) {
739 case IRCALL_lj_strfmt_putfxint:
740 sb = lj_strfmt_putfxint(sb, sf, ir_k64(ira)->u64);
741 break;
742 case IRCALL_lj_strfmt_putfstr:
743 sb = lj_strfmt_putfstr(sb, sf, ir_kstr(ira));
744 break;
745 case IRCALL_lj_strfmt_putfchar:
746 sb = lj_strfmt_putfchar(sb, sf, ira->i);
747 break;
748 case IRCALL_lj_strfmt_putfnum_int:
749 case IRCALL_lj_strfmt_putfnum_uint:
750 case IRCALL_lj_strfmt_putfnum:
751 default: {
752 const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
753 sb = ((SBuf * (*)(SBuf *, SFormat, lua_Number))ci->func)(sb, sf,
754 ir_knum(ira)->n);
755 break;
756 }
757 }
758 fins->o = IR_BUFPUT;
759 fins->op1 = irc->op1;
760 fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
761 return RETRYFOLD;
762 }
763 return EMITFOLD; /* Always emit, CSE later. */
764}
765
531/* -- Constant folding of pointer arithmetic ------------------------------ */ 766/* -- Constant folding of pointer arithmetic ------------------------------ */
532 767
533LJFOLD(ADD KGC KINT) 768LJFOLD(ADD KGC KINT)
@@ -648,21 +883,17 @@ LJFOLD(CONV KNUM IRCONV_INT_NUM)
648LJFOLDF(kfold_conv_knum_int_num) 883LJFOLDF(kfold_conv_knum_int_num)
649{ 884{
650 lua_Number n = knumleft; 885 lua_Number n = knumleft;
651 if (!(fins->op2 & IRCONV_TRUNC)) { 886 int32_t k = lj_num2int(n);
652 int32_t k = lj_num2int(n); 887 if (irt_isguard(fins->t) && n != (lua_Number)k) {
653 if (irt_isguard(fins->t) && n != (lua_Number)k) { 888 /* We're about to create a guard which always fails, like CONV +1.5.
654 /* We're about to create a guard which always fails, like CONV +1.5. 889 ** Some pathological loops cause this during LICM, e.g.:
655 ** Some pathological loops cause this during LICM, e.g.: 890 ** local x,k,t = 0,1.5,{1,[1.5]=2}
656 ** local x,k,t = 0,1.5,{1,[1.5]=2} 891 ** for i=1,200 do x = x+ t[k]; k = k == 1 and 1.5 or 1 end
657 ** for i=1,200 do x = x+ t[k]; k = k == 1 and 1.5 or 1 end 892 ** assert(x == 300)
658 ** assert(x == 300) 893 */
659 */ 894 return FAILFOLD;
660 return FAILFOLD;
661 }
662 return INTFOLD(k);
663 } else {
664 return INTFOLD((int32_t)n);
665 } 895 }
896 return INTFOLD(k);
666} 897}
667 898
668LJFOLD(CONV KNUM IRCONV_U32_NUM) 899LJFOLD(CONV KNUM IRCONV_U32_NUM)
@@ -690,16 +921,18 @@ LJFOLDF(kfold_conv_knum_u64_num)
690 return INT64FOLD(lj_num2u64(knumleft)); 921 return INT64FOLD(lj_num2u64(knumleft));
691} 922}
692 923
693LJFOLD(TOSTR KNUM) 924LJFOLD(TOSTR KNUM any)
694LJFOLDF(kfold_tostr_knum) 925LJFOLDF(kfold_tostr_knum)
695{ 926{
696 return lj_ir_kstr(J, lj_str_fromnum(J->L, &knumleft)); 927 return lj_ir_kstr(J, lj_strfmt_num(J->L, ir_knum(fleft)));
697} 928}
698 929
699LJFOLD(TOSTR KINT) 930LJFOLD(TOSTR KINT any)
700LJFOLDF(kfold_tostr_kint) 931LJFOLDF(kfold_tostr_kint)
701{ 932{
702 return lj_ir_kstr(J, lj_str_fromint(J->L, fleft->i)); 933 return lj_ir_kstr(J, fins->op2 == IRTOSTR_INT ?
934 lj_strfmt_int(J->L, fleft->i) :
935 lj_strfmt_char(J->L, fleft->i));
703} 936}
704 937
705LJFOLD(STRTO KGC) 938LJFOLD(STRTO KGC)
@@ -747,13 +980,13 @@ LJFOLDF(shortcut_round)
747 return NEXTFOLD; 980 return NEXTFOLD;
748} 981}
749 982
750LJFOLD(ABS ABS KNUM) 983LJFOLD(ABS ABS FLOAD)
751LJFOLDF(shortcut_left) 984LJFOLDF(shortcut_left)
752{ 985{
753 return LEFTFOLD; /* f(g(x)) ==> g(x) */ 986 return LEFTFOLD; /* f(g(x)) ==> g(x) */
754} 987}
755 988
756LJFOLD(ABS NEG KNUM) 989LJFOLD(ABS NEG FLOAD)
757LJFOLDF(shortcut_dropleft) 990LJFOLDF(shortcut_dropleft)
758{ 991{
759 PHIBARRIER(fleft); 992 PHIBARRIER(fleft);
@@ -833,8 +1066,10 @@ LJFOLDF(simplify_nummuldiv_k)
833 if (n == 1.0) { /* x o 1 ==> x */ 1066 if (n == 1.0) { /* x o 1 ==> x */
834 return LEFTFOLD; 1067 return LEFTFOLD;
835 } else if (n == -1.0) { /* x o -1 ==> -x */ 1068 } else if (n == -1.0) { /* x o -1 ==> -x */
1069 IRRef op1 = fins->op1;
1070 fins->op2 = (IRRef1)lj_ir_ksimd(J, LJ_KSIMD_NEG); /* Modifies fins. */
1071 fins->op1 = op1;
836 fins->o = IR_NEG; 1072 fins->o = IR_NEG;
837 fins->op2 = (IRRef1)lj_ir_knum_neg(J);
838 return RETRYFOLD; 1073 return RETRYFOLD;
839 } else if (fins->o == IR_MUL && n == 2.0) { /* x * 2 ==> x + x */ 1074 } else if (fins->o == IR_MUL && n == 2.0) { /* x * 2 ==> x + x */
840 fins->o = IR_ADD; 1075 fins->o = IR_ADD;
@@ -874,52 +1109,17 @@ LJFOLDF(simplify_nummuldiv_negneg)
874 return RETRYFOLD; 1109 return RETRYFOLD;
875} 1110}
876 1111
877LJFOLD(POW any KINT) 1112LJFOLD(POW any KNUM)
878LJFOLDF(simplify_numpow_xk) 1113LJFOLDF(simplify_numpow_k)
879{ 1114{
880 int32_t k = fright->i; 1115 if (knumright == 0.0) /* x ^ 0 ==> 1 */
881 TRef ref = fins->op1;
882 if (k == 0) /* x ^ 0 ==> 1 */
883 return lj_ir_knum_one(J); /* Result must be a number, not an int. */ 1116 return lj_ir_knum_one(J); /* Result must be a number, not an int. */
884 if (k == 1) /* x ^ 1 ==> x */ 1117 else if (knumright == 1.0) /* x ^ 1 ==> x */
885 return LEFTFOLD; 1118 return LEFTFOLD;
886 if ((uint32_t)(k+65536) > 2*65536u) /* Limit code explosion. */ 1119 else if (knumright == 2.0) /* x ^ 2 ==> x * x */
1120 return emitir(IRTN(IR_MUL), fins->op1, fins->op1);
1121 else
887 return NEXTFOLD; 1122 return NEXTFOLD;
888 if (k < 0) { /* x ^ (-k) ==> (1/x) ^ k. */
889 ref = emitir(IRTN(IR_DIV), lj_ir_knum_one(J), ref);
890 k = -k;
891 }
892 /* Unroll x^k for 1 <= k <= 65536. */
893 for (; (k & 1) == 0; k >>= 1) /* Handle leading zeros. */
894 ref = emitir(IRTN(IR_MUL), ref, ref);
895 if ((k >>= 1) != 0) { /* Handle trailing bits. */
896 TRef tmp = emitir(IRTN(IR_MUL), ref, ref);
897 for (; k != 1; k >>= 1) {
898 if (k & 1)
899 ref = emitir(IRTN(IR_MUL), ref, tmp);
900 tmp = emitir(IRTN(IR_MUL), tmp, tmp);
901 }
902 ref = emitir(IRTN(IR_MUL), ref, tmp);
903 }
904 return ref;
905}
906
907LJFOLD(POW KNUM any)
908LJFOLDF(simplify_numpow_kx)
909{
910 lua_Number n = knumleft;
911 if (n == 2.0) { /* 2.0 ^ i ==> ldexp(1.0, tonum(i)) */
912 fins->o = IR_CONV;
913#if LJ_TARGET_X86ORX64
914 fins->op1 = fins->op2;
915 fins->op2 = IRCONV_NUM_INT;
916 fins->op2 = (IRRef1)lj_opt_fold(J);
917#endif
918 fins->op1 = (IRRef1)lj_ir_knum_one(J);
919 fins->o = IR_LDEXP;
920 return RETRYFOLD;
921 }
922 return NEXTFOLD;
923} 1123}
924 1124
925/* -- Simplify conversions ------------------------------------------------ */ 1125/* -- Simplify conversions ------------------------------------------------ */
@@ -1004,10 +1204,10 @@ LJFOLDF(simplify_tobit_conv)
1004{ 1204{
1005 /* Fold even across PHI to avoid expensive num->int conversions in loop. */ 1205 /* Fold even across PHI to avoid expensive num->int conversions in loop. */
1006 if ((fleft->op2 & IRCONV_SRCMASK) == IRT_INT) { 1206 if ((fleft->op2 & IRCONV_SRCMASK) == IRT_INT) {
1007 lua_assert(irt_isnum(fleft->t)); 1207 lj_assertJ(irt_isnum(fleft->t), "expected TOBIT number arg");
1008 return fleft->op1; 1208 return fleft->op1;
1009 } else if ((fleft->op2 & IRCONV_SRCMASK) == IRT_U32) { 1209 } else if ((fleft->op2 & IRCONV_SRCMASK) == IRT_U32) {
1010 lua_assert(irt_isnum(fleft->t)); 1210 lj_assertJ(irt_isnum(fleft->t), "expected TOBIT number arg");
1011 fins->o = IR_CONV; 1211 fins->o = IR_CONV;
1012 fins->op1 = fleft->op1; 1212 fins->op1 = fleft->op1;
1013 fins->op2 = (IRT_INT<<5)|IRT_U32; 1213 fins->op2 = (IRT_INT<<5)|IRT_U32;
@@ -1047,7 +1247,7 @@ LJFOLDF(simplify_conv_sext)
1047 /* Use scalar evolution analysis results to strength-reduce sign-extension. */ 1247 /* Use scalar evolution analysis results to strength-reduce sign-extension. */
1048 if (ref == J->scev.idx) { 1248 if (ref == J->scev.idx) {
1049 IRRef lo = J->scev.dir ? J->scev.start : J->scev.stop; 1249 IRRef lo = J->scev.dir ? J->scev.start : J->scev.stop;
1050 lua_assert(irt_isint(J->scev.t)); 1250 lj_assertJ(irt_isint(J->scev.t), "only int SCEV supported");
1051 if (lo && IR(lo)->o == IR_KINT && IR(lo)->i + ofs >= 0) { 1251 if (lo && IR(lo)->o == IR_KINT && IR(lo)->i + ofs >= 0) {
1052 ok_reduce: 1252 ok_reduce:
1053#if LJ_TARGET_X64 1253#if LJ_TARGET_X64
@@ -1078,6 +1278,10 @@ LJFOLD(CONV SUB IRCONV_U32_U64)
1078LJFOLD(CONV MUL IRCONV_U32_U64) 1278LJFOLD(CONV MUL IRCONV_U32_U64)
1079LJFOLDF(simplify_conv_narrow) 1279LJFOLDF(simplify_conv_narrow)
1080{ 1280{
1281#if LJ_64
1282 UNUSED(J);
1283 return NEXTFOLD;
1284#else
1081 IROp op = (IROp)fleft->o; 1285 IROp op = (IROp)fleft->o;
1082 IRType t = irt_type(fins->t); 1286 IRType t = irt_type(fins->t);
1083 IRRef op1 = fleft->op1, op2 = fleft->op2, mode = fins->op2; 1287 IRRef op1 = fleft->op1, op2 = fleft->op2, mode = fins->op2;
@@ -1088,6 +1292,7 @@ LJFOLDF(simplify_conv_narrow)
1088 fins->op1 = op1; 1292 fins->op1 = op1;
1089 fins->op2 = op2; 1293 fins->op2 = op2;
1090 return RETRYFOLD; 1294 return RETRYFOLD;
1295#endif
1091} 1296}
1092 1297
1093/* Special CSE rule for CONV. */ 1298/* Special CSE rule for CONV. */
@@ -1123,7 +1328,8 @@ LJFOLDF(narrow_convert)
1123 /* Narrowing ignores PHIs and repeating it inside the loop is not useful. */ 1328 /* Narrowing ignores PHIs and repeating it inside the loop is not useful. */
1124 if (J->chain[IR_LOOP]) 1329 if (J->chain[IR_LOOP])
1125 return NEXTFOLD; 1330 return NEXTFOLD;
1126 lua_assert(fins->o != IR_CONV || (fins->op2&IRCONV_CONVMASK) != IRCONV_TOBIT); 1331 lj_assertJ(fins->o != IR_CONV || (fins->op2&IRCONV_CONVMASK) != IRCONV_TOBIT,
1332 "unexpected CONV TOBIT");
1127 return lj_opt_narrow_convert(J); 1333 return lj_opt_narrow_convert(J);
1128} 1334}
1129 1335
@@ -1201,7 +1407,9 @@ static TRef simplify_intmul_k(jit_State *J, int32_t k)
1201 ** But this is mainly intended for simple address arithmetic. 1407 ** But this is mainly intended for simple address arithmetic.
1202 ** Also it's easier for the backend to optimize the original multiplies. 1408 ** Also it's easier for the backend to optimize the original multiplies.
1203 */ 1409 */
1204 if (k == 1) { /* i * 1 ==> i */ 1410 if (k == 0) { /* i * 0 ==> 0 */
1411 return RIGHTFOLD;
1412 } else if (k == 1) { /* i * 1 ==> i */
1205 return LEFTFOLD; 1413 return LEFTFOLD;
1206 } else if ((k & (k-1)) == 0) { /* i * 2^k ==> i << k */ 1414 } else if ((k & (k-1)) == 0) { /* i * 2^k ==> i << k */
1207 fins->o = IR_BSHL; 1415 fins->o = IR_BSHL;
@@ -1214,9 +1422,7 @@ static TRef simplify_intmul_k(jit_State *J, int32_t k)
1214LJFOLD(MUL any KINT) 1422LJFOLD(MUL any KINT)
1215LJFOLDF(simplify_intmul_k32) 1423LJFOLDF(simplify_intmul_k32)
1216{ 1424{
1217 if (fright->i == 0) /* i * 0 ==> 0 */ 1425 if (fright->i >= 0)
1218 return INTFOLD(0);
1219 else if (fright->i > 0)
1220 return simplify_intmul_k(J, fright->i); 1426 return simplify_intmul_k(J, fright->i);
1221 return NEXTFOLD; 1427 return NEXTFOLD;
1222} 1428}
@@ -1224,21 +1430,20 @@ LJFOLDF(simplify_intmul_k32)
1224LJFOLD(MUL any KINT64) 1430LJFOLD(MUL any KINT64)
1225LJFOLDF(simplify_intmul_k64) 1431LJFOLDF(simplify_intmul_k64)
1226{ 1432{
1227 if (ir_kint64(fright)->u64 == 0) /* i * 0 ==> 0 */ 1433#if LJ_HASFFI
1228 return INT64FOLD(0); 1434 if (ir_kint64(fright)->u64 < 0x80000000u)
1229#if LJ_64
1230 /* NYI: SPLIT for BSHL and 32 bit backend support. */
1231 else if (ir_kint64(fright)->u64 < 0x80000000u)
1232 return simplify_intmul_k(J, (int32_t)ir_kint64(fright)->u64); 1435 return simplify_intmul_k(J, (int32_t)ir_kint64(fright)->u64);
1233#endif
1234 return NEXTFOLD; 1436 return NEXTFOLD;
1437#else
1438 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
1439#endif
1235} 1440}
1236 1441
1237LJFOLD(MOD any KINT) 1442LJFOLD(MOD any KINT)
1238LJFOLDF(simplify_intmod_k) 1443LJFOLDF(simplify_intmod_k)
1239{ 1444{
1240 int32_t k = fright->i; 1445 int32_t k = fright->i;
1241 lua_assert(k != 0); 1446 lj_assertJ(k != 0, "integer mod 0");
1242 if (k > 0 && (k & (k-1)) == 0) { /* i % (2^k) ==> i & (2^k-1) */ 1447 if (k > 0 && (k & (k-1)) == 0) { /* i % (2^k) ==> i & (2^k-1) */
1243 fins->o = IR_BAND; 1448 fins->o = IR_BAND;
1244 fins->op2 = lj_ir_kint(J, k-1); 1449 fins->op2 = lj_ir_kint(J, k-1);
@@ -1487,6 +1692,15 @@ LJFOLDF(simplify_shiftk_andk)
1487 fins->op2 = (IRRef1)lj_ir_kint(J, k); 1692 fins->op2 = (IRRef1)lj_ir_kint(J, k);
1488 fins->ot = IRTI(IR_BAND); 1693 fins->ot = IRTI(IR_BAND);
1489 return RETRYFOLD; 1694 return RETRYFOLD;
1695 } else if (irk->o == IR_KINT64) {
1696 uint64_t k = kfold_int64arith(J, ir_k64(irk)->u64, fright->i,
1697 (IROp)fins->o);
1698 IROpT ot = fleft->ot;
1699 fins->op1 = fleft->op1;
1700 fins->op1 = (IRRef1)lj_opt_fold(J);
1701 fins->op2 = (IRRef1)lj_ir_kint64(J, k);
1702 fins->ot = ot;
1703 return RETRYFOLD;
1490 } 1704 }
1491 return NEXTFOLD; 1705 return NEXTFOLD;
1492} 1706}
@@ -1502,6 +1716,47 @@ LJFOLDF(simplify_andk_shiftk)
1502 return NEXTFOLD; 1716 return NEXTFOLD;
1503} 1717}
1504 1718
1719LJFOLD(BAND BOR KINT)
1720LJFOLD(BOR BAND KINT)
1721LJFOLDF(simplify_andor_k)
1722{
1723 IRIns *irk = IR(fleft->op2);
1724 PHIBARRIER(fleft);
1725 if (irk->o == IR_KINT) {
1726 int32_t k = kfold_intop(irk->i, fright->i, (IROp)fins->o);
1727 /* (i | k1) & k2 ==> i & k2, if (k1 & k2) == 0. */
1728 /* (i & k1) | k2 ==> i | k2, if (k1 | k2) == -1. */
1729 if (k == (fins->o == IR_BAND ? 0 : -1)) {
1730 fins->op1 = fleft->op1;
1731 return RETRYFOLD;
1732 }
1733 }
1734 return NEXTFOLD;
1735}
1736
1737LJFOLD(BAND BOR KINT64)
1738LJFOLD(BOR BAND KINT64)
1739LJFOLDF(simplify_andor_k64)
1740{
1741#if LJ_HASFFI
1742 IRIns *irk = IR(fleft->op2);
1743 PHIBARRIER(fleft);
1744 if (irk->o == IR_KINT64) {
1745 uint64_t k = kfold_int64arith(J, ir_k64(irk)->u64, ir_k64(fright)->u64,
1746 (IROp)fins->o);
1747 /* (i | k1) & k2 ==> i & k2, if (k1 & k2) == 0. */
1748 /* (i & k1) | k2 ==> i | k2, if (k1 | k2) == -1. */
1749 if (k == (fins->o == IR_BAND ? (uint64_t)0 : ~(uint64_t)0)) {
1750 fins->op1 = fleft->op1;
1751 return RETRYFOLD;
1752 }
1753 }
1754 return NEXTFOLD;
1755#else
1756 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
1757#endif
1758}
1759
1505/* -- Reassociation ------------------------------------------------------- */ 1760/* -- Reassociation ------------------------------------------------------- */
1506 1761
1507LJFOLD(ADD ADD KINT) 1762LJFOLD(ADD ADD KINT)
@@ -1531,11 +1786,11 @@ LJFOLD(BOR BOR KINT64)
1531LJFOLD(BXOR BXOR KINT64) 1786LJFOLD(BXOR BXOR KINT64)
1532LJFOLDF(reassoc_intarith_k64) 1787LJFOLDF(reassoc_intarith_k64)
1533{ 1788{
1534#if LJ_HASFFI || LJ_64 1789#if LJ_HASFFI
1535 IRIns *irk = IR(fleft->op2); 1790 IRIns *irk = IR(fleft->op2);
1536 if (irk->o == IR_KINT64) { 1791 if (irk->o == IR_KINT64) {
1537 uint64_t k = kfold_int64arith(ir_k64(irk)->u64, 1792 uint64_t k = kfold_int64arith(J, ir_k64(irk)->u64, ir_k64(fright)->u64,
1538 ir_k64(fright)->u64, (IROp)fins->o); 1793 (IROp)fins->o);
1539 PHIBARRIER(fleft); 1794 PHIBARRIER(fleft);
1540 fins->op1 = fleft->op1; 1795 fins->op1 = fleft->op1;
1541 fins->op2 = (IRRef1)lj_ir_kint64(J, k); 1796 fins->op2 = (IRRef1)lj_ir_kint64(J, k);
@@ -1543,12 +1798,10 @@ LJFOLDF(reassoc_intarith_k64)
1543 } 1798 }
1544 return NEXTFOLD; 1799 return NEXTFOLD;
1545#else 1800#else
1546 UNUSED(J); lua_assert(0); return FAILFOLD; 1801 UNUSED(J); lj_assertJ(0, "FFI IR op without FFI"); return FAILFOLD;
1547#endif 1802#endif
1548} 1803}
1549 1804
1550LJFOLD(MIN MIN any)
1551LJFOLD(MAX MAX any)
1552LJFOLD(BAND BAND any) 1805LJFOLD(BAND BAND any)
1553LJFOLD(BOR BOR any) 1806LJFOLD(BOR BOR any)
1554LJFOLDF(reassoc_dup) 1807LJFOLDF(reassoc_dup)
@@ -1558,6 +1811,15 @@ LJFOLDF(reassoc_dup)
1558 return NEXTFOLD; 1811 return NEXTFOLD;
1559} 1812}
1560 1813
1814LJFOLD(MIN MIN any)
1815LJFOLD(MAX MAX any)
1816LJFOLDF(reassoc_dup_minmax)
1817{
1818 if (fins->op2 == fleft->op2)
1819 return LEFTFOLD; /* (a o b) o b ==> a o b */
1820 return NEXTFOLD;
1821}
1822
1561LJFOLD(BXOR BXOR any) 1823LJFOLD(BXOR BXOR any)
1562LJFOLDF(reassoc_bxor) 1824LJFOLDF(reassoc_bxor)
1563{ 1825{
@@ -1596,23 +1858,12 @@ LJFOLDF(reassoc_shift)
1596 return NEXTFOLD; 1858 return NEXTFOLD;
1597} 1859}
1598 1860
1599LJFOLD(MIN MIN KNUM)
1600LJFOLD(MAX MAX KNUM)
1601LJFOLD(MIN MIN KINT) 1861LJFOLD(MIN MIN KINT)
1602LJFOLD(MAX MAX KINT) 1862LJFOLD(MAX MAX KINT)
1603LJFOLDF(reassoc_minmax_k) 1863LJFOLDF(reassoc_minmax_k)
1604{ 1864{
1605 IRIns *irk = IR(fleft->op2); 1865 IRIns *irk = IR(fleft->op2);
1606 if (irk->o == IR_KNUM) { 1866 if (irk->o == IR_KINT) {
1607 lua_Number a = ir_knum(irk)->n;
1608 lua_Number y = lj_vm_foldarith(a, knumright, fins->o - IR_ADD);
1609 if (a == y) /* (x o k1) o k2 ==> x o k1, if (k1 o k2) == k1. */
1610 return LEFTFOLD;
1611 PHIBARRIER(fleft);
1612 fins->op1 = fleft->op1;
1613 fins->op2 = (IRRef1)lj_ir_knum(J, y);
1614 return RETRYFOLD; /* (x o k1) o k2 ==> x o (k1 o k2) */
1615 } else if (irk->o == IR_KINT) {
1616 int32_t a = irk->i; 1867 int32_t a = irk->i;
1617 int32_t y = kfold_intop(a, fright->i, fins->o); 1868 int32_t y = kfold_intop(a, fright->i, fins->o);
1618 if (a == y) /* (x o k1) o k2 ==> x o k1, if (k1 o k2) == k1. */ 1869 if (a == y) /* (x o k1) o k2 ==> x o k1, if (k1 o k2) == k1. */
@@ -1625,24 +1876,6 @@ LJFOLDF(reassoc_minmax_k)
1625 return NEXTFOLD; 1876 return NEXTFOLD;
1626} 1877}
1627 1878
1628LJFOLD(MIN MAX any)
1629LJFOLD(MAX MIN any)
1630LJFOLDF(reassoc_minmax_left)
1631{
1632 if (fins->op2 == fleft->op1 || fins->op2 == fleft->op2)
1633 return RIGHTFOLD; /* (b o1 a) o2 b ==> b; (a o1 b) o2 b ==> b */
1634 return NEXTFOLD;
1635}
1636
1637LJFOLD(MIN any MAX)
1638LJFOLD(MAX any MIN)
1639LJFOLDF(reassoc_minmax_right)
1640{
1641 if (fins->op1 == fright->op1 || fins->op1 == fright->op2)
1642 return LEFTFOLD; /* a o2 (a o1 b) ==> a; a o2 (b o1 a) ==> a */
1643 return NEXTFOLD;
1644}
1645
1646/* -- Array bounds check elimination -------------------------------------- */ 1879/* -- Array bounds check elimination -------------------------------------- */
1647 1880
1648/* Eliminate ABC across PHIs to handle t[i-1] forwarding case. 1881/* Eliminate ABC across PHIs to handle t[i-1] forwarding case.
@@ -1769,8 +2002,6 @@ LJFOLDF(comm_comp)
1769 2002
1770LJFOLD(BAND any any) 2003LJFOLD(BAND any any)
1771LJFOLD(BOR any any) 2004LJFOLD(BOR any any)
1772LJFOLD(MIN any any)
1773LJFOLD(MAX any any)
1774LJFOLDF(comm_dup) 2005LJFOLDF(comm_dup)
1775{ 2006{
1776 if (fins->op1 == fins->op2) /* x o x ==> x */ 2007 if (fins->op1 == fins->op2) /* x o x ==> x */
@@ -1778,6 +2009,15 @@ LJFOLDF(comm_dup)
1778 return fold_comm_swap(J); 2009 return fold_comm_swap(J);
1779} 2010}
1780 2011
2012LJFOLD(MIN any any)
2013LJFOLD(MAX any any)
2014LJFOLDF(comm_dup_minmax)
2015{
2016 if (fins->op1 == fins->op2) /* x o x ==> x */
2017 return LEFTFOLD;
2018 return NEXTFOLD;
2019}
2020
1781LJFOLD(BXOR any any) 2021LJFOLD(BXOR any any)
1782LJFOLDF(comm_bxor) 2022LJFOLDF(comm_bxor)
1783{ 2023{
@@ -1814,7 +2054,7 @@ LJFOLDF(merge_eqne_snew_kgc)
1814{ 2054{
1815 GCstr *kstr = ir_kstr(fright); 2055 GCstr *kstr = ir_kstr(fright);
1816 int32_t len = (int32_t)kstr->len; 2056 int32_t len = (int32_t)kstr->len;
1817 lua_assert(irt_isstr(fins->t)); 2057 lj_assertJ(irt_isstr(fins->t), "bad equality IR type");
1818 2058
1819#if LJ_TARGET_UNALIGNED 2059#if LJ_TARGET_UNALIGNED
1820#define FOLD_SNEW_MAX_LEN 4 /* Handle string lengths 0, 1, 2, 3, 4. */ 2060#define FOLD_SNEW_MAX_LEN 4 /* Handle string lengths 0, 1, 2, 3, 4. */
@@ -1878,7 +2118,7 @@ LJFOLD(HLOAD KKPTR)
1878LJFOLDF(kfold_hload_kkptr) 2118LJFOLDF(kfold_hload_kkptr)
1879{ 2119{
1880 UNUSED(J); 2120 UNUSED(J);
1881 lua_assert(ir_kptr(fleft) == niltvg(J2G(J))); 2121 lj_assertJ(ir_kptr(fleft) == niltvg(J2G(J)), "expected niltv");
1882 return TREF_NIL; 2122 return TREF_NIL;
1883} 2123}
1884 2124
@@ -1888,8 +2128,8 @@ LJFOLDX(lj_opt_fwd_hload)
1888LJFOLD(ULOAD any) 2128LJFOLD(ULOAD any)
1889LJFOLDX(lj_opt_fwd_uload) 2129LJFOLDX(lj_opt_fwd_uload)
1890 2130
1891LJFOLD(CALLL any IRCALL_lj_tab_len) 2131LJFOLD(ALEN any any)
1892LJFOLDX(lj_opt_fwd_tab_len) 2132LJFOLDX(lj_opt_fwd_alen)
1893 2133
1894/* Upvalue refs are really loads, but there are no corresponding stores. 2134/* Upvalue refs are really loads, but there are no corresponding stores.
1895** So CSE is ok for them, except for UREFO across a GC step (see below). 2135** So CSE is ok for them, except for UREFO across a GC step (see below).
@@ -1950,6 +2190,7 @@ LJFOLDF(fwd_href_tdup)
1950** an aliased table, as it may invalidate all of the pointers and fields. 2190** an aliased table, as it may invalidate all of the pointers and fields.
1951** Only HREF needs the NEWREF check -- AREF and HREFK already depend on 2191** Only HREF needs the NEWREF check -- AREF and HREFK already depend on
1952** FLOADs. And NEWREF itself is treated like a store (see below). 2192** FLOADs. And NEWREF itself is treated like a store (see below).
2193** LREF is constant (per trace) since coroutine switches are not inlined.
1953*/ 2194*/
1954LJFOLD(FLOAD TNEW IRFL_TAB_ASIZE) 2195LJFOLD(FLOAD TNEW IRFL_TAB_ASIZE)
1955LJFOLDF(fload_tab_tnew_asize) 2196LJFOLDF(fload_tab_tnew_asize)
@@ -2013,6 +2254,35 @@ LJFOLDF(fload_str_len_snew)
2013 return NEXTFOLD; 2254 return NEXTFOLD;
2014} 2255}
2015 2256
2257LJFOLD(FLOAD TOSTR IRFL_STR_LEN)
2258LJFOLDF(fload_str_len_tostr)
2259{
2260 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && fleft->op2 == IRTOSTR_CHAR)
2261 return INTFOLD(1);
2262 return NEXTFOLD;
2263}
2264
2265LJFOLD(FLOAD any IRFL_SBUF_W)
2266LJFOLD(FLOAD any IRFL_SBUF_E)
2267LJFOLD(FLOAD any IRFL_SBUF_B)
2268LJFOLD(FLOAD any IRFL_SBUF_L)
2269LJFOLD(FLOAD any IRFL_SBUF_REF)
2270LJFOLD(FLOAD any IRFL_SBUF_R)
2271LJFOLDF(fload_sbuf)
2272{
2273 TRef tr = lj_opt_fwd_fload(J);
2274 return lj_opt_fwd_sbuf(J, tref_ref(tr)) ? tr : EMITFOLD;
2275}
2276
2277/* The fast function ID of function objects is immutable. */
2278LJFOLD(FLOAD KGC IRFL_FUNC_FFID)
2279LJFOLDF(fload_func_ffid_kgc)
2280{
2281 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD))
2282 return INTFOLD((int32_t)ir_kfunc(fleft)->c.ffid);
2283 return NEXTFOLD;
2284}
2285
2016/* The C type ID of cdata objects is immutable. */ 2286/* The C type ID of cdata objects is immutable. */
2017LJFOLD(FLOAD KGC IRFL_CDATA_CTYPEID) 2287LJFOLD(FLOAD KGC IRFL_CDATA_CTYPEID)
2018LJFOLDF(fload_cdata_typeid_kgc) 2288LJFOLDF(fload_cdata_typeid_kgc)
@@ -2059,6 +2329,8 @@ LJFOLDF(fload_cdata_ptr_int64_cnew)
2059} 2329}
2060 2330
2061LJFOLD(FLOAD any IRFL_STR_LEN) 2331LJFOLD(FLOAD any IRFL_STR_LEN)
2332LJFOLD(FLOAD any IRFL_FUNC_ENV)
2333LJFOLD(FLOAD any IRFL_THREAD_ENV)
2062LJFOLD(FLOAD any IRFL_CDATA_CTYPEID) 2334LJFOLD(FLOAD any IRFL_CDATA_CTYPEID)
2063LJFOLD(FLOAD any IRFL_CDATA_PTR) 2335LJFOLD(FLOAD any IRFL_CDATA_PTR)
2064LJFOLD(FLOAD any IRFL_CDATA_INT) 2336LJFOLD(FLOAD any IRFL_CDATA_INT)
@@ -2078,7 +2350,7 @@ LJFOLDF(fwd_sload)
2078 TRef tr = lj_opt_cse(J); 2350 TRef tr = lj_opt_cse(J);
2079 return tref_ref(tr) < J->chain[IR_RETF] ? EMITFOLD : tr; 2351 return tref_ref(tr) < J->chain[IR_RETF] ? EMITFOLD : tr;
2080 } else { 2352 } else {
2081 lua_assert(J->slot[fins->op1] != 0); 2353 lj_assertJ(J->slot[fins->op1] != 0, "uninitialized slot accessed");
2082 return J->slot[fins->op1]; 2354 return J->slot[fins->op1];
2083 } 2355 }
2084} 2356}
@@ -2135,6 +2407,17 @@ LJFOLDF(barrier_tnew_tdup)
2135 return DROPFOLD; 2407 return DROPFOLD;
2136} 2408}
2137 2409
2410/* -- Profiling ----------------------------------------------------------- */
2411
2412LJFOLD(PROF any any)
2413LJFOLDF(prof)
2414{
2415 IRRef ref = J->chain[IR_PROF];
2416 if (ref+1 == J->cur.nins) /* Drop neighbouring IR_PROF. */
2417 return ref;
2418 return EMITFOLD;
2419}
2420
2138/* -- Stores and allocations ---------------------------------------------- */ 2421/* -- Stores and allocations ---------------------------------------------- */
2139 2422
2140/* Stores and allocations cannot be folded or passed on to CSE in general. 2423/* Stores and allocations cannot be folded or passed on to CSE in general.
@@ -2157,8 +2440,10 @@ LJFOLD(XSTORE any any)
2157LJFOLDX(lj_opt_dse_xstore) 2440LJFOLDX(lj_opt_dse_xstore)
2158 2441
2159LJFOLD(NEWREF any any) /* Treated like a store. */ 2442LJFOLD(NEWREF any any) /* Treated like a store. */
2160LJFOLD(CALLS any any) 2443LJFOLD(TMPREF any any)
2444LJFOLD(CALLA any any)
2161LJFOLD(CALLL any any) /* Safeguard fallback. */ 2445LJFOLD(CALLL any any) /* Safeguard fallback. */
2446LJFOLD(CALLS any any)
2162LJFOLD(CALLXS any any) 2447LJFOLD(CALLXS any any)
2163LJFOLD(XBAR) 2448LJFOLD(XBAR)
2164LJFOLD(RETF any any) /* Modifies BASE. */ 2449LJFOLD(RETF any any) /* Modifies BASE. */
@@ -2191,8 +2476,9 @@ TRef LJ_FASTCALL lj_opt_fold(jit_State *J)
2191 IRRef ref; 2476 IRRef ref;
2192 2477
2193 if (LJ_UNLIKELY((J->flags & JIT_F_OPT_MASK) != JIT_F_OPT_DEFAULT)) { 2478 if (LJ_UNLIKELY((J->flags & JIT_F_OPT_MASK) != JIT_F_OPT_DEFAULT)) {
2194 lua_assert(((JIT_F_OPT_FOLD|JIT_F_OPT_FWD|JIT_F_OPT_CSE|JIT_F_OPT_DSE) | 2479 lj_assertJ(((JIT_F_OPT_FOLD|JIT_F_OPT_FWD|JIT_F_OPT_CSE|JIT_F_OPT_DSE) |
2195 JIT_F_OPT_DEFAULT) == JIT_F_OPT_DEFAULT); 2480 JIT_F_OPT_DEFAULT) == JIT_F_OPT_DEFAULT,
2481 "bad JIT_F_OPT_DEFAULT");
2196 /* Folding disabled? Chain to CSE, but not for loads/stores/allocs. */ 2482 /* Folding disabled? Chain to CSE, but not for loads/stores/allocs. */
2197 if (!(J->flags & JIT_F_OPT_FOLD) && irm_kind(lj_ir_mode[fins->o]) == IRM_N) 2483 if (!(J->flags & JIT_F_OPT_FOLD) && irm_kind(lj_ir_mode[fins->o]) == IRM_N)
2198 return lj_opt_cse(J); 2484 return lj_opt_cse(J);
@@ -2217,10 +2503,14 @@ retry:
2217 if (fins->op1 >= J->cur.nk) { 2503 if (fins->op1 >= J->cur.nk) {
2218 key += (uint32_t)IR(fins->op1)->o << 10; 2504 key += (uint32_t)IR(fins->op1)->o << 10;
2219 *fleft = *IR(fins->op1); 2505 *fleft = *IR(fins->op1);
2506 if (fins->op1 < REF_TRUE)
2507 fleft[1] = IR(fins->op1)[1];
2220 } 2508 }
2221 if (fins->op2 >= J->cur.nk) { 2509 if (fins->op2 >= J->cur.nk) {
2222 key += (uint32_t)IR(fins->op2)->o; 2510 key += (uint32_t)IR(fins->op2)->o;
2223 *fright = *IR(fins->op2); 2511 *fright = *IR(fins->op2);
2512 if (fins->op2 < REF_TRUE)
2513 fright[1] = IR(fins->op2)[1];
2224 } else { 2514 } else {
2225 key += (fins->op2 & 0x3ffu); /* Literal mask. Must include IRCONV_*MASK. */ 2515 key += (fins->op2 & 0x3ffu); /* Literal mask. Must include IRCONV_*MASK. */
2226 } 2516 }
@@ -2250,7 +2540,7 @@ retry:
2250 return lj_ir_kint(J, fins->i); 2540 return lj_ir_kint(J, fins->i);
2251 if (ref == FAILFOLD) 2541 if (ref == FAILFOLD)
2252 lj_trace_err(J, LJ_TRERR_GFAIL); 2542 lj_trace_err(J, LJ_TRERR_GFAIL);
2253 lua_assert(ref == DROPFOLD); 2543 lj_assertJ(ref == DROPFOLD, "bad fold result");
2254 return REF_DROP; 2544 return REF_DROP;
2255} 2545}
2256 2546
diff --git a/src/lj_opt_loop.c b/src/lj_opt_loop.c
index 8a0e611f..01c2b306 100644
--- a/src/lj_opt_loop.c
+++ b/src/lj_opt_loop.c
@@ -11,7 +11,7 @@
11#if LJ_HASJIT 11#if LJ_HASJIT
12 12
13#include "lj_err.h" 13#include "lj_err.h"
14#include "lj_str.h" 14#include "lj_buf.h"
15#include "lj_ir.h" 15#include "lj_ir.h"
16#include "lj_jit.h" 16#include "lj_jit.h"
17#include "lj_iropt.h" 17#include "lj_iropt.h"
@@ -225,6 +225,7 @@ static void loop_subst_snap(jit_State *J, SnapShot *osnap,
225 /* Setup new snapshot. */ 225 /* Setup new snapshot. */
226 snap->mapofs = (uint32_t)nmapofs; 226 snap->mapofs = (uint32_t)nmapofs;
227 snap->ref = (IRRef1)J->cur.nins; 227 snap->ref = (IRRef1)J->cur.nins;
228 snap->mcofs = 0;
228 snap->nslots = nslots; 229 snap->nslots = nslots;
229 snap->topslot = osnap->topslot; 230 snap->topslot = osnap->topslot;
230 snap->count = 0; 231 snap->count = 0;
@@ -254,9 +255,16 @@ static void loop_subst_snap(jit_State *J, SnapShot *osnap,
254 J->cur.nsnapmap = (uint32_t)(nmap - J->cur.snapmap); 255 J->cur.nsnapmap = (uint32_t)(nmap - J->cur.snapmap);
255} 256}
256 257
258typedef struct LoopState {
259 jit_State *J;
260 IRRef1 *subst;
261 MSize sizesubst;
262} LoopState;
263
257/* Unroll loop. */ 264/* Unroll loop. */
258static void loop_unroll(jit_State *J) 265static void loop_unroll(LoopState *lps)
259{ 266{
267 jit_State *J = lps->J;
260 IRRef1 phi[LJ_MAX_PHI]; 268 IRRef1 phi[LJ_MAX_PHI];
261 uint32_t nphi = 0; 269 uint32_t nphi = 0;
262 IRRef1 *subst; 270 IRRef1 *subst;
@@ -265,13 +273,13 @@ static void loop_unroll(jit_State *J)
265 SnapEntry *loopmap, *psentinel; 273 SnapEntry *loopmap, *psentinel;
266 IRRef ins, invar; 274 IRRef ins, invar;
267 275
268 /* Use temp buffer for substitution table. 276 /* Allocate substitution table.
269 ** Only non-constant refs in [REF_BIAS,invar) are valid indexes. 277 ** Only non-constant refs in [REF_BIAS,invar) are valid indexes.
270 ** Caveat: don't call into the VM or run the GC or the buffer may be gone.
271 */ 278 */
272 invar = J->cur.nins; 279 invar = J->cur.nins;
273 subst = (IRRef1 *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, 280 lps->sizesubst = invar - REF_BIAS;
274 (invar-REF_BIAS)*sizeof(IRRef1)) - REF_BIAS; 281 lps->subst = lj_mem_newvec(J->L, lps->sizesubst, IRRef1);
282 subst = lps->subst - REF_BIAS;
275 subst[REF_BASE] = REF_BASE; 283 subst[REF_BASE] = REF_BASE;
276 284
277 /* LOOP separates the pre-roll from the loop body. */ 285 /* LOOP separates the pre-roll from the loop body. */
@@ -292,7 +300,8 @@ static void loop_unroll(jit_State *J)
292 loopmap = &J->cur.snapmap[loopsnap->mapofs]; 300 loopmap = &J->cur.snapmap[loopsnap->mapofs];
293 /* The PC of snapshot #0 and the loop snapshot must match. */ 301 /* The PC of snapshot #0 and the loop snapshot must match. */
294 psentinel = &loopmap[loopsnap->nent]; 302 psentinel = &loopmap[loopsnap->nent];
295 lua_assert(*psentinel == J->cur.snapmap[J->cur.snap[0].nent]); 303 lj_assertJ(*psentinel == J->cur.snapmap[J->cur.snap[0].nent],
304 "mismatched PC for loop snapshot");
296 *psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */ 305 *psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */
297 306
298 /* Start substitution with snapshot #1 (#0 is empty for root traces). */ 307 /* Start substitution with snapshot #1 (#0 is empty for root traces). */
@@ -345,10 +354,12 @@ static void loop_unroll(jit_State *J)
345 irr = IR(ref); 354 irr = IR(ref);
346 goto phiconv; 355 goto phiconv;
347 } 356 }
348 } else if (ref != REF_DROP && irr->o == IR_CONV && 357 } else if (ref != REF_DROP && ref > invar &&
349 ref > invar && irr->op1 < invar) { 358 ((irr->o == IR_CONV && irr->op1 < invar) ||
350 /* May need an extra PHI for a CONV. */ 359 (irr->o == IR_ALEN && irr->op2 < invar &&
351 ref = irr->op1; 360 irr->op2 != REF_NIL))) {
361 /* May need an extra PHI for a CONV or ALEN hint. */
362 ref = irr->o == IR_CONV ? irr->op1 : irr->op2;
352 irr = IR(ref); 363 irr = IR(ref);
353 phiconv: 364 phiconv:
354 if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) { 365 if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) {
@@ -363,7 +374,7 @@ static void loop_unroll(jit_State *J)
363 } 374 }
364 if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */ 375 if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */
365 J->cur.nsnapmap = (uint32_t)J->cur.snap[--J->cur.nsnap].mapofs; 376 J->cur.nsnapmap = (uint32_t)J->cur.snap[--J->cur.nsnap].mapofs;
366 lua_assert(J->cur.nsnapmap <= J->sizesnapmap); 377 lj_assertJ(J->cur.nsnapmap <= J->sizesnapmap, "bad snapshot map index");
367 *psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */ 378 *psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */
368 379
369 loop_emit_phi(J, subst, phi, nphi, onsnap); 380 loop_emit_phi(J, subst, phi, nphi, onsnap);
@@ -396,7 +407,7 @@ static void loop_undo(jit_State *J, IRRef ins, SnapNo nsnap, MSize nsnapmap)
396static TValue *cploop_opt(lua_State *L, lua_CFunction dummy, void *ud) 407static TValue *cploop_opt(lua_State *L, lua_CFunction dummy, void *ud)
397{ 408{
398 UNUSED(L); UNUSED(dummy); 409 UNUSED(L); UNUSED(dummy);
399 loop_unroll((jit_State *)ud); 410 loop_unroll((LoopState *)ud);
400 return NULL; 411 return NULL;
401} 412}
402 413
@@ -406,7 +417,13 @@ int lj_opt_loop(jit_State *J)
406 IRRef nins = J->cur.nins; 417 IRRef nins = J->cur.nins;
407 SnapNo nsnap = J->cur.nsnap; 418 SnapNo nsnap = J->cur.nsnap;
408 MSize nsnapmap = J->cur.nsnapmap; 419 MSize nsnapmap = J->cur.nsnapmap;
409 int errcode = lj_vm_cpcall(J->L, NULL, J, cploop_opt); 420 LoopState lps;
421 int errcode;
422 lps.J = J;
423 lps.subst = NULL;
424 lps.sizesubst = 0;
425 errcode = lj_vm_cpcall(J->L, NULL, &lps, cploop_opt);
426 lj_mem_freevec(J2G(J), lps.subst, lps.sizesubst, IRRef1);
410 if (LJ_UNLIKELY(errcode)) { 427 if (LJ_UNLIKELY(errcode)) {
411 lua_State *L = J->L; 428 lua_State *L = J->L;
412 if (errcode == LUA_ERRRUN && tvisnumber(L->top-1)) { /* Trace error? */ 429 if (errcode == LUA_ERRRUN && tvisnumber(L->top-1)) { /* Trace error? */
diff --git a/src/lj_opt_mem.c b/src/lj_opt_mem.c
index dc74a06d..351d958c 100644
--- a/src/lj_opt_mem.c
+++ b/src/lj_opt_mem.c
@@ -17,12 +17,14 @@
17#include "lj_ir.h" 17#include "lj_ir.h"
18#include "lj_jit.h" 18#include "lj_jit.h"
19#include "lj_iropt.h" 19#include "lj_iropt.h"
20#include "lj_ircall.h"
21#include "lj_dispatch.h"
20 22
21/* Some local macros to save typing. Undef'd at the end. */ 23/* Some local macros to save typing. Undef'd at the end. */
22#define IR(ref) (&J->cur.ir[(ref)]) 24#define IR(ref) (&J->cur.ir[(ref)])
23#define fins (&J->fold.ins) 25#define fins (&J->fold.ins)
24#define fleft (&J->fold.left) 26#define fleft (J->fold.left)
25#define fright (&J->fold.right) 27#define fright (J->fold.right)
26 28
27/* 29/*
28** Caveat #1: return value is not always a TRef -- only use with tref_ref(). 30** Caveat #1: return value is not always a TRef -- only use with tref_ref().
@@ -55,8 +57,8 @@ static AliasRet aa_table(jit_State *J, IRRef ta, IRRef tb)
55{ 57{
56 IRIns *taba = IR(ta), *tabb = IR(tb); 58 IRIns *taba = IR(ta), *tabb = IR(tb);
57 int newa, newb; 59 int newa, newb;
58 lua_assert(ta != tb); 60 lj_assertJ(ta != tb, "bad usage");
59 lua_assert(irt_istab(taba->t) && irt_istab(tabb->t)); 61 lj_assertJ(irt_istab(taba->t) && irt_istab(tabb->t), "bad usage");
60 /* Disambiguate new allocations. */ 62 /* Disambiguate new allocations. */
61 newa = (taba->o == IR_TNEW || taba->o == IR_TDUP); 63 newa = (taba->o == IR_TNEW || taba->o == IR_TDUP);
62 newb = (tabb->o == IR_TNEW || tabb->o == IR_TDUP); 64 newb = (tabb->o == IR_TNEW || tabb->o == IR_TDUP);
@@ -70,6 +72,34 @@ static AliasRet aa_table(jit_State *J, IRRef ta, IRRef tb)
70 return aa_escape(J, taba, tabb); 72 return aa_escape(J, taba, tabb);
71} 73}
72 74
75/* Check whether there's no aliasing table.clear. */
76static int fwd_aa_tab_clear(jit_State *J, IRRef lim, IRRef ta)
77{
78 IRRef ref = J->chain[IR_CALLS];
79 while (ref > lim) {
80 IRIns *calls = IR(ref);
81 if (calls->op2 == IRCALL_lj_tab_clear &&
82 (ta == calls->op1 || aa_table(J, ta, calls->op1) != ALIAS_NO))
83 return 0; /* Conflict. */
84 ref = calls->prev;
85 }
86 return 1; /* No conflict. Can safely FOLD/CSE. */
87}
88
89/* Check whether there's no aliasing NEWREF/table.clear for the left operand. */
90int LJ_FASTCALL lj_opt_fwd_tptr(jit_State *J, IRRef lim)
91{
92 IRRef ta = fins->op1;
93 IRRef ref = J->chain[IR_NEWREF];
94 while (ref > lim) {
95 IRIns *newref = IR(ref);
96 if (ta == newref->op1 || aa_table(J, ta, newref->op1) != ALIAS_NO)
97 return 0; /* Conflict. */
98 ref = newref->prev;
99 }
100 return fwd_aa_tab_clear(J, lim, ta);
101}
102
73/* Alias analysis for array and hash access using key-based disambiguation. */ 103/* Alias analysis for array and hash access using key-based disambiguation. */
74static AliasRet aa_ahref(jit_State *J, IRIns *refa, IRIns *refb) 104static AliasRet aa_ahref(jit_State *J, IRIns *refa, IRIns *refb)
75{ 105{
@@ -98,7 +128,7 @@ static AliasRet aa_ahref(jit_State *J, IRIns *refa, IRIns *refb)
98 /* Disambiguate array references based on index arithmetic. */ 128 /* Disambiguate array references based on index arithmetic. */
99 int32_t ofsa = 0, ofsb = 0; 129 int32_t ofsa = 0, ofsb = 0;
100 IRRef basea = ka, baseb = kb; 130 IRRef basea = ka, baseb = kb;
101 lua_assert(refb->o == IR_AREF); 131 lj_assertJ(refb->o == IR_AREF, "expected AREF");
102 /* Gather base and offset from t[base] or t[base+-ofs]. */ 132 /* Gather base and offset from t[base] or t[base+-ofs]. */
103 if (keya->o == IR_ADD && irref_isk(keya->op2)) { 133 if (keya->o == IR_ADD && irref_isk(keya->op2)) {
104 basea = keya->op1; 134 basea = keya->op1;
@@ -116,8 +146,9 @@ static AliasRet aa_ahref(jit_State *J, IRIns *refa, IRIns *refb)
116 return ALIAS_NO; /* t[base+-o1] vs. t[base+-o2] and o1 != o2. */ 146 return ALIAS_NO; /* t[base+-o1] vs. t[base+-o2] and o1 != o2. */
117 } else { 147 } else {
118 /* Disambiguate hash references based on the type of their keys. */ 148 /* Disambiguate hash references based on the type of their keys. */
119 lua_assert((refa->o==IR_HREF || refa->o==IR_HREFK || refa->o==IR_NEWREF) && 149 lj_assertJ((refa->o==IR_HREF || refa->o==IR_HREFK || refa->o==IR_NEWREF) &&
120 (refb->o==IR_HREF || refb->o==IR_HREFK || refb->o==IR_NEWREF)); 150 (refb->o==IR_HREF || refb->o==IR_HREFK || refb->o==IR_NEWREF),
151 "bad xREF IR op %d or %d", refa->o, refb->o);
121 if (!irt_sametype(keya->t, keyb->t)) 152 if (!irt_sametype(keya->t, keyb->t))
122 return ALIAS_NO; /* Different key types. */ 153 return ALIAS_NO; /* Different key types. */
123 } 154 }
@@ -151,7 +182,8 @@ static TRef fwd_ahload(jit_State *J, IRRef xref)
151 IRIns *ir = (xr->o == IR_HREFK || xr->o == IR_AREF) ? IR(xr->op1) : xr; 182 IRIns *ir = (xr->o == IR_HREFK || xr->o == IR_AREF) ? IR(xr->op1) : xr;
152 IRRef tab = ir->op1; 183 IRRef tab = ir->op1;
153 ir = IR(tab); 184 ir = IR(tab);
154 if (ir->o == IR_TNEW || (ir->o == IR_TDUP && irref_isk(xr->op2))) { 185 if ((ir->o == IR_TNEW || (ir->o == IR_TDUP && irref_isk(xr->op2))) &&
186 fwd_aa_tab_clear(J, tab, tab)) {
155 /* A NEWREF with a number key may end up pointing to the array part. 187 /* A NEWREF with a number key may end up pointing to the array part.
156 ** But it's referenced from HSTORE and not found in the ASTORE chain. 188 ** But it's referenced from HSTORE and not found in the ASTORE chain.
157 ** Or a NEWREF may rehash the table and move unrelated number keys. 189 ** Or a NEWREF may rehash the table and move unrelated number keys.
@@ -272,7 +304,7 @@ TRef LJ_FASTCALL lj_opt_fwd_hrefk(jit_State *J)
272 while (ref > tab) { 304 while (ref > tab) {
273 IRIns *newref = IR(ref); 305 IRIns *newref = IR(ref);
274 if (tab == newref->op1) { 306 if (tab == newref->op1) {
275 if (fright->op1 == newref->op2) 307 if (fright->op1 == newref->op2 && fwd_aa_tab_clear(J, ref, tab))
276 return ref; /* Forward from NEWREF. */ 308 return ref; /* Forward from NEWREF. */
277 else 309 else
278 goto docse; 310 goto docse;
@@ -282,7 +314,7 @@ TRef LJ_FASTCALL lj_opt_fwd_hrefk(jit_State *J)
282 ref = newref->prev; 314 ref = newref->prev;
283 } 315 }
284 /* No conflicting NEWREF: key location unchanged for HREFK of TDUP. */ 316 /* No conflicting NEWREF: key location unchanged for HREFK of TDUP. */
285 if (IR(tab)->o == IR_TDUP) 317 if (IR(tab)->o == IR_TDUP && fwd_aa_tab_clear(J, tab, tab))
286 fins->t.irt &= ~IRT_GUARD; /* Drop HREFK guard. */ 318 fins->t.irt &= ~IRT_GUARD; /* Drop HREFK guard. */
287docse: 319docse:
288 return CSEFOLD; 320 return CSEFOLD;
@@ -316,20 +348,6 @@ int LJ_FASTCALL lj_opt_fwd_href_nokey(jit_State *J)
316 return 1; /* No conflict. Can fold to niltv. */ 348 return 1; /* No conflict. Can fold to niltv. */
317} 349}
318 350
319/* Check whether there's no aliasing NEWREF for the left operand. */
320int LJ_FASTCALL lj_opt_fwd_tptr(jit_State *J, IRRef lim)
321{
322 IRRef ta = fins->op1;
323 IRRef ref = J->chain[IR_NEWREF];
324 while (ref > lim) {
325 IRIns *newref = IR(ref);
326 if (ta == newref->op1 || aa_table(J, ta, newref->op1) != ALIAS_NO)
327 return 0; /* Conflict. */
328 ref = newref->prev;
329 }
330 return 1; /* No conflict. Can safely FOLD/CSE. */
331}
332
333/* ASTORE/HSTORE elimination. */ 351/* ASTORE/HSTORE elimination. */
334TRef LJ_FASTCALL lj_opt_dse_ahstore(jit_State *J) 352TRef LJ_FASTCALL lj_opt_dse_ahstore(jit_State *J)
335{ 353{
@@ -353,9 +371,12 @@ TRef LJ_FASTCALL lj_opt_dse_ahstore(jit_State *J)
353 /* Different value: try to eliminate the redundant store. */ 371 /* Different value: try to eliminate the redundant store. */
354 if (ref > J->chain[IR_LOOP]) { /* Quick check to avoid crossing LOOP. */ 372 if (ref > J->chain[IR_LOOP]) { /* Quick check to avoid crossing LOOP. */
355 IRIns *ir; 373 IRIns *ir;
356 /* Check for any intervening guards (includes conflicting loads). */ 374 /* Check for any intervening guards (includes conflicting loads).
375 ** Note that lj_tab_keyindex and lj_vm_next don't need guards,
376 ** since they are followed by at least one guarded VLOAD.
377 */
357 for (ir = IR(J->cur.nins-1); ir > store; ir--) 378 for (ir = IR(J->cur.nins-1); ir > store; ir--)
358 if (irt_isguard(ir->t) || ir->o == IR_CALLL) 379 if (irt_isguard(ir->t) || ir->o == IR_ALEN)
359 goto doemit; /* No elimination possible. */ 380 goto doemit; /* No elimination possible. */
360 /* Remove redundant store from chain and replace with NOP. */ 381 /* Remove redundant store from chain and replace with NOP. */
361 *refp = store->prev; 382 *refp = store->prev;
@@ -370,6 +391,67 @@ doemit:
370 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */ 391 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */
371} 392}
372 393
394/* ALEN forwarding. */
395TRef LJ_FASTCALL lj_opt_fwd_alen(jit_State *J)
396{
397 IRRef tab = fins->op1; /* Table reference. */
398 IRRef lim = tab; /* Search limit. */
399 IRRef ref;
400
401 /* Search for conflicting HSTORE with numeric key. */
402 ref = J->chain[IR_HSTORE];
403 while (ref > lim) {
404 IRIns *store = IR(ref);
405 IRIns *href = IR(store->op1);
406 IRIns *key = IR(href->op2);
407 if (irt_isnum(key->o == IR_KSLOT ? IR(key->op1)->t : key->t)) {
408 lim = ref; /* Conflicting store found, limits search for ALEN. */
409 break;
410 }
411 ref = store->prev;
412 }
413
414 /* Try to find a matching ALEN. */
415 ref = J->chain[IR_ALEN];
416 while (ref > lim) {
417 /* CSE for ALEN only depends on the table, not the hint. */
418 if (IR(ref)->op1 == tab) {
419 IRRef sref;
420
421 /* Search for aliasing table.clear. */
422 if (!fwd_aa_tab_clear(J, ref, tab))
423 break;
424
425 /* Search for hint-forwarding or conflicting store. */
426 sref = J->chain[IR_ASTORE];
427 while (sref > ref) {
428 IRIns *store = IR(sref);
429 IRIns *aref = IR(store->op1);
430 IRIns *fref = IR(aref->op1);
431 if (tab == fref->op1) { /* ASTORE to the same table. */
432 /* Detect t[#t+1] = x idiom for push. */
433 IRIns *idx = IR(aref->op2);
434 if (!irt_isnil(store->t) &&
435 idx->o == IR_ADD && idx->op1 == ref &&
436 IR(idx->op2)->o == IR_KINT && IR(idx->op2)->i == 1) {
437 /* Note: this requires an extra PHI check in loop unroll. */
438 fins->op2 = aref->op2; /* Set ALEN hint. */
439 }
440 goto doemit; /* Conflicting store, possibly giving a hint. */
441 } else if (aa_table(J, tab, fref->op1) != ALIAS_NO) {
442 goto doemit; /* Conflicting store. */
443 }
444 sref = store->prev;
445 }
446
447 return ref; /* Plain ALEN forwarding. */
448 }
449 ref = IR(ref)->prev;
450 }
451doemit:
452 return EMITFOLD;
453}
454
373/* -- ULOAD forwarding ---------------------------------------------------- */ 455/* -- ULOAD forwarding ---------------------------------------------------- */
374 456
375/* The current alias analysis for upvalues is very simplistic. It only 457/* The current alias analysis for upvalues is very simplistic. It only
@@ -419,7 +501,6 @@ TRef LJ_FASTCALL lj_opt_fwd_uload(jit_State *J)
419 501
420cselim: 502cselim:
421 /* Try to find a matching load. Below the conflicting store, if any. */ 503 /* Try to find a matching load. Below the conflicting store, if any. */
422
423 ref = J->chain[IR_ULOAD]; 504 ref = J->chain[IR_ULOAD];
424 while (ref > lim) { 505 while (ref > lim) {
425 IRIns *ir = IR(ref); 506 IRIns *ir = IR(ref);
@@ -549,8 +630,9 @@ TRef LJ_FASTCALL lj_opt_dse_fstore(jit_State *J)
549 goto doemit; 630 goto doemit;
550 break; /* Otherwise continue searching. */ 631 break; /* Otherwise continue searching. */
551 case ALIAS_MUST: 632 case ALIAS_MUST:
552 if (store->op2 == val) /* Same value: drop the new store. */ 633 if (store->op2 == val &&
553 return DROPFOLD; 634 !(xr->op2 >= IRFL_SBUF_W && xr->op2 <= IRFL_SBUF_R))
635 return DROPFOLD; /* Same value: drop the new store. */
554 /* Different value: try to eliminate the redundant store. */ 636 /* Different value: try to eliminate the redundant store. */
555 if (ref > J->chain[IR_LOOP]) { /* Quick check to avoid crossing LOOP. */ 637 if (ref > J->chain[IR_LOOP]) { /* Quick check to avoid crossing LOOP. */
556 IRIns *ir; 638 IRIns *ir;
@@ -571,6 +653,29 @@ doemit:
571 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */ 653 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */
572} 654}
573 655
656/* Check whether there's no aliasing buffer op between IRFL_SBUF_*. */
657int LJ_FASTCALL lj_opt_fwd_sbuf(jit_State *J, IRRef lim)
658{
659 IRRef ref;
660 if (J->chain[IR_BUFPUT] > lim)
661 return 0; /* Conflict. */
662 ref = J->chain[IR_CALLS];
663 while (ref > lim) {
664 IRIns *ir = IR(ref);
665 if (ir->op2 >= IRCALL_lj_strfmt_putint && ir->op2 < IRCALL_lj_buf_tostr)
666 return 0; /* Conflict. */
667 ref = ir->prev;
668 }
669 ref = J->chain[IR_CALLL];
670 while (ref > lim) {
671 IRIns *ir = IR(ref);
672 if (ir->op2 >= IRCALL_lj_strfmt_putint && ir->op2 < IRCALL_lj_buf_tostr)
673 return 0; /* Conflict. */
674 ref = ir->prev;
675 }
676 return 1; /* No conflict. Can safely FOLD/CSE. */
677}
678
574/* -- XLOAD forwarding and XSTORE elimination ----------------------------- */ 679/* -- XLOAD forwarding and XSTORE elimination ----------------------------- */
575 680
576/* Find cdata allocation for a reference (if any). */ 681/* Find cdata allocation for a reference (if any). */
@@ -822,35 +927,6 @@ doemit:
822 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */ 927 return EMITFOLD; /* Otherwise we have a conflict or simply no match. */
823} 928}
824 929
825/* -- Forwarding of lj_tab_len -------------------------------------------- */
826
827/* This is rather simplistic right now, but better than nothing. */
828TRef LJ_FASTCALL lj_opt_fwd_tab_len(jit_State *J)
829{
830 IRRef tab = fins->op1; /* Table reference. */
831 IRRef lim = tab; /* Search limit. */
832 IRRef ref;
833
834 /* Any ASTORE is a conflict and limits the search. */
835 if (J->chain[IR_ASTORE] > lim) lim = J->chain[IR_ASTORE];
836
837 /* Search for conflicting HSTORE with numeric key. */
838 ref = J->chain[IR_HSTORE];
839 while (ref > lim) {
840 IRIns *store = IR(ref);
841 IRIns *href = IR(store->op1);
842 IRIns *key = IR(href->op2);
843 if (irt_isnum(key->o == IR_KSLOT ? IR(key->op1)->t : key->t)) {
844 lim = ref; /* Conflicting store found, limits search for TLEN. */
845 break;
846 }
847 ref = store->prev;
848 }
849
850 /* Try to find a matching load. Below the conflicting store, if any. */
851 return lj_opt_cselim(J, lim);
852}
853
854/* -- ASTORE/HSTORE previous type analysis -------------------------------- */ 930/* -- ASTORE/HSTORE previous type analysis -------------------------------- */
855 931
856/* Check whether the previous value for a table store is non-nil. 932/* Check whether the previous value for a table store is non-nil.
diff --git a/src/lj_opt_narrow.c b/src/lj_opt_narrow.c
index 700c23d4..1172df2b 100644
--- a/src/lj_opt_narrow.c
+++ b/src/lj_opt_narrow.c
@@ -372,17 +372,17 @@ static IRRef narrow_conv_emit(jit_State *J, NarrowConv *nc)
372 } else if (op == NARROW_CONV) { 372 } else if (op == NARROW_CONV) {
373 *sp++ = emitir_raw(convot, ref, convop2); /* Raw emit avoids a loop. */ 373 *sp++ = emitir_raw(convot, ref, convop2); /* Raw emit avoids a loop. */
374 } else if (op == NARROW_SEXT) { 374 } else if (op == NARROW_SEXT) {
375 lua_assert(sp >= nc->stack+1); 375 lj_assertJ(sp >= nc->stack+1, "stack underflow");
376 sp[-1] = emitir(IRT(IR_CONV, IRT_I64), sp[-1], 376 sp[-1] = emitir(IRT(IR_CONV, IRT_I64), sp[-1],
377 (IRT_I64<<5)|IRT_INT|IRCONV_SEXT); 377 (IRT_I64<<5)|IRT_INT|IRCONV_SEXT);
378 } else if (op == NARROW_INT) { 378 } else if (op == NARROW_INT) {
379 lua_assert(next < last); 379 lj_assertJ(next < last, "missing arg to NARROW_INT");
380 *sp++ = nc->t == IRT_I64 ? 380 *sp++ = nc->t == IRT_I64 ?
381 lj_ir_kint64(J, (int64_t)(int32_t)*next++) : 381 lj_ir_kint64(J, (int64_t)(int32_t)*next++) :
382 lj_ir_kint(J, *next++); 382 lj_ir_kint(J, *next++);
383 } else { /* Regular IROpT. Pops two operands and pushes one result. */ 383 } else { /* Regular IROpT. Pops two operands and pushes one result. */
384 IRRef mode = nc->mode; 384 IRRef mode = nc->mode;
385 lua_assert(sp >= nc->stack+2); 385 lj_assertJ(sp >= nc->stack+2, "stack underflow");
386 sp--; 386 sp--;
387 /* Omit some overflow checks for array indexing. See comments above. */ 387 /* Omit some overflow checks for array indexing. See comments above. */
388 if ((mode & IRCONV_CONVMASK) == IRCONV_INDEX) { 388 if ((mode & IRCONV_CONVMASK) == IRCONV_INDEX) {
@@ -398,7 +398,7 @@ static IRRef narrow_conv_emit(jit_State *J, NarrowConv *nc)
398 narrow_bpc_set(J, narrow_ref(ref), narrow_ref(sp[-1]), mode); 398 narrow_bpc_set(J, narrow_ref(ref), narrow_ref(sp[-1]), mode);
399 } 399 }
400 } 400 }
401 lua_assert(sp == nc->stack+1); 401 lj_assertJ(sp == nc->stack+1, "stack misalignment");
402 return nc->stack[0]; 402 return nc->stack[0];
403} 403}
404 404
@@ -452,7 +452,7 @@ static TRef narrow_stripov(jit_State *J, TRef tr, int lastop, IRRef mode)
452TRef LJ_FASTCALL lj_opt_narrow_index(jit_State *J, TRef tr) 452TRef LJ_FASTCALL lj_opt_narrow_index(jit_State *J, TRef tr)
453{ 453{
454 IRIns *ir; 454 IRIns *ir;
455 lua_assert(tref_isnumber(tr)); 455 lj_assertJ(tref_isnumber(tr), "expected number type");
456 if (tref_isnum(tr)) /* Conversion may be narrowed, too. See above. */ 456 if (tref_isnum(tr)) /* Conversion may be narrowed, too. See above. */
457 return emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_INDEX); 457 return emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_INDEX);
458 /* Omit some overflow checks for array indexing. See comments above. */ 458 /* Omit some overflow checks for array indexing. See comments above. */
@@ -499,7 +499,7 @@ TRef LJ_FASTCALL lj_opt_narrow_tobit(jit_State *J, TRef tr)
499/* Narrow C array index (overflow undefined). */ 499/* Narrow C array index (overflow undefined). */
500TRef LJ_FASTCALL lj_opt_narrow_cindex(jit_State *J, TRef tr) 500TRef LJ_FASTCALL lj_opt_narrow_cindex(jit_State *J, TRef tr)
501{ 501{
502 lua_assert(tref_isnumber(tr)); 502 lj_assertJ(tref_isnumber(tr), "expected number type");
503 if (tref_isnum(tr)) 503 if (tref_isnum(tr))
504 return emitir(IRT(IR_CONV, IRT_INTP), tr, (IRT_INTP<<5)|IRT_NUM|IRCONV_ANY); 504 return emitir(IRT(IR_CONV, IRT_INTP), tr, (IRT_INTP<<5)|IRT_NUM|IRCONV_ANY);
505 /* Undefined overflow semantics allow stripping of ADDOV, SUBOV and MULOV. */ 505 /* Undefined overflow semantics allow stripping of ADDOV, SUBOV and MULOV. */
@@ -551,11 +551,16 @@ TRef lj_opt_narrow_unm(jit_State *J, TRef rc, TValue *vc)
551{ 551{
552 rc = conv_str_tonum(J, rc, vc); 552 rc = conv_str_tonum(J, rc, vc);
553 if (tref_isinteger(rc)) { 553 if (tref_isinteger(rc)) {
554 if ((uint32_t)numberVint(vc) != 0x80000000u) 554 uint32_t k = (uint32_t)numberVint(vc);
555 return emitir(IRTGI(IR_SUBOV), lj_ir_kint(J, 0), rc); 555 if ((LJ_DUALNUM || k != 0) && k != 0x80000000u) {
556 TRef zero = lj_ir_kint(J, 0);
557 if (!LJ_DUALNUM)
558 emitir(IRTGI(IR_NE), rc, zero);
559 return emitir(IRTGI(IR_SUBOV), zero, rc);
560 }
556 rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT); 561 rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
557 } 562 }
558 return emitir(IRTN(IR_NEG), rc, lj_ir_knum_neg(J)); 563 return emitir(IRTN(IR_NEG), rc, lj_ir_ksimd(J, LJ_KSIMD_NEG));
559} 564}
560 565
561/* Narrowing of modulo operator. */ 566/* Narrowing of modulo operator. */
@@ -579,44 +584,6 @@ TRef lj_opt_narrow_mod(jit_State *J, TRef rb, TRef rc, TValue *vb, TValue *vc)
579 return emitir(IRTN(IR_SUB), rb, tmp); 584 return emitir(IRTN(IR_SUB), rb, tmp);
580} 585}
581 586
582/* Narrowing of power operator or math.pow. */
583TRef lj_opt_narrow_pow(jit_State *J, TRef rb, TRef rc, TValue *vb, TValue *vc)
584{
585 rb = conv_str_tonum(J, rb, vb);
586 rb = lj_ir_tonum(J, rb); /* Left arg is always treated as an FP number. */
587 rc = conv_str_tonum(J, rc, vc);
588 /* Narrowing must be unconditional to preserve (-x)^i semantics. */
589 if (tvisint(vc) || numisint(numV(vc))) {
590 int checkrange = 0;
591 /* Split pow is faster for bigger exponents. But do this only for (+k)^i. */
592 if (tref_isk(rb) && (int32_t)ir_knum(IR(tref_ref(rb)))->u32.hi >= 0) {
593 int32_t k = numberVint(vc);
594 if (!(k >= -65536 && k <= 65536)) goto split_pow;
595 checkrange = 1;
596 }
597 if (!tref_isinteger(rc)) {
598 /* Guarded conversion to integer! */
599 rc = emitir(IRTGI(IR_CONV), rc, IRCONV_INT_NUM|IRCONV_CHECK);
600 }
601 if (checkrange && !tref_isk(rc)) { /* Range guard: -65536 <= i <= 65536 */
602 TRef tmp = emitir(IRTI(IR_ADD), rc, lj_ir_kint(J, 65536));
603 emitir(IRTGI(IR_ULE), tmp, lj_ir_kint(J, 2*65536));
604 }
605 return emitir(IRTN(IR_POW), rb, rc);
606 }
607split_pow:
608 /* FOLD covers most cases, but some are easier to do here. */
609 if (tref_isk(rb) && tvispone(ir_knum(IR(tref_ref(rb)))))
610 return rb; /* 1 ^ x ==> 1 */
611 rc = lj_ir_tonum(J, rc);
612 if (tref_isk(rc) && ir_knum(IR(tref_ref(rc)))->n == 0.5)
613 return emitir(IRTN(IR_FPMATH), rb, IRFPM_SQRT); /* x ^ 0.5 ==> sqrt(x) */
614 /* Split up b^c into exp2(c*log2(b)). Assembler may rejoin later. */
615 rb = emitir(IRTN(IR_FPMATH), rb, IRFPM_LOG2);
616 rc = emitir(IRTN(IR_MUL), rb, rc);
617 return emitir(IRTN(IR_FPMATH), rc, IRFPM_EXP2);
618}
619
620/* -- Predictive narrowing of induction variables ------------------------- */ 587/* -- Predictive narrowing of induction variables ------------------------- */
621 588
622/* Narrow a single runtime value. */ 589/* Narrow a single runtime value. */
@@ -630,9 +597,10 @@ static int narrow_forl(jit_State *J, cTValue *o)
630/* Narrow the FORL index type by looking at the runtime values. */ 597/* Narrow the FORL index type by looking at the runtime values. */
631IRType lj_opt_narrow_forl(jit_State *J, cTValue *tv) 598IRType lj_opt_narrow_forl(jit_State *J, cTValue *tv)
632{ 599{
633 lua_assert(tvisnumber(&tv[FORL_IDX]) && 600 lj_assertJ(tvisnumber(&tv[FORL_IDX]) &&
634 tvisnumber(&tv[FORL_STOP]) && 601 tvisnumber(&tv[FORL_STOP]) &&
635 tvisnumber(&tv[FORL_STEP])); 602 tvisnumber(&tv[FORL_STEP]),
603 "expected number types");
636 /* Narrow only if the runtime values of start/stop/step are all integers. */ 604 /* Narrow only if the runtime values of start/stop/step are all integers. */
637 if (narrow_forl(J, &tv[FORL_IDX]) && 605 if (narrow_forl(J, &tv[FORL_IDX]) &&
638 narrow_forl(J, &tv[FORL_STOP]) && 606 narrow_forl(J, &tv[FORL_STOP]) &&
diff --git a/src/lj_opt_sink.c b/src/lj_opt_sink.c
index a9feddad..642ed750 100644
--- a/src/lj_opt_sink.c
+++ b/src/lj_opt_sink.c
@@ -86,8 +86,7 @@ static void sink_mark_ins(jit_State *J)
86 switch (ir->o) { 86 switch (ir->o) {
87 case IR_BASE: 87 case IR_BASE:
88 return; /* Finished. */ 88 return; /* Finished. */
89 case IR_CALLL: /* IRCALL_lj_tab_len */ 89 case IR_ALOAD: case IR_HLOAD: case IR_XLOAD: case IR_TBAR: case IR_ALEN:
90 case IR_ALOAD: case IR_HLOAD: case IR_XLOAD: case IR_TBAR:
91 irt_setmark(IR(ir->op1)->t); /* Mark ref for remaining loads. */ 90 irt_setmark(IR(ir->op1)->t); /* Mark ref for remaining loads. */
92 break; 91 break;
93 case IR_FLOAD: 92 case IR_FLOAD:
@@ -173,8 +172,8 @@ static void sink_remark_phi(jit_State *J)
173/* Sweep instructions and tag sunken allocations and stores. */ 172/* Sweep instructions and tag sunken allocations and stores. */
174static void sink_sweep_ins(jit_State *J) 173static void sink_sweep_ins(jit_State *J)
175{ 174{
176 IRIns *ir, *irfirst = IR(J->cur.nk); 175 IRIns *ir, *irbase = IR(REF_BASE);
177 for (ir = IR(J->cur.nins-1) ; ir >= irfirst; ir--) { 176 for (ir = IR(J->cur.nins-1) ; ir >= irbase; ir--) {
178 switch (ir->o) { 177 switch (ir->o) {
179 case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: { 178 case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: {
180 IRIns *ira = sink_checkalloc(J, ir); 179 IRIns *ira = sink_checkalloc(J, ir);
@@ -224,6 +223,13 @@ static void sink_sweep_ins(jit_State *J)
224 break; 223 break;
225 } 224 }
226 } 225 }
226 for (ir = IR(J->cur.nk); ir < irbase; ir++) {
227 irt_clearmark(ir->t);
228 ir->prev = REGSP_INIT;
229 /* The false-positive of irt_is64() for ASMREF_L (REF_NIL) is OK here. */
230 if (irt_is64(ir->t) && ir->o != IR_KNULL)
231 ir++;
232 }
227} 233}
228 234
229/* Allocation sinking and store sinking. 235/* Allocation sinking and store sinking.
diff --git a/src/lj_opt_split.c b/src/lj_opt_split.c
index 1f2ffe54..338a61a2 100644
--- a/src/lj_opt_split.c
+++ b/src/lj_opt_split.c
@@ -8,14 +8,15 @@
8 8
9#include "lj_obj.h" 9#include "lj_obj.h"
10 10
11#if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) 11#if LJ_HASJIT && (LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI))
12 12
13#include "lj_err.h" 13#include "lj_err.h"
14#include "lj_str.h" 14#include "lj_buf.h"
15#include "lj_ir.h" 15#include "lj_ir.h"
16#include "lj_jit.h" 16#include "lj_jit.h"
17#include "lj_ircall.h" 17#include "lj_ircall.h"
18#include "lj_iropt.h" 18#include "lj_iropt.h"
19#include "lj_dispatch.h"
19#include "lj_vm.h" 20#include "lj_vm.h"
20 21
21/* SPLIT pass: 22/* SPLIT pass:
@@ -139,6 +140,7 @@ static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
139 ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id); 140 ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
140 return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp); 141 return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
141} 142}
143#endif
142 144
143/* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */ 145/* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
144static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir, 146static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -155,7 +157,6 @@ static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
155 ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id); 157 ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
156 return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp); 158 return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
157} 159}
158#endif
159 160
160/* Emit a CALLN with two split 64 bit arguments. */ 161/* Emit a CALLN with two split 64 bit arguments. */
161static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir, 162static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -192,9 +193,121 @@ static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
192 nref = ir->op1; 193 nref = ir->op1;
193 if (ofs == 0) return nref; 194 if (ofs == 0) return nref;
194 } 195 }
195 return split_emit(J, IRTI(IR_ADD), nref, lj_ir_kint(J, ofs)); 196 return split_emit(J, IRT(IR_ADD, IRT_PTR), nref, lj_ir_kint(J, ofs));
196} 197}
197 198
199#if LJ_HASFFI
200static IRRef split_bitshift(jit_State *J, IRRef1 *hisubst,
201 IRIns *oir, IRIns *nir, IRIns *ir)
202{
203 IROp op = ir->o;
204 IRRef kref = nir->op2;
205 if (irref_isk(kref)) { /* Optimize constant shifts. */
206 int32_t k = (IR(kref)->i & 63);
207 IRRef lo = nir->op1, hi = hisubst[ir->op1];
208 if (op == IR_BROL || op == IR_BROR) {
209 if (op == IR_BROR) k = (-k & 63);
210 if (k >= 32) { IRRef t = lo; lo = hi; hi = t; k -= 32; }
211 if (k == 0) {
212 passthrough:
213 J->cur.nins--;
214 ir->prev = lo;
215 return hi;
216 } else {
217 TRef k1, k2;
218 IRRef t1, t2, t3, t4;
219 J->cur.nins--;
220 k1 = lj_ir_kint(J, k);
221 k2 = lj_ir_kint(J, (-k & 31));
222 t1 = split_emit(J, IRTI(IR_BSHL), lo, k1);
223 t2 = split_emit(J, IRTI(IR_BSHL), hi, k1);
224 t3 = split_emit(J, IRTI(IR_BSHR), lo, k2);
225 t4 = split_emit(J, IRTI(IR_BSHR), hi, k2);
226 ir->prev = split_emit(J, IRTI(IR_BOR), t1, t4);
227 return split_emit(J, IRTI(IR_BOR), t2, t3);
228 }
229 } else if (k == 0) {
230 goto passthrough;
231 } else if (k < 32) {
232 if (op == IR_BSHL) {
233 IRRef t1 = split_emit(J, IRTI(IR_BSHL), hi, kref);
234 IRRef t2 = split_emit(J, IRTI(IR_BSHR), lo, lj_ir_kint(J, (-k&31)));
235 return split_emit(J, IRTI(IR_BOR), t1, t2);
236 } else {
237 IRRef t1 = ir->prev, t2;
238 lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
239 nir->o = IR_BSHR;
240 t2 = split_emit(J, IRTI(IR_BSHL), hi, lj_ir_kint(J, (-k&31)));
241 ir->prev = split_emit(J, IRTI(IR_BOR), t1, t2);
242 return split_emit(J, IRTI(op), hi, kref);
243 }
244 } else {
245 if (op == IR_BSHL) {
246 if (k == 32)
247 J->cur.nins--;
248 else
249 lo = ir->prev;
250 ir->prev = lj_ir_kint(J, 0);
251 return lo;
252 } else {
253 lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
254 if (k == 32) {
255 J->cur.nins--;
256 ir->prev = hi;
257 } else {
258 nir->op1 = hi;
259 }
260 if (op == IR_BSHR)
261 return lj_ir_kint(J, 0);
262 else
263 return split_emit(J, IRTI(IR_BSAR), hi, lj_ir_kint(J, 31));
264 }
265 }
266 }
267 return split_call_li(J, hisubst, oir, ir,
268 op - IR_BSHL + IRCALL_lj_carith_shl64);
269}
270
271static IRRef split_bitop(jit_State *J, IRRef1 *hisubst,
272 IRIns *nir, IRIns *ir)
273{
274 IROp op = ir->o;
275 IRRef hi, kref = nir->op2;
276 if (irref_isk(kref)) { /* Optimize bit operations with lo constant. */
277 int32_t k = IR(kref)->i;
278 if (k == 0 || k == -1) {
279 if (op == IR_BAND) k = ~k;
280 if (k == 0) {
281 J->cur.nins--;
282 ir->prev = nir->op1;
283 } else if (op == IR_BXOR) {
284 nir->o = IR_BNOT;
285 nir->op2 = 0;
286 } else {
287 J->cur.nins--;
288 ir->prev = kref;
289 }
290 }
291 }
292 hi = hisubst[ir->op1];
293 kref = hisubst[ir->op2];
294 if (irref_isk(kref)) { /* Optimize bit operations with hi constant. */
295 int32_t k = IR(kref)->i;
296 if (k == 0 || k == -1) {
297 if (op == IR_BAND) k = ~k;
298 if (k == 0) {
299 return hi;
300 } else if (op == IR_BXOR) {
301 return split_emit(J, IRTI(IR_BNOT), hi, 0);
302 } else {
303 return kref;
304 }
305 }
306 }
307 return split_emit(J, IRTI(op), hi, kref);
308}
309#endif
310
198/* Substitute references of a snapshot. */ 311/* Substitute references of a snapshot. */
199static void split_subst_snap(jit_State *J, SnapShot *snap, IRIns *oir) 312static void split_subst_snap(jit_State *J, SnapShot *snap, IRIns *oir)
200{ 313{
@@ -214,7 +327,7 @@ static void split_ir(jit_State *J)
214 IRRef nins = J->cur.nins, nk = J->cur.nk; 327 IRRef nins = J->cur.nins, nk = J->cur.nk;
215 MSize irlen = nins - nk; 328 MSize irlen = nins - nk;
216 MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1)); 329 MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
217 IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need); 330 IRIns *oir = (IRIns *)lj_buf_tmp(J->L, need);
218 IRRef1 *hisubst; 331 IRRef1 *hisubst;
219 IRRef ref, snref; 332 IRRef ref, snref;
220 SnapShot *snap; 333 SnapShot *snap;
@@ -241,6 +354,8 @@ static void split_ir(jit_State *J)
241 ir->prev = ref; /* Identity substitution for loword. */ 354 ir->prev = ref; /* Identity substitution for loword. */
242 hisubst[ref] = 0; 355 hisubst[ref] = 0;
243 } 356 }
357 if (irt_is64(ir->t) && ir->o != IR_KNULL)
358 ref++;
244 } 359 }
245 360
246 /* Process old IR instructions. */ 361 /* Process old IR instructions. */
@@ -285,35 +400,11 @@ static void split_ir(jit_State *J)
285 hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div); 400 hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
286 break; 401 break;
287 case IR_POW: 402 case IR_POW:
288 hi = split_call_li(J, hisubst, oir, ir, IRCALL_lj_vm_powi); 403 hi = split_call_ll(J, hisubst, oir, ir, IRCALL_pow);
289 break; 404 break;
290 case IR_FPMATH: 405 case IR_FPMATH:
291 /* Try to rejoin pow from EXP2, MUL and LOG2. */
292 if (nir->op2 == IRFPM_EXP2 && nir->op1 > J->loopref) {
293 IRIns *irp = IR(nir->op1);
294 if (irp->o == IR_CALLN && irp->op2 == IRCALL_softfp_mul) {
295 IRIns *irm4 = IR(irp->op1);
296 IRIns *irm3 = IR(irm4->op1);
297 IRIns *irm12 = IR(irm3->op1);
298 IRIns *irl1 = IR(irm12->op1);
299 if (irm12->op1 > J->loopref && irl1->o == IR_CALLN &&
300 irl1->op2 == IRCALL_lj_vm_log2) {
301 IRRef tmp = irl1->op1; /* Recycle first two args from LOG2. */
302 IRRef arg3 = irm3->op2, arg4 = irm4->op2;
303 J->cur.nins--;
304 tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg3);
305 tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg4);
306 ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_pow);
307 hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
308 break;
309 }
310 }
311 }
312 hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2); 406 hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
313 break; 407 break;
314 case IR_ATAN2:
315 hi = split_call_ll(J, hisubst, oir, ir, IRCALL_atan2);
316 break;
317 case IR_LDEXP: 408 case IR_LDEXP:
318 hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp); 409 hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
319 break; 410 break;
@@ -321,7 +412,8 @@ static void split_ir(jit_State *J)
321 nir->o = IR_CONV; /* Pass through loword. */ 412 nir->o = IR_CONV; /* Pass through loword. */
322 nir->op2 = (IRT_INT << 5) | IRT_INT; 413 nir->op2 = (IRT_INT << 5) | IRT_INT;
323 hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP), 414 hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
324 hisubst[ir->op1], hisubst[ir->op2]); 415 hisubst[ir->op1],
416 lj_ir_kint(J, (int32_t)(0x7fffffffu + (ir->o == IR_NEG))));
325 break; 417 break;
326 case IR_SLOAD: 418 case IR_SLOAD:
327 if ((nir->op2 & IRSLOAD_CONVERT)) { /* Convert from int to number. */ 419 if ((nir->op2 & IRSLOAD_CONVERT)) { /* Convert from int to number. */
@@ -336,15 +428,24 @@ static void split_ir(jit_State *J)
336 case IR_STRTO: 428 case IR_STRTO:
337 hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref); 429 hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
338 break; 430 break;
431 case IR_FLOAD:
432 lj_assertJ(ir->op1 == REF_NIL, "expected FLOAD from GG_State");
433 hi = lj_ir_kint(J, *(int32_t*)((char*)J2GG(J) + ir->op2 + LJ_LE*4));
434 nir->op2 += LJ_BE*4;
435 break;
339 case IR_XLOAD: { 436 case IR_XLOAD: {
340 IRIns inslo = *nir; /* Save/undo the emit of the lo XLOAD. */ 437 IRIns inslo = *nir; /* Save/undo the emit of the lo XLOAD. */
341 J->cur.nins--; 438 J->cur.nins--;
342 hi = split_ptr(J, oir, ir->op1); /* Insert the hiref ADD. */ 439 hi = split_ptr(J, oir, ir->op1); /* Insert the hiref ADD. */
440#if LJ_BE
441 hi = split_emit(J, IRT(IR_XLOAD, IRT_INT), hi, ir->op2);
442 inslo.t.irt = IRT_SOFTFP | (inslo.t.irt & IRT_GUARD);
443#endif
343 nref = lj_ir_nextins(J); 444 nref = lj_ir_nextins(J);
344 nir = IR(nref); 445 nir = IR(nref);
345 *nir = inslo; /* Re-emit lo XLOAD immediately before hi XLOAD. */ 446 *nir = inslo; /* Re-emit lo XLOAD. */
346 hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
347#if LJ_LE 447#if LJ_LE
448 hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
348 ir->prev = nref; 449 ir->prev = nref;
349#else 450#else
350 ir->prev = hi; hi = nref; 451 ir->prev = hi; hi = nref;
@@ -364,8 +465,9 @@ static void split_ir(jit_State *J)
364 break; 465 break;
365 } 466 }
366#endif 467#endif
367 lua_assert(st == IRT_INT || 468 lj_assertJ(st == IRT_INT ||
368 (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT))); 469 (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)),
470 "bad source type for CONV");
369 nir->o = IR_CALLN; 471 nir->o = IR_CALLN;
370#if LJ_32 && LJ_HASFFI 472#if LJ_32 && LJ_HASFFI
371 nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d : 473 nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
@@ -395,7 +497,8 @@ static void split_ir(jit_State *J)
395 hi = nir->op2; 497 hi = nir->op2;
396 break; 498 break;
397 default: 499 default:
398 lua_assert(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX); 500 lj_assertJ(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX,
501 "bad IR op %d", ir->o);
399 hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP), 502 hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
400 hisubst[ir->op1], hisubst[ir->op2]); 503 hisubst[ir->op1], hisubst[ir->op2]);
401 break; 504 break;
@@ -438,8 +541,21 @@ static void split_ir(jit_State *J)
438 irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 : 541 irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
439 IRCALL_lj_carith_powu64); 542 IRCALL_lj_carith_powu64);
440 break; 543 break;
544 case IR_BNOT:
545 hi = split_emit(J, IRTI(IR_BNOT), hiref, 0);
546 break;
547 case IR_BSWAP:
548 ir->prev = split_emit(J, IRTI(IR_BSWAP), hiref, 0);
549 hi = nref;
550 break;
551 case IR_BAND: case IR_BOR: case IR_BXOR:
552 hi = split_bitop(J, hisubst, nir, ir);
553 break;
554 case IR_BSHL: case IR_BSHR: case IR_BSAR: case IR_BROL: case IR_BROR:
555 hi = split_bitshift(J, hisubst, oir, nir, ir);
556 break;
441 case IR_FLOAD: 557 case IR_FLOAD:
442 lua_assert(ir->op2 == IRFL_CDATA_INT64); 558 lj_assertJ(ir->op2 == IRFL_CDATA_INT64, "only INT64 supported");
443 hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4); 559 hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
444#if LJ_BE 560#if LJ_BE
445 ir->prev = hi; hi = nref; 561 ir->prev = hi; hi = nref;
@@ -505,7 +621,7 @@ static void split_ir(jit_State *J)
505 hi = nir->op2; 621 hi = nir->op2;
506 break; 622 break;
507 default: 623 default:
508 lua_assert(ir->o <= IR_NE); /* Comparisons. */ 624 lj_assertJ(ir->o <= IR_NE, "bad IR op %d", ir->o); /* Comparisons. */
509 split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]); 625 split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
510 break; 626 break;
511 } 627 }
@@ -529,7 +645,7 @@ static void split_ir(jit_State *J)
529 tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev); 645 tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
530#endif 646#endif
531 ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit); 647 ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
532 } else if (ir->o == IR_TOSTR) { 648 } else if (ir->o == IR_TOSTR || ir->o == IR_TMPREF) {
533 if (hisubst[ir->op1]) { 649 if (hisubst[ir->op1]) {
534 if (irref_isk(ir->op1)) 650 if (irref_isk(ir->op1))
535 nir->op1 = ir->op1; 651 nir->op1 = ir->op1;
@@ -583,7 +699,7 @@ static void split_ir(jit_State *J)
583#if LJ_SOFTFP 699#if LJ_SOFTFP
584 if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) { 700 if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
585 if (irt_isguard(ir->t)) { 701 if (irt_isguard(ir->t)) {
586 lua_assert(st == IRT_NUM && irt_isint(ir->t)); 702 lj_assertJ(st == IRT_NUM && irt_isint(ir->t), "bad CONV types");
587 J->cur.nins--; 703 J->cur.nins--;
588 ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1); 704 ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
589 } else { 705 } else {
@@ -714,7 +830,7 @@ void lj_opt_split(jit_State *J)
714 if (!J->needsplit) 830 if (!J->needsplit)
715 J->needsplit = split_needsplit(J); 831 J->needsplit = split_needsplit(J);
716#else 832#else
717 lua_assert(J->needsplit >= split_needsplit(J)); /* Verify flag. */ 833 lj_assertJ(J->needsplit >= split_needsplit(J), "bad SPLIT state");
718#endif 834#endif
719 if (J->needsplit) { 835 if (J->needsplit) {
720 int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit); 836 int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);
diff --git a/src/lj_parse.c b/src/lj_parse.c
index c0cbd261..6319c272 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -13,6 +13,7 @@
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_debug.h" 15#include "lj_debug.h"
16#include "lj_buf.h"
16#include "lj_str.h" 17#include "lj_str.h"
17#include "lj_tab.h" 18#include "lj_tab.h"
18#include "lj_func.h" 19#include "lj_func.h"
@@ -21,6 +22,7 @@
21#if LJ_HASFFI 22#if LJ_HASFFI
22#include "lj_ctype.h" 23#include "lj_ctype.h"
23#endif 24#endif
25#include "lj_strfmt.h"
24#include "lj_lex.h" 26#include "lj_lex.h"
25#include "lj_parse.h" 27#include "lj_parse.h"
26#include "lj_vm.h" 28#include "lj_vm.h"
@@ -161,16 +163,22 @@ LJ_STATIC_ASSERT((int)BC_MULVV-(int)BC_ADDVV == (int)OPR_MUL-(int)OPR_ADD);
161LJ_STATIC_ASSERT((int)BC_DIVVV-(int)BC_ADDVV == (int)OPR_DIV-(int)OPR_ADD); 163LJ_STATIC_ASSERT((int)BC_DIVVV-(int)BC_ADDVV == (int)OPR_DIV-(int)OPR_ADD);
162LJ_STATIC_ASSERT((int)BC_MODVV-(int)BC_ADDVV == (int)OPR_MOD-(int)OPR_ADD); 164LJ_STATIC_ASSERT((int)BC_MODVV-(int)BC_ADDVV == (int)OPR_MOD-(int)OPR_ADD);
163 165
166#ifdef LUA_USE_ASSERT
167#define lj_assertFS(c, ...) (lj_assertG_(G(fs->L), (c), __VA_ARGS__))
168#else
169#define lj_assertFS(c, ...) ((void)fs)
170#endif
171
164/* -- Error handling ------------------------------------------------------ */ 172/* -- Error handling ------------------------------------------------------ */
165 173
166LJ_NORET LJ_NOINLINE static void err_syntax(LexState *ls, ErrMsg em) 174LJ_NORET LJ_NOINLINE static void err_syntax(LexState *ls, ErrMsg em)
167{ 175{
168 lj_lex_error(ls, ls->token, em); 176 lj_lex_error(ls, ls->tok, em);
169} 177}
170 178
171LJ_NORET LJ_NOINLINE static void err_token(LexState *ls, LexToken token) 179LJ_NORET LJ_NOINLINE static void err_token(LexState *ls, LexToken tok)
172{ 180{
173 lj_lex_error(ls, ls->token, LJ_ERR_XTOKEN, lj_lex_token2str(ls, token)); 181 lj_lex_error(ls, ls->tok, LJ_ERR_XTOKEN, lj_lex_token2str(ls, tok));
174} 182}
175 183
176LJ_NORET static void err_limit(FuncState *fs, uint32_t limit, const char *what) 184LJ_NORET static void err_limit(FuncState *fs, uint32_t limit, const char *what)
@@ -198,7 +206,7 @@ static BCReg const_num(FuncState *fs, ExpDesc *e)
198{ 206{
199 lua_State *L = fs->L; 207 lua_State *L = fs->L;
200 TValue *o; 208 TValue *o;
201 lua_assert(expr_isnumk(e)); 209 lj_assertFS(expr_isnumk(e), "bad usage");
202 o = lj_tab_set(L, fs->kt, &e->u.nval); 210 o = lj_tab_set(L, fs->kt, &e->u.nval);
203 if (tvhaskslot(o)) 211 if (tvhaskslot(o))
204 return tvkslot(o); 212 return tvkslot(o);
@@ -223,7 +231,7 @@ static BCReg const_gc(FuncState *fs, GCobj *gc, uint32_t itype)
223/* Add a string constant. */ 231/* Add a string constant. */
224static BCReg const_str(FuncState *fs, ExpDesc *e) 232static BCReg const_str(FuncState *fs, ExpDesc *e)
225{ 233{
226 lua_assert(expr_isstrk(e) || e->k == VGLOBAL); 234 lj_assertFS(expr_isstrk(e) || e->k == VGLOBAL, "bad usage");
227 return const_gc(fs, obj2gco(e->u.sval), LJ_TSTR); 235 return const_gc(fs, obj2gco(e->u.sval), LJ_TSTR);
228} 236}
229 237
@@ -311,7 +319,7 @@ static void jmp_patchins(FuncState *fs, BCPos pc, BCPos dest)
311{ 319{
312 BCIns *jmp = &fs->bcbase[pc].ins; 320 BCIns *jmp = &fs->bcbase[pc].ins;
313 BCPos offset = dest-(pc+1)+BCBIAS_J; 321 BCPos offset = dest-(pc+1)+BCBIAS_J;
314 lua_assert(dest != NO_JMP); 322 lj_assertFS(dest != NO_JMP, "uninitialized jump target");
315 if (offset > BCMAX_D) 323 if (offset > BCMAX_D)
316 err_syntax(fs->ls, LJ_ERR_XJUMP); 324 err_syntax(fs->ls, LJ_ERR_XJUMP);
317 setbc_d(jmp, offset); 325 setbc_d(jmp, offset);
@@ -360,7 +368,7 @@ static void jmp_patch(FuncState *fs, BCPos list, BCPos target)
360 if (target == fs->pc) { 368 if (target == fs->pc) {
361 jmp_tohere(fs, list); 369 jmp_tohere(fs, list);
362 } else { 370 } else {
363 lua_assert(target < fs->pc); 371 lj_assertFS(target < fs->pc, "bad jump target");
364 jmp_patchval(fs, list, target, NO_REG, target); 372 jmp_patchval(fs, list, target, NO_REG, target);
365 } 373 }
366} 374}
@@ -390,7 +398,7 @@ static void bcreg_free(FuncState *fs, BCReg reg)
390{ 398{
391 if (reg >= fs->nactvar) { 399 if (reg >= fs->nactvar) {
392 fs->freereg--; 400 fs->freereg--;
393 lua_assert(reg == fs->freereg); 401 lj_assertFS(reg == fs->freereg, "bad regfree");
394 } 402 }
395} 403}
396 404
@@ -540,7 +548,7 @@ static void expr_toreg_nobranch(FuncState *fs, ExpDesc *e, BCReg reg)
540 } else if (e->k <= VKTRUE) { 548 } else if (e->k <= VKTRUE) {
541 ins = BCINS_AD(BC_KPRI, reg, const_pri(e)); 549 ins = BCINS_AD(BC_KPRI, reg, const_pri(e));
542 } else { 550 } else {
543 lua_assert(e->k == VVOID || e->k == VJMP); 551 lj_assertFS(e->k == VVOID || e->k == VJMP, "bad expr type %d", e->k);
544 return; 552 return;
545 } 553 }
546 bcemit_INS(fs, ins); 554 bcemit_INS(fs, ins);
@@ -635,7 +643,7 @@ static void bcemit_store(FuncState *fs, ExpDesc *var, ExpDesc *e)
635 ins = BCINS_AD(BC_GSET, ra, const_str(fs, var)); 643 ins = BCINS_AD(BC_GSET, ra, const_str(fs, var));
636 } else { 644 } else {
637 BCReg ra, rc; 645 BCReg ra, rc;
638 lua_assert(var->k == VINDEXED); 646 lj_assertFS(var->k == VINDEXED, "bad expr type %d", var->k);
639 ra = expr_toanyreg(fs, e); 647 ra = expr_toanyreg(fs, e);
640 rc = var->u.s.aux; 648 rc = var->u.s.aux;
641 if ((int32_t)rc < 0) { 649 if ((int32_t)rc < 0) {
@@ -643,10 +651,12 @@ static void bcemit_store(FuncState *fs, ExpDesc *var, ExpDesc *e)
643 } else if (rc > BCMAX_C) { 651 } else if (rc > BCMAX_C) {
644 ins = BCINS_ABC(BC_TSETB, ra, var->u.s.info, rc-(BCMAX_C+1)); 652 ins = BCINS_ABC(BC_TSETB, ra, var->u.s.info, rc-(BCMAX_C+1));
645 } else { 653 } else {
654#ifdef LUA_USE_ASSERT
646 /* Free late alloced key reg to avoid assert on free of value reg. */ 655 /* Free late alloced key reg to avoid assert on free of value reg. */
647 /* This can only happen when called from expr_table(). */ 656 /* This can only happen when called from expr_table(). */
648 lua_assert(e->k != VNONRELOC || ra < fs->nactvar || 657 if (e->k == VNONRELOC && ra >= fs->nactvar && rc >= ra)
649 rc < ra || (bcreg_free(fs, rc),1)); 658 bcreg_free(fs, rc);
659#endif
650 ins = BCINS_ABC(BC_TSETV, ra, var->u.s.info, rc); 660 ins = BCINS_ABC(BC_TSETV, ra, var->u.s.info, rc);
651 } 661 }
652 } 662 }
@@ -660,16 +670,16 @@ static void bcemit_method(FuncState *fs, ExpDesc *e, ExpDesc *key)
660 BCReg idx, func, obj = expr_toanyreg(fs, e); 670 BCReg idx, func, obj = expr_toanyreg(fs, e);
661 expr_free(fs, e); 671 expr_free(fs, e);
662 func = fs->freereg; 672 func = fs->freereg;
663 bcemit_AD(fs, BC_MOV, func+1, obj); /* Copy object to first argument. */ 673 bcemit_AD(fs, BC_MOV, func+1+LJ_FR2, obj); /* Copy object to 1st argument. */
664 lua_assert(expr_isstrk(key)); 674 lj_assertFS(expr_isstrk(key), "bad usage");
665 idx = const_str(fs, key); 675 idx = const_str(fs, key);
666 if (idx <= BCMAX_C) { 676 if (idx <= BCMAX_C) {
667 bcreg_reserve(fs, 2); 677 bcreg_reserve(fs, 2+LJ_FR2);
668 bcemit_ABC(fs, BC_TGETS, func, obj, idx); 678 bcemit_ABC(fs, BC_TGETS, func, obj, idx);
669 } else { 679 } else {
670 bcreg_reserve(fs, 3); 680 bcreg_reserve(fs, 3+LJ_FR2);
671 bcemit_AD(fs, BC_KSTR, func+2, idx); 681 bcemit_AD(fs, BC_KSTR, func+2+LJ_FR2, idx);
672 bcemit_ABC(fs, BC_TGETV, func, obj, func+2); 682 bcemit_ABC(fs, BC_TGETV, func, obj, func+2+LJ_FR2);
673 fs->freereg--; 683 fs->freereg--;
674 } 684 }
675 e->u.s.info = func; 685 e->u.s.info = func;
@@ -801,7 +811,8 @@ static void bcemit_arith(FuncState *fs, BinOpr opr, ExpDesc *e1, ExpDesc *e2)
801 else 811 else
802 rc = expr_toanyreg(fs, e2); 812 rc = expr_toanyreg(fs, e2);
803 /* 1st operand discharged by bcemit_binop_left, but need KNUM/KSHORT. */ 813 /* 1st operand discharged by bcemit_binop_left, but need KNUM/KSHORT. */
804 lua_assert(expr_isnumk(e1) || e1->k == VNONRELOC); 814 lj_assertFS(expr_isnumk(e1) || e1->k == VNONRELOC,
815 "bad expr type %d", e1->k);
805 expr_toval(fs, e1); 816 expr_toval(fs, e1);
806 /* Avoid two consts to satisfy bytecode constraints. */ 817 /* Avoid two consts to satisfy bytecode constraints. */
807 if (expr_isnumk(e1) && !expr_isnumk(e2) && 818 if (expr_isnumk(e1) && !expr_isnumk(e2) &&
@@ -889,19 +900,20 @@ static void bcemit_binop(FuncState *fs, BinOpr op, ExpDesc *e1, ExpDesc *e2)
889 if (op <= OPR_POW) { 900 if (op <= OPR_POW) {
890 bcemit_arith(fs, op, e1, e2); 901 bcemit_arith(fs, op, e1, e2);
891 } else if (op == OPR_AND) { 902 } else if (op == OPR_AND) {
892 lua_assert(e1->t == NO_JMP); /* List must be closed. */ 903 lj_assertFS(e1->t == NO_JMP, "jump list not closed");
893 expr_discharge(fs, e2); 904 expr_discharge(fs, e2);
894 jmp_append(fs, &e2->f, e1->f); 905 jmp_append(fs, &e2->f, e1->f);
895 *e1 = *e2; 906 *e1 = *e2;
896 } else if (op == OPR_OR) { 907 } else if (op == OPR_OR) {
897 lua_assert(e1->f == NO_JMP); /* List must be closed. */ 908 lj_assertFS(e1->f == NO_JMP, "jump list not closed");
898 expr_discharge(fs, e2); 909 expr_discharge(fs, e2);
899 jmp_append(fs, &e2->t, e1->t); 910 jmp_append(fs, &e2->t, e1->t);
900 *e1 = *e2; 911 *e1 = *e2;
901 } else if (op == OPR_CONCAT) { 912 } else if (op == OPR_CONCAT) {
902 expr_toval(fs, e2); 913 expr_toval(fs, e2);
903 if (e2->k == VRELOCABLE && bc_op(*bcptr(fs, e2)) == BC_CAT) { 914 if (e2->k == VRELOCABLE && bc_op(*bcptr(fs, e2)) == BC_CAT) {
904 lua_assert(e1->u.s.info == bc_b(*bcptr(fs, e2))-1); 915 lj_assertFS(e1->u.s.info == bc_b(*bcptr(fs, e2))-1,
916 "bad CAT stack layout");
905 expr_free(fs, e1); 917 expr_free(fs, e1);
906 setbc_b(bcptr(fs, e2), e1->u.s.info); 918 setbc_b(bcptr(fs, e2), e1->u.s.info);
907 e1->u.s.info = e2->u.s.info; 919 e1->u.s.info = e2->u.s.info;
@@ -913,8 +925,9 @@ static void bcemit_binop(FuncState *fs, BinOpr op, ExpDesc *e1, ExpDesc *e2)
913 } 925 }
914 e1->k = VRELOCABLE; 926 e1->k = VRELOCABLE;
915 } else { 927 } else {
916 lua_assert(op == OPR_NE || op == OPR_EQ || 928 lj_assertFS(op == OPR_NE || op == OPR_EQ ||
917 op == OPR_LT || op == OPR_GE || op == OPR_LE || op == OPR_GT); 929 op == OPR_LT || op == OPR_GE || op == OPR_LE || op == OPR_GT,
930 "bad binop %d", op);
918 bcemit_comp(fs, op, e1, e2); 931 bcemit_comp(fs, op, e1, e2);
919 } 932 }
920} 933}
@@ -943,10 +956,10 @@ static void bcemit_unop(FuncState *fs, BCOp op, ExpDesc *e)
943 e->u.s.info = fs->freereg-1; 956 e->u.s.info = fs->freereg-1;
944 e->k = VNONRELOC; 957 e->k = VNONRELOC;
945 } else { 958 } else {
946 lua_assert(e->k == VNONRELOC); 959 lj_assertFS(e->k == VNONRELOC, "bad expr type %d", e->k);
947 } 960 }
948 } else { 961 } else {
949 lua_assert(op == BC_UNM || op == BC_LEN); 962 lj_assertFS(op == BC_UNM || op == BC_LEN, "bad unop %d", op);
950 if (op == BC_UNM && !expr_hasjump(e)) { /* Constant-fold negations. */ 963 if (op == BC_UNM && !expr_hasjump(e)) { /* Constant-fold negations. */
951#if LJ_HASFFI 964#if LJ_HASFFI
952 if (e->k == VKCDATA) { /* Fold in-place since cdata is not interned. */ 965 if (e->k == VKCDATA) { /* Fold in-place since cdata is not interned. */
@@ -986,7 +999,7 @@ static void bcemit_unop(FuncState *fs, BCOp op, ExpDesc *e)
986/* Check and consume optional token. */ 999/* Check and consume optional token. */
987static int lex_opt(LexState *ls, LexToken tok) 1000static int lex_opt(LexState *ls, LexToken tok)
988{ 1001{
989 if (ls->token == tok) { 1002 if (ls->tok == tok) {
990 lj_lex_next(ls); 1003 lj_lex_next(ls);
991 return 1; 1004 return 1;
992 } 1005 }
@@ -996,7 +1009,7 @@ static int lex_opt(LexState *ls, LexToken tok)
996/* Check and consume token. */ 1009/* Check and consume token. */
997static void lex_check(LexState *ls, LexToken tok) 1010static void lex_check(LexState *ls, LexToken tok)
998{ 1011{
999 if (ls->token != tok) 1012 if (ls->tok != tok)
1000 err_token(ls, tok); 1013 err_token(ls, tok);
1001 lj_lex_next(ls); 1014 lj_lex_next(ls);
1002} 1015}
@@ -1010,7 +1023,7 @@ static void lex_match(LexState *ls, LexToken what, LexToken who, BCLine line)
1010 } else { 1023 } else {
1011 const char *swhat = lj_lex_token2str(ls, what); 1024 const char *swhat = lj_lex_token2str(ls, what);
1012 const char *swho = lj_lex_token2str(ls, who); 1025 const char *swho = lj_lex_token2str(ls, who);
1013 lj_lex_error(ls, ls->token, LJ_ERR_XMATCH, swhat, swho, line); 1026 lj_lex_error(ls, ls->tok, LJ_ERR_XMATCH, swhat, swho, line);
1014 } 1027 }
1015 } 1028 }
1016} 1029}
@@ -1019,9 +1032,9 @@ static void lex_match(LexState *ls, LexToken what, LexToken who, BCLine line)
1019static GCstr *lex_str(LexState *ls) 1032static GCstr *lex_str(LexState *ls)
1020{ 1033{
1021 GCstr *s; 1034 GCstr *s;
1022 if (ls->token != TK_name && (LJ_52 || ls->token != TK_goto)) 1035 if (ls->tok != TK_name && (LJ_52 || ls->tok != TK_goto))
1023 err_token(ls, TK_name); 1036 err_token(ls, TK_name);
1024 s = strV(&ls->tokenval); 1037 s = strV(&ls->tokval);
1025 lj_lex_next(ls); 1038 lj_lex_next(ls);
1026 return s; 1039 return s;
1027} 1040}
@@ -1041,8 +1054,9 @@ static void var_new(LexState *ls, BCReg n, GCstr *name)
1041 lj_lex_error(ls, 0, LJ_ERR_XLIMC, LJ_MAX_VSTACK); 1054 lj_lex_error(ls, 0, LJ_ERR_XLIMC, LJ_MAX_VSTACK);
1042 lj_mem_growvec(ls->L, ls->vstack, ls->sizevstack, LJ_MAX_VSTACK, VarInfo); 1055 lj_mem_growvec(ls->L, ls->vstack, ls->sizevstack, LJ_MAX_VSTACK, VarInfo);
1043 } 1056 }
1044 lua_assert((uintptr_t)name < VARNAME__MAX || 1057 lj_assertFS((uintptr_t)name < VARNAME__MAX ||
1045 lj_tab_getstr(fs->kt, name) != NULL); 1058 lj_tab_getstr(fs->kt, name) != NULL,
1059 "unanchored variable name");
1046 /* NOBARRIER: name is anchored in fs->kt and ls->vstack is not a GCobj. */ 1060 /* NOBARRIER: name is anchored in fs->kt and ls->vstack is not a GCobj. */
1047 setgcref(ls->vstack[vtop].name, obj2gco(name)); 1061 setgcref(ls->vstack[vtop].name, obj2gco(name));
1048 fs->varmap[fs->nactvar+n] = (uint16_t)vtop; 1062 fs->varmap[fs->nactvar+n] = (uint16_t)vtop;
@@ -1097,7 +1111,7 @@ static MSize var_lookup_uv(FuncState *fs, MSize vidx, ExpDesc *e)
1097 return i; /* Already exists. */ 1111 return i; /* Already exists. */
1098 /* Otherwise create a new one. */ 1112 /* Otherwise create a new one. */
1099 checklimit(fs, fs->nuv, LJ_MAX_UPVAL, "upvalues"); 1113 checklimit(fs, fs->nuv, LJ_MAX_UPVAL, "upvalues");
1100 lua_assert(e->k == VLOCAL || e->k == VUPVAL); 1114 lj_assertFS(e->k == VLOCAL || e->k == VUPVAL, "bad expr type %d", e->k);
1101 fs->uvmap[n] = (uint16_t)vidx; 1115 fs->uvmap[n] = (uint16_t)vidx;
1102 fs->uvtmp[n] = (uint16_t)(e->k == VLOCAL ? vidx : LJ_MAX_VSTACK+e->u.s.info); 1116 fs->uvtmp[n] = (uint16_t)(e->k == VLOCAL ? vidx : LJ_MAX_VSTACK+e->u.s.info);
1103 fs->nuv = n+1; 1117 fs->nuv = n+1;
@@ -1148,7 +1162,8 @@ static MSize gola_new(LexState *ls, GCstr *name, uint8_t info, BCPos pc)
1148 lj_lex_error(ls, 0, LJ_ERR_XLIMC, LJ_MAX_VSTACK); 1162 lj_lex_error(ls, 0, LJ_ERR_XLIMC, LJ_MAX_VSTACK);
1149 lj_mem_growvec(ls->L, ls->vstack, ls->sizevstack, LJ_MAX_VSTACK, VarInfo); 1163 lj_mem_growvec(ls->L, ls->vstack, ls->sizevstack, LJ_MAX_VSTACK, VarInfo);
1150 } 1164 }
1151 lua_assert(name == NAME_BREAK || lj_tab_getstr(fs->kt, name) != NULL); 1165 lj_assertFS(name == NAME_BREAK || lj_tab_getstr(fs->kt, name) != NULL,
1166 "unanchored label name");
1152 /* NOBARRIER: name is anchored in fs->kt and ls->vstack is not a GCobj. */ 1167 /* NOBARRIER: name is anchored in fs->kt and ls->vstack is not a GCobj. */
1153 setgcref(ls->vstack[vtop].name, obj2gco(name)); 1168 setgcref(ls->vstack[vtop].name, obj2gco(name));
1154 ls->vstack[vtop].startpc = pc; 1169 ls->vstack[vtop].startpc = pc;
@@ -1178,8 +1193,9 @@ static void gola_close(LexState *ls, VarInfo *vg)
1178 FuncState *fs = ls->fs; 1193 FuncState *fs = ls->fs;
1179 BCPos pc = vg->startpc; 1194 BCPos pc = vg->startpc;
1180 BCIns *ip = &fs->bcbase[pc].ins; 1195 BCIns *ip = &fs->bcbase[pc].ins;
1181 lua_assert(gola_isgoto(vg)); 1196 lj_assertFS(gola_isgoto(vg), "expected goto");
1182 lua_assert(bc_op(*ip) == BC_JMP || bc_op(*ip) == BC_UCLO); 1197 lj_assertFS(bc_op(*ip) == BC_JMP || bc_op(*ip) == BC_UCLO,
1198 "bad bytecode op %d", bc_op(*ip));
1183 setbc_a(ip, vg->slot); 1199 setbc_a(ip, vg->slot);
1184 if (bc_op(*ip) == BC_JMP) { 1200 if (bc_op(*ip) == BC_JMP) {
1185 BCPos next = jmp_next(fs, pc); 1201 BCPos next = jmp_next(fs, pc);
@@ -1198,9 +1214,9 @@ static void gola_resolve(LexState *ls, FuncScope *bl, MSize idx)
1198 if (gcrefeq(vg->name, vl->name) && gola_isgoto(vg)) { 1214 if (gcrefeq(vg->name, vl->name) && gola_isgoto(vg)) {
1199 if (vg->slot < vl->slot) { 1215 if (vg->slot < vl->slot) {
1200 GCstr *name = strref(var_get(ls, ls->fs, vg->slot).name); 1216 GCstr *name = strref(var_get(ls, ls->fs, vg->slot).name);
1201 lua_assert((uintptr_t)name >= VARNAME__MAX); 1217 lj_assertLS((uintptr_t)name >= VARNAME__MAX, "expected goto name");
1202 ls->linenumber = ls->fs->bcbase[vg->startpc].line; 1218 ls->linenumber = ls->fs->bcbase[vg->startpc].line;
1203 lua_assert(strref(vg->name) != NAME_BREAK); 1219 lj_assertLS(strref(vg->name) != NAME_BREAK, "unexpected break");
1204 lj_lex_error(ls, 0, LJ_ERR_XGSCOPE, 1220 lj_lex_error(ls, 0, LJ_ERR_XGSCOPE,
1205 strdata(strref(vg->name)), strdata(name)); 1221 strdata(strref(vg->name)), strdata(name));
1206 } 1222 }
@@ -1264,7 +1280,7 @@ static void fscope_begin(FuncState *fs, FuncScope *bl, int flags)
1264 bl->vstart = fs->ls->vtop; 1280 bl->vstart = fs->ls->vtop;
1265 bl->prev = fs->bl; 1281 bl->prev = fs->bl;
1266 fs->bl = bl; 1282 fs->bl = bl;
1267 lua_assert(fs->freereg == fs->nactvar); 1283 lj_assertFS(fs->freereg == fs->nactvar, "bad regalloc");
1268} 1284}
1269 1285
1270/* End a scope. */ 1286/* End a scope. */
@@ -1275,7 +1291,7 @@ static void fscope_end(FuncState *fs)
1275 fs->bl = bl->prev; 1291 fs->bl = bl->prev;
1276 var_remove(ls, bl->nactvar); 1292 var_remove(ls, bl->nactvar);
1277 fs->freereg = fs->nactvar; 1293 fs->freereg = fs->nactvar;
1278 lua_assert(bl->nactvar == fs->nactvar); 1294 lj_assertFS(bl->nactvar == fs->nactvar, "bad regalloc");
1279 if ((bl->flags & (FSCOPE_UPVAL|FSCOPE_NOCLOSE)) == FSCOPE_UPVAL) 1295 if ((bl->flags & (FSCOPE_UPVAL|FSCOPE_NOCLOSE)) == FSCOPE_UPVAL)
1280 bcemit_AJ(fs, BC_UCLO, bl->nactvar, 0); 1296 bcemit_AJ(fs, BC_UCLO, bl->nactvar, 0);
1281 if ((bl->flags & FSCOPE_BREAK)) { 1297 if ((bl->flags & FSCOPE_BREAK)) {
@@ -1362,13 +1378,13 @@ static void fs_fixup_k(FuncState *fs, GCproto *pt, void *kptr)
1362 Node *n = &node[i]; 1378 Node *n = &node[i];
1363 if (tvhaskslot(&n->val)) { 1379 if (tvhaskslot(&n->val)) {
1364 ptrdiff_t kidx = (ptrdiff_t)tvkslot(&n->val); 1380 ptrdiff_t kidx = (ptrdiff_t)tvkslot(&n->val);
1365 lua_assert(!tvisint(&n->key)); 1381 lj_assertFS(!tvisint(&n->key), "unexpected integer key");
1366 if (tvisnum(&n->key)) { 1382 if (tvisnum(&n->key)) {
1367 TValue *tv = &((TValue *)kptr)[kidx]; 1383 TValue *tv = &((TValue *)kptr)[kidx];
1368 if (LJ_DUALNUM) { 1384 if (LJ_DUALNUM) {
1369 lua_Number nn = numV(&n->key); 1385 lua_Number nn = numV(&n->key);
1370 int32_t k = lj_num2int(nn); 1386 int32_t k = lj_num2int(nn);
1371 lua_assert(!tvismzero(&n->key)); 1387 lj_assertFS(!tvismzero(&n->key), "unexpected -0 key");
1372 if ((lua_Number)k == nn) 1388 if ((lua_Number)k == nn)
1373 setintV(tv, k); 1389 setintV(tv, k);
1374 else 1390 else
@@ -1416,98 +1432,66 @@ static void fs_fixup_line(FuncState *fs, GCproto *pt,
1416 uint8_t *li = (uint8_t *)lineinfo; 1432 uint8_t *li = (uint8_t *)lineinfo;
1417 do { 1433 do {
1418 BCLine delta = base[i].line - first; 1434 BCLine delta = base[i].line - first;
1419 lua_assert(delta >= 0 && delta < 256); 1435 lj_assertFS(delta >= 0 && delta < 256, "bad line delta");
1420 li[i] = (uint8_t)delta; 1436 li[i] = (uint8_t)delta;
1421 } while (++i < n); 1437 } while (++i < n);
1422 } else if (LJ_LIKELY(numline < 65536)) { 1438 } else if (LJ_LIKELY(numline < 65536)) {
1423 uint16_t *li = (uint16_t *)lineinfo; 1439 uint16_t *li = (uint16_t *)lineinfo;
1424 do { 1440 do {
1425 BCLine delta = base[i].line - first; 1441 BCLine delta = base[i].line - first;
1426 lua_assert(delta >= 0 && delta < 65536); 1442 lj_assertFS(delta >= 0 && delta < 65536, "bad line delta");
1427 li[i] = (uint16_t)delta; 1443 li[i] = (uint16_t)delta;
1428 } while (++i < n); 1444 } while (++i < n);
1429 } else { 1445 } else {
1430 uint32_t *li = (uint32_t *)lineinfo; 1446 uint32_t *li = (uint32_t *)lineinfo;
1431 do { 1447 do {
1432 BCLine delta = base[i].line - first; 1448 BCLine delta = base[i].line - first;
1433 lua_assert(delta >= 0); 1449 lj_assertFS(delta >= 0, "bad line delta");
1434 li[i] = (uint32_t)delta; 1450 li[i] = (uint32_t)delta;
1435 } while (++i < n); 1451 } while (++i < n);
1436 } 1452 }
1437} 1453}
1438 1454
1439/* Resize buffer if needed. */
1440static LJ_NOINLINE void fs_buf_resize(LexState *ls, MSize len)
1441{
1442 MSize sz = ls->sb.sz * 2;
1443 while (ls->sb.n + len > sz) sz = sz * 2;
1444 lj_str_resizebuf(ls->L, &ls->sb, sz);
1445}
1446
1447static LJ_AINLINE void fs_buf_need(LexState *ls, MSize len)
1448{
1449 if (LJ_UNLIKELY(ls->sb.n + len > ls->sb.sz))
1450 fs_buf_resize(ls, len);
1451}
1452
1453/* Add string to buffer. */
1454static void fs_buf_str(LexState *ls, const char *str, MSize len)
1455{
1456 char *p = ls->sb.buf + ls->sb.n;
1457 MSize i;
1458 ls->sb.n += len;
1459 for (i = 0; i < len; i++) p[i] = str[i];
1460}
1461
1462/* Add ULEB128 value to buffer. */
1463static void fs_buf_uleb128(LexState *ls, uint32_t v)
1464{
1465 MSize n = ls->sb.n;
1466 uint8_t *p = (uint8_t *)ls->sb.buf;
1467 for (; v >= 0x80; v >>= 7)
1468 p[n++] = (uint8_t)((v & 0x7f) | 0x80);
1469 p[n++] = (uint8_t)v;
1470 ls->sb.n = n;
1471}
1472
1473/* Prepare variable info for prototype. */ 1455/* Prepare variable info for prototype. */
1474static size_t fs_prep_var(LexState *ls, FuncState *fs, size_t *ofsvar) 1456static size_t fs_prep_var(LexState *ls, FuncState *fs, size_t *ofsvar)
1475{ 1457{
1476 VarInfo *vs =ls->vstack, *ve; 1458 VarInfo *vs =ls->vstack, *ve;
1477 MSize i, n; 1459 MSize i, n;
1478 BCPos lastpc; 1460 BCPos lastpc;
1479 lj_str_resetbuf(&ls->sb); /* Copy to temp. string buffer. */ 1461 lj_buf_reset(&ls->sb); /* Copy to temp. string buffer. */
1480 /* Store upvalue names. */ 1462 /* Store upvalue names. */
1481 for (i = 0, n = fs->nuv; i < n; i++) { 1463 for (i = 0, n = fs->nuv; i < n; i++) {
1482 GCstr *s = strref(vs[fs->uvmap[i]].name); 1464 GCstr *s = strref(vs[fs->uvmap[i]].name);
1483 MSize len = s->len+1; 1465 MSize len = s->len+1;
1484 fs_buf_need(ls, len); 1466 char *p = lj_buf_more(&ls->sb, len);
1485 fs_buf_str(ls, strdata(s), len); 1467 p = lj_buf_wmem(p, strdata(s), len);
1468 ls->sb.w = p;
1486 } 1469 }
1487 *ofsvar = ls->sb.n; 1470 *ofsvar = sbuflen(&ls->sb);
1488 lastpc = 0; 1471 lastpc = 0;
1489 /* Store local variable names and compressed ranges. */ 1472 /* Store local variable names and compressed ranges. */
1490 for (ve = vs + ls->vtop, vs += fs->vbase; vs < ve; vs++) { 1473 for (ve = vs + ls->vtop, vs += fs->vbase; vs < ve; vs++) {
1491 if (!gola_isgotolabel(vs)) { 1474 if (!gola_isgotolabel(vs)) {
1492 GCstr *s = strref(vs->name); 1475 GCstr *s = strref(vs->name);
1493 BCPos startpc; 1476 BCPos startpc;
1477 char *p;
1494 if ((uintptr_t)s < VARNAME__MAX) { 1478 if ((uintptr_t)s < VARNAME__MAX) {
1495 fs_buf_need(ls, 1 + 2*5); 1479 p = lj_buf_more(&ls->sb, 1 + 2*5);
1496 ls->sb.buf[ls->sb.n++] = (uint8_t)(uintptr_t)s; 1480 *p++ = (char)(uintptr_t)s;
1497 } else { 1481 } else {
1498 MSize len = s->len+1; 1482 MSize len = s->len+1;
1499 fs_buf_need(ls, len + 2*5); 1483 p = lj_buf_more(&ls->sb, len + 2*5);
1500 fs_buf_str(ls, strdata(s), len); 1484 p = lj_buf_wmem(p, strdata(s), len);
1501 } 1485 }
1502 startpc = vs->startpc; 1486 startpc = vs->startpc;
1503 fs_buf_uleb128(ls, startpc-lastpc); 1487 p = lj_strfmt_wuleb128(p, startpc-lastpc);
1504 fs_buf_uleb128(ls, vs->endpc-startpc); 1488 p = lj_strfmt_wuleb128(p, vs->endpc-startpc);
1489 ls->sb.w = p;
1505 lastpc = startpc; 1490 lastpc = startpc;
1506 } 1491 }
1507 } 1492 }
1508 fs_buf_need(ls, 1); 1493 lj_buf_putb(&ls->sb, '\0'); /* Terminator for varinfo. */
1509 ls->sb.buf[ls->sb.n++] = '\0'; /* Terminator for varinfo. */ 1494 return sbuflen(&ls->sb);
1510 return ls->sb.n;
1511} 1495}
1512 1496
1513/* Fixup variable info for prototype. */ 1497/* Fixup variable info for prototype. */
@@ -1515,7 +1499,7 @@ static void fs_fixup_var(LexState *ls, GCproto *pt, uint8_t *p, size_t ofsvar)
1515{ 1499{
1516 setmref(pt->uvinfo, p); 1500 setmref(pt->uvinfo, p);
1517 setmref(pt->varinfo, (char *)p + ofsvar); 1501 setmref(pt->varinfo, (char *)p + ofsvar);
1518 memcpy(p, ls->sb.buf, ls->sb.n); /* Copy from temp. string buffer. */ 1502 memcpy(p, ls->sb.b, sbuflen(&ls->sb)); /* Copy from temp. buffer. */
1519} 1503}
1520#else 1504#else
1521 1505
@@ -1552,7 +1536,7 @@ static void fs_fixup_ret(FuncState *fs)
1552 } 1536 }
1553 fs->bl->flags |= FSCOPE_NOCLOSE; /* Handled above. */ 1537 fs->bl->flags |= FSCOPE_NOCLOSE; /* Handled above. */
1554 fscope_end(fs); 1538 fscope_end(fs);
1555 lua_assert(fs->bl == NULL); 1539 lj_assertFS(fs->bl == NULL, "bad scope nesting");
1556 /* May need to fixup returns encoded before first function was created. */ 1540 /* May need to fixup returns encoded before first function was created. */
1557 if (fs->flags & PROTO_FIXUP_RETURN) { 1541 if (fs->flags & PROTO_FIXUP_RETURN) {
1558 BCPos pc; 1542 BCPos pc;
@@ -1624,7 +1608,7 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
1624 L->top--; /* Pop table of constants. */ 1608 L->top--; /* Pop table of constants. */
1625 ls->vtop = fs->vbase; /* Reset variable stack. */ 1609 ls->vtop = fs->vbase; /* Reset variable stack. */
1626 ls->fs = fs->prev; 1610 ls->fs = fs->prev;
1627 lua_assert(ls->fs != NULL || ls->token == TK_eof); 1611 lj_assertL(ls->fs != NULL || ls->tok == TK_eof, "bad parser state");
1628 return pt; 1612 return pt;
1629} 1613}
1630 1614
@@ -1718,15 +1702,15 @@ static void expr_bracket(LexState *ls, ExpDesc *v)
1718} 1702}
1719 1703
1720/* Get value of constant expression. */ 1704/* Get value of constant expression. */
1721static void expr_kvalue(TValue *v, ExpDesc *e) 1705static void expr_kvalue(FuncState *fs, TValue *v, ExpDesc *e)
1722{ 1706{
1707 UNUSED(fs);
1723 if (e->k <= VKTRUE) { 1708 if (e->k <= VKTRUE) {
1724 setitype(v, ~(uint32_t)e->k); 1709 setpriV(v, ~(uint32_t)e->k);
1725 } else if (e->k == VKSTR) { 1710 } else if (e->k == VKSTR) {
1726 setgcref(v->gcr, obj2gco(e->u.sval)); 1711 setgcVraw(v, obj2gco(e->u.sval), LJ_TSTR);
1727 setitype(v, LJ_TSTR);
1728 } else { 1712 } else {
1729 lua_assert(tvisnumber(expr_numtv(e))); 1713 lj_assertFS(tvisnumber(expr_numtv(e)), "bad number constant");
1730 *v = *expr_numtv(e); 1714 *v = *expr_numtv(e);
1731 } 1715 }
1732} 1716}
@@ -1746,15 +1730,15 @@ static void expr_table(LexState *ls, ExpDesc *e)
1746 bcreg_reserve(fs, 1); 1730 bcreg_reserve(fs, 1);
1747 freg++; 1731 freg++;
1748 lex_check(ls, '{'); 1732 lex_check(ls, '{');
1749 while (ls->token != '}') { 1733 while (ls->tok != '}') {
1750 ExpDesc key, val; 1734 ExpDesc key, val;
1751 vcall = 0; 1735 vcall = 0;
1752 if (ls->token == '[') { 1736 if (ls->tok == '[') {
1753 expr_bracket(ls, &key); /* Already calls expr_toval. */ 1737 expr_bracket(ls, &key); /* Already calls expr_toval. */
1754 if (!expr_isk(&key)) expr_index(fs, e, &key); 1738 if (!expr_isk(&key)) expr_index(fs, e, &key);
1755 if (expr_isnumk(&key) && expr_numiszero(&key)) needarr = 1; else nhash++; 1739 if (expr_isnumk(&key) && expr_numiszero(&key)) needarr = 1; else nhash++;
1756 lex_check(ls, '='); 1740 lex_check(ls, '=');
1757 } else if ((ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) && 1741 } else if ((ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) &&
1758 lj_lex_lookahead(ls) == '=') { 1742 lj_lex_lookahead(ls) == '=') {
1759 expr_str(ls, &key); 1743 expr_str(ls, &key);
1760 lex_check(ls, '='); 1744 lex_check(ls, '=');
@@ -1776,11 +1760,11 @@ static void expr_table(LexState *ls, ExpDesc *e)
1776 fs->bcbase[pc].ins = BCINS_AD(BC_TDUP, freg-1, kidx); 1760 fs->bcbase[pc].ins = BCINS_AD(BC_TDUP, freg-1, kidx);
1777 } 1761 }
1778 vcall = 0; 1762 vcall = 0;
1779 expr_kvalue(&k, &key); 1763 expr_kvalue(fs, &k, &key);
1780 v = lj_tab_set(fs->L, t, &k); 1764 v = lj_tab_set(fs->L, t, &k);
1781 lj_gc_anybarriert(fs->L, t); 1765 lj_gc_anybarriert(fs->L, t);
1782 if (expr_isk_nojump(&val)) { /* Add const key/value to template table. */ 1766 if (expr_isk_nojump(&val)) { /* Add const key/value to template table. */
1783 expr_kvalue(v, &val); 1767 expr_kvalue(fs, v, &val);
1784 } else { /* Otherwise create dummy string key (avoids lj_tab_newkey). */ 1768 } else { /* Otherwise create dummy string key (avoids lj_tab_newkey). */
1785 settabV(fs->L, v, t); /* Preserve key with table itself as value. */ 1769 settabV(fs->L, v, t); /* Preserve key with table itself as value. */
1786 fixt = 1; /* Fix this later, after all resizes. */ 1770 fixt = 1; /* Fix this later, after all resizes. */
@@ -1799,8 +1783,9 @@ static void expr_table(LexState *ls, ExpDesc *e)
1799 if (vcall) { 1783 if (vcall) {
1800 BCInsLine *ilp = &fs->bcbase[fs->pc-1]; 1784 BCInsLine *ilp = &fs->bcbase[fs->pc-1];
1801 ExpDesc en; 1785 ExpDesc en;
1802 lua_assert(bc_a(ilp->ins) == freg && 1786 lj_assertFS(bc_a(ilp->ins) == freg &&
1803 bc_op(ilp->ins) == (narr > 256 ? BC_TSETV : BC_TSETB)); 1787 bc_op(ilp->ins) == (narr > 256 ? BC_TSETV : BC_TSETB),
1788 "bad CALL code generation");
1804 expr_init(&en, VKNUM, 0); 1789 expr_init(&en, VKNUM, 0);
1805 en.u.nval.u32.lo = narr-1; 1790 en.u.nval.u32.lo = narr-1;
1806 en.u.nval.u32.hi = 0x43300000; /* Biased integer to avoid denormals. */ 1791 en.u.nval.u32.hi = 0x43300000; /* Biased integer to avoid denormals. */
@@ -1830,7 +1815,7 @@ static void expr_table(LexState *ls, ExpDesc *e)
1830 for (i = 0; i <= hmask; i++) { 1815 for (i = 0; i <= hmask; i++) {
1831 Node *n = &node[i]; 1816 Node *n = &node[i];
1832 if (tvistab(&n->val)) { 1817 if (tvistab(&n->val)) {
1833 lua_assert(tabV(&n->val) == t); 1818 lj_assertFS(tabV(&n->val) == t, "bad dummy key in template table");
1834 setnilV(&n->val); /* Turn value into nil. */ 1819 setnilV(&n->val); /* Turn value into nil. */
1835 } 1820 }
1836 } 1821 }
@@ -1847,11 +1832,11 @@ static BCReg parse_params(LexState *ls, int needself)
1847 lex_check(ls, '('); 1832 lex_check(ls, '(');
1848 if (needself) 1833 if (needself)
1849 var_new_lit(ls, nparams++, "self"); 1834 var_new_lit(ls, nparams++, "self");
1850 if (ls->token != ')') { 1835 if (ls->tok != ')') {
1851 do { 1836 do {
1852 if (ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) { 1837 if (ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) {
1853 var_new(ls, nparams++, lex_str(ls)); 1838 var_new(ls, nparams++, lex_str(ls));
1854 } else if (ls->token == TK_dots) { 1839 } else if (ls->tok == TK_dots) {
1855 lj_lex_next(ls); 1840 lj_lex_next(ls);
1856 fs->flags |= PROTO_VARARG; 1841 fs->flags |= PROTO_VARARG;
1857 break; 1842 break;
@@ -1861,7 +1846,7 @@ static BCReg parse_params(LexState *ls, int needself)
1861 } while (lex_opt(ls, ',')); 1846 } while (lex_opt(ls, ','));
1862 } 1847 }
1863 var_add(ls, nparams); 1848 var_add(ls, nparams);
1864 lua_assert(fs->nactvar == nparams); 1849 lj_assertFS(fs->nactvar == nparams, "bad regalloc");
1865 bcreg_reserve(fs, nparams); 1850 bcreg_reserve(fs, nparams);
1866 lex_check(ls, ')'); 1851 lex_check(ls, ')');
1867 return nparams; 1852 return nparams;
@@ -1885,7 +1870,7 @@ static void parse_body(LexState *ls, ExpDesc *e, int needself, BCLine line)
1885 fs.bclim = pfs->bclim - pfs->pc; 1870 fs.bclim = pfs->bclim - pfs->pc;
1886 bcemit_AD(&fs, BC_FUNCF, 0, 0); /* Placeholder. */ 1871 bcemit_AD(&fs, BC_FUNCF, 0, 0); /* Placeholder. */
1887 parse_chunk(ls); 1872 parse_chunk(ls);
1888 if (ls->token != TK_end) lex_match(ls, TK_end, TK_function, line); 1873 if (ls->tok != TK_end) lex_match(ls, TK_end, TK_function, line);
1889 pt = fs_finish(ls, (ls->lastline = ls->linenumber)); 1874 pt = fs_finish(ls, (ls->lastline = ls->linenumber));
1890 pfs->bcbase = ls->bcstack + oldbase; /* May have been reallocated. */ 1875 pfs->bcbase = ls->bcstack + oldbase; /* May have been reallocated. */
1891 pfs->bclim = (BCPos)(ls->sizebcstack - oldbase); 1876 pfs->bclim = (BCPos)(ls->sizebcstack - oldbase);
@@ -1924,13 +1909,13 @@ static void parse_args(LexState *ls, ExpDesc *e)
1924 BCIns ins; 1909 BCIns ins;
1925 BCReg base; 1910 BCReg base;
1926 BCLine line = ls->linenumber; 1911 BCLine line = ls->linenumber;
1927 if (ls->token == '(') { 1912 if (ls->tok == '(') {
1928#if !LJ_52 1913#if !LJ_52
1929 if (line != ls->lastline) 1914 if (line != ls->lastline)
1930 err_syntax(ls, LJ_ERR_XAMBIG); 1915 err_syntax(ls, LJ_ERR_XAMBIG);
1931#endif 1916#endif
1932 lj_lex_next(ls); 1917 lj_lex_next(ls);
1933 if (ls->token == ')') { /* f(). */ 1918 if (ls->tok == ')') { /* f(). */
1934 args.k = VVOID; 1919 args.k = VVOID;
1935 } else { 1920 } else {
1936 expr_list(ls, &args); 1921 expr_list(ls, &args);
@@ -1938,24 +1923,24 @@ static void parse_args(LexState *ls, ExpDesc *e)
1938 setbc_b(bcptr(fs, &args), 0); /* Pass on multiple results. */ 1923 setbc_b(bcptr(fs, &args), 0); /* Pass on multiple results. */
1939 } 1924 }
1940 lex_match(ls, ')', '(', line); 1925 lex_match(ls, ')', '(', line);
1941 } else if (ls->token == '{') { 1926 } else if (ls->tok == '{') {
1942 expr_table(ls, &args); 1927 expr_table(ls, &args);
1943 } else if (ls->token == TK_string) { 1928 } else if (ls->tok == TK_string) {
1944 expr_init(&args, VKSTR, 0); 1929 expr_init(&args, VKSTR, 0);
1945 args.u.sval = strV(&ls->tokenval); 1930 args.u.sval = strV(&ls->tokval);
1946 lj_lex_next(ls); 1931 lj_lex_next(ls);
1947 } else { 1932 } else {
1948 err_syntax(ls, LJ_ERR_XFUNARG); 1933 err_syntax(ls, LJ_ERR_XFUNARG);
1949 return; /* Silence compiler. */ 1934 return; /* Silence compiler. */
1950 } 1935 }
1951 lua_assert(e->k == VNONRELOC); 1936 lj_assertFS(e->k == VNONRELOC, "bad expr type %d", e->k);
1952 base = e->u.s.info; /* Base register for call. */ 1937 base = e->u.s.info; /* Base register for call. */
1953 if (args.k == VCALL) { 1938 if (args.k == VCALL) {
1954 ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1); 1939 ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1 - LJ_FR2);
1955 } else { 1940 } else {
1956 if (args.k != VVOID) 1941 if (args.k != VVOID)
1957 expr_tonextreg(fs, &args); 1942 expr_tonextreg(fs, &args);
1958 ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base); 1943 ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base - LJ_FR2);
1959 } 1944 }
1960 expr_init(e, VCALL, bcemit_INS(fs, ins)); 1945 expr_init(e, VCALL, bcemit_INS(fs, ins));
1961 e->u.s.aux = base; 1946 e->u.s.aux = base;
@@ -1968,33 +1953,34 @@ static void expr_primary(LexState *ls, ExpDesc *v)
1968{ 1953{
1969 FuncState *fs = ls->fs; 1954 FuncState *fs = ls->fs;
1970 /* Parse prefix expression. */ 1955 /* Parse prefix expression. */
1971 if (ls->token == '(') { 1956 if (ls->tok == '(') {
1972 BCLine line = ls->linenumber; 1957 BCLine line = ls->linenumber;
1973 lj_lex_next(ls); 1958 lj_lex_next(ls);
1974 expr(ls, v); 1959 expr(ls, v);
1975 lex_match(ls, ')', '(', line); 1960 lex_match(ls, ')', '(', line);
1976 expr_discharge(ls->fs, v); 1961 expr_discharge(ls->fs, v);
1977 } else if (ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) { 1962 } else if (ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) {
1978 var_lookup(ls, v); 1963 var_lookup(ls, v);
1979 } else { 1964 } else {
1980 err_syntax(ls, LJ_ERR_XSYMBOL); 1965 err_syntax(ls, LJ_ERR_XSYMBOL);
1981 } 1966 }
1982 for (;;) { /* Parse multiple expression suffixes. */ 1967 for (;;) { /* Parse multiple expression suffixes. */
1983 if (ls->token == '.') { 1968 if (ls->tok == '.') {
1984 expr_field(ls, v); 1969 expr_field(ls, v);
1985 } else if (ls->token == '[') { 1970 } else if (ls->tok == '[') {
1986 ExpDesc key; 1971 ExpDesc key;
1987 expr_toanyreg(fs, v); 1972 expr_toanyreg(fs, v);
1988 expr_bracket(ls, &key); 1973 expr_bracket(ls, &key);
1989 expr_index(fs, v, &key); 1974 expr_index(fs, v, &key);
1990 } else if (ls->token == ':') { 1975 } else if (ls->tok == ':') {
1991 ExpDesc key; 1976 ExpDesc key;
1992 lj_lex_next(ls); 1977 lj_lex_next(ls);
1993 expr_str(ls, &key); 1978 expr_str(ls, &key);
1994 bcemit_method(fs, v, &key); 1979 bcemit_method(fs, v, &key);
1995 parse_args(ls, v); 1980 parse_args(ls, v);
1996 } else if (ls->token == '(' || ls->token == TK_string || ls->token == '{') { 1981 } else if (ls->tok == '(' || ls->tok == TK_string || ls->tok == '{') {
1997 expr_tonextreg(fs, v); 1982 expr_tonextreg(fs, v);
1983 if (LJ_FR2) bcreg_reserve(fs, 1);
1998 parse_args(ls, v); 1984 parse_args(ls, v);
1999 } else { 1985 } else {
2000 break; 1986 break;
@@ -2005,14 +1991,14 @@ static void expr_primary(LexState *ls, ExpDesc *v)
2005/* Parse simple expression. */ 1991/* Parse simple expression. */
2006static void expr_simple(LexState *ls, ExpDesc *v) 1992static void expr_simple(LexState *ls, ExpDesc *v)
2007{ 1993{
2008 switch (ls->token) { 1994 switch (ls->tok) {
2009 case TK_number: 1995 case TK_number:
2010 expr_init(v, (LJ_HASFFI && tviscdata(&ls->tokenval)) ? VKCDATA : VKNUM, 0); 1996 expr_init(v, (LJ_HASFFI && tviscdata(&ls->tokval)) ? VKCDATA : VKNUM, 0);
2011 copyTV(ls->L, &v->u.nval, &ls->tokenval); 1997 copyTV(ls->L, &v->u.nval, &ls->tokval);
2012 break; 1998 break;
2013 case TK_string: 1999 case TK_string:
2014 expr_init(v, VKSTR, 0); 2000 expr_init(v, VKSTR, 0);
2015 v->u.sval = strV(&ls->tokenval); 2001 v->u.sval = strV(&ls->tokval);
2016 break; 2002 break;
2017 case TK_nil: 2003 case TK_nil:
2018 expr_init(v, VKNIL, 0); 2004 expr_init(v, VKNIL, 0);
@@ -2100,11 +2086,11 @@ static BinOpr expr_binop(LexState *ls, ExpDesc *v, uint32_t limit);
2100static void expr_unop(LexState *ls, ExpDesc *v) 2086static void expr_unop(LexState *ls, ExpDesc *v)
2101{ 2087{
2102 BCOp op; 2088 BCOp op;
2103 if (ls->token == TK_not) { 2089 if (ls->tok == TK_not) {
2104 op = BC_NOT; 2090 op = BC_NOT;
2105 } else if (ls->token == '-') { 2091 } else if (ls->tok == '-') {
2106 op = BC_UNM; 2092 op = BC_UNM;
2107 } else if (ls->token == '#') { 2093 } else if (ls->tok == '#') {
2108 op = BC_LEN; 2094 op = BC_LEN;
2109 } else { 2095 } else {
2110 expr_simple(ls, v); 2096 expr_simple(ls, v);
@@ -2121,7 +2107,7 @@ static BinOpr expr_binop(LexState *ls, ExpDesc *v, uint32_t limit)
2121 BinOpr op; 2107 BinOpr op;
2122 synlevel_begin(ls); 2108 synlevel_begin(ls);
2123 expr_unop(ls, v); 2109 expr_unop(ls, v);
2124 op = token2binop(ls->token); 2110 op = token2binop(ls->tok);
2125 while (op != OPR_NOBINOPR && priority[op].left > limit) { 2111 while (op != OPR_NOBINOPR && priority[op].left > limit) {
2126 ExpDesc v2; 2112 ExpDesc v2;
2127 BinOpr nextop; 2113 BinOpr nextop;
@@ -2310,9 +2296,9 @@ static void parse_func(LexState *ls, BCLine line)
2310 lj_lex_next(ls); /* Skip 'function'. */ 2296 lj_lex_next(ls); /* Skip 'function'. */
2311 /* Parse function name. */ 2297 /* Parse function name. */
2312 var_lookup(ls, &v); 2298 var_lookup(ls, &v);
2313 while (ls->token == '.') /* Multiple dot-separated fields. */ 2299 while (ls->tok == '.') /* Multiple dot-separated fields. */
2314 expr_field(ls, &v); 2300 expr_field(ls, &v);
2315 if (ls->token == ':') { /* Optional colon to signify method call. */ 2301 if (ls->tok == ':') { /* Optional colon to signify method call. */
2316 needself = 1; 2302 needself = 1;
2317 expr_field(ls, &v); 2303 expr_field(ls, &v);
2318 } 2304 }
@@ -2325,9 +2311,9 @@ static void parse_func(LexState *ls, BCLine line)
2325/* -- Control transfer statements ----------------------------------------- */ 2311/* -- Control transfer statements ----------------------------------------- */
2326 2312
2327/* Check for end of block. */ 2313/* Check for end of block. */
2328static int endofblock(LexToken token) 2314static int parse_isend(LexToken tok)
2329{ 2315{
2330 switch (token) { 2316 switch (tok) {
2331 case TK_else: case TK_elseif: case TK_end: case TK_until: case TK_eof: 2317 case TK_else: case TK_elseif: case TK_end: case TK_until: case TK_eof:
2332 return 1; 2318 return 1;
2333 default: 2319 default:
@@ -2342,7 +2328,7 @@ static void parse_return(LexState *ls)
2342 FuncState *fs = ls->fs; 2328 FuncState *fs = ls->fs;
2343 lj_lex_next(ls); /* Skip 'return'. */ 2329 lj_lex_next(ls); /* Skip 'return'. */
2344 fs->flags |= PROTO_HAS_RETURN; 2330 fs->flags |= PROTO_HAS_RETURN;
2345 if (endofblock(ls->token) || ls->token == ';') { /* Bare return. */ 2331 if (parse_isend(ls->tok) || ls->tok == ';') { /* Bare return. */
2346 ins = BCINS_AD(BC_RET0, 0, 1); 2332 ins = BCINS_AD(BC_RET0, 0, 1);
2347 } else { /* Return with one or more values. */ 2333 } else { /* Return with one or more values. */
2348 ExpDesc e; /* Receives the _last_ expression in the list. */ 2334 ExpDesc e; /* Receives the _last_ expression in the list. */
@@ -2408,18 +2394,18 @@ static void parse_label(LexState *ls)
2408 lex_check(ls, TK_label); 2394 lex_check(ls, TK_label);
2409 /* Recursively parse trailing statements: labels and ';' (Lua 5.2 only). */ 2395 /* Recursively parse trailing statements: labels and ';' (Lua 5.2 only). */
2410 for (;;) { 2396 for (;;) {
2411 if (ls->token == TK_label) { 2397 if (ls->tok == TK_label) {
2412 synlevel_begin(ls); 2398 synlevel_begin(ls);
2413 parse_label(ls); 2399 parse_label(ls);
2414 synlevel_end(ls); 2400 synlevel_end(ls);
2415 } else if (LJ_52 && ls->token == ';') { 2401 } else if (LJ_52 && ls->tok == ';') {
2416 lj_lex_next(ls); 2402 lj_lex_next(ls);
2417 } else { 2403 } else {
2418 break; 2404 break;
2419 } 2405 }
2420 } 2406 }
2421 /* Trailing label is considered to be outside of scope. */ 2407 /* Trailing label is considered to be outside of scope. */
2422 if (endofblock(ls->token) && ls->token != TK_until) 2408 if (parse_isend(ls->tok) && ls->tok != TK_until)
2423 ls->vstack[idx].slot = fs->bl->nactvar; 2409 ls->vstack[idx].slot = fs->bl->nactvar;
2424 gola_resolve(ls, fs->bl, idx); 2410 gola_resolve(ls, fs->bl, idx);
2425} 2411}
@@ -2576,7 +2562,8 @@ static void parse_for_iter(LexState *ls, GCstr *indexname)
2576 lex_check(ls, TK_in); 2562 lex_check(ls, TK_in);
2577 line = ls->linenumber; 2563 line = ls->linenumber;
2578 assign_adjust(ls, 3, expr_list(ls, &e), &e); 2564 assign_adjust(ls, 3, expr_list(ls, &e), &e);
2579 bcreg_bump(fs, 3); /* The iterator needs another 3 slots (func + 2 args). */ 2565 /* The iterator needs another 3 [4] slots (func [pc] | state ctl). */
2566 bcreg_bump(fs, 3+LJ_FR2);
2580 isnext = (nvars <= 5 && predict_next(ls, fs, exprpc)); 2567 isnext = (nvars <= 5 && predict_next(ls, fs, exprpc));
2581 var_add(ls, 3); /* Hidden control variables. */ 2568 var_add(ls, 3); /* Hidden control variables. */
2582 lex_check(ls, TK_do); 2569 lex_check(ls, TK_do);
@@ -2604,9 +2591,9 @@ static void parse_for(LexState *ls, BCLine line)
2604 fscope_begin(fs, &bl, FSCOPE_LOOP); 2591 fscope_begin(fs, &bl, FSCOPE_LOOP);
2605 lj_lex_next(ls); /* Skip 'for'. */ 2592 lj_lex_next(ls); /* Skip 'for'. */
2606 varname = lex_str(ls); /* Get first variable name. */ 2593 varname = lex_str(ls); /* Get first variable name. */
2607 if (ls->token == '=') 2594 if (ls->tok == '=')
2608 parse_for_num(ls, varname, line); 2595 parse_for_num(ls, varname, line);
2609 else if (ls->token == ',' || ls->token == TK_in) 2596 else if (ls->tok == ',' || ls->tok == TK_in)
2610 parse_for_iter(ls, varname); 2597 parse_for_iter(ls, varname);
2611 else 2598 else
2612 err_syntax(ls, LJ_ERR_XFOR); 2599 err_syntax(ls, LJ_ERR_XFOR);
@@ -2632,12 +2619,12 @@ static void parse_if(LexState *ls, BCLine line)
2632 BCPos flist; 2619 BCPos flist;
2633 BCPos escapelist = NO_JMP; 2620 BCPos escapelist = NO_JMP;
2634 flist = parse_then(ls); 2621 flist = parse_then(ls);
2635 while (ls->token == TK_elseif) { /* Parse multiple 'elseif' blocks. */ 2622 while (ls->tok == TK_elseif) { /* Parse multiple 'elseif' blocks. */
2636 jmp_append(fs, &escapelist, bcemit_jmp(fs)); 2623 jmp_append(fs, &escapelist, bcemit_jmp(fs));
2637 jmp_tohere(fs, flist); 2624 jmp_tohere(fs, flist);
2638 flist = parse_then(ls); 2625 flist = parse_then(ls);
2639 } 2626 }
2640 if (ls->token == TK_else) { /* Parse optional 'else' block. */ 2627 if (ls->tok == TK_else) { /* Parse optional 'else' block. */
2641 jmp_append(fs, &escapelist, bcemit_jmp(fs)); 2628 jmp_append(fs, &escapelist, bcemit_jmp(fs));
2642 jmp_tohere(fs, flist); 2629 jmp_tohere(fs, flist);
2643 lj_lex_next(ls); /* Skip 'else'. */ 2630 lj_lex_next(ls); /* Skip 'else'. */
@@ -2655,7 +2642,7 @@ static void parse_if(LexState *ls, BCLine line)
2655static int parse_stmt(LexState *ls) 2642static int parse_stmt(LexState *ls)
2656{ 2643{
2657 BCLine line = ls->linenumber; 2644 BCLine line = ls->linenumber;
2658 switch (ls->token) { 2645 switch (ls->tok) {
2659 case TK_if: 2646 case TK_if:
2660 parse_if(ls, line); 2647 parse_if(ls, line);
2661 break; 2648 break;
@@ -2714,11 +2701,12 @@ static void parse_chunk(LexState *ls)
2714{ 2701{
2715 int islast = 0; 2702 int islast = 0;
2716 synlevel_begin(ls); 2703 synlevel_begin(ls);
2717 while (!islast && !endofblock(ls->token)) { 2704 while (!islast && !parse_isend(ls->tok)) {
2718 islast = parse_stmt(ls); 2705 islast = parse_stmt(ls);
2719 lex_opt(ls, ';'); 2706 lex_opt(ls, ';');
2720 lua_assert(ls->fs->framesize >= ls->fs->freereg && 2707 lj_assertLS(ls->fs->framesize >= ls->fs->freereg &&
2721 ls->fs->freereg >= ls->fs->nactvar); 2708 ls->fs->freereg >= ls->fs->nactvar,
2709 "bad regalloc");
2722 ls->fs->freereg = ls->fs->nactvar; /* Free registers after each stmt. */ 2710 ls->fs->freereg = ls->fs->nactvar; /* Free registers after each stmt. */
2723 } 2711 }
2724 synlevel_end(ls); 2712 synlevel_end(ls);
@@ -2749,13 +2737,12 @@ GCproto *lj_parse(LexState *ls)
2749 bcemit_AD(&fs, BC_FUNCV, 0, 0); /* Placeholder. */ 2737 bcemit_AD(&fs, BC_FUNCV, 0, 0); /* Placeholder. */
2750 lj_lex_next(ls); /* Read-ahead first token. */ 2738 lj_lex_next(ls); /* Read-ahead first token. */
2751 parse_chunk(ls); 2739 parse_chunk(ls);
2752 if (ls->token != TK_eof) 2740 if (ls->tok != TK_eof)
2753 err_token(ls, TK_eof); 2741 err_token(ls, TK_eof);
2754 pt = fs_finish(ls, ls->linenumber); 2742 pt = fs_finish(ls, ls->linenumber);
2755 L->top--; /* Drop chunkname. */ 2743 L->top--; /* Drop chunkname. */
2756 lua_assert(fs.prev == NULL); 2744 lj_assertL(fs.prev == NULL && ls->fs == NULL, "mismatched frame nesting");
2757 lua_assert(ls->fs == NULL); 2745 lj_assertL(pt->sizeuv == 0, "toplevel proto has upvalues");
2758 lua_assert(pt->sizeuv == 0);
2759 return pt; 2746 return pt;
2760} 2747}
2761 2748
diff --git a/src/lj_prng.c b/src/lj_prng.c
new file mode 100644
index 00000000..326b41e6
--- /dev/null
+++ b/src/lj_prng.c
@@ -0,0 +1,259 @@
1/*
2** Pseudo-random number generation.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_prng_c
7#define LUA_CORE
8
9/* To get the syscall prototype. */
10#if defined(__linux__) && !defined(_GNU_SOURCE)
11#define _GNU_SOURCE
12#endif
13
14#include "lj_def.h"
15#include "lj_arch.h"
16#include "lj_prng.h"
17
18/* -- PRNG step function -------------------------------------------------- */
19
20/* This implements a Tausworthe PRNG with period 2^223. Based on:
21** Tables of maximally-equidistributed combined LFSR generators,
22** Pierre L'Ecuyer, 1991, table 3, 1st entry.
23** Full-period ME-CF generator with L=64, J=4, k=223, N1=49.
24**
25** Important note: This PRNG is NOT suitable for cryptographic use!
26**
27** But it works fine for math.random(), which has an API that's not
28** suitable for cryptography, anyway.
29**
30** When used as a securely seeded global PRNG, it substantially raises
31** the difficulty for various attacks on the VM.
32*/
33
34/* Update generator i and compute a running xor of all states. */
35#define TW223_GEN(rs, z, r, i, k, q, s) \
36 z = rs->u[i]; \
37 z = (((z<<q)^z) >> (k-s)) ^ ((z&((uint64_t)(int64_t)-1 << (64-k)))<<s); \
38 r ^= z; rs->u[i] = z;
39
40#define TW223_STEP(rs, z, r) \
41 TW223_GEN(rs, z, r, 0, 63, 31, 18) \
42 TW223_GEN(rs, z, r, 1, 58, 19, 28) \
43 TW223_GEN(rs, z, r, 2, 55, 24, 7) \
44 TW223_GEN(rs, z, r, 3, 47, 21, 8)
45
46/* PRNG step function with uint64_t result. */
47LJ_NOINLINE uint64_t LJ_FASTCALL lj_prng_u64(PRNGState *rs)
48{
49 uint64_t z, r = 0;
50 TW223_STEP(rs, z, r)
51 return r;
52}
53
54/* PRNG step function with double in uint64_t result. */
55LJ_NOINLINE uint64_t LJ_FASTCALL lj_prng_u64d(PRNGState *rs)
56{
57 uint64_t z, r = 0;
58 TW223_STEP(rs, z, r)
59 /* Returns a double bit pattern in the range 1.0 <= d < 2.0. */
60 return (r & U64x(000fffff,ffffffff)) | U64x(3ff00000,00000000);
61}
62
63/* Condition seed: ensure k[i] MSB of u[i] are non-zero. */
64static LJ_AINLINE void lj_prng_condition(PRNGState *rs)
65{
66 if (rs->u[0] < (1u << 1)) rs->u[0] += (1u << 1);
67 if (rs->u[1] < (1u << 6)) rs->u[1] += (1u << 6);
68 if (rs->u[2] < (1u << 9)) rs->u[2] += (1u << 9);
69 if (rs->u[3] < (1u << 17)) rs->u[3] += (1u << 17);
70}
71
72/* -- PRNG seeding from OS ------------------------------------------------ */
73
74#if LUAJIT_SECURITY_PRNG == 0
75
76/* Nothing to define. */
77
78#elif LJ_TARGET_XBOX360
79
80extern int XNetRandom(void *buf, unsigned int len);
81
82#elif LJ_TARGET_PS3
83
84extern int sys_get_random_number(void *buf, uint64_t len);
85
86#elif LJ_TARGET_PS4 || LJ_TARGET_PS5 || LJ_TARGET_PSVITA
87
88extern int sceRandomGetRandomNumber(void *buf, size_t len);
89
90#elif LJ_TARGET_NX
91
92#include <unistd.h>
93
94#elif LJ_TARGET_WINDOWS || LJ_TARGET_XBOXONE
95
96#define WIN32_LEAN_AND_MEAN
97#include <windows.h>
98
99#if LJ_TARGET_UWP || LJ_TARGET_XBOXONE
100/* Must use BCryptGenRandom. */
101#include <bcrypt.h>
102#pragma comment(lib, "bcrypt.lib")
103#else
104/* If you wonder about this mess, then search online for RtlGenRandom. */
105typedef BOOLEAN (WINAPI *PRGR)(void *buf, ULONG len);
106static PRGR libfunc_rgr;
107#endif
108
109#elif LJ_TARGET_POSIX
110
111#if LJ_TARGET_LINUX
112/* Avoid a dependency on glibc 2.25+ and use the getrandom syscall instead. */
113#include <sys/syscall.h>
114#else
115
116#if LJ_TARGET_OSX && !LJ_TARGET_IOS
117/*
118** In their infinite wisdom Apple decided to disallow getentropy() in the
119** iOS App Store. Even though the call is common to all BSD-ish OS, it's
120** recommended by Apple in their own security-related docs, and, to top
121** off the foolery, /dev/urandom is handled by the same kernel code,
122** yet accessing it is actually permitted (but less efficient).
123*/
124#include <Availability.h>
125#if __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200
126#define LJ_TARGET_HAS_GETENTROPY 1
127#endif
128#elif (LJ_TARGET_BSD && !defined(__NetBSD__)) || LJ_TARGET_SOLARIS || LJ_TARGET_CYGWIN || LJ_TARGET_QNX
129#define LJ_TARGET_HAS_GETENTROPY 1
130#endif
131
132#if LJ_TARGET_HAS_GETENTROPY
133extern int getentropy(void *buf, size_t len)
134#ifdef __ELF__
135 __attribute__((weak))
136#endif
137;
138#endif
139
140#endif
141
142/* For the /dev/urandom fallback. */
143#include <fcntl.h>
144#include <unistd.h>
145
146#endif
147
148#if LUAJIT_SECURITY_PRNG == 0
149
150/* If you really don't care about security, then define
151** LUAJIT_SECURITY_PRNG=0. This yields a predictable seed
152** and provides NO SECURITY against various attacks on the VM.
153**
154** BTW: This is NOT the way to get predictable table iteration,
155** predictable trace generation, predictable bytecode generation, etc.
156*/
157int LJ_FASTCALL lj_prng_seed_secure(PRNGState *rs)
158{
159 lj_prng_seed_fixed(rs); /* The fixed seed is already conditioned. */
160 return 1;
161}
162
163#else
164
165/* Securely seed PRNG from system entropy. Returns 0 on failure. */
166int LJ_FASTCALL lj_prng_seed_secure(PRNGState *rs)
167{
168#if LJ_TARGET_XBOX360
169
170 if (XNetRandom(rs->u, (unsigned int)sizeof(rs->u)) == 0)
171 goto ok;
172
173#elif LJ_TARGET_PS3
174
175 if (sys_get_random_number(rs->u, sizeof(rs->u)) == 0)
176 goto ok;
177
178#elif LJ_TARGET_PS4 || LJ_TARGET_PS5 || LJ_TARGET_PSVITA
179
180 if (sceRandomGetRandomNumber(rs->u, sizeof(rs->u)) == 0)
181 goto ok;
182
183#elif LJ_TARGET_NX
184
185 if (getentropy(rs->u, sizeof(rs->u)) == 0)
186 goto ok;
187
188#elif LJ_TARGET_UWP || LJ_TARGET_XBOXONE
189
190 if (BCryptGenRandom(NULL, (PUCHAR)(rs->u), (ULONG)sizeof(rs->u),
191 BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0)
192 goto ok;
193
194#elif LJ_TARGET_WINDOWS
195
196 /* Keep the library loaded in case multiple VMs are started. */
197 if (!libfunc_rgr) {
198 HMODULE lib = LJ_WIN_LOADLIBA("advapi32.dll");
199 if (!lib) return 0;
200 libfunc_rgr = (PRGR)GetProcAddress(lib, "SystemFunction036");
201 if (!libfunc_rgr) return 0;
202 }
203 if (libfunc_rgr(rs->u, (ULONG)sizeof(rs->u)))
204 goto ok;
205
206#elif LJ_TARGET_POSIX
207
208#if LJ_TARGET_LINUX && defined(SYS_getrandom)
209
210 if (syscall(SYS_getrandom, rs->u, sizeof(rs->u), 0) == (long)sizeof(rs->u))
211 goto ok;
212
213#elif LJ_TARGET_HAS_GETENTROPY
214
215#ifdef __ELF__
216 if (&getentropy && getentropy(rs->u, sizeof(rs->u)) == 0)
217 goto ok;
218#else
219 if (getentropy(rs->u, sizeof(rs->u)) == 0)
220 goto ok;
221#endif
222
223#endif
224
225 /* Fallback to /dev/urandom. This may fail if the device is not
226 ** existent or accessible in a chroot or container, or if the process
227 ** or the OS ran out of file descriptors.
228 */
229 {
230 int fd = open("/dev/urandom", O_RDONLY|O_CLOEXEC);
231 if (fd != -1) {
232 ssize_t n = read(fd, rs->u, sizeof(rs->u));
233 (void)close(fd);
234 if (n == (ssize_t)sizeof(rs->u))
235 goto ok;
236 }
237 }
238
239#else
240
241 /* Add an elif above for your OS with a secure PRNG seed.
242 ** Note that fiddling around with rand(), getpid(), time() or coercing
243 ** ASLR to yield a few bits of randomness is not helpful.
244 ** If you don't want any security, then don't pretend you have any
245 ** and simply define LUAJIT_SECURITY_PRNG=0 for the build.
246 */
247#error "Missing secure PRNG seed for this OS"
248
249#endif
250 return 0; /* Fail. */
251
252ok:
253 lj_prng_condition(rs);
254 (void)lj_prng_u64(rs);
255 return 1; /* Success. */
256}
257
258#endif
259
diff --git a/src/lj_prng.h b/src/lj_prng.h
new file mode 100644
index 00000000..3dd9dbc0
--- /dev/null
+++ b/src/lj_prng.h
@@ -0,0 +1,24 @@
1/*
2** Pseudo-random number generation.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_PRNG_H
7#define _LJ_PRNG_H
8
9#include "lj_def.h"
10
11LJ_FUNC int LJ_FASTCALL lj_prng_seed_secure(PRNGState *rs);
12LJ_FUNC uint64_t LJ_FASTCALL lj_prng_u64(PRNGState *rs);
13LJ_FUNC uint64_t LJ_FASTCALL lj_prng_u64d(PRNGState *rs);
14
15/* This is just the precomputed result of lib_math.c:random_seed(rs, 0.0). */
16static LJ_AINLINE void lj_prng_seed_fixed(PRNGState *rs)
17{
18 rs->u[0] = U64x(a0d27757,0a345b8c);
19 rs->u[1] = U64x(764a296c,5d4aa64f);
20 rs->u[2] = U64x(51220704,070adeaa);
21 rs->u[3] = U64x(2a2717b5,a7b7b927);
22}
23
24#endif
diff --git a/src/lj_profile.c b/src/lj_profile.c
new file mode 100644
index 00000000..8cefd5fb
--- /dev/null
+++ b/src/lj_profile.c
@@ -0,0 +1,371 @@
1/*
2** Low-overhead profiling.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_profile_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASPROFILE
12
13#include "lj_buf.h"
14#include "lj_frame.h"
15#include "lj_debug.h"
16#include "lj_dispatch.h"
17#if LJ_HASJIT
18#include "lj_jit.h"
19#include "lj_trace.h"
20#endif
21#include "lj_profile.h"
22
23#include "luajit.h"
24
25#if LJ_PROFILE_SIGPROF
26
27#include <sys/time.h>
28#include <signal.h>
29#define profile_lock(ps) UNUSED(ps)
30#define profile_unlock(ps) UNUSED(ps)
31
32#elif LJ_PROFILE_PTHREAD
33
34#include <pthread.h>
35#include <time.h>
36#if LJ_TARGET_PS3
37#include <sys/timer.h>
38#endif
39#define profile_lock(ps) pthread_mutex_lock(&ps->lock)
40#define profile_unlock(ps) pthread_mutex_unlock(&ps->lock)
41
42#elif LJ_PROFILE_WTHREAD
43
44#define WIN32_LEAN_AND_MEAN
45#if LJ_TARGET_XBOX360
46#include <xtl.h>
47#include <xbox.h>
48#else
49#include <windows.h>
50#endif
51typedef unsigned int (WINAPI *WMM_TPFUNC)(unsigned int);
52#define profile_lock(ps) EnterCriticalSection(&ps->lock)
53#define profile_unlock(ps) LeaveCriticalSection(&ps->lock)
54
55#endif
56
57/* Profiler state. */
58typedef struct ProfileState {
59 global_State *g; /* VM state that started the profiler. */
60 luaJIT_profile_callback cb; /* Profiler callback. */
61 void *data; /* Profiler callback data. */
62 SBuf sb; /* String buffer for stack dumps. */
63 int interval; /* Sample interval in milliseconds. */
64 int samples; /* Number of samples for next callback. */
65 int vmstate; /* VM state when profile timer triggered. */
66#if LJ_PROFILE_SIGPROF
67 struct sigaction oldsa; /* Previous SIGPROF state. */
68#elif LJ_PROFILE_PTHREAD
69 pthread_mutex_t lock; /* g->hookmask update lock. */
70 pthread_t thread; /* Timer thread. */
71 int abort; /* Abort timer thread. */
72#elif LJ_PROFILE_WTHREAD
73#if LJ_TARGET_WINDOWS
74 HINSTANCE wmm; /* WinMM library handle. */
75 WMM_TPFUNC wmm_tbp; /* WinMM timeBeginPeriod function. */
76 WMM_TPFUNC wmm_tep; /* WinMM timeEndPeriod function. */
77#endif
78 CRITICAL_SECTION lock; /* g->hookmask update lock. */
79 HANDLE thread; /* Timer thread. */
80 int abort; /* Abort timer thread. */
81#endif
82} ProfileState;
83
84/* Sadly, we have to use a static profiler state.
85**
86** The SIGPROF variant needs a static pointer to the global state, anyway.
87** And it would be hard to extend for multiple threads. You can still use
88** multiple VMs in multiple threads, but only profile one at a time.
89*/
90static ProfileState profile_state;
91
92/* Default sample interval in milliseconds. */
93#define LJ_PROFILE_INTERVAL_DEFAULT 10
94
95/* -- Profiler/hook interaction ------------------------------------------- */
96
97#if !LJ_PROFILE_SIGPROF
98void LJ_FASTCALL lj_profile_hook_enter(global_State *g)
99{
100 ProfileState *ps = &profile_state;
101 if (ps->g) {
102 profile_lock(ps);
103 hook_enter(g);
104 profile_unlock(ps);
105 } else {
106 hook_enter(g);
107 }
108}
109
110void LJ_FASTCALL lj_profile_hook_leave(global_State *g)
111{
112 ProfileState *ps = &profile_state;
113 if (ps->g) {
114 profile_lock(ps);
115 hook_leave(g);
116 profile_unlock(ps);
117 } else {
118 hook_leave(g);
119 }
120}
121#endif
122
123/* -- Profile callbacks --------------------------------------------------- */
124
125/* Callback from profile hook (HOOK_PROFILE already cleared). */
126void LJ_FASTCALL lj_profile_interpreter(lua_State *L)
127{
128 ProfileState *ps = &profile_state;
129 global_State *g = G(L);
130 uint8_t mask;
131 profile_lock(ps);
132 mask = (g->hookmask & ~HOOK_PROFILE);
133 if (!(mask & HOOK_VMEVENT)) {
134 int samples = ps->samples;
135 ps->samples = 0;
136 g->hookmask = HOOK_VMEVENT;
137 lj_dispatch_update(g);
138 profile_unlock(ps);
139 ps->cb(ps->data, L, samples, ps->vmstate); /* Invoke user callback. */
140 profile_lock(ps);
141 mask |= (g->hookmask & HOOK_PROFILE);
142 }
143 g->hookmask = mask;
144 lj_dispatch_update(g);
145 profile_unlock(ps);
146}
147
148/* Trigger profile hook. Asynchronous call from OS-specific profile timer. */
149static void profile_trigger(ProfileState *ps)
150{
151 global_State *g = ps->g;
152 uint8_t mask;
153 profile_lock(ps);
154 ps->samples++; /* Always increment number of samples. */
155 mask = g->hookmask;
156 if (!(mask & (HOOK_PROFILE|HOOK_VMEVENT|HOOK_GC))) { /* Set profile hook. */
157 int st = g->vmstate;
158 ps->vmstate = st >= 0 ? 'N' :
159 st == ~LJ_VMST_INTERP ? 'I' :
160 st == ~LJ_VMST_C ? 'C' :
161 st == ~LJ_VMST_GC ? 'G' : 'J';
162 g->hookmask = (mask | HOOK_PROFILE);
163 lj_dispatch_update(g);
164 }
165 profile_unlock(ps);
166}
167
168/* -- OS-specific profile timer handling ---------------------------------- */
169
170#if LJ_PROFILE_SIGPROF
171
172/* SIGPROF handler. */
173static void profile_signal(int sig)
174{
175 UNUSED(sig);
176 profile_trigger(&profile_state);
177}
178
179/* Start profiling timer. */
180static void profile_timer_start(ProfileState *ps)
181{
182 int interval = ps->interval;
183 struct itimerval tm;
184 struct sigaction sa;
185 tm.it_value.tv_sec = tm.it_interval.tv_sec = interval / 1000;
186 tm.it_value.tv_usec = tm.it_interval.tv_usec = (interval % 1000) * 1000;
187 setitimer(ITIMER_PROF, &tm, NULL);
188#if LJ_TARGET_QNX
189 sa.sa_flags = 0;
190#else
191 sa.sa_flags = SA_RESTART;
192#endif
193 sa.sa_handler = profile_signal;
194 sigemptyset(&sa.sa_mask);
195 sigaction(SIGPROF, &sa, &ps->oldsa);
196}
197
198/* Stop profiling timer. */
199static void profile_timer_stop(ProfileState *ps)
200{
201 struct itimerval tm;
202 tm.it_value.tv_sec = tm.it_interval.tv_sec = 0;
203 tm.it_value.tv_usec = tm.it_interval.tv_usec = 0;
204 setitimer(ITIMER_PROF, &tm, NULL);
205 sigaction(SIGPROF, &ps->oldsa, NULL);
206}
207
208#elif LJ_PROFILE_PTHREAD
209
210/* POSIX timer thread. */
211static void *profile_thread(ProfileState *ps)
212{
213 int interval = ps->interval;
214#if !LJ_TARGET_PS3
215 struct timespec ts;
216 ts.tv_sec = interval / 1000;
217 ts.tv_nsec = (interval % 1000) * 1000000;
218#endif
219 while (1) {
220#if LJ_TARGET_PS3
221 sys_timer_usleep(interval * 1000);
222#else
223 nanosleep(&ts, NULL);
224#endif
225 if (ps->abort) break;
226 profile_trigger(ps);
227 }
228 return NULL;
229}
230
231/* Start profiling timer thread. */
232static void profile_timer_start(ProfileState *ps)
233{
234 pthread_mutex_init(&ps->lock, 0);
235 ps->abort = 0;
236 pthread_create(&ps->thread, NULL, (void *(*)(void *))profile_thread, ps);
237}
238
239/* Stop profiling timer thread. */
240static void profile_timer_stop(ProfileState *ps)
241{
242 ps->abort = 1;
243 pthread_join(ps->thread, NULL);
244 pthread_mutex_destroy(&ps->lock);
245}
246
247#elif LJ_PROFILE_WTHREAD
248
249/* Windows timer thread. */
250static DWORD WINAPI profile_thread(void *psx)
251{
252 ProfileState *ps = (ProfileState *)psx;
253 int interval = ps->interval;
254#if LJ_TARGET_WINDOWS && !LJ_TARGET_UWP
255 ps->wmm_tbp(interval);
256#endif
257 while (1) {
258 Sleep(interval);
259 if (ps->abort) break;
260 profile_trigger(ps);
261 }
262#if LJ_TARGET_WINDOWS && !LJ_TARGET_UWP
263 ps->wmm_tep(interval);
264#endif
265 return 0;
266}
267
268/* Start profiling timer thread. */
269static void profile_timer_start(ProfileState *ps)
270{
271#if LJ_TARGET_WINDOWS && !LJ_TARGET_UWP
272 if (!ps->wmm) { /* Load WinMM library on-demand. */
273 ps->wmm = LJ_WIN_LOADLIBA("winmm.dll");
274 if (ps->wmm) {
275 ps->wmm_tbp = (WMM_TPFUNC)GetProcAddress(ps->wmm, "timeBeginPeriod");
276 ps->wmm_tep = (WMM_TPFUNC)GetProcAddress(ps->wmm, "timeEndPeriod");
277 if (!ps->wmm_tbp || !ps->wmm_tep) {
278 ps->wmm = NULL;
279 return;
280 }
281 }
282 }
283#endif
284 InitializeCriticalSection(&ps->lock);
285 ps->abort = 0;
286 ps->thread = CreateThread(NULL, 0, profile_thread, ps, 0, NULL);
287}
288
289/* Stop profiling timer thread. */
290static void profile_timer_stop(ProfileState *ps)
291{
292 ps->abort = 1;
293 WaitForSingleObject(ps->thread, INFINITE);
294 DeleteCriticalSection(&ps->lock);
295}
296
297#endif
298
299/* -- Public profiling API ------------------------------------------------ */
300
301/* Start profiling. */
302LUA_API void luaJIT_profile_start(lua_State *L, const char *mode,
303 luaJIT_profile_callback cb, void *data)
304{
305 ProfileState *ps = &profile_state;
306 int interval = LJ_PROFILE_INTERVAL_DEFAULT;
307 while (*mode) {
308 int m = *mode++;
309 switch (m) {
310 case 'i':
311 interval = 0;
312 while (*mode >= '0' && *mode <= '9')
313 interval = interval * 10 + (*mode++ - '0');
314 if (interval <= 0) interval = 1;
315 break;
316#if LJ_HASJIT
317 case 'l': case 'f':
318 L2J(L)->prof_mode = m;
319 lj_trace_flushall(L);
320 break;
321#endif
322 default: /* Ignore unknown mode chars. */
323 break;
324 }
325 }
326 if (ps->g) {
327 luaJIT_profile_stop(L);
328 if (ps->g) return; /* Profiler in use by another VM. */
329 }
330 ps->g = G(L);
331 ps->interval = interval;
332 ps->cb = cb;
333 ps->data = data;
334 ps->samples = 0;
335 lj_buf_init(L, &ps->sb);
336 profile_timer_start(ps);
337}
338
339/* Stop profiling. */
340LUA_API void luaJIT_profile_stop(lua_State *L)
341{
342 ProfileState *ps = &profile_state;
343 global_State *g = ps->g;
344 if (G(L) == g) { /* Only stop profiler if started by this VM. */
345 profile_timer_stop(ps);
346 g->hookmask &= ~HOOK_PROFILE;
347 lj_dispatch_update(g);
348#if LJ_HASJIT
349 G2J(g)->prof_mode = 0;
350 lj_trace_flushall(L);
351#endif
352 lj_buf_free(g, &ps->sb);
353 ps->sb.w = ps->sb.e = NULL;
354 ps->g = NULL;
355 }
356}
357
358/* Return a compact stack dump. */
359LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt,
360 int depth, size_t *len)
361{
362 ProfileState *ps = &profile_state;
363 SBuf *sb = &ps->sb;
364 setsbufL(sb, L);
365 lj_buf_reset(sb);
366 lj_debug_dumpstack(L, sb, fmt, depth);
367 *len = (size_t)sbuflen(sb);
368 return sb->b;
369}
370
371#endif
diff --git a/src/lj_profile.h b/src/lj_profile.h
new file mode 100644
index 00000000..68bb9a1f
--- /dev/null
+++ b/src/lj_profile.h
@@ -0,0 +1,21 @@
1/*
2** Low-overhead profiling.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_PROFILE_H
7#define _LJ_PROFILE_H
8
9#include "lj_obj.h"
10
11#if LJ_HASPROFILE
12
13LJ_FUNC void LJ_FASTCALL lj_profile_interpreter(lua_State *L);
14#if !LJ_PROFILE_SIGPROF
15LJ_FUNC void LJ_FASTCALL lj_profile_hook_enter(global_State *g);
16LJ_FUNC void LJ_FASTCALL lj_profile_hook_leave(global_State *g);
17#endif
18
19#endif
20
21#endif
diff --git a/src/lj_record.c b/src/lj_record.c
index 0e14382c..9d0021a6 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -20,6 +20,9 @@
20#endif 20#endif
21#include "lj_bc.h" 21#include "lj_bc.h"
22#include "lj_ff.h" 22#include "lj_ff.h"
23#if LJ_HASPROFILE
24#include "lj_debug.h"
25#endif
23#include "lj_ir.h" 26#include "lj_ir.h"
24#include "lj_jit.h" 27#include "lj_jit.h"
25#include "lj_ircall.h" 28#include "lj_ircall.h"
@@ -30,6 +33,7 @@
30#include "lj_snap.h" 33#include "lj_snap.h"
31#include "lj_dispatch.h" 34#include "lj_dispatch.h"
32#include "lj_vm.h" 35#include "lj_vm.h"
36#include "lj_prng.h"
33 37
34/* Some local macros to save typing. Undef'd at the end. */ 38/* Some local macros to save typing. Undef'd at the end. */
35#define IR(ref) (&J->cur.ir[(ref)]) 39#define IR(ref) (&J->cur.ir[(ref)])
@@ -47,31 +51,52 @@
47static void rec_check_ir(jit_State *J) 51static void rec_check_ir(jit_State *J)
48{ 52{
49 IRRef i, nins = J->cur.nins, nk = J->cur.nk; 53 IRRef i, nins = J->cur.nins, nk = J->cur.nk;
50 lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536); 54 lj_assertJ(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536,
51 for (i = nins-1; i >= nk; i--) { 55 "inconsistent IR layout");
56 for (i = nk; i < nins; i++) {
52 IRIns *ir = IR(i); 57 IRIns *ir = IR(i);
53 uint32_t mode = lj_ir_mode[ir->o]; 58 uint32_t mode = lj_ir_mode[ir->o];
54 IRRef op1 = ir->op1; 59 IRRef op1 = ir->op1;
55 IRRef op2 = ir->op2; 60 IRRef op2 = ir->op2;
61 const char *err = NULL;
56 switch (irm_op1(mode)) { 62 switch (irm_op1(mode)) {
57 case IRMnone: lua_assert(op1 == 0); break; 63 case IRMnone:
58 case IRMref: lua_assert(op1 >= nk); 64 if (op1 != 0) err = "IRMnone op1 used";
59 lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break; 65 break;
66 case IRMref:
67 if (op1 < nk || (i >= REF_BIAS ? op1 >= i : op1 <= i))
68 err = "IRMref op1 out of range";
69 break;
60 case IRMlit: break; 70 case IRMlit: break;
61 case IRMcst: lua_assert(i < REF_BIAS); continue; 71 case IRMcst:
72 if (i >= REF_BIAS) { err = "constant in IR range"; break; }
73 if (irt_is64(ir->t) && ir->o != IR_KNULL)
74 i++;
75 continue;
62 } 76 }
63 switch (irm_op2(mode)) { 77 switch (irm_op2(mode)) {
64 case IRMnone: lua_assert(op2 == 0); break; 78 case IRMnone:
65 case IRMref: lua_assert(op2 >= nk); 79 if (op2) err = "IRMnone op2 used";
66 lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break; 80 break;
81 case IRMref:
82 if (op2 < nk || (i >= REF_BIAS ? op2 >= i : op2 <= i))
83 err = "IRMref op2 out of range";
84 break;
67 case IRMlit: break; 85 case IRMlit: break;
68 case IRMcst: lua_assert(0); break; 86 case IRMcst: err = "IRMcst op2"; break;
69 } 87 }
70 if (ir->prev) { 88 if (!err && ir->prev) {
71 lua_assert(ir->prev >= nk); 89 if (ir->prev < nk || (i >= REF_BIAS ? ir->prev >= i : ir->prev <= i))
72 lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i); 90 err = "chain out of range";
73 lua_assert(ir->o == IR_NOP || IR(ir->prev)->o == ir->o); 91 else if (ir->o != IR_NOP && IR(ir->prev)->o != ir->o)
92 err = "chain to different op";
74 } 93 }
94 lj_assertJ(!err, "bad IR %04d op %d(%04d,%04d): %s",
95 i-REF_BIAS,
96 ir->o,
97 irm_op1(mode) == IRMref ? op1-REF_BIAS : op1,
98 irm_op2(mode) == IRMref ? op2-REF_BIAS : op2,
99 err);
75 } 100 }
76} 101}
77 102
@@ -81,48 +106,80 @@ static void rec_check_slots(jit_State *J)
81 BCReg s, nslots = J->baseslot + J->maxslot; 106 BCReg s, nslots = J->baseslot + J->maxslot;
82 int32_t depth = 0; 107 int32_t depth = 0;
83 cTValue *base = J->L->base - J->baseslot; 108 cTValue *base = J->L->base - J->baseslot;
84 lua_assert(J->baseslot >= 1); 109 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot");
85 lua_assert(J->baseslot == 1 || (J->slot[J->baseslot-1] & TREF_FRAME)); 110 lj_assertJ(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME),
86 lua_assert(nslots <= LJ_MAX_JSLOTS); 111 "baseslot does not point to frame");
112 lj_assertJ(nslots <= LJ_MAX_JSLOTS, "slot overflow");
87 for (s = 0; s < nslots; s++) { 113 for (s = 0; s < nslots; s++) {
88 TRef tr = J->slot[s]; 114 TRef tr = J->slot[s];
89 if (tr) { 115 if (tr) {
90 cTValue *tv = &base[s]; 116 cTValue *tv = &base[s];
91 IRRef ref = tref_ref(tr); 117 IRRef ref = tref_ref(tr);
92 IRIns *ir; 118 IRIns *ir = NULL; /* Silence compiler. */
93 lua_assert(ref >= J->cur.nk && ref < J->cur.nins); 119 lj_assertJ(tv < J->L->top, "slot %d above top of Lua stack", s);
94 ir = IR(ref); 120 if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) {
95 lua_assert(irt_t(ir->t) == tref_t(tr)); 121 lj_assertJ(ref >= J->cur.nk && ref < J->cur.nins,
122 "slot %d ref %04d out of range", s, ref - REF_BIAS);
123 ir = IR(ref);
124 lj_assertJ(irt_t(ir->t) == tref_t(tr), "slot %d IR type mismatch", s);
125 }
96 if (s == 0) { 126 if (s == 0) {
97 lua_assert(tref_isfunc(tr)); 127 lj_assertJ(tref_isfunc(tr), "frame slot 0 is not a function");
128#if LJ_FR2
129 } else if (s == 1) {
130 lj_assertJ((tr & ~TREF_FRAME) == 0, "bad frame slot 1");
131#endif
98 } else if ((tr & TREF_FRAME)) { 132 } else if ((tr & TREF_FRAME)) {
99 GCfunc *fn = gco2func(frame_gc(tv)); 133 GCfunc *fn = gco2func(frame_gc(tv));
100 BCReg delta = (BCReg)(tv - frame_prev(tv)); 134 BCReg delta = (BCReg)(tv - frame_prev(tv));
101 lua_assert(tref_isfunc(tr)); 135#if LJ_FR2
102 if (tref_isk(tr)) lua_assert(fn == ir_kfunc(ir)); 136 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
103 lua_assert(s > delta ? (J->slot[s-delta] & TREF_FRAME) : (s == delta)); 137 "frame slot %d PC mismatch", s);
138 tr = J->slot[s-1];
139 ir = IR(tref_ref(tr));
140#endif
141 lj_assertJ(tref_isfunc(tr),
142 "frame slot %d is not a function", s-LJ_FR2);
143 lj_assertJ(!tref_isk(tr) || fn == ir_kfunc(ir),
144 "frame slot %d function mismatch", s-LJ_FR2);
145 lj_assertJ(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME)
146 : (s == delta + LJ_FR2),
147 "frame slot %d broken chain", s-LJ_FR2);
104 depth++; 148 depth++;
105 } else if ((tr & TREF_CONT)) { 149 } else if ((tr & TREF_CONT)) {
106 lua_assert(ir_kptr(ir) == gcrefp(tv->gcr, void)); 150#if LJ_FR2
107 lua_assert((J->slot[s+1] & TREF_FRAME)); 151 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
152 "cont slot %d continuation mismatch", s);
153#else
154 lj_assertJ(ir_kptr(ir) == gcrefp(tv->gcr, void),
155 "cont slot %d continuation mismatch", s);
156#endif
157 lj_assertJ((J->slot[s+1+LJ_FR2] & TREF_FRAME),
158 "cont slot %d not followed by frame", s);
108 depth++; 159 depth++;
160 } else if ((tr & TREF_KEYINDEX)) {
161 lj_assertJ(tref_isint(tr), "keyindex slot %d bad type %d",
162 s, tref_type(tr));
109 } else { 163 } else {
110 if (tvisnumber(tv)) 164 /* Number repr. may differ, but other types must be the same. */
111 lua_assert(tref_isnumber(tr)); /* Could be IRT_INT etc., too. */ 165 lj_assertJ(tvisnumber(tv) ? tref_isnumber(tr) :
112 else 166 itype2irt(tv) == tref_type(tr),
113 lua_assert(itype2irt(tv) == tref_type(tr)); 167 "slot %d type mismatch: stack type %d vs IR type %d",
168 s, itypemap(tv), tref_type(tr));
114 if (tref_isk(tr)) { /* Compare constants. */ 169 if (tref_isk(tr)) { /* Compare constants. */
115 TValue tvk; 170 TValue tvk;
116 lj_ir_kvalue(J->L, &tvk, ir); 171 lj_ir_kvalue(J->L, &tvk, ir);
117 if (!(tvisnum(&tvk) && tvisnan(&tvk))) 172 lj_assertJ((tvisnum(&tvk) && tvisnan(&tvk)) ?
118 lua_assert(lj_obj_equal(tv, &tvk)); 173 (tvisnum(tv) && tvisnan(tv)) :
119 else 174 lj_obj_equal(tv, &tvk),
120 lua_assert(tvisnum(tv) && tvisnan(tv)); 175 "slot %d const mismatch: stack %016llx vs IR %016llx",
176 s, tv->u64, tvk.u64);
121 } 177 }
122 } 178 }
123 } 179 }
124 } 180 }
125 lua_assert(J->framedepth == depth); 181 lj_assertJ(J->framedepth == depth,
182 "frame depth mismatch %d vs %d", J->framedepth, depth);
126} 183}
127#endif 184#endif
128 185
@@ -156,10 +213,11 @@ static TRef sload(jit_State *J, int32_t slot)
156/* Get TRef for current function. */ 213/* Get TRef for current function. */
157static TRef getcurrf(jit_State *J) 214static TRef getcurrf(jit_State *J)
158{ 215{
159 if (J->base[-1]) 216 if (J->base[-1-LJ_FR2])
160 return J->base[-1]; 217 return J->base[-1-LJ_FR2];
161 lua_assert(J->baseslot == 1); 218 /* Non-base frame functions ought to be loaded already. */
162 return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY); 219 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot");
220 return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY);
163} 221}
164 222
165/* Compare for raw object equality. 223/* Compare for raw object equality.
@@ -205,6 +263,14 @@ TRef lj_record_constify(jit_State *J, cTValue *o)
205 return 0; /* Can't represent lightuserdata (pointless). */ 263 return 0; /* Can't represent lightuserdata (pointless). */
206} 264}
207 265
266/* Emit a VLOAD with the correct type. */
267TRef lj_record_vload(jit_State *J, TRef ref, MSize idx, IRType t)
268{
269 TRef tr = emitir(IRTG(IR_VLOAD, t), ref, idx);
270 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */
271 return tr;
272}
273
208/* -- Record loop ops ----------------------------------------------------- */ 274/* -- Record loop ops ----------------------------------------------------- */
209 275
210/* Loop event. */ 276/* Loop event. */
@@ -221,17 +287,21 @@ static void canonicalize_slots(jit_State *J)
221 if (LJ_DUALNUM) return; 287 if (LJ_DUALNUM) return;
222 for (s = J->baseslot+J->maxslot-1; s >= 1; s--) { 288 for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
223 TRef tr = J->slot[s]; 289 TRef tr = J->slot[s];
224 if (tref_isinteger(tr)) { 290 if (tref_isinteger(tr) && !(tr & TREF_KEYINDEX)) {
225 IRIns *ir = IR(tref_ref(tr)); 291 IRIns *ir = IR(tref_ref(tr));
226 if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY))) 292 if (!(ir->o == IR_SLOAD && (ir->op2 & (IRSLOAD_READONLY))))
227 J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT); 293 J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
228 } 294 }
229 } 295 }
230} 296}
231 297
232/* Stop recording. */ 298/* Stop recording. */
233static void rec_stop(jit_State *J, TraceLink linktype, TraceNo lnk) 299void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
234{ 300{
301#ifdef LUAJIT_ENABLE_TABLE_BUMP
302 if (J->retryrec)
303 lj_trace_err(J, LJ_TRERR_RETRY);
304#endif
235 lj_trace_end(J); 305 lj_trace_end(J);
236 J->cur.linktype = (uint8_t)linktype; 306 J->cur.linktype = (uint8_t)linktype;
237 J->cur.link = (uint16_t)lnk; 307 J->cur.link = (uint16_t)lnk;
@@ -399,7 +469,8 @@ static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
399 TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode); 469 TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
400 TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode); 470 TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
401 int tc, dir = rec_for_direction(&tv[FORL_STEP]); 471 int tc, dir = rec_for_direction(&tv[FORL_STEP]);
402 lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI); 472 lj_assertJ(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI,
473 "bad bytecode %d instead of FORI/JFORI", bc_op(*fori));
403 scev->t.irt = t; 474 scev->t.irt = t;
404 scev->dir = dir; 475 scev->dir = dir;
405 scev->stop = tref_ref(stop); 476 scev->stop = tref_ref(stop);
@@ -455,7 +526,7 @@ static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
455 IRT_NUM; 526 IRT_NUM;
456 for (i = FORL_IDX; i <= FORL_STEP; i++) { 527 for (i = FORL_IDX; i <= FORL_STEP; i++) {
457 if (!tr[i]) sload(J, ra+i); 528 if (!tr[i]) sload(J, ra+i);
458 lua_assert(tref_isnumber_str(tr[i])); 529 lj_assertJ(tref_isnumber_str(tr[i]), "bad FORI argument type");
459 if (tref_isstr(tr[i])) 530 if (tref_isstr(tr[i]))
460 tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0); 531 tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
461 if (t == IRT_INT) { 532 if (t == IRT_INT) {
@@ -499,8 +570,7 @@ static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
499static LoopEvent rec_iterl(jit_State *J, const BCIns iterins) 570static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
500{ 571{
501 BCReg ra = bc_a(iterins); 572 BCReg ra = bc_a(iterins);
502 lua_assert(J->base[ra] != 0); 573 if (!tref_isnil(getslot(J, ra))) { /* Looping back? */
503 if (!tref_isnil(J->base[ra])) { /* Looping back? */
504 J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */ 574 J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */
505 J->maxslot = ra-1+bc_b(J->pc[-1]); 575 J->maxslot = ra-1+bc_b(J->pc[-1]);
506 J->pc += bc_j(iterins)+1; 576 J->pc += bc_j(iterins)+1;
@@ -538,12 +608,13 @@ static int innerloopleft(jit_State *J, const BCIns *pc)
538/* Handle the case when an interpreted loop op is hit. */ 608/* Handle the case when an interpreted loop op is hit. */
539static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev) 609static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
540{ 610{
541 if (J->parent == 0) { 611 if (J->parent == 0 && J->exitno == 0) {
542 if (pc == J->startpc && J->framedepth + J->retdepth == 0) { 612 if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
613 if (bc_op(J->cur.startins) == BC_ITERN) return; /* See rec_itern(). */
543 /* Same loop? */ 614 /* Same loop? */
544 if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */ 615 if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */
545 lj_trace_err(J, LJ_TRERR_LLEAVE); 616 lj_trace_err(J, LJ_TRERR_LLEAVE);
546 rec_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping root trace. */ 617 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */
547 } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */ 618 } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */
548 /* It's usually better to abort here and wait until the inner loop 619 /* It's usually better to abort here and wait until the inner loop
549 ** is traced. But if the inner loop repeatedly didn't loop back, 620 ** is traced. But if the inner loop repeatedly didn't loop back,
@@ -568,18 +639,136 @@ static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
568/* Handle the case when an already compiled loop op is hit. */ 639/* Handle the case when an already compiled loop op is hit. */
569static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev) 640static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
570{ 641{
571 if (J->parent == 0) { /* Root trace hit an inner loop. */ 642 if (J->parent == 0 && J->exitno == 0) { /* Root trace hit an inner loop. */
572 /* Better let the inner loop spawn a side trace back here. */ 643 /* Better let the inner loop spawn a side trace back here. */
573 lj_trace_err(J, LJ_TRERR_LINNER); 644 lj_trace_err(J, LJ_TRERR_LINNER);
574 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */ 645 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */
575 J->instunroll = 0; /* Cannot continue across a compiled loop op. */ 646 J->instunroll = 0; /* Cannot continue across a compiled loop op. */
576 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) 647 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
577 rec_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form an extra loop. */ 648 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form extra loop. */
578 else 649 else
579 rec_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */ 650 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */
580 } /* Side trace continues across a loop that's left or not entered. */ 651 } /* Side trace continues across a loop that's left or not entered. */
581} 652}
582 653
654/* Record ITERN. */
655static LoopEvent rec_itern(jit_State *J, BCReg ra, BCReg rb)
656{
657#if LJ_BE
658 /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
659 ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
660 */
661 UNUSED(ra); UNUSED(rb);
662 setintV(&J->errinfo, (int32_t)BC_ITERN);
663 lj_trace_err_info(J, LJ_TRERR_NYIBC);
664#else
665 RecordIndex ix;
666 /* Since ITERN is recorded at the start, we need our own loop detection. */
667 if (J->pc == J->startpc &&
668 J->framedepth + J->retdepth == 0 && J->parent == 0 && J->exitno == 0) {
669 IRRef ref = REF_FIRST + LJ_HASPROFILE;
670#ifdef LUAJIT_ENABLE_CHECKHOOK
671 ref += 3;
672#endif
673 if (J->cur.nins > ref ||
674 (LJ_HASPROFILE && J->cur.nins == ref && J->cur.ir[ref-1].o != IR_PROF)) {
675 J->instunroll = 0; /* Cannot continue unrolling across an ITERN. */
676 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */
677 return LOOPEV_ENTER;
678 }
679 }
680 J->maxslot = ra;
681 lj_snap_add(J); /* Required to make JLOOP the first ins in a side-trace. */
682 ix.tab = getslot(J, ra-2);
683 ix.key = J->base[ra-1] ? J->base[ra-1] :
684 sloadt(J, (int32_t)(ra-1), IRT_GUARD|IRT_INT,
685 IRSLOAD_TYPECHECK|IRSLOAD_KEYINDEX);
686 copyTV(J->L, &ix.tabv, &J->L->base[ra-2]);
687 copyTV(J->L, &ix.keyv, &J->L->base[ra-1]);
688 ix.idxchain = (rb < 3); /* Omit value type check, if unused. */
689 ix.mobj = 1; /* We need the next index, too. */
690 J->maxslot = ra + lj_record_next(J, &ix);
691 J->needsnap = 1;
692 if (!tref_isnil(ix.key)) { /* Looping back? */
693 J->base[ra-1] = ix.mobj | TREF_KEYINDEX; /* Control var has next index. */
694 J->base[ra] = ix.key;
695 J->base[ra+1] = ix.val;
696 J->pc += bc_j(J->pc[1])+2;
697 return LOOPEV_ENTER;
698 } else {
699 J->maxslot = ra-3;
700 J->pc += 2;
701 return LOOPEV_LEAVE;
702 }
703#endif
704}
705
706/* Record ISNEXT. */
707static void rec_isnext(jit_State *J, BCReg ra)
708{
709 cTValue *b = &J->L->base[ra-3];
710 if (tvisfunc(b) && funcV(b)->c.ffid == FF_next &&
711 tvistab(b+1) && tvisnil(b+2)) {
712 /* These checks are folded away for a compiled pairs(). */
713 TRef func = getslot(J, ra-3);
714 TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), func, IRFL_FUNC_FFID);
715 emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, FF_next));
716 (void)getslot(J, ra-2); /* Type check for table. */
717 (void)getslot(J, ra-1); /* Type check for nil key. */
718 J->base[ra-1] = lj_ir_kint(J, 0) | TREF_KEYINDEX;
719 J->maxslot = ra;
720 } else { /* Abort trace. Interpreter will despecialize bytecode. */
721 lj_trace_err(J, LJ_TRERR_RECERR);
722 }
723}
724
725/* -- Record profiler hook checks ----------------------------------------- */
726
727#if LJ_HASPROFILE
728
729/* Need to insert profiler hook check? */
730static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
731{
732 GCproto *ppt;
733 lj_assertJ(J->prof_mode == 'f' || J->prof_mode == 'l',
734 "bad profiler mode %c", J->prof_mode);
735 if (!pt)
736 return 0;
737 ppt = J->prev_pt;
738 J->prev_pt = pt;
739 if (pt != ppt && ppt) {
740 J->prev_line = -1;
741 return 1;
742 }
743 if (J->prof_mode == 'l') {
744 BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
745 BCLine pline = J->prev_line;
746 J->prev_line = line;
747 if (pline != line)
748 return 1;
749 }
750 return 0;
751}
752
753static void rec_profile_ins(jit_State *J, const BCIns *pc)
754{
755 if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
756 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
757 lj_snap_add(J);
758 }
759}
760
761static void rec_profile_ret(jit_State *J)
762{
763 if (J->prof_mode == 'f') {
764 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
765 J->prev_pt = NULL;
766 lj_snap_add(J);
767 }
768}
769
770#endif
771
583/* -- Record calls and returns -------------------------------------------- */ 772/* -- Record calls and returns -------------------------------------------- */
584 773
585/* Specialize to the runtime value of the called function or its prototype. */ 774/* Specialize to the runtime value of the called function or its prototype. */
@@ -590,11 +779,26 @@ static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
590 GCproto *pt = funcproto(fn); 779 GCproto *pt = funcproto(fn);
591 /* Too many closures created? Probably not a monomorphic function. */ 780 /* Too many closures created? Probably not a monomorphic function. */
592 if (pt->flags >= PROTO_CLC_POLY) { /* Specialize to prototype instead. */ 781 if (pt->flags >= PROTO_CLC_POLY) { /* Specialize to prototype instead. */
593 TRef trpt = emitir(IRT(IR_FLOAD, IRT_P32), tr, IRFL_FUNC_PC); 782 TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC);
594 emitir(IRTG(IR_EQ, IRT_P32), trpt, lj_ir_kptr(J, proto_bc(pt))); 783 emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt)));
595 (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */ 784 (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */
596 return tr; 785 return tr;
597 } 786 }
787 } else {
788 /* Don't specialize to non-monomorphic builtins. */
789 switch (fn->c.ffid) {
790 case FF_coroutine_wrap_aux:
791 case FF_string_gmatch_aux:
792 /* NYI: io_file_iter doesn't have an ffid, yet. */
793 { /* Specialize to the ffid. */
794 TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
795 emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, fn->c.ffid));
796 }
797 return tr;
798 default:
799 /* NYI: don't specialize to non-monomorphic C functions. */
800 break;
801 }
598 } 802 }
599 /* Otherwise specialize to the function (closure) value itself. */ 803 /* Otherwise specialize to the function (closure) value itself. */
600 kfunc = lj_ir_kfunc(J, fn); 804 kfunc = lj_ir_kfunc(J, fn);
@@ -607,21 +811,31 @@ static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
607{ 811{
608 RecordIndex ix; 812 RecordIndex ix;
609 TValue *functv = &J->L->base[func]; 813 TValue *functv = &J->L->base[func];
610 TRef *fbase = &J->base[func]; 814 TRef kfunc, *fbase = &J->base[func];
611 ptrdiff_t i; 815 ptrdiff_t i;
612 for (i = 0; i <= nargs; i++) 816 (void)getslot(J, func); /* Ensure func has a reference. */
613 (void)getslot(J, func+i); /* Ensure func and all args have a reference. */ 817 for (i = 1; i <= nargs; i++)
818 (void)getslot(J, func+LJ_FR2+i); /* Ensure all args have a reference. */
614 if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */ 819 if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */
615 ix.tab = fbase[0]; 820 ix.tab = fbase[0];
616 copyTV(J->L, &ix.tabv, functv); 821 copyTV(J->L, &ix.tabv, functv);
617 if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj)) 822 if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
618 lj_trace_err(J, LJ_TRERR_NOMM); 823 lj_trace_err(J, LJ_TRERR_NOMM);
619 for (i = ++nargs; i > 0; i--) /* Shift arguments up. */ 824 for (i = ++nargs; i > LJ_FR2; i--) /* Shift arguments up. */
620 fbase[i] = fbase[i-1]; 825 fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1];
826#if LJ_FR2
827 fbase[2] = fbase[0];
828#endif
621 fbase[0] = ix.mobj; /* Replace function. */ 829 fbase[0] = ix.mobj; /* Replace function. */
622 functv = &ix.mobjv; 830 functv = &ix.mobjv;
623 } 831 }
624 fbase[0] = TREF_FRAME | rec_call_specialize(J, funcV(functv), fbase[0]); 832 kfunc = rec_call_specialize(J, funcV(functv), fbase[0]);
833#if LJ_FR2
834 fbase[0] = kfunc;
835 fbase[1] = TREF_FRAME;
836#else
837 fbase[0] = kfunc | TREF_FRAME;
838#endif
625 J->maxslot = (BCReg)nargs; 839 J->maxslot = (BCReg)nargs;
626} 840}
627 841
@@ -631,8 +845,8 @@ void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
631 rec_call_setup(J, func, nargs); 845 rec_call_setup(J, func, nargs);
632 /* Bump frame. */ 846 /* Bump frame. */
633 J->framedepth++; 847 J->framedepth++;
634 J->base += func+1; 848 J->base += func+1+LJ_FR2;
635 J->baseslot += func+1; 849 J->baseslot += func+1+LJ_FR2;
636 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) 850 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
637 lj_trace_err(J, LJ_TRERR_STACKOV); 851 lj_trace_err(J, LJ_TRERR_STACKOV);
638} 852}
@@ -650,7 +864,9 @@ void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
650 func += cbase; 864 func += cbase;
651 } 865 }
652 /* Move func + args down. */ 866 /* Move func + args down. */
653 memmove(&J->base[-1], &J->base[func], sizeof(TRef)*(J->maxslot+1)); 867 if (LJ_FR2 && J->baseslot == 2)
868 J->base[func+1] = TREF_FRAME;
869 memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2));
654 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */ 870 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
655 /* Tailcalls can form a loop, so count towards the loop unroll limit. */ 871 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
656 if (++J->tailcalled > J->loopunroll) 872 if (++J->tailcalled > J->loopunroll)
@@ -680,6 +896,8 @@ static int check_downrec_unroll(jit_State *J, GCproto *pt)
680 return 0; 896 return 0;
681} 897}
682 898
899static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
900
683/* Record return. */ 901/* Record return. */
684void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults) 902void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
685{ 903{
@@ -691,30 +909,32 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
691 BCReg cbase = (BCReg)frame_delta(frame); 909 BCReg cbase = (BCReg)frame_delta(frame);
692 if (--J->framedepth <= 0) 910 if (--J->framedepth <= 0)
693 lj_trace_err(J, LJ_TRERR_NYIRETL); 911 lj_trace_err(J, LJ_TRERR_NYIRETL);
694 lua_assert(J->baseslot > 1); 912 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
695 gotresults++; 913 gotresults++;
696 rbase += cbase; 914 rbase += cbase;
697 J->baseslot -= (BCReg)cbase; 915 J->baseslot -= (BCReg)cbase;
698 J->base -= cbase; 916 J->base -= cbase;
699 J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */ 917 J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */
700 frame = frame_prevd(frame); 918 frame = frame_prevd(frame);
919 J->needsnap = 1; /* Stop catching on-trace errors. */
701 } 920 }
702 /* Return to lower frame via interpreter for unhandled cases. */ 921 /* Return to lower frame via interpreter for unhandled cases. */
703 if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) && 922 if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
704 (!frame_islua(frame) || 923 (!frame_islua(frame) ||
705 (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))))) { 924 (J->parent == 0 && J->exitno == 0 &&
925 !bc_isret(bc_op(J->cur.startins))))) {
706 /* NYI: specialize to frame type and return directly, not via RET*. */ 926 /* NYI: specialize to frame type and return directly, not via RET*. */
707 for (i = 0; i < (ptrdiff_t)rbase; i++) 927 for (i = 0; i < (ptrdiff_t)rbase; i++)
708 J->base[i] = 0; /* Purge dead slots. */ 928 J->base[i] = 0; /* Purge dead slots. */
709 J->maxslot = rbase + (BCReg)gotresults; 929 J->maxslot = rbase + (BCReg)gotresults;
710 rec_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */ 930 lj_record_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */
711 return; 931 return;
712 } 932 }
713 if (frame_isvarg(frame)) { 933 if (frame_isvarg(frame)) {
714 BCReg cbase = (BCReg)frame_delta(frame); 934 BCReg cbase = (BCReg)frame_delta(frame);
715 if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */ 935 if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */
716 lj_trace_err(J, LJ_TRERR_NYIRETL); 936 lj_trace_err(J, LJ_TRERR_NYIRETL);
717 lua_assert(J->baseslot > 1); 937 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
718 rbase += cbase; 938 rbase += cbase;
719 J->baseslot -= (BCReg)cbase; 939 J->baseslot -= (BCReg)cbase;
720 J->base -= cbase; 940 J->base -= cbase;
@@ -724,27 +944,28 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
724 BCIns callins = *(frame_pc(frame)-1); 944 BCIns callins = *(frame_pc(frame)-1);
725 ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults; 945 ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
726 BCReg cbase = bc_a(callins); 946 BCReg cbase = bc_a(callins);
727 GCproto *pt = funcproto(frame_func(frame - (cbase+1))); 947 GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2)));
728 if ((pt->flags & PROTO_NOJIT)) 948 if ((pt->flags & PROTO_NOJIT))
729 lj_trace_err(J, LJ_TRERR_CJITOFF); 949 lj_trace_err(J, LJ_TRERR_CJITOFF);
730 if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) { 950 if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
731 if (check_downrec_unroll(J, pt)) { 951 if (check_downrec_unroll(J, pt)) {
732 J->maxslot = (BCReg)(rbase + gotresults); 952 J->maxslot = (BCReg)(rbase + gotresults);
733 lj_snap_purge(J); 953 lj_snap_purge(J);
734 rec_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-recursion. */ 954 lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-rec. */
735 return; 955 return;
736 } 956 }
737 lj_snap_add(J); 957 lj_snap_add(J);
738 } 958 }
739 for (i = 0; i < nresults; i++) /* Adjust results. */ 959 for (i = 0; i < nresults; i++) /* Adjust results. */
740 J->base[i-1] = i < gotresults ? J->base[rbase+i] : TREF_NIL; 960 J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
741 J->maxslot = cbase+(BCReg)nresults; 961 J->maxslot = cbase+(BCReg)nresults;
742 if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */ 962 if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */
743 J->framedepth--; 963 J->framedepth--;
744 lua_assert(J->baseslot > cbase+1); 964 lj_assertJ(J->baseslot > cbase+1+LJ_FR2, "bad baseslot for return");
745 J->baseslot -= cbase+1; 965 J->baseslot -= cbase+1+LJ_FR2;
746 J->base -= cbase+1; 966 J->base -= cbase+1+LJ_FR2;
747 } else if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) { 967 } else if (J->parent == 0 && J->exitno == 0 &&
968 !bc_isret(bc_op(J->cur.startins))) {
748 /* Return to lower frame would leave the loop in a root trace. */ 969 /* Return to lower frame would leave the loop in a root trace. */
749 lj_trace_err(J, LJ_TRERR_LLEAVE); 970 lj_trace_err(J, LJ_TRERR_LLEAVE);
750 } else if (J->needsnap) { /* Tailcalled to ff with side-effects. */ 971 } else if (J->needsnap) { /* Tailcalled to ff with side-effects. */
@@ -752,13 +973,13 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
752 } else { /* Return to lower frame. Guard for the target we return to. */ 973 } else { /* Return to lower frame. Guard for the target we return to. */
753 TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); 974 TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
754 TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame)); 975 TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
755 emitir(IRTG(IR_RETF, IRT_P32), trpt, trpc); 976 emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc);
756 J->retdepth++; 977 J->retdepth++;
757 J->needsnap = 1; 978 J->needsnap = 1;
758 lua_assert(J->baseslot == 1); 979 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot for return");
759 /* Shift result slots up and clear the slots of the new frame below. */ 980 /* Shift result slots up and clear the slots of the new frame below. */
760 memmove(J->base + cbase, J->base-1, sizeof(TRef)*nresults); 981 memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults);
761 memset(J->base-1, 0, sizeof(TRef)*(cbase+1)); 982 memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2));
762 } 983 }
763 } else if (frame_iscont(frame)) { /* Return to continuation frame. */ 984 } else if (frame_iscont(frame)) { /* Return to continuation frame. */
764 ASMFunction cont = frame_contf(frame); 985 ASMFunction cont = frame_contf(frame);
@@ -767,24 +988,52 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
767 lj_trace_err(J, LJ_TRERR_NYIRETL); 988 lj_trace_err(J, LJ_TRERR_NYIRETL);
768 J->baseslot -= (BCReg)cbase; 989 J->baseslot -= (BCReg)cbase;
769 J->base -= cbase; 990 J->base -= cbase;
770 J->maxslot = cbase-2; 991 J->maxslot = cbase-(2<<LJ_FR2);
771 if (cont == lj_cont_ra) { 992 if (cont == lj_cont_ra) {
772 /* Copy result to destination slot. */ 993 /* Copy result to destination slot. */
773 BCReg dst = bc_a(*(frame_contpc(frame)-1)); 994 BCReg dst = bc_a(*(frame_contpc(frame)-1));
774 J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL; 995 J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
775 if (dst >= J->maxslot) J->maxslot = dst+1; 996 if (dst >= J->maxslot) {
997 J->maxslot = dst+1;
998 }
776 } else if (cont == lj_cont_nop) { 999 } else if (cont == lj_cont_nop) {
777 /* Nothing to do here. */ 1000 /* Nothing to do here. */
778 } else if (cont == lj_cont_cat) { 1001 } else if (cont == lj_cont_cat) {
779 lua_assert(0); 1002 BCReg bslot = bc_b(*(frame_contpc(frame)-1));
1003 TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
1004 if (bslot != J->maxslot) { /* Concatenate the remainder. */
1005 TValue *b = J->L->base, save; /* Simulate lower frame and result. */
1006 /* Can't handle MM_concat + CALLT + fast func side-effects. */
1007 if (J->postproc != LJ_POST_NONE)
1008 lj_trace_err(J, LJ_TRERR_NYIRETL);
1009 J->base[J->maxslot] = tr;
1010 copyTV(J->L, &save, b-(2<<LJ_FR2));
1011 if (gotresults)
1012 copyTV(J->L, b-(2<<LJ_FR2), b+rbase);
1013 else
1014 setnilV(b-(2<<LJ_FR2));
1015 J->L->base = b - cbase;
1016 tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
1017 b = J->L->base + cbase; /* Undo. */
1018 J->L->base = b;
1019 copyTV(J->L, b-(2<<LJ_FR2), &save);
1020 }
1021 if (tr) { /* Store final result. */
1022 BCReg dst = bc_a(*(frame_contpc(frame)-1));
1023 J->base[dst] = tr;
1024 if (dst >= J->maxslot) {
1025 J->maxslot = dst+1;
1026 }
1027 } /* Otherwise continue with another __concat call. */
780 } else { 1028 } else {
781 /* Result type already specialized. */ 1029 /* Result type already specialized. */
782 lua_assert(cont == lj_cont_condf || cont == lj_cont_condt); 1030 lj_assertJ(cont == lj_cont_condf || cont == lj_cont_condt,
1031 "bad continuation type");
783 } 1032 }
784 } else { 1033 } else {
785 lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */ 1034 lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */
786 } 1035 }
787 lua_assert(J->baseslot >= 1); 1036 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot for return");
788} 1037}
789 1038
790/* -- Metamethod handling ------------------------------------------------- */ 1039/* -- Metamethod handling ------------------------------------------------- */
@@ -792,19 +1041,17 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
792/* Prepare to record call to metamethod. */ 1041/* Prepare to record call to metamethod. */
793static BCReg rec_mm_prep(jit_State *J, ASMFunction cont) 1042static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
794{ 1043{
795 BCReg s, top = curr_proto(J->L)->framesize; 1044 BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
796 TRef trcont; 1045#if LJ_FR2
797 setcont(&J->L->base[top], cont); 1046 J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
798#if LJ_64 1047 J->base[top+1] = TREF_CONT;
799 trcont = lj_ir_kptr(J, (void *)((int64_t)cont - (int64_t)lj_vm_asm_begin));
800#else 1048#else
801 trcont = lj_ir_kptr(J, (void *)cont); 1049 J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
802#endif 1050#endif
803 J->base[top] = trcont | TREF_CONT;
804 J->framedepth++; 1051 J->framedepth++;
805 for (s = J->maxslot; s < top; s++) 1052 for (s = J->maxslot; s < top; s++)
806 J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */ 1053 J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */
807 return top+1; 1054 return top+1+LJ_FR2;
808} 1055}
809 1056
810/* Record metamethod lookup. */ 1057/* Record metamethod lookup. */
@@ -823,7 +1070,7 @@ int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
823 cTValue *mo; 1070 cTValue *mo;
824 if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) { 1071 if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
825 /* Specialize to the C library namespace object. */ 1072 /* Specialize to the C library namespace object. */
826 emitir(IRTG(IR_EQ, IRT_P32), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv))); 1073 emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
827 } else { 1074 } else {
828 /* Specialize to the type of userdata. */ 1075 /* Specialize to the type of userdata. */
829 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE); 1076 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
@@ -852,7 +1099,8 @@ int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
852 } 1099 }
853 /* The cdata metatable is treated as immutable. */ 1100 /* The cdata metatable is treated as immutable. */
854 if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt; 1101 if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt;
855 ix->mt = mix.tab = lj_ir_ktab(J, mt); 1102 ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB,
1103 GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)]));
856 goto nocheck; 1104 goto nocheck;
857 } 1105 }
858 ix->mt = mt ? mix.tab : TREF_NIL; 1106 ix->mt = mt ? mix.tab : TREF_NIL;
@@ -879,12 +1127,12 @@ nocheck:
879static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm) 1127static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
880{ 1128{
881 /* Set up metamethod call first to save ix->tab and ix->tabv. */ 1129 /* Set up metamethod call first to save ix->tab and ix->tabv. */
882 BCReg func = rec_mm_prep(J, lj_cont_ra); 1130 BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
883 TRef *base = J->base + func; 1131 TRef *base = J->base + func;
884 TValue *basev = J->L->base + func; 1132 TValue *basev = J->L->base + func;
885 base[1] = ix->tab; base[2] = ix->key; 1133 base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key;
886 copyTV(J->L, basev+1, &ix->tabv); 1134 copyTV(J->L, basev+1+LJ_FR2, &ix->tabv);
887 copyTV(J->L, basev+2, &ix->keyv); 1135 copyTV(J->L, basev+2+LJ_FR2, &ix->keyv);
888 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ 1136 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
889 if (mm != MM_unm) { 1137 if (mm != MM_unm) {
890 ix->tab = ix->key; 1138 ix->tab = ix->key;
@@ -896,6 +1144,9 @@ static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
896 } 1144 }
897ok: 1145ok:
898 base[0] = ix->mobj; 1146 base[0] = ix->mobj;
1147#if LJ_FR2
1148 base[1] = 0;
1149#endif
899 copyTV(J->L, basev+0, &ix->mobjv); 1150 copyTV(J->L, basev+0, &ix->mobjv);
900 lj_record_call(J, func, 2); 1151 lj_record_call(J, func, 2);
901 return 0; /* No result yet. */ 1152 return 0; /* No result yet. */
@@ -912,6 +1163,8 @@ static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
912 TRef *base = J->base + func; 1163 TRef *base = J->base + func;
913 TValue *basev = J->L->base + func; 1164 TValue *basev = J->L->base + func;
914 base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv); 1165 base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
1166 base += LJ_FR2;
1167 basev += LJ_FR2;
915 base[1] = tr; copyTV(J->L, basev+1, tv); 1168 base[1] = tr; copyTV(J->L, basev+1, tv);
916#if LJ_52 1169#if LJ_52
917 base[2] = tr; copyTV(J->L, basev+2, tv); 1170 base[2] = tr; copyTV(J->L, basev+2, tv);
@@ -921,7 +1174,7 @@ static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
921 lj_record_call(J, func, 2); 1174 lj_record_call(J, func, 2);
922 } else { 1175 } else {
923 if (LJ_52 && tref_istab(tr)) 1176 if (LJ_52 && tref_istab(tr))
924 return lj_ir_call(J, IRCALL_lj_tab_len, tr); 1177 return emitir(IRTI(IR_ALEN), tr, TREF_NIL);
925 lj_trace_err(J, LJ_TRERR_NOMM); 1178 lj_trace_err(J, LJ_TRERR_NOMM);
926 } 1179 }
927 return 0; /* No result yet. */ 1180 return 0; /* No result yet. */
@@ -931,10 +1184,10 @@ static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
931static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op) 1184static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
932{ 1185{
933 BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt); 1186 BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
934 TRef *base = J->base + func; 1187 TRef *base = J->base + func + LJ_FR2;
935 TValue *tv = J->L->base + func; 1188 TValue *tv = J->L->base + func + LJ_FR2;
936 base[0] = ix->mobj; base[1] = ix->val; base[2] = ix->key; 1189 base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
937 copyTV(J->L, tv+0, &ix->mobjv); 1190 copyTV(J->L, tv-LJ_FR2, &ix->mobjv);
938 copyTV(J->L, tv+1, &ix->valv); 1191 copyTV(J->L, tv+1, &ix->valv);
939 copyTV(J->L, tv+2, &ix->keyv); 1192 copyTV(J->L, tv+2, &ix->keyv);
940 lj_record_call(J, func, 2); 1193 lj_record_call(J, func, 2);
@@ -1030,7 +1283,7 @@ static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1030 ix->tab = ix->val; 1283 ix->tab = ix->val;
1031 copyTV(J->L, &ix->tabv, &ix->valv); 1284 copyTV(J->L, &ix->tabv, &ix->valv);
1032 } else { 1285 } else {
1033 lua_assert(tref_iscdata(ix->key)); 1286 lj_assertJ(tref_iscdata(ix->key), "cdata expected");
1034 ix->tab = ix->key; 1287 ix->tab = ix->key;
1035 copyTV(J->L, &ix->tabv, &ix->keyv); 1288 copyTV(J->L, &ix->tabv, &ix->keyv);
1036 } 1289 }
@@ -1041,6 +1294,72 @@ static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1041 1294
1042/* -- Indexed access ------------------------------------------------------ */ 1295/* -- Indexed access ------------------------------------------------------ */
1043 1296
1297#ifdef LUAJIT_ENABLE_TABLE_BUMP
1298/* Bump table allocations in bytecode when they grow during recording. */
1299static void rec_idx_bump(jit_State *J, RecordIndex *ix)
1300{
1301 RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
1302 if (tref_ref(ix->tab) == rbc->ref) {
1303 const BCIns *pc = mref(rbc->pc, const BCIns);
1304 GCtab *tb = tabV(&ix->tabv);
1305 uint32_t nhbits;
1306 IRIns *ir;
1307 if (!tvisnil(&ix->keyv))
1308 (void)lj_tab_set(J->L, tb, &ix->keyv); /* Grow table right now. */
1309 nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
1310 ir = IR(tref_ref(ix->tab));
1311 if (ir->o == IR_TNEW) {
1312 uint32_t ah = bc_d(*pc);
1313 uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
1314 if (nhbits > hbits) hbits = nhbits;
1315 if (tb->asize > asize) {
1316 asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
1317 }
1318 if ((asize | (hbits<<11)) != ah) { /* Has the size changed? */
1319 /* Patch bytecode, but continue recording (for more patching). */
1320 setbc_d(pc, (asize | (hbits<<11)));
1321 /* Patching TNEW operands is only safe if the trace is aborted. */
1322 ir->op1 = asize; ir->op2 = hbits;
1323 J->retryrec = 1; /* Abort the trace at the end of recording. */
1324 }
1325 } else if (ir->o == IR_TDUP) {
1326 GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc)));
1327 /* Grow template table, but preserve keys with nil values. */
1328 if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) ||
1329 (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) {
1330 Node *node = noderef(tpl->node);
1331 uint32_t i, hmask = tpl->hmask, asize;
1332 TValue *array;
1333 for (i = 0; i <= hmask; i++) {
1334 if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
1335 settabV(J->L, &node[i].val, tpl);
1336 }
1337 if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
1338 TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
1339 if (tvisnil(o)) settabV(J->L, o, tpl);
1340 }
1341 lj_tab_resize(J->L, tpl, tb->asize, nhbits);
1342 node = noderef(tpl->node);
1343 hmask = tpl->hmask;
1344 for (i = 0; i <= hmask; i++) {
1345 /* This is safe, since template tables only hold immutable values. */
1346 if (tvistab(&node[i].val))
1347 setnilV(&node[i].val);
1348 }
1349 /* The shape of the table may have changed. Clean up array part, too. */
1350 asize = tpl->asize;
1351 array = tvref(tpl->array);
1352 for (i = 0; i < asize; i++) {
1353 if (tvistab(&array[i]))
1354 setnilV(&array[i]);
1355 }
1356 J->retryrec = 1; /* Abort the trace at the end of recording. */
1357 }
1358 }
1359 }
1360}
1361#endif
1362
1044/* Record bounds-check. */ 1363/* Record bounds-check. */
1045static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize) 1364static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1046{ 1365{
@@ -1061,7 +1380,8 @@ static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1061 /* Got scalar evolution analysis results for this reference? */ 1380 /* Got scalar evolution analysis results for this reference? */
1062 if (ref == J->scev.idx) { 1381 if (ref == J->scev.idx) {
1063 int32_t stop; 1382 int32_t stop;
1064 lua_assert(irt_isint(J->scev.t) && ir->o == IR_SLOAD); 1383 lj_assertJ(irt_isint(J->scev.t) && ir->o == IR_SLOAD,
1384 "only int SCEV supported");
1065 stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]); 1385 stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
1066 /* Runtime value for stop of loop is within bounds? */ 1386 /* Runtime value for stop of loop is within bounds? */
1067 if ((uint64_t)stop + ofs < (uint64_t)asize) { 1387 if ((uint64_t)stop + ofs < (uint64_t)asize) {
@@ -1080,11 +1400,14 @@ static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1080} 1400}
1081 1401
1082/* Record indexed key lookup. */ 1402/* Record indexed key lookup. */
1083static TRef rec_idx_key(jit_State *J, RecordIndex *ix) 1403static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref,
1404 IRType1 *rbguard)
1084{ 1405{
1085 TRef key; 1406 TRef key;
1086 GCtab *t = tabV(&ix->tabv); 1407 GCtab *t = tabV(&ix->tabv);
1087 ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */ 1408 ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */
1409 *rbref = 0;
1410 rbguard->irt = 0;
1088 1411
1089 /* Integer keys are looked up in the array part first. */ 1412 /* Integer keys are looked up in the array part first. */
1090 key = ix->key; 1413 key = ix->key;
@@ -1098,8 +1421,8 @@ static TRef rec_idx_key(jit_State *J, RecordIndex *ix)
1098 if ((MSize)k < t->asize) { /* Currently an array key? */ 1421 if ((MSize)k < t->asize) { /* Currently an array key? */
1099 TRef arrayref; 1422 TRef arrayref;
1100 rec_idx_abc(J, asizeref, ikey, t->asize); 1423 rec_idx_abc(J, asizeref, ikey, t->asize);
1101 arrayref = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_ARRAY); 1424 arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY);
1102 return emitir(IRT(IR_AREF, IRT_P32), arrayref, ikey); 1425 return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey);
1103 } else { /* Currently not in array (may be an array extension)? */ 1426 } else { /* Currently not in array (may be an array extension)? */
1104 emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */ 1427 emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */
1105 if (k == 0 && tref_isk(key)) 1428 if (k == 0 && tref_isk(key))
@@ -1131,19 +1454,21 @@ static TRef rec_idx_key(jit_State *J, RecordIndex *ix)
1131 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); 1454 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1132 if (tref_isk(key)) { 1455 if (tref_isk(key)) {
1133 /* Optimize lookup of constant hash keys. */ 1456 /* Optimize lookup of constant hash keys. */
1134 MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val); 1457 GCSize hslot = (GCSize)((char *)ix->oldv-(char *)&noderef(t->node)[0].val);
1135 if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) && 1458 if (hslot <= t->hmask*(GCSize)sizeof(Node) &&
1136 hslot <= 65535*(MSize)sizeof(Node)) { 1459 hslot <= 65535*(GCSize)sizeof(Node)) {
1137 TRef node, kslot; 1460 TRef node, kslot, hm;
1138 TRef hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); 1461 *rbref = J->cur.nins; /* Mark possible rollback point. */
1462 *rbguard = J->guardemit;
1463 hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1139 emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask)); 1464 emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
1140 node = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_NODE); 1465 node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE);
1141 kslot = lj_ir_kslot(J, key, hslot / sizeof(Node)); 1466 kslot = lj_ir_kslot(J, key, (IRRef)(hslot / sizeof(Node)));
1142 return emitir(IRTG(IR_HREFK, IRT_P32), node, kslot); 1467 return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot);
1143 } 1468 }
1144 } 1469 }
1145 /* Fall back to a regular hash lookup. */ 1470 /* Fall back to a regular hash lookup. */
1146 return emitir(IRT(IR_HREF, IRT_P32), ix->tab, key); 1471 return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key);
1147} 1472}
1148 1473
1149/* Determine whether a key is NOT one of the fast metamethod names. */ 1474/* Determine whether a key is NOT one of the fast metamethod names. */
@@ -1168,20 +1493,22 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1168{ 1493{
1169 TRef xref; 1494 TRef xref;
1170 IROp xrefop, loadop; 1495 IROp xrefop, loadop;
1496 IRRef rbref;
1497 IRType1 rbguard;
1171 cTValue *oldv; 1498 cTValue *oldv;
1172 1499
1173 while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */ 1500 while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
1174 /* Never call raw lj_record_idx() on non-table. */ 1501 /* Never call raw lj_record_idx() on non-table. */
1175 lua_assert(ix->idxchain != 0); 1502 lj_assertJ(ix->idxchain != 0, "bad usage");
1176 if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index)) 1503 if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
1177 lj_trace_err(J, LJ_TRERR_NOMM); 1504 lj_trace_err(J, LJ_TRERR_NOMM);
1178 handlemm: 1505 handlemm:
1179 if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */ 1506 if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */
1180 BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra); 1507 BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
1181 TRef *base = J->base + func; 1508 TRef *base = J->base + func + LJ_FR2;
1182 TValue *tv = J->L->base + func; 1509 TValue *tv = J->L->base + func + LJ_FR2;
1183 base[0] = ix->mobj; base[1] = ix->tab; base[2] = ix->key; 1510 base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
1184 setfuncV(J->L, tv+0, funcV(&ix->mobjv)); 1511 setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv));
1185 copyTV(J->L, tv+1, &ix->tabv); 1512 copyTV(J->L, tv+1, &ix->tabv);
1186 copyTV(J->L, tv+2, &ix->keyv); 1513 copyTV(J->L, tv+2, &ix->keyv);
1187 if (ix->val) { 1514 if (ix->val) {
@@ -1194,6 +1521,16 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1194 return 0; /* No result yet. */ 1521 return 0; /* No result yet. */
1195 } 1522 }
1196 } 1523 }
1524#if LJ_HASBUFFER
1525 /* The index table of buffer objects is treated as immutable. */
1526 if (ix->mt == TREF_NIL && !ix->val &&
1527 tref_isudata(ix->tab) && udataV(&ix->tabv)->udtype == UDTYPE_BUFFER &&
1528 tref_istab(ix->mobj) && tref_isstr(ix->key) && tref_isk(ix->key)) {
1529 cTValue *val = lj_tab_getstr(tabV(&ix->mobjv), strV(&ix->keyv));
1530 TRef tr = lj_record_constify(J, val);
1531 if (tr) return tr; /* Specialize to the value, i.e. a method. */
1532 }
1533#endif
1197 /* Otherwise retry lookup with metaobject. */ 1534 /* Otherwise retry lookup with metaobject. */
1198 ix->tab = ix->mobj; 1535 ix->tab = ix->mobj;
1199 copyTV(J->L, &ix->tabv, &ix->mobjv); 1536 copyTV(J->L, &ix->tabv, &ix->mobjv);
@@ -1213,7 +1550,7 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1213 } 1550 }
1214 1551
1215 /* Record the key lookup. */ 1552 /* Record the key lookup. */
1216 xref = rec_idx_key(J, ix); 1553 xref = rec_idx_key(J, ix, &rbref, &rbguard);
1217 xrefop = IR(tref_ref(xref))->o; 1554 xrefop = IR(tref_ref(xref))->o;
1218 loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD; 1555 loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
1219 /* The lj_meta_tset() inconsistency is gone, but better play safe. */ 1556 /* The lj_meta_tset() inconsistency is gone, but better play safe. */
@@ -1223,11 +1560,15 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1223 IRType t = itype2irt(oldv); 1560 IRType t = itype2irt(oldv);
1224 TRef res; 1561 TRef res;
1225 if (oldv == niltvg(J2G(J))) { 1562 if (oldv == niltvg(J2G(J))) {
1226 emitir(IRTG(IR_EQ, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); 1563 emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1227 res = TREF_NIL; 1564 res = TREF_NIL;
1228 } else { 1565 } else {
1229 res = emitir(IRTG(loadop, t), xref, 0); 1566 res = emitir(IRTG(loadop, t), xref, 0);
1230 } 1567 }
1568 if (tref_ref(res) < rbref) { /* HREFK + load forwarded? */
1569 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1570 J->guardemit = rbguard;
1571 }
1231 if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index)) 1572 if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1232 goto handlemm; 1573 goto handlemm;
1233 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */ 1574 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */
@@ -1235,6 +1576,10 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1235 } else { /* Indexed store. */ 1576 } else { /* Indexed store. */
1236 GCtab *mt = tabref(tabV(&ix->tabv)->metatable); 1577 GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
1237 int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val); 1578 int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
1579 if (tref_ref(xref) < rbref) { /* HREFK forwarded? */
1580 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1581 J->guardemit = rbguard;
1582 }
1238 if (tvisnil(oldv)) { /* Previous value was nil? */ 1583 if (tvisnil(oldv)) { /* Previous value was nil? */
1239 /* Need to duplicate the hasmm check for the early guards. */ 1584 /* Need to duplicate the hasmm check for the early guards. */
1240 int hasmm = 0; 1585 int hasmm = 0;
@@ -1245,26 +1590,30 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1245 if (hasmm) 1590 if (hasmm)
1246 emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */ 1591 emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */
1247 else if (xrefop == IR_HREF) 1592 else if (xrefop == IR_HREF)
1248 emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_P32), 1593 emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC),
1249 xref, lj_ir_kkptr(J, niltvg(J2G(J)))); 1594 xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1250 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) { 1595 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
1251 lua_assert(hasmm); 1596 lj_assertJ(hasmm, "inconsistent metamethod handling");
1252 goto handlemm; 1597 goto handlemm;
1253 } 1598 }
1254 lua_assert(!hasmm); 1599 lj_assertJ(!hasmm, "inconsistent metamethod handling");
1255 if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */ 1600 if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */
1256 TRef key = ix->key; 1601 TRef key = ix->key;
1257 if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */ 1602 if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */
1258 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); 1603 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1259 else if (tref_isnumber(key) && tref_isk(key) && tvismzero(&ix->keyv)) 1604 else if (tref_isnumber(key) && tref_isk(key) && tvismzero(&ix->keyv))
1260 key = lj_ir_knum_zero(J); /* Canonicalize -0.0 to +0.0. */ 1605 key = lj_ir_knum_zero(J); /* Canonicalize -0.0 to +0.0. */
1261 xref = emitir(IRT(IR_NEWREF, IRT_P32), ix->tab, key); 1606 xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key);
1262 keybarrier = 0; /* NEWREF already takes care of the key barrier. */ 1607 keybarrier = 0; /* NEWREF already takes care of the key barrier. */
1608#ifdef LUAJIT_ENABLE_TABLE_BUMP
1609 if ((J->flags & JIT_F_OPT_SINK)) /* Avoid a separate flag. */
1610 rec_idx_bump(J, ix);
1611#endif
1263 } 1612 }
1264 } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) { 1613 } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
1265 /* Cannot derive that the previous value was non-nil, must do checks. */ 1614 /* Cannot derive that the previous value was non-nil, must do checks. */
1266 if (xrefop == IR_HREF) /* Guard against store to niltv. */ 1615 if (xrefop == IR_HREF) /* Guard against store to niltv. */
1267 emitir(IRTG(IR_NE, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); 1616 emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1268 if (ix->idxchain) { /* Metamethod lookup required? */ 1617 if (ix->idxchain) { /* Metamethod lookup required? */
1269 /* A check for NULL metatable is cheaper (hoistable) than a load. */ 1618 /* A check for NULL metatable is cheaper (hoistable) than a load. */
1270 if (!mt) { 1619 if (!mt) {
@@ -1286,7 +1635,7 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1286 emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0); 1635 emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
1287 /* Invalidate neg. metamethod cache for stores with certain string keys. */ 1636 /* Invalidate neg. metamethod cache for stores with certain string keys. */
1288 if (!nommstr(J, ix->key)) { 1637 if (!nommstr(J, ix->key)) {
1289 TRef fref = emitir(IRT(IR_FREF, IRT_P32), ix->tab, IRFL_TAB_NOMM); 1638 TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM);
1290 emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0)); 1639 emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
1291 } 1640 }
1292 J->needsnap = 1; 1641 J->needsnap = 1;
@@ -1294,6 +1643,72 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1294 } 1643 }
1295} 1644}
1296 1645
1646/* Determine result type of table traversal. */
1647static IRType rec_next_types(GCtab *t, uint32_t idx)
1648{
1649 for (; idx < t->asize; idx++) {
1650 cTValue *a = arrayslot(t, idx);
1651 if (LJ_LIKELY(!tvisnil(a)))
1652 return (LJ_DUALNUM ? IRT_INT : IRT_NUM) + (itype2irt(a) << 8);
1653 }
1654 idx -= t->asize;
1655 for (; idx <= t->hmask; idx++) {
1656 Node *n = &noderef(t->node)[idx];
1657 if (!tvisnil(&n->val))
1658 return itype2irt(&n->key) + (itype2irt(&n->val) << 8);
1659 }
1660 return IRT_NIL + (IRT_NIL << 8);
1661}
1662
1663/* Record a table traversal step aka next(). */
1664int lj_record_next(jit_State *J, RecordIndex *ix)
1665{
1666 IRType t, tkey, tval;
1667 TRef trvk;
1668 t = rec_next_types(tabV(&ix->tabv), ix->keyv.u32.lo);
1669 tkey = (t & 0xff); tval = (t >> 8);
1670 trvk = lj_ir_call(J, IRCALL_lj_vm_next, ix->tab, ix->key);
1671 if (ix->mobj || tkey == IRT_NIL) {
1672 TRef idx = emitir(IRTI(IR_HIOP), trvk, trvk);
1673 /* Always check for invalid key from next() for nil result. */
1674 if (!ix->mobj) emitir(IRTGI(IR_NE), idx, lj_ir_kint(J, -1));
1675 ix->mobj = idx;
1676 }
1677 ix->key = lj_record_vload(J, trvk, 1, tkey);
1678 if (tkey == IRT_NIL || ix->idxchain) { /* Omit value type check. */
1679 ix->val = TREF_NIL;
1680 return 1;
1681 } else { /* Need value. */
1682 ix->val = lj_record_vload(J, trvk, 0, tval);
1683 return 2;
1684 }
1685}
1686
1687static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
1688{
1689 RecordIndex ix;
1690 cTValue *basev = J->L->base;
1691 GCtab *t = tabV(&basev[ra-1]);
1692 settabV(J->L, &ix.tabv, t);
1693 ix.tab = getslot(J, ra-1);
1694 ix.idxchain = 0;
1695#ifdef LUAJIT_ENABLE_TABLE_BUMP
1696 if ((J->flags & JIT_F_OPT_SINK)) {
1697 if (t->asize < i+rn-ra)
1698 lj_tab_reasize(J->L, t, i+rn-ra);
1699 setnilV(&ix.keyv);
1700 rec_idx_bump(J, &ix);
1701 }
1702#endif
1703 for (; ra < rn; i++, ra++) {
1704 setintV(&ix.keyv, i);
1705 ix.key = lj_ir_kint(J, i);
1706 copyTV(J->L, &ix.valv, &basev[ra]);
1707 ix.val = getslot(J, ra);
1708 lj_record_idx(J, &ix);
1709 }
1710}
1711
1297/* -- Upvalue access ------------------------------------------------------ */ 1712/* -- Upvalue access ------------------------------------------------------ */
1298 1713
1299/* Check whether upvalue is immutable and ok to constify. */ 1714/* Check whether upvalue is immutable and ok to constify. */
@@ -1330,13 +1745,17 @@ static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
1330 int needbarrier = 0; 1745 int needbarrier = 0;
1331 if (rec_upvalue_constify(J, uvp)) { /* Try to constify immutable upvalue. */ 1746 if (rec_upvalue_constify(J, uvp)) { /* Try to constify immutable upvalue. */
1332 TRef tr, kfunc; 1747 TRef tr, kfunc;
1333 lua_assert(val == 0); 1748 lj_assertJ(val == 0, "bad usage");
1334 if (!tref_isk(fn)) { /* Late specialization of current function. */ 1749 if (!tref_isk(fn)) { /* Late specialization of current function. */
1335 if (J->pt->flags >= PROTO_CLC_POLY) 1750 if (J->pt->flags >= PROTO_CLC_POLY)
1336 goto noconstify; 1751 goto noconstify;
1337 kfunc = lj_ir_kfunc(J, J->fn); 1752 kfunc = lj_ir_kfunc(J, J->fn);
1338 emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc); 1753 emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
1339 J->base[-1] = TREF_FRAME | kfunc; 1754#if LJ_FR2
1755 J->base[-2] = kfunc;
1756#else
1757 J->base[-1] = kfunc | TREF_FRAME;
1758#endif
1340 fn = kfunc; 1759 fn = kfunc;
1341 } 1760 }
1342 tr = lj_record_constify(J, uvval(uvp)); 1761 tr = lj_record_constify(J, uvval(uvp));
@@ -1347,16 +1766,16 @@ noconstify:
1347 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */ 1766 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1348 uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff); 1767 uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
1349 if (!uvp->closed) { 1768 if (!uvp->closed) {
1350 uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_P32), fn, uv)); 1769 uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PGC), fn, uv));
1351 /* In current stack? */ 1770 /* In current stack? */
1352 if (uvval(uvp) >= tvref(J->L->stack) && 1771 if (uvval(uvp) >= tvref(J->L->stack) &&
1353 uvval(uvp) < tvref(J->L->maxstack)) { 1772 uvval(uvp) < tvref(J->L->maxstack)) {
1354 int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot)); 1773 int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
1355 if (slot >= 0) { /* Aliases an SSA slot? */ 1774 if (slot >= 0) { /* Aliases an SSA slot? */
1356 emitir(IRTG(IR_EQ, IRT_P32), 1775 emitir(IRTG(IR_EQ, IRT_PGC),
1357 REF_BASE, 1776 REF_BASE,
1358 emitir(IRT(IR_ADD, IRT_P32), uref, 1777 emitir(IRT(IR_ADD, IRT_PGC), uref,
1359 lj_ir_kint(J, (slot - 1) * -8))); 1778 lj_ir_kint(J, (slot - 1 - LJ_FR2) * -8)));
1360 slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */ 1779 slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */
1361 if (val == 0) { 1780 if (val == 0) {
1362 return getslot(J, slot); 1781 return getslot(J, slot);
@@ -1367,12 +1786,12 @@ noconstify:
1367 } 1786 }
1368 } 1787 }
1369 } 1788 }
1370 emitir(IRTG(IR_UGT, IRT_P32), 1789 emitir(IRTG(IR_UGT, IRT_PGC),
1371 emitir(IRT(IR_SUB, IRT_P32), uref, REF_BASE), 1790 emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE),
1372 lj_ir_kint(J, (J->baseslot + J->maxslot) * 8)); 1791 lj_ir_kint(J, (J->baseslot + J->maxslot) * 8));
1373 } else { 1792 } else {
1374 needbarrier = 1; 1793 needbarrier = 1;
1375 uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_P32), fn, uv)); 1794 uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PGC), fn, uv));
1376 } 1795 }
1377 if (val == 0) { /* Upvalue load */ 1796 if (val == 0) { /* Upvalue load */
1378 IRType t = itype2irt(uvval(uvp)); 1797 IRType t = itype2irt(uvval(uvp));
@@ -1411,16 +1830,16 @@ static void check_call_unroll(jit_State *J, TraceNo lnk)
1411 if (count + J->tailcalled > J->param[JIT_P_recunroll]) { 1830 if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
1412 J->pc++; 1831 J->pc++;
1413 if (J->framedepth + J->retdepth == 0) 1832 if (J->framedepth + J->retdepth == 0)
1414 rec_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-recursion. */ 1833 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-rec. */
1415 else 1834 else
1416 rec_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */ 1835 lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */
1417 } 1836 }
1418 } else { 1837 } else {
1419 if (count > J->param[JIT_P_callunroll]) { 1838 if (count > J->param[JIT_P_callunroll]) {
1420 if (lnk) { /* Possible tail- or up-recursion. */ 1839 if (lnk) { /* Possible tail- or up-recursion. */
1421 lj_trace_flush(J, lnk); /* Flush trace that only returns. */ 1840 lj_trace_flush(J, lnk); /* Flush trace that only returns. */
1422 /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */ 1841 /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1423 hotcount_set(J2GG(J), J->pc+1, LJ_PRNG_BITS(J, 4)); 1842 hotcount_set(J2GG(J), J->pc+1, lj_prng_u64(&J2G(J)->prng) & 15u);
1424 } 1843 }
1425 lj_trace_err(J, LJ_TRERR_CUNROLL); 1844 lj_trace_err(J, LJ_TRERR_CUNROLL);
1426 } 1845 }
@@ -1447,11 +1866,14 @@ static void rec_func_setup(jit_State *J)
1447static void rec_func_vararg(jit_State *J) 1866static void rec_func_vararg(jit_State *J)
1448{ 1867{
1449 GCproto *pt = J->pt; 1868 GCproto *pt = J->pt;
1450 BCReg s, fixargs, vframe = J->maxslot+1; 1869 BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2;
1451 lua_assert((pt->flags & PROTO_VARARG)); 1870 lj_assertJ((pt->flags & PROTO_VARARG), "FUNCV in non-vararg function");
1452 if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS) 1871 if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
1453 lj_trace_err(J, LJ_TRERR_STACKOV); 1872 lj_trace_err(J, LJ_TRERR_STACKOV);
1454 J->base[vframe-1] = J->base[-1]; /* Copy function up. */ 1873 J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2]; /* Copy function up. */
1874#if LJ_FR2
1875 J->base[vframe-1] = TREF_FRAME;
1876#endif
1455 /* Copy fixarg slots up and set their original slots to nil. */ 1877 /* Copy fixarg slots up and set their original slots to nil. */
1456 fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot; 1878 fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
1457 for (s = 0; s < fixargs; s++) { 1879 for (s = 0; s < fixargs; s++) {
@@ -1487,9 +1909,9 @@ static void rec_func_jit(jit_State *J, TraceNo lnk)
1487 } 1909 }
1488 J->instunroll = 0; /* Cannot continue across a compiled function. */ 1910 J->instunroll = 0; /* Cannot continue across a compiled function. */
1489 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) 1911 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
1490 rec_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-recursion. */ 1912 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-rec. */
1491 else 1913 else
1492 rec_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */ 1914 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */
1493} 1915}
1494 1916
1495/* -- Vararg handling ----------------------------------------------------- */ 1917/* -- Vararg handling ----------------------------------------------------- */
@@ -1513,8 +1935,10 @@ static int select_detect(jit_State *J)
1513static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults) 1935static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1514{ 1936{
1515 int32_t numparams = J->pt->numparams; 1937 int32_t numparams = J->pt->numparams;
1516 ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1; 1938 ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2;
1517 lua_assert(frame_isvarg(J->L->base-1)); 1939 lj_assertJ(frame_isvarg(J->L->base-1), "VARG in non-vararg frame");
1940 if (LJ_FR2 && dst > J->maxslot)
1941 J->base[dst-1] = 0; /* Prevent resurrection of unrelated slot. */
1518 if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */ 1942 if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */
1519 ptrdiff_t i; 1943 ptrdiff_t i;
1520 if (nvararg < 0) nvararg = 0; 1944 if (nvararg < 0) nvararg = 0;
@@ -1527,10 +1951,10 @@ static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1527 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) 1951 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
1528 lj_trace_err(J, LJ_TRERR_STACKOV); 1952 lj_trace_err(J, LJ_TRERR_STACKOV);
1529 for (i = 0; i < nresults; i++) 1953 for (i = 0; i < nresults; i++)
1530 J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1) : TREF_NIL; 1954 J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
1531 } else { /* Unknown number of varargs passed to trace. */ 1955 } else { /* Unknown number of varargs passed to trace. */
1532 TRef fr = emitir(IRTI(IR_SLOAD), 0, IRSLOAD_READONLY|IRSLOAD_FRAME); 1956 TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME);
1533 int32_t frofs = 8*(1+numparams)+FRAME_VARG; 1957 int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG;
1534 if (nresults >= 0) { /* Known fixed number of results. */ 1958 if (nresults >= 0) { /* Known fixed number of results. */
1535 ptrdiff_t i; 1959 ptrdiff_t i;
1536 if (nvararg > 0) { 1960 if (nvararg > 0) {
@@ -1539,16 +1963,13 @@ static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1539 if (nvararg >= nresults) 1963 if (nvararg >= nresults)
1540 emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults)); 1964 emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
1541 else 1965 else
1542 emitir(IRTGI(IR_EQ), fr, lj_ir_kint(J, frame_ftsz(J->L->base-1))); 1966 emitir(IRTGI(IR_EQ), fr,
1543 vbase = emitir(IRTI(IR_SUB), REF_BASE, fr); 1967 lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
1544 vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8)); 1968 vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1969 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase, lj_ir_kint(J, frofs-8*(1+LJ_FR2)));
1545 for (i = 0; i < nload; i++) { 1970 for (i = 0; i < nload; i++) {
1546 IRType t = itype2irt(&J->L->base[i-1-nvararg]); 1971 IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]);
1547 TRef aref = emitir(IRT(IR_AREF, IRT_P32), 1972 J->base[dst+i] = lj_record_vload(J, vbase, (MSize)i, t);
1548 vbase, lj_ir_kint(J, (int32_t)i));
1549 TRef tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
1550 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */
1551 J->base[dst+i] = tr;
1552 } 1973 }
1553 } else { 1974 } else {
1554 emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs)); 1975 emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
@@ -1591,15 +2012,15 @@ static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1591 } 2012 }
1592 if (idx != 0 && idx <= nvararg) { 2013 if (idx != 0 && idx <= nvararg) {
1593 IRType t; 2014 IRType t;
1594 TRef aref, vbase = emitir(IRTI(IR_SUB), REF_BASE, fr); 2015 TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1595 vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8)); 2016 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
1596 t = itype2irt(&J->L->base[idx-2-nvararg]); 2017 lj_ir_kint(J, frofs-(8<<LJ_FR2)));
1597 aref = emitir(IRT(IR_AREF, IRT_P32), vbase, tridx); 2018 t = itype2irt(&J->L->base[idx-2-LJ_FR2-nvararg]);
1598 tr = emitir(IRTG(IR_VLOAD, t), aref, 0); 2019 aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx);
1599 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */ 2020 tr = lj_record_vload(J, aref, 0, t);
1600 } 2021 }
1601 J->base[dst-2] = tr; 2022 J->base[dst-2-LJ_FR2] = tr;
1602 J->maxslot = dst-1; 2023 J->maxslot = dst-1-LJ_FR2;
1603 J->bcskip = 2; /* Skip CALLM + select. */ 2024 J->bcskip = 2; /* Skip CALLM + select. */
1604 } else { 2025 } else {
1605 nyivarg: 2026 nyivarg:
@@ -1615,8 +2036,63 @@ static TRef rec_tnew(jit_State *J, uint32_t ah)
1615{ 2036{
1616 uint32_t asize = ah & 0x7ff; 2037 uint32_t asize = ah & 0x7ff;
1617 uint32_t hbits = ah >> 11; 2038 uint32_t hbits = ah >> 11;
2039 TRef tr;
1618 if (asize == 0x7ff) asize = 0x801; 2040 if (asize == 0x7ff) asize = 0x801;
1619 return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits); 2041 tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
2042#ifdef LUAJIT_ENABLE_TABLE_BUMP
2043 J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
2044 setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
2045 setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2046#endif
2047 return tr;
2048}
2049
2050/* -- Concatenation ------------------------------------------------------- */
2051
2052static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
2053{
2054 TRef *top = &J->base[topslot];
2055 TValue savetv[5+LJ_FR2];
2056 BCReg s;
2057 RecordIndex ix;
2058 lj_assertJ(baseslot < topslot, "bad CAT arg");
2059 for (s = baseslot; s <= topslot; s++)
2060 (void)getslot(J, s); /* Ensure all arguments have a reference. */
2061 if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
2062 TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
2063 /* First convert numbers to strings. */
2064 for (trp = top; trp >= base; trp--) {
2065 if (tref_isnumber(*trp))
2066 *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
2067 tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
2068 else if (!tref_isstr(*trp))
2069 break;
2070 }
2071 xbase = ++trp;
2072 tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC),
2073 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
2074 do {
2075 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, *trp++);
2076 } while (trp <= top);
2077 tr = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
2078 J->maxslot = (BCReg)(xbase - J->base);
2079 if (xbase == base) return tr; /* Return simple concatenation result. */
2080 /* Pass partial result. */
2081 topslot = J->maxslot--;
2082 *xbase = tr;
2083 top = xbase;
2084 setstrV(J->L, &ix.keyv, &J2G(J)->strempty); /* Simulate string result. */
2085 } else {
2086 J->maxslot = topslot-1;
2087 copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
2088 }
2089 copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
2090 ix.tab = top[-1];
2091 ix.key = top[0];
2092 memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv)); /* Save slots. */
2093 rec_mm_arith(J, &ix, MM_concat); /* Call __concat metamethod. */
2094 memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv)); /* Restore slots. */
2095 return 0; /* No result yet. */
1620} 2096}
1621 2097
1622/* -- Record bytecode ops ------------------------------------------------- */ 2098/* -- Record bytecode ops ------------------------------------------------- */
@@ -1637,7 +2113,15 @@ static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
1637 const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0); 2113 const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
1638 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; 2114 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
1639 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */ 2115 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
2116#if LJ_FR2
2117 SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent];
2118 uint64_t pcbase;
2119 memcpy(&pcbase, flink, sizeof(uint64_t));
2120 pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8);
2121 memcpy(flink, &pcbase, sizeof(uint64_t));
2122#else
1640 J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc); 2123 J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
2124#endif
1641 J->needsnap = 1; 2125 J->needsnap = 1;
1642 if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins); 2126 if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
1643 lj_snap_shrink(J); /* Shrink last snapshot if possible. */ 2127 lj_snap_shrink(J); /* Shrink last snapshot if possible. */
@@ -1657,7 +2141,7 @@ void lj_record_ins(jit_State *J)
1657 if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) { 2141 if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
1658 switch (J->postproc) { 2142 switch (J->postproc) {
1659 case LJ_POST_FIXCOMP: /* Fixup comparison. */ 2143 case LJ_POST_FIXCOMP: /* Fixup comparison. */
1660 pc = frame_pc(&J2G(J)->tmptv); 2144 pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
1661 rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1))); 2145 rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
1662 /* fallthrough */ 2146 /* fallthrough */
1663 case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */ 2147 case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */
@@ -1695,7 +2179,7 @@ void lj_record_ins(jit_State *J)
1695 if (bc_op(*J->pc) >= BC__MAX) 2179 if (bc_op(*J->pc) >= BC__MAX)
1696 return; 2180 return;
1697 break; 2181 break;
1698 default: lua_assert(0); break; 2182 default: lj_assertJ(0, "bad post-processing mode"); break;
1699 } 2183 }
1700 J->postproc = LJ_POST_NONE; 2184 J->postproc = LJ_POST_NONE;
1701 } 2185 }
@@ -1703,7 +2187,7 @@ void lj_record_ins(jit_State *J)
1703 /* Need snapshot before recording next bytecode (e.g. after a store). */ 2187 /* Need snapshot before recording next bytecode (e.g. after a store). */
1704 if (J->needsnap) { 2188 if (J->needsnap) {
1705 J->needsnap = 0; 2189 J->needsnap = 0;
1706 lj_snap_purge(J); 2190 if (J->pt) lj_snap_purge(J);
1707 lj_snap_add(J); 2191 lj_snap_add(J);
1708 J->mergesnap = 1; 2192 J->mergesnap = 1;
1709 } 2193 }
@@ -1725,6 +2209,10 @@ void lj_record_ins(jit_State *J)
1725 rec_check_ir(J); 2209 rec_check_ir(J);
1726#endif 2210#endif
1727 2211
2212#if LJ_HASPROFILE
2213 rec_profile_ins(J, pc);
2214#endif
2215
1728 /* Keep a copy of the runtime values of var/num/str operands. */ 2216 /* Keep a copy of the runtime values of var/num/str operands. */
1729#define rav (&ix.valv) 2217#define rav (&ix.valv)
1730#define rbv (&ix.tabv) 2218#define rbv (&ix.tabv)
@@ -1751,9 +2239,10 @@ void lj_record_ins(jit_State *J)
1751 switch (bcmode_c(op)) { 2239 switch (bcmode_c(op)) {
1752 case BCMvar: 2240 case BCMvar:
1753 copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break; 2241 copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
1754 case BCMpri: setitype(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break; 2242 case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
1755 case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc); 2243 case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
1756 copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) : 2244 copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
2245 tv->u32.hi == LJ_KEYINDEX ? (lj_ir_kint(J, 0) | TREF_KEYINDEX) :
1757 lj_ir_knumint(J, numV(tv)); } break; 2246 lj_ir_knumint(J, numV(tv)); } break;
1758 case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc)); 2247 case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
1759 setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break; 2248 setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
@@ -1846,6 +2335,18 @@ void lj_record_ins(jit_State *J)
1846 J->maxslot = bc_a(pc[1]); /* Shrink used slots. */ 2335 J->maxslot = bc_a(pc[1]); /* Shrink used slots. */
1847 break; 2336 break;
1848 2337
2338 case BC_ISTYPE: case BC_ISNUM:
2339 /* These coercions need to correspond with lj_meta_istype(). */
2340 if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
2341 ra = lj_opt_narrow_toint(J, ra);
2342 else if (rc == ~LJ_TNUMX+2)
2343 ra = lj_ir_tonum(J, ra);
2344 else if (rc == ~LJ_TSTR+1)
2345 ra = lj_ir_tostr(J, ra);
2346 /* else: type specialization suffices. */
2347 J->base[bc_a(ins)] = ra;
2348 break;
2349
1849 /* -- Unary ops --------------------------------------------------------- */ 2350 /* -- Unary ops --------------------------------------------------------- */
1850 2351
1851 case BC_NOT: 2352 case BC_NOT:
@@ -1857,7 +2358,7 @@ void lj_record_ins(jit_State *J)
1857 if (tref_isstr(rc)) 2358 if (tref_isstr(rc))
1858 rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN); 2359 rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
1859 else if (!LJ_52 && tref_istab(rc)) 2360 else if (!LJ_52 && tref_istab(rc))
1860 rc = lj_ir_call(J, IRCALL_lj_tab_len, rc); 2361 rc = emitir(IRTI(IR_ALEN), rc, TREF_NIL);
1861 else 2362 else
1862 rc = rec_mm_len(J, rc, rcv); 2363 rc = rec_mm_len(J, rc, rcv);
1863 break; 2364 break;
@@ -1904,16 +2405,28 @@ void lj_record_ins(jit_State *J)
1904 2405
1905 case BC_POW: 2406 case BC_POW:
1906 if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) 2407 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
1907 rc = lj_opt_narrow_pow(J, rb, rc, rbv, rcv); 2408 rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, IR_POW);
1908 else 2409 else
1909 rc = rec_mm_arith(J, &ix, MM_pow); 2410 rc = rec_mm_arith(J, &ix, MM_pow);
1910 break; 2411 break;
1911 2412
2413 /* -- Miscellaneous ops ------------------------------------------------- */
2414
2415 case BC_CAT:
2416 rc = rec_cat(J, rb, rc);
2417 break;
2418
1912 /* -- Constant and move ops --------------------------------------------- */ 2419 /* -- Constant and move ops --------------------------------------------- */
1913 2420
1914 case BC_MOV: 2421 case BC_MOV:
1915 /* Clear gap of method call to avoid resurrecting previous refs. */ 2422 /* Clear gap of method call to avoid resurrecting previous refs. */
1916 if (ra > J->maxslot) J->base[ra-1] = 0; 2423 if (ra > J->maxslot) {
2424#if LJ_FR2
2425 memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef));
2426#else
2427 J->base[ra-1] = 0;
2428#endif
2429 }
1917 break; 2430 break;
1918 case BC_KSTR: case BC_KNUM: case BC_KPRI: 2431 case BC_KSTR: case BC_KNUM: case BC_KPRI:
1919 break; 2432 break;
@@ -1921,6 +2434,8 @@ void lj_record_ins(jit_State *J)
1921 rc = lj_ir_kint(J, (int32_t)(int16_t)rc); 2434 rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
1922 break; 2435 break;
1923 case BC_KNIL: 2436 case BC_KNIL:
2437 if (LJ_FR2 && ra > J->maxslot)
2438 J->base[ra-1] = 0;
1924 while (ra <= rc) 2439 while (ra <= rc)
1925 J->base[ra++] = TREF_NIL; 2440 J->base[ra++] = TREF_NIL;
1926 if (rc >= J->maxslot) J->maxslot = rc+1; 2441 if (rc >= J->maxslot) J->maxslot = rc+1;
@@ -1957,6 +2472,15 @@ void lj_record_ins(jit_State *J)
1957 ix.idxchain = LJ_MAX_IDXCHAIN; 2472 ix.idxchain = LJ_MAX_IDXCHAIN;
1958 rc = lj_record_idx(J, &ix); 2473 rc = lj_record_idx(J, &ix);
1959 break; 2474 break;
2475 case BC_TGETR: case BC_TSETR:
2476 ix.idxchain = 0;
2477 rc = lj_record_idx(J, &ix);
2478 break;
2479
2480 case BC_TSETM:
2481 rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
2482 J->maxslot = ra; /* The table slot at ra-1 is the highest used slot. */
2483 break;
1960 2484
1961 case BC_TNEW: 2485 case BC_TNEW:
1962 rc = rec_tnew(J, rc); 2486 rc = rec_tnew(J, rc);
@@ -1964,33 +2488,38 @@ void lj_record_ins(jit_State *J)
1964 case BC_TDUP: 2488 case BC_TDUP:
1965 rc = emitir(IRTG(IR_TDUP, IRT_TAB), 2489 rc = emitir(IRTG(IR_TDUP, IRT_TAB),
1966 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0); 2490 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
2491#ifdef LUAJIT_ENABLE_TABLE_BUMP
2492 J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
2493 setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
2494 setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2495#endif
1967 break; 2496 break;
1968 2497
1969 /* -- Calls and vararg handling ----------------------------------------- */ 2498 /* -- Calls and vararg handling ----------------------------------------- */
1970 2499
1971 case BC_ITERC: 2500 case BC_ITERC:
1972 J->base[ra] = getslot(J, ra-3); 2501 J->base[ra] = getslot(J, ra-3);
1973 J->base[ra+1] = getslot(J, ra-2); 2502 J->base[ra+1+LJ_FR2] = getslot(J, ra-2);
1974 J->base[ra+2] = getslot(J, ra-1); 2503 J->base[ra+2+LJ_FR2] = getslot(J, ra-1);
1975 { /* Do the actual copy now because lj_record_call needs the values. */ 2504 { /* Do the actual copy now because lj_record_call needs the values. */
1976 TValue *b = &J->L->base[ra]; 2505 TValue *b = &J->L->base[ra];
1977 copyTV(J->L, b, b-3); 2506 copyTV(J->L, b, b-3);
1978 copyTV(J->L, b+1, b-2); 2507 copyTV(J->L, b+1+LJ_FR2, b-2);
1979 copyTV(J->L, b+2, b-1); 2508 copyTV(J->L, b+2+LJ_FR2, b-1);
1980 } 2509 }
1981 lj_record_call(J, ra, (ptrdiff_t)rc-1); 2510 lj_record_call(J, ra, (ptrdiff_t)rc-1);
1982 break; 2511 break;
1983 2512
1984 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */ 2513 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
1985 case BC_CALLM: 2514 case BC_CALLM:
1986 rc = (BCReg)(J->L->top - J->L->base) - ra; 2515 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
1987 /* fallthrough */ 2516 /* fallthrough */
1988 case BC_CALL: 2517 case BC_CALL:
1989 lj_record_call(J, ra, (ptrdiff_t)rc-1); 2518 lj_record_call(J, ra, (ptrdiff_t)rc-1);
1990 break; 2519 break;
1991 2520
1992 case BC_CALLMT: 2521 case BC_CALLMT:
1993 rc = (BCReg)(J->L->top - J->L->base) - ra; 2522 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
1994 /* fallthrough */ 2523 /* fallthrough */
1995 case BC_CALLT: 2524 case BC_CALLT:
1996 lj_record_tailcall(J, ra, (ptrdiff_t)rc-1); 2525 lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
@@ -2007,6 +2536,9 @@ void lj_record_ins(jit_State *J)
2007 rc = (BCReg)(J->L->top - J->L->base) - ra + 1; 2536 rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
2008 /* fallthrough */ 2537 /* fallthrough */
2009 case BC_RET: case BC_RET0: case BC_RET1: 2538 case BC_RET: case BC_RET0: case BC_RET1:
2539#if LJ_HASPROFILE
2540 rec_profile_ret(J);
2541#endif
2010 lj_record_ret(J, ra, (ptrdiff_t)rc-1); 2542 lj_record_ret(J, ra, (ptrdiff_t)rc-1);
2011 break; 2543 break;
2012 2544
@@ -2017,9 +2549,10 @@ void lj_record_ins(jit_State *J)
2017 J->loopref = J->cur.nins; 2549 J->loopref = J->cur.nins;
2018 break; 2550 break;
2019 case BC_JFORI: 2551 case BC_JFORI:
2020 lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL); 2552 lj_assertJ(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL,
2553 "JFORI does not point to JFORL");
2021 if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */ 2554 if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */
2022 rec_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J])); 2555 lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
2023 /* Continue tracing if the loop is not entered. */ 2556 /* Continue tracing if the loop is not entered. */
2024 break; 2557 break;
2025 2558
@@ -2029,6 +2562,9 @@ void lj_record_ins(jit_State *J)
2029 case BC_ITERL: 2562 case BC_ITERL:
2030 rec_loop_interp(J, pc, rec_iterl(J, *pc)); 2563 rec_loop_interp(J, pc, rec_iterl(J, *pc));
2031 break; 2564 break;
2565 case BC_ITERN:
2566 rec_loop_interp(J, pc, rec_itern(J, ra, rb));
2567 break;
2032 case BC_LOOP: 2568 case BC_LOOP:
2033 rec_loop_interp(J, pc, rec_loop(J, ra, 1)); 2569 rec_loop_interp(J, pc, rec_loop(J, ra, 1));
2034 break; 2570 break;
@@ -2041,7 +2577,8 @@ void lj_record_ins(jit_State *J)
2041 break; 2577 break;
2042 case BC_JLOOP: 2578 case BC_JLOOP:
2043 rec_loop_jit(J, rc, rec_loop(J, ra, 2579 rec_loop_jit(J, rc, rec_loop(J, ra,
2044 !bc_isret(bc_op(traceref(J, rc)->startins)))); 2580 !bc_isret(bc_op(traceref(J, rc)->startins)) &&
2581 bc_op(traceref(J, rc)->startins) != BC_ITERN));
2045 break; 2582 break;
2046 2583
2047 case BC_IFORL: 2584 case BC_IFORL:
@@ -2057,6 +2594,10 @@ void lj_record_ins(jit_State *J)
2057 J->maxslot = ra; /* Shrink used slots. */ 2594 J->maxslot = ra; /* Shrink used slots. */
2058 break; 2595 break;
2059 2596
2597 case BC_ISNEXT:
2598 rec_isnext(J, ra);
2599 break;
2600
2060 /* -- Function headers -------------------------------------------------- */ 2601 /* -- Function headers -------------------------------------------------- */
2061 2602
2062 case BC_FUNCF: 2603 case BC_FUNCF:
@@ -2071,7 +2612,8 @@ void lj_record_ins(jit_State *J)
2071 rec_func_lua(J); 2612 rec_func_lua(J);
2072 break; 2613 break;
2073 case BC_JFUNCV: 2614 case BC_JFUNCV:
2074 lua_assert(0); /* Cannot happen. No hotcall counting for varag funcs. */ 2615 /* Cannot happen. No hotcall counting for varag funcs. */
2616 lj_assertJ(0, "unsupported vararg hotcall");
2075 break; 2617 break;
2076 2618
2077 case BC_FUNCC: 2619 case BC_FUNCC:
@@ -2085,12 +2627,8 @@ void lj_record_ins(jit_State *J)
2085 break; 2627 break;
2086 } 2628 }
2087 /* fallthrough */ 2629 /* fallthrough */
2088 case BC_ITERN:
2089 case BC_ISNEXT:
2090 case BC_CAT:
2091 case BC_UCLO: 2630 case BC_UCLO:
2092 case BC_FNEW: 2631 case BC_FNEW:
2093 case BC_TSETM:
2094 setintV(&J->errinfo, (int32_t)op); 2632 setintV(&J->errinfo, (int32_t)op);
2095 lj_trace_err_info(J, LJ_TRERR_NYIBC); 2633 lj_trace_err_info(J, LJ_TRERR_NYIBC);
2096 break; 2634 break;
@@ -2099,15 +2637,21 @@ void lj_record_ins(jit_State *J)
2099 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */ 2637 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2100 if (bcmode_a(op) == BCMdst && rc) { 2638 if (bcmode_a(op) == BCMdst && rc) {
2101 J->base[ra] = rc; 2639 J->base[ra] = rc;
2102 if (ra >= J->maxslot) J->maxslot = ra+1; 2640 if (ra >= J->maxslot) {
2641#if LJ_FR2
2642 if (ra > J->maxslot) J->base[ra-1] = 0;
2643#endif
2644 J->maxslot = ra+1;
2645 }
2103 } 2646 }
2104 2647
2105#undef rav 2648#undef rav
2106#undef rbv 2649#undef rbv
2107#undef rcv 2650#undef rcv
2108 2651
2109 /* Limit the number of recorded IR instructions. */ 2652 /* Limit the number of recorded IR instructions and constants. */
2110 if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord]) 2653 if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] ||
2654 J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst])
2111 lj_trace_err(J, LJ_TRERR_TRACEOV); 2655 lj_trace_err(J, LJ_TRERR_TRACEOV);
2112} 2656}
2113 2657
@@ -2127,13 +2671,22 @@ static const BCIns *rec_setup_root(jit_State *J)
2127 J->bc_min = pc; 2671 J->bc_min = pc;
2128 break; 2672 break;
2129 case BC_ITERL: 2673 case BC_ITERL:
2130 lua_assert(bc_op(pc[-1]) == BC_ITERC); 2674 if (bc_op(pc[-1]) == BC_JLOOP)
2675 lj_trace_err(J, LJ_TRERR_LINNER);
2676 lj_assertJ(bc_op(pc[-1]) == BC_ITERC, "no ITERC before ITERL");
2131 J->maxslot = ra + bc_b(pc[-1]) - 1; 2677 J->maxslot = ra + bc_b(pc[-1]) - 1;
2132 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); 2678 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2133 pc += 1+bc_j(ins); 2679 pc += 1+bc_j(ins);
2134 lua_assert(bc_op(pc[-1]) == BC_JMP); 2680 lj_assertJ(bc_op(pc[-1]) == BC_JMP, "ITERL does not point to JMP+1");
2135 J->bc_min = pc; 2681 J->bc_min = pc;
2136 break; 2682 break;
2683 case BC_ITERN:
2684 lj_assertJ(bc_op(pc[1]) == BC_ITERL, "no ITERL after ITERN");
2685 J->maxslot = ra;
2686 J->bc_extent = (MSize)(-bc_j(pc[1]))*sizeof(BCIns);
2687 J->bc_min = pc+2 + bc_j(pc[1]);
2688 J->state = LJ_TRACE_RECORD_1ST; /* Record the first ITERN, too. */
2689 break;
2137 case BC_LOOP: 2690 case BC_LOOP:
2138 /* Only check BC range for real loops, but not for "repeat until true". */ 2691 /* Only check BC range for real loops, but not for "repeat until true". */
2139 pcj = pc + bc_j(ins); 2692 pcj = pc + bc_j(ins);
@@ -2156,8 +2709,14 @@ static const BCIns *rec_setup_root(jit_State *J)
2156 J->maxslot = J->pt->numparams; 2709 J->maxslot = J->pt->numparams;
2157 pc++; 2710 pc++;
2158 break; 2711 break;
2712 case BC_CALLM:
2713 case BC_CALL:
2714 case BC_ITERC:
2715 /* No bytecode range check for stitched traces. */
2716 pc++;
2717 break;
2159 default: 2718 default:
2160 lua_assert(0); 2719 lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins));
2161 break; 2720 break;
2162 } 2721 }
2163 return pc; 2722 return pc;
@@ -2171,11 +2730,14 @@ void lj_record_setup(jit_State *J)
2171 /* Initialize state related to current trace. */ 2730 /* Initialize state related to current trace. */
2172 memset(J->slot, 0, sizeof(J->slot)); 2731 memset(J->slot, 0, sizeof(J->slot));
2173 memset(J->chain, 0, sizeof(J->chain)); 2732 memset(J->chain, 0, sizeof(J->chain));
2733#ifdef LUAJIT_ENABLE_TABLE_BUMP
2734 memset(J->rbchash, 0, sizeof(J->rbchash));
2735#endif
2174 memset(J->bpropcache, 0, sizeof(J->bpropcache)); 2736 memset(J->bpropcache, 0, sizeof(J->bpropcache));
2175 J->scev.idx = REF_NIL; 2737 J->scev.idx = REF_NIL;
2176 setmref(J->scev.pc, NULL); 2738 setmref(J->scev.pc, NULL);
2177 2739
2178 J->baseslot = 1; /* Invoking function is at base[-1]. */ 2740 J->baseslot = 1+LJ_FR2; /* Invoking function is at base[-1-LJ_FR2]. */
2179 J->base = J->slot + J->baseslot; 2741 J->base = J->slot + J->baseslot;
2180 J->maxslot = 0; 2742 J->maxslot = 0;
2181 J->framedepth = 0; 2743 J->framedepth = 0;
@@ -2190,7 +2752,7 @@ void lj_record_setup(jit_State *J)
2190 J->bc_extent = ~(MSize)0; 2752 J->bc_extent = ~(MSize)0;
2191 2753
2192 /* Emit instructions for fixed references. Also triggers initial IR alloc. */ 2754 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2193 emitir_raw(IRT(IR_BASE, IRT_P32), J->parent, J->exitno); 2755 emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno);
2194 for (i = 0; i <= 2; i++) { 2756 for (i = 0; i <= 2; i++) {
2195 IRIns *ir = IR(REF_NIL-i); 2757 IRIns *ir = IR(REF_NIL-i);
2196 ir->i = 0; 2758 ir->i = 0;
@@ -2221,10 +2783,15 @@ void lj_record_setup(jit_State *J)
2221 } 2783 }
2222 lj_snap_replay(J, T); 2784 lj_snap_replay(J, T);
2223 sidecheck: 2785 sidecheck:
2224 if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] || 2786 if ((traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
2225 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] + 2787 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2226 J->param[JIT_P_tryside]) { 2788 J->param[JIT_P_tryside])) {
2227 rec_stop(J, LJ_TRLINK_INTERP, 0); 2789 if (bc_op(*J->pc) == BC_JLOOP) {
2790 BCIns startins = traceref(J, bc_d(*J->pc))->startins;
2791 if (bc_op(startins) == BC_ITERN)
2792 rec_itern(J, bc_a(startins), bc_b(startins));
2793 }
2794 lj_record_stop(J, LJ_TRLINK_INTERP, 0);
2228 } 2795 }
2229 } else { /* Root trace. */ 2796 } else { /* Root trace. */
2230 J->cur.root = 0; 2797 J->cur.root = 0;
@@ -2232,13 +2799,20 @@ void lj_record_setup(jit_State *J)
2232 J->pc = rec_setup_root(J); 2799 J->pc = rec_setup_root(J);
2233 /* Note: the loop instruction itself is recorded at the end and not 2800 /* Note: the loop instruction itself is recorded at the end and not
2234 ** at the start! So snapshot #0 needs to point to the *next* instruction. 2801 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2802 ** The one exception is BC_ITERN, which sets LJ_TRACE_RECORD_1ST.
2235 */ 2803 */
2236 lj_snap_add(J); 2804 lj_snap_add(J);
2237 if (bc_op(J->cur.startins) == BC_FORL) 2805 if (bc_op(J->cur.startins) == BC_FORL)
2238 rec_for_loop(J, J->pc-1, &J->scev, 1); 2806 rec_for_loop(J, J->pc-1, &J->scev, 1);
2807 else if (bc_op(J->cur.startins) == BC_ITERC)
2808 J->startpc = NULL;
2239 if (1 + J->pt->framesize >= LJ_MAX_JSLOTS) 2809 if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2240 lj_trace_err(J, LJ_TRERR_STACKOV); 2810 lj_trace_err(J, LJ_TRERR_STACKOV);
2241 } 2811 }
2812#if LJ_HASPROFILE
2813 J->prev_pt = NULL;
2814 J->prev_line = -1;
2815#endif
2242#ifdef LUAJIT_ENABLE_CHECKHOOK 2816#ifdef LUAJIT_ENABLE_CHECKHOOK
2243 /* Regularly check for instruction/line hooks from compiled code and 2817 /* Regularly check for instruction/line hooks from compiled code and
2244 ** exit to the interpreter if the hooks are set. 2818 ** exit to the interpreter if the hooks are set.
diff --git a/src/lj_record.h b/src/lj_record.h
index 08b7ba20..61ccb395 100644
--- a/src/lj_record.h
+++ b/src/lj_record.h
@@ -28,7 +28,9 @@ typedef struct RecordIndex {
28 28
29LJ_FUNC int lj_record_objcmp(jit_State *J, TRef a, TRef b, 29LJ_FUNC int lj_record_objcmp(jit_State *J, TRef a, TRef b,
30 cTValue *av, cTValue *bv); 30 cTValue *av, cTValue *bv);
31LJ_FUNC void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk);
31LJ_FUNC TRef lj_record_constify(jit_State *J, cTValue *o); 32LJ_FUNC TRef lj_record_constify(jit_State *J, cTValue *o);
33LJ_FUNC TRef lj_record_vload(jit_State *J, TRef ref, MSize idx, IRType t);
32 34
33LJ_FUNC void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs); 35LJ_FUNC void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs);
34LJ_FUNC void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs); 36LJ_FUNC void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs);
@@ -36,6 +38,7 @@ LJ_FUNC void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults);
36 38
37LJ_FUNC int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm); 39LJ_FUNC int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm);
38LJ_FUNC TRef lj_record_idx(jit_State *J, RecordIndex *ix); 40LJ_FUNC TRef lj_record_idx(jit_State *J, RecordIndex *ix);
41LJ_FUNC int lj_record_next(jit_State *J, RecordIndex *ix);
39 42
40LJ_FUNC void lj_record_ins(jit_State *J); 43LJ_FUNC void lj_record_ins(jit_State *J);
41LJ_FUNC void lj_record_setup(jit_State *J); 44LJ_FUNC void lj_record_setup(jit_State *J);
diff --git a/src/lj_serialize.c b/src/lj_serialize.c
new file mode 100644
index 00000000..83881766
--- /dev/null
+++ b/src/lj_serialize.c
@@ -0,0 +1,539 @@
1/*
2** Object de/serialization.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_serialize_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASBUFFER
12#include "lj_err.h"
13#include "lj_buf.h"
14#include "lj_str.h"
15#include "lj_tab.h"
16#include "lj_udata.h"
17#if LJ_HASFFI
18#include "lj_ctype.h"
19#include "lj_cdata.h"
20#endif
21#if LJ_HASJIT
22#include "lj_ir.h"
23#endif
24#include "lj_serialize.h"
25
26/* Tags for internal serialization format. */
27enum {
28 SER_TAG_NIL, /* 0x00 */
29 SER_TAG_FALSE,
30 SER_TAG_TRUE,
31 SER_TAG_NULL,
32 SER_TAG_LIGHTUD32,
33 SER_TAG_LIGHTUD64,
34 SER_TAG_INT,
35 SER_TAG_NUM,
36 SER_TAG_TAB, /* 0x08 */
37 SER_TAG_DICT_MT = SER_TAG_TAB+6,
38 SER_TAG_DICT_STR,
39 SER_TAG_INT64, /* 0x10 */
40 SER_TAG_UINT64,
41 SER_TAG_COMPLEX,
42 SER_TAG_0x13,
43 SER_TAG_0x14,
44 SER_TAG_0x15,
45 SER_TAG_0x16,
46 SER_TAG_0x17,
47 SER_TAG_0x18, /* 0x18 */
48 SER_TAG_0x19,
49 SER_TAG_0x1a,
50 SER_TAG_0x1b,
51 SER_TAG_0x1c,
52 SER_TAG_0x1d,
53 SER_TAG_0x1e,
54 SER_TAG_0x1f,
55 SER_TAG_STR, /* 0x20 + str->len */
56};
57LJ_STATIC_ASSERT((SER_TAG_TAB & 7) == 0);
58
59/* -- Helper functions ---------------------------------------------------- */
60
61static LJ_AINLINE char *serialize_more(char *w, SBufExt *sbx, MSize sz)
62{
63 if (LJ_UNLIKELY(sz > (MSize)(sbx->e - w))) {
64 sbx->w = w;
65 w = lj_buf_more2((SBuf *)sbx, sz);
66 }
67 return w;
68}
69
70/* Write U124 to buffer. */
71static LJ_NOINLINE char *serialize_wu124_(char *w, uint32_t v)
72{
73 if (v < 0x1fe0) {
74 v -= 0xe0;
75 *w++ = (char)(0xe0 | (v >> 8)); *w++ = (char)v;
76 } else {
77 *w++ = (char)0xff;
78#if LJ_BE
79 v = lj_bswap(v);
80#endif
81 memcpy(w, &v, 4); w += 4;
82 }
83 return w;
84}
85
86static LJ_AINLINE char *serialize_wu124(char *w, uint32_t v)
87{
88 if (LJ_LIKELY(v < 0xe0)) {
89 *w++ = (char)v;
90 return w;
91 } else {
92 return serialize_wu124_(w, v);
93 }
94}
95
96static LJ_NOINLINE char *serialize_ru124_(char *r, char *w, uint32_t *pv)
97{
98 uint32_t v = *pv;
99 if (v != 0xff) {
100 if (r >= w) return NULL;
101 v = ((v & 0x1f) << 8) + *(uint8_t *)r + 0xe0; r++;
102 } else {
103 if (r + 4 > w) return NULL;
104 v = lj_getu32(r); r += 4;
105#if LJ_BE
106 v = lj_bswap(v);
107#endif
108 }
109 *pv = v;
110 return r;
111}
112
113static LJ_AINLINE char *serialize_ru124(char *r, char *w, uint32_t *pv)
114{
115 if (LJ_LIKELY(r < w)) {
116 uint32_t v = *(uint8_t *)r; r++;
117 *pv = v;
118 if (LJ_UNLIKELY(v >= 0xe0)) {
119 r = serialize_ru124_(r, w, pv);
120 }
121 return r;
122 }
123 return NULL;
124}
125
126/* Prepare string dictionary for use (once). */
127void LJ_FASTCALL lj_serialize_dict_prep_str(lua_State *L, GCtab *dict)
128{
129 if (!dict->hmask) { /* No hash part means not prepared, yet. */
130 MSize i, len = lj_tab_len(dict);
131 if (!len) return;
132 lj_tab_resize(L, dict, dict->asize, hsize2hbits(len));
133 for (i = 1; i <= len && i < dict->asize; i++) {
134 cTValue *o = arrayslot(dict, i);
135 if (tvisstr(o)) {
136 if (!lj_tab_getstr(dict, strV(o))) { /* Ignore dups. */
137 lj_tab_newkey(L, dict, o)->u64 = (uint64_t)(i-1);
138 }
139 } else if (!tvisfalse(o)) {
140 lj_err_caller(L, LJ_ERR_BUFFER_BADOPT);
141 }
142 }
143 }
144}
145
146/* Prepare metatable dictionary for use (once). */
147void LJ_FASTCALL lj_serialize_dict_prep_mt(lua_State *L, GCtab *dict)
148{
149 if (!dict->hmask) { /* No hash part means not prepared, yet. */
150 MSize i, len = lj_tab_len(dict);
151 if (!len) return;
152 lj_tab_resize(L, dict, dict->asize, hsize2hbits(len));
153 for (i = 1; i <= len && i < dict->asize; i++) {
154 cTValue *o = arrayslot(dict, i);
155 if (tvistab(o)) {
156 if (tvisnil(lj_tab_get(L, dict, o))) { /* Ignore dups. */
157 lj_tab_newkey(L, dict, o)->u64 = (uint64_t)(i-1);
158 }
159 } else if (!tvisfalse(o)) {
160 lj_err_caller(L, LJ_ERR_BUFFER_BADOPT);
161 }
162 }
163 }
164}
165
166/* -- Internal serializer ------------------------------------------------- */
167
168/* Put serialized object into buffer. */
169static char *serialize_put(char *w, SBufExt *sbx, cTValue *o)
170{
171 if (LJ_LIKELY(tvisstr(o))) {
172 const GCstr *str = strV(o);
173 MSize len = str->len;
174 w = serialize_more(w, sbx, 5+len);
175 w = serialize_wu124(w, SER_TAG_STR + len);
176 w = lj_buf_wmem(w, strdata(str), len);
177 } else if (tvisint(o)) {
178 uint32_t x = LJ_BE ? lj_bswap((uint32_t)intV(o)) : (uint32_t)intV(o);
179 w = serialize_more(w, sbx, 1+4);
180 *w++ = SER_TAG_INT; memcpy(w, &x, 4); w += 4;
181 } else if (tvisnum(o)) {
182 uint64_t x = LJ_BE ? lj_bswap64(o->u64) : o->u64;
183 w = serialize_more(w, sbx, 1+sizeof(lua_Number));
184 *w++ = SER_TAG_NUM; memcpy(w, &x, 8); w += 8;
185 } else if (tvispri(o)) {
186 w = serialize_more(w, sbx, 1);
187 *w++ = (char)(SER_TAG_NIL + ~itype(o));
188 } else if (tvistab(o)) {
189 const GCtab *t = tabV(o);
190 uint32_t narray = 0, nhash = 0, one = 2;
191 if (sbx->depth <= 0) lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_DEPTH);
192 sbx->depth--;
193 if (t->asize > 0) { /* Determine max. length of array part. */
194 ptrdiff_t i;
195 TValue *array = tvref(t->array);
196 for (i = (ptrdiff_t)t->asize-1; i >= 0; i--)
197 if (!tvisnil(&array[i]))
198 break;
199 narray = (uint32_t)(i+1);
200 if (narray && tvisnil(&array[0])) one = 4;
201 }
202 if (t->hmask > 0) { /* Count number of used hash slots. */
203 uint32_t i, hmask = t->hmask;
204 Node *node = noderef(t->node);
205 for (i = 0; i <= hmask; i++)
206 nhash += !tvisnil(&node[i].val);
207 }
208 /* Write metatable index. */
209 if (LJ_UNLIKELY(tabref(sbx->dict_mt)) && tabref(t->metatable)) {
210 TValue mto;
211 Node *n;
212 settabV(sbufL(sbx), &mto, tabref(t->metatable));
213 n = hashgcref(tabref(sbx->dict_mt), mto.gcr);
214 do {
215 if (n->key.u64 == mto.u64) {
216 uint32_t idx = n->val.u32.lo;
217 w = serialize_more(w, sbx, 1+5);
218 *w++ = SER_TAG_DICT_MT;
219 w = serialize_wu124(w, idx);
220 break;
221 }
222 } while ((n = nextnode(n)));
223 }
224 /* Write number of array slots and hash slots. */
225 w = serialize_more(w, sbx, 1+2*5);
226 *w++ = (char)(SER_TAG_TAB + (nhash ? 1 : 0) + (narray ? one : 0));
227 if (narray) w = serialize_wu124(w, narray);
228 if (nhash) w = serialize_wu124(w, nhash);
229 if (narray) { /* Write array entries. */
230 cTValue *oa = tvref(t->array) + (one >> 2);
231 cTValue *oe = tvref(t->array) + narray;
232 while (oa < oe) w = serialize_put(w, sbx, oa++);
233 }
234 if (nhash) { /* Write hash entries. */
235 const Node *node = noderef(t->node) + t->hmask;
236 GCtab *dict_str = tabref(sbx->dict_str);
237 if (LJ_UNLIKELY(dict_str)) {
238 for (;; node--)
239 if (!tvisnil(&node->val)) {
240 if (LJ_LIKELY(tvisstr(&node->key))) {
241 /* Inlined lj_tab_getstr is 30% faster. */
242 const GCstr *str = strV(&node->key);
243 Node *n = hashstr(dict_str, str);
244 do {
245 if (tvisstr(&n->key) && strV(&n->key) == str) {
246 uint32_t idx = n->val.u32.lo;
247 w = serialize_more(w, sbx, 1+5);
248 *w++ = SER_TAG_DICT_STR;
249 w = serialize_wu124(w, idx);
250 break;
251 }
252 n = nextnode(n);
253 if (!n) {
254 MSize len = str->len;
255 w = serialize_more(w, sbx, 5+len);
256 w = serialize_wu124(w, SER_TAG_STR + len);
257 w = lj_buf_wmem(w, strdata(str), len);
258 break;
259 }
260 } while (1);
261 } else {
262 w = serialize_put(w, sbx, &node->key);
263 }
264 w = serialize_put(w, sbx, &node->val);
265 if (--nhash == 0) break;
266 }
267 } else {
268 for (;; node--)
269 if (!tvisnil(&node->val)) {
270 w = serialize_put(w, sbx, &node->key);
271 w = serialize_put(w, sbx, &node->val);
272 if (--nhash == 0) break;
273 }
274 }
275 }
276 sbx->depth++;
277#if LJ_HASFFI
278 } else if (tviscdata(o)) {
279 CTState *cts = ctype_cts(sbufL(sbx));
280 CType *s = ctype_raw(cts, cdataV(o)->ctypeid);
281 uint8_t *sp = cdataptr(cdataV(o));
282 if (ctype_isinteger(s->info) && s->size == 8) {
283 w = serialize_more(w, sbx, 1+8);
284 *w++ = (s->info & CTF_UNSIGNED) ? SER_TAG_UINT64 : SER_TAG_INT64;
285#if LJ_BE
286 { uint64_t u = lj_bswap64(*(uint64_t *)sp); memcpy(w, &u, 8); }
287#else
288 memcpy(w, sp, 8);
289#endif
290 w += 8;
291 } else if (ctype_iscomplex(s->info) && s->size == 16) {
292 w = serialize_more(w, sbx, 1+16);
293 *w++ = SER_TAG_COMPLEX;
294#if LJ_BE
295 { /* Only swap the doubles. The re/im order stays the same. */
296 uint64_t u = lj_bswap64(((uint64_t *)sp)[0]); memcpy(w, &u, 8);
297 u = lj_bswap64(((uint64_t *)sp)[1]); memcpy(w+8, &u, 8);
298 }
299#else
300 memcpy(w, sp, 16);
301#endif
302 w += 16;
303 } else {
304 goto badenc; /* NYI other cdata */
305 }
306#endif
307 } else if (tvislightud(o)) {
308 uintptr_t ud = (uintptr_t)lightudV(G(sbufL(sbx)), o);
309 w = serialize_more(w, sbx, 1+sizeof(ud));
310 if (ud == 0) {
311 *w++ = SER_TAG_NULL;
312 } else if (LJ_32 || checku32(ud)) {
313#if LJ_BE && LJ_64
314 ud = lj_bswap64(ud);
315#elif LJ_BE
316 ud = lj_bswap(ud);
317#endif
318 *w++ = SER_TAG_LIGHTUD32; memcpy(w, &ud, 4); w += 4;
319#if LJ_64
320 } else {
321#if LJ_BE
322 ud = lj_bswap64(ud);
323#endif
324 *w++ = SER_TAG_LIGHTUD64; memcpy(w, &ud, 8); w += 8;
325#endif
326 }
327 } else {
328 /* NYI userdata */
329#if LJ_HASFFI
330 badenc:
331#endif
332 lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADENC, lj_typename(o));
333 }
334 return w;
335}
336
337/* Get serialized object from buffer. */
338static char *serialize_get(char *r, SBufExt *sbx, TValue *o)
339{
340 char *w = sbx->w;
341 uint32_t tp;
342 r = serialize_ru124(r, w, &tp); if (LJ_UNLIKELY(!r)) goto eob;
343 if (LJ_LIKELY(tp >= SER_TAG_STR)) {
344 uint32_t len = tp - SER_TAG_STR;
345 if (LJ_UNLIKELY(len > (uint32_t)(w - r))) goto eob;
346 setstrV(sbufL(sbx), o, lj_str_new(sbufL(sbx), r, len));
347 r += len;
348 } else if (tp == SER_TAG_INT) {
349 if (LJ_UNLIKELY(r + 4 > w)) goto eob;
350 setintV(o, (int32_t)(LJ_BE ? lj_bswap(lj_getu32(r)) : lj_getu32(r)));
351 r += 4;
352 } else if (tp == SER_TAG_NUM) {
353 if (LJ_UNLIKELY(r + 8 > w)) goto eob;
354 memcpy(o, r, 8); r += 8;
355#if LJ_BE
356 o->u64 = lj_bswap64(o->u64);
357#endif
358 if (!tvisnum(o)) setnanV(o); /* Fix non-canonical NaNs. */
359 } else if (tp <= SER_TAG_TRUE) {
360 setpriV(o, ~tp);
361 } else if (tp == SER_TAG_DICT_STR) {
362 GCtab *dict_str;
363 uint32_t idx;
364 r = serialize_ru124(r, w, &idx); if (LJ_UNLIKELY(!r)) goto eob;
365 idx++;
366 dict_str = tabref(sbx->dict_str);
367 if (dict_str && idx < dict_str->asize && tvisstr(arrayslot(dict_str, idx)))
368 copyTV(sbufL(sbx), o, arrayslot(dict_str, idx));
369 else
370 lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADDICTX, idx);
371 } else if (tp >= SER_TAG_TAB && tp <= SER_TAG_DICT_MT) {
372 uint32_t narray = 0, nhash = 0;
373 GCtab *t, *mt = NULL;
374 if (sbx->depth <= 0) lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_DEPTH);
375 sbx->depth--;
376 if (tp == SER_TAG_DICT_MT) {
377 GCtab *dict_mt;
378 uint32_t idx;
379 r = serialize_ru124(r, w, &idx); if (LJ_UNLIKELY(!r)) goto eob;
380 idx++;
381 dict_mt = tabref(sbx->dict_mt);
382 if (dict_mt && idx < dict_mt->asize && tvistab(arrayslot(dict_mt, idx)))
383 mt = tabV(arrayslot(dict_mt, idx));
384 else
385 lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADDICTX, idx);
386 r = serialize_ru124(r, w, &tp); if (LJ_UNLIKELY(!r)) goto eob;
387 if (!(tp >= SER_TAG_TAB && tp < SER_TAG_DICT_MT)) goto badtag;
388 }
389 if (tp >= SER_TAG_TAB+2) {
390 r = serialize_ru124(r, w, &narray); if (LJ_UNLIKELY(!r)) goto eob;
391 }
392 if ((tp & 1)) {
393 r = serialize_ru124(r, w, &nhash); if (LJ_UNLIKELY(!r)) goto eob;
394 }
395 t = lj_tab_new(sbufL(sbx), narray, hsize2hbits(nhash));
396 /* NOBARRIER: The table is new (marked white). */
397 setgcref(t->metatable, obj2gco(mt));
398 settabV(sbufL(sbx), o, t);
399 if (narray) {
400 TValue *oa = tvref(t->array) + (tp >= SER_TAG_TAB+4);
401 TValue *oe = tvref(t->array) + narray;
402 while (oa < oe) r = serialize_get(r, sbx, oa++);
403 }
404 if (nhash) {
405 do {
406 TValue k, *v;
407 r = serialize_get(r, sbx, &k);
408 v = lj_tab_set(sbufL(sbx), t, &k);
409 if (LJ_UNLIKELY(!tvisnil(v)))
410 lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_DUPKEY);
411 r = serialize_get(r, sbx, v);
412 } while (--nhash);
413 }
414 sbx->depth++;
415#if LJ_HASFFI
416 } else if (tp >= SER_TAG_INT64 && tp <= SER_TAG_COMPLEX) {
417 uint32_t sz = tp == SER_TAG_COMPLEX ? 16 : 8;
418 GCcdata *cd;
419 if (LJ_UNLIKELY(r + sz > w)) goto eob;
420 if (LJ_UNLIKELY(!ctype_ctsG(G(sbufL(sbx))))) goto badtag;
421 cd = lj_cdata_new_(sbufL(sbx),
422 tp == SER_TAG_INT64 ? CTID_INT64 :
423 tp == SER_TAG_UINT64 ? CTID_UINT64 : CTID_COMPLEX_DOUBLE,
424 sz);
425 memcpy(cdataptr(cd), r, sz); r += sz;
426#if LJ_BE
427 *(uint64_t *)cdataptr(cd) = lj_bswap64(*(uint64_t *)cdataptr(cd));
428 if (sz == 16)
429 ((uint64_t *)cdataptr(cd))[1] = lj_bswap64(((uint64_t *)cdataptr(cd))[1]);
430#endif
431 if (sz == 16) { /* Fix non-canonical NaNs. */
432 TValue *cdo = (TValue *)cdataptr(cd);
433 if (!tvisnum(&cdo[0])) setnanV(&cdo[0]);
434 if (!tvisnum(&cdo[1])) setnanV(&cdo[1]);
435 }
436 setcdataV(sbufL(sbx), o, cd);
437#endif
438 } else if (tp <= (LJ_64 ? SER_TAG_LIGHTUD64 : SER_TAG_LIGHTUD32)) {
439 uintptr_t ud = 0;
440 if (tp == SER_TAG_LIGHTUD32) {
441 if (LJ_UNLIKELY(r + 4 > w)) goto eob;
442 ud = (uintptr_t)(LJ_BE ? lj_bswap(lj_getu32(r)) : lj_getu32(r));
443 r += 4;
444 }
445#if LJ_64
446 else if (tp == SER_TAG_LIGHTUD64) {
447 if (LJ_UNLIKELY(r + 8 > w)) goto eob;
448 memcpy(&ud, r, 8); r += 8;
449#if LJ_BE
450 ud = lj_bswap64(ud);
451#endif
452 }
453 setrawlightudV(o, lj_lightud_intern(sbufL(sbx), (void *)ud));
454#else
455 setrawlightudV(o, (void *)ud);
456#endif
457 } else {
458badtag:
459 lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADDEC, tp);
460 }
461 return r;
462eob:
463 lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_EOB);
464 return NULL;
465}
466
467/* -- External serialization API ------------------------------------------ */
468
469/* Encode to buffer. */
470SBufExt * LJ_FASTCALL lj_serialize_put(SBufExt *sbx, cTValue *o)
471{
472 sbx->depth = LJ_SERIALIZE_DEPTH;
473 sbx->w = serialize_put(sbx->w, sbx, o);
474 return sbx;
475}
476
477/* Decode from buffer. */
478char * LJ_FASTCALL lj_serialize_get(SBufExt *sbx, TValue *o)
479{
480 sbx->depth = LJ_SERIALIZE_DEPTH;
481 return serialize_get(sbx->r, sbx, o);
482}
483
484/* Stand-alone encoding, borrowing from global temporary buffer. */
485GCstr * LJ_FASTCALL lj_serialize_encode(lua_State *L, cTValue *o)
486{
487 SBufExt sbx;
488 char *w;
489 memset(&sbx, 0, sizeof(SBufExt));
490 lj_bufx_set_borrow(L, &sbx, &G(L)->tmpbuf);
491 sbx.depth = LJ_SERIALIZE_DEPTH;
492 w = serialize_put(sbx.w, &sbx, o);
493 return lj_str_new(L, sbx.b, (size_t)(w - sbx.b));
494}
495
496/* Stand-alone decoding, copy-on-write from string. */
497void lj_serialize_decode(lua_State *L, TValue *o, GCstr *str)
498{
499 SBufExt sbx;
500 char *r;
501 memset(&sbx, 0, sizeof(SBufExt));
502 lj_bufx_set_cow(L, &sbx, strdata(str), str->len);
503 /* No need to set sbx.cowref here. */
504 sbx.depth = LJ_SERIALIZE_DEPTH;
505 r = serialize_get(sbx.r, &sbx, o);
506 if (r != sbx.w) lj_err_caller(L, LJ_ERR_BUFFER_LEFTOV);
507}
508
509#if LJ_HASJIT
510/* Peek into buffer to find the result IRType for specialization purposes. */
511LJ_FUNC MSize LJ_FASTCALL lj_serialize_peektype(SBufExt *sbx)
512{
513 uint32_t tp;
514 if (serialize_ru124(sbx->r, sbx->w, &tp)) {
515 /* This must match the handling of all tags in the decoder above. */
516 switch (tp) {
517 case SER_TAG_NIL: return IRT_NIL;
518 case SER_TAG_FALSE: return IRT_FALSE;
519 case SER_TAG_TRUE: return IRT_TRUE;
520 case SER_TAG_NULL: case SER_TAG_LIGHTUD32: case SER_TAG_LIGHTUD64:
521 return IRT_LIGHTUD;
522 case SER_TAG_INT: return LJ_DUALNUM ? IRT_INT : IRT_NUM;
523 case SER_TAG_NUM: return IRT_NUM;
524 case SER_TAG_TAB: case SER_TAG_TAB+1: case SER_TAG_TAB+2:
525 case SER_TAG_TAB+3: case SER_TAG_TAB+4: case SER_TAG_TAB+5:
526 case SER_TAG_DICT_MT:
527 return IRT_TAB;
528 case SER_TAG_INT64: case SER_TAG_UINT64: case SER_TAG_COMPLEX:
529 return IRT_CDATA;
530 case SER_TAG_DICT_STR:
531 default:
532 return IRT_STR;
533 }
534 }
535 return IRT_NIL; /* Will fail on actual decode. */
536}
537#endif
538
539#endif
diff --git a/src/lj_serialize.h b/src/lj_serialize.h
new file mode 100644
index 00000000..da823573
--- /dev/null
+++ b/src/lj_serialize.h
@@ -0,0 +1,28 @@
1/*
2** Object de/serialization.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_SERIALIZE_H
7#define _LJ_SERIALIZE_H
8
9#include "lj_obj.h"
10#include "lj_buf.h"
11
12#if LJ_HASBUFFER
13
14#define LJ_SERIALIZE_DEPTH 100 /* Default depth. */
15
16LJ_FUNC void LJ_FASTCALL lj_serialize_dict_prep_str(lua_State *L, GCtab *dict);
17LJ_FUNC void LJ_FASTCALL lj_serialize_dict_prep_mt(lua_State *L, GCtab *dict);
18LJ_FUNC SBufExt * LJ_FASTCALL lj_serialize_put(SBufExt *sbx, cTValue *o);
19LJ_FUNC char * LJ_FASTCALL lj_serialize_get(SBufExt *sbx, TValue *o);
20LJ_FUNC GCstr * LJ_FASTCALL lj_serialize_encode(lua_State *L, cTValue *o);
21LJ_FUNC void lj_serialize_decode(lua_State *L, TValue *o, GCstr *str);
22#if LJ_HASJIT
23LJ_FUNC MSize LJ_FASTCALL lj_serialize_peektype(SBufExt *sbx);
24#endif
25
26#endif
27
28#endif
diff --git a/src/lj_snap.c b/src/lj_snap.c
index 1ef75e83..877ce099 100644
--- a/src/lj_snap.c
+++ b/src/lj_snap.c
@@ -68,20 +68,37 @@ static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
68 for (s = 0; s < nslots; s++) { 68 for (s = 0; s < nslots; s++) {
69 TRef tr = J->slot[s]; 69 TRef tr = J->slot[s];
70 IRRef ref = tref_ref(tr); 70 IRRef ref = tref_ref(tr);
71#if LJ_FR2
72 if (s == 1) { /* Ignore slot 1 in LJ_FR2 mode, except if tailcalled. */
73 if ((tr & TREF_FRAME))
74 map[n++] = SNAP(1, SNAP_FRAME | SNAP_NORESTORE, REF_NIL);
75 continue;
76 }
77 if ((tr & (TREF_FRAME | TREF_CONT)) && !ref) {
78 cTValue *base = J->L->base - J->baseslot;
79 tr = J->slot[s] = (tr & 0xff0000) | lj_ir_k64(J, IR_KNUM, base[s].u64);
80 ref = tref_ref(tr);
81 }
82#endif
71 if (ref) { 83 if (ref) {
72 SnapEntry sn = SNAP_TR(s, tr); 84 SnapEntry sn = SNAP_TR(s, tr);
73 IRIns *ir = &J->cur.ir[ref]; 85 IRIns *ir = &J->cur.ir[ref];
74 if (!(sn & (SNAP_CONT|SNAP_FRAME)) && 86 if ((LJ_FR2 || !(sn & (SNAP_CONT|SNAP_FRAME))) &&
75 ir->o == IR_SLOAD && ir->op1 == s && ref > retf) { 87 ir->o == IR_SLOAD && ir->op1 == s && ref > retf) {
76 /* No need to snapshot unmodified non-inherited slots. */ 88 /*
77 if (!(ir->op2 & IRSLOAD_INHERIT)) 89 ** No need to snapshot unmodified non-inherited slots.
90 ** But always snapshot the function below a frame in LJ_FR2 mode.
91 */
92 if (!(ir->op2 & IRSLOAD_INHERIT) &&
93 (!LJ_FR2 || s == 0 || s+1 == nslots ||
94 !(J->slot[s+1] & (TREF_CONT|TREF_FRAME))))
78 continue; 95 continue;
79 /* No need to restore readonly slots and unmodified non-parent slots. */ 96 /* No need to restore readonly slots and unmodified non-parent slots. */
80 if (!(LJ_DUALNUM && (ir->op2 & IRSLOAD_CONVERT)) && 97 if (!(LJ_DUALNUM && (ir->op2 & IRSLOAD_CONVERT)) &&
81 (ir->op2 & (IRSLOAD_READONLY|IRSLOAD_PARENT)) != IRSLOAD_PARENT) 98 (ir->op2 & (IRSLOAD_READONLY|IRSLOAD_PARENT)) != IRSLOAD_PARENT)
82 sn |= SNAP_NORESTORE; 99 sn |= SNAP_NORESTORE;
83 } 100 }
84 if (LJ_SOFTFP && irt_isnum(ir->t)) 101 if (LJ_SOFTFP32 && irt_isnum(ir->t))
85 sn |= SNAP_SOFTFPNUM; 102 sn |= SNAP_SOFTFPNUM;
86 map[n++] = sn; 103 map[n++] = sn;
87 } 104 }
@@ -90,35 +107,54 @@ static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
90} 107}
91 108
92/* Add frame links at the end of the snapshot. */ 109/* Add frame links at the end of the snapshot. */
93static BCReg snapshot_framelinks(jit_State *J, SnapEntry *map) 110static MSize snapshot_framelinks(jit_State *J, SnapEntry *map, uint8_t *topslot)
94{ 111{
95 cTValue *frame = J->L->base - 1; 112 cTValue *frame = J->L->base - 1;
96 cTValue *lim = J->L->base - J->baseslot; 113 cTValue *lim = J->L->base - J->baseslot + LJ_FR2;
97 cTValue *ftop = frame + funcproto(frame_func(frame))->framesize; 114 GCfunc *fn = frame_func(frame);
115 cTValue *ftop = isluafunc(fn) ? (frame+funcproto(fn)->framesize) : J->L->top;
116#if LJ_FR2
117 uint64_t pcbase = (u64ptr(J->pc) << 8) | (J->baseslot - 2);
118 lj_assertJ(2 <= J->baseslot && J->baseslot <= 257, "bad baseslot");
119 memcpy(map, &pcbase, sizeof(uint64_t));
120#else
98 MSize f = 0; 121 MSize f = 0;
99 map[f++] = SNAP_MKPC(J->pc); /* The current PC is always the first entry. */ 122 map[f++] = SNAP_MKPC(J->pc); /* The current PC is always the first entry. */
100 lua_assert(!J->pt || 123#endif
124 lj_assertJ(!J->pt ||
101 (J->pc >= proto_bc(J->pt) && 125 (J->pc >= proto_bc(J->pt) &&
102 J->pc < proto_bc(J->pt) + J->pt->sizebc)); 126 J->pc < proto_bc(J->pt) + J->pt->sizebc), "bad snapshot PC");
103 while (frame > lim) { /* Backwards traversal of all frames above base. */ 127 while (frame > lim) { /* Backwards traversal of all frames above base. */
104 if (frame_islua(frame)) { 128 if (frame_islua(frame)) {
129#if !LJ_FR2
105 map[f++] = SNAP_MKPC(frame_pc(frame)); 130 map[f++] = SNAP_MKPC(frame_pc(frame));
131#endif
106 frame = frame_prevl(frame); 132 frame = frame_prevl(frame);
107 } else if (frame_iscont(frame)) { 133 } else if (frame_iscont(frame)) {
134#if !LJ_FR2
108 map[f++] = SNAP_MKFTSZ(frame_ftsz(frame)); 135 map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
109 map[f++] = SNAP_MKPC(frame_contpc(frame)); 136 map[f++] = SNAP_MKPC(frame_contpc(frame));
137#endif
110 frame = frame_prevd(frame); 138 frame = frame_prevd(frame);
111 } else { 139 } else {
112 lua_assert(!frame_isc(frame)); 140 lj_assertJ(!frame_isc(frame), "broken frame chain");
141#if !LJ_FR2
113 map[f++] = SNAP_MKFTSZ(frame_ftsz(frame)); 142 map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
143#endif
114 frame = frame_prevd(frame); 144 frame = frame_prevd(frame);
115 continue; 145 continue;
116 } 146 }
117 if (frame + funcproto(frame_func(frame))->framesize > ftop) 147 if (frame + funcproto(frame_func(frame))->framesize > ftop)
118 ftop = frame + funcproto(frame_func(frame))->framesize; 148 ftop = frame + funcproto(frame_func(frame))->framesize;
119 } 149 }
120 lua_assert(f == (MSize)(1 + J->framedepth)); 150 *topslot = (uint8_t)(ftop - lim);
121 return (BCReg)(ftop - lim); 151#if LJ_FR2
152 lj_assertJ(sizeof(SnapEntry) * 2 == sizeof(uint64_t), "bad SnapEntry def");
153 return 2;
154#else
155 lj_assertJ(f == (MSize)(1 + J->framedepth), "miscalculated snapshot size");
156 return f;
157#endif
122} 158}
123 159
124/* Take a snapshot of the current stack. */ 160/* Take a snapshot of the current stack. */
@@ -128,16 +164,17 @@ static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
128 MSize nent; 164 MSize nent;
129 SnapEntry *p; 165 SnapEntry *p;
130 /* Conservative estimate. */ 166 /* Conservative estimate. */
131 lj_snap_grow_map(J, nsnapmap + nslots + (MSize)J->framedepth+1); 167 lj_snap_grow_map(J, nsnapmap + nslots + (MSize)(LJ_FR2?2:J->framedepth+1));
132 p = &J->cur.snapmap[nsnapmap]; 168 p = &J->cur.snapmap[nsnapmap];
133 nent = snapshot_slots(J, p, nslots); 169 nent = snapshot_slots(J, p, nslots);
134 snap->topslot = (uint8_t)snapshot_framelinks(J, p + nent); 170 snap->nent = (uint8_t)nent;
171 nent += snapshot_framelinks(J, p + nent, &snap->topslot);
135 snap->mapofs = (uint32_t)nsnapmap; 172 snap->mapofs = (uint32_t)nsnapmap;
136 snap->ref = (IRRef1)J->cur.nins; 173 snap->ref = (IRRef1)J->cur.nins;
137 snap->nent = (uint8_t)nent; 174 snap->mcofs = 0;
138 snap->nslots = (uint8_t)nslots; 175 snap->nslots = (uint8_t)nslots;
139 snap->count = 0; 176 snap->count = 0;
140 J->cur.nsnapmap = (uint32_t)(nsnapmap + nent + 1 + J->framedepth); 177 J->cur.nsnapmap = (uint32_t)(nsnapmap + nent);
141} 178}
142 179
143/* Add or merge a snapshot. */ 180/* Add or merge a snapshot. */
@@ -146,8 +183,8 @@ void lj_snap_add(jit_State *J)
146 MSize nsnap = J->cur.nsnap; 183 MSize nsnap = J->cur.nsnap;
147 MSize nsnapmap = J->cur.nsnapmap; 184 MSize nsnapmap = J->cur.nsnapmap;
148 /* Merge if no ins. inbetween or if requested and no guard inbetween. */ 185 /* Merge if no ins. inbetween or if requested and no guard inbetween. */
149 if (J->mergesnap ? !irt_isguard(J->guardemit) : 186 if ((nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins) ||
150 (nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) { 187 (J->mergesnap && !irt_isguard(J->guardemit))) {
151 if (nsnap == 1) { /* But preserve snap #0 PC. */ 188 if (nsnap == 1) { /* But preserve snap #0 PC. */
152 emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0); 189 emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
153 goto nomerge; 190 goto nomerge;
@@ -194,7 +231,8 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
194#define DEF_SLOT(s) udf[(s)] *= 3 231#define DEF_SLOT(s) udf[(s)] *= 3
195 232
196 /* Scan through following bytecode and check for uses/defs. */ 233 /* Scan through following bytecode and check for uses/defs. */
197 lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc); 234 lj_assertJ(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc,
235 "snapshot PC out of range");
198 for (;;) { 236 for (;;) {
199 BCIns ins = *pc++; 237 BCIns ins = *pc++;
200 BCOp op = bc_op(ins); 238 BCOp op = bc_op(ins);
@@ -205,7 +243,7 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
205 switch (bcmode_c(op)) { 243 switch (bcmode_c(op)) {
206 case BCMvar: USE_SLOT(bc_c(ins)); break; 244 case BCMvar: USE_SLOT(bc_c(ins)); break;
207 case BCMrbase: 245 case BCMrbase:
208 lua_assert(op == BC_CAT); 246 lj_assertJ(op == BC_CAT, "unhandled op %d with RC rbase", op);
209 for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s); 247 for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s);
210 for (; s < maxslot; s++) DEF_SLOT(s); 248 for (; s < maxslot; s++) DEF_SLOT(s);
211 break; 249 break;
@@ -245,7 +283,8 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
245 case BCMbase: 283 case BCMbase:
246 if (op >= BC_CALLM && op <= BC_ITERN) { 284 if (op >= BC_CALLM && op <= BC_ITERN) {
247 BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ? 285 BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ?
248 maxslot : (bc_a(ins) + bc_c(ins)); 286 maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2);
287 if (LJ_FR2) DEF_SLOT(bc_a(ins)+1);
249 s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0); 288 s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0);
250 for (; s < top; s++) USE_SLOT(s); 289 for (; s < top; s++) USE_SLOT(s);
251 for (; s < maxslot; s++) DEF_SLOT(s); 290 for (; s < maxslot; s++) DEF_SLOT(s);
@@ -263,7 +302,8 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
263 break; 302 break;
264 default: break; 303 default: break;
265 } 304 }
266 lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc); 305 lj_assertJ(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc,
306 "use/def analysis PC out of range");
267 } 307 }
268 308
269#undef USE_SLOT 309#undef USE_SLOT
@@ -321,8 +361,8 @@ void lj_snap_shrink(jit_State *J)
321 MSize n, m, nlim, nent = snap->nent; 361 MSize n, m, nlim, nent = snap->nent;
322 uint8_t udf[SNAP_USEDEF_SLOTS]; 362 uint8_t udf[SNAP_USEDEF_SLOTS];
323 BCReg maxslot = J->maxslot; 363 BCReg maxslot = J->maxslot;
324 BCReg minslot = snap_usedef(J, udf, snap_pc(map[nent]), maxslot);
325 BCReg baseslot = J->baseslot; 364 BCReg baseslot = J->baseslot;
365 BCReg minslot = snap_usedef(J, udf, snap_pc(&map[nent]), maxslot);
326 if (minslot < maxslot) snap_useuv(J->pt, udf); 366 if (minslot < maxslot) snap_useuv(J->pt, udf);
327 maxslot += baseslot; 367 maxslot += baseslot;
328 minslot += baseslot; 368 minslot += baseslot;
@@ -365,25 +405,26 @@ static RegSP snap_renameref(GCtrace *T, SnapNo lim, IRRef ref, RegSP rs)
365} 405}
366 406
367/* Copy RegSP from parent snapshot to the parent links of the IR. */ 407/* Copy RegSP from parent snapshot to the parent links of the IR. */
368IRIns *lj_snap_regspmap(GCtrace *T, SnapNo snapno, IRIns *ir) 408IRIns *lj_snap_regspmap(jit_State *J, GCtrace *T, SnapNo snapno, IRIns *ir)
369{ 409{
370 SnapShot *snap = &T->snap[snapno]; 410 SnapShot *snap = &T->snap[snapno];
371 SnapEntry *map = &T->snapmap[snap->mapofs]; 411 SnapEntry *map = &T->snapmap[snap->mapofs];
372 BloomFilter rfilt = snap_renamefilter(T, snapno); 412 BloomFilter rfilt = snap_renamefilter(T, snapno);
373 MSize n = 0; 413 MSize n = 0;
374 IRRef ref = 0; 414 IRRef ref = 0;
415 UNUSED(J);
375 for ( ; ; ir++) { 416 for ( ; ; ir++) {
376 uint32_t rs; 417 uint32_t rs;
377 if (ir->o == IR_SLOAD) { 418 if (ir->o == IR_SLOAD) {
378 if (!(ir->op2 & IRSLOAD_PARENT)) break; 419 if (!(ir->op2 & IRSLOAD_PARENT)) break;
379 for ( ; ; n++) { 420 for ( ; ; n++) {
380 lua_assert(n < snap->nent); 421 lj_assertJ(n < snap->nent, "slot %d not found in snapshot", ir->op1);
381 if (snap_slot(map[n]) == ir->op1) { 422 if (snap_slot(map[n]) == ir->op1) {
382 ref = snap_ref(map[n++]); 423 ref = snap_ref(map[n++]);
383 break; 424 break;
384 } 425 }
385 } 426 }
386 } else if (LJ_SOFTFP && ir->o == IR_HIOP) { 427 } else if (LJ_SOFTFP32 && ir->o == IR_HIOP) {
387 ref++; 428 ref++;
388 } else if (ir->o == IR_PVAL) { 429 } else if (ir->o == IR_PVAL) {
389 ref = ir->op1 + REF_BIAS; 430 ref = ir->op1 + REF_BIAS;
@@ -394,7 +435,7 @@ IRIns *lj_snap_regspmap(GCtrace *T, SnapNo snapno, IRIns *ir)
394 if (bloomtest(rfilt, ref)) 435 if (bloomtest(rfilt, ref))
395 rs = snap_renameref(T, snapno, ref, rs); 436 rs = snap_renameref(T, snapno, ref, rs);
396 ir->prev = (uint16_t)rs; 437 ir->prev = (uint16_t)rs;
397 lua_assert(regsp_used(rs)); 438 lj_assertJ(regsp_used(rs), "unused IR %04d in snapshot", ref - REF_BIAS);
398 } 439 }
399 return ir; 440 return ir;
400} 441}
@@ -409,10 +450,10 @@ static TRef snap_replay_const(jit_State *J, IRIns *ir)
409 case IR_KPRI: return TREF_PRI(irt_type(ir->t)); 450 case IR_KPRI: return TREF_PRI(irt_type(ir->t));
410 case IR_KINT: return lj_ir_kint(J, ir->i); 451 case IR_KINT: return lj_ir_kint(J, ir->i);
411 case IR_KGC: return lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); 452 case IR_KGC: return lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t));
412 case IR_KNUM: return lj_ir_k64(J, IR_KNUM, ir_knum(ir)); 453 case IR_KNUM: case IR_KINT64:
413 case IR_KINT64: return lj_ir_k64(J, IR_KINT64, ir_kint64(ir)); 454 return lj_ir_k64(J, (IROp)ir->o, ir_k64(ir)->u64);
414 case IR_KPTR: return lj_ir_kptr(J, ir_kptr(ir)); /* Continuation. */ 455 case IR_KPTR: return lj_ir_kptr(J, ir_kptr(ir)); /* Continuation. */
415 default: lua_assert(0); return TREF_NIL; break; 456 default: lj_assertJ(0, "bad IR constant op %d", ir->o); return TREF_NIL;
416 } 457 }
417} 458}
418 459
@@ -422,7 +463,7 @@ static TRef snap_dedup(jit_State *J, SnapEntry *map, MSize nmax, IRRef ref)
422 MSize j; 463 MSize j;
423 for (j = 0; j < nmax; j++) 464 for (j = 0; j < nmax; j++)
424 if (snap_ref(map[j]) == ref) 465 if (snap_ref(map[j]) == ref)
425 return J->slot[snap_slot(map[j])] & ~(SNAP_CONT|SNAP_FRAME); 466 return J->slot[snap_slot(map[j])] & ~(SNAP_KEYINDEX|SNAP_CONT|SNAP_FRAME);
426 return 0; 467 return 0;
427} 468}
428 469
@@ -483,21 +524,27 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
483 goto setslot; 524 goto setslot;
484 bloomset(seen, ref); 525 bloomset(seen, ref);
485 if (irref_isk(ref)) { 526 if (irref_isk(ref)) {
486 tr = snap_replay_const(J, ir); 527 /* See special treatment of LJ_FR2 slot 1 in snapshot_slots() above. */
528 if (LJ_FR2 && (sn == SNAP(1, SNAP_FRAME | SNAP_NORESTORE, REF_NIL)))
529 tr = 0;
530 else
531 tr = snap_replay_const(J, ir);
487 } else if (!regsp_used(ir->prev)) { 532 } else if (!regsp_used(ir->prev)) {
488 pass23 = 1; 533 pass23 = 1;
489 lua_assert(s != 0); 534 lj_assertJ(s != 0, "unused slot 0 in snapshot");
490 tr = s; 535 tr = s;
491 } else { 536 } else {
492 IRType t = irt_type(ir->t); 537 IRType t = irt_type(ir->t);
493 uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT; 538 uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
494 if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM; 539 if (LJ_SOFTFP32 && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
495 if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY); 540 if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
541 if ((sn & SNAP_KEYINDEX)) mode |= IRSLOAD_KEYINDEX;
496 tr = emitir_raw(IRT(IR_SLOAD, t), s, mode); 542 tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
497 } 543 }
498 setslot: 544 setslot:
499 J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */ 545 /* Same as TREF_* flags. */
500 J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s); 546 J->slot[s] = tr | (sn&(SNAP_KEYINDEX|SNAP_CONT|SNAP_FRAME));
547 J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && (s != LJ_FR2));
501 if ((sn & SNAP_FRAME)) 548 if ((sn & SNAP_FRAME))
502 J->baseslot = s+1; 549 J->baseslot = s+1;
503 } 550 }
@@ -512,8 +559,9 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
512 if (regsp_reg(ir->r) == RID_SUNK) { 559 if (regsp_reg(ir->r) == RID_SUNK) {
513 if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue; 560 if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
514 pass23 = 1; 561 pass23 = 1;
515 lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || 562 lj_assertJ(ir->o == IR_TNEW || ir->o == IR_TDUP ||
516 ir->o == IR_CNEW || ir->o == IR_CNEWI); 563 ir->o == IR_CNEW || ir->o == IR_CNEWI,
564 "sunk parent IR %04d has bad op %d", refp - REF_BIAS, ir->o);
517 if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1); 565 if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
518 if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2); 566 if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
519 if (LJ_HASFFI && ir->o == IR_CNEWI) { 567 if (LJ_HASFFI && ir->o == IR_CNEWI) {
@@ -525,13 +573,14 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
525 if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { 573 if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
526 if (snap_pref(J, T, map, nent, seen, irs->op2) == 0) 574 if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
527 snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1); 575 snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
528 else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) && 576 else if ((LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)) &&
529 irs+1 < irlast && (irs+1)->o == IR_HIOP) 577 irs+1 < irlast && (irs+1)->o == IR_HIOP)
530 snap_pref(J, T, map, nent, seen, (irs+1)->op2); 578 snap_pref(J, T, map, nent, seen, (irs+1)->op2);
531 } 579 }
532 } 580 }
533 } else if (!irref_isk(refp) && !regsp_used(ir->prev)) { 581 } else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
534 lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT); 582 lj_assertJ(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
583 "sunk parent IR %04d has bad op %d", refp - REF_BIAS, ir->o);
535 J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1); 584 J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
536 } 585 }
537 } 586 }
@@ -581,20 +630,21 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
581 val = snap_pref(J, T, map, nent, seen, irs->op2); 630 val = snap_pref(J, T, map, nent, seen, irs->op2);
582 if (val == 0) { 631 if (val == 0) {
583 IRIns *irc = &T->ir[irs->op2]; 632 IRIns *irc = &T->ir[irs->op2];
584 lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT); 633 lj_assertJ(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT,
634 "sunk store for parent IR %04d with bad op %d",
635 refp - REF_BIAS, irc->o);
585 val = snap_pref(J, T, map, nent, seen, irc->op1); 636 val = snap_pref(J, T, map, nent, seen, irc->op1);
586 val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT); 637 val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
587 } else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) && 638 } else if ((LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)) &&
588 irs+1 < irlast && (irs+1)->o == IR_HIOP) { 639 irs+1 < irlast && (irs+1)->o == IR_HIOP) {
589 IRType t = IRT_I64; 640 IRType t = IRT_I64;
590 if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP) 641 if (LJ_SOFTFP32 && irt_type((irs+1)->t) == IRT_SOFTFP)
591 t = IRT_NUM; 642 t = IRT_NUM;
592 lj_needsplit(J); 643 lj_needsplit(J);
593 if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) { 644 if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
594 uint64_t k = (uint32_t)T->ir[irs->op2].i + 645 uint64_t k = (uint32_t)T->ir[irs->op2].i +
595 ((uint64_t)T->ir[(irs+1)->op2].i << 32); 646 ((uint64_t)T->ir[(irs+1)->op2].i << 32);
596 val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM, 647 val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM, k);
597 lj_ir_k64_find(J, k));
598 } else { 648 } else {
599 val = emitir_raw(IRT(IR_HIOP, t), val, 649 val = emitir_raw(IRT(IR_HIOP, t), val,
600 snap_pref(J, T, map, nent, seen, (irs+1)->op2)); 650 snap_pref(J, T, map, nent, seen, (irs+1)->op2));
@@ -632,7 +682,14 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
632 IRType1 t = ir->t; 682 IRType1 t = ir->t;
633 RegSP rs = ir->prev; 683 RegSP rs = ir->prev;
634 if (irref_isk(ref)) { /* Restore constant slot. */ 684 if (irref_isk(ref)) { /* Restore constant slot. */
635 lj_ir_kvalue(J->L, o, ir); 685 if (ir->o == IR_KPTR) {
686 o->u64 = (uint64_t)(uintptr_t)ir_kptr(ir);
687 } else {
688 lj_assertJ(!(ir->o == IR_KKPTR || ir->o == IR_KNULL),
689 "restore of const from IR %04d with bad op %d",
690 ref - REF_BIAS, ir->o);
691 lj_ir_kvalue(J->L, o, ir);
692 }
636 return; 693 return;
637 } 694 }
638 if (LJ_UNLIKELY(bloomtest(rfilt, ref))) 695 if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
@@ -641,22 +698,24 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
641 int32_t *sps = &ex->spill[regsp_spill(rs)]; 698 int32_t *sps = &ex->spill[regsp_spill(rs)];
642 if (irt_isinteger(t)) { 699 if (irt_isinteger(t)) {
643 setintV(o, *sps); 700 setintV(o, *sps);
644#if !LJ_SOFTFP 701#if !LJ_SOFTFP32
645 } else if (irt_isnum(t)) { 702 } else if (irt_isnum(t)) {
646 o->u64 = *(uint64_t *)sps; 703 o->u64 = *(uint64_t *)sps;
647#endif 704#endif
648 } else if (LJ_64 && irt_islightud(t)) { 705#if LJ_64 && !LJ_GC64
706 } else if (irt_islightud(t)) {
649 /* 64 bit lightuserdata which may escape already has the tag bits. */ 707 /* 64 bit lightuserdata which may escape already has the tag bits. */
650 o->u64 = *(uint64_t *)sps; 708 o->u64 = *(uint64_t *)sps;
709#endif
651 } else { 710 } else {
652 lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ 711 lj_assertJ(!irt_ispri(t), "PRI ref with spill slot");
653 setgcrefi(o->gcr, *sps); 712 setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
654 setitype(o, irt_toitype(t));
655 } 713 }
656 } else { /* Restore from register. */ 714 } else { /* Restore from register. */
657 Reg r = regsp_reg(rs); 715 Reg r = regsp_reg(rs);
658 if (ra_noreg(r)) { 716 if (ra_noreg(r)) {
659 lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT); 717 lj_assertJ(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
718 "restore from IR %04d has no reg", ref - REF_BIAS);
660 snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o); 719 snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
661 if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o)); 720 if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
662 return; 721 return;
@@ -665,21 +724,26 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
665#if !LJ_SOFTFP 724#if !LJ_SOFTFP
666 } else if (irt_isnum(t)) { 725 } else if (irt_isnum(t)) {
667 setnumV(o, ex->fpr[r-RID_MIN_FPR]); 726 setnumV(o, ex->fpr[r-RID_MIN_FPR]);
727#elif LJ_64 /* && LJ_SOFTFP */
728 } else if (irt_isnum(t)) {
729 o->u64 = ex->gpr[r-RID_MIN_GPR];
668#endif 730#endif
669 } else if (LJ_64 && irt_islightud(t)) { 731#if LJ_64 && !LJ_GC64
670 /* 64 bit lightuserdata which may escape already has the tag bits. */ 732 } else if (irt_is64(t)) {
733 /* 64 bit values that already have the tag bits. */
671 o->u64 = ex->gpr[r-RID_MIN_GPR]; 734 o->u64 = ex->gpr[r-RID_MIN_GPR];
735#endif
736 } else if (irt_ispri(t)) {
737 setpriV(o, irt_toitype(t));
672 } else { 738 } else {
673 if (!irt_ispri(t)) 739 setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
674 setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
675 setitype(o, irt_toitype(t));
676 } 740 }
677 } 741 }
678} 742}
679 743
680#if LJ_HASFFI 744#if LJ_HASFFI
681/* Restore raw data from the trace exit state. */ 745/* Restore raw data from the trace exit state. */
682static void snap_restoredata(GCtrace *T, ExitState *ex, 746static void snap_restoredata(jit_State *J, GCtrace *T, ExitState *ex,
683 SnapNo snapno, BloomFilter rfilt, 747 SnapNo snapno, BloomFilter rfilt,
684 IRRef ref, void *dst, CTSize sz) 748 IRRef ref, void *dst, CTSize sz)
685{ 749{
@@ -687,9 +751,10 @@ static void snap_restoredata(GCtrace *T, ExitState *ex,
687 RegSP rs = ir->prev; 751 RegSP rs = ir->prev;
688 int32_t *src; 752 int32_t *src;
689 uint64_t tmp; 753 uint64_t tmp;
754 UNUSED(J);
690 if (irref_isk(ref)) { 755 if (irref_isk(ref)) {
691 if (ir->o == IR_KNUM || ir->o == IR_KINT64) { 756 if (ir_isk64(ir)) {
692 src = mref(ir->ptr, int32_t); 757 src = (int32_t *)&ir[1];
693 } else if (sz == 8) { 758 } else if (sz == 8) {
694 tmp = (uint64_t)(uint32_t)ir->i; 759 tmp = (uint64_t)(uint32_t)ir->i;
695 src = (int32_t *)&tmp; 760 src = (int32_t *)&tmp;
@@ -709,8 +774,9 @@ static void snap_restoredata(GCtrace *T, ExitState *ex,
709 Reg r = regsp_reg(rs); 774 Reg r = regsp_reg(rs);
710 if (ra_noreg(r)) { 775 if (ra_noreg(r)) {
711 /* Note: this assumes CNEWI is never used for SOFTFP split numbers. */ 776 /* Note: this assumes CNEWI is never used for SOFTFP split numbers. */
712 lua_assert(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT); 777 lj_assertJ(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
713 snap_restoredata(T, ex, snapno, rfilt, ir->op1, dst, 4); 778 "restore from IR %04d has no reg", ref - REF_BIAS);
779 snap_restoredata(J, T, ex, snapno, rfilt, ir->op1, dst, 4);
714 *(lua_Number *)dst = (lua_Number)*(int32_t *)dst; 780 *(lua_Number *)dst = (lua_Number)*(int32_t *)dst;
715 return; 781 return;
716 } 782 }
@@ -726,11 +792,13 @@ static void snap_restoredata(GCtrace *T, ExitState *ex,
726#else 792#else
727 if (LJ_BE && sz == 4) src++; 793 if (LJ_BE && sz == 4) src++;
728#endif 794#endif
729 } 795 } else
730#endif 796#endif
797 if (LJ_64 && LJ_BE && sz == 4) src++;
731 } 798 }
732 } 799 }
733 lua_assert(sz == 1 || sz == 2 || sz == 4 || sz == 8); 800 lj_assertJ(sz == 1 || sz == 2 || sz == 4 || sz == 8,
801 "restore from IR %04d with bad size %d", ref - REF_BIAS, sz);
734 if (sz == 4) *(int32_t *)dst = *src; 802 if (sz == 4) *(int32_t *)dst = *src;
735 else if (sz == 8) *(int64_t *)dst = *(int64_t *)src; 803 else if (sz == 8) *(int64_t *)dst = *(int64_t *)src;
736 else if (sz == 1) *(int8_t *)dst = (int8_t)*src; 804 else if (sz == 1) *(int8_t *)dst = (int8_t)*src;
@@ -743,24 +811,27 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
743 SnapNo snapno, BloomFilter rfilt, 811 SnapNo snapno, BloomFilter rfilt,
744 IRIns *ir, TValue *o) 812 IRIns *ir, TValue *o)
745{ 813{
746 lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || 814 lj_assertJ(ir->o == IR_TNEW || ir->o == IR_TDUP ||
747 ir->o == IR_CNEW || ir->o == IR_CNEWI); 815 ir->o == IR_CNEW || ir->o == IR_CNEWI,
816 "sunk allocation with bad op %d", ir->o);
748#if LJ_HASFFI 817#if LJ_HASFFI
749 if (ir->o == IR_CNEW || ir->o == IR_CNEWI) { 818 if (ir->o == IR_CNEW || ir->o == IR_CNEWI) {
750 CTState *cts = ctype_cts(J->L); 819 CTState *cts = ctype_cts(J->L);
751 CTypeID id = (CTypeID)T->ir[ir->op1].i; 820 CTypeID id = (CTypeID)T->ir[ir->op1].i;
752 CTSize sz = lj_ctype_size(cts, id); 821 CTSize sz;
753 GCcdata *cd = lj_cdata_new(cts, id, sz); 822 CTInfo info = lj_ctype_info(cts, id, &sz);
823 GCcdata *cd = lj_cdata_newx(cts, id, sz, info);
754 setcdataV(J->L, o, cd); 824 setcdataV(J->L, o, cd);
755 if (ir->o == IR_CNEWI) { 825 if (ir->o == IR_CNEWI) {
756 uint8_t *p = (uint8_t *)cdataptr(cd); 826 uint8_t *p = (uint8_t *)cdataptr(cd);
757 lua_assert(sz == 4 || sz == 8); 827 lj_assertJ(sz == 4 || sz == 8, "sunk cdata with bad size %d", sz);
758 if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) { 828 if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) {
759 snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4); 829 snap_restoredata(J, T, ex, snapno, rfilt, (ir+1)->op2,
830 LJ_LE ? p+4 : p, 4);
760 if (LJ_BE) p += 4; 831 if (LJ_BE) p += 4;
761 sz = 4; 832 sz = 4;
762 } 833 }
763 snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz); 834 snap_restoredata(J, T, ex, snapno, rfilt, ir->op2, p, sz);
764 } else { 835 } else {
765 IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref]; 836 IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
766 for (irs = ir+1; irs < irlast; irs++) 837 for (irs = ir+1; irs < irlast; irs++)
@@ -768,8 +839,11 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
768 IRIns *iro = &T->ir[T->ir[irs->op1].op2]; 839 IRIns *iro = &T->ir[T->ir[irs->op1].op2];
769 uint8_t *p = (uint8_t *)cd; 840 uint8_t *p = (uint8_t *)cd;
770 CTSize szs; 841 CTSize szs;
771 lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD); 842 lj_assertJ(irs->o == IR_XSTORE, "sunk store with bad op %d", irs->o);
772 lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64); 843 lj_assertJ(T->ir[irs->op1].o == IR_ADD,
844 "sunk store with bad add op %d", T->ir[irs->op1].o);
845 lj_assertJ(iro->o == IR_KINT || iro->o == IR_KINT64,
846 "sunk store with bad const offset op %d", iro->o);
773 if (irt_is64(irs->t)) szs = 8; 847 if (irt_is64(irs->t)) szs = 8;
774 else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1; 848 else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1;
775 else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2; 849 else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2;
@@ -778,14 +852,16 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
778 p += (int64_t)ir_k64(iro)->u64; 852 p += (int64_t)ir_k64(iro)->u64;
779 else 853 else
780 p += iro->i; 854 p += iro->i;
781 lua_assert(p >= (uint8_t *)cdataptr(cd) && 855 lj_assertJ(p >= (uint8_t *)cdataptr(cd) &&
782 p + szs <= (uint8_t *)cdataptr(cd) + sz); 856 p + szs <= (uint8_t *)cdataptr(cd) + sz,
857 "sunk store with offset out of range");
783 if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { 858 if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
784 lua_assert(szs == 4); 859 lj_assertJ(szs == 4, "sunk store with bad size %d", szs);
785 snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4); 860 snap_restoredata(J, T, ex, snapno, rfilt, (irs+1)->op2,
861 LJ_LE ? p+4 : p, 4);
786 if (LJ_BE) p += 4; 862 if (LJ_BE) p += 4;
787 } 863 }
788 snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs); 864 snap_restoredata(J, T, ex, snapno, rfilt, irs->op2, p, szs);
789 } 865 }
790 } 866 }
791 } else 867 } else
@@ -800,10 +876,12 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
800 if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { 876 if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
801 IRIns *irk = &T->ir[irs->op1]; 877 IRIns *irk = &T->ir[irs->op1];
802 TValue tmp, *val; 878 TValue tmp, *val;
803 lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE || 879 lj_assertJ(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
804 irs->o == IR_FSTORE); 880 irs->o == IR_FSTORE,
881 "sunk store with bad op %d", irs->o);
805 if (irk->o == IR_FREF) { 882 if (irk->o == IR_FREF) {
806 lua_assert(irk->op2 == IRFL_TAB_META); 883 lj_assertJ(irk->op2 == IRFL_TAB_META,
884 "sunk store with bad field %d", irk->op2);
807 snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp); 885 snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp);
808 /* NOBARRIER: The table is new (marked white). */ 886 /* NOBARRIER: The table is new (marked white). */
809 setgcref(t->metatable, obj2gco(tabV(&tmp))); 887 setgcref(t->metatable, obj2gco(tabV(&tmp)));
@@ -814,7 +892,7 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
814 val = lj_tab_set(J->L, t, &tmp); 892 val = lj_tab_set(J->L, t, &tmp);
815 /* NOBARRIER: The table is new (marked white). */ 893 /* NOBARRIER: The table is new (marked white). */
816 snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val); 894 snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val);
817 if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { 895 if (LJ_SOFTFP32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
818 snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp); 896 snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp);
819 val->u32.hi = tmp.u32.lo; 897 val->u32.hi = tmp.u32.lo;
820 } 898 }
@@ -832,11 +910,15 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr)
832 SnapShot *snap = &T->snap[snapno]; 910 SnapShot *snap = &T->snap[snapno];
833 MSize n, nent = snap->nent; 911 MSize n, nent = snap->nent;
834 SnapEntry *map = &T->snapmap[snap->mapofs]; 912 SnapEntry *map = &T->snapmap[snap->mapofs];
835 SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1]; 913#if !LJ_FR2 || defined(LUA_USE_ASSERT)
836 int32_t ftsz0; 914 SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1-LJ_FR2];
915#endif
916#if !LJ_FR2
917 ptrdiff_t ftsz0;
918#endif
837 TValue *frame; 919 TValue *frame;
838 BloomFilter rfilt = snap_renamefilter(T, snapno); 920 BloomFilter rfilt = snap_renamefilter(T, snapno);
839 const BCIns *pc = snap_pc(map[nent]); 921 const BCIns *pc = snap_pc(&map[nent]);
840 lua_State *L = J->L; 922 lua_State *L = J->L;
841 923
842 /* Set interpreter PC to the next PC to get correct error messages. */ 924 /* Set interpreter PC to the next PC to get correct error messages. */
@@ -849,8 +931,10 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr)
849 } 931 }
850 932
851 /* Fill stack slots with data from the registers and spill slots. */ 933 /* Fill stack slots with data from the registers and spill slots. */
852 frame = L->base-1; 934 frame = L->base-1-LJ_FR2;
935#if !LJ_FR2
853 ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ 936 ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
937#endif
854 for (n = 0; n < nent; n++) { 938 for (n = 0; n < nent; n++) {
855 SnapEntry sn = map[n]; 939 SnapEntry sn = map[n];
856 if (!(sn & SNAP_NORESTORE)) { 940 if (!(sn & SNAP_NORESTORE)) {
@@ -869,18 +953,27 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr)
869 continue; 953 continue;
870 } 954 }
871 snap_restoreval(J, T, ex, snapno, rfilt, ref, o); 955 snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
872 if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) { 956 if (LJ_SOFTFP32 && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
873 TValue tmp; 957 TValue tmp;
874 snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp); 958 snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
875 o->u32.hi = tmp.u32.lo; 959 o->u32.hi = tmp.u32.lo;
960#if !LJ_FR2
876 } else if ((sn & (SNAP_CONT|SNAP_FRAME))) { 961 } else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
877 /* Overwrite tag with frame link. */ 962 /* Overwrite tag with frame link. */
878 o->fr.tp.ftsz = snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0; 963 setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
879 L->base = o+1; 964 L->base = o+1;
965#endif
966 } else if ((sn & SNAP_KEYINDEX)) {
967 /* A IRT_INT key index slot is restored as a number. Undo this. */
968 o->u32.lo = (uint32_t)(LJ_DUALNUM ? intV(o) : lj_num2int(numV(o)));
969 o->u32.hi = LJ_KEYINDEX;
880 } 970 }
881 } 971 }
882 } 972 }
883 lua_assert(map + nent == flinks); 973#if LJ_FR2
974 L->base += (map[nent+LJ_BE] & 0xff);
975#endif
976 lj_assertJ(map + nent == flinks, "inconsistent frames in snapshot");
884 977
885 /* Compute current stack top. */ 978 /* Compute current stack top. */
886 switch (bc_op(*pc)) { 979 switch (bc_op(*pc)) {
diff --git a/src/lj_snap.h b/src/lj_snap.h
index 6c406f20..089d86bb 100644
--- a/src/lj_snap.h
+++ b/src/lj_snap.h
@@ -13,7 +13,8 @@
13LJ_FUNC void lj_snap_add(jit_State *J); 13LJ_FUNC void lj_snap_add(jit_State *J);
14LJ_FUNC void lj_snap_purge(jit_State *J); 14LJ_FUNC void lj_snap_purge(jit_State *J);
15LJ_FUNC void lj_snap_shrink(jit_State *J); 15LJ_FUNC void lj_snap_shrink(jit_State *J);
16LJ_FUNC IRIns *lj_snap_regspmap(GCtrace *T, SnapNo snapno, IRIns *ir); 16LJ_FUNC IRIns *lj_snap_regspmap(jit_State *J, GCtrace *T, SnapNo snapno,
17 IRIns *ir);
17LJ_FUNC void lj_snap_replay(jit_State *J, GCtrace *T); 18LJ_FUNC void lj_snap_replay(jit_State *J, GCtrace *T);
18LJ_FUNC const BCIns *lj_snap_restore(jit_State *J, void *exptr); 19LJ_FUNC const BCIns *lj_snap_restore(jit_State *J, void *exptr);
19LJ_FUNC void lj_snap_grow_buf_(jit_State *J, MSize need); 20LJ_FUNC void lj_snap_grow_buf_(jit_State *J, MSize need);
diff --git a/src/lj_state.c b/src/lj_state.c
index d7befaff..b45a2043 100644
--- a/src/lj_state.c
+++ b/src/lj_state.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#include "lj_tab.h" 17#include "lj_tab.h"
17#include "lj_func.h" 18#include "lj_func.h"
@@ -24,8 +25,10 @@
24#include "lj_trace.h" 25#include "lj_trace.h"
25#include "lj_dispatch.h" 26#include "lj_dispatch.h"
26#include "lj_vm.h" 27#include "lj_vm.h"
28#include "lj_prng.h"
27#include "lj_lex.h" 29#include "lj_lex.h"
28#include "lj_alloc.h" 30#include "lj_alloc.h"
31#include "luajit.h"
29 32
30/* -- Stack handling ------------------------------------------------------ */ 33/* -- Stack handling ------------------------------------------------------ */
31 34
@@ -47,6 +50,7 @@
47** one extra slot if mobj is not a function. Only lj_meta_tset needs 5 50** one extra slot if mobj is not a function. Only lj_meta_tset needs 5
48** slots above top, but then mobj is always a function. So we can get by 51** slots above top, but then mobj is always a function. So we can get by
49** with 5 extra slots. 52** with 5 extra slots.
53** LJ_FR2: We need 2 more slots for the frame PC and the continuation PC.
50*/ 54*/
51 55
52/* Resize stack slots and adjust pointers in state. */ 56/* Resize stack slots and adjust pointers in state. */
@@ -57,9 +61,10 @@ static void resizestack(lua_State *L, MSize n)
57 MSize oldsize = L->stacksize; 61 MSize oldsize = L->stacksize;
58 MSize realsize = n + 1 + LJ_STACK_EXTRA; 62 MSize realsize = n + 1 + LJ_STACK_EXTRA;
59 GCobj *up; 63 GCobj *up;
60 lua_assert((MSize)(tvref(L->maxstack)-oldst)==L->stacksize-LJ_STACK_EXTRA-1); 64 lj_assertL((MSize)(tvref(L->maxstack)-oldst) == L->stacksize-LJ_STACK_EXTRA-1,
65 "inconsistent stack size");
61 st = (TValue *)lj_mem_realloc(L, tvref(L->stack), 66 st = (TValue *)lj_mem_realloc(L, tvref(L->stack),
62 (MSize)(L->stacksize*sizeof(TValue)), 67 (MSize)(oldsize*sizeof(TValue)),
63 (MSize)(realsize*sizeof(TValue))); 68 (MSize)(realsize*sizeof(TValue)));
64 setmref(L->stack, st); 69 setmref(L->stack, st);
65 delta = (char *)st - (char *)oldst; 70 delta = (char *)st - (char *)oldst;
@@ -67,12 +72,12 @@ static void resizestack(lua_State *L, MSize n)
67 while (oldsize < realsize) /* Clear new slots. */ 72 while (oldsize < realsize) /* Clear new slots. */
68 setnilV(st + oldsize++); 73 setnilV(st + oldsize++);
69 L->stacksize = realsize; 74 L->stacksize = realsize;
75 if ((size_t)(mref(G(L)->jit_base, char) - (char *)oldst) < oldsize)
76 setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta);
70 L->base = (TValue *)((char *)L->base + delta); 77 L->base = (TValue *)((char *)L->base + delta);
71 L->top = (TValue *)((char *)L->top + delta); 78 L->top = (TValue *)((char *)L->top + delta);
72 for (up = gcref(L->openupval); up != NULL; up = gcnext(up)) 79 for (up = gcref(L->openupval); up != NULL; up = gcnext(up))
73 setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta)); 80 setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta));
74 if (obj2gco(L) == gcref(G(L)->jit_L))
75 setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta);
76} 81}
77 82
78/* Relimit stack after error, in case the limit was overdrawn. */ 83/* Relimit stack after error, in case the limit was overdrawn. */
@@ -89,7 +94,8 @@ void lj_state_shrinkstack(lua_State *L, MSize used)
89 return; /* Avoid stack shrinking while handling stack overflow. */ 94 return; /* Avoid stack shrinking while handling stack overflow. */
90 if (4*used < L->stacksize && 95 if (4*used < L->stacksize &&
91 2*(LJ_STACK_START+LJ_STACK_EXTRA) < L->stacksize && 96 2*(LJ_STACK_START+LJ_STACK_EXTRA) < L->stacksize &&
92 obj2gco(L) != gcref(G(L)->jit_L)) /* Don't shrink stack of live trace. */ 97 /* Don't shrink stack of live trace. */
98 (tvref(G(L)->jit_base) == NULL || obj2gco(L) != gcref(G(L)->cur_L)))
93 resizestack(L, L->stacksize >> 1); 99 resizestack(L, L->stacksize >> 1);
94} 100}
95 101
@@ -125,8 +131,9 @@ static void stack_init(lua_State *L1, lua_State *L)
125 L1->stacksize = LJ_STACK_START + LJ_STACK_EXTRA; 131 L1->stacksize = LJ_STACK_START + LJ_STACK_EXTRA;
126 stend = st + L1->stacksize; 132 stend = st + L1->stacksize;
127 setmref(L1->maxstack, stend - LJ_STACK_EXTRA - 1); 133 setmref(L1->maxstack, stend - LJ_STACK_EXTRA - 1);
128 L1->base = L1->top = st+1; 134 setthreadV(L1, st++, L1); /* Needed for curr_funcisL() on empty stack. */
129 setthreadV(L1, st, L1); /* Needed for curr_funcisL() on empty stack. */ 135 if (LJ_FR2) setnilV(st++);
136 L1->base = L1->top = st;
130 while (st < stend) /* Clear new slots. */ 137 while (st < stend) /* Clear new slots. */
131 setnilV(st++); 138 setnilV(st++);
132} 139}
@@ -143,12 +150,13 @@ static TValue *cpluaopen(lua_State *L, lua_CFunction dummy, void *ud)
143 /* NOBARRIER: State initialization, all objects are white. */ 150 /* NOBARRIER: State initialization, all objects are white. */
144 setgcref(L->env, obj2gco(lj_tab_new(L, 0, LJ_MIN_GLOBAL))); 151 setgcref(L->env, obj2gco(lj_tab_new(L, 0, LJ_MIN_GLOBAL)));
145 settabV(L, registry(L), lj_tab_new(L, 0, LJ_MIN_REGISTRY)); 152 settabV(L, registry(L), lj_tab_new(L, 0, LJ_MIN_REGISTRY));
146 lj_str_resize(L, LJ_MIN_STRTAB-1); 153 lj_str_init(L);
147 lj_meta_init(L); 154 lj_meta_init(L);
148 lj_lex_init(L); 155 lj_lex_init(L);
149 fixstring(lj_err_str(L, LJ_ERR_ERRMEM)); /* Preallocate memory error msg. */ 156 fixstring(lj_err_str(L, LJ_ERR_ERRMEM)); /* Preallocate memory error msg. */
150 g->gc.threshold = 4*g->gc.total; 157 g->gc.threshold = 4*g->gc.total;
151 lj_trace_initstate(g); 158 lj_trace_initstate(g);
159 lj_err_verify();
152 return NULL; 160 return NULL;
153} 161}
154 162
@@ -157,16 +165,25 @@ static void close_state(lua_State *L)
157 global_State *g = G(L); 165 global_State *g = G(L);
158 lj_func_closeuv(L, tvref(L->stack)); 166 lj_func_closeuv(L, tvref(L->stack));
159 lj_gc_freeall(g); 167 lj_gc_freeall(g);
160 lua_assert(gcref(g->gc.root) == obj2gco(L)); 168 lj_assertG(gcref(g->gc.root) == obj2gco(L),
161 lua_assert(g->strnum == 0); 169 "main thread is not first GC object");
170 lj_assertG(g->str.num == 0, "leaked %d strings", g->str.num);
162 lj_trace_freestate(g); 171 lj_trace_freestate(g);
163#if LJ_HASFFI 172#if LJ_HASFFI
164 lj_ctype_freestate(g); 173 lj_ctype_freestate(g);
165#endif 174#endif
166 lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef); 175 lj_str_freetab(g);
167 lj_str_freebuf(g, &g->tmpbuf); 176 lj_buf_free(g, &g->tmpbuf);
168 lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue); 177 lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue);
169 lua_assert(g->gc.total == sizeof(GG_State)); 178#if LJ_64
179 if (mref(g->gc.lightudseg, uint32_t)) {
180 MSize segnum = g->gc.lightudnum ? (2 << lj_fls(g->gc.lightudnum)) : 2;
181 lj_mem_freevec(g, mref(g->gc.lightudseg, uint32_t), segnum, uint32_t);
182 }
183#endif
184 lj_assertG(g->gc.total == sizeof(GG_State),
185 "memory leak of %lld bytes",
186 (long long)(g->gc.total - sizeof(GG_State)));
170#ifndef LUAJIT_USE_SYSMALLOC 187#ifndef LUAJIT_USE_SYSMALLOC
171 if (g->allocf == lj_alloc_f) 188 if (g->allocf == lj_alloc_f)
172 lj_alloc_destroy(g->allocd); 189 lj_alloc_destroy(g->allocd);
@@ -175,17 +192,34 @@ static void close_state(lua_State *L)
175 g->allocf(g->allocd, G2GG(g), sizeof(GG_State), 0); 192 g->allocf(g->allocd, G2GG(g), sizeof(GG_State), 0);
176} 193}
177 194
178#if LJ_64 && !(defined(LUAJIT_USE_VALGRIND) && defined(LUAJIT_USE_SYSMALLOC)) 195#if LJ_64 && !LJ_GC64 && !(defined(LUAJIT_USE_VALGRIND) && defined(LUAJIT_USE_SYSMALLOC))
179lua_State *lj_state_newstate(lua_Alloc f, void *ud) 196lua_State *lj_state_newstate(lua_Alloc allocf, void *allocd)
180#else 197#else
181LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) 198LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd)
182#endif 199#endif
183{ 200{
184 GG_State *GG = (GG_State *)f(ud, NULL, 0, sizeof(GG_State)); 201 PRNGState prng;
185 lua_State *L = &GG->L; 202 GG_State *GG;
186 global_State *g = &GG->g; 203 lua_State *L;
187 if (GG == NULL || !checkptr32(GG)) return NULL; 204 global_State *g;
205 /* We need the PRNG for the memory allocator, so initialize this first. */
206 if (!lj_prng_seed_secure(&prng)) {
207 lj_assertX(0, "secure PRNG seeding failed");
208 /* Can only return NULL here, so this errors with "not enough memory". */
209 return NULL;
210 }
211#ifndef LUAJIT_USE_SYSMALLOC
212 if (allocf == LJ_ALLOCF_INTERNAL) {
213 allocd = lj_alloc_create(&prng);
214 if (!allocd) return NULL;
215 allocf = lj_alloc_f;
216 }
217#endif
218 GG = (GG_State *)allocf(allocd, NULL, 0, sizeof(GG_State));
219 if (GG == NULL || !checkptrGC(GG)) return NULL;
188 memset(GG, 0, sizeof(GG_State)); 220 memset(GG, 0, sizeof(GG_State));
221 L = &GG->L;
222 g = &GG->g;
189 L->gct = ~LJ_TTHREAD; 223 L->gct = ~LJ_TTHREAD;
190 L->marked = LJ_GC_WHITE0 | LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */ 224 L->marked = LJ_GC_WHITE0 | LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */
191 L->dummy_ffid = FF_C; 225 L->dummy_ffid = FF_C;
@@ -193,17 +227,25 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
193 g->gc.currentwhite = LJ_GC_WHITE0 | LJ_GC_FIXED; 227 g->gc.currentwhite = LJ_GC_WHITE0 | LJ_GC_FIXED;
194 g->strempty.marked = LJ_GC_WHITE0; 228 g->strempty.marked = LJ_GC_WHITE0;
195 g->strempty.gct = ~LJ_TSTR; 229 g->strempty.gct = ~LJ_TSTR;
196 g->allocf = f; 230 g->allocf = allocf;
197 g->allocd = ud; 231 g->allocd = allocd;
232 g->prng = prng;
233#ifndef LUAJIT_USE_SYSMALLOC
234 if (allocf == lj_alloc_f) {
235 lj_alloc_setprng(allocd, &g->prng);
236 }
237#endif
198 setgcref(g->mainthref, obj2gco(L)); 238 setgcref(g->mainthref, obj2gco(L));
199 setgcref(g->uvhead.prev, obj2gco(&g->uvhead)); 239 setgcref(g->uvhead.prev, obj2gco(&g->uvhead));
200 setgcref(g->uvhead.next, obj2gco(&g->uvhead)); 240 setgcref(g->uvhead.next, obj2gco(&g->uvhead));
201 g->strmask = ~(MSize)0; 241 g->str.mask = ~(MSize)0;
202 setnilV(registry(L)); 242 setnilV(registry(L));
203 setnilV(&g->nilnode.val); 243 setnilV(&g->nilnode.val);
204 setnilV(&g->nilnode.key); 244 setnilV(&g->nilnode.key);
245#if !LJ_GC64
205 setmref(g->nilnode.freetop, &g->nilnode); 246 setmref(g->nilnode.freetop, &g->nilnode);
206 lj_str_initbuf(&g->tmpbuf); 247#endif
248 lj_buf_init(NULL, &g->tmpbuf);
207 g->gc.state = GCSpause; 249 g->gc.state = GCSpause;
208 setgcref(g->gc.root, obj2gco(L)); 250 setgcref(g->gc.root, obj2gco(L));
209 setmref(g->gc.sweep, &g->gc.root); 251 setmref(g->gc.sweep, &g->gc.root);
@@ -217,7 +259,7 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
217 close_state(L); 259 close_state(L);
218 return NULL; 260 return NULL;
219 } 261 }
220 L->status = 0; 262 L->status = LUA_OK;
221 return L; 263 return L;
222} 264}
223 265
@@ -236,6 +278,10 @@ LUA_API void lua_close(lua_State *L)
236 global_State *g = G(L); 278 global_State *g = G(L);
237 int i; 279 int i;
238 L = mainthread(g); /* Only the main thread can be closed. */ 280 L = mainthread(g); /* Only the main thread can be closed. */
281#if LJ_HASPROFILE
282 luaJIT_profile_stop(L);
283#endif
284 setgcrefnull(g->cur_L);
239 lj_func_closeuv(L, tvref(L->stack)); 285 lj_func_closeuv(L, tvref(L->stack));
240 lj_gc_separateudata(g, 1); /* Separate udata which have GC metamethods. */ 286 lj_gc_separateudata(g, 1); /* Separate udata which have GC metamethods. */
241#if LJ_HASJIT 287#if LJ_HASJIT
@@ -245,10 +291,10 @@ LUA_API void lua_close(lua_State *L)
245#endif 291#endif
246 for (i = 0;;) { 292 for (i = 0;;) {
247 hook_enter(g); 293 hook_enter(g);
248 L->status = 0; 294 L->status = LUA_OK;
295 L->base = L->top = tvref(L->stack) + 1 + LJ_FR2;
249 L->cframe = NULL; 296 L->cframe = NULL;
250 L->base = L->top = tvref(L->stack) + 1; 297 if (lj_vm_cpcall(L, NULL, NULL, cpfinalize) == LUA_OK) {
251 if (lj_vm_cpcall(L, NULL, NULL, cpfinalize) == 0) {
252 if (++i >= 10) break; 298 if (++i >= 10) break;
253 lj_gc_separateudata(g, 1); /* Separate udata again. */ 299 lj_gc_separateudata(g, 1); /* Separate udata again. */
254 if (gcref(g->gc.mmudata) == NULL) /* Until nothing is left to do. */ 300 if (gcref(g->gc.mmudata) == NULL) /* Until nothing is left to do. */
@@ -263,7 +309,7 @@ lua_State *lj_state_new(lua_State *L)
263 lua_State *L1 = lj_mem_newobj(L, lua_State); 309 lua_State *L1 = lj_mem_newobj(L, lua_State);
264 L1->gct = ~LJ_TTHREAD; 310 L1->gct = ~LJ_TTHREAD;
265 L1->dummy_ffid = FF_C; 311 L1->dummy_ffid = FF_C;
266 L1->status = 0; 312 L1->status = LUA_OK;
267 L1->stacksize = 0; 313 L1->stacksize = 0;
268 setmref(L1->stack, NULL); 314 setmref(L1->stack, NULL);
269 L1->cframe = NULL; 315 L1->cframe = NULL;
@@ -272,15 +318,17 @@ lua_State *lj_state_new(lua_State *L)
272 setmrefr(L1->glref, L->glref); 318 setmrefr(L1->glref, L->glref);
273 setgcrefr(L1->env, L->env); 319 setgcrefr(L1->env, L->env);
274 stack_init(L1, L); /* init stack */ 320 stack_init(L1, L); /* init stack */
275 lua_assert(iswhite(obj2gco(L1))); 321 lj_assertL(iswhite(obj2gco(L1)), "new thread object is not white");
276 return L1; 322 return L1;
277} 323}
278 324
279void LJ_FASTCALL lj_state_free(global_State *g, lua_State *L) 325void LJ_FASTCALL lj_state_free(global_State *g, lua_State *L)
280{ 326{
281 lua_assert(L != mainthread(g)); 327 lj_assertG(L != mainthread(g), "free of main thread");
328 if (obj2gco(L) == gcref(g->cur_L))
329 setgcrefnull(g->cur_L);
282 lj_func_closeuv(L, tvref(L->stack)); 330 lj_func_closeuv(L, tvref(L->stack));
283 lua_assert(gcref(L->openupval) == NULL); 331 lj_assertG(gcref(L->openupval) == NULL, "stale open upvalues");
284 lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue); 332 lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue);
285 lj_mem_freet(g, L); 333 lj_mem_freet(g, L);
286} 334}
diff --git a/src/lj_state.h b/src/lj_state.h
index dea9f58e..db67f03b 100644
--- a/src/lj_state.h
+++ b/src/lj_state.h
@@ -28,8 +28,10 @@ static LJ_AINLINE void lj_state_checkstack(lua_State *L, MSize need)
28 28
29LJ_FUNC lua_State *lj_state_new(lua_State *L); 29LJ_FUNC lua_State *lj_state_new(lua_State *L);
30LJ_FUNC void LJ_FASTCALL lj_state_free(global_State *g, lua_State *L); 30LJ_FUNC void LJ_FASTCALL lj_state_free(global_State *g, lua_State *L);
31#if LJ_64 31#if LJ_64 && !LJ_GC64 && !(defined(LUAJIT_USE_VALGRIND) && defined(LUAJIT_USE_SYSMALLOC))
32LJ_FUNC lua_State *lj_state_newstate(lua_Alloc f, void *ud); 32LJ_FUNC lua_State *lj_state_newstate(lua_Alloc f, void *ud);
33#endif 33#endif
34 34
35#define LJ_ALLOCF_INTERNAL ((lua_Alloc)(void *)(uintptr_t)(1237<<4))
36
35#endif 37#endif
diff --git a/src/lj_str.c b/src/lj_str.c
index 7242a8e0..cfdaec6f 100644
--- a/src/lj_str.c
+++ b/src/lj_str.c
@@ -1,13 +1,8 @@
1/* 1/*
2** String handling. 2** String handling.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h 3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4**
5** Portions taken verbatim or adapted from the Lua interpreter.
6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7*/ 4*/
8 5
9#include <stdio.h>
10
11#define lj_str_c 6#define lj_str_c
12#define LUA_CORE 7#define LUA_CORE
13 8
@@ -15,10 +10,10 @@
15#include "lj_gc.h" 10#include "lj_gc.h"
16#include "lj_err.h" 11#include "lj_err.h"
17#include "lj_str.h" 12#include "lj_str.h"
18#include "lj_state.h"
19#include "lj_char.h" 13#include "lj_char.h"
14#include "lj_prng.h"
20 15
21/* -- String interning ---------------------------------------------------- */ 16/* -- String helpers ------------------------------------------------------ */
22 17
23/* Ordered compare of strings. Assumes string data is 4-byte aligned. */ 18/* Ordered compare of strings. Assumes string data is 4-byte aligned. */
24int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b) 19int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b)
@@ -43,297 +38,333 @@ int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b)
43 return (int32_t)(a->len - b->len); 38 return (int32_t)(a->len - b->len);
44} 39}
45 40
46/* Fast string data comparison. Caveat: unaligned access to 1st string! */ 41/* Find fixed string p inside string s. */
47static LJ_AINLINE int str_fastcmp(const char *a, const char *b, MSize len) 42const char *lj_str_find(const char *s, const char *p, MSize slen, MSize plen)
48{ 43{
49 MSize i = 0; 44 if (plen <= slen) {
50 lua_assert(len > 0); 45 if (plen == 0) {
51 lua_assert((((uintptr_t)a+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4); 46 return s;
52 do { /* Note: innocuous access up to end of string + 3. */ 47 } else {
53 uint32_t v = lj_getu32(a+i) ^ *(const uint32_t *)(b+i); 48 int c = *(const uint8_t *)p++;
54 if (v) { 49 plen--; slen -= plen;
55 i -= len; 50 while (slen) {
56#if LJ_LE 51 const char *q = (const char *)memchr(s, c, slen);
57 return (int32_t)i >= -3 ? (v << (32+(i<<3))) : 1; 52 if (!q) break;
58#else 53 if (memcmp(q+1, p, plen) == 0) return q;
59 return (int32_t)i >= -3 ? (v >> (32+(i<<3))) : 1; 54 q++; slen -= (MSize)(q-s); s = q;
60#endif 55 }
61 } 56 }
62 i += 4; 57 }
63 } while (i < len); 58 return NULL;
64 return 0;
65} 59}
66 60
67/* Resize the string hash table (grow and shrink). */ 61/* Check whether a string has a pattern matching character. */
68void lj_str_resize(lua_State *L, MSize newmask) 62int lj_str_haspattern(GCstr *s)
69{ 63{
70 global_State *g = G(L); 64 const char *p = strdata(s), *q = p + s->len;
71 GCRef *newhash; 65 while (p < q) {
72 MSize i; 66 int c = *(const uint8_t *)p++;
73 if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1) 67 if (lj_char_ispunct(c) && strchr("^$*+?.([%-", c))
74 return; /* No resizing during GC traversal or if already too big. */ 68 return 1; /* Found a pattern matching char. */
75 newhash = lj_mem_newvec(L, newmask+1, GCRef);
76 memset(newhash, 0, (newmask+1)*sizeof(GCRef));
77 for (i = g->strmask; i != ~(MSize)0; i--) { /* Rehash old table. */
78 GCobj *p = gcref(g->strhash[i]);
79 while (p) { /* Follow each hash chain and reinsert all strings. */
80 MSize h = gco2str(p)->hash & newmask;
81 GCobj *next = gcnext(p);
82 /* NOBARRIER: The string table is a GC root. */
83 setgcrefr(p->gch.nextgc, newhash[h]);
84 setgcref(newhash[h], p);
85 p = next;
86 }
87 } 69 }
88 lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef); 70 return 0; /* No pattern matching chars found. */
89 g->strmask = newmask;
90 g->strhash = newhash;
91} 71}
92 72
93/* Intern a string and return string object. */ 73/* -- String hashing ------------------------------------------------------ */
94GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) 74
75/* Keyed sparse ARX string hash. Constant time. */
76static StrHash hash_sparse(uint64_t seed, const char *str, MSize len)
95{ 77{
96 global_State *g; 78 /* Constants taken from lookup3 hash by Bob Jenkins. */
97 GCstr *s; 79 StrHash a, b, h = len ^ (StrHash)seed;
98 GCobj *o;
99 MSize len = (MSize)lenx;
100 MSize a, b, h = len;
101 if (lenx >= LJ_MAX_STR)
102 lj_err_msg(L, LJ_ERR_STROV);
103 g = G(L);
104 /* Compute string hash. Constants taken from lookup3 hash by Bob Jenkins. */
105 if (len >= 4) { /* Caveat: unaligned access! */ 80 if (len >= 4) { /* Caveat: unaligned access! */
106 a = lj_getu32(str); 81 a = lj_getu32(str);
107 h ^= lj_getu32(str+len-4); 82 h ^= lj_getu32(str+len-4);
108 b = lj_getu32(str+(len>>1)-2); 83 b = lj_getu32(str+(len>>1)-2);
109 h ^= b; h -= lj_rol(b, 14); 84 h ^= b; h -= lj_rol(b, 14);
110 b += lj_getu32(str+(len>>2)-1); 85 b += lj_getu32(str+(len>>2)-1);
111 } else if (len > 0) { 86 } else {
112 a = *(const uint8_t *)str; 87 a = *(const uint8_t *)str;
113 h ^= *(const uint8_t *)(str+len-1); 88 h ^= *(const uint8_t *)(str+len-1);
114 b = *(const uint8_t *)(str+(len>>1)); 89 b = *(const uint8_t *)(str+(len>>1));
115 h ^= b; h -= lj_rol(b, 14); 90 h ^= b; h -= lj_rol(b, 14);
116 } else {
117 return &g->strempty;
118 } 91 }
119 a ^= h; a -= lj_rol(h, 11); 92 a ^= h; a -= lj_rol(h, 11);
120 b ^= a; b -= lj_rol(a, 25); 93 b ^= a; b -= lj_rol(a, 25);
121 h ^= b; h -= lj_rol(b, 16); 94 h ^= b; h -= lj_rol(b, 16);
122 /* Check if the string has already been interned. */ 95 return h;
123 o = gcref(g->strhash[h & g->strmask]);
124 if (LJ_LIKELY((((uintptr_t)str+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4)) {
125 while (o != NULL) {
126 GCstr *sx = gco2str(o);
127 if (sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) {
128 /* Resurrect if dead. Can only happen with fixstring() (keywords). */
129 if (isdead(g, o)) flipwhite(o);
130 return sx; /* Return existing string. */
131 }
132 o = gcnext(o);
133 }
134 } else { /* Slow path: end of string is too close to a page boundary. */
135 while (o != NULL) {
136 GCstr *sx = gco2str(o);
137 if (sx->len == len && memcmp(str, strdata(sx), len) == 0) {
138 /* Resurrect if dead. Can only happen with fixstring() (keywords). */
139 if (isdead(g, o)) flipwhite(o);
140 return sx; /* Return existing string. */
141 }
142 o = gcnext(o);
143 }
144 }
145 /* Nope, create a new string. */
146 s = lj_mem_newt(L, sizeof(GCstr)+len+1, GCstr);
147 newwhite(g, s);
148 s->gct = ~LJ_TSTR;
149 s->len = len;
150 s->hash = h;
151 s->reserved = 0;
152 memcpy(strdatawr(s), str, len);
153 strdatawr(s)[len] = '\0'; /* Zero-terminate string. */
154 /* Add it to string hash table. */
155 h &= g->strmask;
156 s->nextgc = g->strhash[h];
157 /* NOBARRIER: The string table is a GC root. */
158 setgcref(g->strhash[h], obj2gco(s));
159 if (g->strnum++ > g->strmask) /* Allow a 100% load factor. */
160 lj_str_resize(L, (g->strmask<<1)+1); /* Grow string table. */
161 return s; /* Return newly interned string. */
162} 96}
163 97
164void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s) 98#if LUAJIT_SECURITY_STRHASH
99/* Keyed dense ARX string hash. Linear time. */
100static LJ_NOINLINE StrHash hash_dense(uint64_t seed, StrHash h,
101 const char *str, MSize len)
165{ 102{
166 g->strnum--; 103 StrHash b = lj_bswap(lj_rol(h ^ (StrHash)(seed >> 32), 4));
167 lj_mem_free(g, s, sizestring(s)); 104 if (len > 12) {
105 StrHash a = (StrHash)seed;
106 const char *pe = str+len-12, *p = pe, *q = str;
107 do {
108 a += lj_getu32(p);
109 b += lj_getu32(p+4);
110 h += lj_getu32(p+8);
111 p = q; q += 12;
112 h ^= b; h -= lj_rol(b, 14);
113 a ^= h; a -= lj_rol(h, 11);
114 b ^= a; b -= lj_rol(a, 25);
115 } while (p < pe);
116 h ^= b; h -= lj_rol(b, 16);
117 a ^= h; a -= lj_rol(h, 4);
118 b ^= a; b -= lj_rol(a, 14);
119 }
120 return b;
168} 121}
122#endif
169 123
170/* -- Type conversions ---------------------------------------------------- */ 124/* -- String interning ---------------------------------------------------- */
171 125
172/* Print number to buffer. Canonicalizes non-finite values. */ 126#define LJ_STR_MAXCOLL 32
173size_t LJ_FASTCALL lj_str_bufnum(char *s, cTValue *o)
174{
175 if (LJ_LIKELY((o->u32.hi << 1) < 0xffe00000)) { /* Finite? */
176 lua_Number n = o->n;
177#if __BIONIC__
178 if (tvismzero(o)) { s[0] = '-'; s[1] = '0'; return 2; }
179#endif
180 return (size_t)lua_number2str(s, n);
181 } else if (((o->u32.hi & 0x000fffff) | o->u32.lo) != 0) {
182 s[0] = 'n'; s[1] = 'a'; s[2] = 'n'; return 3;
183 } else if ((o->u32.hi & 0x80000000) == 0) {
184 s[0] = 'i'; s[1] = 'n'; s[2] = 'f'; return 3;
185 } else {
186 s[0] = '-'; s[1] = 'i'; s[2] = 'n'; s[3] = 'f'; return 4;
187 }
188}
189 127
190/* Print integer to buffer. Returns pointer to start. */ 128/* Resize the string interning hash table (grow and shrink). */
191char * LJ_FASTCALL lj_str_bufint(char *p, int32_t k) 129void lj_str_resize(lua_State *L, MSize newmask)
192{ 130{
193 uint32_t u = k < 0 ? ~(uint32_t)k+1u : (uint32_t)k; 131 global_State *g = G(L);
194 p += 1+10; 132 GCRef *newtab, *oldtab = g->str.tab;
195 do { *--p = (char)('0' + u % 10); } while (u /= 10); 133 MSize i;
196 if (k < 0) *--p = '-';
197 return p;
198}
199 134
200/* Convert number to string. */ 135 /* No resizing during GC traversal or if already too big. */
201GCstr * LJ_FASTCALL lj_str_fromnum(lua_State *L, const lua_Number *np) 136 if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1)
202{ 137 return;
203 char buf[LJ_STR_NUMBUF];
204 size_t len = lj_str_bufnum(buf, (TValue *)np);
205 return lj_str_new(L, buf, len);
206}
207 138
208/* Convert integer to string. */ 139 newtab = lj_mem_newvec(L, newmask+1, GCRef);
209GCstr * LJ_FASTCALL lj_str_fromint(lua_State *L, int32_t k) 140 memset(newtab, 0, (newmask+1)*sizeof(GCRef));
210{
211 char s[1+10];
212 char *p = lj_str_bufint(s, k);
213 return lj_str_new(L, p, (size_t)(s+sizeof(s)-p));
214}
215 141
216GCstr * LJ_FASTCALL lj_str_fromnumber(lua_State *L, cTValue *o) 142#if LUAJIT_SECURITY_STRHASH
217{ 143 /* Check which chains need secondary hashes. */
218 return tvisint(o) ? lj_str_fromint(L, intV(o)) : lj_str_fromnum(L, &o->n); 144 if (g->str.second) {
219} 145 int newsecond = 0;
146 /* Compute primary chain lengths. */
147 for (i = g->str.mask; i != ~(MSize)0; i--) {
148 GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1);
149 while (o) {
150 GCstr *s = gco2str(o);
151 MSize hash = s->hashalg ? hash_sparse(g->str.seed, strdata(s), s->len) :
152 s->hash;
153 hash &= newmask;
154 setgcrefp(newtab[hash], gcrefu(newtab[hash]) + 1);
155 o = gcnext(o);
156 }
157 }
158 /* Mark secondary chains. */
159 for (i = newmask; i != ~(MSize)0; i--) {
160 int secondary = gcrefu(newtab[i]) > LJ_STR_MAXCOLL;
161 newsecond |= secondary;
162 setgcrefp(newtab[i], secondary);
163 }
164 g->str.second = newsecond;
165 }
166#endif
220 167
221/* -- String formatting --------------------------------------------------- */ 168 /* Reinsert all strings from the old table into the new table. */
169 for (i = g->str.mask; i != ~(MSize)0; i--) {
170 GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1);
171 while (o) {
172 GCobj *next = gcnext(o);
173 GCstr *s = gco2str(o);
174 MSize hash = s->hash;
175#if LUAJIT_SECURITY_STRHASH
176 uintptr_t u;
177 if (LJ_LIKELY(!s->hashalg)) { /* String hashed with primary hash. */
178 hash &= newmask;
179 u = gcrefu(newtab[hash]);
180 if (LJ_UNLIKELY(u & 1)) { /* Switch string to secondary hash. */
181 s->hash = hash = hash_dense(g->str.seed, s->hash, strdata(s), s->len);
182 s->hashalg = 1;
183 hash &= newmask;
184 u = gcrefu(newtab[hash]);
185 }
186 } else { /* String hashed with secondary hash. */
187 MSize shash = hash_sparse(g->str.seed, strdata(s), s->len);
188 u = gcrefu(newtab[shash & newmask]);
189 if (u & 1) {
190 hash &= newmask;
191 u = gcrefu(newtab[hash]);
192 } else { /* Revert string back to primary hash. */
193 s->hash = shash;
194 s->hashalg = 0;
195 hash = (shash & newmask);
196 }
197 }
198 /* NOBARRIER: The string table is a GC root. */
199 setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1));
200 setgcrefp(newtab[hash], ((uintptr_t)o | (u & 1)));
201#else
202 hash &= newmask;
203 /* NOBARRIER: The string table is a GC root. */
204 setgcrefr(o->gch.nextgc, newtab[hash]);
205 setgcref(newtab[hash], o);
206#endif
207 o = next;
208 }
209 }
210
211 /* Free old table and replace with new table. */
212 lj_str_freetab(g);
213 g->str.tab = newtab;
214 g->str.mask = newmask;
215}
222 216
223static void addstr(lua_State *L, SBuf *sb, const char *str, MSize len) 217#if LUAJIT_SECURITY_STRHASH
218/* Rehash and rechain all strings in a chain. */
219static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc,
220 const char *str, MSize len)
224{ 221{
225 char *p; 222 global_State *g = G(L);
226 MSize i; 223 int ow = g->gc.state == GCSsweepstring ? otherwhite(g) : 0; /* Sweeping? */
227 if (sb->n + len > sb->sz) { 224 GCRef *strtab = g->str.tab;
228 MSize sz = sb->sz * 2; 225 MSize strmask = g->str.mask;
229 while (sb->n + len > sz) sz = sz * 2; 226 GCobj *o = gcref(strtab[hashc & strmask]);
230 lj_str_resizebuf(L, sb, sz); 227 setgcrefp(strtab[hashc & strmask], (void *)((uintptr_t)1));
228 g->str.second = 1;
229 while (o) {
230 uintptr_t u;
231 GCobj *next = gcnext(o);
232 GCstr *s = gco2str(o);
233 StrHash hash;
234 if (ow) { /* Must sweep while rechaining. */
235 if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* String alive? */
236 lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED),
237 "sweep of undead string");
238 makewhite(g, o);
239 } else { /* Free dead string. */
240 lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED,
241 "sweep of unlive string");
242 lj_str_free(g, s);
243 o = next;
244 continue;
245 }
246 }
247 hash = s->hash;
248 if (!s->hashalg) { /* Rehash with secondary hash. */
249 hash = hash_dense(g->str.seed, hash, strdata(s), s->len);
250 s->hash = hash;
251 s->hashalg = 1;
252 }
253 /* Rechain. */
254 hash &= strmask;
255 u = gcrefu(strtab[hash]);
256 setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1));
257 setgcrefp(strtab[hash], ((uintptr_t)o | (u & 1)));
258 o = next;
231 } 259 }
232 p = sb->buf + sb->n; 260 /* Try to insert the pending string again. */
233 sb->n += len; 261 return lj_str_new(L, str, len);
234 for (i = 0; i < len; i++) p[i] = str[i];
235} 262}
263#endif
264
265/* Reseed String ID from PRNG after random interval < 2^bits. */
266#if LUAJIT_SECURITY_STRID == 1
267#define STRID_RESEED_INTERVAL 8
268#elif LUAJIT_SECURITY_STRID == 2
269#define STRID_RESEED_INTERVAL 4
270#elif LUAJIT_SECURITY_STRID >= 3
271#define STRID_RESEED_INTERVAL 0
272#endif
236 273
237static void addchar(lua_State *L, SBuf *sb, int c) 274/* Allocate a new string and add to string interning table. */
275static GCstr *lj_str_alloc(lua_State *L, const char *str, MSize len,
276 StrHash hash, int hashalg)
238{ 277{
239 if (sb->n + 1 > sb->sz) { 278 GCstr *s = lj_mem_newt(L, lj_str_size(len), GCstr);
240 MSize sz = sb->sz * 2; 279 global_State *g = G(L);
241 lj_str_resizebuf(L, sb, sz); 280 uintptr_t u;
281 newwhite(g, s);
282 s->gct = ~LJ_TSTR;
283 s->len = len;
284 s->hash = hash;
285#ifndef STRID_RESEED_INTERVAL
286 s->sid = g->str.id++;
287#elif STRID_RESEED_INTERVAL
288 if (!g->str.idreseed--) {
289 uint64_t r = lj_prng_u64(&g->prng);
290 g->str.id = (StrID)r;
291 g->str.idreseed = (uint8_t)(r >> (64 - STRID_RESEED_INTERVAL));
242 } 292 }
243 sb->buf[sb->n++] = (char)c; 293 s->sid = g->str.id++;
294#else
295 s->sid = (StrID)lj_prng_u64(&g->prng);
296#endif
297 s->reserved = 0;
298 s->hashalg = (uint8_t)hashalg;
299 /* Clear last 4 bytes of allocated memory. Implies zero-termination, too. */
300 *(uint32_t *)(strdatawr(s)+(len & ~(MSize)3)) = 0;
301 memcpy(strdatawr(s), str, len);
302 /* Add to string hash table. */
303 hash &= g->str.mask;
304 u = gcrefu(g->str.tab[hash]);
305 setgcrefp(s->nextgc, (u & ~(uintptr_t)1));
306 /* NOBARRIER: The string table is a GC root. */
307 setgcrefp(g->str.tab[hash], ((uintptr_t)s | (u & 1)));
308 if (g->str.num++ > g->str.mask) /* Allow a 100% load factor. */
309 lj_str_resize(L, (g->str.mask<<1)+1); /* Grow string table. */
310 return s; /* Return newly interned string. */
244} 311}
245 312
246/* Push formatted message as a string object to Lua stack. va_list variant. */ 313/* Intern a string and return string object. */
247const char *lj_str_pushvf(lua_State *L, const char *fmt, va_list argp) 314GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
248{ 315{
249 SBuf *sb = &G(L)->tmpbuf; 316 global_State *g = G(L);
250 lj_str_needbuf(L, sb, (MSize)strlen(fmt)); 317 if (lenx-1 < LJ_MAX_STR-1) {
251 lj_str_resetbuf(sb); 318 MSize len = (MSize)lenx;
252 for (;;) { 319 StrHash hash = hash_sparse(g->str.seed, str, len);
253 const char *e = strchr(fmt, '%'); 320 MSize coll = 0;
254 if (e == NULL) break; 321 int hashalg = 0;
255 addstr(L, sb, fmt, (MSize)(e-fmt)); 322 /* Check if the string has already been interned. */
256 /* This function only handles %s, %c, %d, %f and %p formats. */ 323 GCobj *o = gcref(g->str.tab[hash & g->str.mask]);
257 switch (e[1]) { 324#if LUAJIT_SECURITY_STRHASH
258 case 's': { 325 if (LJ_UNLIKELY((uintptr_t)o & 1)) { /* Secondary hash for this chain? */
259 const char *s = va_arg(argp, char *); 326 hashalg = 1;
260 if (s == NULL) s = "(null)"; 327 hash = hash_dense(g->str.seed, hash, str, len);
261 addstr(L, sb, s, (MSize)strlen(s)); 328 o = (GCobj *)(gcrefu(g->str.tab[hash & g->str.mask]) & ~(uintptr_t)1);
262 break; 329 }
263 }
264 case 'c':
265 addchar(L, sb, va_arg(argp, int));
266 break;
267 case 'd': {
268 char buf[LJ_STR_INTBUF];
269 char *p = lj_str_bufint(buf, va_arg(argp, int32_t));
270 addstr(L, sb, p, (MSize)(buf+LJ_STR_INTBUF-p));
271 break;
272 }
273 case 'f': {
274 char buf[LJ_STR_NUMBUF];
275 TValue tv;
276 MSize len;
277 tv.n = (lua_Number)(va_arg(argp, LUAI_UACNUMBER));
278 len = (MSize)lj_str_bufnum(buf, &tv);
279 addstr(L, sb, buf, len);
280 break;
281 }
282 case 'p': {
283#define FMTP_CHARS (2*sizeof(ptrdiff_t))
284 char buf[2+FMTP_CHARS];
285 ptrdiff_t p = (ptrdiff_t)(va_arg(argp, void *));
286 ptrdiff_t i, lasti = 2+FMTP_CHARS;
287 if (p == 0) {
288 addstr(L, sb, "NULL", 4);
289 break;
290 }
291#if LJ_64
292 /* Shorten output for 64 bit pointers. */
293 lasti = 2+2*4+((p >> 32) ? 2+2*(lj_fls((uint32_t)(p >> 32))>>3) : 0);
294#endif 330#endif
295 buf[0] = '0'; 331 while (o != NULL) {
296 buf[1] = 'x'; 332 GCstr *sx = gco2str(o);
297 for (i = lasti-1; i >= 2; i--, p >>= 4) 333 if (sx->hash == hash && sx->len == len) {
298 buf[i] = "0123456789abcdef"[(p & 15)]; 334 if (memcmp(str, strdata(sx), len) == 0) {
299 addstr(L, sb, buf, (MSize)lasti); 335 if (isdead(g, o)) flipwhite(o); /* Resurrect if dead. */
300 break; 336 return sx; /* Return existing string. */
337 }
338 coll++;
301 } 339 }
302 case '%': 340 coll++;
303 addchar(L, sb, '%'); 341 o = gcnext(o);
304 break; 342 }
305 default: 343#if LUAJIT_SECURITY_STRHASH
306 addchar(L, sb, '%'); 344 /* Rehash chain if there are too many collisions. */
307 addchar(L, sb, e[1]); 345 if (LJ_UNLIKELY(coll > LJ_STR_MAXCOLL) && !hashalg) {
308 break; 346 return lj_str_rehash_chain(L, hash, str, len);
309 } 347 }
310 fmt = e+2; 348#endif
349 /* Otherwise allocate a new string. */
350 return lj_str_alloc(L, str, len, hash, hashalg);
351 } else {
352 if (lenx)
353 lj_err_msg(L, LJ_ERR_STROV);
354 return &g->strempty;
311 } 355 }
312 addstr(L, sb, fmt, (MSize)strlen(fmt));
313 setstrV(L, L->top, lj_str_new(L, sb->buf, sb->n));
314 incr_top(L);
315 return strVdata(L->top - 1);
316} 356}
317 357
318/* Push formatted message as a string object to Lua stack. Vararg variant. */ 358void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s)
319const char *lj_str_pushf(lua_State *L, const char *fmt, ...)
320{ 359{
321 const char *msg; 360 g->str.num--;
322 va_list argp; 361 lj_mem_free(g, s, lj_str_size(s->len));
323 va_start(argp, fmt);
324 msg = lj_str_pushvf(L, fmt, argp);
325 va_end(argp);
326 return msg;
327} 362}
328 363
329/* -- Buffer handling ----------------------------------------------------- */ 364void LJ_FASTCALL lj_str_init(lua_State *L)
330
331char *lj_str_needbuf(lua_State *L, SBuf *sb, MSize sz)
332{ 365{
333 if (sz > sb->sz) { 366 global_State *g = G(L);
334 if (sz < LJ_MIN_SBUF) sz = LJ_MIN_SBUF; 367 g->str.seed = lj_prng_u64(&g->prng);
335 lj_str_resizebuf(L, sb, sz); 368 lj_str_resize(L, LJ_MIN_STRTAB-1);
336 }
337 return sb->buf;
338} 369}
339 370
diff --git a/src/lj_str.h b/src/lj_str.h
index 8689a0d7..2a5a8190 100644
--- a/src/lj_str.h
+++ b/src/lj_str.h
@@ -10,41 +10,22 @@
10 10
11#include "lj_obj.h" 11#include "lj_obj.h"
12 12
13/* String interning. */ 13/* String helpers. */
14LJ_FUNC int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b); 14LJ_FUNC int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b);
15LJ_FUNC const char *lj_str_find(const char *s, const char *f,
16 MSize slen, MSize flen);
17LJ_FUNC int lj_str_haspattern(GCstr *s);
18
19/* String interning. */
15LJ_FUNC void lj_str_resize(lua_State *L, MSize newmask); 20LJ_FUNC void lj_str_resize(lua_State *L, MSize newmask);
16LJ_FUNCA GCstr *lj_str_new(lua_State *L, const char *str, size_t len); 21LJ_FUNCA GCstr *lj_str_new(lua_State *L, const char *str, size_t len);
17LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s); 22LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s);
23LJ_FUNC void LJ_FASTCALL lj_str_init(lua_State *L);
24#define lj_str_freetab(g) \
25 (lj_mem_freevec(g, g->str.tab, g->str.mask+1, GCRef))
18 26
19#define lj_str_newz(L, s) (lj_str_new(L, s, strlen(s))) 27#define lj_str_newz(L, s) (lj_str_new(L, s, strlen(s)))
20#define lj_str_newlit(L, s) (lj_str_new(L, "" s, sizeof(s)-1)) 28#define lj_str_newlit(L, s) (lj_str_new(L, "" s, sizeof(s)-1))
21 29#define lj_str_size(len) (sizeof(GCstr) + (((len)+4) & ~(MSize)3))
22/* Type conversions. */
23LJ_FUNC size_t LJ_FASTCALL lj_str_bufnum(char *s, cTValue *o);
24LJ_FUNC char * LJ_FASTCALL lj_str_bufint(char *p, int32_t k);
25LJ_FUNCA GCstr * LJ_FASTCALL lj_str_fromnum(lua_State *L, const lua_Number *np);
26LJ_FUNC GCstr * LJ_FASTCALL lj_str_fromint(lua_State *L, int32_t k);
27LJ_FUNCA GCstr * LJ_FASTCALL lj_str_fromnumber(lua_State *L, cTValue *o);
28
29#define LJ_STR_INTBUF (1+10)
30#define LJ_STR_NUMBUF LUAI_MAXNUMBER2STR
31
32/* String formatting. */
33LJ_FUNC const char *lj_str_pushvf(lua_State *L, const char *fmt, va_list argp);
34LJ_FUNC const char *lj_str_pushf(lua_State *L, const char *fmt, ...)
35#if defined(__GNUC__)
36 __attribute__ ((format (printf, 2, 3)))
37#endif
38 ;
39
40/* Resizable string buffers. Struct definition in lj_obj.h. */
41LJ_FUNC char *lj_str_needbuf(lua_State *L, SBuf *sb, MSize sz);
42
43#define lj_str_initbuf(sb) ((sb)->buf = NULL, (sb)->sz = 0)
44#define lj_str_resetbuf(sb) ((sb)->n = 0)
45#define lj_str_resizebuf(L, sb, size) \
46 ((sb)->buf = (char *)lj_mem_realloc(L, (sb)->buf, (sb)->sz, (size)), \
47 (sb)->sz = (size))
48#define lj_str_freebuf(g, sb) lj_mem_free(g, (void *)(sb)->buf, (sb)->sz)
49 30
50#endif 31#endif
diff --git a/src/lj_strfmt.c b/src/lj_strfmt.c
new file mode 100644
index 00000000..909255db
--- /dev/null
+++ b/src/lj_strfmt.c
@@ -0,0 +1,606 @@
1/*
2** String formatting.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#include <stdio.h>
7
8#define lj_strfmt_c
9#define LUA_CORE
10
11#include "lj_obj.h"
12#include "lj_err.h"
13#include "lj_buf.h"
14#include "lj_str.h"
15#include "lj_meta.h"
16#include "lj_state.h"
17#include "lj_char.h"
18#include "lj_strfmt.h"
19#if LJ_HASFFI
20#include "lj_ctype.h"
21#endif
22#include "lj_lib.h"
23
24/* -- Format parser ------------------------------------------------------- */
25
26static const uint8_t strfmt_map[('x'-'A')+1] = {
27 STRFMT_A,0,0,0,STRFMT_E,STRFMT_F,STRFMT_G,0,0,0,0,0,0,
28 0,0,0,0,0,0,0,0,0,0,STRFMT_X,0,0,
29 0,0,0,0,0,0,
30 STRFMT_A,0,STRFMT_C,STRFMT_D,STRFMT_E,STRFMT_F,STRFMT_G,0,STRFMT_I,0,0,0,0,
31 0,STRFMT_O,STRFMT_P,STRFMT_Q,0,STRFMT_S,0,STRFMT_U,0,0,STRFMT_X
32};
33
34SFormat LJ_FASTCALL lj_strfmt_parse(FormatState *fs)
35{
36 const uint8_t *p = fs->p, *e = fs->e;
37 fs->str = (const char *)p;
38 for (; p < e; p++) {
39 if (*p == '%') { /* Escape char? */
40 if (p[1] == '%') { /* '%%'? */
41 fs->p = ++p+1;
42 goto retlit;
43 } else {
44 SFormat sf = 0;
45 uint32_t c;
46 if (p != (const uint8_t *)fs->str)
47 break;
48 for (p++; (uint32_t)*p - ' ' <= (uint32_t)('0' - ' '); p++) {
49 /* Parse flags. */
50 if (*p == '-') sf |= STRFMT_F_LEFT;
51 else if (*p == '+') sf |= STRFMT_F_PLUS;
52 else if (*p == '0') sf |= STRFMT_F_ZERO;
53 else if (*p == ' ') sf |= STRFMT_F_SPACE;
54 else if (*p == '#') sf |= STRFMT_F_ALT;
55 else break;
56 }
57 if ((uint32_t)*p - '0' < 10) { /* Parse width. */
58 uint32_t width = (uint32_t)*p++ - '0';
59 if ((uint32_t)*p - '0' < 10)
60 width = (uint32_t)*p++ - '0' + width*10;
61 sf |= (width << STRFMT_SH_WIDTH);
62 }
63 if (*p == '.') { /* Parse precision. */
64 uint32_t prec = 0;
65 p++;
66 if ((uint32_t)*p - '0' < 10) {
67 prec = (uint32_t)*p++ - '0';
68 if ((uint32_t)*p - '0' < 10)
69 prec = (uint32_t)*p++ - '0' + prec*10;
70 }
71 sf |= ((prec+1) << STRFMT_SH_PREC);
72 }
73 /* Parse conversion. */
74 c = (uint32_t)*p - 'A';
75 if (LJ_LIKELY(c <= (uint32_t)('x' - 'A'))) {
76 uint32_t sx = strfmt_map[c];
77 if (sx) {
78 fs->p = p+1;
79 return (sf | sx | ((c & 0x20) ? 0 : STRFMT_F_UPPER));
80 }
81 }
82 /* Return error location. */
83 if (*p >= 32) p++;
84 fs->len = (MSize)(p - (const uint8_t *)fs->str);
85 fs->p = fs->e;
86 return STRFMT_ERR;
87 }
88 }
89 }
90 fs->p = p;
91retlit:
92 fs->len = (MSize)(p - (const uint8_t *)fs->str);
93 return fs->len ? STRFMT_LIT : STRFMT_EOF;
94}
95
96/* -- Raw conversions ----------------------------------------------------- */
97
98#define WINT_R(x, sh, sc) \
99 { uint32_t d = (x*(((1<<sh)+sc-1)/sc))>>sh; x -= d*sc; *p++ = (char)('0'+d); }
100
101/* Write integer to buffer. */
102char * LJ_FASTCALL lj_strfmt_wint(char *p, int32_t k)
103{
104 uint32_t u = (uint32_t)k;
105 if (k < 0) { u = ~u+1u; *p++ = '-'; }
106 if (u < 10000) {
107 if (u < 10) goto dig1;
108 if (u < 100) goto dig2;
109 if (u < 1000) goto dig3;
110 } else {
111 uint32_t v = u / 10000; u -= v * 10000;
112 if (v < 10000) {
113 if (v < 10) goto dig5;
114 if (v < 100) goto dig6;
115 if (v < 1000) goto dig7;
116 } else {
117 uint32_t w = v / 10000; v -= w * 10000;
118 if (w >= 10) WINT_R(w, 10, 10)
119 *p++ = (char)('0'+w);
120 }
121 WINT_R(v, 23, 1000)
122 dig7: WINT_R(v, 12, 100)
123 dig6: WINT_R(v, 10, 10)
124 dig5: *p++ = (char)('0'+v);
125 }
126 WINT_R(u, 23, 1000)
127 dig3: WINT_R(u, 12, 100)
128 dig2: WINT_R(u, 10, 10)
129 dig1: *p++ = (char)('0'+u);
130 return p;
131}
132#undef WINT_R
133
134/* Write pointer to buffer. */
135char * LJ_FASTCALL lj_strfmt_wptr(char *p, const void *v)
136{
137 ptrdiff_t x = (ptrdiff_t)v;
138 MSize i, n = STRFMT_MAXBUF_PTR;
139 if (x == 0) {
140 *p++ = 'N'; *p++ = 'U'; *p++ = 'L'; *p++ = 'L';
141 return p;
142 }
143#if LJ_64
144 /* Shorten output for 64 bit pointers. */
145 n = 2+2*4+((x >> 32) ? 2+2*(lj_fls((uint32_t)(x >> 32))>>3) : 0);
146#endif
147 p[0] = '0';
148 p[1] = 'x';
149 for (i = n-1; i >= 2; i--, x >>= 4)
150 p[i] = "0123456789abcdef"[(x & 15)];
151 return p+n;
152}
153
154/* Write ULEB128 to buffer. */
155char * LJ_FASTCALL lj_strfmt_wuleb128(char *p, uint32_t v)
156{
157 for (; v >= 0x80; v >>= 7)
158 *p++ = (char)((v & 0x7f) | 0x80);
159 *p++ = (char)v;
160 return p;
161}
162
163/* Return string or write number to tmp buffer and return pointer to start. */
164const char *lj_strfmt_wstrnum(lua_State *L, cTValue *o, MSize *lenp)
165{
166 SBuf *sb;
167 if (tvisstr(o)) {
168 *lenp = strV(o)->len;
169 return strVdata(o);
170 } else if (tvisbuf(o)) {
171 SBufExt *sbx = bufV(o);
172 *lenp = sbufxlen(sbx);
173 return sbx->r;
174 } else if (tvisint(o)) {
175 sb = lj_strfmt_putint(lj_buf_tmp_(L), intV(o));
176 } else if (tvisnum(o)) {
177 sb = lj_strfmt_putfnum(lj_buf_tmp_(L), STRFMT_G14, o->n);
178 } else {
179 return NULL;
180 }
181 *lenp = sbuflen(sb);
182 return sb->b;
183}
184
185/* -- Unformatted conversions to buffer ----------------------------------- */
186
187/* Add integer to buffer. */
188SBuf * LJ_FASTCALL lj_strfmt_putint(SBuf *sb, int32_t k)
189{
190 sb->w = lj_strfmt_wint(lj_buf_more(sb, STRFMT_MAXBUF_INT), k);
191 return sb;
192}
193
194#if LJ_HASJIT
195/* Add number to buffer. */
196SBuf * LJ_FASTCALL lj_strfmt_putnum(SBuf *sb, cTValue *o)
197{
198 return lj_strfmt_putfnum(sb, STRFMT_G14, o->n);
199}
200#endif
201
202SBuf * LJ_FASTCALL lj_strfmt_putptr(SBuf *sb, const void *v)
203{
204 sb->w = lj_strfmt_wptr(lj_buf_more(sb, STRFMT_MAXBUF_PTR), v);
205 return sb;
206}
207
208/* Add quoted string to buffer. */
209static SBuf *strfmt_putquotedlen(SBuf *sb, const char *s, MSize len)
210{
211 lj_buf_putb(sb, '"');
212 while (len--) {
213 uint32_t c = (uint32_t)(uint8_t)*s++;
214 char *w = lj_buf_more(sb, 4);
215 if (c == '"' || c == '\\' || c == '\n') {
216 *w++ = '\\';
217 } else if (lj_char_iscntrl(c)) { /* This can only be 0-31 or 127. */
218 uint32_t d;
219 *w++ = '\\';
220 if (c >= 100 || lj_char_isdigit((uint8_t)*s)) {
221 *w++ = (char)('0'+(c >= 100)); if (c >= 100) c -= 100;
222 goto tens;
223 } else if (c >= 10) {
224 tens:
225 d = (c * 205) >> 11; c -= d * 10; *w++ = (char)('0'+d);
226 }
227 c += '0';
228 }
229 *w++ = (char)c;
230 sb->w = w;
231 }
232 lj_buf_putb(sb, '"');
233 return sb;
234}
235
236#if LJ_HASJIT
237SBuf * LJ_FASTCALL lj_strfmt_putquoted(SBuf *sb, GCstr *str)
238{
239 return strfmt_putquotedlen(sb, strdata(str), str->len);
240}
241#endif
242
243/* -- Formatted conversions to buffer ------------------------------------- */
244
245/* Add formatted char to buffer. */
246SBuf *lj_strfmt_putfchar(SBuf *sb, SFormat sf, int32_t c)
247{
248 MSize width = STRFMT_WIDTH(sf);
249 char *w = lj_buf_more(sb, width > 1 ? width : 1);
250 if ((sf & STRFMT_F_LEFT)) *w++ = (char)c;
251 while (width-- > 1) *w++ = ' ';
252 if (!(sf & STRFMT_F_LEFT)) *w++ = (char)c;
253 sb->w = w;
254 return sb;
255}
256
257/* Add formatted string to buffer. */
258static SBuf *strfmt_putfstrlen(SBuf *sb, SFormat sf, const char *s, MSize len)
259{
260 MSize width = STRFMT_WIDTH(sf);
261 char *w;
262 if (len > STRFMT_PREC(sf)) len = STRFMT_PREC(sf);
263 w = lj_buf_more(sb, width > len ? width : len);
264 if ((sf & STRFMT_F_LEFT)) w = lj_buf_wmem(w, s, len);
265 while (width-- > len) *w++ = ' ';
266 if (!(sf & STRFMT_F_LEFT)) w = lj_buf_wmem(w, s, len);
267 sb->w = w;
268 return sb;
269}
270
271#if LJ_HASJIT
272SBuf *lj_strfmt_putfstr(SBuf *sb, SFormat sf, GCstr *str)
273{
274 return strfmt_putfstrlen(sb, sf, strdata(str), str->len);
275}
276#endif
277
278/* Add formatted signed/unsigned integer to buffer. */
279SBuf *lj_strfmt_putfxint(SBuf *sb, SFormat sf, uint64_t k)
280{
281 char buf[STRFMT_MAXBUF_XINT], *q = buf + sizeof(buf), *w;
282#ifdef LUA_USE_ASSERT
283 char *ws;
284#endif
285 MSize prefix = 0, len, prec, pprec, width, need;
286
287 /* Figure out signed prefixes. */
288 if (STRFMT_TYPE(sf) == STRFMT_INT) {
289 if ((int64_t)k < 0) {
290 k = ~k+1u;
291 prefix = 256 + '-';
292 } else if ((sf & STRFMT_F_PLUS)) {
293 prefix = 256 + '+';
294 } else if ((sf & STRFMT_F_SPACE)) {
295 prefix = 256 + ' ';
296 }
297 }
298
299 /* Convert number and store to fixed-size buffer in reverse order. */
300 prec = STRFMT_PREC(sf);
301 if ((int32_t)prec >= 0) sf &= ~STRFMT_F_ZERO;
302 if (k == 0) { /* Special-case zero argument. */
303 if (prec != 0 ||
304 (sf & (STRFMT_T_OCT|STRFMT_F_ALT)) == (STRFMT_T_OCT|STRFMT_F_ALT))
305 *--q = '0';
306 } else if (!(sf & (STRFMT_T_HEX|STRFMT_T_OCT))) { /* Decimal. */
307 uint32_t k2;
308 while ((k >> 32)) { *--q = (char)('0' + k % 10); k /= 10; }
309 k2 = (uint32_t)k;
310 do { *--q = (char)('0' + k2 % 10); k2 /= 10; } while (k2);
311 } else if ((sf & STRFMT_T_HEX)) { /* Hex. */
312 const char *hexdig = (sf & STRFMT_F_UPPER) ? "0123456789ABCDEF" :
313 "0123456789abcdef";
314 do { *--q = hexdig[(k & 15)]; k >>= 4; } while (k);
315 if ((sf & STRFMT_F_ALT)) prefix = 512 + ((sf & STRFMT_F_UPPER) ? 'X' : 'x');
316 } else { /* Octal. */
317 do { *--q = (char)('0' + (uint32_t)(k & 7)); k >>= 3; } while (k);
318 if ((sf & STRFMT_F_ALT)) *--q = '0';
319 }
320
321 /* Calculate sizes. */
322 len = (MSize)(buf + sizeof(buf) - q);
323 if ((int32_t)len >= (int32_t)prec) prec = len;
324 width = STRFMT_WIDTH(sf);
325 pprec = prec + (prefix >> 8);
326 need = width > pprec ? width : pprec;
327 w = lj_buf_more(sb, need);
328#ifdef LUA_USE_ASSERT
329 ws = w;
330#endif
331
332 /* Format number with leading/trailing whitespace and zeros. */
333 if ((sf & (STRFMT_F_LEFT|STRFMT_F_ZERO)) == 0)
334 while (width-- > pprec) *w++ = ' ';
335 if (prefix) {
336 if ((char)prefix >= 'X') *w++ = '0';
337 *w++ = (char)prefix;
338 }
339 if ((sf & (STRFMT_F_LEFT|STRFMT_F_ZERO)) == STRFMT_F_ZERO)
340 while (width-- > pprec) *w++ = '0';
341 while (prec-- > len) *w++ = '0';
342 while (q < buf + sizeof(buf)) *w++ = *q++; /* Add number itself. */
343 if ((sf & STRFMT_F_LEFT))
344 while (width-- > pprec) *w++ = ' ';
345
346 lj_assertX(need == (MSize)(w - ws), "miscalculated format size");
347 sb->w = w;
348 return sb;
349}
350
351/* Add number formatted as signed integer to buffer. */
352SBuf *lj_strfmt_putfnum_int(SBuf *sb, SFormat sf, lua_Number n)
353{
354 int64_t k = (int64_t)n;
355 if (checki32(k) && sf == STRFMT_INT)
356 return lj_strfmt_putint(sb, (int32_t)k); /* Shortcut for plain %d. */
357 else
358 return lj_strfmt_putfxint(sb, sf, (uint64_t)k);
359}
360
361/* Add number formatted as unsigned integer to buffer. */
362SBuf *lj_strfmt_putfnum_uint(SBuf *sb, SFormat sf, lua_Number n)
363{
364 int64_t k;
365 if (n >= 9223372036854775808.0)
366 k = (int64_t)(n - 18446744073709551616.0);
367 else
368 k = (int64_t)n;
369 return lj_strfmt_putfxint(sb, sf, (uint64_t)k);
370}
371
372/* Format stack arguments to buffer. */
373int lj_strfmt_putarg(lua_State *L, SBuf *sb, int arg, int retry)
374{
375 int narg = (int)(L->top - L->base);
376 GCstr *fmt = lj_lib_checkstr(L, arg);
377 FormatState fs;
378 SFormat sf;
379 lj_strfmt_init(&fs, strdata(fmt), fmt->len);
380 while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) {
381 if (sf == STRFMT_LIT) {
382 lj_buf_putmem(sb, fs.str, fs.len);
383 } else if (sf == STRFMT_ERR) {
384 lj_err_callerv(L, LJ_ERR_STRFMT,
385 strdata(lj_str_new(L, fs.str, fs.len)));
386 } else {
387 TValue *o = &L->base[arg++];
388 if (arg > narg)
389 lj_err_arg(L, arg, LJ_ERR_NOVAL);
390 switch (STRFMT_TYPE(sf)) {
391 case STRFMT_INT:
392 if (tvisint(o)) {
393 int32_t k = intV(o);
394 if (sf == STRFMT_INT)
395 lj_strfmt_putint(sb, k); /* Shortcut for plain %d. */
396 else
397 lj_strfmt_putfxint(sb, sf, k);
398 break;
399 }
400#if LJ_HASFFI
401 if (tviscdata(o)) {
402 GCcdata *cd = cdataV(o);
403 if (cd->ctypeid == CTID_INT64 || cd->ctypeid == CTID_UINT64) {
404 lj_strfmt_putfxint(sb, sf, *(uint64_t *)cdataptr(cd));
405 break;
406 }
407 }
408#endif
409 lj_strfmt_putfnum_int(sb, sf, lj_lib_checknum(L, arg));
410 break;
411 case STRFMT_UINT:
412 if (tvisint(o)) {
413 lj_strfmt_putfxint(sb, sf, intV(o));
414 break;
415 }
416#if LJ_HASFFI
417 if (tviscdata(o)) {
418 GCcdata *cd = cdataV(o);
419 if (cd->ctypeid == CTID_INT64 || cd->ctypeid == CTID_UINT64) {
420 lj_strfmt_putfxint(sb, sf, *(uint64_t *)cdataptr(cd));
421 break;
422 }
423 }
424#endif
425 lj_strfmt_putfnum_uint(sb, sf, lj_lib_checknum(L, arg));
426 break;
427 case STRFMT_NUM:
428 lj_strfmt_putfnum(sb, sf, lj_lib_checknum(L, arg));
429 break;
430 case STRFMT_STR: {
431 MSize len;
432 const char *s;
433 cTValue *mo;
434 if (LJ_UNLIKELY(!tvisstr(o) && !tvisbuf(o)) && retry >= 0 &&
435 !tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
436 /* Call __tostring metamethod once. */
437 copyTV(L, L->top++, mo);
438 copyTV(L, L->top++, o);
439 lua_call(L, 1, 1);
440 o = &L->base[arg-1]; /* Stack may have been reallocated. */
441 copyTV(L, o, --L->top); /* Replace inline for retry. */
442 if (retry < 2) { /* Global buffer may have been overwritten. */
443 retry = 1;
444 break;
445 }
446 }
447 if (LJ_LIKELY(tvisstr(o))) {
448 len = strV(o)->len;
449 s = strVdata(o);
450#if LJ_HASBUFFER
451 } else if (tvisbuf(o)) {
452 SBufExt *sbx = bufV(o);
453 if (sbx == (SBufExt *)sb) lj_err_arg(L, arg+1, LJ_ERR_BUFFER_SELF);
454 len = sbufxlen(sbx);
455 s = sbx->r;
456#endif
457 } else {
458 GCstr *str = lj_strfmt_obj(L, o);
459 len = str->len;
460 s = strdata(str);
461 }
462 if ((sf & STRFMT_T_QUOTED))
463 strfmt_putquotedlen(sb, s, len); /* No formatting. */
464 else
465 strfmt_putfstrlen(sb, sf, s, len);
466 break;
467 }
468 case STRFMT_CHAR:
469 lj_strfmt_putfchar(sb, sf, lj_lib_checkint(L, arg));
470 break;
471 case STRFMT_PTR: /* No formatting. */
472 lj_strfmt_putptr(sb, lj_obj_ptr(G(L), o));
473 break;
474 default:
475 lj_assertL(0, "bad string format type");
476 break;
477 }
478 }
479 }
480 return retry;
481}
482
483/* -- Conversions to strings ---------------------------------------------- */
484
485/* Convert integer to string. */
486GCstr * LJ_FASTCALL lj_strfmt_int(lua_State *L, int32_t k)
487{
488 char buf[STRFMT_MAXBUF_INT];
489 MSize len = (MSize)(lj_strfmt_wint(buf, k) - buf);
490 return lj_str_new(L, buf, len);
491}
492
493/* Convert integer or number to string. */
494GCstr * LJ_FASTCALL lj_strfmt_number(lua_State *L, cTValue *o)
495{
496 return tvisint(o) ? lj_strfmt_int(L, intV(o)) : lj_strfmt_num(L, o);
497}
498
499#if LJ_HASJIT
500/* Convert char value to string. */
501GCstr * LJ_FASTCALL lj_strfmt_char(lua_State *L, int c)
502{
503 char buf[1];
504 buf[0] = c;
505 return lj_str_new(L, buf, 1);
506}
507#endif
508
509/* Raw conversion of object to string. */
510GCstr * LJ_FASTCALL lj_strfmt_obj(lua_State *L, cTValue *o)
511{
512 if (tvisstr(o)) {
513 return strV(o);
514 } else if (tvisnumber(o)) {
515 return lj_strfmt_number(L, o);
516 } else if (tvisnil(o)) {
517 return lj_str_newlit(L, "nil");
518 } else if (tvisfalse(o)) {
519 return lj_str_newlit(L, "false");
520 } else if (tvistrue(o)) {
521 return lj_str_newlit(L, "true");
522 } else {
523 char buf[8+2+2+16], *p = buf;
524 p = lj_buf_wmem(p, lj_typename(o), (MSize)strlen(lj_typename(o)));
525 *p++ = ':'; *p++ = ' ';
526 if (tvisfunc(o) && isffunc(funcV(o))) {
527 p = lj_buf_wmem(p, "builtin#", 8);
528 p = lj_strfmt_wint(p, funcV(o)->c.ffid);
529 } else {
530 p = lj_strfmt_wptr(p, lj_obj_ptr(G(L), o));
531 }
532 return lj_str_new(L, buf, (size_t)(p - buf));
533 }
534}
535
536/* -- Internal string formatting ------------------------------------------ */
537
538/*
539** These functions are only used for lua_pushfstring(), lua_pushvfstring()
540** and for internal string formatting (e.g. error messages). Caveat: unlike
541** string.format(), only a limited subset of formats and flags are supported!
542**
543** LuaJIT has support for a couple more formats than Lua 5.1/5.2:
544** - %d %u %o %x with full formatting, 32 bit integers only.
545** - %f and other FP formats are really %.14g.
546** - %s %c %p without formatting.
547*/
548
549/* Push formatted message as a string object to Lua stack. va_list variant. */
550const char *lj_strfmt_pushvf(lua_State *L, const char *fmt, va_list argp)
551{
552 SBuf *sb = lj_buf_tmp_(L);
553 FormatState fs;
554 SFormat sf;
555 GCstr *str;
556 lj_strfmt_init(&fs, fmt, (MSize)strlen(fmt));
557 while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) {
558 switch (STRFMT_TYPE(sf)) {
559 case STRFMT_LIT:
560 lj_buf_putmem(sb, fs.str, fs.len);
561 break;
562 case STRFMT_INT:
563 lj_strfmt_putfxint(sb, sf, va_arg(argp, int32_t));
564 break;
565 case STRFMT_UINT:
566 lj_strfmt_putfxint(sb, sf, va_arg(argp, uint32_t));
567 break;
568 case STRFMT_NUM:
569 lj_strfmt_putfnum(sb, STRFMT_G14, va_arg(argp, lua_Number));
570 break;
571 case STRFMT_STR: {
572 const char *s = va_arg(argp, char *);
573 if (s == NULL) s = "(null)";
574 lj_buf_putmem(sb, s, (MSize)strlen(s));
575 break;
576 }
577 case STRFMT_CHAR:
578 lj_buf_putb(sb, va_arg(argp, int));
579 break;
580 case STRFMT_PTR:
581 lj_strfmt_putptr(sb, va_arg(argp, void *));
582 break;
583 case STRFMT_ERR:
584 default:
585 lj_buf_putb(sb, '?');
586 lj_assertL(0, "bad string format near offset %d", fs.len);
587 break;
588 }
589 }
590 str = lj_buf_str(L, sb);
591 setstrV(L, L->top, str);
592 incr_top(L);
593 return strdata(str);
594}
595
596/* Push formatted message as a string object to Lua stack. Vararg variant. */
597const char *lj_strfmt_pushf(lua_State *L, const char *fmt, ...)
598{
599 const char *msg;
600 va_list argp;
601 va_start(argp, fmt);
602 msg = lj_strfmt_pushvf(L, fmt, argp);
603 va_end(argp);
604 return msg;
605}
606
diff --git a/src/lj_strfmt.h b/src/lj_strfmt.h
new file mode 100644
index 00000000..bd17896e
--- /dev/null
+++ b/src/lj_strfmt.h
@@ -0,0 +1,131 @@
1/*
2** String formatting.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_STRFMT_H
7#define _LJ_STRFMT_H
8
9#include "lj_obj.h"
10
11typedef uint32_t SFormat; /* Format indicator. */
12
13/* Format parser state. */
14typedef struct FormatState {
15 const uint8_t *p; /* Current format string pointer. */
16 const uint8_t *e; /* End of format string. */
17 const char *str; /* Returned literal string. */
18 MSize len; /* Size of literal string. */
19} FormatState;
20
21/* Format types (max. 16). */
22typedef enum FormatType {
23 STRFMT_EOF, STRFMT_ERR, STRFMT_LIT,
24 STRFMT_INT, STRFMT_UINT, STRFMT_NUM, STRFMT_STR, STRFMT_CHAR, STRFMT_PTR
25} FormatType;
26
27/* Format subtypes (bits are reused). */
28#define STRFMT_T_HEX 0x0010 /* STRFMT_UINT */
29#define STRFMT_T_OCT 0x0020 /* STRFMT_UINT */
30#define STRFMT_T_FP_A 0x0000 /* STRFMT_NUM */
31#define STRFMT_T_FP_E 0x0010 /* STRFMT_NUM */
32#define STRFMT_T_FP_F 0x0020 /* STRFMT_NUM */
33#define STRFMT_T_FP_G 0x0030 /* STRFMT_NUM */
34#define STRFMT_T_QUOTED 0x0010 /* STRFMT_STR */
35
36/* Format flags. */
37#define STRFMT_F_LEFT 0x0100
38#define STRFMT_F_PLUS 0x0200
39#define STRFMT_F_ZERO 0x0400
40#define STRFMT_F_SPACE 0x0800
41#define STRFMT_F_ALT 0x1000
42#define STRFMT_F_UPPER 0x2000
43
44/* Format indicator fields. */
45#define STRFMT_SH_WIDTH 16
46#define STRFMT_SH_PREC 24
47
48#define STRFMT_TYPE(sf) ((FormatType)((sf) & 15))
49#define STRFMT_WIDTH(sf) (((sf) >> STRFMT_SH_WIDTH) & 255u)
50#define STRFMT_PREC(sf) ((((sf) >> STRFMT_SH_PREC) & 255u) - 1u)
51#define STRFMT_FP(sf) (((sf) >> 4) & 3)
52
53/* Formats for conversion characters. */
54#define STRFMT_A (STRFMT_NUM|STRFMT_T_FP_A)
55#define STRFMT_C (STRFMT_CHAR)
56#define STRFMT_D (STRFMT_INT)
57#define STRFMT_E (STRFMT_NUM|STRFMT_T_FP_E)
58#define STRFMT_F (STRFMT_NUM|STRFMT_T_FP_F)
59#define STRFMT_G (STRFMT_NUM|STRFMT_T_FP_G)
60#define STRFMT_I STRFMT_D
61#define STRFMT_O (STRFMT_UINT|STRFMT_T_OCT)
62#define STRFMT_P (STRFMT_PTR)
63#define STRFMT_Q (STRFMT_STR|STRFMT_T_QUOTED)
64#define STRFMT_S (STRFMT_STR)
65#define STRFMT_U (STRFMT_UINT)
66#define STRFMT_X (STRFMT_UINT|STRFMT_T_HEX)
67#define STRFMT_G14 (STRFMT_G | ((14+1) << STRFMT_SH_PREC))
68
69/* Maximum buffer sizes for conversions. */
70#define STRFMT_MAXBUF_XINT (1+22) /* '0' prefix + uint64_t in octal. */
71#define STRFMT_MAXBUF_INT (1+10) /* Sign + int32_t in decimal. */
72#define STRFMT_MAXBUF_NUM 32 /* Must correspond with STRFMT_G14. */
73#define STRFMT_MAXBUF_PTR (2+2*sizeof(ptrdiff_t)) /* "0x" + hex ptr. */
74
75/* Format parser. */
76LJ_FUNC SFormat LJ_FASTCALL lj_strfmt_parse(FormatState *fs);
77
78static LJ_AINLINE void lj_strfmt_init(FormatState *fs, const char *p, MSize len)
79{
80 fs->p = (const uint8_t *)p;
81 fs->e = (const uint8_t *)p + len;
82 /* Must be NUL-terminated. May have NULs inside, too. */
83 lj_assertX(*fs->e == 0, "format not NUL-terminated");
84}
85
86/* Raw conversions. */
87LJ_FUNC char * LJ_FASTCALL lj_strfmt_wint(char *p, int32_t k);
88LJ_FUNC char * LJ_FASTCALL lj_strfmt_wptr(char *p, const void *v);
89LJ_FUNC char * LJ_FASTCALL lj_strfmt_wuleb128(char *p, uint32_t v);
90LJ_FUNC const char *lj_strfmt_wstrnum(lua_State *L, cTValue *o, MSize *lenp);
91
92/* Unformatted conversions to buffer. */
93LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putint(SBuf *sb, int32_t k);
94#if LJ_HASJIT
95LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putnum(SBuf *sb, cTValue *o);
96#endif
97LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putptr(SBuf *sb, const void *v);
98#if LJ_HASJIT
99LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putquoted(SBuf *sb, GCstr *str);
100#endif
101
102/* Formatted conversions to buffer. */
103LJ_FUNC SBuf *lj_strfmt_putfxint(SBuf *sb, SFormat sf, uint64_t k);
104LJ_FUNC SBuf *lj_strfmt_putfnum_int(SBuf *sb, SFormat sf, lua_Number n);
105LJ_FUNC SBuf *lj_strfmt_putfnum_uint(SBuf *sb, SFormat sf, lua_Number n);
106LJ_FUNC SBuf *lj_strfmt_putfnum(SBuf *sb, SFormat, lua_Number n);
107LJ_FUNC SBuf *lj_strfmt_putfchar(SBuf *sb, SFormat, int32_t c);
108#if LJ_HASJIT
109LJ_FUNC SBuf *lj_strfmt_putfstr(SBuf *sb, SFormat, GCstr *str);
110#endif
111LJ_FUNC int lj_strfmt_putarg(lua_State *L, SBuf *sb, int arg, int retry);
112
113/* Conversions to strings. */
114LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_int(lua_State *L, int32_t k);
115LJ_FUNCA GCstr * LJ_FASTCALL lj_strfmt_num(lua_State *L, cTValue *o);
116LJ_FUNCA GCstr * LJ_FASTCALL lj_strfmt_number(lua_State *L, cTValue *o);
117#if LJ_HASJIT
118LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_char(lua_State *L, int c);
119#endif
120LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_obj(lua_State *L, cTValue *o);
121
122/* Internal string formatting. */
123LJ_FUNC const char *lj_strfmt_pushvf(lua_State *L, const char *fmt,
124 va_list argp);
125LJ_FUNC const char *lj_strfmt_pushf(lua_State *L, const char *fmt, ...)
126#if defined(__GNUC__) || defined(__clang__)
127 __attribute__ ((format (printf, 2, 3)))
128#endif
129 ;
130
131#endif
diff --git a/src/lj_strfmt_num.c b/src/lj_strfmt_num.c
new file mode 100644
index 00000000..79ec0263
--- /dev/null
+++ b/src/lj_strfmt_num.c
@@ -0,0 +1,592 @@
1/*
2** String formatting for floating-point numbers.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4** Contributed by Peter Cawley.
5*/
6
7#include <stdio.h>
8
9#define lj_strfmt_num_c
10#define LUA_CORE
11
12#include "lj_obj.h"
13#include "lj_buf.h"
14#include "lj_str.h"
15#include "lj_strfmt.h"
16
17/* -- Precomputed tables -------------------------------------------------- */
18
19/* Rescale factors to push the exponent of a number towards zero. */
20#define RESCALE_EXPONENTS(P, N) \
21 P(308), P(289), P(270), P(250), P(231), P(212), P(193), P(173), P(154), \
22 P(135), P(115), P(96), P(77), P(58), P(38), P(0), P(0), P(0), N(39), N(58), \
23 N(77), N(96), N(116), N(135), N(154), N(174), N(193), N(212), N(231), \
24 N(251), N(270), N(289)
25
26#define ONE_E_P(X) 1e+0 ## X
27#define ONE_E_N(X) 1e-0 ## X
28static const int16_t rescale_e[] = { RESCALE_EXPONENTS(-, +) };
29static const double rescale_n[] = { RESCALE_EXPONENTS(ONE_E_P, ONE_E_N) };
30#undef ONE_E_N
31#undef ONE_E_P
32
33/*
34** For p in range -70 through 57, this table encodes pairs (m, e) such that
35** 4*2^p <= (uint8_t)m*10^e, and is the smallest value for which this holds.
36*/
37static const int8_t four_ulp_m_e[] = {
38 34, -21, 68, -21, 14, -20, 28, -20, 55, -20, 2, -19, 3, -19, 5, -19, 9, -19,
39 -82, -18, 35, -18, 7, -17, -117, -17, 28, -17, 56, -17, 112, -16, -33, -16,
40 45, -16, 89, -16, -78, -15, 36, -15, 72, -15, -113, -14, 29, -14, 57, -14,
41 114, -13, -28, -13, 46, -13, 91, -12, -74, -12, 37, -12, 73, -12, 15, -11, 3,
42 -11, 59, -11, 2, -10, 3, -10, 5, -10, 1, -9, -69, -9, 38, -9, 75, -9, 15, -7,
43 3, -7, 6, -7, 12, -6, -17, -7, 48, -7, 96, -7, -65, -6, 39, -6, 77, -6, -103,
44 -5, 31, -5, 62, -5, 123, -4, -11, -4, 49, -4, 98, -4, -60, -3, 4, -2, 79, -3,
45 16, -2, 32, -2, 63, -2, 2, -1, 25, 0, 5, 1, 1, 2, 2, 2, 4, 2, 8, 2, 16, 2,
46 32, 2, 64, 2, -128, 2, 26, 2, 52, 2, 103, 3, -51, 3, 41, 4, 82, 4, -92, 4,
47 33, 4, 66, 4, -124, 5, 27, 5, 53, 5, 105, 6, 21, 6, 42, 6, 84, 6, 17, 7, 34,
48 7, 68, 7, 2, 8, 3, 8, 6, 8, 108, 9, -41, 9, 43, 10, 86, 9, -84, 10, 35, 10,
49 69, 10, -118, 11, 28, 11, 55, 12, 11, 13, 22, 13, 44, 13, 88, 13, -80, 13,
50 36, 13, 71, 13, -115, 14, 29, 14, 57, 14, 113, 15, -30, 15, 46, 15, 91, 15,
51 19, 16, 37, 16, 73, 16, 2, 17, 3, 17, 6, 17
52};
53
54/* min(2^32-1, 10^e-1) for e in range 0 through 10 */
55static uint32_t ndigits_dec_threshold[] = {
56 0, 9U, 99U, 999U, 9999U, 99999U, 999999U,
57 9999999U, 99999999U, 999999999U, 0xffffffffU
58};
59
60/* -- Helper functions ---------------------------------------------------- */
61
62/* Compute the number of digits in the decimal representation of x. */
63static MSize ndigits_dec(uint32_t x)
64{
65 MSize t = ((lj_fls(x | 1) * 77) >> 8) + 1; /* 2^8/77 is roughly log2(10) */
66 return t + (x > ndigits_dec_threshold[t]);
67}
68
69#define WINT_R(x, sh, sc) \
70 { uint32_t d = (x*(((1<<sh)+sc-1)/sc))>>sh; x -= d*sc; *p++ = (char)('0'+d); }
71
72/* Write 9-digit unsigned integer to buffer. */
73static char *lj_strfmt_wuint9(char *p, uint32_t u)
74{
75 uint32_t v = u / 10000, w;
76 u -= v * 10000;
77 w = v / 10000;
78 v -= w * 10000;
79 *p++ = (char)('0'+w);
80 WINT_R(v, 23, 1000)
81 WINT_R(v, 12, 100)
82 WINT_R(v, 10, 10)
83 *p++ = (char)('0'+v);
84 WINT_R(u, 23, 1000)
85 WINT_R(u, 12, 100)
86 WINT_R(u, 10, 10)
87 *p++ = (char)('0'+u);
88 return p;
89}
90#undef WINT_R
91
92/* -- Extended precision arithmetic --------------------------------------- */
93
94/*
95** The "nd" format is a fixed-precision decimal representation for numbers. It
96** consists of up to 64 uint32_t values, with each uint32_t storing a value
97** in the range [0, 1e9). A number in "nd" format consists of three variables:
98**
99** uint32_t nd[64];
100** uint32_t ndlo;
101** uint32_t ndhi;
102**
103** The integral part of the number is stored in nd[0 ... ndhi], the value of
104** which is sum{i in [0, ndhi] | nd[i] * 10^(9*i)}. If the fractional part of
105** the number is zero, ndlo is zero. Otherwise, the fractional part is stored
106** in nd[ndlo ... 63], the value of which is taken to be
107** sum{i in [ndlo, 63] | nd[i] * 10^(9*(i-64))}.
108**
109** If the array part had 128 elements rather than 64, then every double would
110** have an exact representation in "nd" format. With 64 elements, all integral
111** doubles have an exact representation, and all non-integral doubles have
112** enough digits to make both %.99e and %.99f do the right thing.
113*/
114
115#if LJ_64
116#define ND_MUL2K_MAX_SHIFT 29
117#define ND_MUL2K_DIV1E9(val) ((uint32_t)((val) / 1000000000))
118#else
119#define ND_MUL2K_MAX_SHIFT 11
120#define ND_MUL2K_DIV1E9(val) ((uint32_t)((val) >> 9) / 1953125)
121#endif
122
123/* Multiply nd by 2^k and add carry_in (ndlo is assumed to be zero). */
124static uint32_t nd_mul2k(uint32_t* nd, uint32_t ndhi, uint32_t k,
125 uint32_t carry_in, SFormat sf)
126{
127 uint32_t i, ndlo = 0, start = 1;
128 /* Performance hacks. */
129 if (k > ND_MUL2K_MAX_SHIFT*2 && STRFMT_FP(sf) != STRFMT_FP(STRFMT_T_FP_F)) {
130 start = ndhi - (STRFMT_PREC(sf) + 17) / 8;
131 }
132 /* Real logic. */
133 while (k >= ND_MUL2K_MAX_SHIFT) {
134 for (i = ndlo; i <= ndhi; i++) {
135 uint64_t val = ((uint64_t)nd[i] << ND_MUL2K_MAX_SHIFT) | carry_in;
136 carry_in = ND_MUL2K_DIV1E9(val);
137 nd[i] = (uint32_t)val - carry_in * 1000000000;
138 }
139 if (carry_in) {
140 nd[++ndhi] = carry_in; carry_in = 0;
141 if (start++ == ndlo) ++ndlo;
142 }
143 k -= ND_MUL2K_MAX_SHIFT;
144 }
145 if (k) {
146 for (i = ndlo; i <= ndhi; i++) {
147 uint64_t val = ((uint64_t)nd[i] << k) | carry_in;
148 carry_in = ND_MUL2K_DIV1E9(val);
149 nd[i] = (uint32_t)val - carry_in * 1000000000;
150 }
151 if (carry_in) nd[++ndhi] = carry_in;
152 }
153 return ndhi;
154}
155
156/* Divide nd by 2^k (ndlo is assumed to be zero). */
157static uint32_t nd_div2k(uint32_t* nd, uint32_t ndhi, uint32_t k, SFormat sf)
158{
159 uint32_t ndlo = 0, stop1 = ~0, stop2 = ~0;
160 /* Performance hacks. */
161 if (!ndhi) {
162 if (!nd[0]) {
163 return 0;
164 } else {
165 uint32_t s = lj_ffs(nd[0]);
166 if (s >= k) { nd[0] >>= k; return 0; }
167 nd[0] >>= s; k -= s;
168 }
169 }
170 if (k > 18) {
171 if (STRFMT_FP(sf) == STRFMT_FP(STRFMT_T_FP_F)) {
172 stop1 = 63 - (int32_t)STRFMT_PREC(sf) / 9;
173 } else {
174 int32_t floorlog2 = ndhi * 29 + lj_fls(nd[ndhi]) - k;
175 int32_t floorlog10 = (int32_t)(floorlog2 * 0.30102999566398114);
176 stop1 = 62 + (floorlog10 - (int32_t)STRFMT_PREC(sf)) / 9;
177 stop2 = 61 + ndhi - (int32_t)STRFMT_PREC(sf) / 8;
178 }
179 }
180 /* Real logic. */
181 while (k >= 9) {
182 uint32_t i = ndhi, carry = 0;
183 for (;;) {
184 uint32_t val = nd[i];
185 nd[i] = (val >> 9) + carry;
186 carry = (val & 0x1ff) * 1953125;
187 if (i == ndlo) break;
188 i = (i - 1) & 0x3f;
189 }
190 if (ndlo != stop1 && ndlo != stop2) {
191 if (carry) { ndlo = (ndlo - 1) & 0x3f; nd[ndlo] = carry; }
192 if (!nd[ndhi]) { ndhi = (ndhi - 1) & 0x3f; stop2--; }
193 } else if (!nd[ndhi]) {
194 if (ndhi != ndlo) { ndhi = (ndhi - 1) & 0x3f; stop2--; }
195 else return ndlo;
196 }
197 k -= 9;
198 }
199 if (k) {
200 uint32_t mask = (1U << k) - 1, mul = 1000000000 >> k, i = ndhi, carry = 0;
201 for (;;) {
202 uint32_t val = nd[i];
203 nd[i] = (val >> k) + carry;
204 carry = (val & mask) * mul;
205 if (i == ndlo) break;
206 i = (i - 1) & 0x3f;
207 }
208 if (carry) { ndlo = (ndlo - 1) & 0x3f; nd[ndlo] = carry; }
209 }
210 return ndlo;
211}
212
213/* Add m*10^e to nd (assumes ndlo <= e/9 <= ndhi and 0 <= m <= 9). */
214static uint32_t nd_add_m10e(uint32_t* nd, uint32_t ndhi, uint8_t m, int32_t e)
215{
216 uint32_t i, carry;
217 if (e >= 0) {
218 i = (uint32_t)e/9;
219 carry = m * (ndigits_dec_threshold[e - (int32_t)i*9] + 1);
220 } else {
221 int32_t f = (e-8)/9;
222 i = (uint32_t)(64 + f);
223 carry = m * (ndigits_dec_threshold[e - f*9] + 1);
224 }
225 for (;;) {
226 uint32_t val = nd[i] + carry;
227 if (LJ_UNLIKELY(val >= 1000000000)) {
228 val -= 1000000000;
229 nd[i] = val;
230 if (LJ_UNLIKELY(i == ndhi)) {
231 ndhi = (ndhi + 1) & 0x3f;
232 nd[ndhi] = 1;
233 break;
234 }
235 carry = 1;
236 i = (i + 1) & 0x3f;
237 } else {
238 nd[i] = val;
239 break;
240 }
241 }
242 return ndhi;
243}
244
245/* Test whether two "nd" values are equal in their most significant digits. */
246static int nd_similar(uint32_t* nd, uint32_t ndhi, uint32_t* ref, MSize hilen,
247 MSize prec)
248{
249 char nd9[9], ref9[9];
250 if (hilen <= prec) {
251 if (LJ_UNLIKELY(nd[ndhi] != *ref)) return 0;
252 prec -= hilen; ref--; ndhi = (ndhi - 1) & 0x3f;
253 if (prec >= 9) {
254 if (LJ_UNLIKELY(nd[ndhi] != *ref)) return 0;
255 prec -= 9; ref--; ndhi = (ndhi - 1) & 0x3f;
256 }
257 } else {
258 prec -= hilen - 9;
259 }
260 lj_assertX(prec < 9, "bad precision %d", prec);
261 lj_strfmt_wuint9(nd9, nd[ndhi]);
262 lj_strfmt_wuint9(ref9, *ref);
263 return !memcmp(nd9, ref9, prec) && (nd9[prec] < '5') == (ref9[prec] < '5');
264}
265
266/* -- Formatted conversions to buffer ------------------------------------- */
267
268/* Write formatted floating-point number to either sb or p. */
269static char *lj_strfmt_wfnum(SBuf *sb, SFormat sf, lua_Number n, char *p)
270{
271 MSize width = STRFMT_WIDTH(sf), prec = STRFMT_PREC(sf), len;
272 TValue t;
273 t.n = n;
274 if (LJ_UNLIKELY((t.u32.hi << 1) >= 0xffe00000)) {
275 /* Handle non-finite values uniformly for %a, %e, %f, %g. */
276 int prefix = 0, ch = (sf & STRFMT_F_UPPER) ? 0x202020 : 0;
277 if (((t.u32.hi & 0x000fffff) | t.u32.lo) != 0) {
278 ch ^= ('n' << 16) | ('a' << 8) | 'n';
279 if ((sf & STRFMT_F_SPACE)) prefix = ' ';
280 } else {
281 ch ^= ('i' << 16) | ('n' << 8) | 'f';
282 if ((t.u32.hi & 0x80000000)) prefix = '-';
283 else if ((sf & STRFMT_F_PLUS)) prefix = '+';
284 else if ((sf & STRFMT_F_SPACE)) prefix = ' ';
285 }
286 len = 3 + (prefix != 0);
287 if (!p) p = lj_buf_more(sb, width > len ? width : len);
288 if (!(sf & STRFMT_F_LEFT)) while (width-- > len) *p++ = ' ';
289 if (prefix) *p++ = prefix;
290 *p++ = (char)(ch >> 16); *p++ = (char)(ch >> 8); *p++ = (char)ch;
291 } else if (STRFMT_FP(sf) == STRFMT_FP(STRFMT_T_FP_A)) {
292 /* %a */
293 const char *hexdig = (sf & STRFMT_F_UPPER) ? "0123456789ABCDEFPX"
294 : "0123456789abcdefpx";
295 int32_t e = (t.u32.hi >> 20) & 0x7ff;
296 char prefix = 0, eprefix = '+';
297 if (t.u32.hi & 0x80000000) prefix = '-';
298 else if ((sf & STRFMT_F_PLUS)) prefix = '+';
299 else if ((sf & STRFMT_F_SPACE)) prefix = ' ';
300 t.u32.hi &= 0xfffff;
301 if (e) {
302 t.u32.hi |= 0x100000;
303 e -= 1023;
304 } else if (t.u32.lo | t.u32.hi) {
305 /* Non-zero denormal - normalise it. */
306 uint32_t shift = t.u32.hi ? 20-lj_fls(t.u32.hi) : 52-lj_fls(t.u32.lo);
307 e = -1022 - shift;
308 t.u64 <<= shift;
309 }
310 /* abs(n) == t.u64 * 2^(e - 52) */
311 /* If n != 0, bit 52 of t.u64 is set, and is the highest set bit. */
312 if ((int32_t)prec < 0) {
313 /* Default precision: use smallest precision giving exact result. */
314 prec = t.u32.lo ? 13-lj_ffs(t.u32.lo)/4 : 5-lj_ffs(t.u32.hi|0x100000)/4;
315 } else if (prec < 13) {
316 /* Precision is sufficiently low as to maybe require rounding. */
317 t.u64 += (((uint64_t)1) << (51 - prec*4));
318 }
319 if (e < 0) {
320 eprefix = '-';
321 e = -e;
322 }
323 len = 5 + ndigits_dec((uint32_t)e) + prec + (prefix != 0)
324 + ((prec | (sf & STRFMT_F_ALT)) != 0);
325 if (!p) p = lj_buf_more(sb, width > len ? width : len);
326 if (!(sf & (STRFMT_F_LEFT | STRFMT_F_ZERO))) {
327 while (width-- > len) *p++ = ' ';
328 }
329 if (prefix) *p++ = prefix;
330 *p++ = '0';
331 *p++ = hexdig[17]; /* x or X */
332 if ((sf & (STRFMT_F_LEFT | STRFMT_F_ZERO)) == STRFMT_F_ZERO) {
333 while (width-- > len) *p++ = '0';
334 }
335 *p++ = '0' + (t.u32.hi >> 20); /* Usually '1', sometimes '0' or '2'. */
336 if ((prec | (sf & STRFMT_F_ALT))) {
337 /* Emit fractional part. */
338 char *q = p + 1 + prec;
339 *p = '.';
340 if (prec < 13) t.u64 >>= (52 - prec*4);
341 else while (prec > 13) p[prec--] = '0';
342 while (prec) { p[prec--] = hexdig[t.u64 & 15]; t.u64 >>= 4; }
343 p = q;
344 }
345 *p++ = hexdig[16]; /* p or P */
346 *p++ = eprefix; /* + or - */
347 p = lj_strfmt_wint(p, e);
348 } else {
349 /* %e or %f or %g - begin by converting n to "nd" format. */
350 uint32_t nd[64];
351 uint32_t ndhi = 0, ndlo, i;
352 int32_t e = (t.u32.hi >> 20) & 0x7ff, ndebias = 0;
353 char prefix = 0, *q;
354 if (t.u32.hi & 0x80000000) prefix = '-';
355 else if ((sf & STRFMT_F_PLUS)) prefix = '+';
356 else if ((sf & STRFMT_F_SPACE)) prefix = ' ';
357 prec += ((int32_t)prec >> 31) & 7; /* Default precision is 6. */
358 if (STRFMT_FP(sf) == STRFMT_FP(STRFMT_T_FP_G)) {
359 /* %g - decrement precision if non-zero (to make it like %e). */
360 prec--;
361 prec ^= (uint32_t)((int32_t)prec >> 31);
362 }
363 if ((sf & STRFMT_T_FP_E) && prec < 14 && n != 0) {
364 /* Precision is sufficiently low that rescaling will probably work. */
365 if ((ndebias = rescale_e[e >> 6])) {
366 t.n = n * rescale_n[e >> 6];
367 if (LJ_UNLIKELY(!e)) t.n *= 1e10, ndebias -= 10;
368 t.u64 -= 2; /* Convert 2ulp below (later we convert 2ulp above). */
369 nd[0] = 0x100000 | (t.u32.hi & 0xfffff);
370 e = ((t.u32.hi >> 20) & 0x7ff) - 1075 - (ND_MUL2K_MAX_SHIFT < 29);
371 goto load_t_lo; rescale_failed:
372 t.n = n;
373 e = (t.u32.hi >> 20) & 0x7ff;
374 ndebias = ndhi = 0;
375 }
376 }
377 nd[0] = t.u32.hi & 0xfffff;
378 if (e == 0) e++; else nd[0] |= 0x100000;
379 e -= 1043;
380 if (t.u32.lo) {
381 e -= 32 + (ND_MUL2K_MAX_SHIFT < 29); load_t_lo:
382#if ND_MUL2K_MAX_SHIFT >= 29
383 nd[0] = (nd[0] << 3) | (t.u32.lo >> 29);
384 ndhi = nd_mul2k(nd, ndhi, 29, t.u32.lo & 0x1fffffff, sf);
385#elif ND_MUL2K_MAX_SHIFT >= 11
386 ndhi = nd_mul2k(nd, ndhi, 11, t.u32.lo >> 21, sf);
387 ndhi = nd_mul2k(nd, ndhi, 11, (t.u32.lo >> 10) & 0x7ff, sf);
388 ndhi = nd_mul2k(nd, ndhi, 11, (t.u32.lo << 1) & 0x7ff, sf);
389#else
390#error "ND_MUL2K_MAX_SHIFT too small"
391#endif
392 }
393 if (e >= 0) {
394 ndhi = nd_mul2k(nd, ndhi, (uint32_t)e, 0, sf);
395 ndlo = 0;
396 } else {
397 ndlo = nd_div2k(nd, ndhi, (uint32_t)-e, sf);
398 if (ndhi && !nd[ndhi]) ndhi--;
399 }
400 /* abs(n) == nd * 10^ndebias (for slightly loose interpretation of ==) */
401 if ((sf & STRFMT_T_FP_E)) {
402 /* %e or %g - assume %e and start by calculating nd's exponent (nde). */
403 char eprefix = '+';
404 int32_t nde = -1;
405 MSize hilen;
406 if (ndlo && !nd[ndhi]) {
407 ndhi = 64; do {} while (!nd[--ndhi]);
408 nde -= 64 * 9;
409 }
410 hilen = ndigits_dec(nd[ndhi]);
411 nde += ndhi * 9 + hilen;
412 if (ndebias) {
413 /*
414 ** Rescaling was performed, but this introduced some error, and might
415 ** have pushed us across a rounding boundary. We check whether this
416 ** error affected the result by introducing even more error (2ulp in
417 ** either direction), and seeing whether a rounding boundary was
418 ** crossed. Having already converted the -2ulp case, we save off its
419 ** most significant digits, convert the +2ulp case, and compare them.
420 */
421 int32_t eidx = e + 70 + (ND_MUL2K_MAX_SHIFT < 29)
422 + (t.u32.lo >= 0xfffffffe && !(~t.u32.hi << 12));
423 const int8_t *m_e = four_ulp_m_e + eidx * 2;
424 lj_assertG_(G(sbufL(sb)), 0 <= eidx && eidx < 128, "bad eidx %d", eidx);
425 nd[33] = nd[ndhi];
426 nd[32] = nd[(ndhi - 1) & 0x3f];
427 nd[31] = nd[(ndhi - 2) & 0x3f];
428 nd_add_m10e(nd, ndhi, (uint8_t)*m_e, m_e[1]);
429 if (LJ_UNLIKELY(!nd_similar(nd, ndhi, nd + 33, hilen, prec + 1))) {
430 goto rescale_failed;
431 }
432 }
433 if ((int32_t)(prec - nde) < (0x3f & -(int32_t)ndlo) * 9) {
434 /* Precision is sufficiently low as to maybe require rounding. */
435 ndhi = nd_add_m10e(nd, ndhi, 5, nde - prec - 1);
436 nde += (hilen != ndigits_dec(nd[ndhi]));
437 }
438 nde += ndebias;
439 if ((sf & STRFMT_T_FP_F)) {
440 /* %g */
441 if ((int32_t)prec >= nde && nde >= -4) {
442 if (nde < 0) ndhi = 0;
443 prec -= nde;
444 goto g_format_like_f;
445 } else if (!(sf & STRFMT_F_ALT) && prec && width > 5) {
446 /* Decrease precision in order to strip trailing zeroes. */
447 char tail[9];
448 uint32_t maxprec = hilen - 1 + ((ndhi - ndlo) & 0x3f) * 9;
449 if (prec >= maxprec) prec = maxprec;
450 else ndlo = (ndhi - (((int32_t)(prec - hilen) + 9) / 9)) & 0x3f;
451 i = prec - hilen - (((ndhi - ndlo) & 0x3f) * 9) + 10;
452 lj_strfmt_wuint9(tail, nd[ndlo]);
453 while (prec && tail[--i] == '0') {
454 prec--;
455 if (!i) {
456 if (ndlo == ndhi) { prec = 0; break; }
457 lj_strfmt_wuint9(tail, nd[++ndlo]);
458 i = 9;
459 }
460 }
461 }
462 }
463 if (nde < 0) {
464 /* Make nde non-negative. */
465 eprefix = '-';
466 nde = -nde;
467 }
468 len = 3 + prec + (prefix != 0) + ndigits_dec((uint32_t)nde) + (nde < 10)
469 + ((prec | (sf & STRFMT_F_ALT)) != 0);
470 if (!p) p = lj_buf_more(sb, (width > len ? width : len) + 5);
471 if (!(sf & (STRFMT_F_LEFT | STRFMT_F_ZERO))) {
472 while (width-- > len) *p++ = ' ';
473 }
474 if (prefix) *p++ = prefix;
475 if ((sf & (STRFMT_F_LEFT | STRFMT_F_ZERO)) == STRFMT_F_ZERO) {
476 while (width-- > len) *p++ = '0';
477 }
478 q = lj_strfmt_wint(p + 1, nd[ndhi]);
479 p[0] = p[1]; /* Put leading digit in the correct place. */
480 if ((prec | (sf & STRFMT_F_ALT))) {
481 /* Emit fractional part. */
482 p[1] = '.'; p += 2;
483 prec -= (MSize)(q - p); p = q; /* Account for digits already emitted. */
484 /* Then emit chunks of 9 digits (this may emit 8 digits too many). */
485 for (i = ndhi; (int32_t)prec > 0 && i != ndlo; prec -= 9) {
486 i = (i - 1) & 0x3f;
487 p = lj_strfmt_wuint9(p, nd[i]);
488 }
489 if ((sf & STRFMT_T_FP_F) && !(sf & STRFMT_F_ALT)) {
490 /* %g (and not %#g) - strip trailing zeroes. */
491 p += (int32_t)prec & ((int32_t)prec >> 31);
492 while (p[-1] == '0') p--;
493 if (p[-1] == '.') p--;
494 } else {
495 /* %e (or %#g) - emit trailing zeroes. */
496 while ((int32_t)prec > 0) { *p++ = '0'; prec--; }
497 p += (int32_t)prec;
498 }
499 } else {
500 p++;
501 }
502 *p++ = (sf & STRFMT_F_UPPER) ? 'E' : 'e';
503 *p++ = eprefix; /* + or - */
504 if (nde < 10) *p++ = '0'; /* Always at least two digits of exponent. */
505 p = lj_strfmt_wint(p, nde);
506 } else {
507 /* %f (or, shortly, %g in %f style) */
508 if (prec < (MSize)(0x3f & -(int32_t)ndlo) * 9) {
509 /* Precision is sufficiently low as to maybe require rounding. */
510 ndhi = nd_add_m10e(nd, ndhi, 5, 0 - prec - 1);
511 }
512 g_format_like_f:
513 if ((sf & STRFMT_T_FP_E) && !(sf & STRFMT_F_ALT) && prec && width) {
514 /* Decrease precision in order to strip trailing zeroes. */
515 if (ndlo) {
516 /* nd has a fractional part; we need to look at its digits. */
517 char tail[9];
518 uint32_t maxprec = (64 - ndlo) * 9;
519 if (prec >= maxprec) prec = maxprec;
520 else ndlo = 64 - (prec + 8) / 9;
521 i = prec - ((63 - ndlo) * 9);
522 lj_strfmt_wuint9(tail, nd[ndlo]);
523 while (prec && tail[--i] == '0') {
524 prec--;
525 if (!i) {
526 if (ndlo == 63) { prec = 0; break; }
527 lj_strfmt_wuint9(tail, nd[++ndlo]);
528 i = 9;
529 }
530 }
531 } else {
532 /* nd has no fractional part, so precision goes straight to zero. */
533 prec = 0;
534 }
535 }
536 len = ndhi * 9 + ndigits_dec(nd[ndhi]) + prec + (prefix != 0)
537 + ((prec | (sf & STRFMT_F_ALT)) != 0);
538 if (!p) p = lj_buf_more(sb, (width > len ? width : len) + 8);
539 if (!(sf & (STRFMT_F_LEFT | STRFMT_F_ZERO))) {
540 while (width-- > len) *p++ = ' ';
541 }
542 if (prefix) *p++ = prefix;
543 if ((sf & (STRFMT_F_LEFT | STRFMT_F_ZERO)) == STRFMT_F_ZERO) {
544 while (width-- > len) *p++ = '0';
545 }
546 /* Emit integer part. */
547 p = lj_strfmt_wint(p, nd[ndhi]);
548 i = ndhi;
549 while (i) p = lj_strfmt_wuint9(p, nd[--i]);
550 if ((prec | (sf & STRFMT_F_ALT))) {
551 /* Emit fractional part. */
552 *p++ = '.';
553 /* Emit chunks of 9 digits (this may emit 8 digits too many). */
554 while ((int32_t)prec > 0 && i != ndlo) {
555 i = (i - 1) & 0x3f;
556 p = lj_strfmt_wuint9(p, nd[i]);
557 prec -= 9;
558 }
559 if ((sf & STRFMT_T_FP_E) && !(sf & STRFMT_F_ALT)) {
560 /* %g (and not %#g) - strip trailing zeroes. */
561 p += (int32_t)prec & ((int32_t)prec >> 31);
562 while (p[-1] == '0') p--;
563 if (p[-1] == '.') p--;
564 } else {
565 /* %f (or %#g) - emit trailing zeroes. */
566 while ((int32_t)prec > 0) { *p++ = '0'; prec--; }
567 p += (int32_t)prec;
568 }
569 }
570 }
571 }
572 if ((sf & STRFMT_F_LEFT)) while (width-- > len) *p++ = ' ';
573 return p;
574}
575
576/* Add formatted floating-point number to buffer. */
577SBuf *lj_strfmt_putfnum(SBuf *sb, SFormat sf, lua_Number n)
578{
579 sb->w = lj_strfmt_wfnum(sb, sf, n, NULL);
580 return sb;
581}
582
583/* -- Conversions to strings ---------------------------------------------- */
584
585/* Convert number to string. */
586GCstr * LJ_FASTCALL lj_strfmt_num(lua_State *L, cTValue *o)
587{
588 char buf[STRFMT_MAXBUF_NUM];
589 MSize len = (MSize)(lj_strfmt_wfnum(NULL, STRFMT_G14, o->n, buf) - buf);
590 return lj_str_new(L, buf, len);
591}
592
diff --git a/src/lj_strscan.c b/src/lj_strscan.c
index 79c0c569..9dbf477a 100644
--- a/src/lj_strscan.c
+++ b/src/lj_strscan.c
@@ -80,7 +80,7 @@ static void strscan_double(uint64_t x, TValue *o, int32_t ex2, int32_t neg)
80 /* Avoid double rounding for denormals. */ 80 /* Avoid double rounding for denormals. */
81 if (LJ_UNLIKELY(ex2 <= -1075 && x != 0)) { 81 if (LJ_UNLIKELY(ex2 <= -1075 && x != 0)) {
82 /* NYI: all of this generates way too much code on 32 bit CPUs. */ 82 /* NYI: all of this generates way too much code on 32 bit CPUs. */
83#if defined(__GNUC__) && LJ_64 83#if (defined(__GNUC__) || defined(__clang__)) && LJ_64
84 int32_t b = (int32_t)(__builtin_clzll(x)^63); 84 int32_t b = (int32_t)(__builtin_clzll(x)^63);
85#else 85#else
86 int32_t b = (x>>32) ? 32+(int32_t)lj_fls((uint32_t)(x>>32)) : 86 int32_t b = (x>>32) ? 32+(int32_t)lj_fls((uint32_t)(x>>32)) :
@@ -94,7 +94,7 @@ static void strscan_double(uint64_t x, TValue *o, int32_t ex2, int32_t neg)
94 } 94 }
95 95
96 /* Convert to double using a signed int64_t conversion, then rescale. */ 96 /* Convert to double using a signed int64_t conversion, then rescale. */
97 lua_assert((int64_t)x >= 0); 97 lj_assertX((int64_t)x >= 0, "bad double conversion");
98 n = (double)(int64_t)x; 98 n = (double)(int64_t)x;
99 if (neg) n = -n; 99 if (neg) n = -n;
100 if (ex2) n = ldexp(n, ex2); 100 if (ex2) n = ldexp(n, ex2);
@@ -142,7 +142,7 @@ static StrScanFmt strscan_hex(const uint8_t *p, TValue *o,
142 break; 142 break;
143 } 143 }
144 144
145 /* Reduce range then convert to double. */ 145 /* Reduce range, then convert to double. */
146 if ((x & U64x(c0000000,0000000))) { x = (x >> 2) | (x & 3); ex2 += 2; } 146 if ((x & U64x(c0000000,0000000))) { x = (x >> 2) | (x & 3); ex2 += 2; }
147 strscan_double(x, o, ex2, neg); 147 strscan_double(x, o, ex2, neg);
148 return fmt; 148 return fmt;
@@ -264,7 +264,7 @@ static StrScanFmt strscan_dec(const uint8_t *p, TValue *o,
264 uint32_t hi = 0, lo = (uint32_t)(xip-xi); 264 uint32_t hi = 0, lo = (uint32_t)(xip-xi);
265 int32_t ex2 = 0, idig = (int32_t)lo + (ex10 >> 1); 265 int32_t ex2 = 0, idig = (int32_t)lo + (ex10 >> 1);
266 266
267 lua_assert(lo > 0 && (ex10 & 1) == 0); 267 lj_assertX(lo > 0 && (ex10 & 1) == 0, "bad lo %d ex10 %d", lo, ex10);
268 268
269 /* Handle simple overflow/underflow. */ 269 /* Handle simple overflow/underflow. */
270 if (idig > 310/2) { if (neg) setminfV(o); else setpinfV(o); return fmt; } 270 if (idig > 310/2) { if (neg) setminfV(o); else setpinfV(o); return fmt; }
@@ -328,10 +328,55 @@ static StrScanFmt strscan_dec(const uint8_t *p, TValue *o,
328 return fmt; 328 return fmt;
329} 329}
330 330
331/* Parse binary number. */
332static StrScanFmt strscan_bin(const uint8_t *p, TValue *o,
333 StrScanFmt fmt, uint32_t opt,
334 int32_t ex2, int32_t neg, uint32_t dig)
335{
336 uint64_t x = 0;
337 uint32_t i;
338
339 if (ex2 || dig > 64) return STRSCAN_ERROR;
340
341 /* Scan binary digits. */
342 for (i = dig; i; i--, p++) {
343 if ((*p & ~1) != '0') return STRSCAN_ERROR;
344 x = (x << 1) | (*p & 1);
345 }
346
347 /* Format-specific handling. */
348 switch (fmt) {
349 case STRSCAN_INT:
350 if (!(opt & STRSCAN_OPT_TONUM) && x < 0x80000000u+neg) {
351 o->i = neg ? (int32_t)(~x+1u) : (int32_t)x;
352 return STRSCAN_INT; /* Fast path for 32 bit integers. */
353 }
354 if (!(opt & STRSCAN_OPT_C)) { fmt = STRSCAN_NUM; break; }
355 /* fallthrough */
356 case STRSCAN_U32:
357 if (dig > 32) return STRSCAN_ERROR;
358 o->i = neg ? (int32_t)(~x+1u) : (int32_t)x;
359 return STRSCAN_U32;
360 case STRSCAN_I64:
361 case STRSCAN_U64:
362 o->u64 = neg ? ~x+1u : x;
363 return fmt;
364 default:
365 break;
366 }
367
368 /* Reduce range, then convert to double. */
369 if ((x & U64x(c0000000,0000000))) { x = (x >> 2) | (x & 3); ex2 += 2; }
370 strscan_double(x, o, ex2, neg);
371 return fmt;
372}
373
331/* Scan string containing a number. Returns format. Returns value in o. */ 374/* Scan string containing a number. Returns format. Returns value in o. */
332StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt) 375StrScanFmt lj_strscan_scan(const uint8_t *p, MSize len, TValue *o,
376 uint32_t opt)
333{ 377{
334 int32_t neg = 0; 378 int32_t neg = 0;
379 const uint8_t *pe = p + len;
335 380
336 /* Remove leading space, parse sign and non-numbers. */ 381 /* Remove leading space, parse sign and non-numbers. */
337 if (LJ_UNLIKELY(!lj_char_isdigit(*p))) { 382 if (LJ_UNLIKELY(!lj_char_isdigit(*p))) {
@@ -349,7 +394,7 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
349 p += 3; 394 p += 3;
350 } 395 }
351 while (lj_char_isspace(*p)) p++; 396 while (lj_char_isspace(*p)) p++;
352 if (*p) return STRSCAN_ERROR; 397 if (*p || p < pe) return STRSCAN_ERROR;
353 o->u64 = tmp.u64; 398 o->u64 = tmp.u64;
354 return STRSCAN_NUM; 399 return STRSCAN_NUM;
355 } 400 }
@@ -366,8 +411,12 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
366 411
367 /* Determine base and skip leading zeros. */ 412 /* Determine base and skip leading zeros. */
368 if (LJ_UNLIKELY(*p <= '0')) { 413 if (LJ_UNLIKELY(*p <= '0')) {
369 if (*p == '0' && casecmp(p[1], 'x')) 414 if (*p == '0') {
370 base = 16, cmask = LJ_CHAR_XDIGIT, p += 2; 415 if (casecmp(p[1], 'x'))
416 base = 16, cmask = LJ_CHAR_XDIGIT, p += 2;
417 else if (casecmp(p[1], 'b'))
418 base = 2, cmask = LJ_CHAR_DIGIT, p += 2;
419 }
371 for ( ; ; p++) { 420 for ( ; ; p++) {
372 if (*p == '0') { 421 if (*p == '0') {
373 hasdig = 1; 422 hasdig = 1;
@@ -396,6 +445,7 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
396 445
397 /* Handle decimal point. */ 446 /* Handle decimal point. */
398 if (dp) { 447 if (dp) {
448 if (base == 2) return STRSCAN_ERROR;
399 fmt = STRSCAN_NUM; 449 fmt = STRSCAN_NUM;
400 if (dig) { 450 if (dig) {
401 ex = (int32_t)(dp-(p-1)); dp = p-1; 451 ex = (int32_t)(dp-(p-1)); dp = p-1;
@@ -406,7 +456,7 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
406 } 456 }
407 457
408 /* Parse exponent. */ 458 /* Parse exponent. */
409 if (casecmp(*p, (uint32_t)(base == 16 ? 'p' : 'e'))) { 459 if (base >= 10 && casecmp(*p, (uint32_t)(base == 16 ? 'p' : 'e'))) {
410 uint32_t xx; 460 uint32_t xx;
411 int negx = 0; 461 int negx = 0;
412 fmt = STRSCAN_NUM; p++; 462 fmt = STRSCAN_NUM; p++;
@@ -445,6 +495,7 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
445 while (lj_char_isspace(*p)) p++; 495 while (lj_char_isspace(*p)) p++;
446 if (*p) return STRSCAN_ERROR; 496 if (*p) return STRSCAN_ERROR;
447 } 497 }
498 if (p < pe) return STRSCAN_ERROR;
448 499
449 /* Fast path for decimal 32 bit integers. */ 500 /* Fast path for decimal 32 bit integers. */
450 if (fmt == STRSCAN_INT && base == 10 && 501 if (fmt == STRSCAN_INT && base == 10 &&
@@ -466,6 +517,8 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
466 return strscan_oct(sp, o, fmt, neg, dig); 517 return strscan_oct(sp, o, fmt, neg, dig);
467 if (base == 16) 518 if (base == 16)
468 fmt = strscan_hex(sp, o, fmt, opt, ex, neg, dig); 519 fmt = strscan_hex(sp, o, fmt, opt, ex, neg, dig);
520 else if (base == 2)
521 fmt = strscan_bin(sp, o, fmt, opt, ex, neg, dig);
469 else 522 else
470 fmt = strscan_dec(sp, o, fmt, opt, ex, neg, dig); 523 fmt = strscan_dec(sp, o, fmt, opt, ex, neg, dig);
471 524
@@ -481,18 +534,19 @@ StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt)
481 534
482int LJ_FASTCALL lj_strscan_num(GCstr *str, TValue *o) 535int LJ_FASTCALL lj_strscan_num(GCstr *str, TValue *o)
483{ 536{
484 StrScanFmt fmt = lj_strscan_scan((const uint8_t *)strdata(str), o, 537 StrScanFmt fmt = lj_strscan_scan((const uint8_t *)strdata(str), str->len, o,
485 STRSCAN_OPT_TONUM); 538 STRSCAN_OPT_TONUM);
486 lua_assert(fmt == STRSCAN_ERROR || fmt == STRSCAN_NUM); 539 lj_assertX(fmt == STRSCAN_ERROR || fmt == STRSCAN_NUM, "bad scan format");
487 return (fmt != STRSCAN_ERROR); 540 return (fmt != STRSCAN_ERROR);
488} 541}
489 542
490#if LJ_DUALNUM 543#if LJ_DUALNUM
491int LJ_FASTCALL lj_strscan_number(GCstr *str, TValue *o) 544int LJ_FASTCALL lj_strscan_number(GCstr *str, TValue *o)
492{ 545{
493 StrScanFmt fmt = lj_strscan_scan((const uint8_t *)strdata(str), o, 546 StrScanFmt fmt = lj_strscan_scan((const uint8_t *)strdata(str), str->len, o,
494 STRSCAN_OPT_TOINT); 547 STRSCAN_OPT_TOINT);
495 lua_assert(fmt == STRSCAN_ERROR || fmt == STRSCAN_NUM || fmt == STRSCAN_INT); 548 lj_assertX(fmt == STRSCAN_ERROR || fmt == STRSCAN_NUM || fmt == STRSCAN_INT,
549 "bad scan format");
496 if (fmt == STRSCAN_INT) setitype(o, LJ_TISNUM); 550 if (fmt == STRSCAN_INT) setitype(o, LJ_TISNUM);
497 return (fmt != STRSCAN_ERROR); 551 return (fmt != STRSCAN_ERROR);
498} 552}
diff --git a/src/lj_strscan.h b/src/lj_strscan.h
index 8d036e6f..a6047cf2 100644
--- a/src/lj_strscan.h
+++ b/src/lj_strscan.h
@@ -22,7 +22,8 @@ typedef enum {
22 STRSCAN_INT, STRSCAN_U32, STRSCAN_I64, STRSCAN_U64, 22 STRSCAN_INT, STRSCAN_U32, STRSCAN_I64, STRSCAN_U64,
23} StrScanFmt; 23} StrScanFmt;
24 24
25LJ_FUNC StrScanFmt lj_strscan_scan(const uint8_t *p, TValue *o, uint32_t opt); 25LJ_FUNC StrScanFmt lj_strscan_scan(const uint8_t *p, MSize len, TValue *o,
26 uint32_t opt);
26LJ_FUNC int LJ_FASTCALL lj_strscan_num(GCstr *str, TValue *o); 27LJ_FUNC int LJ_FASTCALL lj_strscan_num(GCstr *str, TValue *o);
27#if LJ_DUALNUM 28#if LJ_DUALNUM
28LJ_FUNC int LJ_FASTCALL lj_strscan_number(GCstr *str, TValue *o); 29LJ_FUNC int LJ_FASTCALL lj_strscan_number(GCstr *str, TValue *o);
diff --git a/src/lj_tab.c b/src/lj_tab.c
index 26485dcd..a9b02dcc 100644
--- a/src/lj_tab.c
+++ b/src/lj_tab.c
@@ -16,25 +16,10 @@
16 16
17/* -- Object hashing ------------------------------------------------------ */ 17/* -- Object hashing ------------------------------------------------------ */
18 18
19/* Hash values are masked with the table hash mask and used as an index. */
20static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash)
21{
22 Node *n = noderef(t->node);
23 return &n[hash & t->hmask];
24}
25
26/* String hashes are precomputed when they are interned. */
27#define hashstr(t, s) hashmask(t, (s)->hash)
28
29#define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi)))
30#define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1))
31#define hashptr(t, p) hashlohi((t), u32ptr(p), u32ptr(p) + HASH_BIAS)
32#define hashgcref(t, r) hashlohi((t), gcrefu(r), gcrefu(r) + HASH_BIAS)
33
34/* Hash an arbitrary key and return its anchor position in the hash table. */ 19/* Hash an arbitrary key and return its anchor position in the hash table. */
35static Node *hashkey(const GCtab *t, cTValue *key) 20static Node *hashkey(const GCtab *t, cTValue *key)
36{ 21{
37 lua_assert(!tvisint(key)); 22 lj_assertX(!tvisint(key), "attempt to hash integer");
38 if (tvisstr(key)) 23 if (tvisstr(key))
39 return hashstr(t, strV(key)); 24 return hashstr(t, strV(key));
40 else if (tvisnum(key)) 25 else if (tvisnum(key))
@@ -53,13 +38,13 @@ static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits)
53{ 38{
54 uint32_t hsize; 39 uint32_t hsize;
55 Node *node; 40 Node *node;
56 lua_assert(hbits != 0); 41 lj_assertL(hbits != 0, "zero hash size");
57 if (hbits > LJ_MAX_HBITS) 42 if (hbits > LJ_MAX_HBITS)
58 lj_err_msg(L, LJ_ERR_TABOV); 43 lj_err_msg(L, LJ_ERR_TABOV);
59 hsize = 1u << hbits; 44 hsize = 1u << hbits;
60 node = lj_mem_newvec(L, hsize, Node); 45 node = lj_mem_newvec(L, hsize, Node);
61 setmref(node->freetop, &node[hsize]);
62 setmref(t->node, node); 46 setmref(t->node, node);
47 setfreetop(t, node, &node[hsize]);
63 t->hmask = hsize-1; 48 t->hmask = hsize-1;
64} 49}
65 50
@@ -74,7 +59,7 @@ static LJ_AINLINE void clearhpart(GCtab *t)
74{ 59{
75 uint32_t i, hmask = t->hmask; 60 uint32_t i, hmask = t->hmask;
76 Node *node = noderef(t->node); 61 Node *node = noderef(t->node);
77 lua_assert(t->hmask != 0); 62 lj_assertX(t->hmask != 0, "empty hash part");
78 for (i = 0; i <= hmask; i++) { 63 for (i = 0; i <= hmask; i++) {
79 Node *n = &node[i]; 64 Node *n = &node[i];
80 setmref(n->next, NULL); 65 setmref(n->next, NULL);
@@ -98,7 +83,8 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
98 GCtab *t; 83 GCtab *t;
99 /* First try to colocate the array part. */ 84 /* First try to colocate the array part. */
100 if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) { 85 if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) {
101 lua_assert((sizeof(GCtab) & 7) == 0); 86 Node *nilnode;
87 lj_assertL((sizeof(GCtab) & 7) == 0, "bad GCtab size");
102 t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize)); 88 t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize));
103 t->gct = ~LJ_TTAB; 89 t->gct = ~LJ_TTAB;
104 t->nomm = (uint8_t)~0; 90 t->nomm = (uint8_t)~0;
@@ -107,8 +93,13 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
107 setgcrefnull(t->metatable); 93 setgcrefnull(t->metatable);
108 t->asize = asize; 94 t->asize = asize;
109 t->hmask = 0; 95 t->hmask = 0;
110 setmref(t->node, &G(L)->nilnode); 96 nilnode = &G(L)->nilnode;
97 setmref(t->node, nilnode);
98#if LJ_GC64
99 setmref(t->freetop, nilnode);
100#endif
111 } else { /* Otherwise separately allocate the array part. */ 101 } else { /* Otherwise separately allocate the array part. */
102 Node *nilnode;
112 t = lj_mem_newobj(L, GCtab); 103 t = lj_mem_newobj(L, GCtab);
113 t->gct = ~LJ_TTAB; 104 t->gct = ~LJ_TTAB;
114 t->nomm = (uint8_t)~0; 105 t->nomm = (uint8_t)~0;
@@ -117,7 +108,11 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
117 setgcrefnull(t->metatable); 108 setgcrefnull(t->metatable);
118 t->asize = 0; /* In case the array allocation fails. */ 109 t->asize = 0; /* In case the array allocation fails. */
119 t->hmask = 0; 110 t->hmask = 0;
120 setmref(t->node, &G(L)->nilnode); 111 nilnode = &G(L)->nilnode;
112 setmref(t->node, nilnode);
113#if LJ_GC64
114 setmref(t->freetop, nilnode);
115#endif
121 if (asize > 0) { 116 if (asize > 0) {
122 if (asize > LJ_MAX_ASIZE) 117 if (asize > LJ_MAX_ASIZE)
123 lj_err_msg(L, LJ_ERR_TABOV); 118 lj_err_msg(L, LJ_ERR_TABOV);
@@ -149,6 +144,12 @@ GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits)
149 return t; 144 return t;
150} 145}
151 146
147/* The API of this function conforms to lua_createtable(). */
148GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h)
149{
150 return lj_tab_new(L, (uint32_t)(a > 0 ? a+1 : 0), hsize2hbits(h));
151}
152
152#if LJ_HASJIT 153#if LJ_HASJIT
153GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize) 154GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize)
154{ 155{
@@ -165,7 +166,8 @@ GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt)
165 GCtab *t; 166 GCtab *t;
166 uint32_t asize, hmask; 167 uint32_t asize, hmask;
167 t = newtab(L, kt->asize, kt->hmask > 0 ? lj_fls(kt->hmask)+1 : 0); 168 t = newtab(L, kt->asize, kt->hmask > 0 ? lj_fls(kt->hmask)+1 : 0);
168 lua_assert(kt->asize == t->asize && kt->hmask == t->hmask); 169 lj_assertL(kt->asize == t->asize && kt->hmask == t->hmask,
170 "mismatched size of table and template");
169 t->nomm = 0; /* Keys with metamethod names may be present. */ 171 t->nomm = 0; /* Keys with metamethod names may be present. */
170 asize = kt->asize; 172 asize = kt->asize;
171 if (asize > 0) { 173 if (asize > 0) {
@@ -185,7 +187,7 @@ GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt)
185 Node *node = noderef(t->node); 187 Node *node = noderef(t->node);
186 Node *knode = noderef(kt->node); 188 Node *knode = noderef(kt->node);
187 ptrdiff_t d = (char *)node - (char *)knode; 189 ptrdiff_t d = (char *)node - (char *)knode;
188 setmref(node->freetop, (Node *)((char *)noderef(knode->freetop) + d)); 190 setfreetop(t, node, (Node *)((char *)getfreetop(kt, knode) + d));
189 for (i = 0; i <= hmask; i++) { 191 for (i = 0; i <= hmask; i++) {
190 Node *kn = &knode[i]; 192 Node *kn = &knode[i];
191 Node *n = &node[i]; 193 Node *n = &node[i];
@@ -198,6 +200,17 @@ GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt)
198 return t; 200 return t;
199} 201}
200 202
203/* Clear a table. */
204void LJ_FASTCALL lj_tab_clear(GCtab *t)
205{
206 clearapart(t);
207 if (t->hmask > 0) {
208 Node *node = noderef(t->node);
209 setfreetop(t, node, &node[t->hmask+1]);
210 clearhpart(t);
211 }
212}
213
201/* Free a table. */ 214/* Free a table. */
202void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t) 215void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t)
203{ 216{
@@ -214,7 +227,7 @@ void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t)
214/* -- Table resizing ------------------------------------------------------ */ 227/* -- Table resizing ------------------------------------------------------ */
215 228
216/* Resize a table to fit the new array/hash part sizes. */ 229/* Resize a table to fit the new array/hash part sizes. */
217static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) 230void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
218{ 231{
219 Node *oldnode = noderef(t->node); 232 Node *oldnode = noderef(t->node);
220 uint32_t oldasize = t->asize; 233 uint32_t oldasize = t->asize;
@@ -247,6 +260,9 @@ static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
247 } else { 260 } else {
248 global_State *g = G(L); 261 global_State *g = G(L);
249 setmref(t->node, &g->nilnode); 262 setmref(t->node, &g->nilnode);
263#if LJ_GC64
264 setmref(t->freetop, &g->nilnode);
265#endif
250 t->hmask = 0; 266 t->hmask = 0;
251 } 267 }
252 if (asize < oldasize) { /* Array part shrinks? */ 268 if (asize < oldasize) { /* Array part shrinks? */
@@ -276,7 +292,7 @@ static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
276 292
277static uint32_t countint(cTValue *key, uint32_t *bins) 293static uint32_t countint(cTValue *key, uint32_t *bins)
278{ 294{
279 lua_assert(!tvisint(key)); 295 lj_assertX(!tvisint(key), "bad integer key");
280 if (tvisnum(key)) { 296 if (tvisnum(key)) {
281 lua_Number nk = numV(key); 297 lua_Number nk = numV(key);
282 int32_t k = lj_num2int(nk); 298 int32_t k = lj_num2int(nk);
@@ -348,7 +364,7 @@ static void rehashtab(lua_State *L, GCtab *t, cTValue *ek)
348 asize += countint(ek, bins); 364 asize += countint(ek, bins);
349 na = bestasize(bins, &asize); 365 na = bestasize(bins, &asize);
350 total -= na; 366 total -= na;
351 resizetab(L, t, asize, hsize2hbits(total)); 367 lj_tab_resize(L, t, asize, hsize2hbits(total));
352} 368}
353 369
354#if LJ_HASFFI 370#if LJ_HASFFI
@@ -360,7 +376,7 @@ void lj_tab_rehash(lua_State *L, GCtab *t)
360 376
361void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize) 377void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize)
362{ 378{
363 resizetab(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0); 379 lj_tab_resize(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0);
364} 380}
365 381
366/* -- Table getters ------------------------------------------------------- */ 382/* -- Table getters ------------------------------------------------------- */
@@ -378,7 +394,7 @@ cTValue * LJ_FASTCALL lj_tab_getinth(GCtab *t, int32_t key)
378 return NULL; 394 return NULL;
379} 395}
380 396
381cTValue *lj_tab_getstr(GCtab *t, GCstr *key) 397cTValue *lj_tab_getstr(GCtab *t, const GCstr *key)
382{ 398{
383 Node *n = hashstr(t, key); 399 Node *n = hashstr(t, key);
384 do { 400 do {
@@ -428,16 +444,17 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key)
428 Node *n = hashkey(t, key); 444 Node *n = hashkey(t, key);
429 if (!tvisnil(&n->val) || t->hmask == 0) { 445 if (!tvisnil(&n->val) || t->hmask == 0) {
430 Node *nodebase = noderef(t->node); 446 Node *nodebase = noderef(t->node);
431 Node *collide, *freenode = noderef(nodebase->freetop); 447 Node *collide, *freenode = getfreetop(t, nodebase);
432 lua_assert(freenode >= nodebase && freenode <= nodebase+t->hmask+1); 448 lj_assertL(freenode >= nodebase && freenode <= nodebase+t->hmask+1,
449 "bad freenode");
433 do { 450 do {
434 if (freenode == nodebase) { /* No free node found? */ 451 if (freenode == nodebase) { /* No free node found? */
435 rehashtab(L, t, key); /* Rehash table. */ 452 rehashtab(L, t, key); /* Rehash table. */
436 return lj_tab_set(L, t, key); /* Retry key insertion. */ 453 return lj_tab_set(L, t, key); /* Retry key insertion. */
437 } 454 }
438 } while (!tvisnil(&(--freenode)->key)); 455 } while (!tvisnil(&(--freenode)->key));
439 setmref(nodebase->freetop, freenode); 456 setfreetop(t, nodebase, freenode);
440 lua_assert(freenode != &G(L)->nilnode); 457 lj_assertL(freenode != &G(L)->nilnode, "store to fallback hash");
441 collide = hashkey(t, &n->key); 458 collide = hashkey(t, &n->key);
442 if (collide != n) { /* Colliding node not the main node? */ 459 if (collide != n) { /* Colliding node not the main node? */
443 while (noderef(collide->next) != n) /* Find predecessor. */ 460 while (noderef(collide->next) != n) /* Find predecessor. */
@@ -493,7 +510,7 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key)
493 if (LJ_UNLIKELY(tvismzero(&n->key))) 510 if (LJ_UNLIKELY(tvismzero(&n->key)))
494 n->key.u64 = 0; 511 n->key.u64 = 0;
495 lj_gc_anybarriert(L, t); 512 lj_gc_anybarriert(L, t);
496 lua_assert(tvisnil(&n->val)); 513 lj_assertL(tvisnil(&n->val), "new hash slot is not empty");
497 return &n->val; 514 return &n->val;
498} 515}
499 516
@@ -510,7 +527,7 @@ TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key)
510 return lj_tab_newkey(L, t, &k); 527 return lj_tab_newkey(L, t, &k);
511} 528}
512 529
513TValue *lj_tab_setstr(lua_State *L, GCtab *t, GCstr *key) 530TValue *lj_tab_setstr(lua_State *L, GCtab *t, const GCstr *key)
514{ 531{
515 TValue k; 532 TValue k;
516 Node *n = hashstr(t, key); 533 Node *n = hashstr(t, key);
@@ -551,103 +568,126 @@ TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key)
551 568
552/* -- Table traversal ----------------------------------------------------- */ 569/* -- Table traversal ----------------------------------------------------- */
553 570
554/* Get the traversal index of a key. */ 571/* Table traversal indexes:
555static uint32_t keyindex(lua_State *L, GCtab *t, cTValue *key) 572**
573** Array key index: [0 .. t->asize-1]
574** Hash key index: [t->asize .. t->asize+t->hmask]
575** Invalid key: ~0
576*/
577
578/* Get the successor traversal index of a key. */
579uint32_t LJ_FASTCALL lj_tab_keyindex(GCtab *t, cTValue *key)
556{ 580{
557 TValue tmp; 581 TValue tmp;
558 if (tvisint(key)) { 582 if (tvisint(key)) {
559 int32_t k = intV(key); 583 int32_t k = intV(key);
560 if ((uint32_t)k < t->asize) 584 if ((uint32_t)k < t->asize)
561 return (uint32_t)k; /* Array key indexes: [0..t->asize-1] */ 585 return (uint32_t)k + 1;
562 setnumV(&tmp, (lua_Number)k); 586 setnumV(&tmp, (lua_Number)k);
563 key = &tmp; 587 key = &tmp;
564 } else if (tvisnum(key)) { 588 } else if (tvisnum(key)) {
565 lua_Number nk = numV(key); 589 lua_Number nk = numV(key);
566 int32_t k = lj_num2int(nk); 590 int32_t k = lj_num2int(nk);
567 if ((uint32_t)k < t->asize && nk == (lua_Number)k) 591 if ((uint32_t)k < t->asize && nk == (lua_Number)k)
568 return (uint32_t)k; /* Array key indexes: [0..t->asize-1] */ 592 return (uint32_t)k + 1;
569 } 593 }
570 if (!tvisnil(key)) { 594 if (!tvisnil(key)) {
571 Node *n = hashkey(t, key); 595 Node *n = hashkey(t, key);
572 do { 596 do {
573 if (lj_obj_equal(&n->key, key)) 597 if (lj_obj_equal(&n->key, key))
574 return t->asize + (uint32_t)(n - noderef(t->node)); 598 return t->asize + (uint32_t)((n+1) - noderef(t->node));
575 /* Hash key indexes: [t->asize..t->asize+t->nmask] */
576 } while ((n = nextnode(n))); 599 } while ((n = nextnode(n)));
577 if (key->u32.hi == 0xfffe7fff) /* ITERN was despecialized while running. */ 600 if (key->u32.hi == LJ_KEYINDEX) /* Despecialized ITERN while running. */
578 return key->u32.lo - 1; 601 return key->u32.lo;
579 lj_err_msg(L, LJ_ERR_NEXTIDX); 602 return ~0u; /* Invalid key to next. */
580 return 0; /* unreachable */
581 } 603 }
582 return ~0u; /* A nil key starts the traversal. */ 604 return 0; /* A nil key starts the traversal. */
583} 605}
584 606
585/* Advance to the next step in a table traversal. */ 607/* Get the next key/value pair of a table traversal. */
586int lj_tab_next(lua_State *L, GCtab *t, TValue *key) 608int lj_tab_next(GCtab *t, cTValue *key, TValue *o)
587{ 609{
588 uint32_t i = keyindex(L, t, key); /* Find predecessor key index. */ 610 uint32_t idx = lj_tab_keyindex(t, key); /* Find successor index of key. */
589 for (i++; i < t->asize; i++) /* First traverse the array keys. */ 611 /* First traverse the array part. */
590 if (!tvisnil(arrayslot(t, i))) { 612 for (; idx < t->asize; idx++) {
591 setintV(key, i); 613 cTValue *a = arrayslot(t, idx);
592 copyTV(L, key+1, arrayslot(t, i)); 614 if (LJ_LIKELY(!tvisnil(a))) {
615 setintV(o, idx);
616 o[1] = *a;
593 return 1; 617 return 1;
594 } 618 }
595 for (i -= t->asize; i <= t->hmask; i++) { /* Then traverse the hash keys. */ 619 }
596 Node *n = &noderef(t->node)[i]; 620 idx -= t->asize;
621 /* Then traverse the hash part. */
622 for (; idx <= t->hmask; idx++) {
623 Node *n = &noderef(t->node)[idx];
597 if (!tvisnil(&n->val)) { 624 if (!tvisnil(&n->val)) {
598 copyTV(L, key, &n->key); 625 o[0] = n->key;
599 copyTV(L, key+1, &n->val); 626 o[1] = n->val;
600 return 1; 627 return 1;
601 } 628 }
602 } 629 }
603 return 0; /* End of traversal. */ 630 return (int32_t)idx < 0 ? -1 : 0; /* Invalid key or end of traversal. */
604} 631}
605 632
606/* -- Table length calculation -------------------------------------------- */ 633/* -- Table length calculation -------------------------------------------- */
607 634
608static MSize unbound_search(GCtab *t, MSize j) 635/* Compute table length. Slow path with mixed array/hash lookups. */
636LJ_NOINLINE static MSize tab_len_slow(GCtab *t, size_t hi)
609{ 637{
610 cTValue *tv; 638 cTValue *tv;
611 MSize i = j; /* i is zero or a present index */ 639 size_t lo = hi;
612 j++; 640 hi++;
613 /* find `i' and `j' such that i is present and j is not */ 641 /* Widening search for an upper bound. */
614 while ((tv = lj_tab_getint(t, (int32_t)j)) && !tvisnil(tv)) { 642 while ((tv = lj_tab_getint(t, (int32_t)hi)) && !tvisnil(tv)) {
615 i = j; 643 lo = hi;
616 j *= 2; 644 hi += hi;
617 if (j > (MSize)(INT_MAX-2)) { /* overflow? */ 645 if (hi > (size_t)(INT_MAX-2)) { /* Punt and do a linear search. */
618 /* table was built with bad purposes: resort to linear search */ 646 lo = 1;
619 i = 1; 647 while ((tv = lj_tab_getint(t, (int32_t)lo)) && !tvisnil(tv)) lo++;
620 while ((tv = lj_tab_getint(t, (int32_t)i)) && !tvisnil(tv)) i++; 648 return (MSize)(lo - 1);
621 return i - 1;
622 } 649 }
623 } 650 }
624 /* now do a binary search between them */ 651 /* Binary search to find a non-nil to nil transition. */
625 while (j - i > 1) { 652 while (hi - lo > 1) {
626 MSize m = (i+j)/2; 653 size_t mid = (lo+hi) >> 1;
627 cTValue *tvb = lj_tab_getint(t, (int32_t)m); 654 cTValue *tvb = lj_tab_getint(t, (int32_t)mid);
628 if (tvb && !tvisnil(tvb)) i = m; else j = m; 655 if (tvb && !tvisnil(tvb)) lo = mid; else hi = mid;
629 } 656 }
630 return i; 657 return (MSize)lo;
631} 658}
632 659
633/* 660/* Compute table length. Fast path. */
634** Try to find a boundary in table `t'. A `boundary' is an integer index
635** such that t[i] is non-nil and t[i+1] is nil (and 0 if t[1] is nil).
636*/
637MSize LJ_FASTCALL lj_tab_len(GCtab *t) 661MSize LJ_FASTCALL lj_tab_len(GCtab *t)
638{ 662{
639 MSize j = (MSize)t->asize; 663 size_t hi = (size_t)t->asize;
640 if (j > 1 && tvisnil(arrayslot(t, j-1))) { 664 if (hi) hi--;
641 MSize i = 1; 665 /* In a growing array the last array element is very likely nil. */
642 while (j - i > 1) { 666 if (hi > 0 && LJ_LIKELY(tvisnil(arrayslot(t, hi)))) {
643 MSize m = (i+j)/2; 667 /* Binary search to find a non-nil to nil transition in the array. */
644 if (tvisnil(arrayslot(t, m-1))) j = m; else i = m; 668 size_t lo = 0;
669 while (hi - lo > 1) {
670 size_t mid = (lo+hi) >> 1;
671 if (tvisnil(arrayslot(t, mid))) hi = mid; else lo = mid;
645 } 672 }
646 return i-1; 673 return (MSize)lo;
647 } 674 }
648 if (j) j--; 675 /* Without a hash part, there's an implicit nil after the last element. */
649 if (t->hmask <= 0) 676 return t->hmask ? tab_len_slow(t, hi) : (MSize)hi;
650 return j;
651 return unbound_search(t, j);
652} 677}
653 678
679#if LJ_HASJIT
680/* Verify hinted table length or compute it. */
681MSize LJ_FASTCALL lj_tab_len_hint(GCtab *t, size_t hint)
682{
683 size_t asize = (size_t)t->asize;
684 cTValue *tv = arrayslot(t, hint);
685 if (LJ_LIKELY(hint+1 < asize)) {
686 if (LJ_LIKELY(!tvisnil(tv) && tvisnil(tv+1))) return (MSize)hint;
687 } else if (hint+1 <= asize && LJ_LIKELY(t->hmask == 0) && !tvisnil(tv)) {
688 return (MSize)hint;
689 }
690 return lj_tab_len(t);
691}
692#endif
693
diff --git a/src/lj_tab.h b/src/lj_tab.h
index bb3e273d..cddc4765 100644
--- a/src/lj_tab.h
+++ b/src/lj_tab.h
@@ -31,30 +31,52 @@ static LJ_AINLINE uint32_t hashrot(uint32_t lo, uint32_t hi)
31 return hi; 31 return hi;
32} 32}
33 33
34/* Hash values are masked with the table hash mask and used as an index. */
35static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash)
36{
37 Node *n = noderef(t->node);
38 return &n[hash & t->hmask];
39}
40
41/* String IDs are generated when a string is interned. */
42#define hashstr(t, s) hashmask(t, (s)->sid)
43
44#define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi)))
45#define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1))
46#if LJ_GC64
47#define hashgcref(t, r) \
48 hashlohi((t), (uint32_t)gcrefu(r), (uint32_t)(gcrefu(r) >> 32))
49#else
50#define hashgcref(t, r) hashlohi((t), gcrefu(r), gcrefu(r) + HASH_BIAS)
51#endif
52
34#define hsize2hbits(s) ((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0) 53#define hsize2hbits(s) ((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0)
35 54
36LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits); 55LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits);
56LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h);
37#if LJ_HASJIT 57#if LJ_HASJIT
38LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize); 58LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize);
39#endif 59#endif
40LJ_FUNCA GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt); 60LJ_FUNCA GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt);
61LJ_FUNC void LJ_FASTCALL lj_tab_clear(GCtab *t);
41LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t); 62LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t);
42#if LJ_HASFFI 63#if LJ_HASFFI
43LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t); 64LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t);
44#endif 65#endif
66LJ_FUNC void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits);
45LJ_FUNCA void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize); 67LJ_FUNCA void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize);
46 68
47/* Caveat: all getters except lj_tab_get() can return NULL! */ 69/* Caveat: all getters except lj_tab_get() can return NULL! */
48 70
49LJ_FUNCA cTValue * LJ_FASTCALL lj_tab_getinth(GCtab *t, int32_t key); 71LJ_FUNCA cTValue * LJ_FASTCALL lj_tab_getinth(GCtab *t, int32_t key);
50LJ_FUNC cTValue *lj_tab_getstr(GCtab *t, GCstr *key); 72LJ_FUNC cTValue *lj_tab_getstr(GCtab *t, const GCstr *key);
51LJ_FUNCA cTValue *lj_tab_get(lua_State *L, GCtab *t, cTValue *key); 73LJ_FUNCA cTValue *lj_tab_get(lua_State *L, GCtab *t, cTValue *key);
52 74
53/* Caveat: all setters require a write barrier for the stored value. */ 75/* Caveat: all setters require a write barrier for the stored value. */
54 76
55LJ_FUNCA TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key); 77LJ_FUNCA TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key);
56LJ_FUNC TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key); 78LJ_FUNCA TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key);
57LJ_FUNC TValue *lj_tab_setstr(lua_State *L, GCtab *t, GCstr *key); 79LJ_FUNC TValue *lj_tab_setstr(lua_State *L, GCtab *t, const GCstr *key);
58LJ_FUNC TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key); 80LJ_FUNC TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key);
59 81
60#define inarray(t, key) ((MSize)(key) < (MSize)(t)->asize) 82#define inarray(t, key) ((MSize)(key) < (MSize)(t)->asize)
@@ -64,7 +86,11 @@ LJ_FUNC TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key);
64#define lj_tab_setint(L, t, key) \ 86#define lj_tab_setint(L, t, key) \
65 (inarray((t), (key)) ? arrayslot((t), (key)) : lj_tab_setinth(L, (t), (key))) 87 (inarray((t), (key)) ? arrayslot((t), (key)) : lj_tab_setinth(L, (t), (key)))
66 88
67LJ_FUNCA int lj_tab_next(lua_State *L, GCtab *t, TValue *key); 89LJ_FUNC uint32_t LJ_FASTCALL lj_tab_keyindex(GCtab *t, cTValue *key);
90LJ_FUNCA int lj_tab_next(GCtab *t, cTValue *key, TValue *o);
68LJ_FUNCA MSize LJ_FASTCALL lj_tab_len(GCtab *t); 91LJ_FUNCA MSize LJ_FASTCALL lj_tab_len(GCtab *t);
92#if LJ_HASJIT
93LJ_FUNC MSize LJ_FASTCALL lj_tab_len_hint(GCtab *t, size_t hint);
94#endif
69 95
70#endif 96#endif
diff --git a/src/lj_target.h b/src/lj_target.h
index 8c881652..2f4d21c1 100644
--- a/src/lj_target.h
+++ b/src/lj_target.h
@@ -55,7 +55,7 @@ typedef uint32_t RegSP;
55/* Bitset for registers. 32 registers suffice for most architectures. 55/* Bitset for registers. 32 registers suffice for most architectures.
56** Note that one set holds bits for both GPRs and FPRs. 56** Note that one set holds bits for both GPRs and FPRs.
57*/ 57*/
58#if LJ_TARGET_PPC || LJ_TARGET_MIPS 58#if LJ_TARGET_PPC || LJ_TARGET_MIPS || LJ_TARGET_ARM64
59typedef uint64_t RegSet; 59typedef uint64_t RegSet;
60#else 60#else
61typedef uint32_t RegSet; 61typedef uint32_t RegSet;
@@ -69,7 +69,7 @@ typedef uint32_t RegSet;
69#define rset_set(rs, r) (rs |= RID2RSET(r)) 69#define rset_set(rs, r) (rs |= RID2RSET(r))
70#define rset_clear(rs, r) (rs &= ~RID2RSET(r)) 70#define rset_clear(rs, r) (rs &= ~RID2RSET(r))
71#define rset_exclude(rs, r) (rs & ~RID2RSET(r)) 71#define rset_exclude(rs, r) (rs & ~RID2RSET(r))
72#if LJ_TARGET_PPC || LJ_TARGET_MIPS 72#if LJ_TARGET_PPC || LJ_TARGET_MIPS || LJ_TARGET_ARM64
73#define rset_picktop(rs) ((Reg)(__builtin_clzll(rs)^63)) 73#define rset_picktop(rs) ((Reg)(__builtin_clzll(rs)^63))
74#define rset_pickbot(rs) ((Reg)__builtin_ctzll(rs)) 74#define rset_pickbot(rs) ((Reg)__builtin_ctzll(rs))
75#else 75#else
@@ -138,6 +138,8 @@ typedef uint32_t RegCost;
138#include "lj_target_x86.h" 138#include "lj_target_x86.h"
139#elif LJ_TARGET_ARM 139#elif LJ_TARGET_ARM
140#include "lj_target_arm.h" 140#include "lj_target_arm.h"
141#elif LJ_TARGET_ARM64
142#include "lj_target_arm64.h"
141#elif LJ_TARGET_PPC 143#elif LJ_TARGET_PPC
142#include "lj_target_ppc.h" 144#include "lj_target_ppc.h"
143#elif LJ_TARGET_MIPS 145#elif LJ_TARGET_MIPS
@@ -150,7 +152,8 @@ typedef uint32_t RegCost;
150/* Return the address of an exit stub. */ 152/* Return the address of an exit stub. */
151static LJ_AINLINE char *exitstub_addr_(char **group, uint32_t exitno) 153static LJ_AINLINE char *exitstub_addr_(char **group, uint32_t exitno)
152{ 154{
153 lua_assert(group[exitno / EXITSTUBS_PER_GROUP] != NULL); 155 lj_assertX(group[exitno / EXITSTUBS_PER_GROUP] != NULL,
156 "exit stub group for exit %d uninitialized", exitno);
154 return (char *)group[exitno / EXITSTUBS_PER_GROUP] + 157 return (char *)group[exitno / EXITSTUBS_PER_GROUP] +
155 EXITSTUB_SPACING*(exitno % EXITSTUBS_PER_GROUP); 158 EXITSTUB_SPACING*(exitno % EXITSTUBS_PER_GROUP);
156} 159}
diff --git a/src/lj_target_arm.h b/src/lj_target_arm.h
index cdc8776d..7170fcab 100644
--- a/src/lj_target_arm.h
+++ b/src/lj_target_arm.h
@@ -211,6 +211,7 @@ typedef enum ARMIns {
211 /* ARMv6T2 */ 211 /* ARMv6T2 */
212 ARMI_MOVW = 0xe3000000, 212 ARMI_MOVW = 0xe3000000,
213 ARMI_MOVT = 0xe3400000, 213 ARMI_MOVT = 0xe3400000,
214 ARMI_BFI = 0xe7c00010,
214 215
215 /* VFP */ 216 /* VFP */
216 ARMI_VMOV_D = 0xeeb00b40, 217 ARMI_VMOV_D = 0xeeb00b40,
@@ -243,10 +244,6 @@ typedef enum ARMIns {
243 ARMI_VCVT_S32_F64 = 0xeebd0bc0, 244 ARMI_VCVT_S32_F64 = 0xeebd0bc0,
244 ARMI_VCVT_U32_F32 = 0xeebc0ac0, 245 ARMI_VCVT_U32_F32 = 0xeebc0ac0,
245 ARMI_VCVT_U32_F64 = 0xeebc0bc0, 246 ARMI_VCVT_U32_F64 = 0xeebc0bc0,
246 ARMI_VCVTR_S32_F32 = 0xeebd0a40,
247 ARMI_VCVTR_S32_F64 = 0xeebd0b40,
248 ARMI_VCVTR_U32_F32 = 0xeebc0a40,
249 ARMI_VCVTR_U32_F64 = 0xeebc0b40,
250 ARMI_VCVT_F32_S32 = 0xeeb80ac0, 247 ARMI_VCVT_F32_S32 = 0xeeb80ac0,
251 ARMI_VCVT_F64_S32 = 0xeeb80bc0, 248 ARMI_VCVT_F64_S32 = 0xeeb80bc0,
252 ARMI_VCVT_F32_U32 = 0xeeb80a40, 249 ARMI_VCVT_F32_U32 = 0xeeb80a40,
diff --git a/src/lj_target_arm64.h b/src/lj_target_arm64.h
new file mode 100644
index 00000000..65a14307
--- /dev/null
+++ b/src/lj_target_arm64.h
@@ -0,0 +1,342 @@
1/*
2** Definitions for ARM64 CPUs.
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_TARGET_ARM64_H
7#define _LJ_TARGET_ARM64_H
8
9/* -- Registers IDs ------------------------------------------------------- */
10
11#define GPRDEF(_) \
12 _(X0) _(X1) _(X2) _(X3) _(X4) _(X5) _(X6) _(X7) \
13 _(X8) _(X9) _(X10) _(X11) _(X12) _(X13) _(X14) _(X15) \
14 _(X16) _(X17) _(X18) _(X19) _(X20) _(X21) _(X22) _(X23) \
15 _(X24) _(X25) _(X26) _(X27) _(X28) _(FP) _(LR) _(SP)
16#define FPRDEF(_) \
17 _(D0) _(D1) _(D2) _(D3) _(D4) _(D5) _(D6) _(D7) \
18 _(D8) _(D9) _(D10) _(D11) _(D12) _(D13) _(D14) _(D15) \
19 _(D16) _(D17) _(D18) _(D19) _(D20) _(D21) _(D22) _(D23) \
20 _(D24) _(D25) _(D26) _(D27) _(D28) _(D29) _(D30) _(D31)
21#define VRIDDEF(_)
22
23#define RIDENUM(name) RID_##name,
24
25enum {
26 GPRDEF(RIDENUM) /* General-purpose registers (GPRs). */
27 FPRDEF(RIDENUM) /* Floating-point registers (FPRs). */
28 RID_MAX,
29 RID_TMP = RID_LR,
30 RID_ZERO = RID_SP,
31
32 /* Calling conventions. */
33 RID_RET = RID_X0,
34 RID_RETLO = RID_X0,
35 RID_RETHI = RID_X1,
36 RID_FPRET = RID_D0,
37
38 /* These definitions must match with the *.dasc file(s): */
39 RID_BASE = RID_X19, /* Interpreter BASE. */
40 RID_LPC = RID_X21, /* Interpreter PC. */
41 RID_GL = RID_X22, /* Interpreter GL. */
42 RID_LREG = RID_X23, /* Interpreter L. */
43
44 /* Register ranges [min, max) and number of registers. */
45 RID_MIN_GPR = RID_X0,
46 RID_MAX_GPR = RID_SP+1,
47 RID_MIN_FPR = RID_MAX_GPR,
48 RID_MAX_FPR = RID_D31+1,
49 RID_NUM_GPR = RID_MAX_GPR - RID_MIN_GPR,
50 RID_NUM_FPR = RID_MAX_FPR - RID_MIN_FPR
51};
52
53#define RID_NUM_KREF RID_NUM_GPR
54#define RID_MIN_KREF RID_X0
55
56/* -- Register sets ------------------------------------------------------- */
57
58/* Make use of all registers, except for x18, fp, lr and sp. */
59#define RSET_FIXED \
60 (RID2RSET(RID_X18)|RID2RSET(RID_FP)|RID2RSET(RID_LR)|RID2RSET(RID_SP)|\
61 RID2RSET(RID_GL))
62#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR) - RSET_FIXED)
63#define RSET_FPR RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR)
64#define RSET_ALL (RSET_GPR|RSET_FPR)
65#define RSET_INIT RSET_ALL
66
67/* lr is an implicit scratch register. */
68#define RSET_SCRATCH_GPR (RSET_RANGE(RID_X0, RID_X17+1))
69#define RSET_SCRATCH_FPR \
70 (RSET_RANGE(RID_D0, RID_D7+1)|RSET_RANGE(RID_D16, RID_D31+1))
71#define RSET_SCRATCH (RSET_SCRATCH_GPR|RSET_SCRATCH_FPR)
72#define REGARG_FIRSTGPR RID_X0
73#define REGARG_LASTGPR RID_X7
74#define REGARG_NUMGPR 8
75#define REGARG_FIRSTFPR RID_D0
76#define REGARG_LASTFPR RID_D7
77#define REGARG_NUMFPR 8
78
79/* -- Spill slots --------------------------------------------------------- */
80
81/* Spill slots are 32 bit wide. An even/odd pair is used for FPRs.
82**
83** SPS_FIXED: Available fixed spill slots in interpreter frame.
84** This definition must match with the vm_arm64.dasc file.
85** Pre-allocate some slots to avoid sp adjust in every root trace.
86**
87** SPS_FIRST: First spill slot for general use. Reserve min. two 32 bit slots.
88*/
89#define SPS_FIXED 4
90#define SPS_FIRST 2
91
92#define SPOFS_TMP 0
93
94#define sps_scale(slot) (4 * (int32_t)(slot))
95#define sps_align(slot) (((slot) - SPS_FIXED + 3) & ~3)
96
97/* -- Exit state ---------------------------------------------------------- */
98
99/* This definition must match with the *.dasc file(s). */
100typedef struct {
101 lua_Number fpr[RID_NUM_FPR]; /* Floating-point registers. */
102 intptr_t gpr[RID_NUM_GPR]; /* General-purpose registers. */
103 int32_t spill[256]; /* Spill slots. */
104} ExitState;
105
106/* Highest exit + 1 indicates stack check. */
107#define EXITSTATE_CHECKEXIT 1
108
109/* Return the address of a per-trace exit stub. */
110static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno)
111{
112 while (*p == (LJ_LE ? 0xd503201f : 0x1f2003d5)) p++; /* Skip A64I_NOP. */
113 return p + 3 + exitno;
114}
115/* Avoid dependence on lj_jit.h if only including lj_target.h. */
116#define exitstub_trace_addr(T, exitno) \
117 exitstub_trace_addr_((MCode *)((char *)(T)->mcode + (T)->szmcode), (exitno))
118
119/* -- Instructions -------------------------------------------------------- */
120
121/* ARM64 instructions are always little-endian. Swap for ARM64BE. */
122#if LJ_BE
123#define A64I_LE(x) (lj_bswap(x))
124#else
125#define A64I_LE(x) (x)
126#endif
127
128/* Instruction fields. */
129#define A64F_D(r) (r)
130#define A64F_N(r) ((r) << 5)
131#define A64F_A(r) ((r) << 10)
132#define A64F_M(r) ((r) << 16)
133#define A64F_IMMS(x) ((x) << 10)
134#define A64F_IMMR(x) ((x) << 16)
135#define A64F_U16(x) ((x) << 5)
136#define A64F_U12(x) ((x) << 10)
137#define A64F_S26(x) (((uint32_t)(x) & 0x03ffffffu))
138#define A64F_S19(x) (((uint32_t)(x) & 0x7ffffu) << 5)
139#define A64F_S14(x) (((uint32_t)(x) & 0x3fffu) << 5)
140#define A64F_S9(x) ((x) << 12)
141#define A64F_BIT(x) ((x) << 19)
142#define A64F_SH(sh, x) (((sh) << 22) | ((x) << 10))
143#define A64F_EX(ex) (A64I_EX | ((ex) << 13))
144#define A64F_EXSH(ex,x) (A64I_EX | ((ex) << 13) | ((x) << 10))
145#define A64F_FP8(x) ((x) << 13)
146#define A64F_CC(cc) ((cc) << 12)
147#define A64F_LSL16(x) (((x) / 16) << 21)
148#define A64F_BSH(sh) ((sh) << 10)
149
150/* Check for valid field range. */
151#define A64F_S_OK(x, b) ((((x) + (1 << (b-1))) >> (b)) == 0)
152
153typedef enum A64Ins {
154 A64I_S = 0x20000000,
155 A64I_X = 0x80000000,
156 A64I_EX = 0x00200000,
157 A64I_ON = 0x00200000,
158 A64I_K12 = 0x1a000000,
159 A64I_K13 = 0x18000000,
160 A64I_LS_U = 0x01000000,
161 A64I_LS_S = 0x00800000,
162 A64I_LS_R = 0x01200800,
163 A64I_LS_SH = 0x00001000,
164 A64I_LS_UXTWx = 0x00004000,
165 A64I_LS_SXTWx = 0x0000c000,
166 A64I_LS_SXTXx = 0x0000e000,
167 A64I_LS_LSLx = 0x00006000,
168
169 A64I_ADDw = 0x0b000000,
170 A64I_ADDx = 0x8b000000,
171 A64I_ADDSw = 0x2b000000,
172 A64I_ADDSx = 0xab000000,
173 A64I_NEGw = 0x4b0003e0,
174 A64I_NEGx = 0xcb0003e0,
175 A64I_SUBw = 0x4b000000,
176 A64I_SUBx = 0xcb000000,
177 A64I_SUBSw = 0x6b000000,
178 A64I_SUBSx = 0xeb000000,
179
180 A64I_MULw = 0x1b007c00,
181 A64I_MULx = 0x9b007c00,
182 A64I_SMULL = 0x9b207c00,
183
184 A64I_ANDw = 0x0a000000,
185 A64I_ANDx = 0x8a000000,
186 A64I_ANDSw = 0x6a000000,
187 A64I_ANDSx = 0xea000000,
188 A64I_EORw = 0x4a000000,
189 A64I_EORx = 0xca000000,
190 A64I_ORRw = 0x2a000000,
191 A64I_ORRx = 0xaa000000,
192 A64I_TSTw = 0x6a00001f,
193 A64I_TSTx = 0xea00001f,
194
195 A64I_CMPw = 0x6b00001f,
196 A64I_CMPx = 0xeb00001f,
197 A64I_CMNw = 0x2b00001f,
198 A64I_CMNx = 0xab00001f,
199 A64I_CCMPw = 0x7a400000,
200 A64I_CCMPx = 0xfa400000,
201 A64I_CSELw = 0x1a800000,
202 A64I_CSELx = 0x9a800000,
203
204 A64I_ASRw = 0x13007c00,
205 A64I_ASRx = 0x9340fc00,
206 A64I_LSLx = 0xd3400000,
207 A64I_LSRx = 0xd340fc00,
208 A64I_SHRw = 0x1ac02000,
209 A64I_SHRx = 0x9ac02000, /* lsl/lsr/asr/ror x0, x0, x0 */
210 A64I_REVw = 0x5ac00800,
211 A64I_REVx = 0xdac00c00,
212
213 A64I_EXTRw = 0x13800000,
214 A64I_EXTRx = 0x93c00000,
215 A64I_BFMw = 0x33000000,
216 A64I_BFMx = 0xb3400000,
217 A64I_SBFMw = 0x13000000,
218 A64I_SBFMx = 0x93400000,
219 A64I_SXTBw = 0x13001c00,
220 A64I_SXTHw = 0x13003c00,
221 A64I_SXTW = 0x93407c00,
222 A64I_UBFMw = 0x53000000,
223 A64I_UBFMx = 0xd3400000,
224 A64I_UXTBw = 0x53001c00,
225 A64I_UXTHw = 0x53003c00,
226
227 A64I_MOVw = 0x2a0003e0,
228 A64I_MOVx = 0xaa0003e0,
229 A64I_MVNw = 0x2a2003e0,
230 A64I_MVNx = 0xaa2003e0,
231 A64I_MOVKw = 0x72800000,
232 A64I_MOVKx = 0xf2800000,
233 A64I_MOVZw = 0x52800000,
234 A64I_MOVZx = 0xd2800000,
235 A64I_MOVNw = 0x12800000,
236 A64I_MOVNx = 0x92800000,
237
238 A64I_LDRB = 0x39400000,
239 A64I_LDRH = 0x79400000,
240 A64I_LDRw = 0xb9400000,
241 A64I_LDRx = 0xf9400000,
242 A64I_LDRLw = 0x18000000,
243 A64I_LDRLx = 0x58000000,
244 A64I_STRB = 0x39000000,
245 A64I_STRH = 0x79000000,
246 A64I_STRw = 0xb9000000,
247 A64I_STRx = 0xf9000000,
248 A64I_STPw = 0x29000000,
249 A64I_STPx = 0xa9000000,
250 A64I_LDPw = 0x29400000,
251 A64I_LDPx = 0xa9400000,
252
253 A64I_B = 0x14000000,
254 A64I_BCC = 0x54000000,
255 A64I_BL = 0x94000000,
256 A64I_BR = 0xd61f0000,
257 A64I_BLR = 0xd63f0000,
258 A64I_TBZ = 0x36000000,
259 A64I_TBNZ = 0x37000000,
260 A64I_CBZ = 0x34000000,
261 A64I_CBNZ = 0x35000000,
262
263 A64I_BRAAZ = 0xd61f081f,
264 A64I_BLRAAZ = 0xd63f081f,
265
266 A64I_NOP = 0xd503201f,
267
268 /* FP */
269 A64I_FADDd = 0x1e602800,
270 A64I_FSUBd = 0x1e603800,
271 A64I_FMADDd = 0x1f400000,
272 A64I_FMSUBd = 0x1f408000,
273 A64I_FNMADDd = 0x1f600000,
274 A64I_FNMSUBd = 0x1f608000,
275 A64I_FMULd = 0x1e600800,
276 A64I_FDIVd = 0x1e601800,
277 A64I_FNEGd = 0x1e614000,
278 A64I_FABS = 0x1e60c000,
279 A64I_FSQRTd = 0x1e61c000,
280 A64I_LDRs = 0xbd400000,
281 A64I_LDRd = 0xfd400000,
282 A64I_STRs = 0xbd000000,
283 A64I_STRd = 0xfd000000,
284 A64I_LDPs = 0x2d400000,
285 A64I_LDPd = 0x6d400000,
286 A64I_STPs = 0x2d000000,
287 A64I_STPd = 0x6d000000,
288 A64I_FCMPd = 0x1e602000,
289 A64I_FCMPZd = 0x1e602008,
290 A64I_FCSELd = 0x1e600c00,
291 A64I_FRINTMd = 0x1e654000,
292 A64I_FRINTPd = 0x1e64c000,
293 A64I_FRINTZd = 0x1e65c000,
294
295 A64I_FCVT_F32_F64 = 0x1e624000,
296 A64I_FCVT_F64_F32 = 0x1e22c000,
297 A64I_FCVT_F32_S32 = 0x1e220000,
298 A64I_FCVT_F64_S32 = 0x1e620000,
299 A64I_FCVT_F32_U32 = 0x1e230000,
300 A64I_FCVT_F64_U32 = 0x1e630000,
301 A64I_FCVT_F32_S64 = 0x9e220000,
302 A64I_FCVT_F64_S64 = 0x9e620000,
303 A64I_FCVT_F32_U64 = 0x9e230000,
304 A64I_FCVT_F64_U64 = 0x9e630000,
305 A64I_FCVT_S32_F64 = 0x1e780000,
306 A64I_FCVT_S32_F32 = 0x1e380000,
307 A64I_FCVT_U32_F64 = 0x1e790000,
308 A64I_FCVT_U32_F32 = 0x1e390000,
309 A64I_FCVT_S64_F64 = 0x9e780000,
310 A64I_FCVT_S64_F32 = 0x9e380000,
311 A64I_FCVT_U64_F64 = 0x9e790000,
312 A64I_FCVT_U64_F32 = 0x9e390000,
313
314 A64I_FMOV_S = 0x1e204000,
315 A64I_FMOV_D = 0x1e604000,
316 A64I_FMOV_R_S = 0x1e260000,
317 A64I_FMOV_S_R = 0x1e270000,
318 A64I_FMOV_R_D = 0x9e660000,
319 A64I_FMOV_D_R = 0x9e670000,
320 A64I_FMOV_DI = 0x1e601000,
321} A64Ins;
322
323#define A64I_BR_AUTH (LJ_ABI_PAUTH ? A64I_BRAAZ : A64I_BR)
324#define A64I_BLR_AUTH (LJ_ABI_PAUTH ? A64I_BLRAAZ : A64I_BLR)
325
326typedef enum A64Shift {
327 A64SH_LSL, A64SH_LSR, A64SH_ASR, A64SH_ROR
328} A64Shift;
329
330typedef enum A64Extend {
331 A64EX_UXTB, A64EX_UXTH, A64EX_UXTW, A64EX_UXTX,
332 A64EX_SXTB, A64EX_SXTH, A64EX_SXTW, A64EX_SXTX,
333} A64Extend;
334
335/* ARM condition codes. */
336typedef enum A64CC {
337 CC_EQ, CC_NE, CC_CS, CC_CC, CC_MI, CC_PL, CC_VS, CC_VC,
338 CC_HI, CC_LS, CC_GE, CC_LT, CC_GT, CC_LE, CC_AL,
339 CC_HS = CC_CS, CC_LO = CC_CC
340} A64CC;
341
342#endif
diff --git a/src/lj_target_mips.h b/src/lj_target_mips.h
index 01ec0117..a3353884 100644
--- a/src/lj_target_mips.h
+++ b/src/lj_target_mips.h
@@ -13,11 +13,15 @@
13 _(R8) _(R9) _(R10) _(R11) _(R12) _(R13) _(R14) _(R15) \ 13 _(R8) _(R9) _(R10) _(R11) _(R12) _(R13) _(R14) _(R15) \
14 _(R16) _(R17) _(R18) _(R19) _(R20) _(R21) _(R22) _(R23) \ 14 _(R16) _(R17) _(R18) _(R19) _(R20) _(R21) _(R22) _(R23) \
15 _(R24) _(R25) _(SYS1) _(SYS2) _(R28) _(SP) _(R30) _(RA) 15 _(R24) _(R25) _(SYS1) _(SYS2) _(R28) _(SP) _(R30) _(RA)
16#if LJ_SOFTFP
17#define FPRDEF(_)
18#else
16#define FPRDEF(_) \ 19#define FPRDEF(_) \
17 _(F0) _(F1) _(F2) _(F3) _(F4) _(F5) _(F6) _(F7) \ 20 _(F0) _(F1) _(F2) _(F3) _(F4) _(F5) _(F6) _(F7) \
18 _(F8) _(F9) _(F10) _(F11) _(F12) _(F13) _(F14) _(F15) \ 21 _(F8) _(F9) _(F10) _(F11) _(F12) _(F13) _(F14) _(F15) \
19 _(F16) _(F17) _(F18) _(F19) _(F20) _(F21) _(F22) _(F23) \ 22 _(F16) _(F17) _(F18) _(F19) _(F20) _(F21) _(F22) _(F23) \
20 _(F24) _(F25) _(F26) _(F27) _(F28) _(F29) _(F30) _(F31) 23 _(F24) _(F25) _(F26) _(F27) _(F28) _(F29) _(F30) _(F31)
24#endif
21#define VRIDDEF(_) 25#define VRIDDEF(_)
22 26
23#define RIDENUM(name) RID_##name, 27#define RIDENUM(name) RID_##name,
@@ -39,7 +43,11 @@ enum {
39 RID_RETHI = RID_R2, 43 RID_RETHI = RID_R2,
40 RID_RETLO = RID_R3, 44 RID_RETLO = RID_R3,
41#endif 45#endif
46#if LJ_SOFTFP
47 RID_FPRET = RID_R2,
48#else
42 RID_FPRET = RID_F0, 49 RID_FPRET = RID_F0,
50#endif
43 RID_CFUNCADDR = RID_R25, 51 RID_CFUNCADDR = RID_R25,
44 52
45 /* These definitions must match with the *.dasc file(s): */ 53 /* These definitions must match with the *.dasc file(s): */
@@ -52,8 +60,12 @@ enum {
52 /* Register ranges [min, max) and number of registers. */ 60 /* Register ranges [min, max) and number of registers. */
53 RID_MIN_GPR = RID_R0, 61 RID_MIN_GPR = RID_R0,
54 RID_MAX_GPR = RID_RA+1, 62 RID_MAX_GPR = RID_RA+1,
55 RID_MIN_FPR = RID_F0, 63 RID_MIN_FPR = RID_MAX_GPR,
64#if LJ_SOFTFP
65 RID_MAX_FPR = RID_MIN_FPR,
66#else
56 RID_MAX_FPR = RID_F31+1, 67 RID_MAX_FPR = RID_F31+1,
68#endif
57 RID_NUM_GPR = RID_MAX_GPR - RID_MIN_GPR, 69 RID_NUM_GPR = RID_MAX_GPR - RID_MIN_GPR,
58 RID_NUM_FPR = RID_MAX_FPR - RID_MIN_FPR /* Only even regs are used. */ 70 RID_NUM_FPR = RID_MAX_FPR - RID_MIN_FPR /* Only even regs are used. */
59}; 71};
@@ -68,28 +80,60 @@ enum {
68 (RID2RSET(RID_ZERO)|RID2RSET(RID_TMP)|RID2RSET(RID_SP)|\ 80 (RID2RSET(RID_ZERO)|RID2RSET(RID_TMP)|RID2RSET(RID_SP)|\
69 RID2RSET(RID_SYS1)|RID2RSET(RID_SYS2)|RID2RSET(RID_JGL)|RID2RSET(RID_GP)) 81 RID2RSET(RID_SYS1)|RID2RSET(RID_SYS2)|RID2RSET(RID_JGL)|RID2RSET(RID_GP))
70#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR) - RSET_FIXED) 82#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR) - RSET_FIXED)
83#if LJ_SOFTFP
84#define RSET_FPR 0
85#else
86#if LJ_32
71#define RSET_FPR \ 87#define RSET_FPR \
72 (RID2RSET(RID_F0)|RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(RID_F6)|\ 88 (RID2RSET(RID_F0)|RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(RID_F6)|\
73 RID2RSET(RID_F8)|RID2RSET(RID_F10)|RID2RSET(RID_F12)|RID2RSET(RID_F14)|\ 89 RID2RSET(RID_F8)|RID2RSET(RID_F10)|RID2RSET(RID_F12)|RID2RSET(RID_F14)|\
74 RID2RSET(RID_F16)|RID2RSET(RID_F18)|RID2RSET(RID_F20)|RID2RSET(RID_F22)|\ 90 RID2RSET(RID_F16)|RID2RSET(RID_F18)|RID2RSET(RID_F20)|RID2RSET(RID_F22)|\
75 RID2RSET(RID_F24)|RID2RSET(RID_F26)|RID2RSET(RID_F28)|RID2RSET(RID_F30)) 91 RID2RSET(RID_F24)|RID2RSET(RID_F26)|RID2RSET(RID_F28)|RID2RSET(RID_F30))
76#define RSET_ALL (RSET_GPR|RSET_FPR) 92#else
77#define RSET_INIT RSET_ALL 93#define RSET_FPR RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR)
94#endif
95#endif
96#define RSET_ALL (RSET_GPR|RSET_FPR)
97#define RSET_INIT RSET_ALL
78 98
79#define RSET_SCRATCH_GPR \ 99#define RSET_SCRATCH_GPR \
80 (RSET_RANGE(RID_R1, RID_R15+1)|\ 100 (RSET_RANGE(RID_R1, RID_R15+1)|\
81 RID2RSET(RID_R24)|RID2RSET(RID_R25)) 101 RID2RSET(RID_R24)|RID2RSET(RID_R25))
102#if LJ_SOFTFP
103#define RSET_SCRATCH_FPR 0
104#else
105#if LJ_32
82#define RSET_SCRATCH_FPR \ 106#define RSET_SCRATCH_FPR \
83 (RID2RSET(RID_F0)|RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(RID_F6)|\ 107 (RID2RSET(RID_F0)|RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(RID_F6)|\
84 RID2RSET(RID_F8)|RID2RSET(RID_F10)|RID2RSET(RID_F12)|RID2RSET(RID_F14)|\ 108 RID2RSET(RID_F8)|RID2RSET(RID_F10)|RID2RSET(RID_F12)|RID2RSET(RID_F14)|\
85 RID2RSET(RID_F16)|RID2RSET(RID_F18)) 109 RID2RSET(RID_F16)|RID2RSET(RID_F18))
110#else
111#define RSET_SCRATCH_FPR RSET_RANGE(RID_F0, RID_F24)
112#endif
113#endif
86#define RSET_SCRATCH (RSET_SCRATCH_GPR|RSET_SCRATCH_FPR) 114#define RSET_SCRATCH (RSET_SCRATCH_GPR|RSET_SCRATCH_FPR)
87#define REGARG_FIRSTGPR RID_R4 115#define REGARG_FIRSTGPR RID_R4
116#if LJ_32
88#define REGARG_LASTGPR RID_R7 117#define REGARG_LASTGPR RID_R7
89#define REGARG_NUMGPR 4 118#define REGARG_NUMGPR 4
119#else
120#define REGARG_LASTGPR RID_R11
121#define REGARG_NUMGPR 8
122#endif
123#if LJ_ABI_SOFTFP
124#define REGARG_FIRSTFPR 0
125#define REGARG_LASTFPR 0
126#define REGARG_NUMFPR 0
127#else
90#define REGARG_FIRSTFPR RID_F12 128#define REGARG_FIRSTFPR RID_F12
129#if LJ_32
91#define REGARG_LASTFPR RID_F14 130#define REGARG_LASTFPR RID_F14
92#define REGARG_NUMFPR 2 131#define REGARG_NUMFPR 2
132#else
133#define REGARG_LASTFPR RID_F19
134#define REGARG_NUMFPR 8
135#endif
136#endif
93 137
94/* -- Spill slots --------------------------------------------------------- */ 138/* -- Spill slots --------------------------------------------------------- */
95 139
@@ -100,7 +144,11 @@ enum {
100** 144**
101** SPS_FIRST: First spill slot for general use. 145** SPS_FIRST: First spill slot for general use.
102*/ 146*/
147#if LJ_32
103#define SPS_FIXED 5 148#define SPS_FIXED 5
149#else
150#define SPS_FIXED 4
151#endif
104#define SPS_FIRST 4 152#define SPS_FIRST 4
105 153
106#define SPOFS_TMP 0 154#define SPOFS_TMP 0
@@ -112,8 +160,10 @@ enum {
112 160
113/* This definition must match with the *.dasc file(s). */ 161/* This definition must match with the *.dasc file(s). */
114typedef struct { 162typedef struct {
163#if !LJ_SOFTFP
115 lua_Number fpr[RID_NUM_FPR]; /* Floating-point registers. */ 164 lua_Number fpr[RID_NUM_FPR]; /* Floating-point registers. */
116 int32_t gpr[RID_NUM_GPR]; /* General-purpose registers. */ 165#endif
166 intptr_t gpr[RID_NUM_GPR]; /* General-purpose registers. */
117 int32_t spill[256]; /* Spill slots. */ 167 int32_t spill[256]; /* Spill slots. */
118} ExitState; 168} ExitState;
119 169
@@ -142,52 +192,87 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p)
142#define MIPSF_F(r) ((r) << 6) 192#define MIPSF_F(r) ((r) << 6)
143#define MIPSF_A(n) ((n) << 6) 193#define MIPSF_A(n) ((n) << 6)
144#define MIPSF_M(n) ((n) << 11) 194#define MIPSF_M(n) ((n) << 11)
195#define MIPSF_L(n) ((n) << 6)
145 196
146typedef enum MIPSIns { 197typedef enum MIPSIns {
198 MIPSI_D = 0x38,
199 MIPSI_DV = 0x10,
200 MIPSI_D32 = 0x3c,
147 /* Integer instructions. */ 201 /* Integer instructions. */
148 MIPSI_MOVE = 0x00000021, 202 MIPSI_MOVE = 0x00000025,
149 MIPSI_NOP = 0x00000000, 203 MIPSI_NOP = 0x00000000,
150 204
151 MIPSI_LI = 0x24000000, 205 MIPSI_LI = 0x24000000,
152 MIPSI_LU = 0x34000000, 206 MIPSI_LU = 0x34000000,
153 MIPSI_LUI = 0x3c000000, 207 MIPSI_LUI = 0x3c000000,
154 208
155 MIPSI_ADDIU = 0x24000000, 209 MIPSI_AND = 0x00000024,
156 MIPSI_ANDI = 0x30000000, 210 MIPSI_ANDI = 0x30000000,
211 MIPSI_OR = 0x00000025,
157 MIPSI_ORI = 0x34000000, 212 MIPSI_ORI = 0x34000000,
213 MIPSI_XOR = 0x00000026,
158 MIPSI_XORI = 0x38000000, 214 MIPSI_XORI = 0x38000000,
215 MIPSI_NOR = 0x00000027,
216
217 MIPSI_SLT = 0x0000002a,
218 MIPSI_SLTU = 0x0000002b,
159 MIPSI_SLTI = 0x28000000, 219 MIPSI_SLTI = 0x28000000,
160 MIPSI_SLTIU = 0x2c000000, 220 MIPSI_SLTIU = 0x2c000000,
161 221
162 MIPSI_ADDU = 0x00000021, 222 MIPSI_ADDU = 0x00000021,
223 MIPSI_ADDIU = 0x24000000,
224 MIPSI_SUB = 0x00000022,
163 MIPSI_SUBU = 0x00000023, 225 MIPSI_SUBU = 0x00000023,
226
227#if !LJ_TARGET_MIPSR6
164 MIPSI_MUL = 0x70000002, 228 MIPSI_MUL = 0x70000002,
165 MIPSI_AND = 0x00000024, 229 MIPSI_DIV = 0x0000001a,
166 MIPSI_OR = 0x00000025, 230 MIPSI_DIVU = 0x0000001b,
167 MIPSI_XOR = 0x00000026, 231
168 MIPSI_NOR = 0x00000027,
169 MIPSI_SLT = 0x0000002a,
170 MIPSI_SLTU = 0x0000002b,
171 MIPSI_MOVZ = 0x0000000a, 232 MIPSI_MOVZ = 0x0000000a,
172 MIPSI_MOVN = 0x0000000b, 233 MIPSI_MOVN = 0x0000000b,
234 MIPSI_MFHI = 0x00000010,
235 MIPSI_MFLO = 0x00000012,
236 MIPSI_MULT = 0x00000018,
237#else
238 MIPSI_MUL = 0x00000098,
239 MIPSI_MUH = 0x000000d8,
240 MIPSI_DIV = 0x0000009a,
241 MIPSI_DIVU = 0x0000009b,
242
243 MIPSI_SELEQZ = 0x00000035,
244 MIPSI_SELNEZ = 0x00000037,
245#endif
173 246
174 MIPSI_SLL = 0x00000000, 247 MIPSI_SLL = 0x00000000,
175 MIPSI_SRL = 0x00000002, 248 MIPSI_SRL = 0x00000002,
176 MIPSI_SRA = 0x00000003, 249 MIPSI_SRA = 0x00000003,
177 MIPSI_ROTR = 0x00200002, /* MIPS32R2 */ 250 MIPSI_ROTR = 0x00200002, /* MIPSXXR2 */
251 MIPSI_DROTR = 0x0020003a,
252 MIPSI_DROTR32 = 0x0020003e,
178 MIPSI_SLLV = 0x00000004, 253 MIPSI_SLLV = 0x00000004,
179 MIPSI_SRLV = 0x00000006, 254 MIPSI_SRLV = 0x00000006,
180 MIPSI_SRAV = 0x00000007, 255 MIPSI_SRAV = 0x00000007,
181 MIPSI_ROTRV = 0x00000046, /* MIPS32R2 */ 256 MIPSI_ROTRV = 0x00000046, /* MIPSXXR2 */
257 MIPSI_DROTRV = 0x00000056,
258
259 MIPSI_INS = 0x7c000004, /* MIPSXXR2 */
182 260
183 MIPSI_SEB = 0x7c000420, /* MIPS32R2 */ 261 MIPSI_SEB = 0x7c000420, /* MIPSXXR2 */
184 MIPSI_SEH = 0x7c000620, /* MIPS32R2 */ 262 MIPSI_SEH = 0x7c000620, /* MIPSXXR2 */
185 MIPSI_WSBH = 0x7c0000a0, /* MIPS32R2 */ 263 MIPSI_WSBH = 0x7c0000a0, /* MIPSXXR2 */
264 MIPSI_DSBH = 0x7c0000a4,
186 265
187 MIPSI_B = 0x10000000, 266 MIPSI_B = 0x10000000,
188 MIPSI_J = 0x08000000, 267 MIPSI_J = 0x08000000,
189 MIPSI_JAL = 0x0c000000, 268 MIPSI_JAL = 0x0c000000,
269#if !LJ_TARGET_MIPSR6
270 MIPSI_JALX = 0x74000000,
190 MIPSI_JR = 0x00000008, 271 MIPSI_JR = 0x00000008,
272#else
273 MIPSI_JR = 0x00000009,
274 MIPSI_BALC = 0xe8000000,
275#endif
191 MIPSI_JALR = 0x0000f809, 276 MIPSI_JALR = 0x0000f809,
192 277
193 MIPSI_BEQ = 0x10000000, 278 MIPSI_BEQ = 0x10000000,
@@ -199,7 +284,9 @@ typedef enum MIPSIns {
199 284
200 /* Load/store instructions. */ 285 /* Load/store instructions. */
201 MIPSI_LW = 0x8c000000, 286 MIPSI_LW = 0x8c000000,
287 MIPSI_LD = 0xdc000000,
202 MIPSI_SW = 0xac000000, 288 MIPSI_SW = 0xac000000,
289 MIPSI_SD = 0xfc000000,
203 MIPSI_LB = 0x80000000, 290 MIPSI_LB = 0x80000000,
204 MIPSI_SB = 0xa0000000, 291 MIPSI_SB = 0xa0000000,
205 MIPSI_LH = 0x84000000, 292 MIPSI_LH = 0x84000000,
@@ -211,11 +298,69 @@ typedef enum MIPSIns {
211 MIPSI_LDC1 = 0xd4000000, 298 MIPSI_LDC1 = 0xd4000000,
212 MIPSI_SDC1 = 0xf4000000, 299 MIPSI_SDC1 = 0xf4000000,
213 300
301 /* MIPS64 instructions. */
302 MIPSI_DADD = 0x0000002c,
303 MIPSI_DADDU = 0x0000002d,
304 MIPSI_DADDIU = 0x64000000,
305 MIPSI_DSUB = 0x0000002e,
306 MIPSI_DSUBU = 0x0000002f,
307#if !LJ_TARGET_MIPSR6
308 MIPSI_DDIV = 0x0000001e,
309 MIPSI_DDIVU = 0x0000001f,
310 MIPSI_DMULT = 0x0000001c,
311 MIPSI_DMULTU = 0x0000001d,
312#else
313 MIPSI_DDIV = 0x0000009e,
314 MIPSI_DMOD = 0x000000de,
315 MIPSI_DDIVU = 0x0000009f,
316 MIPSI_DMODU = 0x000000df,
317 MIPSI_DMUL = 0x0000009c,
318 MIPSI_DMUH = 0x000000dc,
319#endif
320
321 MIPSI_DSLL = 0x00000038,
322 MIPSI_DSRL = 0x0000003a,
323 MIPSI_DSLLV = 0x00000014,
324 MIPSI_DSRLV = 0x00000016,
325 MIPSI_DSRA = 0x0000003b,
326 MIPSI_DSRAV = 0x00000017,
327 MIPSI_DSRA32 = 0x0000003f,
328 MIPSI_DSLL32 = 0x0000003c,
329 MIPSI_DSRL32 = 0x0000003e,
330 MIPSI_DSHD = 0x7c000164,
331
332 MIPSI_AADDU = LJ_32 ? MIPSI_ADDU : MIPSI_DADDU,
333 MIPSI_AADDIU = LJ_32 ? MIPSI_ADDIU : MIPSI_DADDIU,
334 MIPSI_ASUBU = LJ_32 ? MIPSI_SUBU : MIPSI_DSUBU,
335 MIPSI_AL = LJ_32 ? MIPSI_LW : MIPSI_LD,
336 MIPSI_AS = LJ_32 ? MIPSI_SW : MIPSI_SD,
337#if LJ_TARGET_MIPSR6
338 MIPSI_LSA = 0x00000005,
339 MIPSI_DLSA = 0x00000015,
340 MIPSI_ALSA = LJ_32 ? MIPSI_LSA : MIPSI_DLSA,
341#endif
342
343 /* Extract/insert instructions. */
344 MIPSI_DEXTM = 0x7c000001,
345 MIPSI_DEXTU = 0x7c000002,
346 MIPSI_DEXT = 0x7c000003,
347 MIPSI_DINSM = 0x7c000005,
348 MIPSI_DINSU = 0x7c000006,
349 MIPSI_DINS = 0x7c000007,
350
351 MIPSI_FLOOR_D = 0x4620000b,
352
214 /* FP instructions. */ 353 /* FP instructions. */
215 MIPSI_MOV_S = 0x46000006, 354 MIPSI_MOV_S = 0x46000006,
216 MIPSI_MOV_D = 0x46200006, 355 MIPSI_MOV_D = 0x46200006,
356#if !LJ_TARGET_MIPSR6
217 MIPSI_MOVT_D = 0x46210011, 357 MIPSI_MOVT_D = 0x46210011,
218 MIPSI_MOVF_D = 0x46200011, 358 MIPSI_MOVF_D = 0x46200011,
359#else
360 MIPSI_MIN_D = 0x4620001C,
361 MIPSI_MAX_D = 0x4620001E,
362 MIPSI_SEL_D = 0x46200010,
363#endif
219 364
220 MIPSI_ABS_D = 0x46200005, 365 MIPSI_ABS_D = 0x46200005,
221 MIPSI_NEG_D = 0x46200007, 366 MIPSI_NEG_D = 0x46200007,
@@ -235,23 +380,37 @@ typedef enum MIPSIns {
235 MIPSI_CVT_W_D = 0x46200024, 380 MIPSI_CVT_W_D = 0x46200024,
236 MIPSI_CVT_S_W = 0x46800020, 381 MIPSI_CVT_S_W = 0x46800020,
237 MIPSI_CVT_D_W = 0x46800021, 382 MIPSI_CVT_D_W = 0x46800021,
383 MIPSI_CVT_S_L = 0x46a00020,
384 MIPSI_CVT_D_L = 0x46a00021,
238 385
239 MIPSI_TRUNC_W_S = 0x4600000d, 386 MIPSI_TRUNC_W_S = 0x4600000d,
240 MIPSI_TRUNC_W_D = 0x4620000d, 387 MIPSI_TRUNC_W_D = 0x4620000d,
388 MIPSI_TRUNC_L_S = 0x46000009,
389 MIPSI_TRUNC_L_D = 0x46200009,
241 MIPSI_FLOOR_W_S = 0x4600000f, 390 MIPSI_FLOOR_W_S = 0x4600000f,
242 MIPSI_FLOOR_W_D = 0x4620000f, 391 MIPSI_FLOOR_W_D = 0x4620000f,
243 392
244 MIPSI_MFC1 = 0x44000000, 393 MIPSI_MFC1 = 0x44000000,
245 MIPSI_MTC1 = 0x44800000, 394 MIPSI_MTC1 = 0x44800000,
395 MIPSI_DMTC1 = 0x44a00000,
396 MIPSI_DMFC1 = 0x44200000,
246 397
398#if !LJ_TARGET_MIPSR6
247 MIPSI_BC1F = 0x45000000, 399 MIPSI_BC1F = 0x45000000,
248 MIPSI_BC1T = 0x45010000, 400 MIPSI_BC1T = 0x45010000,
249
250 MIPSI_C_EQ_D = 0x46200032, 401 MIPSI_C_EQ_D = 0x46200032,
402 MIPSI_C_OLT_S = 0x46000034,
251 MIPSI_C_OLT_D = 0x46200034, 403 MIPSI_C_OLT_D = 0x46200034,
252 MIPSI_C_ULT_D = 0x46200035, 404 MIPSI_C_ULT_D = 0x46200035,
253 MIPSI_C_OLE_D = 0x46200036, 405 MIPSI_C_OLE_D = 0x46200036,
254 MIPSI_C_ULE_D = 0x46200037, 406 MIPSI_C_ULE_D = 0x46200037,
407#else
408 MIPSI_BC1EQZ = 0x45200000,
409 MIPSI_BC1NEZ = 0x45a00000,
410 MIPSI_CMP_EQ_D = 0x46a00002,
411 MIPSI_CMP_LT_S = 0x46800004,
412 MIPSI_CMP_LT_D = 0x46a00004,
413#endif
255 414
256} MIPSIns; 415} MIPSIns;
257 416
diff --git a/src/lj_target_ppc.h b/src/lj_target_ppc.h
index d881c540..50620c50 100644
--- a/src/lj_target_ppc.h
+++ b/src/lj_target_ppc.h
@@ -104,7 +104,7 @@ enum {
104/* This definition must match with the *.dasc file(s). */ 104/* This definition must match with the *.dasc file(s). */
105typedef struct { 105typedef struct {
106 lua_Number fpr[RID_NUM_FPR]; /* Floating-point registers. */ 106 lua_Number fpr[RID_NUM_FPR]; /* Floating-point registers. */
107 int32_t gpr[RID_NUM_GPR]; /* General-purpose registers. */ 107 intptr_t gpr[RID_NUM_GPR]; /* General-purpose registers. */
108 int32_t spill[256]; /* Spill slots. */ 108 int32_t spill[256]; /* Spill slots. */
109} ExitState; 109} ExitState;
110 110
diff --git a/src/lj_target_x86.h b/src/lj_target_x86.h
index 7d0e5e6d..7b8d62ad 100644
--- a/src/lj_target_x86.h
+++ b/src/lj_target_x86.h
@@ -22,7 +22,7 @@
22 _(XMM0) _(XMM1) _(XMM2) _(XMM3) _(XMM4) _(XMM5) _(XMM6) _(XMM7) 22 _(XMM0) _(XMM1) _(XMM2) _(XMM3) _(XMM4) _(XMM5) _(XMM6) _(XMM7)
23#endif 23#endif
24#define VRIDDEF(_) \ 24#define VRIDDEF(_) \
25 _(MRM) 25 _(MRM) _(RIP)
26 26
27#define RIDENUM(name) RID_##name, 27#define RIDENUM(name) RID_##name,
28 28
@@ -31,15 +31,16 @@ enum {
31 FPRDEF(RIDENUM) /* Floating-point registers (FPRs). */ 31 FPRDEF(RIDENUM) /* Floating-point registers (FPRs). */
32 RID_MAX, 32 RID_MAX,
33 RID_MRM = RID_MAX, /* Pseudo-id for ModRM operand. */ 33 RID_MRM = RID_MAX, /* Pseudo-id for ModRM operand. */
34 RID_RIP = RID_MAX+5, /* Pseudo-id for RIP (x64 only), rm bits = 5. */
34 35
35 /* Calling conventions. */ 36 /* Calling conventions. */
37 RID_SP = RID_ESP,
36 RID_RET = RID_EAX, 38 RID_RET = RID_EAX,
37#if LJ_64 39#if LJ_64
38 RID_FPRET = RID_XMM0, 40 RID_FPRET = RID_XMM0,
39#else 41#endif
40 RID_RETLO = RID_EAX, 42 RID_RETLO = RID_EAX,
41 RID_RETHI = RID_EDX, 43 RID_RETHI = RID_EDX,
42#endif
43 44
44 /* These definitions must match with the *.dasc file(s): */ 45 /* These definitions must match with the *.dasc file(s): */
45 RID_BASE = RID_EDX, /* Interpreter BASE. */ 46 RID_BASE = RID_EDX, /* Interpreter BASE. */
@@ -62,8 +63,10 @@ enum {
62 63
63/* -- Register sets ------------------------------------------------------- */ 64/* -- Register sets ------------------------------------------------------- */
64 65
65/* Make use of all registers, except the stack pointer. */ 66/* Make use of all registers, except the stack pointer (and maybe DISPATCH). */
66#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR)-RID2RSET(RID_ESP)) 67#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR) \
68 - RID2RSET(RID_ESP) \
69 - LJ_GC64*RID2RSET(RID_DISPATCH))
67#define RSET_FPR (RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR)) 70#define RSET_FPR (RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR))
68#define RSET_ALL (RSET_GPR|RSET_FPR) 71#define RSET_ALL (RSET_GPR|RSET_FPR)
69#define RSET_INIT RSET_ALL 72#define RSET_INIT RSET_ALL
@@ -131,7 +134,11 @@ enum {
131#define SPS_FIXED (4*2) 134#define SPS_FIXED (4*2)
132#define SPS_FIRST (4*2) /* Don't use callee register save area. */ 135#define SPS_FIRST (4*2) /* Don't use callee register save area. */
133#else 136#else
137#if LJ_GC64
138#define SPS_FIXED 2
139#else
134#define SPS_FIXED 4 140#define SPS_FIXED 4
141#endif
135#define SPS_FIRST 2 142#define SPS_FIRST 2
136#endif 143#endif
137#else 144#else
@@ -157,6 +164,8 @@ typedef struct {
157#define EXITSTUB_SPACING (2+2) 164#define EXITSTUB_SPACING (2+2)
158#define EXITSTUBS_PER_GROUP 32 165#define EXITSTUBS_PER_GROUP 32
159 166
167#define EXITTRACE_VMSTATE 1 /* g->vmstate has traceno on exit. */
168
160/* -- x86 ModRM operand encoding ------------------------------------------ */ 169/* -- x86 ModRM operand encoding ------------------------------------------ */
161 170
162typedef enum { 171typedef enum {
@@ -184,12 +193,18 @@ typedef struct {
184#define XO_f20f(o) ((uint32_t)(0x0ff2fc + (0x##o<<24))) 193#define XO_f20f(o) ((uint32_t)(0x0ff2fc + (0x##o<<24)))
185#define XO_f30f(o) ((uint32_t)(0x0ff3fc + (0x##o<<24))) 194#define XO_f30f(o) ((uint32_t)(0x0ff3fc + (0x##o<<24)))
186 195
196#define XV_660f38(o) ((uint32_t)(0x79e2c4 + (0x##o<<24)))
197#define XV_f20f38(o) ((uint32_t)(0x7be2c4 + (0x##o<<24)))
198#define XV_f20f3a(o) ((uint32_t)(0x7be3c4 + (0x##o<<24)))
199#define XV_f30f38(o) ((uint32_t)(0x7ae2c4 + (0x##o<<24)))
200
187/* This list of x86 opcodes is not intended to be complete. Opcodes are only 201/* This list of x86 opcodes is not intended to be complete. Opcodes are only
188** included when needed. Take a look at DynASM or jit.dis_x86 to see the 202** included when needed. Take a look at DynASM or jit.dis_x86 to see the
189** whole mess. 203** whole mess.
190*/ 204*/
191typedef enum { 205typedef enum {
192 /* Fixed length opcodes. XI_* prefix. */ 206 /* Fixed length opcodes. XI_* prefix. */
207 XI_O16 = 0x66,
193 XI_NOP = 0x90, 208 XI_NOP = 0x90,
194 XI_XCHGa = 0x90, 209 XI_XCHGa = 0x90,
195 XI_CALL = 0xe8, 210 XI_CALL = 0xe8,
@@ -207,26 +222,28 @@ typedef enum {
207 XI_PUSHi8 = 0x6a, 222 XI_PUSHi8 = 0x6a,
208 XI_TESTb = 0x84, 223 XI_TESTb = 0x84,
209 XI_TEST = 0x85, 224 XI_TEST = 0x85,
225 XI_INT3 = 0xcc,
210 XI_MOVmi = 0xc7, 226 XI_MOVmi = 0xc7,
211 XI_GROUP5 = 0xff, 227 XI_GROUP5 = 0xff,
212 228
213 /* Note: little-endian byte-order! */ 229 /* Note: little-endian byte-order! */
214 XI_FLDZ = 0xeed9, 230 XI_FLDZ = 0xeed9,
215 XI_FLD1 = 0xe8d9, 231 XI_FLD1 = 0xe8d9,
216 XI_FLDLG2 = 0xecd9,
217 XI_FLDLN2 = 0xedd9,
218 XI_FDUP = 0xc0d9, /* Really fld st0. */ 232 XI_FDUP = 0xc0d9, /* Really fld st0. */
219 XI_FPOP = 0xd8dd, /* Really fstp st0. */ 233 XI_FPOP = 0xd8dd, /* Really fstp st0. */
220 XI_FPOP1 = 0xd9dd, /* Really fstp st1. */ 234 XI_FPOP1 = 0xd9dd, /* Really fstp st1. */
221 XI_FRNDINT = 0xfcd9, 235 XI_FRNDINT = 0xfcd9,
222 XI_FSIN = 0xfed9,
223 XI_FCOS = 0xffd9,
224 XI_FPTAN = 0xf2d9,
225 XI_FPATAN = 0xf3d9,
226 XI_FSCALE = 0xfdd9, 236 XI_FSCALE = 0xfdd9,
227 XI_FYL2X = 0xf1d9, 237 XI_FYL2X = 0xf1d9,
228 238
239 /* VEX-encoded instructions. XV_* prefix. */
240 XV_RORX = XV_f20f3a(f0),
241 XV_SARX = XV_f30f38(f7),
242 XV_SHLX = XV_660f38(f7),
243 XV_SHRX = XV_f20f38(f7),
244
229 /* Variable-length opcodes. XO_* prefix. */ 245 /* Variable-length opcodes. XO_* prefix. */
246 XO_OR = XO_(0b),
230 XO_MOV = XO_(8b), 247 XO_MOV = XO_(8b),
231 XO_MOVto = XO_(89), 248 XO_MOVto = XO_(89),
232 XO_MOVtow = XO_66(89), 249 XO_MOVtow = XO_66(89),
@@ -277,10 +294,8 @@ typedef enum {
277 XO_ROUNDSD = 0x0b3a0ffc, /* Really 66 0f 3a 0b. See asm_fpmath. */ 294 XO_ROUNDSD = 0x0b3a0ffc, /* Really 66 0f 3a 0b. See asm_fpmath. */
278 XO_UCOMISD = XO_660f(2e), 295 XO_UCOMISD = XO_660f(2e),
279 XO_CVTSI2SD = XO_f20f(2a), 296 XO_CVTSI2SD = XO_f20f(2a),
280 XO_CVTSD2SI = XO_f20f(2d),
281 XO_CVTTSD2SI= XO_f20f(2c), 297 XO_CVTTSD2SI= XO_f20f(2c),
282 XO_CVTSI2SS = XO_f30f(2a), 298 XO_CVTSI2SS = XO_f30f(2a),
283 XO_CVTSS2SI = XO_f30f(2d),
284 XO_CVTTSS2SI= XO_f30f(2c), 299 XO_CVTTSS2SI= XO_f30f(2c),
285 XO_CVTSS2SD = XO_f30f(5a), 300 XO_CVTSS2SD = XO_f30f(5a),
286 XO_CVTSD2SS = XO_f20f(5a), 301 XO_CVTSD2SS = XO_f20f(5a),
diff --git a/src/lj_trace.c b/src/lj_trace.c
index a72e73a3..f311d54b 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -30,6 +30,7 @@
30#include "lj_vm.h" 30#include "lj_vm.h"
31#include "lj_vmevent.h" 31#include "lj_vmevent.h"
32#include "lj_target.h" 32#include "lj_target.h"
33#include "lj_prng.h"
33 34
34/* -- Error handling ------------------------------------------------------ */ 35/* -- Error handling ------------------------------------------------------ */
35 36
@@ -104,7 +105,8 @@ static void perftools_addtrace(GCtrace *T)
104 name++; 105 name++;
105 else 106 else
106 name = "(string)"; 107 name = "(string)";
107 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc); 108 lj_assertX(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc,
109 "trace PC out of range");
108 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); 110 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
109 if (!fp) { 111 if (!fp) {
110 char fname[40]; 112 char fname[40];
@@ -117,15 +119,26 @@ static void perftools_addtrace(GCtrace *T)
117} 119}
118#endif 120#endif
119 121
120/* Allocate space for copy of trace. */ 122/* Allocate space for copy of T. */
121static GCtrace *trace_save_alloc(jit_State *J) 123GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
122{ 124{
123 size_t sztr = ((sizeof(GCtrace)+7)&~7); 125 size_t sztr = ((sizeof(GCtrace)+7)&~7);
124 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); 126 size_t szins = (T->nins-T->nk)*sizeof(IRIns);
125 size_t sz = sztr + szins + 127 size_t sz = sztr + szins +
126 J->cur.nsnap*sizeof(SnapShot) + 128 T->nsnap*sizeof(SnapShot) +
127 J->cur.nsnapmap*sizeof(SnapEntry); 129 T->nsnapmap*sizeof(SnapEntry);
128 return lj_mem_newt(J->L, (MSize)sz, GCtrace); 130 GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
131 char *p = (char *)T2 + sztr;
132 T2->gct = ~LJ_TTRACE;
133 T2->marked = 0;
134 T2->traceno = 0;
135 T2->ir = (IRIns *)p - T->nk;
136 T2->nins = T->nins;
137 T2->nk = T->nk;
138 T2->nsnap = T->nsnap;
139 T2->nsnapmap = T->nsnapmap;
140 memcpy(p, T->ir + T->nk, szins);
141 return T2;
129} 142}
130 143
131/* Save current trace by copying and compacting it. */ 144/* Save current trace by copying and compacting it. */
@@ -139,12 +152,15 @@ static void trace_save(jit_State *J, GCtrace *T)
139 setgcrefp(J2G(J)->gc.root, T); 152 setgcrefp(J2G(J)->gc.root, T);
140 newwhite(J2G(J), T); 153 newwhite(J2G(J), T);
141 T->gct = ~LJ_TTRACE; 154 T->gct = ~LJ_TTRACE;
142 T->ir = (IRIns *)p - J->cur.nk; 155 T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */
143 memcpy(p, J->cur.ir+J->cur.nk, szins); 156#if LJ_ABI_PAUTH
157 T->mcauth = lj_ptr_sign((ASMFunction)T->mcode, T);
158#endif
144 p += szins; 159 p += szins;
145 TRACE_APPENDVEC(snap, nsnap, SnapShot) 160 TRACE_APPENDVEC(snap, nsnap, SnapShot)
146 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) 161 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
147 J->cur.traceno = 0; 162 J->cur.traceno = 0;
163 J->curfinal = NULL;
148 setgcrefp(J->trace[T->traceno], T); 164 setgcrefp(J->trace[T->traceno], T);
149 lj_gc_barriertrace(J2G(J), T->traceno); 165 lj_gc_barriertrace(J2G(J), T->traceno);
150 lj_gdbjit_addtrace(J, T); 166 lj_gdbjit_addtrace(J, T);
@@ -172,7 +188,7 @@ void lj_trace_reenableproto(GCproto *pt)
172{ 188{
173 if ((pt->flags & PROTO_ILOOP)) { 189 if ((pt->flags & PROTO_ILOOP)) {
174 BCIns *bc = proto_bc(pt); 190 BCIns *bc = proto_bc(pt);
175 BCPos i, sizebc = pt->sizebc;; 191 BCPos i, sizebc = pt->sizebc;
176 pt->flags &= ~PROTO_ILOOP; 192 pt->flags &= ~PROTO_ILOOP;
177 if (bc_op(bc[0]) == BC_IFUNCF) 193 if (bc_op(bc[0]) == BC_IFUNCF)
178 setbc_op(&bc[0], BC_FUNCF); 194 setbc_op(&bc[0], BC_FUNCF);
@@ -194,27 +210,28 @@ static void trace_unpatch(jit_State *J, GCtrace *T)
194 return; /* No need to unpatch branches in parent traces (yet). */ 210 return; /* No need to unpatch branches in parent traces (yet). */
195 switch (bc_op(*pc)) { 211 switch (bc_op(*pc)) {
196 case BC_JFORL: 212 case BC_JFORL:
197 lua_assert(traceref(J, bc_d(*pc)) == T); 213 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JFORL references other trace");
198 *pc = T->startins; 214 *pc = T->startins;
199 pc += bc_j(T->startins); 215 pc += bc_j(T->startins);
200 lua_assert(bc_op(*pc) == BC_JFORI); 216 lj_assertJ(bc_op(*pc) == BC_JFORI, "FORL does not point to JFORI");
201 setbc_op(pc, BC_FORI); 217 setbc_op(pc, BC_FORI);
202 break; 218 break;
203 case BC_JITERL: 219 case BC_JITERL:
204 case BC_JLOOP: 220 case BC_JLOOP:
205 lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op)); 221 lj_assertJ(op == BC_ITERL || op == BC_ITERN || op == BC_LOOP ||
222 bc_isret(op), "bad original bytecode %d", op);
206 *pc = T->startins; 223 *pc = T->startins;
207 break; 224 break;
208 case BC_JMP: 225 case BC_JMP:
209 lua_assert(op == BC_ITERL); 226 lj_assertJ(op == BC_ITERL, "bad original bytecode %d", op);
210 pc += bc_j(*pc)+2; 227 pc += bc_j(*pc)+2;
211 if (bc_op(*pc) == BC_JITERL) { 228 if (bc_op(*pc) == BC_JITERL) {
212 lua_assert(traceref(J, bc_d(*pc)) == T); 229 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JITERL references other trace");
213 *pc = T->startins; 230 *pc = T->startins;
214 } 231 }
215 break; 232 break;
216 case BC_JFUNCF: 233 case BC_JFUNCF:
217 lua_assert(op == BC_FUNCF); 234 lj_assertJ(op == BC_FUNCF, "bad original bytecode %d", op);
218 *pc = T->startins; 235 *pc = T->startins;
219 break; 236 break;
220 default: /* Already unpatched. */ 237 default: /* Already unpatched. */
@@ -226,7 +243,8 @@ static void trace_unpatch(jit_State *J, GCtrace *T)
226static void trace_flushroot(jit_State *J, GCtrace *T) 243static void trace_flushroot(jit_State *J, GCtrace *T)
227{ 244{
228 GCproto *pt = &gcref(T->startpt)->pt; 245 GCproto *pt = &gcref(T->startpt)->pt;
229 lua_assert(T->root == 0 && pt != NULL); 246 lj_assertJ(T->root == 0, "not a root trace");
247 lj_assertJ(pt != NULL, "trace has no prototype");
230 /* First unpatch any modified bytecode. */ 248 /* First unpatch any modified bytecode. */
231 trace_unpatch(J, T); 249 trace_unpatch(J, T);
232 /* Unlink root trace from chain anchored in prototype. */ 250 /* Unlink root trace from chain anchored in prototype. */
@@ -274,7 +292,7 @@ int lj_trace_flushall(lua_State *L)
274 if (T->root == 0) 292 if (T->root == 0)
275 trace_flushroot(J, T); 293 trace_flushroot(J, T);
276 lj_gdbjit_deltrace(J, T); 294 lj_gdbjit_deltrace(J, T);
277 T->traceno = 0; 295 T->traceno = T->link = 0; /* Blacklist the link for cont_stitch. */
278 setgcrefnull(J->trace[i]); 296 setgcrefnull(J->trace[i]);
279 } 297 }
280 } 298 }
@@ -296,13 +314,42 @@ void lj_trace_initstate(global_State *g)
296{ 314{
297 jit_State *J = G2J(g); 315 jit_State *J = G2J(g);
298 TValue *tv; 316 TValue *tv;
299 /* Initialize SIMD constants. */ 317
318 /* Initialize aligned SIMD constants. */
300 tv = LJ_KSIMD(J, LJ_KSIMD_ABS); 319 tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
301 tv[0].u64 = U64x(7fffffff,ffffffff); 320 tv[0].u64 = U64x(7fffffff,ffffffff);
302 tv[1].u64 = U64x(7fffffff,ffffffff); 321 tv[1].u64 = U64x(7fffffff,ffffffff);
303 tv = LJ_KSIMD(J, LJ_KSIMD_NEG); 322 tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
304 tv[0].u64 = U64x(80000000,00000000); 323 tv[0].u64 = U64x(80000000,00000000);
305 tv[1].u64 = U64x(80000000,00000000); 324 tv[1].u64 = U64x(80000000,00000000);
325
326 /* Initialize 32/64 bit constants. */
327#if LJ_TARGET_X86ORX64
328 J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
329#if LJ_32
330 J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
331#endif
332 J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
333 J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
334#endif
335#if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
336 J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
337#endif
338#if LJ_TARGET_PPC
339 J->k32[LJ_K32_2P52_2P31] = 0x59800004;
340 J->k32[LJ_K32_2P52] = 0x59800000;
341#endif
342#if LJ_TARGET_PPC || LJ_TARGET_MIPS
343 J->k32[LJ_K32_2P31] = 0x4f000000;
344#endif
345#if LJ_TARGET_MIPS
346 J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
347#if LJ_64
348 J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
349 J->k32[LJ_K32_2P63] = 0x5f000000;
350 J->k32[LJ_K32_M2P64] = 0xdf800000;
351#endif
352#endif
306} 353}
307 354
308/* Free everything associated with the JIT compiler state. */ 355/* Free everything associated with the JIT compiler state. */
@@ -313,11 +360,11 @@ void lj_trace_freestate(global_State *g)
313 { /* This assumes all traces have already been freed. */ 360 { /* This assumes all traces have already been freed. */
314 ptrdiff_t i; 361 ptrdiff_t i;
315 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++) 362 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
316 lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL); 363 lj_assertG(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL,
364 "trace still allocated");
317 } 365 }
318#endif 366#endif
319 lj_mcode_free(J); 367 lj_mcode_free(J);
320 lj_ir_k64_freeall(J);
321 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry); 368 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
322 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot); 369 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
323 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns); 370 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
@@ -329,8 +376,13 @@ void lj_trace_freestate(global_State *g)
329/* Blacklist a bytecode instruction. */ 376/* Blacklist a bytecode instruction. */
330static void blacklist_pc(GCproto *pt, BCIns *pc) 377static void blacklist_pc(GCproto *pt, BCIns *pc)
331{ 378{
332 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP); 379 if (bc_op(*pc) == BC_ITERN) {
333 pt->flags |= PROTO_ILOOP; 380 setbc_op(pc, BC_ITERC);
381 setbc_op(pc+1+bc_j(pc[1]), BC_JMP);
382 } else {
383 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
384 pt->flags |= PROTO_ILOOP;
385 }
334} 386}
335 387
336/* Penalize a bytecode instruction. */ 388/* Penalize a bytecode instruction. */
@@ -341,7 +393,7 @@ static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
341 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */ 393 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */
342 /* First try to bump its hotcount several times. */ 394 /* First try to bump its hotcount several times. */
343 val = ((uint32_t)J->penalty[i].val << 1) + 395 val = ((uint32_t)J->penalty[i].val << 1) +
344 LJ_PRNG_BITS(J, PENALTY_RNDBITS); 396 (lj_prng_u64(&J2G(J)->prng) & ((1u<<PENALTY_RNDBITS)-1));
345 if (val > PENALTY_MAX) { 397 if (val > PENALTY_MAX) {
346 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */ 398 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */
347 return; 399 return;
@@ -367,10 +419,11 @@ static void trace_start(jit_State *J)
367 TraceNo traceno; 419 TraceNo traceno;
368 420
369 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */ 421 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */
370 if (J->parent == 0) { 422 if (J->parent == 0 && J->exitno == 0 && bc_op(*J->pc) != BC_ITERN) {
371 /* Lazy bytecode patching to disable hotcount events. */ 423 /* Lazy bytecode patching to disable hotcount events. */
372 lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL || 424 lj_assertJ(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
373 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF); 425 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF,
426 "bad hot bytecode %d", bc_op(*J->pc));
374 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP); 427 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
375 J->pt->flags |= PROTO_ILOOP; 428 J->pt->flags |= PROTO_ILOOP;
376 } 429 }
@@ -378,10 +431,17 @@ static void trace_start(jit_State *J)
378 return; 431 return;
379 } 432 }
380 433
434 /* Ensuring forward progress for BC_ITERN can trigger hotcount again. */
435 if (!J->parent && bc_op(*J->pc) == BC_JLOOP) { /* Already compiled. */
436 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
437 return;
438 }
439
381 /* Get a new trace number. */ 440 /* Get a new trace number. */
382 traceno = trace_findfree(J); 441 traceno = trace_findfree(J);
383 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */ 442 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */
384 lua_assert((J2G(J)->hookmask & HOOK_GC) == 0); 443 lj_assertJ((J2G(J)->hookmask & HOOK_GC) == 0,
444 "recorder called from GC hook");
385 lj_trace_flushall(J->L); 445 lj_trace_flushall(J->L);
386 J->state = LJ_TRACE_IDLE; /* Silently ignored. */ 446 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
387 return; 447 return;
@@ -401,6 +461,8 @@ static void trace_start(jit_State *J)
401 J->guardemit.irt = 0; 461 J->guardemit.irt = 0;
402 J->postproc = LJ_POST_NONE; 462 J->postproc = LJ_POST_NONE;
403 lj_resetsplit(J); 463 lj_resetsplit(J);
464 J->retryrec = 0;
465 J->ktrace = 0;
404 setgcref(J->cur.startpt, obj2gco(J->pt)); 466 setgcref(J->cur.startpt, obj2gco(J->pt));
405 467
406 L = J->L; 468 L = J->L;
@@ -412,6 +474,12 @@ static void trace_start(jit_State *J)
412 if (J->parent) { 474 if (J->parent) {
413 setintV(L->top++, J->parent); 475 setintV(L->top++, J->parent);
414 setintV(L->top++, J->exitno); 476 setintV(L->top++, J->exitno);
477 } else {
478 BCOp op = bc_op(*J->pc);
479 if (op == BC_CALLM || op == BC_CALL || op == BC_ITERC) {
480 setintV(L->top++, J->exitno); /* Parent of stitched trace. */
481 setintV(L->top++, -1);
482 }
415 } 483 }
416 ); 484 );
417 lj_record_setup(J); 485 lj_record_setup(J);
@@ -424,7 +492,7 @@ static void trace_stop(jit_State *J)
424 BCOp op = bc_op(J->cur.startins); 492 BCOp op = bc_op(J->cur.startins);
425 GCproto *pt = &gcref(J->cur.startpt)->pt; 493 GCproto *pt = &gcref(J->cur.startpt)->pt;
426 TraceNo traceno = J->cur.traceno; 494 TraceNo traceno = J->cur.traceno;
427 GCtrace *T = trace_save_alloc(J); /* Do this first. May throw OOM. */ 495 GCtrace *T = J->curfinal;
428 lua_State *L; 496 lua_State *L;
429 497
430 switch (op) { 498 switch (op) {
@@ -442,6 +510,7 @@ static void trace_stop(jit_State *J)
442 J->cur.nextroot = pt->trace; 510 J->cur.nextroot = pt->trace;
443 pt->trace = (TraceNo1)traceno; 511 pt->trace = (TraceNo1)traceno;
444 break; 512 break;
513 case BC_ITERN:
445 case BC_RET: 514 case BC_RET:
446 case BC_RET0: 515 case BC_RET0:
447 case BC_RET1: 516 case BC_RET1:
@@ -449,7 +518,7 @@ static void trace_stop(jit_State *J)
449 goto addroot; 518 goto addroot;
450 case BC_JMP: 519 case BC_JMP:
451 /* Patch exit branch in parent to side trace entry. */ 520 /* Patch exit branch in parent to side trace entry. */
452 lua_assert(J->parent != 0 && J->cur.root != 0); 521 lj_assertJ(J->parent != 0 && J->cur.root != 0, "not a side trace");
453 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode); 522 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
454 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */ 523 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
455 { 524 {
@@ -465,8 +534,14 @@ static void trace_stop(jit_State *J)
465 root->nextside = (TraceNo1)traceno; 534 root->nextside = (TraceNo1)traceno;
466 } 535 }
467 break; 536 break;
537 case BC_CALLM:
538 case BC_CALL:
539 case BC_ITERC:
540 /* Trace stitching: patch link of previous trace. */
541 traceref(J, J->exitno)->link = traceno;
542 break;
468 default: 543 default:
469 lua_assert(0); 544 lj_assertJ(0, "bad stop bytecode %d", op);
470 break; 545 break;
471 } 546 }
472 547
@@ -479,6 +554,7 @@ static void trace_stop(jit_State *J)
479 lj_vmevent_send(L, TRACE, 554 lj_vmevent_send(L, TRACE,
480 setstrV(L, L->top++, lj_str_newlit(L, "stop")); 555 setstrV(L, L->top++, lj_str_newlit(L, "stop"));
481 setintV(L->top++, traceno); 556 setintV(L->top++, traceno);
557 setfuncV(L, L->top++, J->fn);
482 ); 558 );
483} 559}
484 560
@@ -486,8 +562,8 @@ static void trace_stop(jit_State *J)
486static int trace_downrec(jit_State *J) 562static int trace_downrec(jit_State *J)
487{ 563{
488 /* Restart recording at the return instruction. */ 564 /* Restart recording at the return instruction. */
489 lua_assert(J->pt != NULL); 565 lj_assertJ(J->pt != NULL, "no active prototype");
490 lua_assert(bc_isret(bc_op(*J->pc))); 566 lj_assertJ(bc_isret(bc_op(*J->pc)), "not at a return bytecode");
491 if (bc_op(*J->pc) == BC_RETM) 567 if (bc_op(*J->pc) == BC_RETM)
492 return 0; /* NYI: down-recursion with RETM. */ 568 return 0; /* NYI: down-recursion with RETM. */
493 J->parent = 0; 569 J->parent = 0;
@@ -506,6 +582,10 @@ static int trace_abort(jit_State *J)
506 582
507 J->postproc = LJ_POST_NONE; 583 J->postproc = LJ_POST_NONE;
508 lj_mcode_abort(J); 584 lj_mcode_abort(J);
585 if (J->curfinal) {
586 lj_trace_free(J2G(J), J->curfinal);
587 J->curfinal = NULL;
588 }
509 if (tvisnumber(L->top-1)) 589 if (tvisnumber(L->top-1))
510 e = (TraceError)numberVint(L->top-1); 590 e = (TraceError)numberVint(L->top-1);
511 if (e == LJ_TRERR_MCODELM) { 591 if (e == LJ_TRERR_MCODELM) {
@@ -514,8 +594,17 @@ static int trace_abort(jit_State *J)
514 return 1; /* Retry ASM with new MCode area. */ 594 return 1; /* Retry ASM with new MCode area. */
515 } 595 }
516 /* Penalize or blacklist starting bytecode instruction. */ 596 /* Penalize or blacklist starting bytecode instruction. */
517 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) 597 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
518 penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e); 598 if (J->exitno == 0) {
599 BCIns *startpc = mref(J->cur.startpc, BCIns);
600 if (e == LJ_TRERR_RETRY)
601 hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */
602 else
603 penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
604 } else {
605 traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */
606 }
607 }
519 608
520 /* Is there anything to abort? */ 609 /* Is there anything to abort? */
521 traceno = J->cur.traceno; 610 traceno = J->cur.traceno;
@@ -581,8 +670,13 @@ static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
581 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */ 670 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */
582 trace_start(J); 671 trace_start(J);
583 lj_dispatch_update(J2G(J)); 672 lj_dispatch_update(J2G(J));
584 break; 673 if (J->state != LJ_TRACE_RECORD_1ST)
674 break;
675 /* fallthrough */
585 676
677 case LJ_TRACE_RECORD_1ST:
678 J->state = LJ_TRACE_RECORD;
679 /* fallthrough */
586 case LJ_TRACE_RECORD: 680 case LJ_TRACE_RECORD:
587 trace_pendpatch(J, 0); 681 trace_pendpatch(J, 0);
588 setvmstate(J2G(J), RECORD); 682 setvmstate(J2G(J), RECORD);
@@ -688,15 +782,30 @@ static void trace_hotside(jit_State *J, const BCIns *pc)
688{ 782{
689 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno]; 783 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
690 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) && 784 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
785 isluafunc(curr_func(J->L)) &&
691 snap->count != SNAPCOUNT_DONE && 786 snap->count != SNAPCOUNT_DONE &&
692 ++snap->count >= J->param[JIT_P_hotexit]) { 787 ++snap->count >= J->param[JIT_P_hotexit]) {
693 lua_assert(J->state == LJ_TRACE_IDLE); 788 lj_assertJ(J->state == LJ_TRACE_IDLE, "hot side exit while recording");
694 /* J->parent is non-zero for a side trace. */ 789 /* J->parent is non-zero for a side trace. */
695 J->state = LJ_TRACE_START; 790 J->state = LJ_TRACE_START;
696 lj_trace_ins(J, pc); 791 lj_trace_ins(J, pc);
697 } 792 }
698} 793}
699 794
795/* Stitch a new trace to the previous trace. */
796void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
797{
798 /* Only start a new trace if not recording or inside __gc call or vmevent. */
799 if (J->state == LJ_TRACE_IDLE &&
800 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
801 J->parent = 0; /* Have to treat it like a root trace. */
802 /* J->exitno is set to the invoking trace. */
803 J->state = LJ_TRACE_START;
804 lj_trace_ins(J, pc);
805 }
806}
807
808
700/* Tiny struct to pass data to protected call. */ 809/* Tiny struct to pass data to protected call. */
701typedef struct ExitDataCP { 810typedef struct ExitDataCP {
702 jit_State *J; 811 jit_State *J;
@@ -740,7 +849,7 @@ static void trace_exit_regs(lua_State *L, ExitState *ex)
740} 849}
741#endif 850#endif
742 851
743#ifdef EXITSTATE_PCREG 852#if defined(EXITSTATE_PCREG) || (LJ_UNWIND_JIT && !EXITTRACE_VMSTATE)
744/* Determine trace number from pc of exit instruction. */ 853/* Determine trace number from pc of exit instruction. */
745static TraceNo trace_exit_find(jit_State *J, MCode *pc) 854static TraceNo trace_exit_find(jit_State *J, MCode *pc)
746{ 855{
@@ -750,7 +859,7 @@ static TraceNo trace_exit_find(jit_State *J, MCode *pc)
750 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode)) 859 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
751 return traceno; 860 return traceno;
752 } 861 }
753 lua_assert(0); 862 lj_assertJ(0, "bad exit pc");
754 return 0; 863 return 0;
755} 864}
756#endif 865#endif
@@ -762,68 +871,81 @@ int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
762 lua_State *L = J->L; 871 lua_State *L = J->L;
763 ExitState *ex = (ExitState *)exptr; 872 ExitState *ex = (ExitState *)exptr;
764 ExitDataCP exd; 873 ExitDataCP exd;
765 int errcode; 874 int errcode, exitcode = J->exitcode;
766 const BCIns *pc; 875 TValue exiterr;
876 const BCIns *pc, *retpc;
767 void *cf; 877 void *cf;
768 GCtrace *T; 878 GCtrace *T;
879
880 setnilV(&exiterr);
881 if (exitcode) { /* Trace unwound with error code. */
882 J->exitcode = 0;
883 copyTV(L, &exiterr, L->top-1);
884 }
885
769#ifdef EXITSTATE_PCREG 886#ifdef EXITSTATE_PCREG
770 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]); 887 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
771#endif 888#endif
772 T = traceref(J, J->parent); UNUSED(T); 889 T = traceref(J, J->parent); UNUSED(T);
773#ifdef EXITSTATE_CHECKEXIT 890#ifdef EXITSTATE_CHECKEXIT
774 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */ 891 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */
775 lua_assert(T->root != 0); 892 lj_assertJ(T->root != 0, "stack check in root trace");
776 J->exitno = T->ir[REF_BASE].op2; 893 J->exitno = T->ir[REF_BASE].op2;
777 J->parent = T->ir[REF_BASE].op1; 894 J->parent = T->ir[REF_BASE].op1;
778 T = traceref(J, J->parent); 895 T = traceref(J, J->parent);
779 } 896 }
780#endif 897#endif
781 lua_assert(T != NULL && J->exitno < T->nsnap); 898 lj_assertJ(T != NULL && J->exitno < T->nsnap, "bad trace or exit number");
782 exd.J = J; 899 exd.J = J;
783 exd.exptr = exptr; 900 exd.exptr = exptr;
784 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp); 901 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
785 if (errcode) 902 if (errcode)
786 return -errcode; /* Return negated error code. */ 903 return -errcode; /* Return negated error code. */
787 904
788 lj_vmevent_send(L, TEXIT, 905 if (exitcode) copyTV(L, L->top++, &exiterr); /* Anchor the error object. */
789 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK); 906
790 setintV(L->top++, J->parent); 907 if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
791 setintV(L->top++, J->exitno); 908 lj_vmevent_send(L, TEXIT,
792 trace_exit_regs(L, ex); 909 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
793 ); 910 setintV(L->top++, J->parent);
911 setintV(L->top++, J->exitno);
912 trace_exit_regs(L, ex);
913 );
794 914
795 pc = exd.pc; 915 pc = exd.pc;
796 cf = cframe_raw(L->cframe); 916 cf = cframe_raw(L->cframe);
797 setcframe_pc(cf, pc); 917 setcframe_pc(cf, pc);
798 if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) { 918 if (exitcode) {
919 return -exitcode;
920 } else if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
921 /* Just exit to interpreter. */
922 } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
799 if (!(G(L)->hookmask & HOOK_GC)) 923 if (!(G(L)->hookmask & HOOK_GC))
800 lj_gc_step(L); /* Exited because of GC: drive GC forward. */ 924 lj_gc_step(L); /* Exited because of GC: drive GC forward. */
801 } else { 925 } else {
802 trace_hotside(J, pc); 926 trace_hotside(J, pc);
803 } 927 }
804 if (bc_op(*pc) == BC_JLOOP) { 928 /* Return MULTRES or 0 or -17. */
805 BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
806 if (bc_isret(bc_op(*retpc))) {
807 if (J->state == LJ_TRACE_RECORD) {
808 J->patchins = *pc;
809 J->patchpc = (BCIns *)pc;
810 *J->patchpc = *retpc;
811 J->bcskip = 1;
812 } else {
813 pc = retpc;
814 setcframe_pc(cf, pc);
815 }
816 }
817 }
818 /* Return MULTRES or 0. */
819 ERRNO_RESTORE 929 ERRNO_RESTORE
820 switch (bc_op(*pc)) { 930 switch (bc_op(*pc)) {
821 case BC_CALLM: case BC_CALLMT: 931 case BC_CALLM: case BC_CALLMT:
822 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc)); 932 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) - LJ_FR2);
823 case BC_RETM: 933 case BC_RETM:
824 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc)); 934 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
825 case BC_TSETM: 935 case BC_TSETM:
826 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc)); 936 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
937 case BC_JLOOP:
938 retpc = &traceref(J, bc_d(*pc))->startins;
939 if (bc_isret(bc_op(*retpc)) || bc_op(*retpc) == BC_ITERN) {
940 /* Dispatch to original ins to ensure forward progress. */
941 if (J->state != LJ_TRACE_RECORD) return -17;
942 /* Unpatch bytecode when recording. */
943 J->patchins = *pc;
944 J->patchpc = (BCIns *)pc;
945 *J->patchpc = *retpc;
946 J->bcskip = 1;
947 }
948 return 0;
827 default: 949 default:
828 if (bc_op(*pc) >= BC_FUNCF) 950 if (bc_op(*pc) >= BC_FUNCF)
829 return (int)((BCReg)(L->top - L->base) + 1); 951 return (int)((BCReg)(L->top - L->base) + 1);
@@ -831,4 +953,41 @@ int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
831 } 953 }
832} 954}
833 955
956#if LJ_UNWIND_JIT
957/* Given an mcode address determine trace exit address for unwinding. */
958uintptr_t LJ_FASTCALL lj_trace_unwind(jit_State *J, uintptr_t addr, ExitNo *ep)
959{
960#if EXITTRACE_VMSTATE
961 TraceNo traceno = J2G(J)->vmstate;
962#else
963 TraceNo traceno = trace_exit_find(J, (MCode *)addr);
964#endif
965 GCtrace *T = traceref(J, traceno);
966 if (T
967#if EXITTRACE_VMSTATE
968 && addr >= (uintptr_t)T->mcode && addr < (uintptr_t)T->mcode + T->szmcode
969#endif
970 ) {
971 SnapShot *snap = T->snap;
972 SnapNo lo = 0, exitno = T->nsnap;
973 uintptr_t ofs = (uintptr_t)((MCode *)addr - T->mcode); /* MCode units! */
974 /* Rightmost binary search for mcode offset to determine exit number. */
975 do {
976 SnapNo mid = (lo+exitno) >> 1;
977 if (ofs < snap[mid].mcofs) exitno = mid; else lo = mid + 1;
978 } while (lo < exitno);
979 exitno--;
980 *ep = exitno;
981#ifdef EXITSTUBS_PER_GROUP
982 return (uintptr_t)exitstub_addr(J, exitno);
983#else
984 return (uintptr_t)exitstub_trace_addr(T, exitno);
985#endif
986 }
987 /* Cannot correlate addr with trace/exit. This will be fatal. */
988 lj_assertJ(0, "bad exit pc");
989 return 0;
990}
991#endif
992
834#endif 993#endif
diff --git a/src/lj_trace.h b/src/lj_trace.h
index d708a217..5dcd365e 100644
--- a/src/lj_trace.h
+++ b/src/lj_trace.h
@@ -23,6 +23,7 @@ LJ_FUNC_NORET void lj_trace_err(jit_State *J, TraceError e);
23LJ_FUNC_NORET void lj_trace_err_info(jit_State *J, TraceError e); 23LJ_FUNC_NORET void lj_trace_err_info(jit_State *J, TraceError e);
24 24
25/* Trace management. */ 25/* Trace management. */
26LJ_FUNC GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T);
26LJ_FUNC void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T); 27LJ_FUNC void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T);
27LJ_FUNC void lj_trace_reenableproto(GCproto *pt); 28LJ_FUNC void lj_trace_reenableproto(GCproto *pt);
28LJ_FUNC void lj_trace_flushproto(global_State *g, GCproto *pt); 29LJ_FUNC void lj_trace_flushproto(global_State *g, GCproto *pt);
@@ -34,7 +35,11 @@ LJ_FUNC void lj_trace_freestate(global_State *g);
34/* Event handling. */ 35/* Event handling. */
35LJ_FUNC void lj_trace_ins(jit_State *J, const BCIns *pc); 36LJ_FUNC void lj_trace_ins(jit_State *J, const BCIns *pc);
36LJ_FUNCA void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc); 37LJ_FUNCA void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc);
38LJ_FUNCA void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc);
37LJ_FUNCA int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr); 39LJ_FUNCA int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr);
40#if LJ_UNWIND_EXT
41LJ_FUNC uintptr_t LJ_FASTCALL lj_trace_unwind(jit_State *J, uintptr_t addr, ExitNo *ep);
42#endif
38 43
39/* Signal asynchronous abort of trace or end of trace. */ 44/* Signal asynchronous abort of trace or end of trace. */
40#define lj_trace_abort(g) (G2J(g)->state &= ~LJ_TRACE_ACTIVE) 45#define lj_trace_abort(g) (G2J(g)->state &= ~LJ_TRACE_ACTIVE)
diff --git a/src/lj_traceerr.h b/src/lj_traceerr.h
index 8b28e65c..19ce30ad 100644
--- a/src/lj_traceerr.h
+++ b/src/lj_traceerr.h
@@ -7,10 +7,12 @@
7 7
8/* Recording. */ 8/* Recording. */
9TREDEF(RECERR, "error thrown or hook called during recording") 9TREDEF(RECERR, "error thrown or hook called during recording")
10TREDEF(TRACEUV, "trace too short")
10TREDEF(TRACEOV, "trace too long") 11TREDEF(TRACEOV, "trace too long")
11TREDEF(STACKOV, "trace too deep") 12TREDEF(STACKOV, "trace too deep")
12TREDEF(SNAPOV, "too many snapshots") 13TREDEF(SNAPOV, "too many snapshots")
13TREDEF(BLACKL, "blacklisted") 14TREDEF(BLACKL, "blacklisted")
15TREDEF(RETRY, "retry recording")
14TREDEF(NYIBC, "NYI: bytecode %d") 16TREDEF(NYIBC, "NYI: bytecode %d")
15 17
16/* Recording loop ops. */ 18/* Recording loop ops. */
@@ -23,8 +25,6 @@ TREDEF(BADTYPE, "bad argument type")
23TREDEF(CJITOFF, "JIT compilation disabled for function") 25TREDEF(CJITOFF, "JIT compilation disabled for function")
24TREDEF(CUNROLL, "call unroll limit reached") 26TREDEF(CUNROLL, "call unroll limit reached")
25TREDEF(DOWNREC, "down-recursion, restarting") 27TREDEF(DOWNREC, "down-recursion, restarting")
26TREDEF(NYICF, "NYI: C function %s")
27TREDEF(NYIFF, "NYI: FastFunc %s")
28TREDEF(NYIFFU, "NYI: unsupported variant of FastFunc %s") 28TREDEF(NYIFFU, "NYI: unsupported variant of FastFunc %s")
29TREDEF(NYIRETL, "NYI: return to lower frame") 29TREDEF(NYIRETL, "NYI: return to lower frame")
30 30
diff --git a/src/lj_udata.c b/src/lj_udata.c
index b17c2529..7acd9e32 100644
--- a/src/lj_udata.c
+++ b/src/lj_udata.c
@@ -8,6 +8,7 @@
8 8
9#include "lj_obj.h" 9#include "lj_obj.h"
10#include "lj_gc.h" 10#include "lj_gc.h"
11#include "lj_err.h"
11#include "lj_udata.h" 12#include "lj_udata.h"
12 13
13GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env) 14GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env)
@@ -32,3 +33,30 @@ void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud)
32 lj_mem_free(g, ud, sizeudata(ud)); 33 lj_mem_free(g, ud, sizeudata(ud));
33} 34}
34 35
36#if LJ_64
37void *lj_lightud_intern(lua_State *L, void *p)
38{
39 global_State *g = G(L);
40 uint64_t u = (uint64_t)p;
41 uint32_t up = lightudup(u);
42 uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
43 MSize segnum = g->gc.lightudnum;
44 if (segmap) {
45 MSize seg;
46 for (seg = 0; seg <= segnum; seg++)
47 if (segmap[seg] == up) /* Fast path. */
48 return (void *)(((uint64_t)seg << LJ_LIGHTUD_BITS_LO) | lightudlo(u));
49 segnum++;
50 /* Leave last segment unused to avoid clash with ITERN key. */
51 if (segnum >= (1 << LJ_LIGHTUD_BITS_SEG)-1) lj_err_msg(L, LJ_ERR_BADLU);
52 }
53 if (!((segnum-1) & segnum) && segnum != 1) {
54 lj_mem_reallocvec(L, segmap, segnum, segnum ? 2*segnum : 2u, uint32_t);
55 setmref(g->gc.lightudseg, segmap);
56 }
57 g->gc.lightudnum = segnum;
58 segmap[segnum] = up;
59 return (void *)(((uint64_t)segnum << LJ_LIGHTUD_BITS_LO) | lightudlo(u));
60}
61#endif
62
diff --git a/src/lj_udata.h b/src/lj_udata.h
index f1c2ca7f..143bf81d 100644
--- a/src/lj_udata.h
+++ b/src/lj_udata.h
@@ -10,5 +10,8 @@
10 10
11LJ_FUNC GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env); 11LJ_FUNC GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env);
12LJ_FUNC void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud); 12LJ_FUNC void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud);
13#if LJ_64
14LJ_FUNC void * LJ_FASTCALL lj_lightud_intern(lua_State *L, void *p);
15#endif
13 16
14#endif 17#endif
diff --git a/src/lj_vm.h b/src/lj_vm.h
index 133a78cc..63d09439 100644
--- a/src/lj_vm.h
+++ b/src/lj_vm.h
@@ -17,11 +17,18 @@ LJ_ASMF int lj_vm_cpcall(lua_State *L, lua_CFunction func, void *ud,
17LJ_ASMF int lj_vm_resume(lua_State *L, TValue *base, int nres1, ptrdiff_t ef); 17LJ_ASMF int lj_vm_resume(lua_State *L, TValue *base, int nres1, ptrdiff_t ef);
18LJ_ASMF_NORET void LJ_FASTCALL lj_vm_unwind_c(void *cframe, int errcode); 18LJ_ASMF_NORET void LJ_FASTCALL lj_vm_unwind_c(void *cframe, int errcode);
19LJ_ASMF_NORET void LJ_FASTCALL lj_vm_unwind_ff(void *cframe); 19LJ_ASMF_NORET void LJ_FASTCALL lj_vm_unwind_ff(void *cframe);
20#if LJ_ABI_WIN && LJ_TARGET_X86
21LJ_ASMF_NORET void LJ_FASTCALL lj_vm_rtlunwind(void *cframe, void *excptrec,
22 void *unwinder, int errcode);
23#endif
20LJ_ASMF void lj_vm_unwind_c_eh(void); 24LJ_ASMF void lj_vm_unwind_c_eh(void);
21LJ_ASMF void lj_vm_unwind_ff_eh(void); 25LJ_ASMF void lj_vm_unwind_ff_eh(void);
22#if LJ_TARGET_X86ORX64 26#if LJ_TARGET_X86ORX64
23LJ_ASMF void lj_vm_unwind_rethrow(void); 27LJ_ASMF void lj_vm_unwind_rethrow(void);
24#endif 28#endif
29#if LJ_TARGET_MIPS
30LJ_ASMF void lj_vm_unwind_stub(void);
31#endif
25 32
26/* Miscellaneous functions. */ 33/* Miscellaneous functions. */
27#if LJ_TARGET_X86ORX64 34#if LJ_TARGET_X86ORX64
@@ -43,13 +50,15 @@ LJ_ASMF void lj_vm_record(void);
43LJ_ASMF void lj_vm_inshook(void); 50LJ_ASMF void lj_vm_inshook(void);
44LJ_ASMF void lj_vm_rethook(void); 51LJ_ASMF void lj_vm_rethook(void);
45LJ_ASMF void lj_vm_callhook(void); 52LJ_ASMF void lj_vm_callhook(void);
53LJ_ASMF void lj_vm_profhook(void);
54LJ_ASMF void lj_vm_IITERN(void);
46 55
47/* Trace exit handling. */ 56/* Trace exit handling. */
48LJ_ASMF void lj_vm_exit_handler(void); 57LJ_ASMF char lj_vm_exit_handler[];
49LJ_ASMF void lj_vm_exit_interp(void); 58LJ_ASMF char lj_vm_exit_interp[];
50 59
51/* Internal math helper functions. */ 60/* Internal math helper functions. */
52#if LJ_TARGET_X86ORX64 || LJ_TARGET_PPC 61#if LJ_TARGET_PPC || LJ_TARGET_ARM64 || (LJ_TARGET_MIPS && LJ_ABI_SOFTFP)
53#define lj_vm_floor floor 62#define lj_vm_floor floor
54#define lj_vm_ceil ceil 63#define lj_vm_ceil ceil
55#else 64#else
@@ -60,23 +69,22 @@ LJ_ASMF double lj_vm_floor_sf(double);
60LJ_ASMF double lj_vm_ceil_sf(double); 69LJ_ASMF double lj_vm_ceil_sf(double);
61#endif 70#endif
62#endif 71#endif
63#if defined(LUAJIT_NO_LOG2) || LJ_TARGET_X86ORX64 72#ifdef LUAJIT_NO_LOG2
64LJ_ASMF double lj_vm_log2(double); 73LJ_ASMF double lj_vm_log2(double);
65#else 74#else
66#define lj_vm_log2 log2 75#define lj_vm_log2 log2
67#endif 76#endif
77#if !(defined(_LJ_DISPATCH_H) && LJ_TARGET_MIPS)
78LJ_ASMF int32_t LJ_FASTCALL lj_vm_modi(int32_t, int32_t);
79#endif
68 80
69#if LJ_HASJIT 81#if LJ_HASJIT
70#if LJ_TARGET_X86ORX64 82#if LJ_TARGET_X86ORX64
71LJ_ASMF void lj_vm_floor_sse(void); 83LJ_ASMF void lj_vm_floor_sse(void);
72LJ_ASMF void lj_vm_ceil_sse(void); 84LJ_ASMF void lj_vm_ceil_sse(void);
73LJ_ASMF void lj_vm_trunc_sse(void); 85LJ_ASMF void lj_vm_trunc_sse(void);
74LJ_ASMF void lj_vm_exp_x87(void); 86#endif
75LJ_ASMF void lj_vm_exp2_x87(void); 87#if LJ_TARGET_PPC || LJ_TARGET_ARM64
76LJ_ASMF void lj_vm_pow_sse(void);
77LJ_ASMF void lj_vm_powi_sse(void);
78#else
79#if LJ_TARGET_PPC
80#define lj_vm_trunc trunc 88#define lj_vm_trunc trunc
81#else 89#else
82LJ_ASMF double lj_vm_trunc(double); 90LJ_ASMF double lj_vm_trunc(double);
@@ -84,17 +92,10 @@ LJ_ASMF double lj_vm_trunc(double);
84LJ_ASMF double lj_vm_trunc_sf(double); 92LJ_ASMF double lj_vm_trunc_sf(double);
85#endif 93#endif
86#endif 94#endif
87LJ_ASMF double lj_vm_powi(double, int32_t);
88#ifdef LUAJIT_NO_EXP2
89LJ_ASMF double lj_vm_exp2(double);
90#else
91#define lj_vm_exp2 exp2
92#endif
93#endif
94LJ_ASMF int32_t LJ_FASTCALL lj_vm_modi(int32_t, int32_t);
95#if LJ_HASFFI 95#if LJ_HASFFI
96LJ_ASMF int lj_vm_errno(void); 96LJ_ASMF int lj_vm_errno(void);
97#endif 97#endif
98LJ_ASMF TValue *lj_vm_next(GCtab *t, uint32_t idx);
98#endif 99#endif
99 100
100/* Continuations for metamethods. */ 101/* Continuations for metamethods. */
@@ -104,13 +105,12 @@ LJ_ASMF void lj_cont_nop(void); /* Do nothing, just continue execution. */
104LJ_ASMF void lj_cont_condt(void); /* Branch if result is true. */ 105LJ_ASMF void lj_cont_condt(void); /* Branch if result is true. */
105LJ_ASMF void lj_cont_condf(void); /* Branch if result is false. */ 106LJ_ASMF void lj_cont_condf(void); /* Branch if result is false. */
106LJ_ASMF void lj_cont_hook(void); /* Continue from hook yield. */ 107LJ_ASMF void lj_cont_hook(void); /* Continue from hook yield. */
107 108LJ_ASMF void lj_cont_stitch(void); /* Trace stitching. */
108enum { LJ_CONT_TAILCALL, LJ_CONT_FFI_CALLBACK }; /* Special continuations. */
109 109
110/* Start of the ASM code. */ 110/* Start of the ASM code. */
111LJ_ASMF char lj_vm_asm_begin[]; 111LJ_ASMF char lj_vm_asm_begin[];
112 112
113/* Bytecode offsets are relative to lj_vm_asm_begin. */ 113/* Bytecode offsets are relative to lj_vm_asm_begin. */
114#define makeasmfunc(ofs) ((ASMFunction)(lj_vm_asm_begin + (ofs))) 114#define makeasmfunc(ofs) lj_ptr_sign((ASMFunction)(lj_vm_asm_begin + (ofs)), 0)
115 115
116#endif 116#endif
diff --git a/src/lj_vmevent.c b/src/lj_vmevent.c
index a25547ee..477ee79b 100644
--- a/src/lj_vmevent.c
+++ b/src/lj_vmevent.c
@@ -27,6 +27,7 @@ ptrdiff_t lj_vmevent_prepare(lua_State *L, VMEvent ev)
27 if (tv && tvisfunc(tv)) { 27 if (tv && tvisfunc(tv)) {
28 lj_state_checkstack(L, LUA_MINSTACK); 28 lj_state_checkstack(L, LUA_MINSTACK);
29 setfuncV(L, L->top++, funcV(tv)); 29 setfuncV(L, L->top++, funcV(tv));
30 if (LJ_FR2) setnilV(L->top++);
30 return savestack(L, L->top); 31 return savestack(L, L->top);
31 } 32 }
32 } 33 }
diff --git a/src/lj_vmevent.h b/src/lj_vmevent.h
index 13fb9b92..d757a8b5 100644
--- a/src/lj_vmevent.h
+++ b/src/lj_vmevent.h
@@ -24,9 +24,10 @@
24/* VM event IDs. */ 24/* VM event IDs. */
25typedef enum { 25typedef enum {
26 VMEVENT_DEF(BC, 0x00003883), 26 VMEVENT_DEF(BC, 0x00003883),
27 VMEVENT_DEF(TRACE, 0xb2d91467), 27 VMEVENT_DEF(TRACE, 0x12d91467),
28 VMEVENT_DEF(RECORD, 0x9284bf4f), 28 VMEVENT_DEF(RECORD, 0x1284bf4f),
29 VMEVENT_DEF(TEXIT, 0xb29df2b0), 29 VMEVENT_DEF(TEXIT, 0x129df2b0),
30 VMEVENT_DEF(ERRFIN, 0x12d93888),
30 LJ_VMEVENT__MAX 31 LJ_VMEVENT__MAX
31} VMEvent; 32} VMEvent;
32 33
diff --git a/src/lj_vmmath.c b/src/lj_vmmath.c
index 2da5f6b7..3351e72b 100644
--- a/src/lj_vmmath.c
+++ b/src/lj_vmmath.c
@@ -13,16 +13,40 @@
13#include "lj_ir.h" 13#include "lj_ir.h"
14#include "lj_vm.h" 14#include "lj_vm.h"
15 15
16/* -- Helper functions for generated machine code ------------------------- */ 16/* -- Wrapper functions --------------------------------------------------- */
17 17
18#if LJ_TARGET_X86ORX64 18#if LJ_TARGET_X86 && __ELF__ && __PIC__
19/* Wrapper functions to avoid linker issues on OSX. */ 19/* Wrapper functions to deal with the ELF/x86 PIC disaster. */
20LJ_FUNCA double lj_vm_sinh(double x) { return sinh(x); } 20LJ_FUNCA double lj_wrap_log(double x) { return log(x); }
21LJ_FUNCA double lj_vm_cosh(double x) { return cosh(x); } 21LJ_FUNCA double lj_wrap_log10(double x) { return log10(x); }
22LJ_FUNCA double lj_vm_tanh(double x) { return tanh(x); } 22LJ_FUNCA double lj_wrap_exp(double x) { return exp(x); }
23LJ_FUNCA double lj_wrap_sin(double x) { return sin(x); }
24LJ_FUNCA double lj_wrap_cos(double x) { return cos(x); }
25LJ_FUNCA double lj_wrap_tan(double x) { return tan(x); }
26LJ_FUNCA double lj_wrap_asin(double x) { return asin(x); }
27LJ_FUNCA double lj_wrap_acos(double x) { return acos(x); }
28LJ_FUNCA double lj_wrap_atan(double x) { return atan(x); }
29LJ_FUNCA double lj_wrap_sinh(double x) { return sinh(x); }
30LJ_FUNCA double lj_wrap_cosh(double x) { return cosh(x); }
31LJ_FUNCA double lj_wrap_tanh(double x) { return tanh(x); }
32LJ_FUNCA double lj_wrap_atan2(double x, double y) { return atan2(x, y); }
33LJ_FUNCA double lj_wrap_pow(double x, double y) { return pow(x, y); }
34LJ_FUNCA double lj_wrap_fmod(double x, double y) { return fmod(x, y); }
23#endif 35#endif
24 36
25#if !LJ_TARGET_X86ORX64 37/* -- Helper functions ---------------------------------------------------- */
38
39/* Required to prevent the C compiler from applying FMA optimizations.
40**
41** Yes, there's -ffp-contract and the FP_CONTRACT pragma ... in theory.
42** But the current state of C compilers is a mess in this regard.
43** Also, this function is not performance sensitive at all.
44*/
45LJ_NOINLINE static double lj_vm_floormul(double x, double y)
46{
47 return lj_vm_floor(x / y) * y;
48}
49
26double lj_vm_foldarith(double x, double y, int op) 50double lj_vm_foldarith(double x, double y, int op)
27{ 51{
28 switch (op) { 52 switch (op) {
@@ -30,42 +54,27 @@ double lj_vm_foldarith(double x, double y, int op)
30 case IR_SUB - IR_ADD: return x-y; break; 54 case IR_SUB - IR_ADD: return x-y; break;
31 case IR_MUL - IR_ADD: return x*y; break; 55 case IR_MUL - IR_ADD: return x*y; break;
32 case IR_DIV - IR_ADD: return x/y; break; 56 case IR_DIV - IR_ADD: return x/y; break;
33 case IR_MOD - IR_ADD: return x-lj_vm_floor(x/y)*y; break; 57 case IR_MOD - IR_ADD: return x-lj_vm_floormul(x, y); break;
34 case IR_POW - IR_ADD: return pow(x, y); break; 58 case IR_POW - IR_ADD: return pow(x, y); break;
35 case IR_NEG - IR_ADD: return -x; break; 59 case IR_NEG - IR_ADD: return -x; break;
36 case IR_ABS - IR_ADD: return fabs(x); break; 60 case IR_ABS - IR_ADD: return fabs(x); break;
37#if LJ_HASJIT 61#if LJ_HASJIT
38 case IR_ATAN2 - IR_ADD: return atan2(x, y); break;
39 case IR_LDEXP - IR_ADD: return ldexp(x, (int)y); break; 62 case IR_LDEXP - IR_ADD: return ldexp(x, (int)y); break;
40 case IR_MIN - IR_ADD: return x > y ? y : x; break; 63 case IR_MIN - IR_ADD: return x < y ? x : y; break;
41 case IR_MAX - IR_ADD: return x < y ? y : x; break; 64 case IR_MAX - IR_ADD: return x > y ? x : y; break;
42#endif 65#endif
43 default: return x; 66 default: return x;
44 } 67 }
45} 68}
46#endif
47 69
48#if LJ_HASJIT 70/* -- Helper functions for generated machine code ------------------------- */
49
50#ifdef LUAJIT_NO_LOG2
51double lj_vm_log2(double a)
52{
53 return log(a) * 1.4426950408889634074;
54}
55#endif
56
57#ifdef LUAJIT_NO_EXP2
58double lj_vm_exp2(double a)
59{
60 return exp(a * 0.6931471805599453);
61}
62#endif
63 71
64#if !(LJ_TARGET_ARM || LJ_TARGET_PPC) 72#if (LJ_HASJIT && !(LJ_TARGET_ARM || LJ_TARGET_ARM64 || LJ_TARGET_PPC)) || LJ_TARGET_MIPS
65int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b) 73int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b)
66{ 74{
67 uint32_t y, ua, ub; 75 uint32_t y, ua, ub;
68 lua_assert(b != 0); /* This must be checked before using this function. */ 76 /* This must be checked before using this function. */
77 lj_assertX(b != 0, "modulo with zero divisor");
69 ua = a < 0 ? ~(uint32_t)a+1u : (uint32_t)a; 78 ua = a < 0 ? ~(uint32_t)a+1u : (uint32_t)a;
70 ub = b < 0 ? ~(uint32_t)b+1u : (uint32_t)b; 79 ub = b < 0 ? ~(uint32_t)b+1u : (uint32_t)b;
71 y = ua % ub; 80 y = ua % ub;
@@ -75,38 +84,14 @@ int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b)
75} 84}
76#endif 85#endif
77 86
78#if !LJ_TARGET_X86ORX64 87#if LJ_HASJIT
79/* Unsigned x^k. */
80static double lj_vm_powui(double x, uint32_t k)
81{
82 double y;
83 lua_assert(k != 0);
84 for (; (k & 1) == 0; k >>= 1) x *= x;
85 y = x;
86 if ((k >>= 1) != 0) {
87 for (;;) {
88 x *= x;
89 if (k == 1) break;
90 if (k & 1) y *= x;
91 k >>= 1;
92 }
93 y *= x;
94 }
95 return y;
96}
97 88
98/* Signed x^k. */ 89#ifdef LUAJIT_NO_LOG2
99double lj_vm_powi(double x, int32_t k) 90double lj_vm_log2(double a)
100{ 91{
101 if (k > 1) 92 return log(a) * 1.4426950408889634074;
102 return lj_vm_powui(x, (uint32_t)k);
103 else if (k == 1)
104 return x;
105 else if (k == 0)
106 return 1.0;
107 else
108 return 1.0 / lj_vm_powui(x, ~(uint32_t)k+1u);
109} 93}
94#endif
110 95
111/* Computes fpm(x) for extended math functions. */ 96/* Computes fpm(x) for extended math functions. */
112double lj_vm_foldfpm(double x, int fpm) 97double lj_vm_foldfpm(double x, int fpm)
@@ -116,19 +101,12 @@ double lj_vm_foldfpm(double x, int fpm)
116 case IRFPM_CEIL: return lj_vm_ceil(x); 101 case IRFPM_CEIL: return lj_vm_ceil(x);
117 case IRFPM_TRUNC: return lj_vm_trunc(x); 102 case IRFPM_TRUNC: return lj_vm_trunc(x);
118 case IRFPM_SQRT: return sqrt(x); 103 case IRFPM_SQRT: return sqrt(x);
119 case IRFPM_EXP: return exp(x);
120 case IRFPM_EXP2: return lj_vm_exp2(x);
121 case IRFPM_LOG: return log(x); 104 case IRFPM_LOG: return log(x);
122 case IRFPM_LOG2: return lj_vm_log2(x); 105 case IRFPM_LOG2: return lj_vm_log2(x);
123 case IRFPM_LOG10: return log10(x); 106 default: lj_assertX(0, "bad fpm %d", fpm);
124 case IRFPM_SIN: return sin(x);
125 case IRFPM_COS: return cos(x);
126 case IRFPM_TAN: return tan(x);
127 default: lua_assert(0);
128 } 107 }
129 return 0; 108 return 0;
130} 109}
131#endif
132 110
133#if LJ_HASFFI 111#if LJ_HASFFI
134int lj_vm_errno(void) 112int lj_vm_errno(void)
diff --git a/src/ljamalg.c b/src/ljamalg.c
index f9315d5c..f1dce6a3 100644
--- a/src/ljamalg.c
+++ b/src/ljamalg.c
@@ -3,16 +3,6 @@
3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h 3** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4*/ 4*/
5 5
6/*
7+--------------------------------------------------------------------------+
8| WARNING: Compiling the amalgamation needs a lot of virtual memory |
9| (around 300 MB with GCC 4.x)! If you don't have enough physical memory |
10| your machine will start swapping to disk and the compile will not finish |
11| within a reasonable amount of time. |
12| So either compile on a bigger machine or use the non-amalgamated build. |
13+--------------------------------------------------------------------------+
14*/
15
16#define ljamalg_c 6#define ljamalg_c
17#define LUA_CORE 7#define LUA_CORE
18 8
@@ -28,23 +18,30 @@
28#include "lua.h" 18#include "lua.h"
29#include "lauxlib.h" 19#include "lauxlib.h"
30 20
21#include "lj_assert.c"
31#include "lj_gc.c" 22#include "lj_gc.c"
32#include "lj_err.c" 23#include "lj_err.c"
33#include "lj_char.c" 24#include "lj_char.c"
34#include "lj_bc.c" 25#include "lj_bc.c"
35#include "lj_obj.c" 26#include "lj_obj.c"
27#include "lj_buf.c"
36#include "lj_str.c" 28#include "lj_str.c"
37#include "lj_tab.c" 29#include "lj_tab.c"
38#include "lj_func.c" 30#include "lj_func.c"
39#include "lj_udata.c" 31#include "lj_udata.c"
40#include "lj_meta.c" 32#include "lj_meta.c"
41#include "lj_debug.c" 33#include "lj_debug.c"
34#include "lj_prng.c"
42#include "lj_state.c" 35#include "lj_state.c"
43#include "lj_dispatch.c" 36#include "lj_dispatch.c"
44#include "lj_vmevent.c" 37#include "lj_vmevent.c"
45#include "lj_vmmath.c" 38#include "lj_vmmath.c"
46#include "lj_strscan.c" 39#include "lj_strscan.c"
40#include "lj_strfmt.c"
41#include "lj_strfmt_num.c"
42#include "lj_serialize.c"
47#include "lj_api.c" 43#include "lj_api.c"
44#include "lj_profile.c"
48#include "lj_lex.c" 45#include "lj_lex.c"
49#include "lj_parse.c" 46#include "lj_parse.c"
50#include "lj_bcread.c" 47#include "lj_bcread.c"
@@ -89,5 +86,6 @@
89#include "lib_bit.c" 86#include "lib_bit.c"
90#include "lib_jit.c" 87#include "lib_jit.c"
91#include "lib_ffi.c" 88#include "lib_ffi.c"
89#include "lib_buffer.c"
92#include "lib_init.c" 90#include "lib_init.c"
93 91
diff --git a/src/lua.h b/src/lua.h
index 2bd683c2..6d1634d1 100644
--- a/src/lua.h
+++ b/src/lua.h
@@ -39,7 +39,8 @@
39#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i)) 39#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i))
40 40
41 41
42/* thread status; 0 is OK */ 42/* thread status */
43#define LUA_OK 0
43#define LUA_YIELD 1 44#define LUA_YIELD 1
44#define LUA_ERRRUN 2 45#define LUA_ERRRUN 2
45#define LUA_ERRSYNTAX 3 46#define LUA_ERRSYNTAX 3
@@ -226,6 +227,7 @@ LUA_API int (lua_status) (lua_State *L);
226#define LUA_GCSTEP 5 227#define LUA_GCSTEP 5
227#define LUA_GCSETPAUSE 6 228#define LUA_GCSETPAUSE 6
228#define LUA_GCSETSTEPMUL 7 229#define LUA_GCSETSTEPMUL 7
230#define LUA_GCISRUNNING 9
229 231
230LUA_API int (lua_gc) (lua_State *L, int what, int data); 232LUA_API int (lua_gc) (lua_State *L, int what, int data);
231 233
@@ -346,6 +348,13 @@ LUA_API void *lua_upvalueid (lua_State *L, int idx, int n);
346LUA_API void lua_upvaluejoin (lua_State *L, int idx1, int n1, int idx2, int n2); 348LUA_API void lua_upvaluejoin (lua_State *L, int idx1, int n1, int idx2, int n2);
347LUA_API int lua_loadx (lua_State *L, lua_Reader reader, void *dt, 349LUA_API int lua_loadx (lua_State *L, lua_Reader reader, void *dt,
348 const char *chunkname, const char *mode); 350 const char *chunkname, const char *mode);
351LUA_API const lua_Number *lua_version (lua_State *L);
352LUA_API void lua_copy (lua_State *L, int fromidx, int toidx);
353LUA_API lua_Number lua_tonumberx (lua_State *L, int idx, int *isnum);
354LUA_API lua_Integer lua_tointegerx (lua_State *L, int idx, int *isnum);
355
356/* From Lua 5.3. */
357LUA_API int lua_isyieldable (lua_State *L);
349 358
350 359
351struct lua_Debug { 360struct lua_Debug {
diff --git a/src/luaconf.h b/src/luaconf.h
index 8fc3eee5..f47f0680 100644
--- a/src/luaconf.h
+++ b/src/luaconf.h
@@ -37,7 +37,7 @@
37#endif 37#endif
38#define LUA_LROOT "/usr/local" 38#define LUA_LROOT "/usr/local"
39#define LUA_LUADIR "/lua/5.1/" 39#define LUA_LUADIR "/lua/5.1/"
40#define LUA_LJDIR "/luajit-2.0/" 40#define LUA_LJDIR "/luajit-2.1/"
41 41
42#ifdef LUA_ROOT 42#ifdef LUA_ROOT
43#define LUA_JROOT LUA_ROOT 43#define LUA_JROOT LUA_ROOT
@@ -79,7 +79,7 @@
79#define LUA_IGMARK "-" 79#define LUA_IGMARK "-"
80#define LUA_PATH_CONFIG \ 80#define LUA_PATH_CONFIG \
81 LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" \ 81 LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" \
82 LUA_EXECDIR "\n" LUA_IGMARK 82 LUA_EXECDIR "\n" LUA_IGMARK "\n"
83 83
84/* Quoting in error messages. */ 84/* Quoting in error messages. */
85#define LUA_QL(x) "'" x "'" 85#define LUA_QL(x) "'" x "'"
@@ -92,10 +92,6 @@
92#define LUAI_GCMUL 200 /* Run GC at 200% of allocation speed. */ 92#define LUAI_GCMUL 200 /* Run GC at 200% of allocation speed. */
93#define LUA_MAXCAPTURES 32 /* Max. pattern captures. */ 93#define LUA_MAXCAPTURES 32 /* Max. pattern captures. */
94 94
95/* Compatibility with older library function names. */
96#define LUA_COMPAT_MOD /* OLD: math.mod, NEW: math.fmod */
97#define LUA_COMPAT_GFIND /* OLD: string.gfind, NEW: string.gmatch */
98
99/* Configuration for the frontend (the luajit executable). */ 95/* Configuration for the frontend (the luajit executable). */
100#if defined(luajit_c) 96#if defined(luajit_c)
101#define LUA_PROGNAME "luajit" /* Fallback frontend name. */ 97#define LUA_PROGNAME "luajit" /* Fallback frontend name. */
@@ -140,7 +136,7 @@
140 136
141#define LUALIB_API LUA_API 137#define LUALIB_API LUA_API
142 138
143/* Support for internal assertions. */ 139/* Compatibility support for assertions. */
144#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK) 140#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
145#include <assert.h> 141#include <assert.h>
146#endif 142#endif
diff --git a/src/luajit.c b/src/luajit.c
index 85579cda..73e29d44 100644
--- a/src/luajit.c
+++ b/src/luajit.c
@@ -62,8 +62,9 @@ static void laction(int i)
62 62
63static void print_usage(void) 63static void print_usage(void)
64{ 64{
65 fprintf(stderr, 65 fputs("usage: ", stderr);
66 "usage: %s [options]... [script [args]...].\n" 66 fputs(progname, stderr);
67 fputs(" [options]... [script [args]...].\n"
67 "Available options are:\n" 68 "Available options are:\n"
68 " -e chunk Execute string " LUA_QL("chunk") ".\n" 69 " -e chunk Execute string " LUA_QL("chunk") ".\n"
69 " -l name Require library " LUA_QL("name") ".\n" 70 " -l name Require library " LUA_QL("name") ".\n"
@@ -74,16 +75,14 @@ static void print_usage(void)
74 " -v Show version information.\n" 75 " -v Show version information.\n"
75 " -E Ignore environment variables.\n" 76 " -E Ignore environment variables.\n"
76 " -- Stop handling options.\n" 77 " -- Stop handling options.\n"
77 " - Execute stdin and stop handling options.\n" 78 " - Execute stdin and stop handling options.\n", stderr);
78 ,
79 progname);
80 fflush(stderr); 79 fflush(stderr);
81} 80}
82 81
83static void l_message(const char *msg) 82static void l_message(const char *msg)
84{ 83{
85 if (progname) fprintf(stderr, "%s: ", progname); 84 if (progname) { fputs(progname, stderr); fputc(':', stderr); fputc(' ', stderr); }
86 fprintf(stderr, "%s\n", msg); 85 fputs(msg, stderr); fputc('\n', stderr);
87 fflush(stderr); 86 fflush(stderr);
88} 87}
89 88
@@ -126,7 +125,7 @@ static int docall(lua_State *L, int narg, int clear)
126#endif 125#endif
127 lua_remove(L, base); /* remove traceback function */ 126 lua_remove(L, base); /* remove traceback function */
128 /* force a complete garbage collection in case of errors */ 127 /* force a complete garbage collection in case of errors */
129 if (status != 0) lua_gc(L, LUA_GCCOLLECT, 0); 128 if (status != LUA_OK) lua_gc(L, LUA_GCCOLLECT, 0);
130 return status; 129 return status;
131} 130}
132 131
@@ -155,22 +154,15 @@ static void print_jit_status(lua_State *L)
155 lua_settop(L, 0); /* clear stack */ 154 lua_settop(L, 0); /* clear stack */
156} 155}
157 156
158static int getargs(lua_State *L, char **argv, int n) 157static void createargtable(lua_State *L, char **argv, int argc, int argf)
159{ 158{
160 int narg;
161 int i; 159 int i;
162 int argc = 0; 160 lua_createtable(L, argc - argf, argf);
163 while (argv[argc]) argc++; /* count total number of arguments */
164 narg = argc - (n + 1); /* number of arguments to the script */
165 luaL_checkstack(L, narg + 3, "too many arguments to script");
166 for (i = n+1; i < argc; i++)
167 lua_pushstring(L, argv[i]);
168 lua_createtable(L, narg, n + 1);
169 for (i = 0; i < argc; i++) { 161 for (i = 0; i < argc; i++) {
170 lua_pushstring(L, argv[i]); 162 lua_pushstring(L, argv[i]);
171 lua_rawseti(L, -2, i - n); 163 lua_rawseti(L, -2, i - argf);
172 } 164 }
173 return narg; 165 lua_setglobal(L, "arg");
174} 166}
175 167
176static int dofile(lua_State *L, const char *name) 168static int dofile(lua_State *L, const char *name)
@@ -259,9 +251,9 @@ static void dotty(lua_State *L)
259 const char *oldprogname = progname; 251 const char *oldprogname = progname;
260 progname = NULL; 252 progname = NULL;
261 while ((status = loadline(L)) != -1) { 253 while ((status = loadline(L)) != -1) {
262 if (status == 0) status = docall(L, 0, 0); 254 if (status == LUA_OK) status = docall(L, 0, 0);
263 report(L, status); 255 report(L, status);
264 if (status == 0 && lua_gettop(L) > 0) { /* any result to print? */ 256 if (status == LUA_OK && lua_gettop(L) > 0) { /* any result to print? */
265 lua_getglobal(L, "print"); 257 lua_getglobal(L, "print");
266 lua_insert(L, 1); 258 lua_insert(L, 1);
267 if (lua_pcall(L, lua_gettop(L)-1, 0, 0) != 0) 259 if (lua_pcall(L, lua_gettop(L)-1, 0, 0) != 0)
@@ -275,21 +267,30 @@ static void dotty(lua_State *L)
275 progname = oldprogname; 267 progname = oldprogname;
276} 268}
277 269
278static int handle_script(lua_State *L, char **argv, int n) 270static int handle_script(lua_State *L, char **argx)
279{ 271{
280 int status; 272 int status;
281 const char *fname; 273 const char *fname = argx[0];
282 int narg = getargs(L, argv, n); /* collect arguments */ 274 if (strcmp(fname, "-") == 0 && strcmp(argx[-1], "--") != 0)
283 lua_setglobal(L, "arg");
284 fname = argv[n];
285 if (strcmp(fname, "-") == 0 && strcmp(argv[n-1], "--") != 0)
286 fname = NULL; /* stdin */ 275 fname = NULL; /* stdin */
287 status = luaL_loadfile(L, fname); 276 status = luaL_loadfile(L, fname);
288 lua_insert(L, -(narg+1)); 277 if (status == LUA_OK) {
289 if (status == 0) 278 /* Fetch args from arg table. LUA_INIT or -e might have changed them. */
279 int narg = 0;
280 lua_getglobal(L, "arg");
281 if (lua_istable(L, -1)) {
282 do {
283 narg++;
284 lua_rawgeti(L, -narg, narg);
285 } while (!lua_isnil(L, -1));
286 lua_pop(L, 1);
287 lua_remove(L, -narg);
288 narg--;
289 } else {
290 lua_pop(L, 1);
291 }
290 status = docall(L, narg, 0); 292 status = docall(L, narg, 0);
291 else 293 }
292 lua_pop(L, narg);
293 return report(L, status); 294 return report(L, status);
294} 295}
295 296
@@ -385,7 +386,8 @@ static int dobytecode(lua_State *L, char **argv)
385 } 386 }
386 for (argv++; *argv != NULL; narg++, argv++) 387 for (argv++; *argv != NULL; narg++, argv++)
387 lua_pushstring(L, *argv); 388 lua_pushstring(L, *argv);
388 return report(L, lua_pcall(L, narg, 0, 0)); 389 report(L, lua_pcall(L, narg, 0, 0));
390 return -1;
389} 391}
390 392
391/* check that argument has no extra characters at the end */ 393/* check that argument has no extra characters at the end */
@@ -406,7 +408,7 @@ static int collectargs(char **argv, int *flags)
406 switch (argv[i][1]) { /* Check option. */ 408 switch (argv[i][1]) { /* Check option. */
407 case '-': 409 case '-':
408 notail(argv[i]); 410 notail(argv[i]);
409 return (argv[i+1] != NULL ? i+1 : 0); 411 return i+1;
410 case '\0': 412 case '\0':
411 return i; 413 return i;
412 case 'i': 414 case 'i':
@@ -432,23 +434,23 @@ static int collectargs(char **argv, int *flags)
432 case 'b': /* LuaJIT extension */ 434 case 'b': /* LuaJIT extension */
433 if (*flags) return -1; 435 if (*flags) return -1;
434 *flags |= FLAGS_EXEC; 436 *flags |= FLAGS_EXEC;
435 return 0; 437 return i+1;
436 case 'E': 438 case 'E':
437 *flags |= FLAGS_NOENV; 439 *flags |= FLAGS_NOENV;
438 break; 440 break;
439 default: return -1; /* invalid option */ 441 default: return -1; /* invalid option */
440 } 442 }
441 } 443 }
442 return 0; 444 return i;
443} 445}
444 446
445static int runargs(lua_State *L, char **argv, int n) 447static int runargs(lua_State *L, char **argv, int argn)
446{ 448{
447 int i; 449 int i;
448 for (i = 1; i < n; i++) { 450 for (i = 1; i < argn; i++) {
449 if (argv[i] == NULL) continue; 451 if (argv[i] == NULL) continue;
450 lua_assert(argv[i][0] == '-'); 452 lua_assert(argv[i][0] == '-');
451 switch (argv[i][1]) { /* option */ 453 switch (argv[i][1]) {
452 case 'e': { 454 case 'e': {
453 const char *chunk = argv[i] + 2; 455 const char *chunk = argv[i] + 2;
454 if (*chunk == '\0') chunk = argv[++i]; 456 if (*chunk == '\0') chunk = argv[++i];
@@ -462,10 +464,10 @@ static int runargs(lua_State *L, char **argv, int n)
462 if (*filename == '\0') filename = argv[++i]; 464 if (*filename == '\0') filename = argv[++i];
463 lua_assert(filename != NULL); 465 lua_assert(filename != NULL);
464 if (dolibrary(L, filename)) 466 if (dolibrary(L, filename))
465 return 1; /* stop if file fails */ 467 return 1;
466 break; 468 break;
467 } 469 }
468 case 'j': { /* LuaJIT extension */ 470 case 'j': { /* LuaJIT extension. */
469 const char *cmd = argv[i] + 2; 471 const char *cmd = argv[i] + 2;
470 if (*cmd == '\0') cmd = argv[++i]; 472 if (*cmd == '\0') cmd = argv[++i];
471 lua_assert(cmd != NULL); 473 lua_assert(cmd != NULL);
@@ -473,16 +475,16 @@ static int runargs(lua_State *L, char **argv, int n)
473 return 1; 475 return 1;
474 break; 476 break;
475 } 477 }
476 case 'O': /* LuaJIT extension */ 478 case 'O': /* LuaJIT extension. */
477 if (dojitopt(L, argv[i] + 2)) 479 if (dojitopt(L, argv[i] + 2))
478 return 1; 480 return 1;
479 break; 481 break;
480 case 'b': /* LuaJIT extension */ 482 case 'b': /* LuaJIT extension. */
481 return dobytecode(L, argv+i); 483 return dobytecode(L, argv+i);
482 default: break; 484 default: break;
483 } 485 }
484 } 486 }
485 return 0; 487 return LUA_OK;
486} 488}
487 489
488static int handle_luainit(lua_State *L) 490static int handle_luainit(lua_State *L)
@@ -493,7 +495,7 @@ static int handle_luainit(lua_State *L)
493 const char *init = getenv(LUA_INIT); 495 const char *init = getenv(LUA_INIT);
494#endif 496#endif
495 if (init == NULL) 497 if (init == NULL)
496 return 0; /* status OK */ 498 return LUA_OK;
497 else if (init[0] == '@') 499 else if (init[0] == '@')
498 return dofile(L, init+1); 500 return dofile(L, init+1);
499 else 501 else
@@ -510,44 +512,55 @@ static int pmain(lua_State *L)
510{ 512{
511 struct Smain *s = &smain; 513 struct Smain *s = &smain;
512 char **argv = s->argv; 514 char **argv = s->argv;
513 int script; 515 int argn;
514 int flags = 0; 516 int flags = 0;
515 globalL = L; 517 globalL = L;
516 LUAJIT_VERSION_SYM(); /* linker-enforced version check */ 518 LUAJIT_VERSION_SYM(); /* Linker-enforced version check. */
517 script = collectargs(argv, &flags); 519
518 if (script < 0) { /* invalid args? */ 520 argn = collectargs(argv, &flags);
521 if (argn < 0) { /* Invalid args? */
519 print_usage(); 522 print_usage();
520 s->status = 1; 523 s->status = 1;
521 return 0; 524 return 0;
522 } 525 }
526
523 if ((flags & FLAGS_NOENV)) { 527 if ((flags & FLAGS_NOENV)) {
524 lua_pushboolean(L, 1); 528 lua_pushboolean(L, 1);
525 lua_setfield(L, LUA_REGISTRYINDEX, "LUA_NOENV"); 529 lua_setfield(L, LUA_REGISTRYINDEX, "LUA_NOENV");
526 } 530 }
527 lua_gc(L, LUA_GCSTOP, 0); /* stop collector during initialization */ 531
528 luaL_openlibs(L); /* open libraries */ 532 /* Stop collector during library initialization. */
533 lua_gc(L, LUA_GCSTOP, 0);
534 luaL_openlibs(L);
529 lua_gc(L, LUA_GCRESTART, -1); 535 lua_gc(L, LUA_GCRESTART, -1);
536
537 createargtable(L, argv, s->argc, argn);
538
530 if (!(flags & FLAGS_NOENV)) { 539 if (!(flags & FLAGS_NOENV)) {
531 s->status = handle_luainit(L); 540 s->status = handle_luainit(L);
532 if (s->status != 0) return 0; 541 if (s->status != LUA_OK) return 0;
533 } 542 }
543
534 if ((flags & FLAGS_VERSION)) print_version(); 544 if ((flags & FLAGS_VERSION)) print_version();
535 s->status = runargs(L, argv, (script > 0) ? script : s->argc); 545
536 if (s->status != 0) return 0; 546 s->status = runargs(L, argv, argn);
537 if (script) { 547 if (s->status != LUA_OK) return 0;
538 s->status = handle_script(L, argv, script); 548
539 if (s->status != 0) return 0; 549 if (s->argc > argn) {
550 s->status = handle_script(L, argv + argn);
551 if (s->status != LUA_OK) return 0;
540 } 552 }
553
541 if ((flags & FLAGS_INTERACTIVE)) { 554 if ((flags & FLAGS_INTERACTIVE)) {
542 print_jit_status(L); 555 print_jit_status(L);
543 dotty(L); 556 dotty(L);
544 } else if (script == 0 && !(flags & (FLAGS_EXEC|FLAGS_VERSION))) { 557 } else if (s->argc == argn && !(flags & (FLAGS_EXEC|FLAGS_VERSION))) {
545 if (lua_stdin_is_tty()) { 558 if (lua_stdin_is_tty()) {
546 print_version(); 559 print_version();
547 print_jit_status(L); 560 print_jit_status(L);
548 dotty(L); 561 dotty(L);
549 } else { 562 } else {
550 dofile(L, NULL); /* executes stdin as a file */ 563 dofile(L, NULL); /* Executes stdin as a file. */
551 } 564 }
552 } 565 }
553 return 0; 566 return 0;
@@ -558,7 +571,7 @@ int main(int argc, char **argv)
558 int status; 571 int status;
559 lua_State *L; 572 lua_State *L;
560 if (!argv[0]) argv = empty_argv; else if (argv[0][0]) progname = argv[0]; 573 if (!argv[0]) argv = empty_argv; else if (argv[0][0]) progname = argv[0];
561 L = lua_open(); /* create state */ 574 L = lua_open();
562 if (L == NULL) { 575 if (L == NULL) {
563 l_message("cannot create state: not enough memory"); 576 l_message("cannot create state: not enough memory");
564 return EXIT_FAILURE; 577 return EXIT_FAILURE;
@@ -568,6 +581,6 @@ int main(int argc, char **argv)
568 status = lua_cpcall(L, pmain, NULL); 581 status = lua_cpcall(L, pmain, NULL);
569 report(L, status); 582 report(L, status);
570 lua_close(L); 583 lua_close(L);
571 return (status || smain.status) ? EXIT_FAILURE : EXIT_SUCCESS; 584 return (status || smain.status > 0) ? EXIT_FAILURE : EXIT_SUCCESS;
572} 585}
573 586
diff --git a/src/luajit_rolling.h b/src/luajit_rolling.h
index 27368836..e564477a 100644
--- a/src/luajit_rolling.h
+++ b/src/luajit_rolling.h
@@ -30,9 +30,9 @@
30 30
31#include "lua.h" 31#include "lua.h"
32 32
33#define LUAJIT_VERSION "LuaJIT 2.0.ROLLING" 33#define LUAJIT_VERSION "LuaJIT 2.1.ROLLING"
34#define LUAJIT_VERSION_NUM 20099 /* Deprecated. */ 34#define LUAJIT_VERSION_NUM 20199 /* Deprecated. */
35#define LUAJIT_VERSION_SYM luaJIT_version_2_0_ROLLING 35#define LUAJIT_VERSION_SYM luaJIT_version_2_1_ROLLING
36#define LUAJIT_COPYRIGHT "Copyright (C) 2005-2023 Mike Pall" 36#define LUAJIT_COPYRIGHT "Copyright (C) 2005-2023 Mike Pall"
37#define LUAJIT_URL "https://luajit.org/" 37#define LUAJIT_URL "https://luajit.org/"
38 38
@@ -64,6 +64,15 @@ enum {
64/* Control the JIT engine. */ 64/* Control the JIT engine. */
65LUA_API int luaJIT_setmode(lua_State *L, int idx, int mode); 65LUA_API int luaJIT_setmode(lua_State *L, int idx, int mode);
66 66
67/* Low-overhead profiling API. */
68typedef void (*luaJIT_profile_callback)(void *data, lua_State *L,
69 int samples, int vmstate);
70LUA_API void luaJIT_profile_start(lua_State *L, const char *mode,
71 luaJIT_profile_callback cb, void *data);
72LUA_API void luaJIT_profile_stop(lua_State *L);
73LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt,
74 int depth, size_t *len);
75
67/* Enforce (dynamic) linker error for version mismatches. Call from main. */ 76/* Enforce (dynamic) linker error for version mismatches. Call from main. */
68LUA_API void LUAJIT_VERSION_SYM(void); 77LUA_API void LUAJIT_VERSION_SYM(void);
69 78
diff --git a/src/lualib.h b/src/lualib.h
index 316782fc..ea116eb1 100644
--- a/src/lualib.h
+++ b/src/lualib.h
@@ -33,6 +33,7 @@ LUALIB_API int luaopen_debug(lua_State *L);
33LUALIB_API int luaopen_bit(lua_State *L); 33LUALIB_API int luaopen_bit(lua_State *L);
34LUALIB_API int luaopen_jit(lua_State *L); 34LUALIB_API int luaopen_jit(lua_State *L);
35LUALIB_API int luaopen_ffi(lua_State *L); 35LUALIB_API int luaopen_ffi(lua_State *L);
36LUALIB_API int luaopen_string_buffer(lua_State *L);
36 37
37LUALIB_API void luaL_openlibs(lua_State *L); 38LUALIB_API void luaL_openlibs(lua_State *L);
38 39
diff --git a/src/msvcbuild.bat b/src/msvcbuild.bat
index 6d6c1aa2..dfe98e16 100644
--- a/src/msvcbuild.bat
+++ b/src/msvcbuild.bat
@@ -5,6 +5,7 @@
5@rem Then cd to this directory and run this script. Use the following 5@rem Then cd to this directory and run this script. Use the following
6@rem options (in order), if needed. The default is a dynamic release build. 6@rem options (in order), if needed. The default is a dynamic release build.
7@rem 7@rem
8@rem nogc64 disable LJ_GC64 mode for x64
8@rem debug emit debug symbols 9@rem debug emit debug symbols
9@rem amalg amalgamated build 10@rem amalg amalgamated build
10@rem static static linkage 11@rem static static linkage
@@ -20,10 +21,11 @@
20@set LJLIB=lib /nologo /nodefaultlib 21@set LJLIB=lib /nologo /nodefaultlib
21@set DASMDIR=..\dynasm 22@set DASMDIR=..\dynasm
22@set DASM=%DASMDIR%\dynasm.lua 23@set DASM=%DASMDIR%\dynasm.lua
24@set DASC=vm_x64.dasc
23@set LJDLLNAME=lua51.dll 25@set LJDLLNAME=lua51.dll
24@set LJLIBNAME=lua51.lib 26@set LJLIBNAME=lua51.lib
25@set BUILDTYPE=release 27@set BUILDTYPE=release
26@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c 28@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
27 29
28%LJCOMPILE% host\minilua.c 30%LJCOMPILE% host\minilua.c
29@if errorlevel 1 goto :BAD 31@if errorlevel 1 goto :BAD
@@ -36,10 +38,17 @@ if exist minilua.exe.manifest^
36@set LJARCH=x64 38@set LJARCH=x64
37@minilua 39@minilua
38@if errorlevel 8 goto :X64 40@if errorlevel 8 goto :X64
41@set DASC=vm_x86.dasc
39@set DASMFLAGS=-D WIN -D JIT -D FFI 42@set DASMFLAGS=-D WIN -D JIT -D FFI
40@set LJARCH=x86 43@set LJARCH=x86
44@set LJCOMPILE=%LJCOMPILE% /arch:SSE2
41:X64 45:X64
42minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h vm_x86.dasc 46@if "%1" neq "nogc64" goto :GC64
47@shift
48@set DASC=vm_x86.dasc
49@set LJCOMPILE=%LJCOMPILE% /DLUAJIT_DISABLE_GC64
50:GC64
51minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h %DASC%
43@if errorlevel 1 goto :BAD 52@if errorlevel 1 goto :BAD
44 53
45%LJCOMPILE% /I "." /I %DASMDIR% host\buildvm*.c 54%LJCOMPILE% /I "." /I %DASMDIR% host\buildvm*.c
@@ -68,6 +77,7 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
68@shift 77@shift
69@set BUILDTYPE=debug 78@set BUILDTYPE=debug
70@set LJCOMPILE=%LJCOMPILE% /Zi %DEBUGCFLAGS% 79@set LJCOMPILE=%LJCOMPILE% /Zi %DEBUGCFLAGS%
80@set LJLINK=%LJLINK% /opt:ref /opt:icf /incremental:no
71:NODEBUG 81:NODEBUG
72@set LJLINK=%LJLINK% /%BUILDTYPE% 82@set LJLINK=%LJLINK% /%BUILDTYPE%
73@if "%1"=="amalg" goto :AMALGDLL 83@if "%1"=="amalg" goto :AMALGDLL
diff --git a/src/nxbuild.bat b/src/nxbuild.bat
new file mode 100644
index 00000000..5c7088ee
--- /dev/null
+++ b/src/nxbuild.bat
@@ -0,0 +1,160 @@
1@rem Script to build LuaJIT with NintendoSDK + NX Addon.
2@rem Donated to the public domain by Swyter.
3@rem
4@rem To run this script you must open a "Native Tools Command Prompt for VS".
5@rem
6@rem Either the x86 version for NX32, or x64 for the NX64 target.
7@rem This is because the pointer size of the LuaJIT host tools (buildvm.exe)
8@rem must match the cross-compiled target (32 or 64 bits).
9@rem
10@rem Then cd to this directory and run this script.
11@rem
12@rem Recommended invocation:
13@rem
14@rem nxbuild # release build, amalgamated
15@rem nxbuild debug # debug build, amalgamated
16@rem
17@rem Additional command-line options (not generally recommended):
18@rem
19@rem noamalg # (after debug) non-amalgamated build
20
21@if not defined INCLUDE goto :FAIL
22@if not defined NINTENDO_SDK_ROOT goto :FAIL
23@if not defined PLATFORM goto :FAIL
24
25@if "%platform%" == "x86" goto :DO_NX32
26@if "%platform%" == "x64" goto :DO_NX64
27
28@echo Error: Current host platform is %platform%!
29@echo.
30@goto :FAIL
31
32@setlocal
33
34:DO_NX32
35@set DASC=vm_arm.dasc
36@set DASMFLAGS= -D HFABI -D FPU
37@set DASMTARGET= -D LUAJIT_TARGET=LUAJIT_ARCH_ARM
38@set HOST_PTR_SIZE=4
39goto :BEGIN
40
41:DO_NX64
42@set DASC=vm_arm64.dasc
43@set DASMFLAGS= -D ENDIAN_LE
44@set DASMTARGET= -D LUAJIT_TARGET=LUAJIT_ARCH_ARM64
45@set HOST_PTR_SIZE=8
46
47:BEGIN
48@rem ---- Host compiler ----
49@set LJCOMPILE=cl /nologo /c /MD /O2 /W3 /wo4146 /wo4244 /D_CRT_SECURE_NO_DEPRECATE
50@set LJLINK=link /nologo
51@set LJMT=mt /nologo
52@set DASMDIR=..\dynasm
53@set DASM=%DASMDIR%\dynasm.lua
54@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
55
56%LJCOMPILE% host\minilua.c
57@if errorlevel 1 goto :BAD
58%LJLINK% /out:minilua.exe minilua.obj
59@if errorlevel 1 goto :BAD
60if exist minilua.exe.manifest^
61 %LJMT% -manifest minilua.exe.manifest -outputresource:minilua.exe
62
63@rem Check that we have the right 32/64 bit host compiler to generate the right virtual machine files.
64@minilua
65@if "%ERRORLEVEL%" == "%HOST_PTR_SIZE%" goto :PASSED_PTR_CHECK
66
67@echo The pointer size of the host in bytes (%HOST_PTR_SIZE%) does not match the expected value (%errorlevel%).
68@echo Check that the script is being ran under the correct x86/x64 VS prompt.
69@goto :BAD
70
71:PASSED_PTR_CHECK
72@set DASMFLAGS=%DASMFLAGS% %DASMTARGET% -D LJ_TARGET_NX -D LUAJIT_OS=LUAJIT_OS_OTHER -D LUAJIT_DISABLE_JIT -D LUAJIT_DISABLE_FFI
73minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h %DASC%
74@if errorlevel 1 goto :BAD
75%LJCOMPILE% /I "." /I %DASMDIR% %DASMTARGET% -D LJ_TARGET_NX -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI host\buildvm*.c
76@if errorlevel 1 goto :BAD
77%LJLINK% /out:buildvm.exe buildvm*.obj
78@if errorlevel 1 goto :BAD
79if exist buildvm.exe.manifest^
80 %LJMT% -manifest buildvm.exe.manifest -outputresource:buildvm.exe
81
82buildvm -m elfasm -o lj_vm.s
83@if errorlevel 1 goto :BAD
84buildvm -m bcdef -o lj_bcdef.h %ALL_LIB%
85@if errorlevel 1 goto :BAD
86buildvm -m ffdef -o lj_ffdef.h %ALL_LIB%
87@if errorlevel 1 goto :BAD
88buildvm -m libdef -o lj_libdef.h %ALL_LIB%
89@if errorlevel 1 goto :BAD
90buildvm -m recdef -o lj_recdef.h %ALL_LIB%
91@if errorlevel 1 goto :BAD
92buildvm -m vmdef -o jit\vmdef.lua %ALL_LIB%
93@if errorlevel 1 goto :BAD
94buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
95@if errorlevel 1 goto :BAD
96
97@rem ---- Cross compiler ----
98@if "%platform%" neq "x64" goto :NX32_CROSSBUILD
99@set LJCOMPILE="%NINTENDO_SDK_ROOT%\Compilers\NX\nx\aarch64\bin\clang" -Wall -I%NINTENDO_SDK_ROOT%\Include %DASMTARGET% -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI -DLUAJIT_USE_SYSMALLOC -c
100@set LJLIB="%NINTENDO_SDK_ROOT%\Compilers\NX\nx\aarch64\bin\aarch64-nintendo-nx-elf-ar" rc
101@set TARGETLIB_SUFFIX=nx64
102
103%NINTENDO_SDK_ROOT%\Compilers\NX\nx\aarch64\bin\aarch64-nintendo-nx-elf-as -o lj_vm.o lj_vm.s
104goto :DEBUGCHECK
105
106:NX32_CROSSBUILD
107@set LJCOMPILE="%NINTENDO_SDK_ROOT%\Compilers\NX\nx\armv7l\bin\clang" -Wall -I%NINTENDO_SDK_ROOT%\Include %DASMTARGET% -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI -DLUAJIT_USE_SYSMALLOC -c
108@set LJLIB="%NINTENDO_SDK_ROOT%\Compilers\NX\nx\armv7l\bin\armv7l-nintendo-nx-eabihf-ar" rc
109@set TARGETLIB_SUFFIX=nx32
110
111%NINTENDO_SDK_ROOT%\Compilers\NX\nx\armv7l\bin\armv7l-nintendo-nx-eabihf-as -o lj_vm.o lj_vm.s
112:DEBUGCHECK
113
114@if "%1" neq "debug" goto :NODEBUG
115@shift
116@set LJCOMPILE=%LJCOMPILE% -DNN_SDK_BUILD_DEBUG -g -O0
117@set TARGETLIB=libluajitD_%TARGETLIB_SUFFIX%.a
118goto :BUILD
119:NODEBUG
120@set LJCOMPILE=%LJCOMPILE% -DNN_SDK_BUILD_RELEASE -O3
121@set TARGETLIB=libluajit_%TARGETLIB_SUFFIX%.a
122:BUILD
123del %TARGETLIB%
124@set LJCOMPILE=%LJCOMPILE% -fPIC
125@if "%1" neq "noamalg" goto :AMALG
126for %%f in (lj_*.c lib_*.c) do (
127 %LJCOMPILE% %%f
128 @if errorlevel 1 goto :BAD
129)
130
131%LJLIB% %TARGETLIB% lj_*.o lib_*.o
132@if errorlevel 1 goto :BAD
133@goto :NOAMALG
134:AMALG
135%LJCOMPILE% ljamalg.c
136@if errorlevel 1 goto :BAD
137%LJLIB% %TARGETLIB% ljamalg.o lj_vm.o
138@if errorlevel 1 goto :BAD
139:NOAMALG
140
141@del *.o *.obj *.manifest minilua.exe buildvm.exe
142@echo.
143@echo === Successfully built LuaJIT for Nintendo Switch (%TARGETLIB_SUFFIX%) ===
144
145@goto :END
146:BAD
147@echo.
148@echo *******************************************************
149@echo *** Build FAILED -- Please check the error messages ***
150@echo *******************************************************
151@goto :END
152:FAIL
153@echo To run this script you must open a "Native Tools Command Prompt for VS".
154@echo.
155@echo Either the x86 version for NX32, or x64 for the NX64 target.
156@echo This is because the pointer size of the LuaJIT host tools (buildvm.exe)
157@echo must match the cross-compiled target (32 or 64 bits).
158@echo.
159@echo Keep in mind that NintendoSDK + NX Addon must be installed, too.
160:END
diff --git a/src/ps4build.bat b/src/ps4build.bat
index 337a44fa..034e2c87 100644
--- a/src/ps4build.bat
+++ b/src/ps4build.bat
@@ -2,7 +2,19 @@
2@rem Donated to the public domain. 2@rem Donated to the public domain.
3@rem 3@rem
4@rem Open a "Visual Studio .NET Command Prompt" (64 bit host compiler) 4@rem Open a "Visual Studio .NET Command Prompt" (64 bit host compiler)
5@rem or "VS2015 x64 Native Tools Command Prompt".
6@rem
5@rem Then cd to this directory and run this script. 7@rem Then cd to this directory and run this script.
8@rem
9@rem Recommended invocation:
10@rem
11@rem ps4build release build, amalgamated, 64-bit GC
12@rem ps4build debug debug build, amalgamated, 64-bit GC
13@rem
14@rem Additional command-line options (not generally recommended):
15@rem
16@rem gc32 (before debug) 32-bit GC
17@rem noamalg (after debug) non-amalgamated build
6 18
7@if not defined INCLUDE goto :FAIL 19@if not defined INCLUDE goto :FAIL
8@if not defined SCE_ORBIS_SDK_DIR goto :FAIL 20@if not defined SCE_ORBIS_SDK_DIR goto :FAIL
@@ -14,7 +26,15 @@
14@set LJMT=mt /nologo 26@set LJMT=mt /nologo
15@set DASMDIR=..\dynasm 27@set DASMDIR=..\dynasm
16@set DASM=%DASMDIR%\dynasm.lua 28@set DASM=%DASMDIR%\dynasm.lua
17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c 29@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
30@set GC64=
31@set DASC=vm_x64.dasc
32
33@if "%1" neq "gc32" goto :NOGC32
34@shift
35@set GC64=-DLUAJIT_DISABLE_GC64
36@set DASC=vm_x86.dasc
37:NOGC32
18 38
19%LJCOMPILE% host\minilua.c 39%LJCOMPILE% host\minilua.c
20@if errorlevel 1 goto :BAD 40@if errorlevel 1 goto :BAD
@@ -28,10 +48,10 @@ if exist minilua.exe.manifest^
28@if not errorlevel 8 goto :FAIL 48@if not errorlevel 8 goto :FAIL
29 49
30@set DASMFLAGS=-D P64 -D NO_UNWIND 50@set DASMFLAGS=-D P64 -D NO_UNWIND
31minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h vm_x86.dasc 51minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h %DASC%
32@if errorlevel 1 goto :BAD 52@if errorlevel 1 goto :BAD
33 53
34%LJCOMPILE% /I "." /I %DASMDIR% -DLUAJIT_TARGET=LUAJIT_ARCH_X64 -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI -DLUAJIT_NO_UNWIND host\buildvm*.c 54%LJCOMPILE% /I "." /I %DASMDIR% %GC64% -DLUAJIT_TARGET=LUAJIT_ARCH_X64 -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI -DLUAJIT_USE_SYSMALLOC -DLUAJIT_NO_UNWIND host\buildvm*.c
35@if errorlevel 1 goto :BAD 55@if errorlevel 1 goto :BAD
36%LJLINK% /out:buildvm.exe buildvm*.obj 56%LJLINK% /out:buildvm.exe buildvm*.obj
37@if errorlevel 1 goto :BAD 57@if errorlevel 1 goto :BAD
@@ -54,23 +74,23 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
54@if errorlevel 1 goto :BAD 74@if errorlevel 1 goto :BAD
55 75
56@rem ---- Cross compiler ---- 76@rem ---- Cross compiler ----
57@set LJCOMPILE="%SCE_ORBIS_SDK_DIR%\host_tools\bin\orbis-clang" -c -Wall -DLUAJIT_DISABLE_FFI 77@set LJCOMPILE="%SCE_ORBIS_SDK_DIR%\host_tools\bin\orbis-clang" -c -Wall -DLUAJIT_DISABLE_FFI %GC64%
58@set LJLIB="%SCE_ORBIS_SDK_DIR%\host_tools\bin\orbis-ar" rcus 78@set LJLIB="%SCE_ORBIS_SDK_DIR%\host_tools\bin\orbis-ar" rcus
59@set INCLUDE="" 79@set INCLUDE=""
60 80
61orbis-as -o lj_vm.o lj_vm.s 81"%SCE_ORBIS_SDK_DIR%\host_tools\bin\orbis-as" -o lj_vm.o lj_vm.s
62 82
63@if "%1" neq "debug" goto :NODEBUG 83@if "%1" neq "debug" goto :NODEBUG
64@shift 84@shift
65@set LJCOMPILE=%LJCOMPILE% -g -O0 85@set LJCOMPILE=%LJCOMPILE% -g -O0
66@set TARGETLIB=libluajitD.a 86@set TARGETLIB=libluajitD_ps4.a
67goto :BUILD 87goto :BUILD
68:NODEBUG 88:NODEBUG
69@set LJCOMPILE=%LJCOMPILE% -O2 89@set LJCOMPILE=%LJCOMPILE% -O2
70@set TARGETLIB=libluajit.a 90@set TARGETLIB=libluajit_ps4.a
71:BUILD 91:BUILD
72del %TARGETLIB% 92del %TARGETLIB%
73@if "%1"=="amalg" goto :AMALG 93@if "%1" neq "noamalg" goto :AMALG
74for %%f in (lj_*.c lib_*.c) do ( 94for %%f in (lj_*.c lib_*.c) do (
75 %LJCOMPILE% %%f 95 %LJCOMPILE% %%f
76 @if errorlevel 1 goto :BAD 96 @if errorlevel 1 goto :BAD
diff --git a/src/ps5build.bat b/src/ps5build.bat
new file mode 100644
index 00000000..56818c2e
--- /dev/null
+++ b/src/ps5build.bat
@@ -0,0 +1,123 @@
1@rem Script to build LuaJIT with the PS5 SDK.
2@rem Donated to the public domain.
3@rem
4@rem Open a "Visual Studio .NET Command Prompt" (64 bit host compiler)
5@rem or "VS20xx x64 Native Tools Command Prompt".
6@rem
7@rem Then cd to this directory and run this script.
8@rem
9@rem Recommended invocation:
10@rem
11@rem ps5build release build, amalgamated, 64-bit GC
12@rem ps5build debug debug build, amalgamated, 64-bit GC
13@rem
14@rem Additional command-line options (not generally recommended):
15@rem
16@rem gc32 (before debug) 32-bit GC
17@rem noamalg (after debug) non-amalgamated build
18
19@if not defined INCLUDE goto :FAIL
20@if not defined SCE_PROSPERO_SDK_DIR goto :FAIL
21
22@setlocal
23@rem ---- Host compiler ----
24@set LJCOMPILE=cl /nologo /c /MD /O2 /W3 /D_CRT_SECURE_NO_DEPRECATE
25@set LJLINK=link /nologo
26@set LJMT=mt /nologo
27@set DASMDIR=..\dynasm
28@set DASM=%DASMDIR%\dynasm.lua
29@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
30@set GC64=
31@set DASC=vm_x64.dasc
32
33@if "%1" neq "gc32" goto :NOGC32
34@shift
35@set GC64=-DLUAJIT_DISABLE_GC64
36@set DASC=vm_x86.dasc
37:NOGC32
38
39%LJCOMPILE% host\minilua.c
40@if errorlevel 1 goto :BAD
41%LJLINK% /out:minilua.exe minilua.obj
42@if errorlevel 1 goto :BAD
43if exist minilua.exe.manifest^
44 %LJMT% -manifest minilua.exe.manifest -outputresource:minilua.exe
45
46@rem Check for 64 bit host compiler.
47@minilua
48@if not errorlevel 8 goto :FAIL
49
50@set DASMFLAGS=-D P64 -D NO_UNWIND
51minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h %DASC%
52@if errorlevel 1 goto :BAD
53
54%LJCOMPILE% /I "." /I %DASMDIR% %GC64% -DLUAJIT_TARGET=LUAJIT_ARCH_X64 -DLUAJIT_OS=LUAJIT_OS_OTHER -DLUAJIT_DISABLE_JIT -DLUAJIT_DISABLE_FFI -DLUAJIT_NO_UNWIND host\buildvm*.c
55@if errorlevel 1 goto :BAD
56%LJLINK% /out:buildvm.exe buildvm*.obj
57@if errorlevel 1 goto :BAD
58if exist buildvm.exe.manifest^
59 %LJMT% -manifest buildvm.exe.manifest -outputresource:buildvm.exe
60
61buildvm -m elfasm -o lj_vm.s
62@if errorlevel 1 goto :BAD
63buildvm -m bcdef -o lj_bcdef.h %ALL_LIB%
64@if errorlevel 1 goto :BAD
65buildvm -m ffdef -o lj_ffdef.h %ALL_LIB%
66@if errorlevel 1 goto :BAD
67buildvm -m libdef -o lj_libdef.h %ALL_LIB%
68@if errorlevel 1 goto :BAD
69buildvm -m recdef -o lj_recdef.h %ALL_LIB%
70@if errorlevel 1 goto :BAD
71buildvm -m vmdef -o jit\vmdef.lua %ALL_LIB%
72@if errorlevel 1 goto :BAD
73buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
74@if errorlevel 1 goto :BAD
75
76@rem ---- Cross compiler ----
77@set LJCOMPILE="%SCE_PROSPERO_SDK_DIR%\host_tools\bin\prospero-clang" -c -Wall -DLUAJIT_DISABLE_FFI -DLUAJIT_USE_SYSMALLOC %GC64%
78@set LJLIB="%SCE_PROSPERO_SDK_DIR%\host_tools\bin\prospero-llvm-ar" rcus
79@set INCLUDE=""
80
81"%SCE_PROSPERO_SDK_DIR%\host_tools\bin\prospero-clang" -c -o lj_vm.o lj_vm.s
82
83@if "%1" neq "debug" goto :NODEBUG
84@shift
85@set LJCOMPILE=%LJCOMPILE% -g -O0
86@set TARGETLIB=libluajitD_ps5.a
87goto :BUILD
88:NODEBUG
89@set LJCOMPILE=%LJCOMPILE% -O2
90@set TARGETLIB=libluajit_ps5.a
91:BUILD
92del %TARGETLIB%
93@if "%1" neq "noamalg" goto :AMALG
94for %%f in (lj_*.c lib_*.c) do (
95 %LJCOMPILE% %%f
96 @if errorlevel 1 goto :BAD
97)
98
99%LJLIB% %TARGETLIB% lj_*.o lib_*.o
100@if errorlevel 1 goto :BAD
101@goto :NOAMALG
102:AMALG
103%LJCOMPILE% ljamalg.c
104@if errorlevel 1 goto :BAD
105%LJLIB% %TARGETLIB% ljamalg.o lj_vm.o
106@if errorlevel 1 goto :BAD
107:NOAMALG
108
109@del *.o *.obj *.manifest minilua.exe buildvm.exe
110@echo.
111@echo === Successfully built LuaJIT for PS5 ===
112
113@goto :END
114:BAD
115@echo.
116@echo *******************************************************
117@echo *** Build FAILED -- Please check the error messages ***
118@echo *******************************************************
119@goto :END
120:FAIL
121@echo To run this script you must open a "Visual Studio .NET Command Prompt"
122@echo (64 bit host compiler). The PS5 Prospero SDK must be installed, too.
123:END
diff --git a/src/psvitabuild.bat b/src/psvitabuild.bat
index 3991dc65..2980e157 100644
--- a/src/psvitabuild.bat
+++ b/src/psvitabuild.bat
@@ -14,7 +14,7 @@
14@set LJMT=mt /nologo 14@set LJMT=mt /nologo
15@set DASMDIR=..\dynasm 15@set DASMDIR=..\dynasm
16@set DASM=%DASMDIR%\dynasm.lua 16@set DASM=%DASMDIR%\dynasm.lua
17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c 17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
18 18
19%LJCOMPILE% host\minilua.c 19%LJCOMPILE% host\minilua.c
20@if errorlevel 1 goto :BAD 20@if errorlevel 1 goto :BAD
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index 7dae1a53..4f0798e0 100644
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -99,6 +99,7 @@
99|.type NODE, Node 99|.type NODE, Node
100|.type NARGS8, int 100|.type NARGS8, int
101|.type TRACE, GCtrace 101|.type TRACE, GCtrace
102|.type SBUF, SBuf
102| 103|
103|//----------------------------------------------------------------------- 104|//-----------------------------------------------------------------------
104| 105|
@@ -372,6 +373,17 @@ static void build_subroutines(BuildCtx *ctx)
372 | st_vmstate CARG2 373 | st_vmstate CARG2
373 | b ->vm_returnc 374 | b ->vm_returnc
374 | 375 |
376 |->vm_unwind_ext: // Complete external unwind.
377#if !LJ_NO_UNWIND
378 | push {r0, r1, r2, lr}
379 | bl extern _Unwind_Complete
380 | ldr r0, [sp]
381 | bl extern _Unwind_DeleteException
382 | pop {r0, r1, r2, lr}
383 | mov r0, r1
384 | bx r2
385#endif
386 |
375 |//----------------------------------------------------------------------- 387 |//-----------------------------------------------------------------------
376 |//-- Grow stack for calls ----------------------------------------------- 388 |//-- Grow stack for calls -----------------------------------------------
377 |//----------------------------------------------------------------------- 389 |//-----------------------------------------------------------------------
@@ -418,13 +430,14 @@ static void build_subroutines(BuildCtx *ctx)
418 | add CARG2, sp, #CFRAME_RESUME 430 | add CARG2, sp, #CFRAME_RESUME
419 | ldrb CARG1, L->status 431 | ldrb CARG1, L->status
420 | str CARG3, SAVE_ERRF 432 | str CARG3, SAVE_ERRF
421 | str CARG2, L->cframe 433 | str L, SAVE_PC // Any value outside of bytecode is ok.
422 | str CARG3, SAVE_CFRAME 434 | str CARG3, SAVE_CFRAME
423 | cmp CARG1, #0 435 | cmp CARG1, #0
424 | str L, SAVE_PC // Any value outside of bytecode is ok. 436 | str CARG2, L->cframe
425 | beq >3 437 | beq >3
426 | 438 |
427 | // Resume after yield (like a return). 439 | // Resume after yield (like a return).
440 | str L, [DISPATCH, #DISPATCH_GL(cur_L)]
428 | mov RA, BASE 441 | mov RA, BASE
429 | ldr BASE, L->base 442 | ldr BASE, L->base
430 | ldr CARG1, L->top 443 | ldr CARG1, L->top
@@ -458,14 +471,15 @@ static void build_subroutines(BuildCtx *ctx)
458 | str CARG3, SAVE_NRES 471 | str CARG3, SAVE_NRES
459 | mov L, CARG1 472 | mov L, CARG1
460 | str CARG1, SAVE_L 473 | str CARG1, SAVE_L
461 | mov BASE, CARG2
462 | str sp, L->cframe // Add our C frame to cframe chain.
463 | ldr DISPATCH, L->glref // Setup pointer to dispatch table. 474 | ldr DISPATCH, L->glref // Setup pointer to dispatch table.
475 | mov BASE, CARG2
464 | str CARG1, SAVE_PC // Any value outside of bytecode is ok. 476 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
465 | str RC, SAVE_CFRAME 477 | str RC, SAVE_CFRAME
466 | add DISPATCH, DISPATCH, #GG_G2DISP 478 | add DISPATCH, DISPATCH, #GG_G2DISP
479 | str sp, L->cframe // Add our C frame to cframe chain.
467 | 480 |
468 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype). 481 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
482 | str L, [DISPATCH, #DISPATCH_GL(cur_L)]
469 | ldr RB, L->base // RB = old base (for vmeta_call). 483 | ldr RB, L->base // RB = old base (for vmeta_call).
470 | ldr CARG1, L->top 484 | ldr CARG1, L->top
471 | mov MASKR8, #255 485 | mov MASKR8, #255
@@ -491,20 +505,21 @@ static void build_subroutines(BuildCtx *ctx)
491 | mov L, CARG1 505 | mov L, CARG1
492 | ldr RA, L:CARG1->stack 506 | ldr RA, L:CARG1->stack
493 | str CARG1, SAVE_L 507 | str CARG1, SAVE_L
508 | ldr DISPATCH, L->glref // Setup pointer to dispatch table.
494 | ldr RB, L->top 509 | ldr RB, L->top
495 | str CARG1, SAVE_PC // Any value outside of bytecode is ok. 510 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
496 | ldr RC, L->cframe 511 | ldr RC, L->cframe
512 | add DISPATCH, DISPATCH, #GG_G2DISP
497 | sub RA, RA, RB // Compute -savestack(L, L->top). 513 | sub RA, RA, RB // Compute -savestack(L, L->top).
498 | str sp, L->cframe // Add our C frame to cframe chain.
499 | mov RB, #0 514 | mov RB, #0
500 | str RA, SAVE_NRES // Neg. delta means cframe w/o frame. 515 | str RA, SAVE_NRES // Neg. delta means cframe w/o frame.
501 | str RB, SAVE_ERRF // No error function. 516 | str RB, SAVE_ERRF // No error function.
502 | str RC, SAVE_CFRAME 517 | str RC, SAVE_CFRAME
518 | str sp, L->cframe // Add our C frame to cframe chain.
519 | str L, [DISPATCH, #DISPATCH_GL(cur_L)]
503 | blx CARG4 // (lua_State *L, lua_CFunction func, void *ud) 520 | blx CARG4 // (lua_State *L, lua_CFunction func, void *ud)
504 | ldr DISPATCH, L->glref // Setup pointer to dispatch table.
505 | movs BASE, CRET1 521 | movs BASE, CRET1
506 | mov PC, #FRAME_CP 522 | mov PC, #FRAME_CP
507 | add DISPATCH, DISPATCH, #GG_G2DISP
508 | bne <3 // Else continue with the call. 523 | bne <3 // Else continue with the call.
509 | b ->vm_leave_cp // No base? Just remove C frame. 524 | b ->vm_leave_cp // No base? Just remove C frame.
510 | 525 |
@@ -614,6 +629,16 @@ static void build_subroutines(BuildCtx *ctx)
614 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here. 629 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
615 | b ->vm_call_dispatch_f 630 | b ->vm_call_dispatch_f
616 | 631 |
632 |->vmeta_tgetr:
633 | .IOS mov RC, BASE
634 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
635 | // Returns cTValue * or NULL.
636 | .IOS mov BASE, RC
637 | cmp CRET1, #0
638 | ldrdne CARG12, [CRET1]
639 | mvneq CARG2, #~LJ_TNIL
640 | b ->BC_TGETR_Z
641 |
617 |//----------------------------------------------------------------------- 642 |//-----------------------------------------------------------------------
618 | 643 |
619 |->vmeta_tsets1: 644 |->vmeta_tsets1:
@@ -671,6 +696,16 @@ static void build_subroutines(BuildCtx *ctx)
671 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here. 696 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
672 | b ->vm_call_dispatch_f 697 | b ->vm_call_dispatch_f
673 | 698 |
699 |->vmeta_tsetr:
700 | str BASE, L->base
701 | .IOS mov RC, BASE
702 | mov CARG1, L
703 | str PC, SAVE_PC
704 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
705 | // Returns TValue *.
706 | .IOS mov BASE, RC
707 | b ->BC_TSETR_Z
708 |
674 |//-- Comparison metamethods --------------------------------------------- 709 |//-- Comparison metamethods ---------------------------------------------
675 | 710 |
676 |->vmeta_comp: 711 |->vmeta_comp:
@@ -735,6 +770,17 @@ static void build_subroutines(BuildCtx *ctx)
735 | b <3 770 | b <3
736 |.endif 771 |.endif
737 | 772 |
773 |->vmeta_istype:
774 | sub PC, PC, #4
775 | str BASE, L->base
776 | mov CARG1, L
777 | lsr CARG2, RA, #3
778 | mov CARG3, RC
779 | str PC, SAVE_PC
780 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
781 | .IOS ldr BASE, L->base
782 | b ->cont_nop
783 |
738 |//-- Arithmetic metamethods --------------------------------------------- 784 |//-- Arithmetic metamethods ---------------------------------------------
739 | 785 |
740 |->vmeta_arith_vn: 786 |->vmeta_arith_vn:
@@ -966,9 +1012,9 @@ static void build_subroutines(BuildCtx *ctx)
966 | cmp TAB:RB, #0 1012 | cmp TAB:RB, #0
967 | beq ->fff_restv 1013 | beq ->fff_restv
968 | ldr CARG3, TAB:RB->hmask 1014 | ldr CARG3, TAB:RB->hmask
969 | ldr CARG4, STR:RC->hash 1015 | ldr CARG4, STR:RC->sid
970 | ldr NODE:INS, TAB:RB->node 1016 | ldr NODE:INS, TAB:RB->node
971 | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask 1017 | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask
972 | add CARG3, CARG3, CARG3, lsl #1 1018 | add CARG3, CARG3, CARG3, lsl #1
973 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 1019 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8
974 |3: // Rearranged logic, because we expect _not_ to find the key. 1020 |3: // Rearranged logic, because we expect _not_ to find the key.
@@ -1052,7 +1098,7 @@ static void build_subroutines(BuildCtx *ctx)
1052 | ffgccheck 1098 | ffgccheck
1053 | mov CARG1, L 1099 | mov CARG1, L
1054 | mov CARG2, BASE 1100 | mov CARG2, BASE
1055 | bl extern lj_str_fromnumber // (lua_State *L, cTValue *o) 1101 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1056 | // Returns GCstr *. 1102 | // Returns GCstr *.
1057 | ldr BASE, L->base 1103 | ldr BASE, L->base
1058 | mvn CARG2, #~LJ_TSTR 1104 | mvn CARG2, #~LJ_TSTR
@@ -1065,24 +1111,18 @@ static void build_subroutines(BuildCtx *ctx)
1065 | checktab CARG2, ->fff_fallback 1111 | checktab CARG2, ->fff_fallback
1066 | strd CARG34, [BASE, NARGS8:RC] // Set missing 2nd arg to nil. 1112 | strd CARG34, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1067 | ldr PC, [BASE, FRAME_PC] 1113 | ldr PC, [BASE, FRAME_PC]
1068 | mov CARG2, CARG1 1114 | add CARG2, BASE, #8
1069 | str BASE, L->base // Add frame since C call can throw. 1115 | sub CARG3, BASE, #8
1070 | mov CARG1, L 1116 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1071 | str BASE, L->top // Dummy frame length is ok. 1117 | // Returns 1=found, 0=end, -1=error.
1072 | add CARG3, BASE, #8
1073 | str PC, SAVE_PC
1074 | bl extern lj_tab_next // (lua_State *L, GCtab *t, TValue *key)
1075 | // Returns 0 at end of traversal.
1076 | .IOS ldr BASE, L->base 1118 | .IOS ldr BASE, L->base
1077 | cmp CRET1, #0 1119 | cmp CRET1, #0
1078 | mvneq CRET2, #~LJ_TNIL 1120 | mov RC, #(2+1)*8
1079 | beq ->fff_restv // End of traversal: return nil. 1121 | bgt ->fff_res // Found key/value.
1080 | ldrd CARG12, [BASE, #8] // Copy key and value to results. 1122 | bmi ->fff_fallback // Invalid key.
1081 | ldrd CARG34, [BASE, #16] 1123 | // End of traversal: return nil.
1082 | mov RC, #(2+1)*8 1124 | mvn CRET2, #~LJ_TNIL
1083 | strd CARG12, [BASE, #-8] 1125 | b ->fff_restv
1084 | strd CARG34, [BASE]
1085 | b ->fff_res
1086 | 1126 |
1087 |.ffunc_1 pairs 1127 |.ffunc_1 pairs
1088 | checktab CARG2, ->fff_fallback 1128 | checktab CARG2, ->fff_fallback
@@ -1230,9 +1270,10 @@ static void build_subroutines(BuildCtx *ctx)
1230 | ldr CARG3, L:RA->base 1270 | ldr CARG3, L:RA->base
1231 | mv_vmstate CARG2, INTERP 1271 | mv_vmstate CARG2, INTERP
1232 | ldr CARG4, L:RA->top 1272 | ldr CARG4, L:RA->top
1233 | st_vmstate CARG2
1234 | cmp CRET1, #LUA_YIELD 1273 | cmp CRET1, #LUA_YIELD
1235 | ldr BASE, L->base 1274 | ldr BASE, L->base
1275 | str L, [DISPATCH, #DISPATCH_GL(cur_L)]
1276 | st_vmstate CARG2
1236 | bhi >8 1277 | bhi >8
1237 | subs RC, CARG4, CARG3 1278 | subs RC, CARG4, CARG3
1238 | ldr CARG1, L->maxstack 1279 | ldr CARG1, L->maxstack
@@ -1500,19 +1541,6 @@ static void build_subroutines(BuildCtx *ctx)
1500 | math_extern2 atan2 1541 | math_extern2 atan2
1501 | math_extern2 fmod 1542 | math_extern2 fmod
1502 | 1543 |
1503 |->ff_math_deg:
1504 |.if FPU
1505 | .ffunc_d math_rad
1506 | vldr d1, CFUNC:CARG3->upvalue[0]
1507 | vmul.f64 d0, d0, d1
1508 | b ->fff_resd
1509 |.else
1510 | .ffunc_n math_rad
1511 | ldrd CARG34, CFUNC:CARG3->upvalue[0]
1512 | bl extern __aeabi_dmul
1513 | b ->fff_restv
1514 |.endif
1515 |
1516 |.if HFABI 1544 |.if HFABI
1517 | .ffunc math_ldexp 1545 | .ffunc math_ldexp
1518 | ldr CARG4, [BASE, #4] 1546 | ldr CARG4, [BASE, #4]
@@ -1682,17 +1710,11 @@ static void build_subroutines(BuildCtx *ctx)
1682 |.endif 1710 |.endif
1683 |.endmacro 1711 |.endmacro
1684 | 1712 |
1685 | math_minmax math_min, gt, hi 1713 | math_minmax math_min, gt, pl
1686 | math_minmax math_max, lt, lo 1714 | math_minmax math_max, lt, le
1687 | 1715 |
1688 |//-- String library ----------------------------------------------------- 1716 |//-- String library -----------------------------------------------------
1689 | 1717 |
1690 |.ffunc_1 string_len
1691 | checkstr CARG2, ->fff_fallback
1692 | ldr CARG1, STR:CARG1->len
1693 | mvn CARG2, #~LJ_TISNUM
1694 | b ->fff_restv
1695 |
1696 |.ffunc string_byte // Only handle the 1-arg case here. 1718 |.ffunc string_byte // Only handle the 1-arg case here.
1697 | ldrd CARG12, [BASE] 1719 | ldrd CARG12, [BASE]
1698 | ldr PC, [BASE, FRAME_PC] 1720 | ldr PC, [BASE, FRAME_PC]
@@ -1725,6 +1747,7 @@ static void build_subroutines(BuildCtx *ctx)
1725 | mov CARG1, L 1747 | mov CARG1, L
1726 | str PC, SAVE_PC 1748 | str PC, SAVE_PC
1727 | bl extern lj_str_new // (lua_State *L, char *str, size_t l) 1749 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1750 |->fff_resstr:
1728 | // Returns GCstr *. 1751 | // Returns GCstr *.
1729 | ldr BASE, L->base 1752 | ldr BASE, L->base
1730 | mvn CARG2, #~LJ_TSTR 1753 | mvn CARG2, #~LJ_TSTR
@@ -1768,91 +1791,28 @@ static void build_subroutines(BuildCtx *ctx)
1768 | mvn CARG2, #~LJ_TSTR 1791 | mvn CARG2, #~LJ_TSTR
1769 | b ->fff_restv 1792 | b ->fff_restv
1770 | 1793 |
1771 |.ffunc string_rep // Only handle the 1-char case inline. 1794 |.macro ffstring_op, name
1795 | .ffunc string_ .. name
1772 | ffgccheck 1796 | ffgccheck
1773 | ldrd CARG12, [BASE] 1797 | ldr CARG3, [BASE, #4]
1774 | ldrd CARG34, [BASE, #8]
1775 | cmp NARGS8:RC, #16
1776 | bne ->fff_fallback // Exactly 2 arguments
1777 | checktp CARG2, LJ_TSTR
1778 | checktpeq CARG4, LJ_TISNUM
1779 | bne ->fff_fallback
1780 | subs CARG4, CARG3, #1
1781 | ldr CARG2, STR:CARG1->len
1782 | blt ->fff_emptystr // Count <= 0?
1783 | cmp CARG2, #1
1784 | blo ->fff_emptystr // Zero-length string?
1785 | bne ->fff_fallback // Fallback for > 1-char strings.
1786 | ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)]
1787 | ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)]
1788 | ldr CARG1, STR:CARG1[1]
1789 | cmp RB, CARG3
1790 | blo ->fff_fallback
1791 |1: // Fill buffer with char.
1792 | strb CARG1, [CARG2, CARG4]
1793 | subs CARG4, CARG4, #1
1794 | bge <1
1795 | b ->fff_newstr
1796 |
1797 |.ffunc string_reverse
1798 | ffgccheck
1799 | ldrd CARG12, [BASE]
1800 | cmp NARGS8:RC, #8
1801 | blo ->fff_fallback
1802 | checkstr CARG2, ->fff_fallback
1803 | ldr CARG3, STR:CARG1->len
1804 | ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)]
1805 | ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)]
1806 | mov CARG4, CARG3
1807 | add CARG1, STR:CARG1, #sizeof(GCstr)
1808 | cmp RB, CARG3
1809 | blo ->fff_fallback
1810 |1: // Reverse string copy.
1811 | ldrb RB, [CARG1], #1
1812 | subs CARG4, CARG4, #1
1813 | blt ->fff_newstr
1814 | strb RB, [CARG2, CARG4]
1815 | b <1
1816 |
1817 |.macro ffstring_case, name, lo
1818 | .ffunc name
1819 | ffgccheck
1820 | ldrd CARG12, [BASE]
1821 | cmp NARGS8:RC, #8 1798 | cmp NARGS8:RC, #8
1799 | ldr STR:CARG2, [BASE]
1822 | blo ->fff_fallback 1800 | blo ->fff_fallback
1823 | checkstr CARG2, ->fff_fallback 1801 | sub SBUF:CARG1, DISPATCH, #-DISPATCH_GL(tmpbuf)
1824 | ldr CARG3, STR:CARG1->len 1802 | checkstr CARG3, ->fff_fallback
1825 | ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)] 1803 | ldr CARG4, SBUF:CARG1->b
1826 | ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)] 1804 | str BASE, L->base
1827 | mov CARG4, #0 1805 | str PC, SAVE_PC
1828 | add CARG1, STR:CARG1, #sizeof(GCstr) 1806 | str L, SBUF:CARG1->L
1829 | cmp RB, CARG3 1807 | str CARG4, SBUF:CARG1->w
1830 | blo ->fff_fallback 1808 | bl extern lj_buf_putstr_ .. name
1831 |1: // ASCII case conversion. 1809 | bl extern lj_buf_tostr
1832 | ldrb RB, [CARG1, CARG4] 1810 | b ->fff_resstr
1833 | cmp CARG4, CARG3
1834 | bhs ->fff_newstr
1835 | sub RC, RB, #lo
1836 | cmp RC, #26
1837 | eorlo RB, RB, #0x20
1838 | strb RB, [CARG2, CARG4]
1839 | add CARG4, CARG4, #1
1840 | b <1
1841 |.endmacro 1811 |.endmacro
1842 | 1812 |
1843 |ffstring_case string_lower, 65 1813 |ffstring_op reverse
1844 |ffstring_case string_upper, 97 1814 |ffstring_op lower
1845 | 1815 |ffstring_op upper
1846 |//-- Table library ------------------------------------------------------
1847 |
1848 |.ffunc_1 table_getn
1849 | checktab CARG2, ->fff_fallback
1850 | .IOS mov RA, BASE
1851 | bl extern lj_tab_len // (GCtab *t)
1852 | // Returns uint32_t (but less than 2^31).
1853 | .IOS mov BASE, RA
1854 | mvn CARG2, #~LJ_TISNUM
1855 | b ->fff_restv
1856 | 1816 |
1857 |//-- Bit library -------------------------------------------------------- 1817 |//-- Bit library --------------------------------------------------------
1858 | 1818 |
@@ -2127,6 +2087,66 @@ static void build_subroutines(BuildCtx *ctx)
2127 | ldr INS, [PC, #-4] 2087 | ldr INS, [PC, #-4]
2128 | bx CRET1 2088 | bx CRET1
2129 | 2089 |
2090 |->cont_stitch: // Trace stitching.
2091 |.if JIT
2092 | // RA = resultptr, CARG4 = meta base
2093 | ldr RB, SAVE_MULTRES
2094 | ldr INS, [PC, #-4]
2095 | ldr TRACE:CARG3, [CARG4, #-24] // Save previous trace.
2096 | subs RB, RB, #8
2097 | decode_RA8 RC, INS // Call base.
2098 | beq >2
2099 |1: // Move results down.
2100 | ldrd CARG12, [RA]
2101 | add RA, RA, #8
2102 | subs RB, RB, #8
2103 | strd CARG12, [BASE, RC]
2104 | add RC, RC, #8
2105 | bne <1
2106 |2:
2107 | decode_RA8 RA, INS
2108 | decode_RB8 RB, INS
2109 | add RA, RA, RB
2110 |3:
2111 | cmp RA, RC
2112 | mvn CARG2, #~LJ_TNIL
2113 | bhi >9 // More results wanted?
2114 |
2115 | ldrh RA, TRACE:CARG3->traceno
2116 | ldrh RC, TRACE:CARG3->link
2117 | cmp RC, RA
2118 | beq ->cont_nop // Blacklisted.
2119 | cmp RC, #0
2120 | bne =>BC_JLOOP // Jump to stitched trace.
2121 |
2122 | // Stitch a new trace to the previous trace.
2123 | str RA, [DISPATCH, #DISPATCH_J(exitno)]
2124 | str L, [DISPATCH, #DISPATCH_J(L)]
2125 | str BASE, L->base
2126 | sub CARG1, DISPATCH, #-GG_DISP2J
2127 | mov CARG2, PC
2128 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
2129 | ldr BASE, L->base
2130 | b ->cont_nop
2131 |
2132 |9: // Fill up results with nil.
2133 | strd CARG12, [BASE, RC]
2134 | add RC, RC, #8
2135 | b <3
2136 |.endif
2137 |
2138 |->vm_profhook: // Dispatch target for profiler hook.
2139#if LJ_HASPROFILE
2140 | mov CARG1, L
2141 | str BASE, L->base
2142 | mov CARG2, PC
2143 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
2144 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2145 | ldr BASE, L->base
2146 | sub PC, PC, #4
2147 | b ->cont_nop
2148#endif
2149 |
2130 |//----------------------------------------------------------------------- 2150 |//-----------------------------------------------------------------------
2131 |//-- Trace exit handler ------------------------------------------------- 2151 |//-- Trace exit handler -------------------------------------------------
2132 |//----------------------------------------------------------------------- 2152 |//-----------------------------------------------------------------------
@@ -2151,14 +2171,14 @@ static void build_subroutines(BuildCtx *ctx)
2151 | add CARG1, CARG1, CARG2, asr #6 2171 | add CARG1, CARG1, CARG2, asr #6
2152 | ldr CARG2, [lr, #4] // Load exit stub group offset. 2172 | ldr CARG2, [lr, #4] // Load exit stub group offset.
2153 | sub CARG1, CARG1, lr 2173 | sub CARG1, CARG1, lr
2154 | ldr L, [DISPATCH, #DISPATCH_GL(jit_L)] 2174 | ldr L, [DISPATCH, #DISPATCH_GL(cur_L)]
2155 | add CARG1, CARG2, CARG1, lsr #2 // Compute exit number. 2175 | add CARG1, CARG2, CARG1, lsr #2 // Compute exit number.
2156 | ldr BASE, [DISPATCH, #DISPATCH_GL(jit_base)] 2176 | ldr BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
2157 | str CARG1, [DISPATCH, #DISPATCH_J(exitno)] 2177 | str CARG1, [DISPATCH, #DISPATCH_J(exitno)]
2158 | mov CARG4, #0 2178 | mov CARG4, #0
2159 | str L, [DISPATCH, #DISPATCH_J(L)]
2160 | str BASE, L->base 2179 | str BASE, L->base
2161 | str CARG4, [DISPATCH, #DISPATCH_GL(jit_L)] 2180 | str L, [DISPATCH, #DISPATCH_J(L)]
2181 | str CARG4, [DISPATCH, #DISPATCH_GL(jit_base)]
2162 | sub CARG1, DISPATCH, #-GG_DISP2J 2182 | sub CARG1, DISPATCH, #-GG_DISP2J
2163 | mov CARG2, sp 2183 | mov CARG2, sp
2164 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex) 2184 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex)
@@ -2176,14 +2196,15 @@ static void build_subroutines(BuildCtx *ctx)
2176 |.if JIT 2196 |.if JIT
2177 | ldr L, SAVE_L 2197 | ldr L, SAVE_L
2178 |1: 2198 |1:
2179 | cmp CARG1, #0 2199 | cmn CARG1, #LUA_ERRERR
2180 | blt >3 // Check for error from exit. 2200 | bhs >9 // Check for error from exit.
2181 | lsl RC, CARG1, #3 2201 | lsl RC, CARG1, #3
2182 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC] 2202 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2183 | str RC, SAVE_MULTRES 2203 | str RC, SAVE_MULTRES
2184 | mov CARG3, #0 2204 | mov CARG3, #0
2205 | str BASE, L->base
2185 | ldr CARG2, LFUNC:CARG2->field_pc 2206 | ldr CARG2, LFUNC:CARG2->field_pc
2186 | str CARG3, [DISPATCH, #DISPATCH_GL(jit_L)] 2207 | str CARG3, [DISPATCH, #DISPATCH_GL(jit_base)]
2187 | mv_vmstate CARG4, INTERP 2208 | mv_vmstate CARG4, INTERP
2188 | ldr KBASE, [CARG2, #PC2PROTO(k)] 2209 | ldr KBASE, [CARG2, #PC2PROTO(k)]
2189 | // Modified copy of ins_next which handles function header dispatch, too. 2210 | // Modified copy of ins_next which handles function header dispatch, too.
@@ -2192,17 +2213,48 @@ static void build_subroutines(BuildCtx *ctx)
2192 | ldr INS, [PC], #4 2213 | ldr INS, [PC], #4
2193 | lsl MASKR8, MASKR8, #3 // MASKR8 = 255*8. 2214 | lsl MASKR8, MASKR8, #3 // MASKR8 = 255*8.
2194 | st_vmstate CARG4 2215 | st_vmstate CARG4
2216 | cmn CARG1, #17 // Static dispatch?
2217 | beq >5
2218 | cmp OP, #BC_FUNCC+2 // Fast function?
2219 | bhs >4
2220 |2:
2195 | cmp OP, #BC_FUNCF // Function header? 2221 | cmp OP, #BC_FUNCF // Function header?
2196 | ldr OP, [DISPATCH, OP, lsl #2] 2222 | ldr OP, [DISPATCH, OP, lsl #2]
2197 | decode_RA8 RA, INS 2223 | decode_RA8 RA, INS
2198 | lsrlo RC, INS, #16 // No: Decode operands A*8 and D. 2224 | lsrlo RC, INS, #16 // No: Decode operands A*8 and D.
2199 | subhs RC, RC, #8 2225 | subhs RC, RC, #8
2200 | addhs RA, RA, BASE // Yes: RA = BASE+framesize*8, RC = nargs*8 2226 | addhs RA, RA, BASE // Yes: RA = BASE+framesize*8, RC = nargs*8
2227 | ldrhs CARG3, [BASE, FRAME_FUNC]
2201 | bx OP 2228 | bx OP
2202 | 2229 |
2203 |3: // Rethrow error from the right C frame. 2230 |4: // Check frame below fast function.
2231 | ldr CARG1, [BASE, FRAME_PC]
2232 | ands CARG2, CARG1, #FRAME_TYPE
2233 | bne <2 // Trace stitching continuation?
2234 | // Otherwise set KBASE for Lua function below fast function.
2235 | ldr CARG3, [CARG1, #-4]
2236 | decode_RA8 CARG1, CARG3
2237 | sub CARG2, BASE, CARG1
2238 | ldr LFUNC:CARG3, [CARG2, #-16]
2239 | ldr CARG3, LFUNC:CARG3->field_pc
2240 | ldr KBASE, [CARG3, #PC2PROTO(k)]
2241 | b <2
2242 |
2243 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2244 | ldr CARG1, [DISPATCH, #DISPATCH_J(trace)]
2245 | decode_RD RC, INS
2246 | ldr TRACE:CARG1, [CARG1, RC, lsl #2]
2247 | ldr INS, TRACE:CARG1->startins
2248 | decode_OP OP, INS
2249 | decode_RA8 RA, INS
2250 | add OP, DISPATCH, OP, lsl #2
2251 | decode_RD RC, INS
2252 | ldr pc, [OP, #GG_DISP2STATIC]
2253 |
2254 |9: // Rethrow error from the right C frame.
2255 | rsb CARG2, CARG1, #0
2204 | mov CARG1, L 2256 | mov CARG1, L
2205 | bl extern lj_err_run // (lua_State *L) 2257 | bl extern lj_err_trace // (lua_State *L, int errcode)
2206 |.endif 2258 |.endif
2207 | 2259 |
2208 |//----------------------------------------------------------------------- 2260 |//-----------------------------------------------------------------------
@@ -2385,6 +2437,64 @@ static void build_subroutines(BuildCtx *ctx)
2385 |//-- Miscellaneous functions -------------------------------------------- 2437 |//-- Miscellaneous functions --------------------------------------------
2386 |//----------------------------------------------------------------------- 2438 |//-----------------------------------------------------------------------
2387 | 2439 |
2440 |.define NEXT_TAB, TAB:CARG1
2441 |.define NEXT_RES, CARG1
2442 |.define NEXT_IDX, CARG2
2443 |.define NEXT_TMP0, CARG3
2444 |.define NEXT_TMP1, CARG4
2445 |.define NEXT_LIM, r12
2446 |.define NEXT_RES_PTR, sp
2447 |.define NEXT_RES_VAL, [sp]
2448 |.define NEXT_RES_KEY_I, [sp, #8]
2449 |.define NEXT_RES_KEY_IT, [sp, #12]
2450 |
2451 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2452 |// Next idx returned in CRET2.
2453 |->vm_next:
2454 |.if JIT
2455 | ldr NEXT_TMP0, NEXT_TAB->array
2456 | ldr NEXT_LIM, NEXT_TAB->asize
2457 | add NEXT_TMP0, NEXT_TMP0, NEXT_IDX, lsl #3
2458 |1: // Traverse array part.
2459 | subs NEXT_TMP1, NEXT_IDX, NEXT_LIM
2460 | bhs >5
2461 | ldr NEXT_TMP1, [NEXT_TMP0, #4]
2462 | str NEXT_IDX, NEXT_RES_KEY_I
2463 | add NEXT_TMP0, NEXT_TMP0, #8
2464 | add NEXT_IDX, NEXT_IDX, #1
2465 | checktp NEXT_TMP1, LJ_TNIL
2466 | beq <1 // Skip holes in array part.
2467 | ldr NEXT_TMP0, [NEXT_TMP0, #-8]
2468 | mov NEXT_RES, NEXT_RES_PTR
2469 | strd NEXT_TMP0, NEXT_RES_VAL // Stores NEXT_TMP1, too.
2470 | mvn NEXT_TMP0, #~LJ_TISNUM
2471 | str NEXT_TMP0, NEXT_RES_KEY_IT
2472 | bx lr
2473 |
2474 |5: // Traverse hash part.
2475 | ldr NEXT_TMP0, NEXT_TAB->hmask
2476 | ldr NODE:NEXT_RES, NEXT_TAB->node
2477 | add NEXT_TMP1, NEXT_TMP1, NEXT_TMP1, lsl #1
2478 | add NEXT_LIM, NEXT_LIM, NEXT_TMP0
2479 | add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP1, lsl #3
2480 |6:
2481 | cmp NEXT_IDX, NEXT_LIM
2482 | bhi >9
2483 | ldr NEXT_TMP1, NODE:NEXT_RES->val.it
2484 | checktp NEXT_TMP1, LJ_TNIL
2485 | add NEXT_IDX, NEXT_IDX, #1
2486 | bxne lr
2487 | // Skip holes in hash part.
2488 | add NEXT_RES, NEXT_RES, #sizeof(Node)
2489 | b <6
2490 |
2491 |9: // End of iteration. Set the key to nil (not the value).
2492 | mvn NEXT_TMP0, #0
2493 | mov NEXT_RES, NEXT_RES_PTR
2494 | str NEXT_TMP0, NEXT_RES_KEY_IT
2495 | bx lr
2496 |.endif
2497 |
2388 |//----------------------------------------------------------------------- 2498 |//-----------------------------------------------------------------------
2389 |//-- FFI helper functions ----------------------------------------------- 2499 |//-- FFI helper functions -----------------------------------------------
2390 |//----------------------------------------------------------------------- 2500 |//-----------------------------------------------------------------------
@@ -2832,6 +2942,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2832 | ins_next 2942 | ins_next
2833 break; 2943 break;
2834 2944
2945 case BC_ISTYPE:
2946 | // RA = src*8, RC = -type
2947 | ldrd CARG12, [BASE, RA]
2948 | ins_next1
2949 | cmn CARG2, RC
2950 | ins_next2
2951 | bne ->vmeta_istype
2952 | ins_next3
2953 break;
2954 case BC_ISNUM:
2955 | // RA = src*8, RC = -(TISNUM-1)
2956 | ldrd CARG12, [BASE, RA]
2957 | ins_next1
2958 | checktp CARG2, LJ_TISNUM
2959 | ins_next2
2960 | bhs ->vmeta_istype
2961 | ins_next3
2962 break;
2963
2835 /* -- Unary ops --------------------------------------------------------- */ 2964 /* -- Unary ops --------------------------------------------------------- */
2836 2965
2837 case BC_MOV: 2966 case BC_MOV:
@@ -3436,10 +3565,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3436 |->BC_TGETS_Z: 3565 |->BC_TGETS_Z:
3437 | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 3566 | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8
3438 | ldr CARG3, TAB:CARG1->hmask 3567 | ldr CARG3, TAB:CARG1->hmask
3439 | ldr CARG4, STR:RC->hash 3568 | ldr CARG4, STR:RC->sid
3440 | ldr NODE:INS, TAB:CARG1->node 3569 | ldr NODE:INS, TAB:CARG1->node
3441 | mov TAB:RB, TAB:CARG1 3570 | mov TAB:RB, TAB:CARG1
3442 | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask 3571 | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask
3443 | add CARG3, CARG3, CARG3, lsl #1 3572 | add CARG3, CARG3, CARG3, lsl #1
3444 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 3573 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8
3445 |1: 3574 |1:
@@ -3502,6 +3631,24 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3502 | bne <1 // 'no __index' flag set: done. 3631 | bne <1 // 'no __index' flag set: done.
3503 | b ->vmeta_tgetb 3632 | b ->vmeta_tgetb
3504 break; 3633 break;
3634 case BC_TGETR:
3635 | decode_RB8 RB, INS
3636 | decode_RC8 RC, INS
3637 | // RA = dst*8, RB = table*8, RC = key*8
3638 | ldr TAB:CARG1, [BASE, RB]
3639 | ldr CARG2, [BASE, RC]
3640 | ldr CARG4, TAB:CARG1->array
3641 | ldr CARG3, TAB:CARG1->asize
3642 | add CARG4, CARG4, CARG2, lsl #3
3643 | cmp CARG2, CARG3 // In array part?
3644 | bhs ->vmeta_tgetr
3645 | ldrd CARG12, [CARG4]
3646 |->BC_TGETR_Z:
3647 | ins_next1
3648 | ins_next2
3649 | strd CARG12, [BASE, RA]
3650 | ins_next3
3651 break;
3505 3652
3506 case BC_TSETV: 3653 case BC_TSETV:
3507 | decode_RB8 RB, INS 3654 | decode_RB8 RB, INS
@@ -3565,10 +3712,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3565 |->BC_TSETS_Z: 3712 |->BC_TSETS_Z:
3566 | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 3713 | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8
3567 | ldr CARG3, TAB:CARG1->hmask 3714 | ldr CARG3, TAB:CARG1->hmask
3568 | ldr CARG4, STR:RC->hash 3715 | ldr CARG4, STR:RC->sid
3569 | ldr NODE:INS, TAB:CARG1->node 3716 | ldr NODE:INS, TAB:CARG1->node
3570 | mov TAB:RB, TAB:CARG1 3717 | mov TAB:RB, TAB:CARG1
3571 | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask 3718 | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask
3572 | add CARG3, CARG3, CARG3, lsl #1 3719 | add CARG3, CARG3, CARG3, lsl #1
3573 | mov CARG4, #0 3720 | mov CARG4, #0
3574 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 3721 | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8
@@ -3672,6 +3819,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3672 | barrierback TAB:CARG1, INS, CARG3 3819 | barrierback TAB:CARG1, INS, CARG3
3673 | b <2 3820 | b <2
3674 break; 3821 break;
3822 case BC_TSETR:
3823 | decode_RB8 RB, INS
3824 | decode_RC8 RC, INS
3825 | // RA = src*8, RB = table*8, RC = key*8
3826 | ldr TAB:CARG2, [BASE, RB]
3827 | ldr CARG3, [BASE, RC]
3828 | ldrb INS, TAB:CARG2->marked
3829 | ldr CARG1, TAB:CARG2->array
3830 | ldr CARG4, TAB:CARG2->asize
3831 | tst INS, #LJ_GC_BLACK // isblack(table)
3832 | add CARG1, CARG1, CARG3, lsl #3
3833 | bne >7
3834 |2:
3835 | cmp CARG3, CARG4 // In array part?
3836 | bhs ->vmeta_tsetr
3837 |->BC_TSETR_Z:
3838 | ldrd CARG34, [BASE, RA]
3839 | ins_next1
3840 | ins_next2
3841 | strd CARG34, [CARG1]
3842 | ins_next3
3843 |
3844 |7: // Possible table write barrier for the value. Skip valiswhite check.
3845 | barrierback TAB:CARG2, INS, RB
3846 | b <2
3847 break;
3675 3848
3676 case BC_TSETM: 3849 case BC_TSETM:
3677 | // RA = base*8 (table at base-1), RC = num_const (start index) 3850 | // RA = base*8 (table at base-1), RC = num_const (start index)
@@ -3812,10 +3985,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3812 break; 3985 break;
3813 3986
3814 case BC_ITERN: 3987 case BC_ITERN:
3815 | // RA = base*8, (RB = nresults+1, RC = nargs+1 (2+1))
3816 |.if JIT 3988 |.if JIT
3817 | // NYI: add hotloop, record BC_ITERN. 3989 | hotloop
3818 |.endif 3990 |.endif
3991 |->vm_IITERN:
3992 | // RA = base*8, (RB = nresults+1, RC = nargs+1 (2+1))
3819 | add RA, BASE, RA 3993 | add RA, BASE, RA
3820 | ldr TAB:RB, [RA, #-16] 3994 | ldr TAB:RB, [RA, #-16]
3821 | ldr CARG1, [RA, #-8] // Get index from control var. 3995 | ldr CARG1, [RA, #-8] // Get index from control var.
@@ -3881,7 +4055,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3881 | ins_next1 4055 | ins_next1
3882 | ins_next2 4056 | ins_next2
3883 | mov CARG1, #0 4057 | mov CARG1, #0
3884 | mvn CARG2, #0x00018000 4058 | mvn CARG2, #~LJ_KEYINDEX
3885 | strd CARG1, [RA, #-8] // Initialize control var. 4059 | strd CARG1, [RA, #-8] // Initialize control var.
3886 |1: 4060 |1:
3887 | ins_next3 4061 | ins_next3
@@ -3890,9 +4064,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3890 | mov OP, #BC_ITERC 4064 | mov OP, #BC_ITERC
3891 | strb CARG1, [PC, #-4] 4065 | strb CARG1, [PC, #-4]
3892 | sub PC, RC, #0x20000 4066 | sub PC, RC, #0x20000
4067 |.if JIT
4068 | ldrb CARG1, [PC]
4069 | cmp CARG1, #BC_ITERN
4070 | bne >6
4071 |.endif
3893 | strb OP, [PC] // Subsumes ins_next1. 4072 | strb OP, [PC] // Subsumes ins_next1.
3894 | ins_next2 4073 | ins_next2
3895 | b <1 4074 | b <1
4075 |.if JIT
4076 |6: // Unpatch JLOOP.
4077 | ldr CARG1, [DISPATCH, #DISPATCH_J(trace)]
4078 | ldrh CARG2, [PC, #2]
4079 | ldr TRACE:CARG1, [CARG1, CARG2, lsl #2]
4080 | // Subsumes ins_next1 and ins_next2.
4081 | ldr INS, TRACE:CARG1->startins
4082 | bfi INS, OP, #0, #8
4083 | str INS, [PC], #4
4084 | b <1
4085 |.endif
3896 break; 4086 break;
3897 4087
3898 case BC_VARG: 4088 case BC_VARG:
@@ -4269,7 +4459,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4269 | st_vmstate CARG2 4459 | st_vmstate CARG2
4270 | ldr RA, TRACE:RC->mcode 4460 | ldr RA, TRACE:RC->mcode
4271 | str BASE, [DISPATCH, #DISPATCH_GL(jit_base)] 4461 | str BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
4272 | str L, [DISPATCH, #DISPATCH_GL(jit_L)] 4462 | str L, [DISPATCH, #DISPATCH_GL(tmpbuf.L)]
4273 | bx RA 4463 | bx RA
4274 |.endif 4464 |.endif
4275 break; 4465 break;
@@ -4387,6 +4577,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4387 | ldr BASE, L->base 4577 | ldr BASE, L->base
4388 | mv_vmstate CARG3, INTERP 4578 | mv_vmstate CARG3, INTERP
4389 | ldr CRET2, L->top 4579 | ldr CRET2, L->top
4580 | str L, [DISPATCH, #DISPATCH_GL(cur_L)]
4390 | lsl RC, CRET1, #3 4581 | lsl RC, CRET1, #3
4391 | st_vmstate CARG3 4582 | st_vmstate CARG3
4392 | ldr PC, [BASE, FRAME_PC] 4583 | ldr PC, [BASE, FRAME_PC]
diff --git a/src/vm_arm64.dasc b/src/vm_arm64.dasc
new file mode 100644
index 00000000..a7a9392c
--- /dev/null
+++ b/src/vm_arm64.dasc
@@ -0,0 +1,4201 @@
1|// Low-level VM code for ARM64 CPUs.
2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4|
5|.arch arm64
6|.section code_op, code_sub
7|
8|.actionlist build_actionlist
9|.globals GLOB_
10|.globalnames globnames
11|.externnames extnames
12|
13|// Note: The ragged indentation of the instructions is intentional.
14|// The starting columns indicate data dependencies.
15|
16|//-----------------------------------------------------------------------
17|
18|// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
19|//
20|// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
21|// x18 is reserved on most platforms. Don't use it, save it or restore it.
22|// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
23|// depending on the instruction.
24|// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
25|//
26|// x0-x7/v0-v7 hold parameters and results.
27|
28|// Fixed register assignments for the interpreter.
29|
30|// The following must be C callee-save.
31|.define BASE, x19 // Base of current Lua stack frame.
32|.define KBASE, x20 // Constants of current Lua function.
33|.define PC, x21 // Next PC.
34|.define GLREG, x22 // Global state.
35|.define LREG, x23 // Register holding lua_State (also in SAVE_L).
36|.define TISNUM, x24 // Constant LJ_TISNUM << 47.
37|.define TISNUMhi, x25 // Constant LJ_TISNUM << 15.
38|.define TISNIL, x26 // Constant -1LL.
39|.define fp, x29 // Yes, we have to maintain a frame pointer.
40|
41|.define ST_INTERP, w26 // Constant -1.
42|
43|// The following temporaries are not saved across C calls, except for RA/RC.
44|.define RA, x27
45|.define RC, x28
46|.define RB, x17
47|.define RAw, w27
48|.define RCw, w28
49|.define RBw, w17
50|.define INS, x16
51|.define INSw, w16
52|.define ITYPE, x15
53|.define TMP0, x8
54|.define TMP1, x9
55|.define TMP2, x10
56|.define TMP3, x11
57|.define TMP0w, w8
58|.define TMP1w, w9
59|.define TMP2w, w10
60|.define TMP3w, w11
61|
62|// Calling conventions. Also used as temporaries.
63|.define CARG1, x0
64|.define CARG2, x1
65|.define CARG3, x2
66|.define CARG4, x3
67|.define CARG5, x4
68|.define CARG1w, w0
69|.define CARG2w, w1
70|.define CARG3w, w2
71|.define CARG4w, w3
72|.define CARG5w, w4
73|
74|.define FARG1, d0
75|.define FARG2, d1
76|
77|.define CRET1, x0
78|.define CRET1w, w0
79|
80|//-----------------------------------------------------------------------
81|
82|// ARM64e pointer authentication codes (PAC).
83|.if PAUTH
84|.macro sp_auth; pacibsp; .endmacro
85|.macro br_auth, reg; braaz reg; .endmacro
86|.macro blr_auth, reg; blraaz reg; .endmacro
87|.macro ret_auth; retab; .endmacro
88|.else
89|.macro sp_auth; .endmacro
90|.macro br_auth, reg; br reg; .endmacro
91|.macro blr_auth, reg; blr reg; .endmacro
92|.macro ret_auth; ret; .endmacro
93|.endif
94|
95|//-----------------------------------------------------------------------
96|
97|// Stack layout while in interpreter. Must match with lj_frame.h.
98|
99|.define CFRAME_SPACE, 208
100|//----- 16 byte aligned, <-- sp entering interpreter
101|.define SAVE_FP_LR_, 192
102|.define SAVE_GPR_, 112 // 112+10*8: 64 bit GPR saves
103|.define SAVE_FPR_, 48 // 48+8*8: 64 bit FPR saves
104|// Unused [sp, #44] // 32 bit values
105|.define SAVE_NRES, [sp, #40]
106|.define SAVE_ERRF, [sp, #36]
107|.define SAVE_MULTRES, [sp, #32]
108|.define TMPD, [sp, #24] // 64 bit values
109|.define SAVE_L, [sp, #16]
110|.define SAVE_PC, [sp, #8]
111|.define SAVE_CFRAME, [sp, #0]
112|//----- 16 byte aligned, <-- sp while in interpreter.
113|
114|.define TMPDofs, #24
115|
116|.macro save_, gpr1, gpr2, fpr1, fpr2
117| stp d..fpr2, d..fpr1, [sp, # SAVE_FPR_+(14-fpr1)*8]
118| stp x..gpr2, x..gpr1, [sp, # SAVE_GPR_+(27-gpr1)*8]
119|.endmacro
120|.macro rest_, gpr1, gpr2, fpr1, fpr2
121| ldp d..fpr2, d..fpr1, [sp, # SAVE_FPR_+(14-fpr1)*8]
122| ldp x..gpr2, x..gpr1, [sp, # SAVE_GPR_+(27-gpr1)*8]
123|.endmacro
124|
125|.macro saveregs
126| sp_auth
127| sub sp, sp, # CFRAME_SPACE
128| stp fp, lr, [sp, # SAVE_FP_LR_]
129| add fp, sp, # SAVE_FP_LR_
130| stp x20, x19, [sp, # SAVE_GPR_+(27-19)*8]
131| save_ 21, 22, 8, 9
132| save_ 23, 24, 10, 11
133| save_ 25, 26, 12, 13
134| save_ 27, 28, 14, 15
135|.endmacro
136|.macro restoreregs
137| ldp x20, x19, [sp, # SAVE_GPR_+(27-19)*8]
138| rest_ 21, 22, 8, 9
139| rest_ 23, 24, 10, 11
140| rest_ 25, 26, 12, 13
141| rest_ 27, 28, 14, 15
142| ldp fp, lr, [sp, # SAVE_FP_LR_]
143| add sp, sp, # CFRAME_SPACE
144|.endmacro
145|
146|// Type definitions. Some of these are only used for documentation.
147|.type L, lua_State, LREG
148|.type GL, global_State, GLREG
149|.type TVALUE, TValue
150|.type GCOBJ, GCobj
151|.type STR, GCstr
152|.type TAB, GCtab
153|.type LFUNC, GCfuncL
154|.type CFUNC, GCfuncC
155|.type PROTO, GCproto
156|.type UPVAL, GCupval
157|.type NODE, Node
158|.type NARGS8, int
159|.type TRACE, GCtrace
160|.type SBUF, SBuf
161|
162|//-----------------------------------------------------------------------
163|
164|// Trap for not-yet-implemented parts.
165|.macro NYI; brk; .endmacro
166|
167|//-----------------------------------------------------------------------
168|
169|// Access to frame relative to BASE.
170|.define FRAME_FUNC, #-16
171|.define FRAME_PC, #-8
172|
173|// Endian-specific defines.
174|.if ENDIAN_LE
175|.define LO, 0
176|.define OFS_RD, 2
177|.define OFS_RB, 3
178|.define OFS_RA, 1
179|.define OFS_OP, 0
180|.else
181|.define LO, 4
182|.define OFS_RD, 0
183|.define OFS_RB, 0
184|.define OFS_RA, 2
185|.define OFS_OP, 3
186|.endif
187|
188|.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
189|.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
190|.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
191|.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
192|.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
193|
194|// Instruction decode+dispatch.
195|.macro ins_NEXT
196| ldr INSw, [PC], #4
197| add TMP1, GL, INS, uxtb #3
198| decode_RA RA, INS
199| ldr TMP0, [TMP1, #GG_G2DISP]
200| decode_RD RC, INS
201| br_auth TMP0
202|.endmacro
203|
204|// Instruction footer.
205|.if 1
206| // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
207| .define ins_next, ins_NEXT
208| .define ins_next_, ins_NEXT
209|.else
210| // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
211| // Affects only certain kinds of benchmarks (and only with -j off).
212| .macro ins_next
213| b ->ins_next
214| .endmacro
215| .macro ins_next_
216| ->ins_next:
217| ins_NEXT
218| .endmacro
219|.endif
220|
221|// Call decode and dispatch.
222|.macro ins_callt
223| // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
224| ldr PC, LFUNC:CARG3->pc
225| ldr INSw, [PC], #4
226| add TMP1, GL, INS, uxtb #3
227| decode_RA RA, INS
228| ldr TMP0, [TMP1, #GG_G2DISP]
229| add RA, BASE, RA, lsl #3
230| br_auth TMP0
231|.endmacro
232|
233|.macro ins_call
234| // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
235| str PC, [BASE, FRAME_PC]
236| ins_callt
237|.endmacro
238|
239|//-----------------------------------------------------------------------
240|
241|// Macros to check the TValue type and extract the GCobj. Branch on failure.
242|.macro checktp, reg, tp, target
243| asr ITYPE, reg, #47
244| cmn ITYPE, #-tp
245| and reg, reg, #LJ_GCVMASK
246| bne target
247|.endmacro
248|.macro checktp, dst, reg, tp, target
249| asr ITYPE, reg, #47
250| cmn ITYPE, #-tp
251| and dst, reg, #LJ_GCVMASK
252| bne target
253|.endmacro
254|.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
255|.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
256|.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
257|.macro checkint, reg, target
258| cmp TISNUMhi, reg, lsr #32
259| bne target
260|.endmacro
261|.macro checknum, reg, target
262| cmp TISNUMhi, reg, lsr #32
263| bls target
264|.endmacro
265|.macro checknumber, reg, target
266| cmp TISNUMhi, reg, lsr #32
267| blo target
268|.endmacro
269|
270|.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
271|.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
272|
273#define GL_J(field) (GG_G2J + (int)offsetof(jit_State, field))
274|
275#define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
276|
277|.macro hotcheck, delta
278| lsr CARG1, PC, #1
279| and CARG1, CARG1, #126
280| add CARG1, CARG1, #GG_G2DISP+GG_DISP2HOT
281| ldrh CARG2w, [GL, CARG1]
282| subs CARG2, CARG2, #delta
283| strh CARG2w, [GL, CARG1]
284|.endmacro
285|
286|.macro hotloop
287| hotcheck HOTCOUNT_LOOP
288| blo ->vm_hotloop
289|.endmacro
290|
291|.macro hotcall
292| hotcheck HOTCOUNT_CALL
293| blo ->vm_hotcall
294|.endmacro
295|
296|// Set current VM state.
297|.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
298|.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
299|
300|// Move table write barrier back. Overwrites mark and tmp.
301|.macro barrierback, tab, mark, tmp
302| ldr tmp, GL->gc.grayagain
303| and mark, mark, #~LJ_GC_BLACK // black2gray(tab)
304| str tab, GL->gc.grayagain
305| strb mark, tab->marked
306| str tmp, tab->gclist
307|.endmacro
308|
309|//-----------------------------------------------------------------------
310
311#if !LJ_DUALNUM
312#error "Only dual-number mode supported for ARM64 target"
313#endif
314
315/* Generate subroutines used by opcodes and other parts of the VM. */
316/* The .code_sub section should be last to help static branch prediction. */
317static void build_subroutines(BuildCtx *ctx)
318{
319 |.code_sub
320 |
321 |//-----------------------------------------------------------------------
322 |//-- Return handling ----------------------------------------------------
323 |//-----------------------------------------------------------------------
324 |
325 |->vm_returnp:
326 | // See vm_return. Also: RB = previous base.
327 | tbz PC, #2, ->cont_dispatch // (PC & FRAME_P) == 0?
328 |
329 | // Return from pcall or xpcall fast func.
330 | ldr PC, [RB, FRAME_PC] // Fetch PC of previous frame.
331 | mov_true TMP0
332 | mov BASE, RB
333 | // Prepending may overwrite the pcall frame, so do it at the end.
334 | str TMP0, [RA, #-8]! // Prepend true to results.
335 |
336 |->vm_returnc:
337 | adds RC, RC, #8 // RC = (nresults+1)*8.
338 | mov CRET1, #LUA_YIELD
339 | beq ->vm_unwind_c_eh
340 | str RCw, SAVE_MULTRES
341 | ands CARG1, PC, #FRAME_TYPE
342 | beq ->BC_RET_Z // Handle regular return to Lua.
343 |
344 |->vm_return:
345 | // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
346 | // CARG1 = PC & FRAME_TYPE
347 | and RB, PC, #~FRAME_TYPEP
348 | cmp CARG1, #FRAME_C
349 | sub RB, BASE, RB // RB = previous base.
350 | bne ->vm_returnp
351 |
352 | str RB, L->base
353 | ldrsw CARG2, SAVE_NRES // CARG2 = nresults+1.
354 | mv_vmstate TMP0w, C
355 | sub BASE, BASE, #16
356 | subs TMP2, RC, #8
357 | st_vmstate TMP0w
358 | beq >2
359 |1:
360 | subs TMP2, TMP2, #8
361 | ldr TMP0, [RA], #8
362 | str TMP0, [BASE], #8
363 | bne <1
364 |2:
365 | cmp RC, CARG2, lsl #3 // More/less results wanted?
366 | bne >6
367 |3:
368 | str BASE, L->top // Store new top.
369 |
370 |->vm_leave_cp:
371 | ldr RC, SAVE_CFRAME // Restore previous C frame.
372 | mov CRET1, #0 // Ok return status for vm_pcall.
373 | str RC, L->cframe
374 |
375 |->vm_leave_unw:
376 | restoreregs
377 | ret_auth
378 |
379 |6:
380 | bgt >7 // Less results wanted?
381 | // More results wanted. Check stack size and fill up results with nil.
382 | ldr CARG3, L->maxstack
383 | cmp BASE, CARG3
384 | bhs >8
385 | str TISNIL, [BASE], #8
386 | add RC, RC, #8
387 | b <2
388 |
389 |7: // Less results wanted.
390 | cbz CARG2, <3 // LUA_MULTRET+1 case?
391 | sub CARG1, RC, CARG2, lsl #3
392 | sub BASE, BASE, CARG1 // Shrink top.
393 | b <3
394 |
395 |8: // Corner case: need to grow stack for filling up results.
396 | // This can happen if:
397 | // - A C function grows the stack (a lot).
398 | // - The GC shrinks the stack in between.
399 | // - A return back from a lua_call() with (high) nresults adjustment.
400 | str BASE, L->top // Save current top held in BASE (yes).
401 | mov CARG1, L
402 | bl extern lj_state_growstack // (lua_State *L, int n)
403 | ldr BASE, L->top // Need the (realloced) L->top in BASE.
404 | ldrsw CARG2, SAVE_NRES
405 | b <2
406 |
407 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
408 | // (void *cframe, int errcode)
409 | mov sp, CARG1
410 | mov CRET1, CARG2
411 |->vm_unwind_c_eh: // Landing pad for external unwinder.
412 | ldr L, SAVE_L
413 | mv_vmstate TMP0w, C
414 | ldr GL, L->glref
415 | st_vmstate TMP0w
416 | b ->vm_leave_unw
417 |
418 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
419 | // (void *cframe)
420 | and sp, CARG1, #CFRAME_RAWMASK
421 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
422 | ldr L, SAVE_L
423 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
424 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
425 | movn TISNIL, #0
426 | mov RC, #16 // 2 results: false + error message.
427 | ldr BASE, L->base
428 | ldr GL, L->glref // Setup pointer to global state.
429 | mov_false TMP0
430 | sub RA, BASE, #8 // Results start at BASE-8.
431 | ldr PC, [BASE, FRAME_PC] // Fetch PC of previous frame.
432 | str TMP0, [BASE, #-8] // Prepend false to error message.
433 | st_vmstate ST_INTERP
434 | b ->vm_returnc
435 |
436 |//-----------------------------------------------------------------------
437 |//-- Grow stack for calls -----------------------------------------------
438 |//-----------------------------------------------------------------------
439 |
440 |->vm_growstack_c: // Grow stack for C function.
441 | // CARG1 = L
442 | mov CARG2, #LUA_MINSTACK
443 | b >2
444 |
445 |->vm_growstack_l: // Grow stack for Lua function.
446 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
447 | add RC, BASE, RC
448 | sub RA, RA, BASE
449 | mov CARG1, L
450 | stp BASE, RC, L->base
451 | add PC, PC, #4 // Must point after first instruction.
452 | lsr CARG2, RA, #3
453 |2:
454 | // L->base = new base, L->top = top
455 | str PC, SAVE_PC
456 | bl extern lj_state_growstack // (lua_State *L, int n)
457 | ldp BASE, RC, L->base
458 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
459 | sub NARGS8:RC, RC, BASE
460 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
461 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
462 | ins_callt // Just retry the call.
463 |
464 |//-----------------------------------------------------------------------
465 |//-- Entry points into the assembler VM ---------------------------------
466 |//-----------------------------------------------------------------------
467 |
468 |->vm_resume: // Setup C frame and resume thread.
469 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
470 | saveregs
471 | mov L, CARG1
472 | ldr GL, L->glref // Setup pointer to global state.
473 | mov BASE, CARG2
474 | str L, SAVE_L
475 | mov PC, #FRAME_CP
476 | str wzr, SAVE_NRES
477 | add TMP0, sp, #CFRAME_RESUME
478 | ldrb TMP1w, L->status
479 | str wzr, SAVE_ERRF
480 | str L, SAVE_PC // Any value outside of bytecode is ok.
481 | str xzr, SAVE_CFRAME
482 | str TMP0, L->cframe
483 | cbz TMP1w, >3
484 |
485 | // Resume after yield (like a return).
486 | str L, GL->cur_L
487 | mov RA, BASE
488 | ldp BASE, CARG1, L->base
489 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
490 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
491 | ldr PC, [BASE, FRAME_PC]
492 | strb wzr, L->status
493 | movn TISNIL, #0
494 | sub RC, CARG1, BASE
495 | ands CARG1, PC, #FRAME_TYPE
496 | add RC, RC, #8
497 | st_vmstate ST_INTERP
498 | str RCw, SAVE_MULTRES
499 | beq ->BC_RET_Z
500 | b ->vm_return
501 |
502 |->vm_pcall: // Setup protected C frame and enter VM.
503 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
504 | saveregs
505 | mov PC, #FRAME_CP
506 | str CARG4w, SAVE_ERRF
507 | b >1
508 |
509 |->vm_call: // Setup C frame and enter VM.
510 | // (lua_State *L, TValue *base, int nres1)
511 | saveregs
512 | mov PC, #FRAME_C
513 |
514 |1: // Entry point for vm_pcall above (PC = ftype).
515 | ldr RC, L:CARG1->cframe
516 | str CARG3w, SAVE_NRES
517 | mov L, CARG1
518 | str CARG1, SAVE_L
519 | ldr GL, L->glref // Setup pointer to global state.
520 | mov BASE, CARG2
521 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
522 | add TMP0, sp, #0
523 | str RC, SAVE_CFRAME
524 | str TMP0, L->cframe // Add our C frame to cframe chain.
525 |
526 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
527 | str L, GL->cur_L
528 | ldp RB, CARG1, L->base // RB = old base (for vmeta_call).
529 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
530 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
531 | add PC, PC, BASE
532 | movn TISNIL, #0
533 | sub PC, PC, RB // PC = frame delta + frame type
534 | sub NARGS8:RC, CARG1, BASE
535 | st_vmstate ST_INTERP
536 |
537 |->vm_call_dispatch:
538 | // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
539 | ldr CARG3, [BASE, FRAME_FUNC]
540 | checkfunc CARG3, ->vmeta_call
541 |
542 |->vm_call_dispatch_f:
543 | ins_call
544 | // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
545 |
546 |->vm_cpcall: // Setup protected C frame, call C.
547 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
548 | saveregs
549 | mov L, CARG1
550 | ldr RA, L:CARG1->stack
551 | str CARG1, SAVE_L
552 | ldr GL, L->glref // Setup pointer to global state.
553 | ldr RB, L->top
554 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
555 | ldr RC, L->cframe
556 | sub RA, RA, RB // Compute -savestack(L, L->top).
557 | str RAw, SAVE_NRES // Neg. delta means cframe w/o frame.
558 | str wzr, SAVE_ERRF // No error function.
559 | add TMP0, sp, #0
560 | str RC, SAVE_CFRAME
561 | str TMP0, L->cframe // Add our C frame to cframe chain.
562 | str L, GL->cur_L
563 | blr_auth CARG4 // (lua_State *L, lua_CFunction func, void *ud)
564 | mov BASE, CRET1
565 | mov PC, #FRAME_CP
566 | cbnz BASE, <3 // Else continue with the call.
567 | b ->vm_leave_cp // No base? Just remove C frame.
568 |
569 |//-----------------------------------------------------------------------
570 |//-- Metamethod handling ------------------------------------------------
571 |//-----------------------------------------------------------------------
572 |
573 |//-- Continuation dispatch ----------------------------------------------
574 |
575 |->cont_dispatch:
576 | // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
577 | ldr LFUNC:CARG3, [RB, FRAME_FUNC]
578 | ldr CARG1, [BASE, #-32] // Get continuation.
579 | mov CARG4, BASE
580 | mov BASE, RB // Restore caller BASE.
581 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
582 |.if FFI
583 | cmp CARG1, #1
584 |.endif
585 | ldr PC, [CARG4, #-24] // Restore PC from [cont|PC].
586 | add TMP0, RA, RC
587 | str TISNIL, [TMP0, #-8] // Ensure one valid arg.
588 |.if FFI
589 | bls >1
590 |.endif
591 | ldr CARG3, LFUNC:CARG3->pc
592 | ldr KBASE, [CARG3, #PC2PROTO(k)]
593 | // BASE = base, RA = resultptr, CARG4 = meta base
594 | br_auth CARG1
595 |
596 |.if FFI
597 |1:
598 | beq ->cont_ffi_callback // cont = 1: return from FFI callback.
599 | // cont = 0: tailcall from C function.
600 | sub CARG4, CARG4, #32
601 | sub RC, CARG4, BASE
602 | b ->vm_call_tail
603 |.endif
604 |
605 |->cont_cat: // RA = resultptr, CARG4 = meta base
606 | ldr INSw, [PC, #-4]
607 | sub CARG2, CARG4, #32
608 | ldr TMP0, [RA]
609 | str BASE, L->base
610 | decode_RB RB, INS
611 | decode_RA RA, INS
612 | add TMP1, BASE, RB, lsl #3
613 | subs TMP1, CARG2, TMP1
614 | beq >1
615 | str TMP0, [CARG2]
616 | lsr CARG3, TMP1, #3
617 | b ->BC_CAT_Z
618 |
619 |1:
620 | str TMP0, [BASE, RA, lsl #3]
621 | b ->cont_nop
622 |
623 |//-- Table indexing metamethods -----------------------------------------
624 |
625 |->vmeta_tgets1:
626 | movn CARG4, #~LJ_TSTR
627 | add CARG2, BASE, RB, lsl #3
628 | add CARG4, STR:RC, CARG4, lsl #47
629 | b >2
630 |
631 |->vmeta_tgets:
632 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
633 | str CARG2, GL->tmptv
634 | add CARG2, GL, #offsetof(global_State, tmptv)
635 |2:
636 | add CARG3, sp, TMPDofs
637 | str CARG4, TMPD
638 | b >1
639 |
640 |->vmeta_tgetb: // RB = table, RC = index
641 | add RC, RC, TISNUM
642 | add CARG2, BASE, RB, lsl #3
643 | add CARG3, sp, TMPDofs
644 | str RC, TMPD
645 | b >1
646 |
647 |->vmeta_tgetv: // RB = table, RC = key
648 | add CARG2, BASE, RB, lsl #3
649 | add CARG3, BASE, RC, lsl #3
650 |1:
651 | str BASE, L->base
652 | mov CARG1, L
653 | str PC, SAVE_PC
654 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
655 | // Returns TValue * (finished) or NULL (metamethod).
656 | cbz CRET1, >3
657 | ldr TMP0, [CRET1]
658 | str TMP0, [BASE, RA, lsl #3]
659 | ins_next
660 |
661 |3: // Call __index metamethod.
662 | // BASE = base, L->top = new base, stack = cont/func/t/k
663 | sub TMP1, BASE, #FRAME_CONT
664 | ldr BASE, L->top
665 | mov NARGS8:RC, #16 // 2 args for func(t, k).
666 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
667 | str PC, [BASE, #-24] // [cont|PC]
668 | sub PC, BASE, TMP1
669 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
670 | b ->vm_call_dispatch_f
671 |
672 |->vmeta_tgetr:
673 | sxtw CARG2, TMP1w
674 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
675 | // Returns cTValue * or NULL.
676 | mov TMP0, TISNIL
677 | cbz CRET1, ->BC_TGETR_Z
678 | ldr TMP0, [CRET1]
679 | b ->BC_TGETR_Z
680 |
681 |//-----------------------------------------------------------------------
682 |
683 |->vmeta_tsets1:
684 | movn CARG4, #~LJ_TSTR
685 | add CARG2, BASE, RB, lsl #3
686 | add CARG4, STR:RC, CARG4, lsl #47
687 | b >2
688 |
689 |->vmeta_tsets:
690 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
691 | str CARG2, GL->tmptv
692 | add CARG2, GL, #offsetof(global_State, tmptv)
693 |2:
694 | add CARG3, sp, TMPDofs
695 | str CARG4, TMPD
696 | b >1
697 |
698 |->vmeta_tsetb: // RB = table, RC = index
699 | add RC, RC, TISNUM
700 | add CARG2, BASE, RB, lsl #3
701 | add CARG3, sp, TMPDofs
702 | str RC, TMPD
703 | b >1
704 |
705 |->vmeta_tsetv:
706 | add CARG2, BASE, RB, lsl #3
707 | add CARG3, BASE, RC, lsl #3
708 |1:
709 | str BASE, L->base
710 | mov CARG1, L
711 | str PC, SAVE_PC
712 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
713 | // Returns TValue * (finished) or NULL (metamethod).
714 | ldr TMP0, [BASE, RA, lsl #3]
715 | cbz CRET1, >3
716 | // NOBARRIER: lj_meta_tset ensures the table is not black.
717 | str TMP0, [CRET1]
718 | ins_next
719 |
720 |3: // Call __newindex metamethod.
721 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
722 | sub TMP1, BASE, #FRAME_CONT
723 | ldr BASE, L->top
724 | mov NARGS8:RC, #24 // 3 args for func(t, k, v).
725 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
726 | str TMP0, [BASE, #16] // Copy value to third argument.
727 | str PC, [BASE, #-24] // [cont|PC]
728 | sub PC, BASE, TMP1
729 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
730 | b ->vm_call_dispatch_f
731 |
732 |->vmeta_tsetr:
733 | sxtw CARG3, TMP1w
734 | str BASE, L->base
735 | mov CARG1, L
736 | str PC, SAVE_PC
737 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
738 | // Returns TValue *.
739 | b ->BC_TSETR_Z
740 |
741 |//-- Comparison metamethods ---------------------------------------------
742 |
743 |->vmeta_comp:
744 | add CARG2, BASE, RA, lsl #3
745 | sub PC, PC, #4
746 | add CARG3, BASE, RC, lsl #3
747 | str BASE, L->base
748 | mov CARG1, L
749 | str PC, SAVE_PC
750 | uxtb CARG4w, INSw
751 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
752 | // Returns 0/1 or TValue * (metamethod).
753 |3:
754 | cmp CRET1, #1
755 | bhi ->vmeta_binop
756 |4:
757 | ldrh RBw, [PC, # OFS_RD]
758 | add PC, PC, #4
759 | add RB, PC, RB, lsl #2
760 | sub RB, RB, #0x20000
761 | csel PC, PC, RB, lo
762 |->cont_nop:
763 | ins_next
764 |
765 |->cont_ra: // RA = resultptr
766 | ldr INSw, [PC, #-4]
767 | ldr TMP0, [RA]
768 | decode_RA TMP1, INS
769 | str TMP0, [BASE, TMP1, lsl #3]
770 | b ->cont_nop
771 |
772 |->cont_condt: // RA = resultptr
773 | ldr TMP0, [RA]
774 | mov_true TMP1
775 | cmp TMP1, TMP0 // Branch if result is true.
776 | b <4
777 |
778 |->cont_condf: // RA = resultptr
779 | ldr TMP0, [RA]
780 | mov_false TMP1
781 | cmp TMP0, TMP1 // Branch if result is false.
782 | b <4
783 |
784 |->vmeta_equal:
785 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
786 | and TAB:CARG3, CARG3, #LJ_GCVMASK
787 | sub PC, PC, #4
788 | str BASE, L->base
789 | mov CARG1, L
790 | str PC, SAVE_PC
791 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
792 | // Returns 0/1 or TValue * (metamethod).
793 | b <3
794 |
795 |->vmeta_equal_cd:
796 |.if FFI
797 | sub PC, PC, #4
798 | str BASE, L->base
799 | mov CARG1, L
800 | mov CARG2, INS
801 | str PC, SAVE_PC
802 | bl extern lj_meta_equal_cd // (lua_State *L, BCIns op)
803 | // Returns 0/1 or TValue * (metamethod).
804 | b <3
805 |.endif
806 |
807 |->vmeta_istype:
808 | sub PC, PC, #4
809 | str BASE, L->base
810 | mov CARG1, L
811 | mov CARG2, RA
812 | mov CARG3, RC
813 | str PC, SAVE_PC
814 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
815 | b ->cont_nop
816 |
817 |//-- Arithmetic metamethods ---------------------------------------------
818 |
819 |->vmeta_arith_vn:
820 | add CARG3, BASE, RB, lsl #3
821 | add CARG4, KBASE, RC, lsl #3
822 | b >1
823 |
824 |->vmeta_arith_nv:
825 | add CARG4, BASE, RB, lsl #3
826 | add CARG3, KBASE, RC, lsl #3
827 | b >1
828 |
829 |->vmeta_unm:
830 | add CARG3, BASE, RC, lsl #3
831 | mov CARG4, CARG3
832 | b >1
833 |
834 |->vmeta_arith_vv:
835 | add CARG3, BASE, RB, lsl #3
836 | add CARG4, BASE, RC, lsl #3
837 |1:
838 | uxtb CARG5w, INSw
839 | add CARG2, BASE, RA, lsl #3
840 | str BASE, L->base
841 | mov CARG1, L
842 | str PC, SAVE_PC
843 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
844 | // Returns NULL (finished) or TValue * (metamethod).
845 | cbz CRET1, ->cont_nop
846 |
847 | // Call metamethod for binary op.
848 |->vmeta_binop:
849 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
850 | sub TMP1, CRET1, BASE
851 | str PC, [CRET1, #-24] // [cont|PC]
852 | add PC, TMP1, #FRAME_CONT
853 | mov BASE, CRET1
854 | mov NARGS8:RC, #16 // 2 args for func(o1, o2).
855 | b ->vm_call_dispatch
856 |
857 |->vmeta_len:
858 | add CARG2, BASE, RC, lsl #3
859#if LJ_52
860 | mov TAB:RC, TAB:CARG1 // Save table (ignored for other types).
861#endif
862 | str BASE, L->base
863 | mov CARG1, L
864 | str PC, SAVE_PC
865 | bl extern lj_meta_len // (lua_State *L, TValue *o)
866 | // Returns NULL (retry) or TValue * (metamethod base).
867#if LJ_52
868 | cbnz CRET1, ->vmeta_binop // Binop call for compatibility.
869 | mov TAB:CARG1, TAB:RC
870 | b ->BC_LEN_Z
871#else
872 | b ->vmeta_binop // Binop call for compatibility.
873#endif
874 |
875 |//-- Call metamethod ----------------------------------------------------
876 |
877 |->vmeta_call: // Resolve and call __call metamethod.
878 | // RB = old base, BASE = new base, RC = nargs*8
879 | mov CARG1, L
880 | str RB, L->base // This is the callers base!
881 | sub CARG2, BASE, #16
882 | str PC, SAVE_PC
883 | add CARG3, BASE, NARGS8:RC
884 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
885 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
886 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
887 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
888 | ins_call
889 |
890 |->vmeta_callt: // Resolve __call for BC_CALLT.
891 | // BASE = old base, RA = new base, RC = nargs*8
892 | mov CARG1, L
893 | str BASE, L->base
894 | sub CARG2, RA, #16
895 | str PC, SAVE_PC
896 | add CARG3, RA, NARGS8:RC
897 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
898 | ldr TMP1, [RA, FRAME_FUNC] // Guaranteed to be a function here.
899 | ldr PC, [BASE, FRAME_PC]
900 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
901 | and LFUNC:CARG3, TMP1, #LJ_GCVMASK
902 | b ->BC_CALLT2_Z
903 |
904 |//-- Argument coercion for 'for' statement ------------------------------
905 |
906 |->vmeta_for:
907 | mov CARG1, L
908 | str BASE, L->base
909 | mov CARG2, RA
910 | str PC, SAVE_PC
911 | bl extern lj_meta_for // (lua_State *L, TValue *base)
912 | ldr INSw, [PC, #-4]
913 |.if JIT
914 | uxtb TMP0w, INSw
915 |.endif
916 | decode_RA RA, INS
917 | decode_RD RC, INS
918 |.if JIT
919 | cmp TMP0, #BC_JFORI
920 | beq =>BC_JFORI
921 |.endif
922 | b =>BC_FORI
923 |
924 |//-----------------------------------------------------------------------
925 |//-- Fast functions -----------------------------------------------------
926 |//-----------------------------------------------------------------------
927 |
928 |.macro .ffunc, name
929 |->ff_ .. name:
930 |.endmacro
931 |
932 |.macro .ffunc_1, name
933 |->ff_ .. name:
934 | ldr CARG1, [BASE]
935 | cmp NARGS8:RC, #8
936 | blo ->fff_fallback
937 |.endmacro
938 |
939 |.macro .ffunc_2, name
940 |->ff_ .. name:
941 | ldp CARG1, CARG2, [BASE]
942 | cmp NARGS8:RC, #16
943 | blo ->fff_fallback
944 |.endmacro
945 |
946 |.macro .ffunc_n, name
947 | .ffunc name
948 | ldr CARG1, [BASE]
949 | cmp NARGS8:RC, #8
950 | ldr FARG1, [BASE]
951 | blo ->fff_fallback
952 | checknum CARG1, ->fff_fallback
953 |.endmacro
954 |
955 |.macro .ffunc_nn, name
956 | .ffunc name
957 | ldp CARG1, CARG2, [BASE]
958 | cmp NARGS8:RC, #16
959 | ldp FARG1, FARG2, [BASE]
960 | blo ->fff_fallback
961 | checknum CARG1, ->fff_fallback
962 | checknum CARG2, ->fff_fallback
963 |.endmacro
964 |
965 |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
966 |.macro ffgccheck
967 | ldp CARG1, CARG2, GL->gc.total // Assumes threshold follows total.
968 | cmp CARG1, CARG2
969 | blt >1
970 | bl ->fff_gcstep
971 |1:
972 |.endmacro
973 |
974 |//-- Base library: checks -----------------------------------------------
975 |
976 |.ffunc_1 assert
977 | ldr PC, [BASE, FRAME_PC]
978 | mov_false TMP1
979 | cmp CARG1, TMP1
980 | bhs ->fff_fallback
981 | str CARG1, [BASE, #-16]
982 | sub RB, BASE, #8
983 | subs RA, NARGS8:RC, #8
984 | add RC, NARGS8:RC, #8 // Compute (nresults+1)*8.
985 | cbz RA, ->fff_res // Done if exactly 1 argument.
986 |1:
987 | ldr CARG1, [RB, #16]
988 | sub RA, RA, #8
989 | str CARG1, [RB], #8
990 | cbnz RA, <1
991 | b ->fff_res
992 |
993 |.ffunc_1 type
994 | mov TMP0, #~LJ_TISNUM
995 | asr ITYPE, CARG1, #47
996 | cmn ITYPE, #~LJ_TISNUM
997 | csinv TMP1, TMP0, ITYPE, lo
998 | add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
999 | ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
1000 | b ->fff_restv
1001 |
1002 |//-- Base library: getters and setters ---------------------------------
1003 |
1004 |.ffunc_1 getmetatable
1005 | asr ITYPE, CARG1, #47
1006 | cmn ITYPE, #-LJ_TTAB
1007 | ccmn ITYPE, #-LJ_TUDATA, #4, ne
1008 | and TAB:CARG1, CARG1, #LJ_GCVMASK
1009 | bne >6
1010 |1: // Field metatable must be at same offset for GCtab and GCudata!
1011 | ldr TAB:RB, TAB:CARG1->metatable
1012 |2:
1013 | mov CARG1, TISNIL
1014 | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
1015 | cbz TAB:RB, ->fff_restv
1016 | ldr TMP1w, TAB:RB->hmask
1017 | ldr TMP2w, STR:RC->sid
1018 | ldr NODE:CARG3, TAB:RB->node
1019 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
1020 | add TMP1, TMP1, TMP1, lsl #1
1021 | movn CARG4, #~LJ_TSTR
1022 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
1023 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
1024 |3: // Rearranged logic, because we expect _not_ to find the key.
1025 | ldp CARG1, TMP0, NODE:CARG3->val
1026 | ldr NODE:CARG3, NODE:CARG3->next
1027 | cmp TMP0, CARG4
1028 | beq >5
1029 | cbnz NODE:CARG3, <3
1030 |4:
1031 | mov CARG1, RB // Use metatable as default result.
1032 | movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
1033 | b ->fff_restv
1034 |5:
1035 | cmp TMP0, TISNIL
1036 | bne ->fff_restv
1037 | b <4
1038 |
1039 |6:
1040 | movn TMP0, #~LJ_TISNUM
1041 | cmp ITYPE, TMP0
1042 | csel ITYPE, ITYPE, TMP0, hs
1043 | sub TMP1, GL, ITYPE, lsl #3
1044 | ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1045 | b <2
1046 |
1047 |.ffunc_2 setmetatable
1048 | // Fast path: no mt for table yet and not clearing the mt.
1049 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1050 | ldr TAB:TMP0, TAB:TMP1->metatable
1051 | asr ITYPE, CARG2, #47
1052 | ldrb TMP2w, TAB:TMP1->marked
1053 | cmn ITYPE, #-LJ_TTAB
1054 | and TAB:CARG2, CARG2, #LJ_GCVMASK
1055 | ccmp TAB:TMP0, #0, #0, eq
1056 | bne ->fff_fallback
1057 | str TAB:CARG2, TAB:TMP1->metatable
1058 | tbz TMP2w, #2, ->fff_restv // isblack(table)
1059 | barrierback TAB:TMP1, TMP2w, TMP0
1060 | b ->fff_restv
1061 |
1062 |.ffunc rawget
1063 | ldr CARG2, [BASE]
1064 | cmp NARGS8:RC, #16
1065 | blo ->fff_fallback
1066 | checktab CARG2, ->fff_fallback
1067 | mov CARG1, L
1068 | add CARG3, BASE, #8
1069 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1070 | // Returns cTValue *.
1071 | ldr CARG1, [CRET1]
1072 | b ->fff_restv
1073 |
1074 |//-- Base library: conversions ------------------------------------------
1075 |
1076 |.ffunc tonumber
1077 | // Only handles the number case inline (without a base argument).
1078 | ldr CARG1, [BASE]
1079 | cmp NARGS8:RC, #8
1080 | bne ->fff_fallback
1081 | checknumber CARG1, ->fff_fallback
1082 | b ->fff_restv
1083 |
1084 |.ffunc_1 tostring
1085 | // Only handles the string or number case inline.
1086 | asr ITYPE, CARG1, #47
1087 | cmn ITYPE, #-LJ_TSTR
1088 | // A __tostring method in the string base metatable is ignored.
1089 | beq ->fff_restv
1090 | // Handle numbers inline, unless a number base metatable is present.
1091 | ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1092 | str BASE, L->base
1093 | cmn ITYPE, #-LJ_TISNUM
1094 | ccmp TMP1, #0, #0, ls
1095 | str PC, SAVE_PC // Redundant (but a defined value).
1096 | bne ->fff_fallback
1097 | ffgccheck
1098 | mov CARG1, L
1099 | mov CARG2, BASE
1100 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1101 | // Returns GCstr *.
1102 | movn TMP1, #~LJ_TSTR
1103 | ldr BASE, L->base
1104 | add CARG1, CARG1, TMP1, lsl #47
1105 | b ->fff_restv
1106 |
1107 |//-- Base library: iterators -------------------------------------------
1108 |
1109 |.ffunc_1 next
1110 | checktp CARG1, LJ_TTAB, ->fff_fallback
1111 | str TISNIL, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1112 | ldr PC, [BASE, FRAME_PC]
1113 | add CARG2, BASE, #8
1114 | sub CARG3, BASE, #16
1115 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1116 | // Returns 1=found, 0=end, -1=error.
1117 | mov RC, #(2+1)*8
1118 | tbnz CRET1w, #31, ->fff_fallback // Invalid key.
1119 | cbnz CRET1, ->fff_res // Found key/value.
1120 | // End of traversal: return nil.
1121 | str TISNIL, [BASE, #-16]
1122 | b ->fff_res1
1123 |
1124 |.ffunc_1 pairs
1125 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1126#if LJ_52
1127 | ldr TAB:CARG2, TAB:TMP1->metatable
1128#endif
1129 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1130 | ldr PC, [BASE, FRAME_PC]
1131#if LJ_52
1132 | cbnz TAB:CARG2, ->fff_fallback
1133#endif
1134 | mov RC, #(3+1)*8
1135 | stp CARG1, TISNIL, [BASE, #-8]
1136 | str CFUNC:CARG4, [BASE, #-16]
1137 | b ->fff_res
1138 |
1139 |.ffunc_2 ipairs_aux
1140 | checktab CARG1, ->fff_fallback
1141 | checkint CARG2, ->fff_fallback
1142 | ldr TMP1w, TAB:CARG1->asize
1143 | ldr CARG3, TAB:CARG1->array
1144 | ldr TMP0w, TAB:CARG1->hmask
1145 | add CARG2w, CARG2w, #1
1146 | cmp CARG2w, TMP1w
1147 | ldr PC, [BASE, FRAME_PC]
1148 | add TMP2, CARG2, TISNUM
1149 | mov RC, #(0+1)*8
1150 | str TMP2, [BASE, #-16]
1151 | bhs >2 // Not in array part?
1152 | ldr TMP0, [CARG3, CARG2, lsl #3]
1153 |1:
1154 | mov TMP1, #(2+1)*8
1155 | cmp TMP0, TISNIL
1156 | str TMP0, [BASE, #-8]
1157 | csel RC, RC, TMP1, eq
1158 | b ->fff_res
1159 |2: // Check for empty hash part first. Otherwise call C function.
1160 | cbz TMP0w, ->fff_res
1161 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1162 | // Returns cTValue * or NULL.
1163 | cbz CRET1, ->fff_res
1164 | ldr TMP0, [CRET1]
1165 | b <1
1166 |
1167 |.ffunc_1 ipairs
1168 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1169#if LJ_52
1170 | ldr TAB:CARG2, TAB:TMP1->metatable
1171#endif
1172 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1173 | ldr PC, [BASE, FRAME_PC]
1174#if LJ_52
1175 | cbnz TAB:CARG2, ->fff_fallback
1176#endif
1177 | mov RC, #(3+1)*8
1178 | stp CARG1, TISNUM, [BASE, #-8]
1179 | str CFUNC:CARG4, [BASE, #-16]
1180 | b ->fff_res
1181 |
1182 |//-- Base library: catch errors ----------------------------------------
1183 |
1184 |.ffunc pcall
1185 | cmp NARGS8:RC, #8
1186 | ldrb TMP0w, GL->hookmask
1187 | blo ->fff_fallback
1188 | sub NARGS8:RC, NARGS8:RC, #8
1189 | mov RB, BASE
1190 | add BASE, BASE, #16
1191 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1192 | add PC, TMP0, #16+FRAME_PCALL
1193 | beq ->vm_call_dispatch
1194 |1:
1195 | add TMP2, BASE, NARGS8:RC
1196 |2:
1197 | ldr TMP0, [TMP2, #-16]
1198 | str TMP0, [TMP2, #-8]!
1199 | cmp TMP2, BASE
1200 | bne <2
1201 | b ->vm_call_dispatch
1202 |
1203 |.ffunc xpcall
1204 | ldp CARG1, CARG2, [BASE]
1205 | ldrb TMP0w, GL->hookmask
1206 | subs NARGS8:TMP1, NARGS8:RC, #16
1207 | blo ->fff_fallback
1208 | mov RB, BASE
1209 | asr ITYPE, CARG2, #47
1210 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1211 | cmn ITYPE, #-LJ_TFUNC
1212 | add PC, TMP0, #24+FRAME_PCALL
1213 | bne ->fff_fallback // Traceback must be a function.
1214 | mov NARGS8:RC, NARGS8:TMP1
1215 | add BASE, BASE, #24
1216 | stp CARG2, CARG1, [RB] // Swap function and traceback.
1217 | cbz NARGS8:RC, ->vm_call_dispatch
1218 | b <1
1219 |
1220 |//-- Coroutine library --------------------------------------------------
1221 |
1222 |.macro coroutine_resume_wrap, resume
1223 |.if resume
1224 |.ffunc_1 coroutine_resume
1225 | checktp CARG1, LJ_TTHREAD, ->fff_fallback
1226 |.else
1227 |.ffunc coroutine_wrap_aux
1228 | ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1229 | and L:CARG1, CARG1, #LJ_GCVMASK
1230 |.endif
1231 | ldr PC, [BASE, FRAME_PC]
1232 | str BASE, L->base
1233 | ldp RB, CARG2, L:CARG1->base
1234 | ldrb TMP1w, L:CARG1->status
1235 | add TMP0, CARG2, TMP1
1236 | str PC, SAVE_PC
1237 | cmp TMP0, RB
1238 | beq ->fff_fallback
1239 | cmp TMP1, #LUA_YIELD
1240 | add TMP0, CARG2, #8
1241 | csel CARG2, CARG2, TMP0, hs
1242 | ldr CARG4, L:CARG1->maxstack
1243 | add CARG3, CARG2, NARGS8:RC
1244 | ldr RB, L:CARG1->cframe
1245 | ccmp CARG3, CARG4, #2, ls
1246 | ccmp RB, #0, #2, ls
1247 | bhi ->fff_fallback
1248 |.if resume
1249 | sub CARG3, CARG3, #8 // Keep resumed thread in stack for GC.
1250 | add BASE, BASE, #8
1251 | sub NARGS8:RC, NARGS8:RC, #8
1252 |.endif
1253 | str CARG3, L:CARG1->top
1254 | str BASE, L->top
1255 | cbz NARGS8:RC, >3
1256 |2: // Move args to coroutine.
1257 | ldr TMP0, [BASE, RB]
1258 | cmp RB, NARGS8:RC
1259 | str TMP0, [CARG2, RB]
1260 | add RB, RB, #8
1261 | bne <2
1262 |3:
1263 | mov CARG3, #0
1264 | mov L:RA, L:CARG1
1265 | mov CARG4, #0
1266 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1267 | // Returns thread status.
1268 |4:
1269 | ldp CARG3, CARG4, L:RA->base
1270 | cmp CRET1, #LUA_YIELD
1271 | ldr BASE, L->base
1272 | str L, GL->cur_L
1273 | st_vmstate ST_INTERP
1274 | bhi >8
1275 | sub RC, CARG4, CARG3
1276 | ldr CARG1, L->maxstack
1277 | add CARG2, BASE, RC
1278 | cbz RC, >6 // No results?
1279 | cmp CARG2, CARG1
1280 | mov RB, #0
1281 | bhi >9 // Need to grow stack?
1282 |
1283 | sub CARG4, RC, #8
1284 | str CARG3, L:RA->top // Clear coroutine stack.
1285 |5: // Move results from coroutine.
1286 | ldr TMP0, [CARG3, RB]
1287 | cmp RB, CARG4
1288 | str TMP0, [BASE, RB]
1289 | add RB, RB, #8
1290 | bne <5
1291 |6:
1292 |.if resume
1293 | mov_true TMP1
1294 | add RC, RC, #16
1295 |7:
1296 | str TMP1, [BASE, #-8] // Prepend true/false to results.
1297 | sub RA, BASE, #8
1298 |.else
1299 | mov RA, BASE
1300 | add RC, RC, #8
1301 |.endif
1302 | ands CARG1, PC, #FRAME_TYPE
1303 | str PC, SAVE_PC
1304 | str RCw, SAVE_MULTRES
1305 | beq ->BC_RET_Z
1306 | b ->vm_return
1307 |
1308 |8: // Coroutine returned with error (at co->top-1).
1309 |.if resume
1310 | ldr TMP0, [CARG4, #-8]!
1311 | mov_false TMP1
1312 | mov RC, #(2+1)*8
1313 | str CARG4, L:RA->top // Remove error from coroutine stack.
1314 | str TMP0, [BASE] // Copy error message.
1315 | b <7
1316 |.else
1317 | mov CARG1, L
1318 | mov CARG2, L:RA
1319 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1320 | // Never returns.
1321 |.endif
1322 |
1323 |9: // Handle stack expansion on return from yield.
1324 | mov CARG1, L
1325 | lsr CARG2, RC, #3
1326 | bl extern lj_state_growstack // (lua_State *L, int n)
1327 | mov CRET1, #0
1328 | b <4
1329 |.endmacro
1330 |
1331 | coroutine_resume_wrap 1 // coroutine.resume
1332 | coroutine_resume_wrap 0 // coroutine.wrap
1333 |
1334 |.ffunc coroutine_yield
1335 | ldr TMP0, L->cframe
1336 | add TMP1, BASE, NARGS8:RC
1337 | mov CRET1, #LUA_YIELD
1338 | stp BASE, TMP1, L->base
1339 | tbz TMP0, #0, ->fff_fallback
1340 | str xzr, L->cframe
1341 | strb CRET1w, L->status
1342 | b ->vm_leave_unw
1343 |
1344 |//-- Math library -------------------------------------------------------
1345 |
1346 |.macro math_round, func, round
1347 | .ffunc math_ .. func
1348 | ldr CARG1, [BASE]
1349 | cmp NARGS8:RC, #8
1350 | ldr d0, [BASE]
1351 | blo ->fff_fallback
1352 | cmp TISNUMhi, CARG1, lsr #32
1353 | beq ->fff_restv
1354 | blo ->fff_fallback
1355 | round d0, d0
1356 | b ->fff_resn
1357 |.endmacro
1358 |
1359 | math_round floor, frintm
1360 | math_round ceil, frintp
1361 |
1362 |.ffunc_1 math_abs
1363 | checknumber CARG1, ->fff_fallback
1364 | and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1365 | bne ->fff_restv
1366 | eor CARG2w, CARG1w, CARG1w, asr #31
1367 | movz CARG3, #0x41e0, lsl #48 // 2^31.
1368 | subs CARG1w, CARG2w, CARG1w, asr #31
1369 | add CARG1, CARG1, TISNUM
1370 | csel CARG1, CARG1, CARG3, pl
1371 | // Fallthrough.
1372 |
1373 |->fff_restv:
1374 | // CARG1 = TValue result.
1375 | ldr PC, [BASE, FRAME_PC]
1376 | str CARG1, [BASE, #-16]
1377 |->fff_res1:
1378 | // PC = return.
1379 | mov RC, #(1+1)*8
1380 |->fff_res:
1381 | // RC = (nresults+1)*8, PC = return.
1382 | ands CARG1, PC, #FRAME_TYPE
1383 | str RCw, SAVE_MULTRES
1384 | sub RA, BASE, #16
1385 | bne ->vm_return
1386 | ldr INSw, [PC, #-4]
1387 | decode_RB RB, INS
1388 |5:
1389 | cmp RC, RB, lsl #3 // More results expected?
1390 | blo >6
1391 | decode_RA TMP1, INS
1392 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1393 | sub BASE, RA, TMP1, lsl #3
1394 | ins_next
1395 |
1396 |6: // Fill up results with nil.
1397 | add TMP1, RA, RC
1398 | add RC, RC, #8
1399 | str TISNIL, [TMP1, #-8]
1400 | b <5
1401 |
1402 |.macro math_extern, func
1403 | .ffunc_n math_ .. func
1404 | bl extern func
1405 | b ->fff_resn
1406 |.endmacro
1407 |
1408 |.macro math_extern2, func
1409 | .ffunc_nn math_ .. func
1410 | bl extern func
1411 | b ->fff_resn
1412 |.endmacro
1413 |
1414 |.ffunc_n math_sqrt
1415 | fsqrt d0, d0
1416 |->fff_resn:
1417 | ldr PC, [BASE, FRAME_PC]
1418 | str d0, [BASE, #-16]
1419 | b ->fff_res1
1420 |
1421 |.ffunc math_log
1422 | ldr CARG1, [BASE]
1423 | cmp NARGS8:RC, #8
1424 | ldr FARG1, [BASE]
1425 | bne ->fff_fallback // Need exactly 1 argument.
1426 | checknum CARG1, ->fff_fallback
1427 | bl extern log
1428 | b ->fff_resn
1429 |
1430 | math_extern log10
1431 | math_extern exp
1432 | math_extern sin
1433 | math_extern cos
1434 | math_extern tan
1435 | math_extern asin
1436 | math_extern acos
1437 | math_extern atan
1438 | math_extern sinh
1439 | math_extern cosh
1440 | math_extern tanh
1441 | math_extern2 pow
1442 | math_extern2 atan2
1443 | math_extern2 fmod
1444 |
1445 |.ffunc_2 math_ldexp
1446 | ldr FARG1, [BASE]
1447 | checknum CARG1, ->fff_fallback
1448 | checkint CARG2, ->fff_fallback
1449 | sxtw CARG1, CARG2w
1450 | bl extern ldexp // (double x, int exp)
1451 | b ->fff_resn
1452 |
1453 |.ffunc_n math_frexp
1454 | add CARG1, sp, TMPDofs
1455 | bl extern frexp
1456 | ldr CARG2w, TMPD
1457 | ldr PC, [BASE, FRAME_PC]
1458 | str d0, [BASE, #-16]
1459 | mov RC, #(2+1)*8
1460 | add CARG2, CARG2, TISNUM
1461 | str CARG2, [BASE, #-8]
1462 | b ->fff_res
1463 |
1464 |.ffunc_n math_modf
1465 | sub CARG1, BASE, #16
1466 | ldr PC, [BASE, FRAME_PC]
1467 | bl extern modf
1468 | mov RC, #(2+1)*8
1469 | str d0, [BASE, #-8]
1470 | b ->fff_res
1471 |
1472 |.macro math_minmax, name, cond, fcond
1473 | .ffunc_1 name
1474 | add RB, BASE, RC
1475 | add RA, BASE, #8
1476 | checkint CARG1, >4
1477 |1: // Handle integers.
1478 | ldr CARG2, [RA]
1479 | cmp RA, RB
1480 | bhs ->fff_restv
1481 | checkint CARG2, >3
1482 | cmp CARG1w, CARG2w
1483 | add RA, RA, #8
1484 | csel CARG1, CARG2, CARG1, cond
1485 | b <1
1486 |3: // Convert intermediate result to number and continue below.
1487 | scvtf d0, CARG1w
1488 | blo ->fff_fallback
1489 | ldr d1, [RA]
1490 | b >6
1491 |
1492 |4:
1493 | ldr d0, [BASE]
1494 | blo ->fff_fallback
1495 |5: // Handle numbers.
1496 | ldr CARG2, [RA]
1497 | ldr d1, [RA]
1498 | cmp RA, RB
1499 | bhs ->fff_resn
1500 | checknum CARG2, >7
1501 |6:
1502 | fcmp d0, d1
1503 | add RA, RA, #8
1504 | fcsel d0, d1, d0, fcond
1505 | b <5
1506 |7: // Convert integer to number and continue above.
1507 | scvtf d1, CARG2w
1508 | blo ->fff_fallback
1509 | b <6
1510 |.endmacro
1511 |
1512 | math_minmax math_min, gt, pl
1513 | math_minmax math_max, lt, le
1514 |
1515 |//-- String library -----------------------------------------------------
1516 |
1517 |.ffunc string_byte // Only handle the 1-arg case here.
1518 | ldp PC, CARG1, [BASE, FRAME_PC]
1519 | cmp NARGS8:RC, #8
1520 | asr ITYPE, CARG1, #47
1521 | ccmn ITYPE, #-LJ_TSTR, #0, eq
1522 | and STR:CARG1, CARG1, #LJ_GCVMASK
1523 | bne ->fff_fallback
1524 | ldrb TMP0w, STR:CARG1[1] // Access is always ok (NUL at end).
1525 | ldr CARG3w, STR:CARG1->len
1526 | add TMP0, TMP0, TISNUM
1527 | str TMP0, [BASE, #-16]
1528 | mov RC, #(0+1)*8
1529 | cbz CARG3, ->fff_res
1530 | b ->fff_res1
1531 |
1532 |.ffunc string_char // Only handle the 1-arg case here.
1533 | ffgccheck
1534 | ldp PC, CARG1, [BASE, FRAME_PC]
1535 | cmp CARG1w, #255
1536 | ccmp NARGS8:RC, #8, #0, ls // Need exactly 1 argument.
1537 | bne ->fff_fallback
1538 | checkint CARG1, ->fff_fallback
1539 | mov CARG3, #1
1540 | // Point to the char inside the integer in the stack slot.
1541 |.if ENDIAN_LE
1542 | mov CARG2, BASE
1543 |.else
1544 | add CARG2, BASE, #7
1545 |.endif
1546 |->fff_newstr:
1547 | // CARG2 = str, CARG3 = len.
1548 | str BASE, L->base
1549 | mov CARG1, L
1550 | str PC, SAVE_PC
1551 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1552 |->fff_resstr:
1553 | // Returns GCstr *.
1554 | ldr BASE, L->base
1555 | movn TMP1, #~LJ_TSTR
1556 | add CARG1, CARG1, TMP1, lsl #47
1557 | b ->fff_restv
1558 |
1559 |.ffunc string_sub
1560 | ffgccheck
1561 | ldr CARG1, [BASE]
1562 | ldr CARG3, [BASE, #16]
1563 | cmp NARGS8:RC, #16
1564 | movn RB, #0
1565 | beq >1
1566 | blo ->fff_fallback
1567 | checkint CARG3, ->fff_fallback
1568 | sxtw RB, CARG3w
1569 |1:
1570 | ldr CARG2, [BASE, #8]
1571 | checkstr CARG1, ->fff_fallback
1572 | ldr TMP1w, STR:CARG1->len
1573 | checkint CARG2, ->fff_fallback
1574 | sxtw CARG2, CARG2w
1575 | // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1576 | add TMP2, RB, TMP1
1577 | cmp RB, #0
1578 | add TMP0, CARG2, TMP1
1579 | csinc RB, RB, TMP2, ge // if (end < 0) end += len+1
1580 | cmp CARG2, #0
1581 | csinc CARG2, CARG2, TMP0, ge // if (start < 0) start += len+1
1582 | cmp RB, #0
1583 | csel RB, RB, xzr, ge // if (end < 0) end = 0
1584 | cmp CARG2, #1
1585 | csinc CARG2, CARG2, xzr, ge // if (start < 1) start = 1
1586 | cmp RB, TMP1
1587 | csel RB, RB, TMP1, le // if (end > len) end = len
1588 | add CARG1, STR:CARG1, #sizeof(GCstr)-1
1589 | subs CARG3, RB, CARG2 // len = end - start
1590 | add CARG2, CARG1, CARG2
1591 | add CARG3, CARG3, #1 // len += 1
1592 | bge ->fff_newstr
1593 | add STR:CARG1, GL, #offsetof(global_State, strempty)
1594 | movn TMP1, #~LJ_TSTR
1595 | add CARG1, CARG1, TMP1, lsl #47
1596 | b ->fff_restv
1597 |
1598 |.macro ffstring_op, name
1599 | .ffunc string_ .. name
1600 | ffgccheck
1601 | ldr CARG2, [BASE]
1602 | cmp NARGS8:RC, #8
1603 | asr ITYPE, CARG2, #47
1604 | ccmn ITYPE, #-LJ_TSTR, #0, hs
1605 | and STR:CARG2, CARG2, #LJ_GCVMASK
1606 | bne ->fff_fallback
1607 | ldr TMP0, GL->tmpbuf.b
1608 | add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1609 | str BASE, L->base
1610 | str PC, SAVE_PC
1611 | str L, GL->tmpbuf.L
1612 | str TMP0, GL->tmpbuf.w
1613 | bl extern lj_buf_putstr_ .. name
1614 | bl extern lj_buf_tostr
1615 | b ->fff_resstr
1616 |.endmacro
1617 |
1618 |ffstring_op reverse
1619 |ffstring_op lower
1620 |ffstring_op upper
1621 |
1622 |//-- Bit library --------------------------------------------------------
1623 |
1624 |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
1625 |->vm_tobit_fb:
1626 | bls ->fff_fallback
1627 | add CARG2, CARG1, CARG1
1628 | mov CARG3, #1076
1629 | sub CARG3, CARG3, CARG2, lsr #53
1630 | cmp CARG3, #53
1631 | bhi >1
1632 | and CARG2, CARG2, #U64x(001fffff,ffffffff)
1633 | orr CARG2, CARG2, #U64x(00200000,00000000)
1634 | cmp CARG1, #0
1635 | lsr CARG2, CARG2, CARG3
1636 | cneg CARG1w, CARG2w, mi
1637 | br lr
1638 |1:
1639 | mov CARG1w, #0
1640 | br lr
1641 |
1642 |.macro .ffunc_bit, name
1643 | .ffunc_1 bit_..name
1644 | adr lr, >1
1645 | checkint CARG1, ->vm_tobit_fb
1646 |1:
1647 |.endmacro
1648 |
1649 |.macro .ffunc_bit_op, name, ins
1650 | .ffunc_bit name
1651 | mov RA, #8
1652 | mov TMP0w, CARG1w
1653 | adr lr, >2
1654 |1:
1655 | ldr CARG1, [BASE, RA]
1656 | cmp RA, NARGS8:RC
1657 | add RA, RA, #8
1658 | bge >9
1659 | checkint CARG1, ->vm_tobit_fb
1660 |2:
1661 | ins TMP0w, TMP0w, CARG1w
1662 | b <1
1663 |.endmacro
1664 |
1665 |.ffunc_bit_op band, and
1666 |.ffunc_bit_op bor, orr
1667 |.ffunc_bit_op bxor, eor
1668 |
1669 |.ffunc_bit tobit
1670 | mov TMP0w, CARG1w
1671 |9: // Label reused by .ffunc_bit_op users.
1672 | add CARG1, TMP0, TISNUM
1673 | b ->fff_restv
1674 |
1675 |.ffunc_bit bswap
1676 | rev TMP0w, CARG1w
1677 | add CARG1, TMP0, TISNUM
1678 | b ->fff_restv
1679 |
1680 |.ffunc_bit bnot
1681 | mvn TMP0w, CARG1w
1682 | add CARG1, TMP0, TISNUM
1683 | b ->fff_restv
1684 |
1685 |.macro .ffunc_bit_sh, name, ins, shmod
1686 | .ffunc bit_..name
1687 | ldp TMP0, CARG1, [BASE]
1688 | cmp NARGS8:RC, #16
1689 | blo ->fff_fallback
1690 | adr lr, >1
1691 | checkint CARG1, ->vm_tobit_fb
1692 |1:
1693 |.if shmod == 0
1694 | mov TMP1, CARG1
1695 |.else
1696 | neg TMP1, CARG1
1697 |.endif
1698 | mov CARG1, TMP0
1699 | adr lr, >2
1700 | checkint CARG1, ->vm_tobit_fb
1701 |2:
1702 | ins TMP0w, CARG1w, TMP1w
1703 | add CARG1, TMP0, TISNUM
1704 | b ->fff_restv
1705 |.endmacro
1706 |
1707 |.ffunc_bit_sh lshift, lsl, 0
1708 |.ffunc_bit_sh rshift, lsr, 0
1709 |.ffunc_bit_sh arshift, asr, 0
1710 |.ffunc_bit_sh rol, ror, 1
1711 |.ffunc_bit_sh ror, ror, 0
1712 |
1713 |//-----------------------------------------------------------------------
1714 |
1715 |->fff_fallback: // Call fast function fallback handler.
1716 | // BASE = new base, RC = nargs*8
1717 | ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC] // Fallback may overwrite PC.
1718 | ldr TMP2, L->maxstack
1719 | add TMP1, BASE, NARGS8:RC
1720 | stp BASE, TMP1, L->base
1721 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1722 | add TMP1, TMP1, #8*LUA_MINSTACK
1723 | ldr CARG3, CFUNC:CARG3->f
1724 | str PC, SAVE_PC // Redundant (but a defined value).
1725 | cmp TMP1, TMP2
1726 | mov CARG1, L
1727 | bhi >5 // Need to grow stack.
1728 | blr_auth CARG3 // (lua_State *L)
1729 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1730 | ldr BASE, L->base
1731 | cmp CRET1w, #0
1732 | lsl RC, CRET1, #3
1733 | sub RA, BASE, #16
1734 | bgt ->fff_res // Returned nresults+1?
1735 |1: // Returned 0 or -1: retry fast path.
1736 | ldr CARG1, L->top
1737 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1738 | sub NARGS8:RC, CARG1, BASE
1739 | bne ->vm_call_tail // Returned -1?
1740 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1741 | ins_callt // Returned 0: retry fast path.
1742 |
1743 |// Reconstruct previous base for vmeta_call during tailcall.
1744 |->vm_call_tail:
1745 | ands TMP0, PC, #FRAME_TYPE
1746 | and TMP1, PC, #~FRAME_TYPEP
1747 | bne >3
1748 | ldrb RAw, [PC, #-4+OFS_RA]
1749 | lsl RA, RA, #3
1750 | add TMP1, RA, #16
1751 |3:
1752 | sub RB, BASE, TMP1
1753 | b ->vm_call_dispatch // Resolve again for tailcall.
1754 |
1755 |5: // Grow stack for fallback handler.
1756 | mov CARG2, #LUA_MINSTACK
1757 | bl extern lj_state_growstack // (lua_State *L, int n)
1758 | ldr BASE, L->base
1759 | cmp CARG1, CARG1 // Set zero-flag to force retry.
1760 | b <1
1761 |
1762 |->fff_gcstep: // Call GC step function.
1763 | // BASE = new base, RC = nargs*8
1764 | sp_auth
1765 | add CARG2, BASE, NARGS8:RC // Calculate L->top.
1766 | mov RA, lr
1767 | stp BASE, CARG2, L->base
1768 | str PC, SAVE_PC // Redundant (but a defined value).
1769 | mov CARG1, L
1770 | bl extern lj_gc_step // (lua_State *L)
1771 | ldp BASE, CARG2, L->base
1772 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1773 | mov lr, RA // Help return address predictor.
1774 | sub NARGS8:RC, CARG2, BASE // Calculate nargs*8.
1775 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1776 | ret_auth
1777 |
1778 |//-----------------------------------------------------------------------
1779 |//-- Special dispatch targets -------------------------------------------
1780 |//-----------------------------------------------------------------------
1781 |
1782 |->vm_record: // Dispatch target for recording phase.
1783 |.if JIT
1784 | ldrb CARG1w, GL->hookmask
1785 | tst CARG1, #HOOK_VMEVENT // No recording while in vmevent.
1786 | bne >5
1787 | // Decrement the hookcount for consistency, but always do the call.
1788 | ldr CARG2w, GL->hookcount
1789 | tst CARG1, #HOOK_ACTIVE
1790 | bne >1
1791 | sub CARG2w, CARG2w, #1
1792 | tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
1793 | beq >1
1794 | str CARG2w, GL->hookcount
1795 | b >1
1796 |.endif
1797 |
1798 |->vm_rethook: // Dispatch target for return hooks.
1799 | ldrb TMP2w, GL->hookmask
1800 | tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1 // Hook already active?
1801 |5: // Re-dispatch to static ins.
1802 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1803 | br_auth TMP0
1804 |
1805 |->vm_inshook: // Dispatch target for instr/line hooks.
1806 | ldrb TMP2w, GL->hookmask
1807 | ldr TMP3w, GL->hookcount
1808 | tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5 // Hook already active?
1809 | tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1810 | beq <5
1811 | sub TMP3w, TMP3w, #1
1812 | str TMP3w, GL->hookcount
1813 | cbz TMP3w, >1
1814 | tbz TMP2w, #LUA_HOOKLINE, <5
1815 |1:
1816 | mov CARG1, L
1817 | str BASE, L->base
1818 | mov CARG2, PC
1819 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1820 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1821 |3:
1822 | ldr BASE, L->base
1823 |4: // Re-dispatch to static ins.
1824 | ldr INSw, [PC, #-4]
1825 | add TMP1, GL, INS, uxtb #3
1826 | decode_RA RA, INS
1827 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1828 | decode_RD RC, INS
1829 | br_auth TMP0
1830 |
1831 |->cont_hook: // Continue from hook yield.
1832 | ldr CARG1, [CARG4, #-40]
1833 | add PC, PC, #4
1834 | str CARG1w, SAVE_MULTRES // Restore MULTRES for *M ins.
1835 | b <4
1836 |
1837 |->vm_hotloop: // Hot loop counter underflow.
1838 |.if JIT
1839 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Same as curr_topL(L).
1840 | add CARG1, GL, #GG_G2DISP+GG_DISP2J
1841 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1842 | str PC, SAVE_PC
1843 | ldr CARG3, LFUNC:CARG3->pc
1844 | mov CARG2, PC
1845 | str L, [GL, #GL_J(L)]
1846 | ldrb CARG3w, [CARG3, #PC2PROTO(framesize)]
1847 | str BASE, L->base
1848 | add CARG3, BASE, CARG3, lsl #3
1849 | str CARG3, L->top
1850 | bl extern lj_trace_hot // (jit_State *J, const BCIns *pc)
1851 | b <3
1852 |.endif
1853 |
1854 |->vm_callhook: // Dispatch target for call hooks.
1855 | mov CARG2, PC
1856 |.if JIT
1857 | b >1
1858 |.endif
1859 |
1860 |->vm_hotcall: // Hot call counter underflow.
1861 |.if JIT
1862 | orr CARG2, PC, #1
1863 |1:
1864 |.endif
1865 | add TMP1, BASE, NARGS8:RC
1866 | str PC, SAVE_PC
1867 | mov CARG1, L
1868 | sub RA, RA, BASE
1869 | stp BASE, TMP1, L->base
1870 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1871 | // Returns ASMFunction.
1872 | ldp BASE, TMP1, L->base
1873 | str xzr, SAVE_PC // Invalidate for subsequent line hook.
1874 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1875 | add RA, BASE, RA
1876 | sub NARGS8:RC, TMP1, BASE
1877 | ldr INSw, [PC, #-4]
1878 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1879 | br_auth CRET1
1880 |
1881 |->cont_stitch: // Trace stitching.
1882 |.if JIT
1883 | // RA = resultptr, CARG4 = meta base
1884 | ldr RBw, SAVE_MULTRES
1885 | ldr INSw, [PC, #-4]
1886 | ldr TRACE:CARG3, [CARG4, #-40] // Save previous trace.
1887 | subs RB, RB, #8
1888 | decode_RA RC, INS // Call base.
1889 | and CARG3, CARG3, #LJ_GCVMASK
1890 | beq >2
1891 |1: // Move results down.
1892 | ldr CARG1, [RA]
1893 | add RA, RA, #8
1894 | subs RB, RB, #8
1895 | str CARG1, [BASE, RC, lsl #3]
1896 | add RC, RC, #1
1897 | bne <1
1898 |2:
1899 | decode_RA RA, INS
1900 | decode_RB RB, INS
1901 | add RA, RA, RB
1902 |3:
1903 | cmp RA, RC
1904 | bhi >9 // More results wanted?
1905 |
1906 | ldrh RAw, TRACE:CARG3->traceno
1907 | ldrh RCw, TRACE:CARG3->link
1908 | cmp RCw, RAw
1909 | beq ->cont_nop // Blacklisted.
1910 | cmp RCw, #0
1911 | bne =>BC_JLOOP // Jump to stitched trace.
1912 |
1913 | // Stitch a new trace to the previous trace.
1914 | mov CARG1, #GL_J(exitno)
1915 | str RAw, [GL, CARG1]
1916 | mov CARG1, #GL_J(L)
1917 | str L, [GL, CARG1]
1918 | str BASE, L->base
1919 | add CARG1, GL, #GG_G2J
1920 | mov CARG2, PC
1921 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
1922 | ldr BASE, L->base
1923 | b ->cont_nop
1924 |
1925 |9: // Fill up results with nil.
1926 | str TISNIL, [BASE, RC, lsl #3]
1927 | add RC, RC, #1
1928 | b <3
1929 |.endif
1930 |
1931 |->vm_profhook: // Dispatch target for profiler hook.
1932#if LJ_HASPROFILE
1933 | mov CARG1, L
1934 | str BASE, L->base
1935 | mov CARG2, PC
1936 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
1937 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
1938 | ldr BASE, L->base
1939 | sub PC, PC, #4
1940 | b ->cont_nop
1941#endif
1942 |
1943 |//-----------------------------------------------------------------------
1944 |//-- Trace exit handler -------------------------------------------------
1945 |//-----------------------------------------------------------------------
1946 |
1947 |.macro savex_, a, b
1948 | stp d..a, d..b, [sp, #a*8]
1949 | stp x..a, x..b, [sp, #32*8+a*8]
1950 |.endmacro
1951 |
1952 |->vm_exit_handler:
1953 |.if JIT
1954 | sub sp, sp, #(64*8)
1955 | savex_, 0, 1
1956 | savex_, 2, 3
1957 | savex_, 4, 5
1958 | savex_, 6, 7
1959 | savex_, 8, 9
1960 | savex_, 10, 11
1961 | savex_, 12, 13
1962 | savex_, 14, 15
1963 | savex_, 16, 17
1964 | savex_, 18, 19
1965 | savex_, 20, 21
1966 | savex_, 22, 23
1967 | savex_, 24, 25
1968 | savex_, 26, 27
1969 | savex_, 28, 29
1970 | stp d30, d31, [sp, #30*8]
1971 | ldr CARG1, [sp, #64*8] // Load original value of lr.
1972 | add CARG3, sp, #64*8 // Recompute original value of sp.
1973 | mv_vmstate CARG4w, EXIT
1974 | stp xzr, CARG3, [sp, #62*8] // Store 0/sp in RID_LR/RID_SP.
1975 | sub CARG1, CARG1, lr
1976 | ldr L, GL->cur_L
1977 | lsr CARG1, CARG1, #2
1978 | ldr BASE, GL->jit_base
1979 | sub CARG1, CARG1, #2
1980 | ldr CARG2w, [lr] // Load trace number.
1981 | st_vmstate CARG4w
1982 |.if ENDIAN_BE
1983 | rev32 CARG2, CARG2
1984 |.endif
1985 | str BASE, L->base
1986 | ubfx CARG2w, CARG2w, #5, #16
1987 | str CARG1w, [GL, #GL_J(exitno)]
1988 | str CARG2w, [GL, #GL_J(parent)]
1989 | str L, [GL, #GL_J(L)]
1990 | str xzr, GL->jit_base
1991 | add CARG1, GL, #GG_G2J
1992 | mov CARG2, sp
1993 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex)
1994 | // Returns MULTRES (unscaled) or negated error code.
1995 | ldr CARG2, L->cframe
1996 | ldr BASE, L->base
1997 | and sp, CARG2, #CFRAME_RAWMASK
1998 | ldr PC, SAVE_PC // Get SAVE_PC.
1999 | str L, SAVE_L // Set SAVE_L (on-trace resume/yield).
2000 | b >1
2001 |.endif
2002 |
2003 |->vm_exit_interp:
2004 | // CARG1 = MULTRES or negated error code, BASE, PC and GL set.
2005 |.if JIT
2006 | ldr L, SAVE_L
2007 |1:
2008 | cmn CARG1w, #LUA_ERRERR
2009 | bhs >9 // Check for error from exit.
2010 | lsl RC, CARG1, #3
2011 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2012 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
2013 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
2014 | movn TISNIL, #0
2015 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2016 | str RCw, SAVE_MULTRES
2017 | str BASE, L->base
2018 | ldr CARG2, LFUNC:CARG2->pc
2019 | str xzr, GL->jit_base
2020 | mv_vmstate CARG4w, INTERP
2021 | ldr KBASE, [CARG2, #PC2PROTO(k)]
2022 | // Modified copy of ins_next which handles function header dispatch, too.
2023 | ldrb RBw, [PC, # OFS_OP]
2024 | ldr INSw, [PC], #4
2025 | st_vmstate CARG4w
2026 | cmn CARG1w, #17 // Static dispatch?
2027 | beq >5
2028 | cmp RBw, #BC_FUNCC+2 // Fast function?
2029 | add TMP1, GL, INS, uxtb #3
2030 | bhs >4
2031 |2:
2032 | cmp RBw, #BC_FUNCF // Function header?
2033 | add TMP0, GL, RB, uxtb #3
2034 | ldr RB, [TMP0, #GG_G2DISP]
2035 | decode_RA RA, INS
2036 | lsr TMP0, INS, #16
2037 | csel RC, TMP0, RC, lo
2038 | blo >3
2039 | ldr CARG3, [BASE, FRAME_FUNC]
2040 | sub RC, RC, #8
2041 | add RA, BASE, RA, lsl #3 // Yes: RA = BASE+framesize*8, RC = nargs*8
2042 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2043 |3:
2044 | br_auth RB
2045 |
2046 |4: // Check frame below fast function.
2047 | ldr CARG1, [BASE, FRAME_PC]
2048 | ands CARG2, CARG1, #FRAME_TYPE
2049 | bne <2 // Trace stitching continuation?
2050 | // Otherwise set KBASE for Lua function below fast function.
2051 | ldr CARG3w, [CARG1, #-4]
2052 | decode_RA CARG1, CARG3
2053 | sub CARG2, BASE, CARG1, lsl #3
2054 | ldr LFUNC:CARG3, [CARG2, #-32]
2055 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2056 | ldr CARG3, LFUNC:CARG3->pc
2057 | ldr KBASE, [CARG3, #PC2PROTO(k)]
2058 | b <2
2059 |
2060 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2061 | ldr RA, [GL, #GL_J(trace)]
2062 | decode_RD RC, INS
2063 | ldr TRACE:RA, [RA, RC, lsl #3]
2064 | ldr INSw, TRACE:RA->startins
2065 | add TMP0, GL, INS, uxtb #3
2066 | decode_RA RA, INS
2067 | ldr RB, [TMP0, #GG_G2DISP+GG_DISP2STATIC]
2068 | decode_RD RC, INS
2069 | br_auth RB
2070 |
2071 |9: // Rethrow error from the right C frame.
2072 | neg CARG2w, CARG1w
2073 | mov CARG1, L
2074 | bl extern lj_err_trace // (lua_State *L, int errcode)
2075 |.endif
2076 |
2077 |//-----------------------------------------------------------------------
2078 |//-- Math helper functions ----------------------------------------------
2079 |//-----------------------------------------------------------------------
2080 |
2081 | // int lj_vm_modi(int dividend, int divisor);
2082 |->vm_modi:
2083 | eor CARG4w, CARG1w, CARG2w
2084 | cmp CARG4w, #0
2085 | eor CARG3w, CARG1w, CARG1w, asr #31
2086 | eor CARG4w, CARG2w, CARG2w, asr #31
2087 | sub CARG3w, CARG3w, CARG1w, asr #31
2088 | sub CARG4w, CARG4w, CARG2w, asr #31
2089 | udiv CARG1w, CARG3w, CARG4w
2090 | msub CARG1w, CARG1w, CARG4w, CARG3w
2091 | ccmp CARG1w, #0, #4, mi
2092 | sub CARG3w, CARG1w, CARG4w
2093 | csel CARG1w, CARG1w, CARG3w, eq
2094 | eor CARG3w, CARG1w, CARG2w
2095 | cmp CARG3w, #0
2096 | cneg CARG1w, CARG1w, mi
2097 | ret
2098 |
2099 |//-----------------------------------------------------------------------
2100 |//-- Miscellaneous functions --------------------------------------------
2101 |//-----------------------------------------------------------------------
2102 |
2103 |.define NEXT_TAB, TAB:CARG1
2104 |.define NEXT_RES, CARG1
2105 |.define NEXT_IDX, CARG2w
2106 |.define NEXT_LIM, CARG3w
2107 |.define NEXT_TMP0, TMP0
2108 |.define NEXT_TMP0w, TMP0w
2109 |.define NEXT_TMP1, TMP1
2110 |.define NEXT_TMP1w, TMP1w
2111 |.define NEXT_RES_PTR, sp
2112 |.define NEXT_RES_VAL, [sp]
2113 |.define NEXT_RES_KEY, [sp, #8]
2114 |
2115 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2116 |// Next idx returned in CRET2w.
2117 |->vm_next:
2118 |.if JIT
2119 | ldr NEXT_LIM, NEXT_TAB->asize
2120 | ldr NEXT_TMP1, NEXT_TAB->array
2121 |1: // Traverse array part.
2122 | subs NEXT_TMP0w, NEXT_IDX, NEXT_LIM
2123 | bhs >5 // Index points after array part?
2124 | ldr NEXT_TMP0, [NEXT_TMP1, NEXT_IDX, uxtw #3]
2125 | cmn NEXT_TMP0, #-LJ_TNIL
2126 | cinc NEXT_IDX, NEXT_IDX, eq
2127 | beq <1 // Skip holes in array part.
2128 | str NEXT_TMP0, NEXT_RES_VAL
2129 | movz NEXT_TMP0w, #(LJ_TISNUM>>1)&0xffff, lsl #16
2130 | stp NEXT_IDX, NEXT_TMP0w, NEXT_RES_KEY
2131 | add NEXT_IDX, NEXT_IDX, #1
2132 | mov NEXT_RES, NEXT_RES_PTR
2133 |4:
2134 | ret
2135 |
2136 |5: // Traverse hash part.
2137 | ldr NEXT_TMP1w, NEXT_TAB->hmask
2138 | ldr NODE:NEXT_RES, NEXT_TAB->node
2139 | add NEXT_TMP0w, NEXT_TMP0w, NEXT_TMP0w, lsl #1
2140 | add NEXT_LIM, NEXT_LIM, NEXT_TMP1w
2141 | add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP0w, uxtw #3
2142 |6:
2143 | cmp NEXT_IDX, NEXT_LIM
2144 | bhi >9
2145 | ldr NEXT_TMP0, NODE:NEXT_RES->val
2146 | cmn NEXT_TMP0, #-LJ_TNIL
2147 | add NEXT_IDX, NEXT_IDX, #1
2148 | bne <4
2149 | // Skip holes in hash part.
2150 | add NODE:NEXT_RES, NODE:NEXT_RES, #sizeof(Node)
2151 | b <6
2152 |
2153 |9: // End of iteration. Set the key to nil (not the value).
2154 | movn NEXT_TMP0, #0
2155 | str NEXT_TMP0, NEXT_RES_KEY
2156 | mov NEXT_RES, NEXT_RES_PTR
2157 | ret
2158 |.endif
2159 |
2160 |//-----------------------------------------------------------------------
2161 |//-- FFI helper functions -----------------------------------------------
2162 |//-----------------------------------------------------------------------
2163 |
2164 |// Handler for callback functions.
2165 |// Saveregs already performed. Callback slot number in [sp], g in r12.
2166 |->vm_ffi_callback:
2167 |.if FFI
2168 |.type CTSTATE, CTState, PC
2169 | saveregs
2170 | ldr CTSTATE, GL:x10->ctype_state
2171 | mov GL, x10
2172 | add x10, sp, # CFRAME_SPACE
2173 | str w9, CTSTATE->cb.slot
2174 | stp x0, x1, CTSTATE->cb.gpr[0]
2175 | stp d0, d1, CTSTATE->cb.fpr[0]
2176 | stp x2, x3, CTSTATE->cb.gpr[2]
2177 | stp d2, d3, CTSTATE->cb.fpr[2]
2178 | stp x4, x5, CTSTATE->cb.gpr[4]
2179 | stp d4, d5, CTSTATE->cb.fpr[4]
2180 | stp x6, x7, CTSTATE->cb.gpr[6]
2181 | stp d6, d7, CTSTATE->cb.fpr[6]
2182 | str x10, CTSTATE->cb.stack
2183 | mov CARG1, CTSTATE
2184 | str CTSTATE, SAVE_PC // Any value outside of bytecode is ok.
2185 | mov CARG2, sp
2186 | bl extern lj_ccallback_enter // (CTState *cts, void *cf)
2187 | // Returns lua_State *.
2188 | ldp BASE, RC, L:CRET1->base
2189 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
2190 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
2191 | movn TISNIL, #0
2192 | mov L, CRET1
2193 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2194 | sub RC, RC, BASE
2195 | st_vmstate ST_INTERP
2196 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2197 | ins_callt
2198 |.endif
2199 |
2200 |->cont_ffi_callback: // Return from FFI callback.
2201 |.if FFI
2202 | ldr CTSTATE, GL->ctype_state
2203 | stp BASE, CARG4, L->base
2204 | str L, CTSTATE->L
2205 | mov CARG1, CTSTATE
2206 | mov CARG2, RA
2207 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
2208 | ldp x0, x1, CTSTATE->cb.gpr[0]
2209 | ldp d0, d1, CTSTATE->cb.fpr[0]
2210 | b ->vm_leave_unw
2211 |.endif
2212 |
2213 |->vm_ffi_call: // Call C function via FFI.
2214 | // Caveat: needs special frame unwinding, see below.
2215 |.if FFI
2216 | .type CCSTATE, CCallState, x19
2217 | sp_auth
2218 | stp x20, CCSTATE, [sp, #-32]!
2219 | stp fp, lr, [sp, #16]
2220 | add fp, sp, #16
2221 | mov CCSTATE, x0
2222 | ldr TMP0w, CCSTATE:x0->spadj
2223 | ldrb TMP1w, CCSTATE->nsp
2224 | add TMP2, CCSTATE, #offsetof(CCallState, stack)
2225 | subs TMP1, TMP1, #1
2226 | ldr TMP3, CCSTATE->func
2227 | sub sp, sp, TMP0
2228 | bmi >2
2229 |1: // Copy stack slots
2230 | ldr TMP0, [TMP2, TMP1, lsl #3]
2231 | str TMP0, [sp, TMP1, lsl #3]
2232 | subs TMP1, TMP1, #1
2233 | bpl <1
2234 |2:
2235 | ldp x0, x1, CCSTATE->gpr[0]
2236 | ldp d0, d1, CCSTATE->fpr[0]
2237 | ldp x2, x3, CCSTATE->gpr[2]
2238 | ldp d2, d3, CCSTATE->fpr[2]
2239 | ldp x4, x5, CCSTATE->gpr[4]
2240 | ldp d4, d5, CCSTATE->fpr[4]
2241 | ldp x6, x7, CCSTATE->gpr[6]
2242 | ldp d6, d7, CCSTATE->fpr[6]
2243 | ldr x8, CCSTATE->retp
2244 | blr_auth TMP3
2245 | sub sp, fp, #16
2246 | stp x0, x1, CCSTATE->gpr[0]
2247 | stp d0, d1, CCSTATE->fpr[0]
2248 | stp d2, d3, CCSTATE->fpr[2]
2249 | ldp fp, lr, [sp, #16]
2250 | ldp x20, CCSTATE, [sp], #32
2251 | ret_auth
2252 |.endif
2253 |// Note: vm_ffi_call must be the last function in this object file!
2254 |
2255 |//-----------------------------------------------------------------------
2256}
2257
2258/* Generate the code for a single instruction. */
2259static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2260{
2261 int vk = 0;
2262 |=>defop:
2263
2264 switch (op) {
2265
2266 /* -- Comparison ops ---------------------------------------------------- */
2267
2268 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2269
2270 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2271 | // RA = src1, RC = src2, JMP with RC = target
2272 | ldr CARG1, [BASE, RA, lsl #3]
2273 | ldrh RBw, [PC, # OFS_RD]
2274 | ldr CARG2, [BASE, RC, lsl #3]
2275 | add PC, PC, #4
2276 | add RB, PC, RB, lsl #2
2277 | sub RB, RB, #0x20000
2278 | checkint CARG1, >3
2279 | checkint CARG2, >4
2280 | cmp CARG1w, CARG2w
2281 if (op == BC_ISLT) {
2282 | csel PC, RB, PC, lt
2283 } else if (op == BC_ISGE) {
2284 | csel PC, RB, PC, ge
2285 } else if (op == BC_ISLE) {
2286 | csel PC, RB, PC, le
2287 } else {
2288 | csel PC, RB, PC, gt
2289 }
2290 |1:
2291 | ins_next
2292 |
2293 |3: // RA not int.
2294 | ldr FARG1, [BASE, RA, lsl #3]
2295 | blo ->vmeta_comp
2296 | ldr FARG2, [BASE, RC, lsl #3]
2297 | cmp TISNUMhi, CARG2, lsr #32
2298 | bhi >5
2299 | bne ->vmeta_comp
2300 | // RA number, RC int.
2301 | scvtf FARG2, CARG2w
2302 | b >5
2303 |
2304 |4: // RA int, RC not int
2305 | ldr FARG2, [BASE, RC, lsl #3]
2306 | blo ->vmeta_comp
2307 | // RA int, RC number.
2308 | scvtf FARG1, CARG1w
2309 |
2310 |5: // RA number, RC number
2311 | fcmp FARG1, FARG2
2312 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2313 if (op == BC_ISLT) {
2314 | csel PC, RB, PC, lo
2315 } else if (op == BC_ISGE) {
2316 | csel PC, RB, PC, hs
2317 } else if (op == BC_ISLE) {
2318 | csel PC, RB, PC, ls
2319 } else {
2320 | csel PC, RB, PC, hi
2321 }
2322 | b <1
2323 break;
2324
2325 case BC_ISEQV: case BC_ISNEV:
2326 vk = op == BC_ISEQV;
2327 | // RA = src1, RC = src2, JMP with RC = target
2328 | ldr CARG1, [BASE, RA, lsl #3]
2329 | add RC, BASE, RC, lsl #3
2330 | ldrh RBw, [PC, # OFS_RD]
2331 | ldr CARG3, [RC]
2332 | add PC, PC, #4
2333 | add RB, PC, RB, lsl #2
2334 | sub RB, RB, #0x20000
2335 | asr ITYPE, CARG3, #47
2336 | cmn ITYPE, #-LJ_TISNUM
2337 if (vk) {
2338 | bls ->BC_ISEQN_Z
2339 } else {
2340 | bls ->BC_ISNEN_Z
2341 }
2342 | // RC is not a number.
2343 | asr TMP0, CARG1, #47
2344 |.if FFI
2345 | // Check if RC or RA is a cdata.
2346 | cmn ITYPE, #-LJ_TCDATA
2347 | ccmn TMP0, #-LJ_TCDATA, #4, ne
2348 | beq ->vmeta_equal_cd
2349 |.endif
2350 | cmp CARG1, CARG3
2351 | bne >2
2352 | // Tag and value are equal.
2353 if (vk) {
2354 |->BC_ISEQV_Z:
2355 | mov PC, RB // Perform branch.
2356 }
2357 |1:
2358 | ins_next
2359 |
2360 |2: // Check if the tags are the same and it's a table or userdata.
2361 | cmp ITYPE, TMP0
2362 | ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2363 if (vk) {
2364 | bhi <1
2365 } else {
2366 | bhi ->BC_ISEQV_Z // Reuse code from opposite instruction.
2367 }
2368 | // Different tables or userdatas. Need to check __eq metamethod.
2369 | // Field metatable must be at same offset for GCtab and GCudata!
2370 | and TAB:CARG2, CARG1, #LJ_GCVMASK
2371 | ldr TAB:TMP2, TAB:CARG2->metatable
2372 if (vk) {
2373 | cbz TAB:TMP2, <1 // No metatable?
2374 | ldrb TMP1w, TAB:TMP2->nomm
2375 | mov CARG4, #0 // ne = 0
2376 | tbnz TMP1w, #MM_eq, <1 // 'no __eq' flag set: done.
2377 } else {
2378 | cbz TAB:TMP2, ->BC_ISEQV_Z // No metatable?
2379 | ldrb TMP1w, TAB:TMP2->nomm
2380 | mov CARG4, #1 // ne = 1.
2381 | tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z // 'no __eq' flag set: done.
2382 }
2383 | b ->vmeta_equal
2384 break;
2385
2386 case BC_ISEQS: case BC_ISNES:
2387 vk = op == BC_ISEQS;
2388 | // RA = src, RC = str_const (~), JMP with RC = target
2389 | ldr CARG1, [BASE, RA, lsl #3]
2390 | mvn RC, RC
2391 | ldrh RBw, [PC, # OFS_RD]
2392 | ldr CARG2, [KBASE, RC, lsl #3]
2393 | add PC, PC, #4
2394 | movn TMP0, #~LJ_TSTR
2395 |.if FFI
2396 | asr ITYPE, CARG1, #47
2397 |.endif
2398 | add RB, PC, RB, lsl #2
2399 | add CARG2, CARG2, TMP0, lsl #47
2400 | sub RB, RB, #0x20000
2401 |.if FFI
2402 | cmn ITYPE, #-LJ_TCDATA
2403 | beq ->vmeta_equal_cd
2404 |.endif
2405 | cmp CARG1, CARG2
2406 if (vk) {
2407 | csel PC, RB, PC, eq
2408 } else {
2409 | csel PC, RB, PC, ne
2410 }
2411 | ins_next
2412 break;
2413
2414 case BC_ISEQN: case BC_ISNEN:
2415 vk = op == BC_ISEQN;
2416 | // RA = src, RC = num_const (~), JMP with RC = target
2417 | ldr CARG1, [BASE, RA, lsl #3]
2418 | add RC, KBASE, RC, lsl #3
2419 | ldrh RBw, [PC, # OFS_RD]
2420 | ldr CARG3, [RC]
2421 | add PC, PC, #4
2422 | add RB, PC, RB, lsl #2
2423 | sub RB, RB, #0x20000
2424 if (vk) {
2425 |->BC_ISEQN_Z:
2426 } else {
2427 |->BC_ISNEN_Z:
2428 }
2429 | checkint CARG1, >4
2430 | checkint CARG3, >6
2431 | cmp CARG1w, CARG3w
2432 |1:
2433 if (vk) {
2434 | csel PC, RB, PC, eq
2435 |2:
2436 } else {
2437 |2:
2438 | csel PC, RB, PC, ne
2439 }
2440 |3:
2441 | ins_next
2442 |
2443 |4: // RA not int.
2444 |.if FFI
2445 | blo >7
2446 |.else
2447 | blo <2
2448 |.endif
2449 | ldr FARG1, [BASE, RA, lsl #3]
2450 | ldr FARG2, [RC]
2451 | cmp TISNUMhi, CARG3, lsr #32
2452 | bne >5
2453 | // RA number, RC int.
2454 | scvtf FARG2, CARG3w
2455 |5:
2456 | // RA number, RC number.
2457 | fcmp FARG1, FARG2
2458 | b <1
2459 |
2460 |6: // RA int, RC number
2461 | ldr FARG2, [RC]
2462 | scvtf FARG1, CARG1w
2463 | fcmp FARG1, FARG2
2464 | b <1
2465 |
2466 |.if FFI
2467 |7:
2468 | asr ITYPE, CARG1, #47
2469 | cmn ITYPE, #-LJ_TCDATA
2470 | bne <2
2471 | b ->vmeta_equal_cd
2472 |.endif
2473 break;
2474
2475 case BC_ISEQP: case BC_ISNEP:
2476 vk = op == BC_ISEQP;
2477 | // RA = src, RC = primitive_type (~), JMP with RC = target
2478 | ldr TMP0, [BASE, RA, lsl #3]
2479 | ldrh RBw, [PC, # OFS_RD]
2480 | add PC, PC, #4
2481 | add RC, RC, #1
2482 | add RB, PC, RB, lsl #2
2483 |.if FFI
2484 | asr ITYPE, TMP0, #47
2485 | cmn ITYPE, #-LJ_TCDATA
2486 | beq ->vmeta_equal_cd
2487 | cmn RC, ITYPE
2488 |.else
2489 | cmn RC, TMP0, asr #47
2490 |.endif
2491 | sub RB, RB, #0x20000
2492 if (vk) {
2493 | csel PC, RB, PC, eq
2494 } else {
2495 | csel PC, RB, PC, ne
2496 }
2497 | ins_next
2498 break;
2499
2500 /* -- Unary test and copy ops ------------------------------------------- */
2501
2502 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2503 | // RA = dst or unused, RC = src, JMP with RC = target
2504 | ldrh RBw, [PC, # OFS_RD]
2505 | ldr TMP0, [BASE, RC, lsl #3]
2506 | add PC, PC, #4
2507 | mov_false TMP1
2508 | add RB, PC, RB, lsl #2
2509 | cmp TMP0, TMP1
2510 | sub RB, RB, #0x20000
2511 if (op == BC_ISTC || op == BC_IST) {
2512 if (op == BC_ISTC) {
2513 | csel RA, RA, RC, lo
2514 }
2515 | csel PC, RB, PC, lo
2516 } else {
2517 if (op == BC_ISFC) {
2518 | csel RA, RA, RC, hs
2519 }
2520 | csel PC, RB, PC, hs
2521 }
2522 if (op == BC_ISTC || op == BC_ISFC) {
2523 | str TMP0, [BASE, RA, lsl #3]
2524 }
2525 | ins_next
2526 break;
2527
2528 case BC_ISTYPE:
2529 | // RA = src, RC = -type
2530 | ldr TMP0, [BASE, RA, lsl #3]
2531 | cmn RC, TMP0, asr #47
2532 | bne ->vmeta_istype
2533 | ins_next
2534 break;
2535 case BC_ISNUM:
2536 | // RA = src, RC = -(TISNUM-1)
2537 | ldr TMP0, [BASE, RA]
2538 | checknum TMP0, ->vmeta_istype
2539 | ins_next
2540 break;
2541
2542 /* -- Unary ops --------------------------------------------------------- */
2543
2544 case BC_MOV:
2545 | // RA = dst, RC = src
2546 | ldr TMP0, [BASE, RC, lsl #3]
2547 | str TMP0, [BASE, RA, lsl #3]
2548 | ins_next
2549 break;
2550 case BC_NOT:
2551 | // RA = dst, RC = src
2552 | ldr TMP0, [BASE, RC, lsl #3]
2553 | mov_false TMP1
2554 | mov_true TMP2
2555 | cmp TMP0, TMP1
2556 | csel TMP0, TMP1, TMP2, lo
2557 | str TMP0, [BASE, RA, lsl #3]
2558 | ins_next
2559 break;
2560 case BC_UNM:
2561 | // RA = dst, RC = src
2562 | ldr TMP0, [BASE, RC, lsl #3]
2563 | asr ITYPE, TMP0, #47
2564 | cmn ITYPE, #-LJ_TISNUM
2565 | bhi ->vmeta_unm
2566 | eor TMP0, TMP0, #U64x(80000000,00000000)
2567 | bne >5
2568 | negs TMP0w, TMP0w
2569 | movz CARG3, #0x41e0, lsl #48 // 2^31.
2570 | add TMP0, TMP0, TISNUM
2571 | csel TMP0, TMP0, CARG3, vc
2572 |5:
2573 | str TMP0, [BASE, RA, lsl #3]
2574 | ins_next
2575 break;
2576 case BC_LEN:
2577 | // RA = dst, RC = src
2578 | ldr CARG1, [BASE, RC, lsl #3]
2579 | asr ITYPE, CARG1, #47
2580 | cmn ITYPE, #-LJ_TSTR
2581 | and CARG1, CARG1, #LJ_GCVMASK
2582 | bne >2
2583 | ldr CARG1w, STR:CARG1->len
2584 |1:
2585 | add CARG1, CARG1, TISNUM
2586 | str CARG1, [BASE, RA, lsl #3]
2587 | ins_next
2588 |
2589 |2:
2590 | cmn ITYPE, #-LJ_TTAB
2591 | bne ->vmeta_len
2592#if LJ_52
2593 | ldr TAB:CARG2, TAB:CARG1->metatable
2594 | cbnz TAB:CARG2, >9
2595 |3:
2596#endif
2597 |->BC_LEN_Z:
2598 | bl extern lj_tab_len // (GCtab *t)
2599 | // Returns uint32_t (but less than 2^31).
2600 | b <1
2601 |
2602#if LJ_52
2603 |9:
2604 | ldrb TMP1w, TAB:CARG2->nomm
2605 | tbnz TMP1w, #MM_len, <3 // 'no __len' flag set: done.
2606 | b ->vmeta_len
2607#endif
2608 break;
2609
2610 /* -- Binary ops -------------------------------------------------------- */
2611
2612 |.macro ins_arithcheck_int, target
2613 | checkint CARG1, target
2614 | checkint CARG2, target
2615 |.endmacro
2616 |
2617 |.macro ins_arithcheck_num, target
2618 | checknum CARG1, target
2619 | checknum CARG2, target
2620 |.endmacro
2621 |
2622 |.macro ins_arithcheck_nzdiv, target
2623 | cbz CARG2w, target
2624 |.endmacro
2625 |
2626 |.macro ins_arithhead
2627 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2628 ||if (vk == 1) {
2629 | and RC, RC, #255
2630 | decode_RB RB, INS
2631 ||} else {
2632 | decode_RB RB, INS
2633 | and RC, RC, #255
2634 ||}
2635 |.endmacro
2636 |
2637 |.macro ins_arithload, reg1, reg2
2638 | // RA = dst, RB = src1, RC = src2 | num_const
2639 ||switch (vk) {
2640 ||case 0:
2641 | ldr reg1, [BASE, RB, lsl #3]
2642 | ldr reg2, [KBASE, RC, lsl #3]
2643 || break;
2644 ||case 1:
2645 | ldr reg1, [KBASE, RC, lsl #3]
2646 | ldr reg2, [BASE, RB, lsl #3]
2647 || break;
2648 ||default:
2649 | ldr reg1, [BASE, RB, lsl #3]
2650 | ldr reg2, [BASE, RC, lsl #3]
2651 || break;
2652 ||}
2653 |.endmacro
2654 |
2655 |.macro ins_arithfallback, ins
2656 ||switch (vk) {
2657 ||case 0:
2658 | ins ->vmeta_arith_vn
2659 || break;
2660 ||case 1:
2661 | ins ->vmeta_arith_nv
2662 || break;
2663 ||default:
2664 | ins ->vmeta_arith_vv
2665 || break;
2666 ||}
2667 |.endmacro
2668 |
2669 |.macro ins_arithmod, res, reg1, reg2
2670 | fdiv d2, reg1, reg2
2671 | frintm d2, d2
2672 | // Cannot use fmsub, because FMA is not enabled by default.
2673 | fmul d2, d2, reg2
2674 | fsub res, reg1, d2
2675 |.endmacro
2676 |
2677 |.macro ins_arithdn, intins, fpins
2678 | ins_arithhead
2679 | ins_arithload CARG1, CARG2
2680 | ins_arithcheck_int >5
2681 |.if "intins" == "smull"
2682 | smull CARG1, CARG1w, CARG2w
2683 | cmp CARG1, CARG1, sxtw
2684 | mov CARG1w, CARG1w
2685 | ins_arithfallback bne
2686 |.elif "intins" == "ins_arithmodi"
2687 | ins_arithfallback ins_arithcheck_nzdiv
2688 | bl ->vm_modi
2689 |.else
2690 | intins CARG1w, CARG1w, CARG2w
2691 | ins_arithfallback bvs
2692 |.endif
2693 | add CARG1, CARG1, TISNUM
2694 | str CARG1, [BASE, RA, lsl #3]
2695 |4:
2696 | ins_next
2697 |
2698 |5: // FP variant.
2699 | ins_arithload FARG1, FARG2
2700 | ins_arithfallback ins_arithcheck_num
2701 | fpins FARG1, FARG1, FARG2
2702 | str FARG1, [BASE, RA, lsl #3]
2703 | b <4
2704 |.endmacro
2705 |
2706 |.macro ins_arithfp, fpins
2707 | ins_arithhead
2708 | ins_arithload CARG1, CARG2
2709 | ins_arithload FARG1, FARG2
2710 | ins_arithfallback ins_arithcheck_num
2711 |.if "fpins" == "fpow"
2712 | bl extern pow
2713 |.else
2714 | fpins FARG1, FARG1, FARG2
2715 |.endif
2716 | str FARG1, [BASE, RA, lsl #3]
2717 | ins_next
2718 |.endmacro
2719
2720 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2721 | ins_arithdn adds, fadd
2722 break;
2723 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2724 | ins_arithdn subs, fsub
2725 break;
2726 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2727 | ins_arithdn smull, fmul
2728 break;
2729 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2730 | ins_arithfp fdiv
2731 break;
2732 case BC_MODVN: case BC_MODNV: case BC_MODVV:
2733 | ins_arithdn ins_arithmodi, ins_arithmod
2734 break;
2735 case BC_POW:
2736 | // NYI: (partial) integer arithmetic.
2737 | ins_arithfp fpow
2738 break;
2739
2740 case BC_CAT:
2741 | decode_RB RB, INS
2742 | and RC, RC, #255
2743 | // RA = dst, RB = src_start, RC = src_end
2744 | str BASE, L->base
2745 | sub CARG3, RC, RB
2746 | add CARG2, BASE, RC, lsl #3
2747 |->BC_CAT_Z:
2748 | // RA = dst, CARG2 = top-1, CARG3 = left
2749 | mov CARG1, L
2750 | str PC, SAVE_PC
2751 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2752 | // Returns NULL (finished) or TValue * (metamethod).
2753 | ldrb RBw, [PC, #-4+OFS_RB]
2754 | ldr BASE, L->base
2755 | cbnz CRET1, ->vmeta_binop
2756 | ldr TMP0, [BASE, RB, lsl #3]
2757 | str TMP0, [BASE, RA, lsl #3] // Copy result to RA.
2758 | ins_next
2759 break;
2760
2761 /* -- Constant ops ------------------------------------------------------ */
2762
2763 case BC_KSTR:
2764 | // RA = dst, RC = str_const (~)
2765 | mvn RC, RC
2766 | ldr TMP0, [KBASE, RC, lsl #3]
2767 | movn TMP1, #~LJ_TSTR
2768 | add TMP0, TMP0, TMP1, lsl #47
2769 | str TMP0, [BASE, RA, lsl #3]
2770 | ins_next
2771 break;
2772 case BC_KCDATA:
2773 |.if FFI
2774 | // RA = dst, RC = cdata_const (~)
2775 | mvn RC, RC
2776 | ldr TMP0, [KBASE, RC, lsl #3]
2777 | movn TMP1, #~LJ_TCDATA
2778 | add TMP0, TMP0, TMP1, lsl #47
2779 | str TMP0, [BASE, RA, lsl #3]
2780 | ins_next
2781 |.endif
2782 break;
2783 case BC_KSHORT:
2784 | // RA = dst, RC = int16_literal
2785 | sxth RCw, RCw
2786 | add TMP0, RC, TISNUM
2787 | str TMP0, [BASE, RA, lsl #3]
2788 | ins_next
2789 break;
2790 case BC_KNUM:
2791 | // RA = dst, RC = num_const
2792 | ldr TMP0, [KBASE, RC, lsl #3]
2793 | str TMP0, [BASE, RA, lsl #3]
2794 | ins_next
2795 break;
2796 case BC_KPRI:
2797 | // RA = dst, RC = primitive_type (~)
2798 | mvn TMP0, RC, lsl #47
2799 | str TMP0, [BASE, RA, lsl #3]
2800 | ins_next
2801 break;
2802 case BC_KNIL:
2803 | // RA = base, RC = end
2804 | add RA, BASE, RA, lsl #3
2805 | add RC, BASE, RC, lsl #3
2806 | str TISNIL, [RA], #8
2807 |1:
2808 | cmp RA, RC
2809 | str TISNIL, [RA], #8
2810 | blt <1
2811 | ins_next_
2812 break;
2813
2814 /* -- Upvalue and function ops ------------------------------------------ */
2815
2816 case BC_UGET:
2817 | // RA = dst, RC = uvnum
2818 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2819 | add RC, RC, #offsetof(GCfuncL, uvptr)/8
2820 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2821 | ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2822 | ldr CARG2, UPVAL:CARG2->v
2823 | ldr TMP0, [CARG2]
2824 | str TMP0, [BASE, RA, lsl #3]
2825 | ins_next
2826 break;
2827 case BC_USETV:
2828 | // RA = uvnum, RC = src
2829 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2830 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2831 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2832 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2833 | ldr CARG3, [BASE, RC, lsl #3]
2834 | ldr CARG2, UPVAL:CARG1->v
2835 | ldrb TMP2w, UPVAL:CARG1->marked
2836 | ldrb TMP0w, UPVAL:CARG1->closed
2837 | asr ITYPE, CARG3, #47
2838 | str CARG3, [CARG2]
2839 | add ITYPE, ITYPE, #-LJ_TISGCV
2840 | tst TMP2w, #LJ_GC_BLACK // isblack(uv)
2841 | ccmp TMP0w, #0, #4, ne // && uv->closed
2842 | ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne // && tvisgcv(v)
2843 | bhi >2
2844 |1:
2845 | ins_next
2846 |
2847 |2: // Check if new value is white.
2848 | and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2849 | ldrb TMP1w, GCOBJ:CARG3->gch.marked
2850 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2851 | beq <1
2852 | // Crossed a write barrier. Move the barrier forward.
2853 | mov CARG1, GL
2854 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2855 | b <1
2856 break;
2857 case BC_USETS:
2858 | // RA = uvnum, RC = str_const (~)
2859 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2860 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2861 | mvn RC, RC
2862 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2863 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2864 | ldr STR:CARG3, [KBASE, RC, lsl #3]
2865 | movn TMP0, #~LJ_TSTR
2866 | ldr CARG2, UPVAL:CARG1->v
2867 | ldrb TMP2w, UPVAL:CARG1->marked
2868 | add TMP0, STR:CARG3, TMP0, lsl #47
2869 | ldrb TMP1w, STR:CARG3->marked
2870 | str TMP0, [CARG2]
2871 | tbnz TMP2w, #2, >2 // isblack(uv)
2872 |1:
2873 | ins_next
2874 |
2875 |2: // Check if string is white and ensure upvalue is closed.
2876 | ldrb TMP0w, UPVAL:CARG1->closed
2877 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2878 | ccmp TMP0w, #0, #4, ne
2879 | beq <1
2880 | // Crossed a write barrier. Move the barrier forward.
2881 | mov CARG1, GL
2882 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2883 | b <1
2884 break;
2885 case BC_USETN:
2886 | // RA = uvnum, RC = num_const
2887 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2888 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2889 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2890 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2891 | ldr TMP0, [KBASE, RC, lsl #3]
2892 | ldr CARG2, UPVAL:CARG2->v
2893 | str TMP0, [CARG2]
2894 | ins_next
2895 break;
2896 case BC_USETP:
2897 | // RA = uvnum, RC = primitive_type (~)
2898 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2899 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2900 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2901 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2902 | mvn TMP0, RC, lsl #47
2903 | ldr CARG2, UPVAL:CARG2->v
2904 | str TMP0, [CARG2]
2905 | ins_next
2906 break;
2907
2908 case BC_UCLO:
2909 | // RA = level, RC = target
2910 | ldr CARG3, L->openupval
2911 | add RC, PC, RC, lsl #2
2912 | str BASE, L->base
2913 | sub PC, RC, #0x20000
2914 | cbz CARG3, >1
2915 | mov CARG1, L
2916 | add CARG2, BASE, RA, lsl #3
2917 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2918 | ldr BASE, L->base
2919 |1:
2920 | ins_next
2921 break;
2922
2923 case BC_FNEW:
2924 | // RA = dst, RC = proto_const (~) (holding function prototype)
2925 | mvn RC, RC
2926 | str BASE, L->base
2927 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2928 | str PC, SAVE_PC
2929 | ldr CARG2, [KBASE, RC, lsl #3]
2930 | mov CARG1, L
2931 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2932 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2933 | bl extern lj_func_newL_gc
2934 | // Returns GCfuncL *.
2935 | ldr BASE, L->base
2936 | movn TMP0, #~LJ_TFUNC
2937 | add CRET1, CRET1, TMP0, lsl #47
2938 | str CRET1, [BASE, RA, lsl #3]
2939 | ins_next
2940 break;
2941
2942 /* -- Table ops --------------------------------------------------------- */
2943
2944 case BC_TNEW:
2945 case BC_TDUP:
2946 | // RA = dst, RC = (hbits|asize) | tab_const (~)
2947 | ldp CARG3, CARG4, GL->gc.total // Assumes threshold follows total.
2948 | str BASE, L->base
2949 | str PC, SAVE_PC
2950 | mov CARG1, L
2951 | cmp CARG3, CARG4
2952 | bhs >5
2953 |1:
2954 if (op == BC_TNEW) {
2955 | and CARG2, RC, #0x7ff
2956 | lsr CARG3, RC, #11
2957 | cmp CARG2, #0x7ff
2958 | mov TMP0, #0x801
2959 | csel CARG2, CARG2, TMP0, ne
2960 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2961 | // Returns GCtab *.
2962 } else {
2963 | mvn RC, RC
2964 | ldr CARG2, [KBASE, RC, lsl #3]
2965 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2966 | // Returns GCtab *.
2967 }
2968 | ldr BASE, L->base
2969 | movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
2970 | str CRET1, [BASE, RA, lsl #3]
2971 | ins_next
2972 |
2973 |5:
2974 | bl extern lj_gc_step_fixtop // (lua_State *L)
2975 | mov CARG1, L
2976 | b <1
2977 break;
2978
2979 case BC_GGET:
2980 | // RA = dst, RC = str_const (~)
2981 case BC_GSET:
2982 | // RA = src, RC = str_const (~)
2983 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
2984 | mvn RC, RC
2985 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
2986 | ldr TAB:CARG2, LFUNC:CARG1->env
2987 | ldr STR:RC, [KBASE, RC, lsl #3]
2988 if (op == BC_GGET) {
2989 | b ->BC_TGETS_Z
2990 } else {
2991 | b ->BC_TSETS_Z
2992 }
2993 break;
2994
2995 case BC_TGETV:
2996 | decode_RB RB, INS
2997 | and RC, RC, #255
2998 | // RA = dst, RB = table, RC = key
2999 | ldr CARG2, [BASE, RB, lsl #3]
3000 | ldr TMP1, [BASE, RC, lsl #3]
3001 | checktab CARG2, ->vmeta_tgetv
3002 | checkint TMP1, >9 // Integer key?
3003 | ldr CARG3, TAB:CARG2->array
3004 | ldr CARG1w, TAB:CARG2->asize
3005 | add CARG3, CARG3, TMP1, uxtw #3
3006 | cmp TMP1w, CARG1w // In array part?
3007 | bhs ->vmeta_tgetv
3008 | ldr TMP0, [CARG3]
3009 | cmp TMP0, TISNIL
3010 | beq >5
3011 |1:
3012 | str TMP0, [BASE, RA, lsl #3]
3013 | ins_next
3014 |
3015 |5: // Check for __index if table value is nil.
3016 | ldr TAB:CARG1, TAB:CARG2->metatable
3017 | cbz TAB:CARG1, <1 // No metatable: done.
3018 | ldrb TMP1w, TAB:CARG1->nomm
3019 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3020 | b ->vmeta_tgetv
3021 |
3022 |9:
3023 | asr ITYPE, TMP1, #47
3024 | cmn ITYPE, #-LJ_TSTR // String key?
3025 | bne ->vmeta_tgetv
3026 | and STR:RC, TMP1, #LJ_GCVMASK
3027 | b ->BC_TGETS_Z
3028 break;
3029 case BC_TGETS:
3030 | decode_RB RB, INS
3031 | and RC, RC, #255
3032 | // RA = dst, RB = table, RC = str_const (~)
3033 | ldr CARG2, [BASE, RB, lsl #3]
3034 | mvn RC, RC
3035 | ldr STR:RC, [KBASE, RC, lsl #3]
3036 | checktab CARG2, ->vmeta_tgets1
3037 |->BC_TGETS_Z:
3038 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
3039 | ldr TMP1w, TAB:CARG2->hmask
3040 | ldr TMP2w, STR:RC->sid
3041 | ldr NODE:CARG3, TAB:CARG2->node
3042 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3043 | add TMP1, TMP1, TMP1, lsl #1
3044 | movn CARG4, #~LJ_TSTR
3045 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3046 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3047 |1:
3048 | ldp TMP0, CARG1, NODE:CARG3->val
3049 | ldr NODE:CARG3, NODE:CARG3->next
3050 | cmp CARG1, CARG4
3051 | bne >4
3052 | cmp TMP0, TISNIL
3053 | beq >5
3054 |3:
3055 | str TMP0, [BASE, RA, lsl #3]
3056 | ins_next
3057 |
3058 |4: // Follow hash chain.
3059 | cbnz NODE:CARG3, <1
3060 | // End of hash chain: key not found, nil result.
3061 | mov TMP0, TISNIL
3062 |
3063 |5: // Check for __index if table value is nil.
3064 | ldr TAB:CARG1, TAB:CARG2->metatable
3065 | cbz TAB:CARG1, <3 // No metatable: done.
3066 | ldrb TMP1w, TAB:CARG1->nomm
3067 | tbnz TMP1w, #MM_index, <3 // 'no __index' flag set: done.
3068 | b ->vmeta_tgets
3069 break;
3070 case BC_TGETB:
3071 | decode_RB RB, INS
3072 | and RC, RC, #255
3073 | // RA = dst, RB = table, RC = index
3074 | ldr CARG2, [BASE, RB, lsl #3]
3075 | checktab CARG2, ->vmeta_tgetb
3076 | ldr CARG3, TAB:CARG2->array
3077 | ldr CARG1w, TAB:CARG2->asize
3078 | add CARG3, CARG3, RC, lsl #3
3079 | cmp RCw, CARG1w // In array part?
3080 | bhs ->vmeta_tgetb
3081 | ldr TMP0, [CARG3]
3082 | cmp TMP0, TISNIL
3083 | beq >5
3084 |1:
3085 | str TMP0, [BASE, RA, lsl #3]
3086 | ins_next
3087 |
3088 |5: // Check for __index if table value is nil.
3089 | ldr TAB:CARG1, TAB:CARG2->metatable
3090 | cbz TAB:CARG1, <1 // No metatable: done.
3091 | ldrb TMP1w, TAB:CARG1->nomm
3092 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3093 | b ->vmeta_tgetb
3094 break;
3095 case BC_TGETR:
3096 | decode_RB RB, INS
3097 | and RC, RC, #255
3098 | // RA = dst, RB = table, RC = key
3099 | ldr CARG1, [BASE, RB, lsl #3]
3100 | ldr TMP1, [BASE, RC, lsl #3]
3101 | and TAB:CARG1, CARG1, #LJ_GCVMASK
3102 | ldr CARG3, TAB:CARG1->array
3103 | ldr TMP2w, TAB:CARG1->asize
3104 | add CARG3, CARG3, TMP1w, uxtw #3
3105 | cmp TMP1w, TMP2w // In array part?
3106 | bhs ->vmeta_tgetr
3107 | ldr TMP0, [CARG3]
3108 |->BC_TGETR_Z:
3109 | str TMP0, [BASE, RA, lsl #3]
3110 | ins_next
3111 break;
3112
3113 case BC_TSETV:
3114 | decode_RB RB, INS
3115 | and RC, RC, #255
3116 | // RA = src, RB = table, RC = key
3117 | ldr CARG2, [BASE, RB, lsl #3]
3118 | ldr TMP1, [BASE, RC, lsl #3]
3119 | checktab CARG2, ->vmeta_tsetv
3120 | checkint TMP1, >9 // Integer key?
3121 | ldr CARG3, TAB:CARG2->array
3122 | ldr CARG1w, TAB:CARG2->asize
3123 | add CARG3, CARG3, TMP1, uxtw #3
3124 | cmp TMP1w, CARG1w // In array part?
3125 | bhs ->vmeta_tsetv
3126 | ldr TMP1, [CARG3]
3127 | ldr TMP0, [BASE, RA, lsl #3]
3128 | ldrb TMP2w, TAB:CARG2->marked
3129 | cmp TMP1, TISNIL // Previous value is nil?
3130 | beq >5
3131 |1:
3132 | str TMP0, [CARG3]
3133 | tbnz TMP2w, #2, >7 // isblack(table)
3134 |2:
3135 | ins_next
3136 |
3137 |5: // Check for __newindex if previous value is nil.
3138 | ldr TAB:CARG1, TAB:CARG2->metatable
3139 | cbz TAB:CARG1, <1 // No metatable: done.
3140 | ldrb TMP1w, TAB:CARG1->nomm
3141 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3142 | b ->vmeta_tsetv
3143 |
3144 |7: // Possible table write barrier for the value. Skip valiswhite check.
3145 | barrierback TAB:CARG2, TMP2w, TMP1
3146 | b <2
3147 |
3148 |9:
3149 | asr ITYPE, TMP1, #47
3150 | cmn ITYPE, #-LJ_TSTR // String key?
3151 | bne ->vmeta_tsetv
3152 | and STR:RC, TMP1, #LJ_GCVMASK
3153 | b ->BC_TSETS_Z
3154 break;
3155 case BC_TSETS:
3156 | decode_RB RB, INS
3157 | and RC, RC, #255
3158 | // RA = dst, RB = table, RC = str_const (~)
3159 | ldr CARG2, [BASE, RB, lsl #3]
3160 | mvn RC, RC
3161 | ldr STR:RC, [KBASE, RC, lsl #3]
3162 | checktab CARG2, ->vmeta_tsets1
3163 |->BC_TSETS_Z:
3164 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
3165 | ldr TMP1w, TAB:CARG2->hmask
3166 | ldr TMP2w, STR:RC->sid
3167 | ldr NODE:CARG3, TAB:CARG2->node
3168 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3169 | add TMP1, TMP1, TMP1, lsl #1
3170 | movn CARG4, #~LJ_TSTR
3171 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3172 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3173 | strb wzr, TAB:CARG2->nomm // Clear metamethod cache.
3174 |1:
3175 | ldp TMP1, CARG1, NODE:CARG3->val
3176 | ldr NODE:TMP3, NODE:CARG3->next
3177 | ldrb TMP2w, TAB:CARG2->marked
3178 | cmp CARG1, CARG4
3179 | bne >5
3180 | ldr TMP0, [BASE, RA, lsl #3]
3181 | cmp TMP1, TISNIL // Previous value is nil?
3182 | beq >4
3183 |2:
3184 | str TMP0, NODE:CARG3->val
3185 | tbnz TMP2w, #2, >7 // isblack(table)
3186 |3:
3187 | ins_next
3188 |
3189 |4: // Check for __newindex if previous value is nil.
3190 | ldr TAB:CARG1, TAB:CARG2->metatable
3191 | cbz TAB:CARG1, <2 // No metatable: done.
3192 | ldrb TMP1w, TAB:CARG1->nomm
3193 | tbnz TMP1w, #MM_newindex, <2 // 'no __newindex' flag set: done.
3194 | b ->vmeta_tsets
3195 |
3196 |5: // Follow hash chain.
3197 | mov NODE:CARG3, NODE:TMP3
3198 | cbnz NODE:TMP3, <1
3199 | // End of hash chain: key not found, add a new one.
3200 |
3201 | // But check for __newindex first.
3202 | ldr TAB:CARG1, TAB:CARG2->metatable
3203 | cbz TAB:CARG1, >6 // No metatable: continue.
3204 | ldrb TMP1w, TAB:CARG1->nomm
3205 | // 'no __newindex' flag NOT set: check.
3206 | tbz TMP1w, #MM_newindex, ->vmeta_tsets
3207 |6:
3208 | movn TMP1, #~LJ_TSTR
3209 | str PC, SAVE_PC
3210 | add TMP0, STR:RC, TMP1, lsl #47
3211 | str BASE, L->base
3212 | mov CARG1, L
3213 | str TMP0, TMPD
3214 | add CARG3, sp, TMPDofs
3215 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
3216 | // Returns TValue *.
3217 | ldr BASE, L->base
3218 | ldr TMP0, [BASE, RA, lsl #3]
3219 | str TMP0, [CRET1]
3220 | b <3 // No 2nd write barrier needed.
3221 |
3222 |7: // Possible table write barrier for the value. Skip valiswhite check.
3223 | barrierback TAB:CARG2, TMP2w, TMP1
3224 | b <3
3225 break;
3226 case BC_TSETB:
3227 | decode_RB RB, INS
3228 | and RC, RC, #255
3229 | // RA = src, RB = table, RC = index
3230 | ldr CARG2, [BASE, RB, lsl #3]
3231 | checktab CARG2, ->vmeta_tsetb
3232 | ldr CARG3, TAB:CARG2->array
3233 | ldr CARG1w, TAB:CARG2->asize
3234 | add CARG3, CARG3, RC, lsl #3
3235 | cmp RCw, CARG1w // In array part?
3236 | bhs ->vmeta_tsetb
3237 | ldr TMP1, [CARG3]
3238 | ldr TMP0, [BASE, RA, lsl #3]
3239 | ldrb TMP2w, TAB:CARG2->marked
3240 | cmp TMP1, TISNIL // Previous value is nil?
3241 | beq >5
3242 |1:
3243 | str TMP0, [CARG3]
3244 | tbnz TMP2w, #2, >7 // isblack(table)
3245 |2:
3246 | ins_next
3247 |
3248 |5: // Check for __newindex if previous value is nil.
3249 | ldr TAB:CARG1, TAB:CARG2->metatable
3250 | cbz TAB:CARG1, <1 // No metatable: done.
3251 | ldrb TMP1w, TAB:CARG1->nomm
3252 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3253 | b ->vmeta_tsetb
3254 |
3255 |7: // Possible table write barrier for the value. Skip valiswhite check.
3256 | barrierback TAB:CARG2, TMP2w, TMP1
3257 | b <2
3258 break;
3259 case BC_TSETR:
3260 | decode_RB RB, INS
3261 | and RC, RC, #255
3262 | // RA = src, RB = table, RC = key
3263 | ldr CARG2, [BASE, RB, lsl #3]
3264 | ldr TMP1, [BASE, RC, lsl #3]
3265 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3266 | ldr CARG1, TAB:CARG2->array
3267 | ldrb TMP2w, TAB:CARG2->marked
3268 | ldr CARG4w, TAB:CARG2->asize
3269 | add CARG1, CARG1, TMP1, uxtw #3
3270 | tbnz TMP2w, #2, >7 // isblack(table)
3271 |2:
3272 | cmp TMP1w, CARG4w // In array part?
3273 | bhs ->vmeta_tsetr
3274 |->BC_TSETR_Z:
3275 | ldr TMP0, [BASE, RA, lsl #3]
3276 | str TMP0, [CARG1]
3277 | ins_next
3278 |
3279 |7: // Possible table write barrier for the value. Skip valiswhite check.
3280 | barrierback TAB:CARG2, TMP2w, TMP0
3281 | b <2
3282 break;
3283
3284 case BC_TSETM:
3285 | // RA = base (table at base-1), RC = num_const (start index)
3286 | add RA, BASE, RA, lsl #3
3287 |1:
3288 | ldr RBw, SAVE_MULTRES
3289 | ldr TAB:CARG2, [RA, #-8] // Guaranteed to be a table.
3290 | ldr TMP1, [KBASE, RC, lsl #3] // Integer constant is in lo-word.
3291 | sub RB, RB, #8
3292 | cbz RB, >4 // Nothing to copy?
3293 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3294 | ldr CARG1w, TAB:CARG2->asize
3295 | add CARG3w, TMP1w, RBw, lsr #3
3296 | ldr CARG4, TAB:CARG2->array
3297 | cmp CARG3, CARG1
3298 | add RB, RA, RB
3299 | bhi >5
3300 | add TMP1, CARG4, TMP1w, uxtw #3
3301 | ldrb TMP2w, TAB:CARG2->marked
3302 |3: // Copy result slots to table.
3303 | ldr TMP0, [RA], #8
3304 | str TMP0, [TMP1], #8
3305 | cmp RA, RB
3306 | blo <3
3307 | tbnz TMP2w, #2, >7 // isblack(table)
3308 |4:
3309 | ins_next
3310 |
3311 |5: // Need to resize array part.
3312 | str BASE, L->base
3313 | mov CARG1, L
3314 | str PC, SAVE_PC
3315 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3316 | // Must not reallocate the stack.
3317 | b <1
3318 |
3319 |7: // Possible table write barrier for any value. Skip valiswhite check.
3320 | barrierback TAB:CARG2, TMP2w, TMP1
3321 | b <4
3322 break;
3323
3324 /* -- Calls and vararg handling ----------------------------------------- */
3325
3326 case BC_CALLM:
3327 | // RA = base, (RB = nresults+1,) RC = extra_nargs
3328 | ldr TMP0w, SAVE_MULTRES
3329 | decode_RC8RD NARGS8:RC, RC
3330 | add NARGS8:RC, NARGS8:RC, TMP0
3331 | b ->BC_CALL_Z
3332 break;
3333 case BC_CALL:
3334 | decode_RC8RD NARGS8:RC, RC
3335 | // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3336 |->BC_CALL_Z:
3337 | mov RB, BASE // Save old BASE for vmeta_call.
3338 | add BASE, BASE, RA, lsl #3
3339 | ldr CARG3, [BASE]
3340 | sub NARGS8:RC, NARGS8:RC, #8
3341 | add BASE, BASE, #16
3342 | checkfunc CARG3, ->vmeta_call
3343 | ins_call
3344 break;
3345
3346 case BC_CALLMT:
3347 | // RA = base, (RB = 0,) RC = extra_nargs
3348 | ldr TMP0w, SAVE_MULTRES
3349 | add NARGS8:RC, TMP0, RC, lsl #3
3350 | b ->BC_CALLT1_Z
3351 break;
3352 case BC_CALLT:
3353 | lsl NARGS8:RC, RC, #3
3354 | // RA = base, (RB = 0,) RC = (nargs+1)*8
3355 |->BC_CALLT1_Z:
3356 | add RA, BASE, RA, lsl #3
3357 | ldr TMP1, [RA]
3358 | sub NARGS8:RC, NARGS8:RC, #8
3359 | add RA, RA, #16
3360 | checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3361 | ldr PC, [BASE, FRAME_PC]
3362 |->BC_CALLT2_Z:
3363 | mov RB, #0
3364 | ldrb TMP2w, LFUNC:CARG3->ffid
3365 | tst PC, #FRAME_TYPE
3366 | bne >7
3367 |1:
3368 | str TMP1, [BASE, FRAME_FUNC] // Copy function down, but keep PC.
3369 | cbz NARGS8:RC, >3
3370 |2:
3371 | ldr TMP0, [RA, RB]
3372 | add TMP1, RB, #8
3373 | cmp TMP1, NARGS8:RC
3374 | str TMP0, [BASE, RB]
3375 | mov RB, TMP1
3376 | bne <2
3377 |3:
3378 | cmp TMP2, #1 // (> FF_C) Calling a fast function?
3379 | bhi >5
3380 |4:
3381 | ins_callt
3382 |
3383 |5: // Tailcall to a fast function with a Lua frame below.
3384 | ldrb RAw, [PC, #-4+OFS_RA]
3385 | sub CARG1, BASE, RA, lsl #3
3386 | ldr LFUNC:CARG1, [CARG1, #-32]
3387 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3388 | ldr CARG1, LFUNC:CARG1->pc
3389 | ldr KBASE, [CARG1, #PC2PROTO(k)]
3390 | b <4
3391 |
3392 |7: // Tailcall from a vararg function.
3393 | eor PC, PC, #FRAME_VARG
3394 | tst PC, #FRAME_TYPEP // Vararg frame below?
3395 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3396 | bne <1
3397 | sub BASE, BASE, PC
3398 | ldr PC, [BASE, FRAME_PC]
3399 | tst PC, #FRAME_TYPE
3400 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3401 | b <1
3402 break;
3403
3404 case BC_ITERC:
3405 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3406 | add RA, BASE, RA, lsl #3
3407 | ldr CARG3, [RA, #-24]
3408 | mov RB, BASE // Save old BASE for vmeta_call.
3409 | ldp CARG1, CARG2, [RA, #-16]
3410 | add BASE, RA, #16
3411 | mov NARGS8:RC, #16 // Iterators get 2 arguments.
3412 | str CARG3, [RA] // Copy callable.
3413 | stp CARG1, CARG2, [RA, #16] // Copy state and control var.
3414 | checkfunc CARG3, ->vmeta_call
3415 | ins_call
3416 break;
3417
3418 case BC_ITERN:
3419 |.if JIT
3420 | hotloop
3421 |.endif
3422 |->vm_IITERN:
3423 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3424 | add RA, BASE, RA, lsl #3
3425 | ldr TAB:RB, [RA, #-16]
3426 | ldrh TMP3w, [PC, # OFS_RD]
3427 | ldr CARG1w, [RA, #-8+LO] // Get index from control var.
3428 | add PC, PC, #4
3429 | add TMP3, PC, TMP3, lsl #2
3430 | and TAB:RB, RB, #LJ_GCVMASK
3431 | sub TMP3, TMP3, #0x20000
3432 | ldr TMP1w, TAB:RB->asize
3433 | ldr CARG2, TAB:RB->array
3434 |1: // Traverse array part.
3435 | subs RC, CARG1, TMP1
3436 | add CARG3, CARG2, CARG1, lsl #3
3437 | bhs >5 // Index points after array part?
3438 | ldr TMP0, [CARG3]
3439 | cmp TMP0, TISNIL
3440 | cinc CARG1, CARG1, eq // Skip holes in array part.
3441 | beq <1
3442 | add CARG1, CARG1, TISNUM
3443 | stp CARG1, TMP0, [RA]
3444 | add CARG1, CARG1, #1
3445 |3:
3446 | str CARG1w, [RA, #-8+LO] // Update control var.
3447 | mov PC, TMP3
3448 |4:
3449 | ins_next
3450 |
3451 |5: // Traverse hash part.
3452 | ldr TMP2w, TAB:RB->hmask
3453 | ldr NODE:RB, TAB:RB->node
3454 |6:
3455 | add CARG1, RC, RC, lsl #1
3456 | cmp RC, TMP2 // End of iteration? Branch to ITERN+1.
3457 | add NODE:CARG3, NODE:RB, CARG1, lsl #3 // node = tab->node + idx*3*8
3458 | bhi <4
3459 | ldp TMP0, CARG1, NODE:CARG3->val
3460 | cmp TMP0, TISNIL
3461 | add RC, RC, #1
3462 | beq <6 // Skip holes in hash part.
3463 | stp CARG1, TMP0, [RA]
3464 | add CARG1, RC, TMP1
3465 | b <3
3466 break;
3467
3468 case BC_ISNEXT:
3469 | // RA = base, RC = target (points to ITERN)
3470 | add RA, BASE, RA, lsl #3
3471 | ldr CFUNC:CARG1, [RA, #-24]
3472 | add RC, PC, RC, lsl #2
3473 | ldp TAB:CARG3, CARG4, [RA, #-16]
3474 | sub RC, RC, #0x20000
3475 | checkfunc CFUNC:CARG1, >5
3476 | asr TMP0, TAB:CARG3, #47
3477 | ldrb TMP1w, CFUNC:CARG1->ffid
3478 | cmn TMP0, #-LJ_TTAB
3479 | ccmp CARG4, TISNIL, #0, eq
3480 | ccmp TMP1w, #FF_next_N, #0, eq
3481 | bne >5
3482 | mov TMP0w, #0xfffe7fff // LJ_KEYINDEX
3483 | lsl TMP0, TMP0, #32
3484 | str TMP0, [RA, #-8] // Initialize control var.
3485 |1:
3486 | mov PC, RC
3487 | ins_next
3488 |
3489 |5: // Despecialize bytecode if any of the checks fail.
3490 |.if JIT
3491 | ldrb TMP2w, [RC, # OFS_OP]
3492 |.endif
3493 | mov TMP0, #BC_JMP
3494 | mov TMP1, #BC_ITERC
3495 | strb TMP0w, [PC, #-4+OFS_OP]
3496 |.if JIT
3497 | cmp TMP2w, #BC_ITERN
3498 | bne >6
3499 |.endif
3500 | strb TMP1w, [RC, # OFS_OP]
3501 | b <1
3502 |.if JIT
3503 |6: // Unpatch JLOOP.
3504 | ldr RA, [GL, #GL_J(trace)]
3505 | ldrh TMP2w, [RC, # OFS_RD]
3506 | ldr TRACE:RA, [RA, TMP2, lsl #3]
3507 | ldr TMP2w, TRACE:RA->startins
3508 | bfxil TMP2w, TMP1w, #0, #8
3509 | str TMP2w, [RC]
3510 | b <1
3511 |.endif
3512 break;
3513
3514 case BC_VARG:
3515 | decode_RB RB, INS
3516 | and RC, RC, #255
3517 | // RA = base, RB = (nresults+1), RC = numparams
3518 | ldr TMP1, [BASE, FRAME_PC]
3519 | add RC, BASE, RC, lsl #3
3520 | add RA, BASE, RA, lsl #3
3521 | add RC, RC, #FRAME_VARG
3522 | add TMP2, RA, RB, lsl #3
3523 | sub RC, RC, TMP1 // RC = vbase
3524 | // Note: RC may now be even _above_ BASE if nargs was < numparams.
3525 | sub TMP3, BASE, #16 // TMP3 = vtop
3526 | cbz RB, >5
3527 | sub TMP2, TMP2, #16
3528 |1: // Copy vararg slots to destination slots.
3529 | cmp RC, TMP3
3530 | ldr TMP0, [RC], #8
3531 | csel TMP0, TMP0, TISNIL, lo
3532 | cmp RA, TMP2
3533 | str TMP0, [RA], #8
3534 | blo <1
3535 |2:
3536 | ins_next
3537 |
3538 |5: // Copy all varargs.
3539 | ldr TMP0, L->maxstack
3540 | subs TMP2, TMP3, RC
3541 | csel RB, xzr, TMP2, le // MULTRES = (max(vtop-vbase,0)+1)*8
3542 | add RB, RB, #8
3543 | add TMP1, RA, TMP2
3544 | str RBw, SAVE_MULTRES
3545 | ble <2 // Nothing to copy.
3546 | cmp TMP1, TMP0
3547 | bhi >7
3548 |6:
3549 | ldr TMP0, [RC], #8
3550 | str TMP0, [RA], #8
3551 | cmp RC, TMP3
3552 | blo <6
3553 | b <2
3554 |
3555 |7: // Grow stack for varargs.
3556 | lsr CARG2, TMP2, #3
3557 | stp BASE, RA, L->base
3558 | mov CARG1, L
3559 | sub RC, RC, BASE // Need delta, because BASE may change.
3560 | str PC, SAVE_PC
3561 | bl extern lj_state_growstack // (lua_State *L, int n)
3562 | ldp BASE, RA, L->base
3563 | add RC, BASE, RC
3564 | sub TMP3, BASE, #16
3565 | b <6
3566 break;
3567
3568 /* -- Returns ----------------------------------------------------------- */
3569
3570 case BC_RETM:
3571 | // RA = results, RC = extra results
3572 | ldr TMP0w, SAVE_MULTRES
3573 | ldr PC, [BASE, FRAME_PC]
3574 | add RA, BASE, RA, lsl #3
3575 | add RC, TMP0, RC, lsl #3
3576 | b ->BC_RETM_Z
3577 break;
3578
3579 case BC_RET:
3580 | // RA = results, RC = nresults+1
3581 | ldr PC, [BASE, FRAME_PC]
3582 | lsl RC, RC, #3
3583 | add RA, BASE, RA, lsl #3
3584 |->BC_RETM_Z:
3585 | str RCw, SAVE_MULTRES
3586 |1:
3587 | ands CARG1, PC, #FRAME_TYPE
3588 | eor CARG2, PC, #FRAME_VARG
3589 | bne ->BC_RETV2_Z
3590 |
3591 |->BC_RET_Z:
3592 | // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3593 | ldr INSw, [PC, #-4]
3594 | subs TMP1, RC, #8
3595 | sub CARG3, BASE, #16
3596 | beq >3
3597 |2:
3598 | ldr TMP0, [RA], #8
3599 | add BASE, BASE, #8
3600 | sub TMP1, TMP1, #8
3601 | str TMP0, [BASE, #-24]
3602 | cbnz TMP1, <2
3603 |3:
3604 | decode_RA RA, INS
3605 | sub CARG4, CARG3, RA, lsl #3
3606 | decode_RB RB, INS
3607 | ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3608 |5:
3609 | cmp RC, RB, lsl #3 // More results expected?
3610 | blo >6
3611 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3612 | mov BASE, CARG4
3613 | ldr CARG2, LFUNC:CARG1->pc
3614 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3615 | ins_next
3616 |
3617 |6: // Fill up results with nil.
3618 | add BASE, BASE, #8
3619 | add RC, RC, #8
3620 | str TISNIL, [BASE, #-24]
3621 | b <5
3622 |
3623 |->BC_RETV1_Z: // Non-standard return case.
3624 | add RA, BASE, RA, lsl #3
3625 |->BC_RETV2_Z:
3626 | tst CARG2, #FRAME_TYPEP
3627 | bne ->vm_return
3628 | // Return from vararg function: relocate BASE down.
3629 | sub BASE, BASE, CARG2
3630 | ldr PC, [BASE, FRAME_PC]
3631 | b <1
3632 break;
3633
3634 case BC_RET0: case BC_RET1:
3635 | // RA = results, RC = nresults+1
3636 | ldr PC, [BASE, FRAME_PC]
3637 | lsl RC, RC, #3
3638 | str RCw, SAVE_MULTRES
3639 | ands CARG1, PC, #FRAME_TYPE
3640 | eor CARG2, PC, #FRAME_VARG
3641 | bne ->BC_RETV1_Z
3642 | ldr INSw, [PC, #-4]
3643 if (op == BC_RET1) {
3644 | ldr TMP0, [BASE, RA, lsl #3]
3645 }
3646 | sub CARG4, BASE, #16
3647 | decode_RA RA, INS
3648 | sub BASE, CARG4, RA, lsl #3
3649 if (op == BC_RET1) {
3650 | str TMP0, [CARG4], #8
3651 }
3652 | decode_RB RB, INS
3653 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3654 |5:
3655 | cmp RC, RB, lsl #3
3656 | blo >6
3657 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3658 | ldr CARG2, LFUNC:CARG1->pc
3659 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3660 | ins_next
3661 |
3662 |6: // Fill up results with nil.
3663 | add RC, RC, #8
3664 | str TISNIL, [CARG4], #8
3665 | b <5
3666 break;
3667
3668 /* -- Loops and branches ------------------------------------------------ */
3669
3670 |.define FOR_IDX, [RA]; .define FOR_TIDX, [RA, #4]
3671 |.define FOR_STOP, [RA, #8]; .define FOR_TSTOP, [RA, #12]
3672 |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3673 |.define FOR_EXT, [RA, #24]; .define FOR_TEXT, [RA, #28]
3674
3675 case BC_FORL:
3676 |.if JIT
3677 | hotloop
3678 |.endif
3679 | // Fall through. Assumes BC_IFORL follows.
3680 break;
3681
3682 case BC_JFORI:
3683 case BC_JFORL:
3684#if !LJ_HASJIT
3685 break;
3686#endif
3687 case BC_FORI:
3688 case BC_IFORL:
3689 | // RA = base, RC = target (after end of loop or start of loop)
3690 vk = (op == BC_IFORL || op == BC_JFORL);
3691 | add RA, BASE, RA, lsl #3
3692 | ldp CARG1, CARG2, FOR_IDX // CARG1 = IDX, CARG2 = STOP
3693 | ldr CARG3, FOR_STEP // CARG3 = STEP
3694 if (op != BC_JFORL) {
3695 | add RC, PC, RC, lsl #2
3696 | sub RC, RC, #0x20000
3697 }
3698 | checkint CARG1, >5
3699 if (!vk) {
3700 | checkint CARG2, ->vmeta_for
3701 | checkint CARG3, ->vmeta_for
3702 | tbnz CARG3w, #31, >4
3703 | cmp CARG1w, CARG2w
3704 } else {
3705 | adds CARG1w, CARG1w, CARG3w
3706 | bvs >2
3707 | add TMP0, CARG1, TISNUM
3708 | tbnz CARG3w, #31, >4
3709 | cmp CARG1w, CARG2w
3710 }
3711 |1:
3712 if (op == BC_FORI) {
3713 | csel PC, RC, PC, gt
3714 } else if (op == BC_JFORI) {
3715 | mov PC, RC
3716 | ldrh RCw, [RC, #-4+OFS_RD]
3717 } else if (op == BC_IFORL) {
3718 | csel PC, RC, PC, le
3719 }
3720 if (vk) {
3721 | str TMP0, FOR_IDX
3722 | str TMP0, FOR_EXT
3723 } else {
3724 | str CARG1, FOR_EXT
3725 }
3726 if (op == BC_JFORI || op == BC_JFORL) {
3727 | ble =>BC_JLOOP
3728 }
3729 |2:
3730 | ins_next
3731 |
3732 |4: // Invert check for negative step.
3733 | cmp CARG2w, CARG1w
3734 | b <1
3735 |
3736 |5: // FP loop.
3737 | ldp d0, d1, FOR_IDX
3738 | blo ->vmeta_for
3739 if (!vk) {
3740 | checknum CARG2, ->vmeta_for
3741 | checknum CARG3, ->vmeta_for
3742 | str d0, FOR_EXT
3743 } else {
3744 | ldr d2, FOR_STEP
3745 | fadd d0, d0, d2
3746 }
3747 | tbnz CARG3, #63, >7
3748 | fcmp d0, d1
3749 |6:
3750 if (vk) {
3751 | str d0, FOR_IDX
3752 | str d0, FOR_EXT
3753 }
3754 if (op == BC_FORI) {
3755 | csel PC, RC, PC, hi
3756 } else if (op == BC_JFORI) {
3757 | ldrh RCw, [RC, #-4+OFS_RD]
3758 | bls =>BC_JLOOP
3759 } else if (op == BC_IFORL) {
3760 | csel PC, RC, PC, ls
3761 } else {
3762 | bls =>BC_JLOOP
3763 }
3764 | b <2
3765 |
3766 |7: // Invert check for negative step.
3767 | fcmp d1, d0
3768 | b <6
3769 break;
3770
3771 case BC_ITERL:
3772 |.if JIT
3773 | hotloop
3774 |.endif
3775 | // Fall through. Assumes BC_IITERL follows.
3776 break;
3777
3778 case BC_JITERL:
3779#if !LJ_HASJIT
3780 break;
3781#endif
3782 case BC_IITERL:
3783 | // RA = base, RC = target
3784 | ldr CARG1, [BASE, RA, lsl #3]
3785 | add TMP1, BASE, RA, lsl #3
3786 | cmp CARG1, TISNIL
3787 | beq >1 // Stop if iterator returned nil.
3788 if (op == BC_JITERL) {
3789 | str CARG1, [TMP1, #-8]
3790 | b =>BC_JLOOP
3791 } else {
3792 | add TMP0, PC, RC, lsl #2 // Otherwise save control var + branch.
3793 | sub PC, TMP0, #0x20000
3794 | str CARG1, [TMP1, #-8]
3795 }
3796 |1:
3797 | ins_next
3798 break;
3799
3800 case BC_LOOP:
3801 | // RA = base, RC = target (loop extent)
3802 | // Note: RA/RC is only used by trace recorder to determine scope/extent
3803 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3804 |.if JIT
3805 | hotloop
3806 |.endif
3807 | // Fall through. Assumes BC_ILOOP follows.
3808 break;
3809
3810 case BC_ILOOP:
3811 | // RA = base, RC = target (loop extent)
3812 | ins_next
3813 break;
3814
3815 case BC_JLOOP:
3816 |.if JIT
3817 | // RA = base (ignored), RC = traceno
3818 | ldr CARG1, [GL, #GL_J(trace)]
3819 | mov CARG2w, #0 // Traces on ARM64 don't store the trace #, so use 0.
3820 | ldr TRACE:RC, [CARG1, RC, lsl #3]
3821 | st_vmstate CARG2w
3822 |.if PAUTH
3823 | ldr RA, TRACE:RC->mcauth
3824 |.else
3825 | ldr RA, TRACE:RC->mcode
3826 |.endif
3827 | str BASE, GL->jit_base
3828 | str L, GL->tmpbuf.L
3829 | sub sp, sp, #16 // See SPS_FIXED. Avoids sp adjust in every root trace.
3830 |.if PAUTH
3831 | braa RA, RC
3832 |.else
3833 | br RA
3834 |.endif
3835 |.endif
3836 break;
3837
3838 case BC_JMP:
3839 | // RA = base (only used by trace recorder), RC = target
3840 | add RC, PC, RC, lsl #2
3841 | sub PC, RC, #0x20000
3842 | ins_next
3843 break;
3844
3845 /* -- Function headers -------------------------------------------------- */
3846
3847 case BC_FUNCF:
3848 |.if JIT
3849 | hotcall
3850 |.endif
3851 case BC_FUNCV: /* NYI: compiled vararg functions. */
3852 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3853 break;
3854
3855 case BC_JFUNCF:
3856#if !LJ_HASJIT
3857 break;
3858#endif
3859 case BC_IFUNCF:
3860 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3861 | ldr CARG1, L->maxstack
3862 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3863 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3864 | cmp RA, CARG1
3865 | bhi ->vm_growstack_l
3866 |2:
3867 | cmp NARGS8:RC, TMP1, lsl #3 // Check for missing parameters.
3868 | blo >3
3869 if (op == BC_JFUNCF) {
3870 | decode_RD RC, INS
3871 | b =>BC_JLOOP
3872 } else {
3873 | ins_next
3874 }
3875 |
3876 |3: // Clear missing parameters.
3877 | str TISNIL, [BASE, NARGS8:RC]
3878 | add NARGS8:RC, NARGS8:RC, #8
3879 | b <2
3880 break;
3881
3882 case BC_JFUNCV:
3883#if !LJ_HASJIT
3884 break;
3885#endif
3886 | NYI // NYI: compiled vararg functions
3887 break; /* NYI: compiled vararg functions. */
3888
3889 case BC_IFUNCV:
3890 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3891 | ldr CARG1, L->maxstack
3892 | movn TMP0, #~LJ_TFUNC
3893 | add TMP2, BASE, RC
3894 | add LFUNC:CARG3, CARG3, TMP0, lsl #47
3895 | add RA, RA, RC
3896 | add TMP0, RC, #16+FRAME_VARG
3897 | str LFUNC:CARG3, [TMP2], #8 // Store (tagged) copy of LFUNC.
3898 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3899 | cmp RA, CARG1
3900 | str TMP0, [TMP2], #8 // Store delta + FRAME_VARG.
3901 | bhs ->vm_growstack_l
3902 | sub RC, TMP2, #16
3903 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3904 | mov RA, BASE
3905 | mov BASE, TMP2
3906 | cbz TMP1, >2
3907 |1:
3908 | cmp RA, RC // Less args than parameters?
3909 | bhs >3
3910 | ldr TMP0, [RA]
3911 | sub TMP1, TMP1, #1
3912 | str TISNIL, [RA], #8 // Clear old fixarg slot (help the GC).
3913 | str TMP0, [TMP2], #8
3914 | cbnz TMP1, <1
3915 |2:
3916 | ins_next
3917 |
3918 |3:
3919 | sub TMP1, TMP1, #1
3920 | str TISNIL, [TMP2], #8
3921 | cbz TMP1, <2
3922 | b <3
3923 break;
3924
3925 case BC_FUNCC:
3926 case BC_FUNCCW:
3927 | // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
3928 if (op == BC_FUNCC) {
3929 | ldr CARG4, CFUNC:CARG3->f
3930 } else {
3931 | ldr CARG4, GL->wrapf
3932 }
3933 | add CARG2, RA, NARGS8:RC
3934 | ldr CARG1, L->maxstack
3935 | add RC, BASE, NARGS8:RC
3936 | cmp CARG2, CARG1
3937 | stp BASE, RC, L->base
3938 if (op == BC_FUNCCW) {
3939 | ldr CARG2, CFUNC:CARG3->f
3940 }
3941 | mv_vmstate TMP0w, C
3942 | mov CARG1, L
3943 | bhi ->vm_growstack_c // Need to grow stack.
3944 | st_vmstate TMP0w
3945 | blr_auth CARG4 // (lua_State *L [, lua_CFunction f])
3946 | // Returns nresults.
3947 | ldp BASE, TMP1, L->base
3948 | str L, GL->cur_L
3949 | sbfiz RC, CRET1, #3, #32
3950 | st_vmstate ST_INTERP
3951 | ldr PC, [BASE, FRAME_PC]
3952 | sub RA, TMP1, RC // RA = L->top - nresults*8
3953 | b ->vm_returnc
3954 break;
3955
3956 /* ---------------------------------------------------------------------- */
3957
3958 default:
3959 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3960 exit(2);
3961 break;
3962 }
3963}
3964
3965static int build_backend(BuildCtx *ctx)
3966{
3967 int op;
3968
3969 dasm_growpc(Dst, BC__MAX);
3970
3971 build_subroutines(ctx);
3972
3973 |.code_op
3974 for (op = 0; op < BC__MAX; op++)
3975 build_ins(ctx, (BCOp)op, op);
3976
3977 return BC__MAX;
3978}
3979
3980/* Emit pseudo frame-info for all assembler functions. */
3981static void emit_asm_debug(BuildCtx *ctx)
3982{
3983 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
3984 int i;
3985 switch (ctx->mode) {
3986 case BUILD_elfasm:
3987 fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
3988 fprintf(ctx->fp,
3989 ".Lframe0:\n"
3990 "\t.long .LECIE0-.LSCIE0\n"
3991 ".LSCIE0:\n"
3992 "\t.long 0xffffffff\n"
3993 "\t.byte 0x1\n"
3994 "\t.string \"\"\n"
3995 "\t.uleb128 0x1\n"
3996 "\t.sleb128 -8\n"
3997 "\t.byte 30\n" /* Return address is in lr. */
3998 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
3999 "\t.align 3\n"
4000 ".LECIE0:\n\n");
4001 fprintf(ctx->fp,
4002 ".LSFDE0:\n"
4003 "\t.long .LEFDE0-.LASFDE0\n"
4004 ".LASFDE0:\n"
4005 "\t.long .Lframe0\n"
4006 "\t.quad .Lbegin\n"
4007 "\t.quad %d\n"
4008 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4009 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4010 fcofs);
4011 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4012 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4013 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4014 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4015 64+i, i+(3+(28-19+1)-8));
4016 fprintf(ctx->fp,
4017 "\t.align 3\n"
4018 ".LEFDE0:\n\n");
4019#if LJ_HASFFI
4020 fprintf(ctx->fp,
4021 ".LSFDE1:\n"
4022 "\t.long .LEFDE1-.LASFDE1\n"
4023 ".LASFDE1:\n"
4024 "\t.long .Lframe0\n"
4025 "\t.quad lj_vm_ffi_call\n"
4026 "\t.quad %d\n"
4027 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4028 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4029 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4030 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4031 "\t.align 3\n"
4032 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4033#endif
4034#if !LJ_NO_UNWIND
4035 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
4036 fprintf(ctx->fp,
4037 ".Lframe1:\n"
4038 "\t.long .LECIE1-.LSCIE1\n"
4039 ".LSCIE1:\n"
4040 "\t.long 0\n"
4041 "\t.byte 0x1\n"
4042 "\t.string \"zPR\"\n"
4043 "\t.uleb128 0x1\n"
4044 "\t.sleb128 -8\n"
4045 "\t.byte 30\n" /* Return address is in lr. */
4046 "\t.uleb128 6\n" /* augmentation length */
4047 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4048 "\t.long lj_err_unwind_dwarf-.\n"
4049 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4050 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4051 "\t.align 3\n"
4052 ".LECIE1:\n\n");
4053 fprintf(ctx->fp,
4054 ".LSFDE2:\n"
4055 "\t.long .LEFDE2-.LASFDE2\n"
4056 ".LASFDE2:\n"
4057 "\t.long .LASFDE2-.Lframe1\n"
4058 "\t.long .Lbegin-.\n"
4059 "\t.long %d\n"
4060 "\t.uleb128 0\n" /* augmentation length */
4061 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4062 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4063 fcofs);
4064 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4065 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4066 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4067 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4068 64+i, i+(3+(28-19+1)-8));
4069 fprintf(ctx->fp,
4070 "\t.align 3\n"
4071 ".LEFDE2:\n\n");
4072#if LJ_HASFFI
4073 fprintf(ctx->fp,
4074 ".Lframe2:\n"
4075 "\t.long .LECIE2-.LSCIE2\n"
4076 ".LSCIE2:\n"
4077 "\t.long 0\n"
4078 "\t.byte 0x1\n"
4079 "\t.string \"zR\"\n"
4080 "\t.uleb128 0x1\n"
4081 "\t.sleb128 -8\n"
4082 "\t.byte 30\n" /* Return address is in lr. */
4083 "\t.uleb128 1\n" /* augmentation length */
4084 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4085 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4086 "\t.align 3\n"
4087 ".LECIE2:\n\n");
4088 fprintf(ctx->fp,
4089 ".LSFDE3:\n"
4090 "\t.long .LEFDE3-.LASFDE3\n"
4091 ".LASFDE3:\n"
4092 "\t.long .LASFDE3-.Lframe2\n"
4093 "\t.long lj_vm_ffi_call-.\n"
4094 "\t.long %d\n"
4095 "\t.uleb128 0\n" /* augmentation length */
4096 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4097 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4098 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4099 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4100 "\t.align 3\n"
4101 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4102#endif
4103#endif
4104 break;
4105#if !LJ_NO_UNWIND
4106 case BUILD_machasm: {
4107#if LJ_HASFFI
4108 int fcsize = 0;
4109#endif
4110 int j;
4111 fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4112 fprintf(ctx->fp,
4113 "EH_frame1:\n"
4114 "\t.set L$set$x,LECIEX-LSCIEX\n"
4115 "\t.long L$set$x\n"
4116 "LSCIEX:\n"
4117 "\t.long 0\n"
4118 "\t.byte 0x1\n"
4119 "\t.ascii \"zPR\\0\"\n"
4120 "\t.uleb128 0x1\n"
4121 "\t.sleb128 -8\n"
4122 "\t.byte 30\n" /* Return address is in lr. */
4123 "\t.uleb128 6\n" /* augmentation length */
4124 "\t.byte 0x9b\n" /* indirect|pcrel|sdata4 */
4125 "\t.long _lj_err_unwind_dwarf@GOT-.\n"
4126 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4127 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4128 "\t.align 3\n"
4129 "LECIEX:\n\n");
4130 for (j = 0; j < ctx->nsym; j++) {
4131 const char *name = ctx->sym[j].name;
4132 int32_t size = ctx->sym[j+1].ofs - ctx->sym[j].ofs;
4133 if (size == 0) continue;
4134#if LJ_HASFFI
4135 if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4136#endif
4137 fprintf(ctx->fp,
4138 "LSFDE%d:\n"
4139 "\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4140 "\t.long L$set$%d\n"
4141 "LASFDE%d:\n"
4142 "\t.long LASFDE%d-EH_frame1\n"
4143 "\t.long %s-.\n"
4144 "\t.long %d\n"
4145 "\t.uleb128 0\n" /* augmentation length */
4146 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4147 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4148 j, j, j, j, j, j, j, name, size);
4149 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4150 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4151 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4152 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4153 64+i, i+(3+(28-19+1)-8));
4154 fprintf(ctx->fp,
4155 "\t.align 3\n"
4156 "LEFDE%d:\n\n", j);
4157 }
4158#if LJ_HASFFI
4159 if (fcsize) {
4160 fprintf(ctx->fp,
4161 "EH_frame2:\n"
4162 "\t.set L$set$y,LECIEY-LSCIEY\n"
4163 "\t.long L$set$y\n"
4164 "LSCIEY:\n"
4165 "\t.long 0\n"
4166 "\t.byte 0x1\n"
4167 "\t.ascii \"zR\\0\"\n"
4168 "\t.uleb128 0x1\n"
4169 "\t.sleb128 -8\n"
4170 "\t.byte 30\n" /* Return address is in lr. */
4171 "\t.uleb128 1\n" /* augmentation length */
4172 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4173 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4174 "\t.align 3\n"
4175 "LECIEY:\n\n");
4176 fprintf(ctx->fp,
4177 "LSFDEY:\n"
4178 "\t.set L$set$yy,LEFDEY-LASFDEY\n"
4179 "\t.long L$set$yy\n"
4180 "LASFDEY:\n"
4181 "\t.long LASFDEY-EH_frame2\n"
4182 "\t.long _lj_vm_ffi_call-.\n"
4183 "\t.long %d\n"
4184 "\t.uleb128 0\n" /* augmentation length */
4185 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4186 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4187 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4188 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4189 "\t.align 3\n"
4190 "LEFDEY:\n\n", fcsize);
4191 }
4192#endif
4193 fprintf(ctx->fp, ".subsections_via_symbols\n");
4194 }
4195 break;
4196#endif
4197 default:
4198 break;
4199 }
4200}
4201
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index f6f801f2..94a878b9 100644
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -1,6 +1,9 @@
1|// Low-level VM code for MIPS CPUs. 1|// Low-level VM code for MIPS CPUs.
2|// Bytecode interpreter, fast functions and helper functions. 2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h 3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4|//
5|// MIPS soft-float support contributed by Djordje Kovacevic and
6|// Stefan Pejic from RT-RK.com, sponsored by Cisco Systems, Inc.
4| 7|
5|.arch mips 8|.arch mips
6|.section code_op, code_sub 9|.section code_op, code_sub
@@ -18,6 +21,12 @@
18|// Fixed register assignments for the interpreter. 21|// Fixed register assignments for the interpreter.
19|// Don't use: r0 = 0, r26/r27 = reserved, r28 = gp, r29 = sp, r31 = ra 22|// Don't use: r0 = 0, r26/r27 = reserved, r28 = gp, r29 = sp, r31 = ra
20| 23|
24|.macro .FPU, a, b
25|.if FPU
26| a, b
27|.endif
28|.endmacro
29|
21|// The following must be C callee-save (but BASE is often refetched). 30|// The following must be C callee-save (but BASE is often refetched).
22|.define BASE, r16 // Base of current Lua stack frame. 31|.define BASE, r16 // Base of current Lua stack frame.
23|.define KBASE, r17 // Constants of current Lua function. 32|.define KBASE, r17 // Constants of current Lua function.
@@ -25,13 +34,15 @@
25|.define DISPATCH, r19 // Opcode dispatch table. 34|.define DISPATCH, r19 // Opcode dispatch table.
26|.define LREG, r20 // Register holding lua_State (also in SAVE_L). 35|.define LREG, r20 // Register holding lua_State (also in SAVE_L).
27|.define MULTRES, r21 // Size of multi-result: (nresults+1)*8. 36|.define MULTRES, r21 // Size of multi-result: (nresults+1)*8.
28|// NYI: r22 currently unused.
29| 37|
30|.define JGL, r30 // On-trace: global_State + 32768. 38|.define JGL, r30 // On-trace: global_State + 32768.
31| 39|
32|// Constants for type-comparisons, stores and conversions. C callee-save. 40|// Constants for type-comparisons, stores and conversions. C callee-save.
41|.define TISNUM, r22
33|.define TISNIL, r30 42|.define TISNIL, r30
43|.if FPU
34|.define TOBIT, f30 // 2^52 + 2^51. 44|.define TOBIT, f30 // 2^52 + 2^51.
45|.endif
35| 46|
36|// The following temporaries are not saved across C calls, except for RA. 47|// The following temporaries are not saved across C calls, except for RA.
37|.define RA, r23 // Callee-save. 48|.define RA, r23 // Callee-save.
@@ -46,7 +57,7 @@
46|.define TMP2, r14 57|.define TMP2, r14
47|.define TMP3, r15 58|.define TMP3, r15
48| 59|
49|// Calling conventions. 60|// MIPS o32 calling convention.
50|.define CFUNCADDR, r25 61|.define CFUNCADDR, r25
51|.define CARG1, r4 62|.define CARG1, r4
52|.define CARG2, r5 63|.define CARG2, r5
@@ -56,13 +67,33 @@
56|.define CRET1, r2 67|.define CRET1, r2
57|.define CRET2, r3 68|.define CRET2, r3
58| 69|
70|.if ENDIAN_LE
71|.define SFRETLO, CRET1
72|.define SFRETHI, CRET2
73|.define SFARG1LO, CARG1
74|.define SFARG1HI, CARG2
75|.define SFARG2LO, CARG3
76|.define SFARG2HI, CARG4
77|.else
78|.define SFRETLO, CRET2
79|.define SFRETHI, CRET1
80|.define SFARG1LO, CARG2
81|.define SFARG1HI, CARG1
82|.define SFARG2LO, CARG4
83|.define SFARG2HI, CARG3
84|.endif
85|
86|.if FPU
59|.define FARG1, f12 87|.define FARG1, f12
60|.define FARG2, f14 88|.define FARG2, f14
61| 89|
62|.define FRET1, f0 90|.define FRET1, f0
63|.define FRET2, f2 91|.define FRET2, f2
92|.endif
64| 93|
65|// Stack layout while in interpreter. Must match with lj_frame.h. 94|// Stack layout while in interpreter. Must match with lj_frame.h.
95|.if FPU // MIPS32 hard-float.
96|
66|.define CFRAME_SPACE, 112 // Delta for sp. 97|.define CFRAME_SPACE, 112 // Delta for sp.
67| 98|
68|.define SAVE_ERRF, 124(sp) // 32 bit C frame info. 99|.define SAVE_ERRF, 124(sp) // 32 bit C frame info.
@@ -72,6 +103,20 @@
72|//----- 8 byte aligned, ^^^^ 16 byte register save area, owned by interpreter. 103|//----- 8 byte aligned, ^^^^ 16 byte register save area, owned by interpreter.
73|.define SAVE_GPR_, 72 // .. 72+10*4: 32 bit GPR saves. 104|.define SAVE_GPR_, 72 // .. 72+10*4: 32 bit GPR saves.
74|.define SAVE_FPR_, 24 // .. 24+6*8: 64 bit FPR saves. 105|.define SAVE_FPR_, 24 // .. 24+6*8: 64 bit FPR saves.
106|
107|.else // MIPS32 soft-float
108|
109|.define CFRAME_SPACE, 64 // Delta for sp.
110|
111|.define SAVE_ERRF, 76(sp) // 32 bit C frame info.
112|.define SAVE_NRES, 72(sp)
113|.define SAVE_CFRAME, 68(sp)
114|.define SAVE_L, 64(sp)
115|//----- 8 byte aligned, ^^^^ 16 byte register save area, owned by interpreter.
116|.define SAVE_GPR_, 24 // .. 24+10*4: 32 bit GPR saves.
117|
118|.endif
119|
75|.define SAVE_PC, 20(sp) 120|.define SAVE_PC, 20(sp)
76|.define ARG5, 16(sp) 121|.define ARG5, 16(sp)
77|.define CSAVE_4, 12(sp) 122|.define CSAVE_4, 12(sp)
@@ -83,43 +128,45 @@
83|.define ARG5_OFS, 16 128|.define ARG5_OFS, 16
84|.define SAVE_MULTRES, ARG5 129|.define SAVE_MULTRES, ARG5
85| 130|
131|//-----------------------------------------------------------------------
132|
86|.macro saveregs 133|.macro saveregs
87| addiu sp, sp, -CFRAME_SPACE 134| addiu sp, sp, -CFRAME_SPACE
88| sw ra, SAVE_GPR_+9*4(sp) 135| sw ra, SAVE_GPR_+9*4(sp)
89| sw r30, SAVE_GPR_+8*4(sp) 136| sw r30, SAVE_GPR_+8*4(sp)
90| sdc1 f30, SAVE_FPR_+5*8(sp) 137| .FPU sdc1 f30, SAVE_FPR_+5*8(sp)
91| sw r23, SAVE_GPR_+7*4(sp) 138| sw r23, SAVE_GPR_+7*4(sp)
92| sw r22, SAVE_GPR_+6*4(sp) 139| sw r22, SAVE_GPR_+6*4(sp)
93| sdc1 f28, SAVE_FPR_+4*8(sp) 140| .FPU sdc1 f28, SAVE_FPR_+4*8(sp)
94| sw r21, SAVE_GPR_+5*4(sp) 141| sw r21, SAVE_GPR_+5*4(sp)
95| sw r20, SAVE_GPR_+4*4(sp) 142| sw r20, SAVE_GPR_+4*4(sp)
96| sdc1 f26, SAVE_FPR_+3*8(sp) 143| .FPU sdc1 f26, SAVE_FPR_+3*8(sp)
97| sw r19, SAVE_GPR_+3*4(sp) 144| sw r19, SAVE_GPR_+3*4(sp)
98| sw r18, SAVE_GPR_+2*4(sp) 145| sw r18, SAVE_GPR_+2*4(sp)
99| sdc1 f24, SAVE_FPR_+2*8(sp) 146| .FPU sdc1 f24, SAVE_FPR_+2*8(sp)
100| sw r17, SAVE_GPR_+1*4(sp) 147| sw r17, SAVE_GPR_+1*4(sp)
101| sw r16, SAVE_GPR_+0*4(sp) 148| sw r16, SAVE_GPR_+0*4(sp)
102| sdc1 f22, SAVE_FPR_+1*8(sp) 149| .FPU sdc1 f22, SAVE_FPR_+1*8(sp)
103| sdc1 f20, SAVE_FPR_+0*8(sp) 150| .FPU sdc1 f20, SAVE_FPR_+0*8(sp)
104|.endmacro 151|.endmacro
105| 152|
106|.macro restoreregs_ret 153|.macro restoreregs_ret
107| lw ra, SAVE_GPR_+9*4(sp) 154| lw ra, SAVE_GPR_+9*4(sp)
108| lw r30, SAVE_GPR_+8*4(sp) 155| lw r30, SAVE_GPR_+8*4(sp)
109| ldc1 f30, SAVE_FPR_+5*8(sp) 156| .FPU ldc1 f30, SAVE_FPR_+5*8(sp)
110| lw r23, SAVE_GPR_+7*4(sp) 157| lw r23, SAVE_GPR_+7*4(sp)
111| lw r22, SAVE_GPR_+6*4(sp) 158| lw r22, SAVE_GPR_+6*4(sp)
112| ldc1 f28, SAVE_FPR_+4*8(sp) 159| .FPU ldc1 f28, SAVE_FPR_+4*8(sp)
113| lw r21, SAVE_GPR_+5*4(sp) 160| lw r21, SAVE_GPR_+5*4(sp)
114| lw r20, SAVE_GPR_+4*4(sp) 161| lw r20, SAVE_GPR_+4*4(sp)
115| ldc1 f26, SAVE_FPR_+3*8(sp) 162| .FPU ldc1 f26, SAVE_FPR_+3*8(sp)
116| lw r19, SAVE_GPR_+3*4(sp) 163| lw r19, SAVE_GPR_+3*4(sp)
117| lw r18, SAVE_GPR_+2*4(sp) 164| lw r18, SAVE_GPR_+2*4(sp)
118| ldc1 f24, SAVE_FPR_+2*8(sp) 165| .FPU ldc1 f24, SAVE_FPR_+2*8(sp)
119| lw r17, SAVE_GPR_+1*4(sp) 166| lw r17, SAVE_GPR_+1*4(sp)
120| lw r16, SAVE_GPR_+0*4(sp) 167| lw r16, SAVE_GPR_+0*4(sp)
121| ldc1 f22, SAVE_FPR_+1*8(sp) 168| .FPU ldc1 f22, SAVE_FPR_+1*8(sp)
122| ldc1 f20, SAVE_FPR_+0*8(sp) 169| .FPU ldc1 f20, SAVE_FPR_+0*8(sp)
123| jr ra 170| jr ra
124| addiu sp, sp, CFRAME_SPACE 171| addiu sp, sp, CFRAME_SPACE
125|.endmacro 172|.endmacro
@@ -138,11 +185,12 @@
138|.type NODE, Node 185|.type NODE, Node
139|.type NARGS8, int 186|.type NARGS8, int
140|.type TRACE, GCtrace 187|.type TRACE, GCtrace
188|.type SBUF, SBuf
141| 189|
142|//----------------------------------------------------------------------- 190|//-----------------------------------------------------------------------
143| 191|
144|// Trap for not-yet-implemented parts. 192|// Trap for not-yet-implemented parts.
145|.macro NYI; .long 0xf0f0f0f0; .endmacro 193|.macro NYI; .long 0xec1cf0f0; .endmacro
146| 194|
147|// Macros to mark delay slots. 195|// Macros to mark delay slots.
148|.macro ., a; a; .endmacro 196|.macro ., a; a; .endmacro
@@ -152,13 +200,23 @@
152|//----------------------------------------------------------------------- 200|//-----------------------------------------------------------------------
153| 201|
154|// Endian-specific defines. 202|// Endian-specific defines.
155|.define FRAME_PC, LJ_ENDIAN_SELECT(-4,-8) 203|.if ENDIAN_LE
156|.define FRAME_FUNC, LJ_ENDIAN_SELECT(-8,-4) 204|.define FRAME_PC, -4
157|.define HI, LJ_ENDIAN_SELECT(4,0) 205|.define FRAME_FUNC, -8
158|.define LO, LJ_ENDIAN_SELECT(0,4) 206|.define HI, 4
159|.define OFS_RD, LJ_ENDIAN_SELECT(2,0) 207|.define LO, 0
160|.define OFS_RA, LJ_ENDIAN_SELECT(1,2) 208|.define OFS_RD, 2
161|.define OFS_OP, LJ_ENDIAN_SELECT(0,3) 209|.define OFS_RA, 1
210|.define OFS_OP, 0
211|.else
212|.define FRAME_PC, -8
213|.define FRAME_FUNC, -4
214|.define HI, 0
215|.define LO, 4
216|.define OFS_RD, 0
217|.define OFS_RA, 2
218|.define OFS_OP, 3
219|.endif
162| 220|
163|// Instruction decode. 221|// Instruction decode.
164|.macro decode_OP1, dst, ins; andi dst, ins, 0xff; .endmacro 222|.macro decode_OP1, dst, ins; andi dst, ins, 0xff; .endmacro
@@ -353,9 +411,11 @@ static void build_subroutines(BuildCtx *ctx)
353 |. sll TMP2, TMP2, 3 411 |. sll TMP2, TMP2, 3
354 |1: 412 |1:
355 | addiu TMP1, TMP1, -8 413 | addiu TMP1, TMP1, -8
356 | ldc1 f0, 0(RA) 414 | lw SFRETHI, HI(RA)
415 | lw SFRETLO, LO(RA)
357 | addiu RA, RA, 8 416 | addiu RA, RA, 8
358 | sdc1 f0, 0(BASE) 417 | sw SFRETHI, HI(BASE)
418 | sw SFRETLO, LO(BASE)
359 | bnez TMP1, <1 419 | bnez TMP1, <1
360 |. addiu BASE, BASE, 8 420 |. addiu BASE, BASE, 8
361 | 421 |
@@ -424,15 +484,16 @@ static void build_subroutines(BuildCtx *ctx)
424 | and sp, CARG1, AT 484 | and sp, CARG1, AT
425 |->vm_unwind_ff_eh: // Landing pad for external unwinder. 485 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
426 | lw L, SAVE_L 486 | lw L, SAVE_L
427 | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 487 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
488 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
428 | li TISNIL, LJ_TNIL 489 | li TISNIL, LJ_TNIL
429 | lw BASE, L->base 490 | lw BASE, L->base
430 | lw DISPATCH, L->glref // Setup pointer to dispatch table. 491 | lw DISPATCH, L->glref // Setup pointer to dispatch table.
431 | mtc1 TMP3, TOBIT 492 | .FPU mtc1 TMP3, TOBIT
432 | li TMP1, LJ_TFALSE 493 | li TMP1, LJ_TFALSE
433 | li_vmstate INTERP 494 | li_vmstate INTERP
434 | lw PC, FRAME_PC(BASE) // Fetch PC of previous frame. 495 | lw PC, FRAME_PC(BASE) // Fetch PC of previous frame.
435 | cvt.d.s TOBIT, TOBIT 496 | .FPU cvt.d.s TOBIT, TOBIT
436 | addiu RA, BASE, -8 // Results start at BASE-8. 497 | addiu RA, BASE, -8 // Results start at BASE-8.
437 | addiu DISPATCH, DISPATCH, GG_G2DISP 498 | addiu DISPATCH, DISPATCH, GG_G2DISP
438 | sw TMP1, HI(RA) // Prepend false to error message. 499 | sw TMP1, HI(RA) // Prepend false to error message.
@@ -440,6 +501,10 @@ static void build_subroutines(BuildCtx *ctx)
440 | b ->vm_returnc 501 | b ->vm_returnc
441 |. li RD, 16 // 2 results: false + error message. 502 |. li RD, 16 // 2 results: false + error message.
442 | 503 |
504 |->vm_unwind_stub: // Jump to exit stub from unwinder.
505 | jr CARG1
506 |. move ra, CARG2
507 |
443 |//----------------------------------------------------------------------- 508 |//-----------------------------------------------------------------------
444 |//-- Grow stack for calls ----------------------------------------------- 509 |//-- Grow stack for calls -----------------------------------------------
445 |//----------------------------------------------------------------------- 510 |//-----------------------------------------------------------------------
@@ -486,21 +551,23 @@ static void build_subroutines(BuildCtx *ctx)
486 | addiu DISPATCH, DISPATCH, GG_G2DISP 551 | addiu DISPATCH, DISPATCH, GG_G2DISP
487 | sw r0, SAVE_NRES 552 | sw r0, SAVE_NRES
488 | sw r0, SAVE_ERRF 553 | sw r0, SAVE_ERRF
489 | sw TMP0, L->cframe 554 | sw CARG1, SAVE_PC // Any value outside of bytecode is ok.
490 | sw r0, SAVE_CFRAME 555 | sw r0, SAVE_CFRAME
491 | beqz TMP1, >3 556 | beqz TMP1, >3
492 |. sw CARG1, SAVE_PC // Any value outside of bytecode is ok. 557 |. sw TMP0, L->cframe
493 | 558 |
494 | // Resume after yield (like a return). 559 | // Resume after yield (like a return).
560 | sw L, DISPATCH_GL(cur_L)(DISPATCH)
495 | move RA, BASE 561 | move RA, BASE
496 | lw BASE, L->base 562 | lw BASE, L->base
563 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
497 | lw TMP1, L->top 564 | lw TMP1, L->top
498 | lw PC, FRAME_PC(BASE) 565 | lw PC, FRAME_PC(BASE)
499 | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 566 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
500 | subu RD, TMP1, BASE 567 | subu RD, TMP1, BASE
501 | mtc1 TMP3, TOBIT 568 | .FPU mtc1 TMP3, TOBIT
502 | sb r0, L->status 569 | sb r0, L->status
503 | cvt.d.s TOBIT, TOBIT 570 | .FPU cvt.d.s TOBIT, TOBIT
504 | li_vmstate INTERP 571 | li_vmstate INTERP
505 | addiu RD, RD, 8 572 | addiu RD, RD, 8
506 | st_vmstate 573 | st_vmstate
@@ -525,25 +592,27 @@ static void build_subroutines(BuildCtx *ctx)
525 | 592 |
526 |1: // Entry point for vm_pcall above (PC = ftype). 593 |1: // Entry point for vm_pcall above (PC = ftype).
527 | lw TMP1, L:CARG1->cframe 594 | lw TMP1, L:CARG1->cframe
528 | sw CARG3, SAVE_NRES
529 | move L, CARG1 595 | move L, CARG1
530 | sw CARG1, SAVE_L 596 | sw CARG3, SAVE_NRES
531 | move BASE, CARG2
532 | sw sp, L->cframe // Add our C frame to cframe chain.
533 | lw DISPATCH, L->glref // Setup pointer to dispatch table. 597 | lw DISPATCH, L->glref // Setup pointer to dispatch table.
598 | sw CARG1, SAVE_L
599 | move BASE, CARG2
600 | addiu DISPATCH, DISPATCH, GG_G2DISP
534 | sw CARG1, SAVE_PC // Any value outside of bytecode is ok. 601 | sw CARG1, SAVE_PC // Any value outside of bytecode is ok.
535 | sw TMP1, SAVE_CFRAME 602 | sw TMP1, SAVE_CFRAME
536 | addiu DISPATCH, DISPATCH, GG_G2DISP 603 | sw sp, L->cframe // Add our C frame to cframe chain.
537 | 604 |
538 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype). 605 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
606 | sw L, DISPATCH_GL(cur_L)(DISPATCH)
539 | lw TMP2, L->base // TMP2 = old base (used in vmeta_call). 607 | lw TMP2, L->base // TMP2 = old base (used in vmeta_call).
540 | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 608 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
609 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
541 | lw TMP1, L->top 610 | lw TMP1, L->top
542 | mtc1 TMP3, TOBIT 611 | .FPU mtc1 TMP3, TOBIT
543 | addu PC, PC, BASE 612 | addu PC, PC, BASE
544 | subu NARGS8:RC, TMP1, BASE 613 | subu NARGS8:RC, TMP1, BASE
545 | subu PC, PC, TMP2 // PC = frame delta + frame type 614 | subu PC, PC, TMP2 // PC = frame delta + frame type
546 | cvt.d.s TOBIT, TOBIT 615 | .FPU cvt.d.s TOBIT, TOBIT
547 | li_vmstate INTERP 616 | li_vmstate INTERP
548 | li TISNIL, LJ_TNIL 617 | li TISNIL, LJ_TNIL
549 | st_vmstate 618 | st_vmstate
@@ -566,20 +635,21 @@ static void build_subroutines(BuildCtx *ctx)
566 | lw TMP0, L:CARG1->stack 635 | lw TMP0, L:CARG1->stack
567 | sw CARG1, SAVE_L 636 | sw CARG1, SAVE_L
568 | lw TMP1, L->top 637 | lw TMP1, L->top
638 | lw DISPATCH, L->glref // Setup pointer to dispatch table.
569 | sw CARG1, SAVE_PC // Any value outside of bytecode is ok. 639 | sw CARG1, SAVE_PC // Any value outside of bytecode is ok.
570 | subu TMP0, TMP0, TMP1 // Compute -savestack(L, L->top). 640 | subu TMP0, TMP0, TMP1 // Compute -savestack(L, L->top).
571 | lw TMP1, L->cframe 641 | lw TMP1, L->cframe
572 | sw sp, L->cframe // Add our C frame to cframe chain. 642 | addiu DISPATCH, DISPATCH, GG_G2DISP
573 | sw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame. 643 | sw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame.
574 | sw r0, SAVE_ERRF // No error function. 644 | sw r0, SAVE_ERRF // No error function.
575 | move CFUNCADDR, CARG4 645 | sw TMP1, SAVE_CFRAME
646 | sw sp, L->cframe // Add our C frame to cframe chain.
647 | sw L, DISPATCH_GL(cur_L)(DISPATCH)
576 | jalr CARG4 // (lua_State *L, lua_CFunction func, void *ud) 648 | jalr CARG4 // (lua_State *L, lua_CFunction func, void *ud)
577 |. sw TMP1, SAVE_CFRAME 649 |. move CFUNCADDR, CARG4
578 | move BASE, CRET1 650 | move BASE, CRET1
579 | lw DISPATCH, L->glref // Setup pointer to dispatch table.
580 | li PC, FRAME_CP
581 | bnez CRET1, <3 // Else continue with the call. 651 | bnez CRET1, <3 // Else continue with the call.
582 |. addiu DISPATCH, DISPATCH, GG_G2DISP 652 |. li PC, FRAME_CP
583 | b ->vm_leave_cp // No base? Just remove C frame. 653 | b ->vm_leave_cp // No base? Just remove C frame.
584 |. nop 654 |. nop
585 | 655 |
@@ -624,7 +694,8 @@ static void build_subroutines(BuildCtx *ctx)
624 |->cont_cat: // RA = resultptr, RB = meta base 694 |->cont_cat: // RA = resultptr, RB = meta base
625 | lw INS, -4(PC) 695 | lw INS, -4(PC)
626 | addiu CARG2, RB, -16 696 | addiu CARG2, RB, -16
627 | ldc1 f0, 0(RA) 697 | lw SFRETHI, HI(RA)
698 | lw SFRETLO, LO(RA)
628 | decode_RB8a MULTRES, INS 699 | decode_RB8a MULTRES, INS
629 | decode_RA8a RA, INS 700 | decode_RA8a RA, INS
630 | decode_RB8b MULTRES 701 | decode_RB8b MULTRES
@@ -632,11 +703,13 @@ static void build_subroutines(BuildCtx *ctx)
632 | addu TMP1, BASE, MULTRES 703 | addu TMP1, BASE, MULTRES
633 | sw BASE, L->base 704 | sw BASE, L->base
634 | subu CARG3, CARG2, TMP1 705 | subu CARG3, CARG2, TMP1
706 | sw SFRETHI, HI(CARG2)
635 | bne TMP1, CARG2, ->BC_CAT_Z 707 | bne TMP1, CARG2, ->BC_CAT_Z
636 |. sdc1 f0, 0(CARG2) 708 |. sw SFRETLO, LO(CARG2)
637 | addu RA, BASE, RA 709 | addu RA, BASE, RA
710 | sw SFRETHI, HI(RA)
638 | b ->cont_nop 711 | b ->cont_nop
639 |. sdc1 f0, 0(RA) 712 |. sw SFRETLO, LO(RA)
640 | 713 |
641 |//-- Table indexing metamethods ----------------------------------------- 714 |//-- Table indexing metamethods -----------------------------------------
642 | 715 |
@@ -659,10 +732,9 @@ static void build_subroutines(BuildCtx *ctx)
659 |. sw TMP1, HI(CARG3) 732 |. sw TMP1, HI(CARG3)
660 | 733 |
661 |->vmeta_tgetb: // TMP0 = index 734 |->vmeta_tgetb: // TMP0 = index
662 | mtc1 TMP0, f0
663 | cvt.d.w f0, f0
664 | addiu CARG3, DISPATCH, DISPATCH_GL(tmptv) 735 | addiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
665 | sdc1 f0, 0(CARG3) 736 | sw TMP0, LO(CARG3)
737 | sw TISNUM, HI(CARG3)
666 | 738 |
667 |->vmeta_tgetv: 739 |->vmeta_tgetv:
668 |1: 740 |1:
@@ -674,9 +746,11 @@ static void build_subroutines(BuildCtx *ctx)
674 | // Returns TValue * (finished) or NULL (metamethod). 746 | // Returns TValue * (finished) or NULL (metamethod).
675 | beqz CRET1, >3 747 | beqz CRET1, >3
676 |. addiu TMP1, BASE, -FRAME_CONT 748 |. addiu TMP1, BASE, -FRAME_CONT
677 | ldc1 f0, 0(CRET1) 749 | lw SFARG1HI, HI(CRET1)
750 | lw SFARG2HI, LO(CRET1)
678 | ins_next1 751 | ins_next1
679 | sdc1 f0, 0(RA) 752 | sw SFARG1HI, HI(RA)
753 | sw SFARG2HI, LO(RA)
680 | ins_next2 754 | ins_next2
681 | 755 |
682 |3: // Call __index metamethod. 756 |3: // Call __index metamethod.
@@ -688,6 +762,17 @@ static void build_subroutines(BuildCtx *ctx)
688 | b ->vm_call_dispatch_f 762 | b ->vm_call_dispatch_f
689 |. li NARGS8:RC, 16 // 2 args for func(t, k). 763 |. li NARGS8:RC, 16 // 2 args for func(t, k).
690 | 764 |
765 |->vmeta_tgetr:
766 | load_got lj_tab_getinth
767 | call_intern lj_tab_getinth // (GCtab *t, int32_t key)
768 |. nop
769 | // Returns cTValue * or NULL.
770 | beqz CRET1, ->BC_TGETR_Z
771 |. move SFARG2HI, TISNIL
772 | lw SFARG2HI, HI(CRET1)
773 | b ->BC_TGETR_Z
774 |. lw SFARG2LO, LO(CRET1)
775 |
691 |//----------------------------------------------------------------------- 776 |//-----------------------------------------------------------------------
692 | 777 |
693 |->vmeta_tsets1: 778 |->vmeta_tsets1:
@@ -709,10 +794,9 @@ static void build_subroutines(BuildCtx *ctx)
709 |. sw TMP1, HI(CARG3) 794 |. sw TMP1, HI(CARG3)
710 | 795 |
711 |->vmeta_tsetb: // TMP0 = index 796 |->vmeta_tsetb: // TMP0 = index
712 | mtc1 TMP0, f0
713 | cvt.d.w f0, f0
714 | addiu CARG3, DISPATCH, DISPATCH_GL(tmptv) 797 | addiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
715 | sdc1 f0, 0(CARG3) 798 | sw TMP0, LO(CARG3)
799 | sw TISNUM, HI(CARG3)
716 | 800 |
717 |->vmeta_tsetv: 801 |->vmeta_tsetv:
718 |1: 802 |1:
@@ -722,11 +806,13 @@ static void build_subroutines(BuildCtx *ctx)
722 | call_intern lj_meta_tset // (lua_State *L, TValue *o, TValue *k) 806 | call_intern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
723 |. move CARG1, L 807 |. move CARG1, L
724 | // Returns TValue * (finished) or NULL (metamethod). 808 | // Returns TValue * (finished) or NULL (metamethod).
809 | lw SFARG1HI, HI(RA)
725 | beqz CRET1, >3 810 | beqz CRET1, >3
726 |. ldc1 f0, 0(RA) 811 |. lw SFARG1LO, LO(RA)
727 | // NOBARRIER: lj_meta_tset ensures the table is not black. 812 | // NOBARRIER: lj_meta_tset ensures the table is not black.
728 | ins_next1 813 | ins_next1
729 | sdc1 f0, 0(CRET1) 814 | sw SFARG1HI, HI(CRET1)
815 | sw SFARG1LO, LO(CRET1)
730 | ins_next2 816 | ins_next2
731 | 817 |
732 |3: // Call __newindex metamethod. 818 |3: // Call __newindex metamethod.
@@ -736,14 +822,27 @@ static void build_subroutines(BuildCtx *ctx)
736 | sw PC, -16+HI(BASE) // [cont|PC] 822 | sw PC, -16+HI(BASE) // [cont|PC]
737 | subu PC, BASE, TMP1 823 | subu PC, BASE, TMP1
738 | lw LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here. 824 | lw LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
739 | sdc1 f0, 16(BASE) // Copy value to third argument. 825 | sw SFARG1HI, 16+HI(BASE) // Copy value to third argument.
826 | sw SFARG1LO, 16+LO(BASE)
740 | b ->vm_call_dispatch_f 827 | b ->vm_call_dispatch_f
741 |. li NARGS8:RC, 24 // 3 args for func(t, k, v) 828 |. li NARGS8:RC, 24 // 3 args for func(t, k, v)
742 | 829 |
830 |->vmeta_tsetr:
831 | load_got lj_tab_setinth
832 | sw BASE, L->base
833 | sw PC, SAVE_PC
834 | call_intern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
835 |. move CARG1, L
836 | // Returns TValue *.
837 | b ->BC_TSETR_Z
838 |. nop
839 |
743 |//-- Comparison metamethods --------------------------------------------- 840 |//-- Comparison metamethods ---------------------------------------------
744 | 841 |
745 |->vmeta_comp: 842 |->vmeta_comp:
746 | // CARG2, CARG3 are already set by BC_ISLT/BC_ISGE/BC_ISLE/BC_ISGT. 843 | // RA/RD point to o1/o2.
844 | move CARG2, RA
845 | move CARG3, RD
747 | load_got lj_meta_comp 846 | load_got lj_meta_comp
748 | addiu PC, PC, -4 847 | addiu PC, PC, -4
749 | sw BASE, L->base 848 | sw BASE, L->base
@@ -769,11 +868,13 @@ static void build_subroutines(BuildCtx *ctx)
769 | 868 |
770 |->cont_ra: // RA = resultptr 869 |->cont_ra: // RA = resultptr
771 | lbu TMP1, -4+OFS_RA(PC) 870 | lbu TMP1, -4+OFS_RA(PC)
772 | ldc1 f0, 0(RA) 871 | lw SFRETHI, HI(RA)
872 | lw SFRETLO, LO(RA)
773 | sll TMP1, TMP1, 3 873 | sll TMP1, TMP1, 3
774 | addu TMP1, BASE, TMP1 874 | addu TMP1, BASE, TMP1
875 | sw SFRETHI, HI(TMP1)
775 | b ->cont_nop 876 | b ->cont_nop
776 |. sdc1 f0, 0(TMP1) 877 |. sw SFRETLO, LO(TMP1)
777 | 878 |
778 |->cont_condt: // RA = resultptr 879 |->cont_condt: // RA = resultptr
779 | lw TMP0, HI(RA) 880 | lw TMP0, HI(RA)
@@ -788,8 +889,11 @@ static void build_subroutines(BuildCtx *ctx)
788 |. addiu TMP2, AT, -1 // Branch if result is false. 889 |. addiu TMP2, AT, -1 // Branch if result is false.
789 | 890 |
790 |->vmeta_equal: 891 |->vmeta_equal:
791 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV. 892 | // SFARG1LO/SFARG2LO point to o1/o2. TMP0 is set to 0/1.
792 | load_got lj_meta_equal 893 | load_got lj_meta_equal
894 | move CARG2, SFARG1LO
895 | move CARG3, SFARG2LO
896 | move CARG4, TMP0
793 | addiu PC, PC, -4 897 | addiu PC, PC, -4
794 | sw BASE, L->base 898 | sw BASE, L->base
795 | sw PC, SAVE_PC 899 | sw PC, SAVE_PC
@@ -813,17 +917,31 @@ static void build_subroutines(BuildCtx *ctx)
813 |. nop 917 |. nop
814 |.endif 918 |.endif
815 | 919 |
920 |->vmeta_istype:
921 | load_got lj_meta_istype
922 | addiu PC, PC, -4
923 | sw BASE, L->base
924 | srl CARG2, RA, 3
925 | srl CARG3, RD, 3
926 | sw PC, SAVE_PC
927 | call_intern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
928 |. move CARG1, L
929 | b ->cont_nop
930 |. nop
931 |
816 |//-- Arithmetic metamethods --------------------------------------------- 932 |//-- Arithmetic metamethods ---------------------------------------------
817 | 933 |
818 |->vmeta_unm: 934 |->vmeta_unm:
819 | move CARG4, CARG3 935 | move RC, RB
820 | 936 |
821 |->vmeta_arith: 937 |->vmeta_arith:
822 | load_got lj_meta_arith 938 | load_got lj_meta_arith
823 | decode_OP1 TMP0, INS 939 | decode_OP1 TMP0, INS
824 | sw BASE, L->base 940 | sw BASE, L->base
825 | sw PC, SAVE_PC
826 | move CARG2, RA 941 | move CARG2, RA
942 | sw PC, SAVE_PC
943 | move CARG3, RB
944 | move CARG4, RC
827 | sw TMP0, ARG5 945 | sw TMP0, ARG5
828 | call_intern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op) 946 | call_intern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
829 |. move CARG1, L 947 |. move CARG1, L
@@ -931,40 +1049,52 @@ static void build_subroutines(BuildCtx *ctx)
931 | 1049 |
932 |.macro .ffunc_1, name 1050 |.macro .ffunc_1, name
933 |->ff_ .. name: 1051 |->ff_ .. name:
1052 | lw SFARG1HI, HI(BASE)
934 | beqz NARGS8:RC, ->fff_fallback 1053 | beqz NARGS8:RC, ->fff_fallback
935 |. lw CARG3, HI(BASE) 1054 |. lw SFARG1LO, LO(BASE)
936 | lw CARG1, LO(BASE)
937 |.endmacro 1055 |.endmacro
938 | 1056 |
939 |.macro .ffunc_2, name 1057 |.macro .ffunc_2, name
940 |->ff_ .. name: 1058 |->ff_ .. name:
941 | sltiu AT, NARGS8:RC, 16 1059 | sltiu AT, NARGS8:RC, 16
942 | lw CARG3, HI(BASE) 1060 | lw SFARG1HI, HI(BASE)
943 | bnez AT, ->fff_fallback 1061 | bnez AT, ->fff_fallback
944 |. lw CARG4, 8+HI(BASE) 1062 |. lw SFARG2HI, 8+HI(BASE)
945 | lw CARG1, LO(BASE) 1063 | lw SFARG1LO, LO(BASE)
946 | lw CARG2, 8+LO(BASE) 1064 | lw SFARG2LO, 8+LO(BASE)
947 |.endmacro 1065 |.endmacro
948 | 1066 |
949 |.macro .ffunc_n, name // Caveat: has delay slot! 1067 |.macro .ffunc_n, name // Caveat: has delay slot!
950 |->ff_ .. name: 1068 |->ff_ .. name:
951 | lw CARG3, HI(BASE) 1069 | lw SFARG1HI, HI(BASE)
1070 |.if FPU
1071 | ldc1 FARG1, 0(BASE)
1072 |.else
1073 | lw SFARG1LO, LO(BASE)
1074 |.endif
952 | beqz NARGS8:RC, ->fff_fallback 1075 | beqz NARGS8:RC, ->fff_fallback
953 |. ldc1 FARG1, 0(BASE) 1076 |. sltiu AT, SFARG1HI, LJ_TISNUM
954 | sltiu AT, CARG3, LJ_TISNUM
955 | beqz AT, ->fff_fallback 1077 | beqz AT, ->fff_fallback
956 |.endmacro 1078 |.endmacro
957 | 1079 |
958 |.macro .ffunc_nn, name // Caveat: has delay slot! 1080 |.macro .ffunc_nn, name // Caveat: has delay slot!
959 |->ff_ .. name: 1081 |->ff_ .. name:
960 | sltiu AT, NARGS8:RC, 16 1082 | sltiu AT, NARGS8:RC, 16
961 | lw CARG3, HI(BASE) 1083 | lw SFARG1HI, HI(BASE)
962 | bnez AT, ->fff_fallback 1084 | bnez AT, ->fff_fallback
963 |. lw CARG4, 8+HI(BASE) 1085 |. lw SFARG2HI, 8+HI(BASE)
964 | ldc1 FARG1, 0(BASE) 1086 | sltiu TMP0, SFARG1HI, LJ_TISNUM
965 | ldc1 FARG2, 8(BASE) 1087 |.if FPU
966 | sltiu TMP0, CARG3, LJ_TISNUM 1088 | ldc1 FARG1, 0(BASE)
967 | sltiu TMP1, CARG4, LJ_TISNUM 1089 |.else
1090 | lw SFARG1LO, LO(BASE)
1091 |.endif
1092 | sltiu TMP1, SFARG2HI, LJ_TISNUM
1093 |.if FPU
1094 | ldc1 FARG2, 8(BASE)
1095 |.else
1096 | lw SFARG2LO, 8+LO(BASE)
1097 |.endif
968 | and TMP0, TMP0, TMP1 1098 | and TMP0, TMP0, TMP1
969 | beqz TMP0, ->fff_fallback 1099 | beqz TMP0, ->fff_fallback
970 |.endmacro 1100 |.endmacro
@@ -980,53 +1110,55 @@ static void build_subroutines(BuildCtx *ctx)
980 |//-- Base library: checks ----------------------------------------------- 1110 |//-- Base library: checks -----------------------------------------------
981 | 1111 |
982 |.ffunc_1 assert 1112 |.ffunc_1 assert
983 | sltiu AT, CARG3, LJ_TISTRUECOND 1113 | sltiu AT, SFARG1HI, LJ_TISTRUECOND
984 | beqz AT, ->fff_fallback 1114 | beqz AT, ->fff_fallback
985 |. addiu RA, BASE, -8 1115 |. addiu RA, BASE, -8
986 | lw PC, FRAME_PC(BASE) 1116 | lw PC, FRAME_PC(BASE)
987 | addiu RD, NARGS8:RC, 8 // Compute (nresults+1)*8. 1117 | addiu RD, NARGS8:RC, 8 // Compute (nresults+1)*8.
988 | addu TMP2, RA, NARGS8:RC 1118 | addu TMP2, RA, NARGS8:RC
989 | sw CARG3, HI(RA) 1119 | sw SFARG1HI, HI(RA)
990 | addiu TMP1, BASE, 8 1120 | addiu TMP1, BASE, 8
991 | beq BASE, TMP2, ->fff_res // Done if exactly 1 argument. 1121 | beq BASE, TMP2, ->fff_res // Done if exactly 1 argument.
992 |. sw CARG1, LO(RA) 1122 |. sw SFARG1LO, LO(RA)
993 |1: 1123 |1:
994 | ldc1 f0, 0(TMP1) 1124 | lw SFRETHI, HI(TMP1)
995 | sdc1 f0, -8(TMP1) 1125 | lw SFRETLO, LO(TMP1)
1126 | sw SFRETHI, -8+HI(TMP1)
1127 | sw SFRETLO, -8+LO(TMP1)
996 | bne TMP1, TMP2, <1 1128 | bne TMP1, TMP2, <1
997 |. addiu TMP1, TMP1, 8 1129 |. addiu TMP1, TMP1, 8
998 | b ->fff_res 1130 | b ->fff_res
999 |. nop 1131 |. nop
1000 | 1132 |
1001 |.ffunc type 1133 |.ffunc type
1002 | lw CARG3, HI(BASE) 1134 | lw SFARG1HI, HI(BASE)
1003 | li TMP1, LJ_TISNUM
1004 | beqz NARGS8:RC, ->fff_fallback 1135 | beqz NARGS8:RC, ->fff_fallback
1005 |. sltiu TMP0, CARG3, LJ_TISNUM 1136 |. sltiu TMP0, SFARG1HI, LJ_TISNUM
1006 | movz TMP1, CARG3, TMP0 1137 | movn SFARG1HI, TISNUM, TMP0
1007 | not TMP1, TMP1 1138 | not TMP1, SFARG1HI
1008 | sll TMP1, TMP1, 3 1139 | sll TMP1, TMP1, 3
1009 | addu TMP1, CFUNC:RB, TMP1 1140 | addu TMP1, CFUNC:RB, TMP1
1010 | b ->fff_resn 1141 | lw SFARG1HI, CFUNC:TMP1->upvalue[0].u32.hi
1011 |. ldc1 FRET1, CFUNC:TMP1->upvalue 1142 | b ->fff_restv
1143 |. lw SFARG1LO, CFUNC:TMP1->upvalue[0].u32.lo
1012 | 1144 |
1013 |//-- Base library: getters and setters --------------------------------- 1145 |//-- Base library: getters and setters ---------------------------------
1014 | 1146 |
1015 |.ffunc_1 getmetatable 1147 |.ffunc_1 getmetatable
1016 | li AT, LJ_TTAB 1148 | li AT, LJ_TTAB
1017 | bne CARG3, AT, >6 1149 | bne SFARG1HI, AT, >6
1018 |. li AT, LJ_TUDATA 1150 |. li AT, LJ_TUDATA
1019 |1: // Field metatable must be at same offset for GCtab and GCudata! 1151 |1: // Field metatable must be at same offset for GCtab and GCudata!
1020 | lw TAB:CARG1, TAB:CARG1->metatable 1152 | lw TAB:SFARG1LO, TAB:SFARG1LO->metatable
1021 |2: 1153 |2:
1022 | lw STR:RC, DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])(DISPATCH) 1154 | lw STR:RC, DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])(DISPATCH)
1023 | beqz TAB:CARG1, ->fff_restv 1155 | beqz TAB:SFARG1LO, ->fff_restv
1024 |. li CARG3, LJ_TNIL 1156 |. li SFARG1HI, LJ_TNIL
1025 | lw TMP0, TAB:CARG1->hmask 1157 | lw TMP0, TAB:SFARG1LO->hmask
1026 | li CARG3, LJ_TTAB // Use metatable as default result. 1158 | li SFARG1HI, LJ_TTAB // Use metatable as default result.
1027 | lw TMP1, STR:RC->hash 1159 | lw TMP1, STR:RC->sid
1028 | lw NODE:TMP2, TAB:CARG1->node 1160 | lw NODE:TMP2, TAB:SFARG1LO->node
1029 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 1161 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
1030 | sll TMP0, TMP1, 5 1162 | sll TMP0, TMP1, 5
1031 | sll TMP1, TMP1, 3 1163 | sll TMP1, TMP1, 3
1032 | subu TMP1, TMP0, TMP1 1164 | subu TMP1, TMP0, TMP1
@@ -1037,7 +1169,7 @@ static void build_subroutines(BuildCtx *ctx)
1037 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2) 1169 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2)
1038 | lw NODE:TMP3, NODE:TMP2->next 1170 | lw NODE:TMP3, NODE:TMP2->next
1039 | bne CARG4, AT, >4 1171 | bne CARG4, AT, >4
1040 |. lw CARG2, offsetof(Node, val)+HI(NODE:TMP2) 1172 |. lw CARG3, offsetof(Node, val)+HI(NODE:TMP2)
1041 | beq TMP0, STR:RC, >5 1173 | beq TMP0, STR:RC, >5
1042 |. lw TMP1, offsetof(Node, val)+LO(NODE:TMP2) 1174 |. lw TMP1, offsetof(Node, val)+LO(NODE:TMP2)
1043 |4: 1175 |4:
@@ -1046,36 +1178,35 @@ static void build_subroutines(BuildCtx *ctx)
1046 | b <3 1178 | b <3
1047 |. nop 1179 |. nop
1048 |5: 1180 |5:
1049 | beq CARG2, TISNIL, ->fff_restv // Ditto for nil value. 1181 | beq CARG3, TISNIL, ->fff_restv // Ditto for nil value.
1050 |. nop 1182 |. nop
1051 | move CARG3, CARG2 // Return value of mt.__metatable. 1183 | move SFARG1HI, CARG3 // Return value of mt.__metatable.
1052 | b ->fff_restv 1184 | b ->fff_restv
1053 |. move CARG1, TMP1 1185 |. move SFARG1LO, TMP1
1054 | 1186 |
1055 |6: 1187 |6:
1056 | beq CARG3, AT, <1 1188 | beq SFARG1HI, AT, <1
1057 |. sltiu TMP0, CARG3, LJ_TISNUM 1189 |. sltu AT, TISNUM, SFARG1HI
1058 | li TMP1, LJ_TISNUM 1190 | movz SFARG1HI, TISNUM, AT
1059 | movz TMP1, CARG3, TMP0 1191 | not TMP1, SFARG1HI
1060 | not TMP1, TMP1
1061 | sll TMP1, TMP1, 2 1192 | sll TMP1, TMP1, 2
1062 | addu TMP1, DISPATCH, TMP1 1193 | addu TMP1, DISPATCH, TMP1
1063 | b <2 1194 | b <2
1064 |. lw TAB:CARG1, DISPATCH_GL(gcroot[GCROOT_BASEMT])(TMP1) 1195 |. lw TAB:SFARG1LO, DISPATCH_GL(gcroot[GCROOT_BASEMT])(TMP1)
1065 | 1196 |
1066 |.ffunc_2 setmetatable 1197 |.ffunc_2 setmetatable
1067 | // Fast path: no mt for table yet and not clearing the mt. 1198 | // Fast path: no mt for table yet and not clearing the mt.
1068 | li AT, LJ_TTAB 1199 | li AT, LJ_TTAB
1069 | bne CARG3, AT, ->fff_fallback 1200 | bne SFARG1HI, AT, ->fff_fallback
1070 |. addiu CARG4, CARG4, -LJ_TTAB 1201 |. addiu SFARG2HI, SFARG2HI, -LJ_TTAB
1071 | lw TAB:TMP1, TAB:CARG1->metatable 1202 | lw TAB:TMP1, TAB:SFARG1LO->metatable
1072 | lbu TMP3, TAB:CARG1->marked 1203 | lbu TMP3, TAB:SFARG1LO->marked
1073 | or AT, CARG4, TAB:TMP1 1204 | or AT, SFARG2HI, TAB:TMP1
1074 | bnez AT, ->fff_fallback 1205 | bnez AT, ->fff_fallback
1075 |. andi AT, TMP3, LJ_GC_BLACK // isblack(table) 1206 |. andi AT, TMP3, LJ_GC_BLACK // isblack(table)
1076 | beqz AT, ->fff_restv 1207 | beqz AT, ->fff_restv
1077 |. sw TAB:CARG2, TAB:CARG1->metatable 1208 |. sw TAB:SFARG2LO, TAB:SFARG1LO->metatable
1078 | barrierback TAB:CARG1, TMP3, TMP0, ->fff_restv 1209 | barrierback TAB:SFARG1LO, TMP3, TMP0, ->fff_restv
1079 | 1210 |
1080 |.ffunc rawget 1211 |.ffunc rawget
1081 | lw CARG4, HI(BASE) 1212 | lw CARG4, HI(BASE)
@@ -1089,90 +1220,89 @@ static void build_subroutines(BuildCtx *ctx)
1089 | call_intern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key) 1220 | call_intern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1090 |. move CARG1, L 1221 |. move CARG1, L
1091 | // Returns cTValue *. 1222 | // Returns cTValue *.
1092 | b ->fff_resn 1223 | lw SFARG1HI, HI(CRET1)
1093 |. ldc1 FRET1, 0(CRET1) 1224 | b ->fff_restv
1225 |. lw SFARG1LO, LO(CRET1)
1094 | 1226 |
1095 |//-- Base library: conversions ------------------------------------------ 1227 |//-- Base library: conversions ------------------------------------------
1096 | 1228 |
1097 |.ffunc tonumber 1229 |.ffunc tonumber
1098 | // Only handles the number case inline (without a base argument). 1230 | // Only handles the number case inline (without a base argument).
1099 | lw CARG1, HI(BASE) 1231 | lw CARG1, HI(BASE)
1100 | xori AT, NARGS8:RC, 8 1232 | xori AT, NARGS8:RC, 8 // Exactly one number argument.
1101 | sltiu CARG1, CARG1, LJ_TISNUM 1233 | sltu TMP0, TISNUM, CARG1
1102 | movn CARG1, r0, AT 1234 | or AT, AT, TMP0
1103 | beqz CARG1, ->fff_fallback // Exactly one number argument. 1235 | bnez AT, ->fff_fallback
1104 |. ldc1 FRET1, 0(BASE) 1236 |. lw SFARG1HI, HI(BASE)
1105 | b ->fff_resn 1237 | b ->fff_restv
1106 |. nop 1238 |. lw SFARG1LO, LO(BASE)
1107 | 1239 |
1108 |.ffunc_1 tostring 1240 |.ffunc_1 tostring
1109 | // Only handles the string or number case inline. 1241 | // Only handles the string or number case inline.
1110 | li AT, LJ_TSTR 1242 | li AT, LJ_TSTR
1111 | // A __tostring method in the string base metatable is ignored. 1243 | // A __tostring method in the string base metatable is ignored.
1112 | beq CARG3, AT, ->fff_restv // String key? 1244 | beq SFARG1HI, AT, ->fff_restv // String key?
1113 | // Handle numbers inline, unless a number base metatable is present. 1245 | // Handle numbers inline, unless a number base metatable is present.
1114 |. lw TMP1, DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])(DISPATCH) 1246 |. lw TMP1, DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])(DISPATCH)
1115 | sltiu TMP0, CARG3, LJ_TISNUM 1247 | sltu TMP0, TISNUM, SFARG1HI
1116 | sltiu TMP1, TMP1, 1 1248 | or TMP0, TMP0, TMP1
1117 | and TMP0, TMP0, TMP1 1249 | bnez TMP0, ->fff_fallback
1118 | beqz TMP0, ->fff_fallback
1119 |. sw BASE, L->base // Add frame since C call can throw. 1250 |. sw BASE, L->base // Add frame since C call can throw.
1120 | ffgccheck 1251 | ffgccheck
1121 |. sw PC, SAVE_PC // Redundant (but a defined value). 1252 |. sw PC, SAVE_PC // Redundant (but a defined value).
1122 | load_got lj_str_fromnum 1253 | load_got lj_strfmt_number
1123 | move CARG1, L 1254 | move CARG1, L
1124 | call_intern lj_str_fromnum // (lua_State *L, lua_Number *np) 1255 | call_intern lj_strfmt_number // (lua_State *L, cTValue *o)
1125 |. move CARG2, BASE 1256 |. move CARG2, BASE
1126 | // Returns GCstr *. 1257 | // Returns GCstr *.
1127 | li CARG3, LJ_TSTR 1258 | li SFARG1HI, LJ_TSTR
1128 | b ->fff_restv 1259 | b ->fff_restv
1129 |. move CARG1, CRET1 1260 |. move SFARG1LO, CRET1
1130 | 1261 |
1131 |//-- Base library: iterators ------------------------------------------- 1262 |//-- Base library: iterators -------------------------------------------
1132 | 1263 |
1133 |.ffunc next 1264 |.ffunc next
1134 | lw CARG1, HI(BASE) 1265 | lw CARG2, HI(BASE)
1135 | lw TAB:CARG2, LO(BASE) 1266 | lw TAB:CARG1, LO(BASE)
1136 | beqz NARGS8:RC, ->fff_fallback 1267 | beqz NARGS8:RC, ->fff_fallback
1137 |. addu TMP2, BASE, NARGS8:RC 1268 |. addu TMP2, BASE, NARGS8:RC
1138 | li AT, LJ_TTAB 1269 | li AT, LJ_TTAB
1139 | sw TISNIL, HI(TMP2) // Set missing 2nd arg to nil. 1270 | sw TISNIL, HI(TMP2) // Set missing 2nd arg to nil.
1140 | bne CARG1, AT, ->fff_fallback 1271 | bne CARG2, AT, ->fff_fallback
1141 |. lw PC, FRAME_PC(BASE) 1272 |. lw PC, FRAME_PC(BASE)
1142 | load_got lj_tab_next 1273 | load_got lj_tab_next
1143 | sw BASE, L->base // Add frame since C call can throw. 1274 | addiu CARG2, BASE, 8
1144 | sw BASE, L->top // Dummy frame length is ok. 1275 | call_intern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1145 | addiu CARG3, BASE, 8 1276 |. addiu CARG3, BASE, -8
1146 | sw PC, SAVE_PC 1277 | // Returns 1=found, 0=end, -1=error.
1147 | call_intern lj_tab_next // (lua_State *L, GCtab *t, TValue *key) 1278 | addiu RA, BASE, -8
1148 |. move CARG1, L 1279 | bgtz CRET1, ->fff_res // Found key/value.
1149 | // Returns 0 at end of traversal. 1280 |. li RD, (2+1)*8
1150 | beqz CRET1, ->fff_restv // End of traversal: return nil. 1281 | beqz CRET1, ->fff_restv // End of traversal: return nil.
1151 |. li CARG3, LJ_TNIL 1282 |. li SFARG1HI, LJ_TNIL
1152 | ldc1 f0, 8(BASE) // Copy key and value to results. 1283 | lw CFUNC:RB, FRAME_FUNC(BASE)
1153 | addiu RA, BASE, -8 1284 | b ->fff_fallback // Invalid key.
1154 | ldc1 f2, 16(BASE) 1285 |. li RC, 2*8
1155 | li RD, (2+1)*8
1156 | sdc1 f0, 0(RA)
1157 | b ->fff_res
1158 |. sdc1 f2, 8(RA)
1159 | 1286 |
1160 |.ffunc_1 pairs 1287 |.ffunc_1 pairs
1161 | li AT, LJ_TTAB 1288 | li AT, LJ_TTAB
1162 | bne CARG3, AT, ->fff_fallback 1289 | bne SFARG1HI, AT, ->fff_fallback
1163 |. lw PC, FRAME_PC(BASE) 1290 |. lw PC, FRAME_PC(BASE)
1164#if LJ_52 1291#if LJ_52
1165 | lw TAB:TMP2, TAB:CARG1->metatable 1292 | lw TAB:TMP2, TAB:SFARG1LO->metatable
1166 | ldc1 f0, CFUNC:RB->upvalue[0] 1293 | lw TMP0, CFUNC:RB->upvalue[0].u32.hi
1294 | lw TMP1, CFUNC:RB->upvalue[0].u32.lo
1167 | bnez TAB:TMP2, ->fff_fallback 1295 | bnez TAB:TMP2, ->fff_fallback
1168#else 1296#else
1169 | ldc1 f0, CFUNC:RB->upvalue[0] 1297 | lw TMP0, CFUNC:RB->upvalue[0].u32.hi
1298 | lw TMP1, CFUNC:RB->upvalue[0].u32.lo
1170#endif 1299#endif
1171 |. addiu RA, BASE, -8 1300 |. addiu RA, BASE, -8
1172 | sw TISNIL, 8+HI(BASE) 1301 | sw TISNIL, 8+HI(BASE)
1173 | li RD, (3+1)*8 1302 | sw TMP0, HI(RA)
1303 | sw TMP1, LO(RA)
1174 | b ->fff_res 1304 | b ->fff_res
1175 |. sdc1 f0, 0(RA) 1305 |. li RD, (3+1)*8
1176 | 1306 |
1177 |.ffunc ipairs_aux 1307 |.ffunc ipairs_aux
1178 | sltiu AT, NARGS8:RC, 16 1308 | sltiu AT, NARGS8:RC, 16
@@ -1180,35 +1310,32 @@ static void build_subroutines(BuildCtx *ctx)
1180 | lw TAB:CARG1, LO(BASE) 1310 | lw TAB:CARG1, LO(BASE)
1181 | lw CARG4, 8+HI(BASE) 1311 | lw CARG4, 8+HI(BASE)
1182 | bnez AT, ->fff_fallback 1312 | bnez AT, ->fff_fallback
1183 |. ldc1 FARG2, 8(BASE) 1313 |. addiu CARG3, CARG3, -LJ_TTAB
1184 | addiu CARG3, CARG3, -LJ_TTAB 1314 | xor CARG4, CARG4, TISNUM
1185 | sltiu AT, CARG4, LJ_TISNUM 1315 | and AT, CARG3, CARG4
1186 | li TMP0, 1 1316 | bnez AT, ->fff_fallback
1187 | movn AT, r0, CARG3
1188 | mtc1 TMP0, FARG1
1189 | beqz AT, ->fff_fallback
1190 |. lw PC, FRAME_PC(BASE) 1317 |. lw PC, FRAME_PC(BASE)
1191 | cvt.w.d FRET1, FARG2 1318 | lw TMP2, 8+LO(BASE)
1192 | cvt.d.w FARG1, FARG1
1193 | lw TMP0, TAB:CARG1->asize 1319 | lw TMP0, TAB:CARG1->asize
1194 | lw TMP1, TAB:CARG1->array 1320 | lw TMP1, TAB:CARG1->array
1195 | mfc1 TMP2, FRET1
1196 | addiu RA, BASE, -8
1197 | add.d FARG2, FARG2, FARG1
1198 | addiu TMP2, TMP2, 1 1321 | addiu TMP2, TMP2, 1
1322 | sw TISNUM, -8+HI(BASE)
1199 | sltu AT, TMP2, TMP0 1323 | sltu AT, TMP2, TMP0
1324 | sw TMP2, -8+LO(BASE)
1325 | beqz AT, >2 // Not in array part?
1326 |. addiu RA, BASE, -8
1200 | sll TMP3, TMP2, 3 1327 | sll TMP3, TMP2, 3
1201 | addu TMP3, TMP1, TMP3 1328 | addu TMP3, TMP1, TMP3
1202 | beqz AT, >2 // Not in array part? 1329 | lw TMP1, HI(TMP3)
1203 |. sdc1 FARG2, 0(RA) 1330 | lw TMP2, LO(TMP3)
1204 | lw TMP2, HI(TMP3)
1205 | ldc1 f0, 0(TMP3)
1206 |1: 1331 |1:
1207 | beq TMP2, TISNIL, ->fff_res // End of iteration, return 0 results. 1332 | beq TMP1, TISNIL, ->fff_res // End of iteration, return 0 results.
1208 |. li RD, (0+1)*8 1333 |. li RD, (0+1)*8
1209 | li RD, (2+1)*8 1334 | sw TMP1, 8+HI(RA)
1335 | sw TMP2, 8+LO(RA)
1210 | b ->fff_res 1336 | b ->fff_res
1211 |. sdc1 f0, 8(RA) 1337 |. li RD, (2+1)*8
1338 |
1212 |2: // Check for empty hash part first. Otherwise call C function. 1339 |2: // Check for empty hash part first. Otherwise call C function.
1213 | lw TMP0, TAB:CARG1->hmask 1340 | lw TMP0, TAB:CARG1->hmask
1214 | load_got lj_tab_getinth 1341 | load_got lj_tab_getinth
@@ -1219,27 +1346,30 @@ static void build_subroutines(BuildCtx *ctx)
1219 | // Returns cTValue * or NULL. 1346 | // Returns cTValue * or NULL.
1220 | beqz CRET1, ->fff_res 1347 | beqz CRET1, ->fff_res
1221 |. li RD, (0+1)*8 1348 |. li RD, (0+1)*8
1222 | lw TMP2, HI(CRET1) 1349 | lw TMP1, HI(CRET1)
1223 | b <1 1350 | b <1
1224 |. ldc1 f0, 0(CRET1) 1351 |. lw TMP2, LO(CRET1)
1225 | 1352 |
1226 |.ffunc_1 ipairs 1353 |.ffunc_1 ipairs
1227 | li AT, LJ_TTAB 1354 | li AT, LJ_TTAB
1228 | bne CARG3, AT, ->fff_fallback 1355 | bne SFARG1HI, AT, ->fff_fallback
1229 |. lw PC, FRAME_PC(BASE) 1356 |. lw PC, FRAME_PC(BASE)
1230#if LJ_52 1357#if LJ_52
1231 | lw TAB:TMP2, TAB:CARG1->metatable 1358 | lw TAB:TMP2, TAB:SFARG1LO->metatable
1232 | ldc1 f0, CFUNC:RB->upvalue[0] 1359 | lw TMP0, CFUNC:RB->upvalue[0].u32.hi
1360 | lw TMP1, CFUNC:RB->upvalue[0].u32.lo
1233 | bnez TAB:TMP2, ->fff_fallback 1361 | bnez TAB:TMP2, ->fff_fallback
1234#else 1362#else
1235 | ldc1 f0, CFUNC:RB->upvalue[0] 1363 | lw TMP0, CFUNC:RB->upvalue[0].u32.hi
1364 | lw TMP1, CFUNC:RB->upvalue[0].u32.lo
1236#endif 1365#endif
1237 |. addiu RA, BASE, -8 1366 |. addiu RA, BASE, -8
1238 | sw r0, 8+HI(BASE) 1367 | sw TISNUM, 8+HI(BASE)
1239 | sw r0, 8+LO(BASE) 1368 | sw r0, 8+LO(BASE)
1240 | li RD, (3+1)*8 1369 | sw TMP0, HI(RA)
1370 | sw TMP1, LO(RA)
1241 | b ->fff_res 1371 | b ->fff_res
1242 |. sdc1 f0, 0(RA) 1372 |. li RD, (3+1)*8
1243 | 1373 |
1244 |//-- Base library: catch errors ---------------------------------------- 1374 |//-- Base library: catch errors ----------------------------------------
1245 | 1375 |
@@ -1259,8 +1389,9 @@ static void build_subroutines(BuildCtx *ctx)
1259 | sltiu AT, NARGS8:RC, 16 1389 | sltiu AT, NARGS8:RC, 16
1260 | lw CARG4, 8+HI(BASE) 1390 | lw CARG4, 8+HI(BASE)
1261 | bnez AT, ->fff_fallback 1391 | bnez AT, ->fff_fallback
1262 |. ldc1 FARG2, 8(BASE) 1392 |. lw CARG3, 8+LO(BASE)
1263 | ldc1 FARG1, 0(BASE) 1393 | lw CARG1, LO(BASE)
1394 | lw CARG2, HI(BASE)
1264 | lbu TMP1, DISPATCH_GL(hookmask)(DISPATCH) 1395 | lbu TMP1, DISPATCH_GL(hookmask)(DISPATCH)
1265 | li AT, LJ_TFUNC 1396 | li AT, LJ_TFUNC
1266 | move TMP2, BASE 1397 | move TMP2, BASE
@@ -1268,9 +1399,11 @@ static void build_subroutines(BuildCtx *ctx)
1268 | addiu BASE, BASE, 16 1399 | addiu BASE, BASE, 16
1269 | // Remember active hook before pcall. 1400 | // Remember active hook before pcall.
1270 | srl TMP3, TMP3, HOOK_ACTIVE_SHIFT 1401 | srl TMP3, TMP3, HOOK_ACTIVE_SHIFT
1271 | sdc1 FARG2, 0(TMP2) // Swap function and traceback. 1402 | sw CARG3, LO(TMP2) // Swap function and traceback.
1403 | sw CARG4, HI(TMP2)
1272 | andi TMP3, TMP3, 1 1404 | andi TMP3, TMP3, 1
1273 | sdc1 FARG1, 8(TMP2) 1405 | sw CARG1, 8+LO(TMP2)
1406 | sw CARG2, 8+HI(TMP2)
1274 | addiu PC, TMP3, 16+FRAME_PCALL 1407 | addiu PC, TMP3, 16+FRAME_PCALL
1275 | b ->vm_call_dispatch 1408 | b ->vm_call_dispatch
1276 |. addiu NARGS8:RC, NARGS8:RC, -16 1409 |. addiu NARGS8:RC, NARGS8:RC, -16
@@ -1279,7 +1412,10 @@ static void build_subroutines(BuildCtx *ctx)
1279 | 1412 |
1280 |.macro coroutine_resume_wrap, resume 1413 |.macro coroutine_resume_wrap, resume
1281 |.if resume 1414 |.if resume
1282 |.ffunc_1 coroutine_resume 1415 |.ffunc coroutine_resume
1416 | lw CARG3, HI(BASE)
1417 | beqz NARGS8:RC, ->fff_fallback
1418 |. lw CARG1, LO(BASE)
1283 | li AT, LJ_TTHREAD 1419 | li AT, LJ_TTHREAD
1284 | bne CARG3, AT, ->fff_fallback 1420 | bne CARG3, AT, ->fff_fallback
1285 |.else 1421 |.else
@@ -1314,11 +1450,13 @@ static void build_subroutines(BuildCtx *ctx)
1314 | move CARG3, CARG2 1450 | move CARG3, CARG2
1315 | sw BASE, L->top 1451 | sw BASE, L->top
1316 |2: // Move args to coroutine. 1452 |2: // Move args to coroutine.
1317 | ldc1 f0, 0(BASE) 1453 | lw SFRETHI, HI(BASE)
1454 | lw SFRETLO, LO(BASE)
1318 | sltu AT, BASE, TMP1 1455 | sltu AT, BASE, TMP1
1319 | beqz AT, >3 1456 | beqz AT, >3
1320 |. addiu BASE, BASE, 8 1457 |. addiu BASE, BASE, 8
1321 | sdc1 f0, 0(CARG3) 1458 | sw SFRETHI, HI(CARG3)
1459 | sw SFRETLO, LO(CARG3)
1322 | b <2 1460 | b <2
1323 |. addiu CARG3, CARG3, 8 1461 |. addiu CARG3, CARG3, 8
1324 |3: 1462 |3:
@@ -1331,6 +1469,7 @@ static void build_subroutines(BuildCtx *ctx)
1331 | lw TMP3, L:RA->top 1469 | lw TMP3, L:RA->top
1332 | li_vmstate INTERP 1470 | li_vmstate INTERP
1333 | lw BASE, L->base 1471 | lw BASE, L->base
1472 | sw L, DISPATCH_GL(cur_L)(DISPATCH)
1334 | st_vmstate 1473 | st_vmstate
1335 | beqz AT, >8 1474 | beqz AT, >8
1336 |. subu RD, TMP3, TMP2 1475 |. subu RD, TMP3, TMP2
@@ -1343,10 +1482,12 @@ static void build_subroutines(BuildCtx *ctx)
1343 | sw TMP2, L:RA->top // Clear coroutine stack. 1482 | sw TMP2, L:RA->top // Clear coroutine stack.
1344 | move TMP1, BASE 1483 | move TMP1, BASE
1345 |5: // Move results from coroutine. 1484 |5: // Move results from coroutine.
1346 | ldc1 f0, 0(TMP2) 1485 | lw SFRETHI, HI(TMP2)
1486 | lw SFRETLO, LO(TMP2)
1347 | addiu TMP2, TMP2, 8 1487 | addiu TMP2, TMP2, 8
1348 | sltu AT, TMP2, TMP3 1488 | sltu AT, TMP2, TMP3
1349 | sdc1 f0, 0(TMP1) 1489 | sw SFRETHI, HI(TMP1)
1490 | sw SFRETLO, LO(TMP1)
1350 | bnez AT, <5 1491 | bnez AT, <5
1351 |. addiu TMP1, TMP1, 8 1492 |. addiu TMP1, TMP1, 8
1352 |6: 1493 |6:
@@ -1371,12 +1512,14 @@ static void build_subroutines(BuildCtx *ctx)
1371 |.if resume 1512 |.if resume
1372 | addiu TMP3, TMP3, -8 1513 | addiu TMP3, TMP3, -8
1373 | li TMP1, LJ_TFALSE 1514 | li TMP1, LJ_TFALSE
1374 | ldc1 f0, 0(TMP3) 1515 | lw SFRETHI, HI(TMP3)
1516 | lw SFRETLO, LO(TMP3)
1375 | sw TMP3, L:RA->top // Remove error from coroutine stack. 1517 | sw TMP3, L:RA->top // Remove error from coroutine stack.
1376 | li RD, (2+1)*8 1518 | li RD, (2+1)*8
1377 | sw TMP1, -8+HI(BASE) // Prepend false to results. 1519 | sw TMP1, -8+HI(BASE) // Prepend false to results.
1378 | addiu RA, BASE, -8 1520 | addiu RA, BASE, -8
1379 | sdc1 f0, 0(BASE) // Copy error message. 1521 | sw SFRETHI, HI(BASE) // Copy error message.
1522 | sw SFRETLO, LO(BASE)
1380 | b <7 1523 | b <7
1381 |. andi TMP0, PC, FRAME_TYPE 1524 |. andi TMP0, PC, FRAME_TYPE
1382 |.else 1525 |.else
@@ -1412,20 +1555,29 @@ static void build_subroutines(BuildCtx *ctx)
1412 | 1555 |
1413 |//-- Math library ------------------------------------------------------- 1556 |//-- Math library -------------------------------------------------------
1414 | 1557 |
1415 |.ffunc_n math_abs 1558 |.ffunc_1 math_abs
1416 |. abs.d FRET1, FARG1 1559 | bne SFARG1HI, TISNUM, >1
1417 |->fff_resn: 1560 |. sra TMP0, SFARG1LO, 31
1418 | lw PC, FRAME_PC(BASE) 1561 | xor TMP1, SFARG1LO, TMP0
1419 | addiu RA, BASE, -8 1562 | subu SFARG1LO, TMP1, TMP0
1420 | b ->fff_res1 1563 | bgez SFARG1LO, ->fff_restv
1421 |. sdc1 FRET1, -8(BASE) 1564 |. nop
1565 | lui SFARG1HI, 0x41e0 // 2^31 as a double.
1566 | b ->fff_restv
1567 |. li SFARG1LO, 0
1568 |1:
1569 | sltiu AT, SFARG1HI, LJ_TISNUM
1570 | beqz AT, ->fff_fallback
1571 |. sll SFARG1HI, SFARG1HI, 1
1572 | srl SFARG1HI, SFARG1HI, 1
1573 |// fallthrough
1422 | 1574 |
1423 |->fff_restv: 1575 |->fff_restv:
1424 | // CARG3/CARG1 = TValue result. 1576 | // SFARG1LO/SFARG1HI = TValue result.
1425 | lw PC, FRAME_PC(BASE) 1577 | lw PC, FRAME_PC(BASE)
1426 | sw CARG3, -8+HI(BASE) 1578 | sw SFARG1HI, -8+HI(BASE)
1427 | addiu RA, BASE, -8 1579 | addiu RA, BASE, -8
1428 | sw CARG1, -8+LO(BASE) 1580 | sw SFARG1LO, -8+LO(BASE)
1429 |->fff_res1: 1581 |->fff_res1:
1430 | // RA = results, PC = return. 1582 | // RA = results, PC = return.
1431 | li RD, (1+1)*8 1583 | li RD, (1+1)*8
@@ -1454,15 +1606,19 @@ static void build_subroutines(BuildCtx *ctx)
1454 |. sw TISNIL, -8+HI(TMP1) 1606 |. sw TISNIL, -8+HI(TMP1)
1455 | 1607 |
1456 |.macro math_extern, func 1608 |.macro math_extern, func
1457 |->ff_math_ .. func: 1609 | .ffunc math_ .. func
1458 | lw CARG3, HI(BASE) 1610 | lw SFARG1HI, HI(BASE)
1459 | beqz NARGS8:RC, ->fff_fallback 1611 | beqz NARGS8:RC, ->fff_fallback
1460 |. load_got func 1612 |. load_got func
1461 | sltiu AT, CARG3, LJ_TISNUM 1613 | sltiu AT, SFARG1HI, LJ_TISNUM
1462 | beqz AT, ->fff_fallback 1614 | beqz AT, ->fff_fallback
1463 |. nop 1615 |.if FPU
1464 | call_extern
1465 |. ldc1 FARG1, 0(BASE) 1616 |. ldc1 FARG1, 0(BASE)
1617 |.else
1618 |. lw SFARG1LO, LO(BASE)
1619 |.endif
1620 | call_extern
1621 |. nop
1466 | b ->fff_resn 1622 | b ->fff_resn
1467 |. nop 1623 |. nop
1468 |.endmacro 1624 |.endmacro
@@ -1476,10 +1632,22 @@ static void build_subroutines(BuildCtx *ctx)
1476 |. nop 1632 |. nop
1477 |.endmacro 1633 |.endmacro
1478 | 1634 |
1635 |// TODO: Return integer type if result is integer (own sf implementation).
1479 |.macro math_round, func 1636 |.macro math_round, func
1480 | .ffunc_n math_ .. func 1637 |->ff_math_ .. func:
1481 |. nop 1638 | lw SFARG1HI, HI(BASE)
1639 | beqz NARGS8:RC, ->fff_fallback
1640 |. lw SFARG1LO, LO(BASE)
1641 | beq SFARG1HI, TISNUM, ->fff_restv
1642 |. sltu AT, SFARG1HI, TISNUM
1643 | beqz AT, ->fff_fallback
1644 |.if FPU
1645 |. ldc1 FARG1, 0(BASE)
1482 | bal ->vm_ .. func 1646 | bal ->vm_ .. func
1647 |.else
1648 |. load_got func
1649 | call_extern
1650 |.endif
1483 |. nop 1651 |. nop
1484 | b ->fff_resn 1652 | b ->fff_resn
1485 |. nop 1653 |. nop
@@ -1489,15 +1657,19 @@ static void build_subroutines(BuildCtx *ctx)
1489 | math_round ceil 1657 | math_round ceil
1490 | 1658 |
1491 |.ffunc math_log 1659 |.ffunc math_log
1492 | lw CARG3, HI(BASE)
1493 | li AT, 8 1660 | li AT, 8
1494 | bne NARGS8:RC, AT, ->fff_fallback // Exactly 1 argument. 1661 | bne NARGS8:RC, AT, ->fff_fallback // Exactly 1 argument.
1495 |. load_got log 1662 |. lw SFARG1HI, HI(BASE)
1496 | sltiu AT, CARG3, LJ_TISNUM 1663 | sltiu AT, SFARG1HI, LJ_TISNUM
1497 | beqz AT, ->fff_fallback 1664 | beqz AT, ->fff_fallback
1498 |. nop 1665 |. load_got log
1666 |.if FPU
1499 | call_extern 1667 | call_extern
1500 |. ldc1 FARG1, 0(BASE) 1668 |. ldc1 FARG1, 0(BASE)
1669 |.else
1670 | call_extern
1671 |. lw SFARG1LO, LO(BASE)
1672 |.endif
1501 | b ->fff_resn 1673 | b ->fff_resn
1502 |. nop 1674 |. nop
1503 | 1675 |
@@ -1516,23 +1688,43 @@ static void build_subroutines(BuildCtx *ctx)
1516 | math_extern2 atan2 1688 | math_extern2 atan2
1517 | math_extern2 fmod 1689 | math_extern2 fmod
1518 | 1690 |
1691 |.if FPU
1519 |.ffunc_n math_sqrt 1692 |.ffunc_n math_sqrt
1520 |. sqrt.d FRET1, FARG1 1693 |. sqrt.d FRET1, FARG1
1521 | b ->fff_resn 1694 |// fallthrough to ->fff_resn
1522 |. nop 1695 |.else
1696 | math_extern sqrt
1697 |.endif
1523 | 1698 |
1524 |->ff_math_deg: 1699 |->fff_resn:
1525 |.ffunc_n math_rad 1700 | lw PC, FRAME_PC(BASE)
1526 |. ldc1 FARG2, CFUNC:RB->upvalue[0] 1701 | addiu RA, BASE, -8
1527 | b ->fff_resn 1702 |.if FPU
1528 |. mul.d FRET1, FARG1, FARG2 1703 | b ->fff_res1
1704 |. sdc1 FRET1, -8(BASE)
1705 |.else
1706 | sw SFRETHI, -8+HI(BASE)
1707 | b ->fff_res1
1708 |. sw SFRETLO, -8+LO(BASE)
1709 |.endif
1529 | 1710 |
1530 |.ffunc_nn math_ldexp 1711 |
1531 | cvt.w.d FARG2, FARG2 1712 |.ffunc math_ldexp
1713 | sltiu AT, NARGS8:RC, 16
1714 | lw SFARG1HI, HI(BASE)
1715 | bnez AT, ->fff_fallback
1716 |. lw CARG4, 8+HI(BASE)
1717 | bne CARG4, TISNUM, ->fff_fallback
1532 | load_got ldexp 1718 | load_got ldexp
1533 | mfc1 CARG3, FARG2 1719 |. sltu AT, SFARG1HI, TISNUM
1720 | beqz AT, ->fff_fallback
1721 |.if FPU
1722 |. ldc1 FARG1, 0(BASE)
1723 |.else
1724 |. lw SFARG1LO, LO(BASE)
1725 |.endif
1534 | call_extern 1726 | call_extern
1535 |. nop 1727 |. lw CARG3, 8+LO(BASE)
1536 | b ->fff_resn 1728 | b ->fff_resn
1537 |. nop 1729 |. nop
1538 | 1730 |
@@ -1543,10 +1735,17 @@ static void build_subroutines(BuildCtx *ctx)
1543 |. addiu CARG3, DISPATCH, DISPATCH_GL(tmptv) 1735 |. addiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
1544 | lw TMP1, DISPATCH_GL(tmptv)(DISPATCH) 1736 | lw TMP1, DISPATCH_GL(tmptv)(DISPATCH)
1545 | addiu RA, BASE, -8 1737 | addiu RA, BASE, -8
1738 |.if FPU
1546 | mtc1 TMP1, FARG2 1739 | mtc1 TMP1, FARG2
1547 | sdc1 FRET1, 0(RA) 1740 | sdc1 FRET1, 0(RA)
1548 | cvt.d.w FARG2, FARG2 1741 | cvt.d.w FARG2, FARG2
1549 | sdc1 FARG2, 8(RA) 1742 | sdc1 FARG2, 8(RA)
1743 |.else
1744 | sw SFRETLO, LO(RA)
1745 | sw SFRETHI, HI(RA)
1746 | sw TMP1, 8+LO(RA)
1747 | sw TISNUM, 8+HI(RA)
1748 |.endif
1550 | b ->fff_res 1749 | b ->fff_res
1551 |. li RD, (2+1)*8 1750 |. li RD, (2+1)*8
1552 | 1751 |
@@ -1556,49 +1755,109 @@ static void build_subroutines(BuildCtx *ctx)
1556 | call_extern 1755 | call_extern
1557 |. addiu CARG3, BASE, -8 1756 |. addiu CARG3, BASE, -8
1558 | addiu RA, BASE, -8 1757 | addiu RA, BASE, -8
1758 |.if FPU
1559 | sdc1 FRET1, 0(BASE) 1759 | sdc1 FRET1, 0(BASE)
1760 |.else
1761 | sw SFRETLO, LO(BASE)
1762 | sw SFRETHI, HI(BASE)
1763 |.endif
1560 | b ->fff_res 1764 | b ->fff_res
1561 |. li RD, (2+1)*8 1765 |. li RD, (2+1)*8
1562 | 1766 |
1563 |.macro math_minmax, name, ismax 1767 |.macro math_minmax, name, intins, ismax
1564 |->ff_ .. name: 1768 | .ffunc_1 name
1565 | lw CARG3, HI(BASE) 1769 | addu TMP3, BASE, NARGS8:RC
1566 | beqz NARGS8:RC, ->fff_fallback 1770 | bne SFARG1HI, TISNUM, >5
1567 |. ldc1 FRET1, 0(BASE) 1771 |. addiu TMP2, BASE, 8
1568 | sltiu AT, CARG3, LJ_TISNUM 1772 |1: // Handle integers.
1773 |. lw SFARG2HI, HI(TMP2)
1774 | beq TMP2, TMP3, ->fff_restv
1775 |. lw SFARG2LO, LO(TMP2)
1776 | bne SFARG2HI, TISNUM, >3
1777 |. slt AT, SFARG1LO, SFARG2LO
1778 | intins SFARG1LO, SFARG2LO, AT
1779 | b <1
1780 |. addiu TMP2, TMP2, 8
1781 |
1782 |3: // Convert intermediate result to number and continue with number loop.
1783 | sltiu AT, SFARG2HI, LJ_TISNUM
1569 | beqz AT, ->fff_fallback 1784 | beqz AT, ->fff_fallback
1570 |. addu TMP2, BASE, NARGS8:RC 1785 |.if FPU
1571 | addiu TMP1, BASE, 8 1786 |. mtc1 SFARG1LO, FRET1
1572 | beq TMP1, TMP2, ->fff_resn 1787 | cvt.d.w FRET1, FRET1
1573 |1: 1788 | b >7
1574 |. lw CARG3, HI(TMP1) 1789 |. ldc1 FARG1, 0(TMP2)
1575 | ldc1 FARG1, 0(TMP1) 1790 |.else
1576 | addiu TMP1, TMP1, 8 1791 |. nop
1577 | sltiu AT, CARG3, LJ_TISNUM 1792 | bal ->vm_sfi2d_1
1793 |. nop
1794 | b >7
1795 |. nop
1796 |.endif
1797 |
1798 |5:
1799 |. sltiu AT, SFARG1HI, LJ_TISNUM
1578 | beqz AT, ->fff_fallback 1800 | beqz AT, ->fff_fallback
1801 |.if FPU
1802 |. ldc1 FRET1, 0(BASE)
1803 |.endif
1804 |
1805 |6: // Handle numbers.
1806 |. lw SFARG2HI, HI(TMP2)
1807 |.if FPU
1808 | beq TMP2, TMP3, ->fff_resn
1809 |.else
1810 | beq TMP2, TMP3, ->fff_restv
1811 |.endif
1812 |. sltiu AT, SFARG2HI, LJ_TISNUM
1813 | beqz AT, >8
1814 |.if FPU
1815 |. ldc1 FARG1, 0(TMP2)
1816 |.else
1817 |. lw SFARG2LO, LO(TMP2)
1818 |.endif
1819 |7:
1820 |.if FPU
1579 |.if ismax 1821 |.if ismax
1580 |. c.olt.d FARG1, FRET1 1822 | c.olt.d FARG1, FRET1
1581 |.else 1823 |.else
1582 |. c.olt.d FRET1, FARG1 1824 | c.olt.d FRET1, FARG1
1583 |.endif 1825 |.endif
1584 | bne TMP1, TMP2, <1 1826 | movf.d FRET1, FARG1
1585 |. movf.d FRET1, FARG1 1827 |.else
1586 | b ->fff_resn 1828 |.if ismax
1829 | bal ->vm_sfcmpogt
1830 |.else
1831 | bal ->vm_sfcmpolt
1832 |.endif
1833 |. nop
1834 | movz SFARG1LO, SFARG2LO, CRET1
1835 | movz SFARG1HI, SFARG2HI, CRET1
1836 |.endif
1837 | b <6
1838 |. addiu TMP2, TMP2, 8
1839 |
1840 |8: // Convert integer to number and continue with number loop.
1841 | bne SFARG2HI, TISNUM, ->fff_fallback
1842 |.if FPU
1843 |. lwc1 FARG1, LO(TMP2)
1844 | b <7
1845 |. cvt.d.w FARG1, FARG1
1846 |.else
1847 |. nop
1848 | bal ->vm_sfi2d_2
1587 |. nop 1849 |. nop
1850 | b <7
1851 |. nop
1852 |.endif
1853 |
1588 |.endmacro 1854 |.endmacro
1589 | 1855 |
1590 | math_minmax math_min, 0 1856 | math_minmax math_min, movz, 0
1591 | math_minmax math_max, 1 1857 | math_minmax math_max, movn, 1
1592 | 1858 |
1593 |//-- String library ----------------------------------------------------- 1859 |//-- String library -----------------------------------------------------
1594 | 1860 |
1595 |.ffunc_1 string_len
1596 | li AT, LJ_TSTR
1597 | bne CARG3, AT, ->fff_fallback
1598 |. nop
1599 | b ->fff_resi
1600 |. lw CRET1, STR:CARG1->len
1601 |
1602 |.ffunc string_byte // Only handle the 1-arg case here. 1861 |.ffunc string_byte // Only handle the 1-arg case here.
1603 | lw CARG3, HI(BASE) 1862 | lw CARG3, HI(BASE)
1604 | lw STR:CARG1, LO(BASE) 1863 | lw STR:CARG1, LO(BASE)
@@ -1608,33 +1867,31 @@ static void build_subroutines(BuildCtx *ctx)
1608 | bnez AT, ->fff_fallback // Need exactly 1 string argument. 1867 | bnez AT, ->fff_fallback // Need exactly 1 string argument.
1609 |. nop 1868 |. nop
1610 | lw TMP0, STR:CARG1->len 1869 | lw TMP0, STR:CARG1->len
1611 | lbu TMP1, STR:CARG1[1] // Access is always ok (NUL at end).
1612 | addiu RA, BASE, -8 1870 | addiu RA, BASE, -8
1871 | lw PC, FRAME_PC(BASE)
1613 | sltu RD, r0, TMP0 1872 | sltu RD, r0, TMP0
1614 | mtc1 TMP1, f0 1873 | lbu TMP1, STR:CARG1[1] // Access is always ok (NUL at end).
1615 | addiu RD, RD, 1 1874 | addiu RD, RD, 1
1616 | cvt.d.w f0, f0
1617 | lw PC, FRAME_PC(BASE)
1618 | sll RD, RD, 3 // RD = ((str->len != 0)+1)*8 1875 | sll RD, RD, 3 // RD = ((str->len != 0)+1)*8
1876 | sw TISNUM, HI(RA)
1619 | b ->fff_res 1877 | b ->fff_res
1620 |. sdc1 f0, 0(RA) 1878 |. sw TMP1, LO(RA)
1621 | 1879 |
1622 |.ffunc string_char // Only handle the 1-arg case here. 1880 |.ffunc string_char // Only handle the 1-arg case here.
1623 | ffgccheck 1881 | ffgccheck
1624 |. nop 1882 |. nop
1625 | lw CARG3, HI(BASE) 1883 | lw CARG3, HI(BASE)
1626 | ldc1 FARG1, 0(BASE) 1884 | lw CARG1, LO(BASE)
1627 | li AT, 8 1885 | li TMP1, 255
1628 | bne NARGS8:RC, AT, ->fff_fallback // Exactly 1 argument. 1886 | xori AT, NARGS8:RC, 8 // Exactly 1 argument.
1629 |. sltiu AT, CARG3, LJ_TISNUM 1887 | xor TMP0, CARG3, TISNUM // Integer.
1630 | beqz AT, ->fff_fallback 1888 | sltu TMP1, TMP1, CARG1 // !(255 < n).
1889 | or AT, AT, TMP0
1890 | or AT, AT, TMP1
1891 | bnez AT, ->fff_fallback
1631 |. li CARG3, 1 1892 |. li CARG3, 1
1632 | cvt.w.d FARG1, FARG1
1633 | addiu CARG2, sp, ARG5_OFS 1893 | addiu CARG2, sp, ARG5_OFS
1634 | sltiu AT, TMP0, 256 1894 | sb CARG1, ARG5
1635 | mfc1 TMP0, FARG1
1636 | beqz AT, ->fff_fallback
1637 |. sw TMP0, ARG5
1638 |->fff_newstr: 1895 |->fff_newstr:
1639 | load_got lj_str_new 1896 | load_got lj_str_new
1640 | sw BASE, L->base 1897 | sw BASE, L->base
@@ -1643,35 +1900,30 @@ static void build_subroutines(BuildCtx *ctx)
1643 |. move CARG1, L 1900 |. move CARG1, L
1644 | // Returns GCstr *. 1901 | // Returns GCstr *.
1645 | lw BASE, L->base 1902 | lw BASE, L->base
1646 | move CARG1, CRET1 1903 |->fff_resstr:
1904 | move SFARG1LO, CRET1
1647 | b ->fff_restv 1905 | b ->fff_restv
1648 |. li CARG3, LJ_TSTR 1906 |. li SFARG1HI, LJ_TSTR
1649 | 1907 |
1650 |.ffunc string_sub 1908 |.ffunc string_sub
1651 | ffgccheck 1909 | ffgccheck
1652 |. nop 1910 |. nop
1653 | addiu AT, NARGS8:RC, -16 1911 | addiu AT, NARGS8:RC, -16
1654 | lw CARG3, 16+HI(BASE) 1912 | lw CARG3, 16+HI(BASE)
1655 | ldc1 f0, 16(BASE)
1656 | lw TMP0, HI(BASE) 1913 | lw TMP0, HI(BASE)
1657 | lw STR:CARG1, LO(BASE) 1914 | lw STR:CARG1, LO(BASE)
1658 | bltz AT, ->fff_fallback 1915 | bltz AT, ->fff_fallback
1659 | lw CARG2, 8+HI(BASE) 1916 |. lw CARG2, 8+HI(BASE)
1660 | ldc1 f2, 8(BASE)
1661 | beqz AT, >1 1917 | beqz AT, >1
1662 |. li CARG4, -1 1918 |. li CARG4, -1
1663 | cvt.w.d f0, f0 1919 | bne CARG3, TISNUM, ->fff_fallback
1664 | sltiu AT, CARG3, LJ_TISNUM 1920 |. lw CARG4, 16+LO(BASE)
1665 | beqz AT, ->fff_fallback
1666 |. mfc1 CARG4, f0
1667 |1: 1921 |1:
1668 | sltiu AT, CARG2, LJ_TISNUM 1922 | bne CARG2, TISNUM, ->fff_fallback
1669 | beqz AT, ->fff_fallback
1670 |. li AT, LJ_TSTR 1923 |. li AT, LJ_TSTR
1671 | cvt.w.d f2, f2
1672 | bne TMP0, AT, ->fff_fallback 1924 | bne TMP0, AT, ->fff_fallback
1673 |. lw CARG2, STR:CARG1->len 1925 |. lw CARG3, 8+LO(BASE)
1674 | mfc1 CARG3, f2 1926 | lw CARG2, STR:CARG1->len
1675 | // STR:CARG1 = str, CARG2 = str->len, CARG3 = start, CARG4 = end 1927 | // STR:CARG1 = str, CARG2 = str->len, CARG3 = start, CARG4 = end
1676 | slt AT, CARG4, r0 1928 | slt AT, CARG4, r0
1677 | addiu TMP0, CARG2, 1 1929 | addiu TMP0, CARG2, 1
@@ -1693,139 +1945,130 @@ static void build_subroutines(BuildCtx *ctx)
1693 | bgez CARG3, ->fff_newstr 1945 | bgez CARG3, ->fff_newstr
1694 |. addiu CARG3, CARG3, 1 // len++ 1946 |. addiu CARG3, CARG3, 1 // len++
1695 |->fff_emptystr: // Return empty string. 1947 |->fff_emptystr: // Return empty string.
1696 | addiu STR:CARG1, DISPATCH, DISPATCH_GL(strempty) 1948 | addiu STR:SFARG1LO, DISPATCH, DISPATCH_GL(strempty)
1697 | b ->fff_restv 1949 | b ->fff_restv
1698 |. li CARG3, LJ_TSTR 1950 |. li SFARG1HI, LJ_TSTR
1699 | 1951 |
1700 |.ffunc string_rep // Only handle the 1-char case inline. 1952 |.macro ffstring_op, name
1701 | ffgccheck 1953 | .ffunc string_ .. name
1702 |. nop
1703 | lw TMP0, HI(BASE)
1704 | addiu AT, NARGS8:RC, -16 // Exactly 2 arguments.
1705 | lw CARG4, 8+HI(BASE)
1706 | lw STR:CARG1, LO(BASE)
1707 | addiu TMP0, TMP0, -LJ_TSTR
1708 | ldc1 f0, 8(BASE)
1709 | or AT, AT, TMP0
1710 | bnez AT, ->fff_fallback
1711 |. sltiu AT, CARG4, LJ_TISNUM
1712 | cvt.w.d f0, f0
1713 | beqz AT, ->fff_fallback
1714 |. lw TMP0, STR:CARG1->len
1715 | mfc1 CARG3, f0
1716 | lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
1717 | li AT, 1
1718 | blez CARG3, ->fff_emptystr // Count <= 0?
1719 |. sltu AT, AT, TMP0
1720 | beqz TMP0, ->fff_emptystr // Zero length string?
1721 |. sltu TMP0, TMP1, CARG3
1722 | or AT, AT, TMP0
1723 | lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
1724 | bnez AT, ->fff_fallback // Fallback for > 1-char strings.
1725 |. lbu TMP0, STR:CARG1[1]
1726 | addu TMP2, CARG2, CARG3
1727 |1: // Fill buffer with char. Yes, this is suboptimal code (do you care?).
1728 | addiu TMP2, TMP2, -1
1729 | sltu AT, CARG2, TMP2
1730 | bnez AT, <1
1731 |. sb TMP0, 0(TMP2)
1732 | b ->fff_newstr
1733 |. nop
1734 |
1735 |.ffunc string_reverse
1736 | ffgccheck 1954 | ffgccheck
1737 |. nop 1955 |. nop
1738 | lw CARG3, HI(BASE) 1956 | lw CARG3, HI(BASE)
1739 | lw STR:CARG1, LO(BASE) 1957 | lw STR:CARG2, LO(BASE)
1740 | beqz NARGS8:RC, ->fff_fallback 1958 | beqz NARGS8:RC, ->fff_fallback
1741 |. li AT, LJ_TSTR 1959 |. li AT, LJ_TSTR
1742 | bne CARG3, AT, ->fff_fallback 1960 | bne CARG3, AT, ->fff_fallback
1743 |. lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH) 1961 |. addiu SBUF:CARG1, DISPATCH, DISPATCH_GL(tmpbuf)
1744 | lw CARG3, STR:CARG1->len 1962 | load_got lj_buf_putstr_ .. name
1745 | addiu CARG1, STR:CARG1, #STR 1963 | lw TMP0, SBUF:CARG1->b
1746 | lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH) 1964 | sw L, SBUF:CARG1->L
1747 | sltu AT, TMP1, CARG3 1965 | sw BASE, L->base
1748 | bnez AT, ->fff_fallback 1966 | sw TMP0, SBUF:CARG1->w
1749 |. addu TMP3, CARG1, CARG3 1967 | call_intern extern lj_buf_putstr_ .. name
1750 | addu CARG4, CARG2, CARG3 1968 |. sw PC, SAVE_PC
1751 |1: // Reverse string copy. 1969 | load_got lj_buf_tostr
1752 | lbu TMP1, 0(CARG1) 1970 | call_intern lj_buf_tostr
1753 | sltu AT, CARG1, TMP3 1971 |. move SBUF:CARG1, SBUF:CRET1
1754 | beqz AT, ->fff_newstr 1972 | b ->fff_resstr
1755 |. addiu CARG1, CARG1, 1 1973 |. lw BASE, L->base
1756 | addiu CARG4, CARG4, -1
1757 | b <1
1758 | sb TMP1, 0(CARG4)
1759 |
1760 |.macro ffstring_case, name, lo
1761 | .ffunc name
1762 | ffgccheck
1763 |. nop
1764 | lw CARG3, HI(BASE)
1765 | lw STR:CARG1, LO(BASE)
1766 | beqz NARGS8:RC, ->fff_fallback
1767 |. li AT, LJ_TSTR
1768 | bne CARG3, AT, ->fff_fallback
1769 |. lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
1770 | lw CARG3, STR:CARG1->len
1771 | addiu CARG1, STR:CARG1, #STR
1772 | lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
1773 | sltu AT, TMP1, CARG3
1774 | bnez AT, ->fff_fallback
1775 |. addu TMP3, CARG1, CARG3
1776 | move CARG4, CARG2
1777 |1: // ASCII case conversion.
1778 | lbu TMP1, 0(CARG1)
1779 | sltu AT, CARG1, TMP3
1780 | beqz AT, ->fff_newstr
1781 |. addiu TMP0, TMP1, -lo
1782 | xori TMP2, TMP1, 0x20
1783 | sltiu AT, TMP0, 26
1784 | movn TMP1, TMP2, AT
1785 | addiu CARG1, CARG1, 1
1786 | sb TMP1, 0(CARG4)
1787 | b <1
1788 |. addiu CARG4, CARG4, 1
1789 |.endmacro 1974 |.endmacro
1790 | 1975 |
1791 |ffstring_case string_lower, 65 1976 |ffstring_op reverse
1792 |ffstring_case string_upper, 97 1977 |ffstring_op lower
1978 |ffstring_op upper
1793 | 1979 |
1794 |//-- Table library ------------------------------------------------------ 1980 |//-- Bit library --------------------------------------------------------
1795 | 1981 |
1796 |.ffunc_1 table_getn 1982 |->vm_tobit_fb:
1797 | li AT, LJ_TTAB 1983 | beqz TMP1, ->fff_fallback
1798 | bne CARG3, AT, ->fff_fallback 1984 |.if FPU
1799 |. load_got lj_tab_len 1985 |. ldc1 FARG1, 0(BASE)
1800 | call_intern lj_tab_len // (GCtab *t) 1986 | add.d FARG1, FARG1, TOBIT
1801 |. nop 1987 | jr ra
1802 | // Returns uint32_t (but less than 2^31). 1988 |. mfc1 CRET1, FARG1
1803 | b ->fff_resi 1989 |.else
1990 |// FP number to bit conversion for soft-float.
1991 |->vm_tobit:
1992 | sll TMP0, SFARG1HI, 1
1993 | lui AT, 0x0020
1994 | addu TMP0, TMP0, AT
1995 | slt AT, TMP0, r0
1996 | movz SFARG1LO, r0, AT
1997 | beqz AT, >2
1998 |. li TMP1, 0x3e0
1999 | not TMP1, TMP1
2000 | sra TMP0, TMP0, 21
2001 | subu TMP0, TMP1, TMP0
2002 | slt AT, TMP0, r0
2003 | bnez AT, >1
2004 |. sll TMP1, SFARG1HI, 11
2005 | lui AT, 0x8000
2006 | or TMP1, TMP1, AT
2007 | srl AT, SFARG1LO, 21
2008 | or TMP1, TMP1, AT
2009 | slt AT, SFARG1HI, r0
2010 | beqz AT, >2
2011 |. srlv SFARG1LO, TMP1, TMP0
2012 | subu SFARG1LO, r0, SFARG1LO
2013 |2:
2014 | jr ra
2015 |. move CRET1, SFARG1LO
2016 |1:
2017 | addiu TMP0, TMP0, 21
2018 | srlv TMP1, SFARG1LO, TMP0
2019 | li AT, 20
2020 | subu TMP0, AT, TMP0
2021 | sll SFARG1LO, SFARG1HI, 12
2022 | sllv AT, SFARG1LO, TMP0
2023 | or SFARG1LO, TMP1, AT
2024 | slt AT, SFARG1HI, r0
2025 | beqz AT, <2
1804 |. nop 2026 |. nop
1805 | 2027 | jr ra
1806 |//-- Bit library -------------------------------------------------------- 2028 |. subu CRET1, r0, SFARG1LO
2029 |.endif
1807 | 2030 |
1808 |.macro .ffunc_bit, name 2031 |.macro .ffunc_bit, name
1809 | .ffunc_n bit_..name 2032 | .ffunc_1 bit_..name
1810 |. add.d FARG1, FARG1, TOBIT 2033 | beq SFARG1HI, TISNUM, >6
1811 | mfc1 CRET1, FARG1 2034 |. move CRET1, SFARG1LO
2035 | bal ->vm_tobit_fb
2036 |. sltu TMP1, SFARG1HI, TISNUM
2037 |6:
1812 |.endmacro 2038 |.endmacro
1813 | 2039 |
1814 |.macro .ffunc_bit_op, name, ins 2040 |.macro .ffunc_bit_op, name, ins
1815 | .ffunc_bit name 2041 | .ffunc_bit name
1816 | addiu TMP1, BASE, 8 2042 | addiu TMP2, BASE, 8
1817 | addu TMP2, BASE, NARGS8:RC 2043 | addu TMP3, BASE, NARGS8:RC
1818 |1: 2044 |1:
1819 | lw CARG4, HI(TMP1) 2045 | lw SFARG1HI, HI(TMP2)
1820 | beq TMP1, TMP2, ->fff_resi 2046 | beq TMP2, TMP3, ->fff_resi
1821 |. ldc1 FARG1, 0(TMP1) 2047 |. lw SFARG1LO, LO(TMP2)
1822 | sltiu AT, CARG4, LJ_TISNUM 2048 |.if FPU
1823 | beqz AT, ->fff_fallback 2049 | bne SFARG1HI, TISNUM, >2
1824 | add.d FARG1, FARG1, TOBIT 2050 |. addiu TMP2, TMP2, 8
1825 | mfc1 CARG2, FARG1
1826 | ins CRET1, CRET1, CARG2
1827 | b <1 2051 | b <1
1828 |. addiu TMP1, TMP1, 8 2052 |. ins CRET1, CRET1, SFARG1LO
2053 |2:
2054 | ldc1 FARG1, -8(TMP2)
2055 | sltu TMP1, SFARG1HI, TISNUM
2056 | beqz TMP1, ->fff_fallback
2057 |. add.d FARG1, FARG1, TOBIT
2058 | mfc1 SFARG1LO, FARG1
2059 | b <1
2060 |. ins CRET1, CRET1, SFARG1LO
2061 |.else
2062 | beq SFARG1HI, TISNUM, >2
2063 |. move CRET2, CRET1
2064 | bal ->vm_tobit_fb
2065 |. sltu TMP1, SFARG1HI, TISNUM
2066 | move SFARG1LO, CRET2
2067 |2:
2068 | ins CRET1, CRET1, SFARG1LO
2069 | b <1
2070 |. addiu TMP2, TMP2, 8
2071 |.endif
1829 |.endmacro 2072 |.endmacro
1830 | 2073 |
1831 |.ffunc_bit_op band, and 2074 |.ffunc_bit_op band, and
@@ -1849,24 +2092,28 @@ static void build_subroutines(BuildCtx *ctx)
1849 |. not CRET1, CRET1 2092 |. not CRET1, CRET1
1850 | 2093 |
1851 |.macro .ffunc_bit_sh, name, ins, shmod 2094 |.macro .ffunc_bit_sh, name, ins, shmod
1852 | .ffunc_nn bit_..name 2095 | .ffunc_2 bit_..name
1853 |. add.d FARG1, FARG1, TOBIT 2096 | beq SFARG1HI, TISNUM, >1
1854 | add.d FARG2, FARG2, TOBIT 2097 |. nop
1855 | mfc1 CARG1, FARG1 2098 | bal ->vm_tobit_fb
1856 | mfc1 CARG2, FARG2 2099 |. sltu TMP1, SFARG1HI, TISNUM
2100 | move SFARG1LO, CRET1
2101 |1:
2102 | bne SFARG2HI, TISNUM, ->fff_fallback
2103 |. nop
1857 |.if shmod == 1 2104 |.if shmod == 1
1858 | li AT, 32 2105 | li AT, 32
1859 | subu TMP0, AT, CARG2 2106 | subu TMP0, AT, SFARG2LO
1860 | sllv CARG2, CARG1, CARG2 2107 | sllv SFARG2LO, SFARG1LO, SFARG2LO
1861 | srlv CARG1, CARG1, TMP0 2108 | srlv SFARG1LO, SFARG1LO, TMP0
1862 |.elif shmod == 2 2109 |.elif shmod == 2
1863 | li AT, 32 2110 | li AT, 32
1864 | subu TMP0, AT, CARG2 2111 | subu TMP0, AT, SFARG2LO
1865 | srlv CARG2, CARG1, CARG2 2112 | srlv SFARG2LO, SFARG1LO, SFARG2LO
1866 | sllv CARG1, CARG1, TMP0 2113 | sllv SFARG1LO, SFARG1LO, TMP0
1867 |.endif 2114 |.endif
1868 | b ->fff_resi 2115 | b ->fff_resi
1869 |. ins CRET1, CARG1, CARG2 2116 |. ins CRET1, SFARG1LO, SFARG2LO
1870 |.endmacro 2117 |.endmacro
1871 | 2118 |
1872 |.ffunc_bit_sh lshift, sllv, 0 2119 |.ffunc_bit_sh lshift, sllv, 0
@@ -1878,9 +2125,11 @@ static void build_subroutines(BuildCtx *ctx)
1878 | 2125 |
1879 |.ffunc_bit tobit 2126 |.ffunc_bit tobit
1880 |->fff_resi: 2127 |->fff_resi:
1881 | mtc1 CRET1, FRET1 2128 | lw PC, FRAME_PC(BASE)
1882 | b ->fff_resn 2129 | addiu RA, BASE, -8
1883 |. cvt.d.w FRET1, FRET1 2130 | sw TISNUM, -8+HI(BASE)
2131 | b ->fff_res1
2132 |. sw CRET1, -8+LO(BASE)
1884 | 2133 |
1885 |//----------------------------------------------------------------------- 2134 |//-----------------------------------------------------------------------
1886 | 2135 |
@@ -2067,19 +2316,96 @@ static void build_subroutines(BuildCtx *ctx)
2067 | jr CRET1 2316 | jr CRET1
2068 |. lw INS, -4(PC) 2317 |. lw INS, -4(PC)
2069 | 2318 |
2319 |->cont_stitch: // Trace stitching.
2320 |.if JIT
2321 | // RA = resultptr, RB = meta base
2322 | lw INS, -4(PC)
2323 | lw TMP2, -24+LO(RB) // Save previous trace.
2324 | decode_RA8a RC, INS
2325 | addiu AT, MULTRES, -8
2326 | decode_RA8b RC
2327 | beqz AT, >2
2328 |. addu RC, BASE, RC // Call base.
2329 |1: // Move results down.
2330 | lw SFRETHI, HI(RA)
2331 | lw SFRETLO, LO(RA)
2332 | addiu AT, AT, -8
2333 | addiu RA, RA, 8
2334 | sw SFRETHI, HI(RC)
2335 | sw SFRETLO, LO(RC)
2336 | bnez AT, <1
2337 |. addiu RC, RC, 8
2338 |2:
2339 | decode_RA8a RA, INS
2340 | decode_RB8a RB, INS
2341 | decode_RA8b RA
2342 | decode_RB8b RB
2343 | addu RA, RA, RB
2344 | addu RA, BASE, RA
2345 |3:
2346 | sltu AT, RC, RA
2347 | bnez AT, >9 // More results wanted?
2348 |. nop
2349 |
2350 | lhu TMP3, TRACE:TMP2->traceno
2351 | lhu RD, TRACE:TMP2->link
2352 | beq RD, TMP3, ->cont_nop // Blacklisted.
2353 |. load_got lj_dispatch_stitch
2354 | bnez RD, =>BC_JLOOP // Jump to stitched trace.
2355 |. sll RD, RD, 3
2356 |
2357 | // Stitch a new trace to the previous trace.
2358 | sw TMP3, DISPATCH_J(exitno)(DISPATCH)
2359 | sw L, DISPATCH_J(L)(DISPATCH)
2360 | sw BASE, L->base
2361 | addiu CARG1, DISPATCH, GG_DISP2J
2362 | call_intern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
2363 |. move CARG2, PC
2364 | b ->cont_nop
2365 |. lw BASE, L->base
2366 |
2367 |9:
2368 | sw TISNIL, HI(RC)
2369 | b <3
2370 |. addiu RC, RC, 8
2371 |.endif
2372 |
2373 |->vm_profhook: // Dispatch target for profiler hook.
2374#if LJ_HASPROFILE
2375 | load_got lj_dispatch_profile
2376 | sw MULTRES, SAVE_MULTRES
2377 | move CARG2, PC
2378 | sw BASE, L->base
2379 | call_intern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
2380 |. move CARG1, L
2381 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2382 | addiu PC, PC, -4
2383 | b ->cont_nop
2384 |. lw BASE, L->base
2385#endif
2386 |
2070 |//----------------------------------------------------------------------- 2387 |//-----------------------------------------------------------------------
2071 |//-- Trace exit handler ------------------------------------------------- 2388 |//-- Trace exit handler -------------------------------------------------
2072 |//----------------------------------------------------------------------- 2389 |//-----------------------------------------------------------------------
2073 | 2390 |
2074 |.macro savex_, a, b 2391 |.macro savex_, a, b
2392 |.if FPU
2075 | sdc1 f..a, 16+a*8(sp) 2393 | sdc1 f..a, 16+a*8(sp)
2076 | sw r..a, 16+32*8+a*4(sp) 2394 | sw r..a, 16+32*8+a*4(sp)
2077 | sw r..b, 16+32*8+b*4(sp) 2395 | sw r..b, 16+32*8+b*4(sp)
2396 |.else
2397 | sw r..a, 16+a*4(sp)
2398 | sw r..b, 16+b*4(sp)
2399 |.endif
2078 |.endmacro 2400 |.endmacro
2079 | 2401 |
2080 |->vm_exit_handler: 2402 |->vm_exit_handler:
2081 |.if JIT 2403 |.if JIT
2404 |.if FPU
2082 | addiu sp, sp, -(16+32*8+32*4) 2405 | addiu sp, sp, -(16+32*8+32*4)
2406 |.else
2407 | addiu sp, sp, -(16+32*4)
2408 |.endif
2083 | savex_ 0, 1 2409 | savex_ 0, 1
2084 | savex_ 2, 3 2410 | savex_ 2, 3
2085 | savex_ 4, 5 2411 | savex_ 4, 5
@@ -2094,25 +2420,34 @@ static void build_subroutines(BuildCtx *ctx)
2094 | savex_ 22, 23 2420 | savex_ 22, 23
2095 | savex_ 24, 25 2421 | savex_ 24, 25
2096 | savex_ 26, 27 2422 | savex_ 26, 27
2423 |.if FPU
2097 | sdc1 f28, 16+28*8(sp) 2424 | sdc1 f28, 16+28*8(sp)
2098 | sw r28, 16+32*8+28*4(sp)
2099 | sdc1 f30, 16+30*8(sp) 2425 | sdc1 f30, 16+30*8(sp)
2426 | sw r28, 16+32*8+28*4(sp)
2100 | sw r30, 16+32*8+30*4(sp) 2427 | sw r30, 16+32*8+30*4(sp)
2101 | sw r0, 16+32*8+31*4(sp) // Clear RID_TMP. 2428 | sw r0, 16+32*8+31*4(sp) // Clear RID_TMP.
2429 | addiu TMP2, sp, 16+32*8+32*4 // Recompute original value of sp.
2430 | sw TMP2, 16+32*8+29*4(sp) // Store sp in RID_SP
2431 |.else
2432 | sw r28, 16+28*4(sp)
2433 | sw r30, 16+30*4(sp)
2434 | sw r0, 16+31*4(sp) // Clear RID_TMP.
2435 | addiu TMP2, sp, 16+32*4 // Recompute original value of sp.
2436 | sw TMP2, 16+29*4(sp) // Store sp in RID_SP
2437 |.endif
2102 | li_vmstate EXIT 2438 | li_vmstate EXIT
2103 | addiu TMP2, sp, 16+32*8+32*4 // Recompute original value of sp.
2104 | addiu DISPATCH, JGL, -GG_DISP2G-32768 2439 | addiu DISPATCH, JGL, -GG_DISP2G-32768
2105 | lw TMP1, 0(TMP2) // Load exit number. 2440 | lw TMP1, 0(TMP2) // Load exit number.
2106 | st_vmstate 2441 | st_vmstate
2107 | sw TMP2, 16+32*8+29*4(sp) // Store sp in RID_SP. 2442 | lw L, DISPATCH_GL(cur_L)(DISPATCH)
2108 | lw L, DISPATCH_GL(jit_L)(DISPATCH) 2443 | lw BASE, DISPATCH_GL(jit_base)(DISPATCH)
2109 | lw BASE, DISPATCH_GL(jit_base)(DISPATCH)
2110 | load_got lj_trace_exit 2444 | load_got lj_trace_exit
2111 | sw L, DISPATCH_J(L)(DISPATCH) 2445 | sw L, DISPATCH_J(L)(DISPATCH)
2112 | sw ra, DISPATCH_J(parent)(DISPATCH) // Store trace number. 2446 | sw ra, DISPATCH_J(parent)(DISPATCH) // Store trace number.
2447 | sw BASE, L->base
2113 | sw TMP1, DISPATCH_J(exitno)(DISPATCH) // Store exit number. 2448 | sw TMP1, DISPATCH_J(exitno)(DISPATCH) // Store exit number.
2114 | addiu CARG1, DISPATCH, GG_DISP2J 2449 | addiu CARG1, DISPATCH, GG_DISP2J
2115 | sw BASE, L->base 2450 | sw r0, DISPATCH_GL(jit_base)(DISPATCH)
2116 | call_intern lj_trace_exit // (jit_State *J, ExitState *ex) 2451 | call_intern lj_trace_exit // (jit_State *J, ExitState *ex)
2117 |. addiu CARG2, sp, 16 2452 |. addiu CARG2, sp, 16
2118 | // Returns MULTRES (unscaled) or negated error code. 2453 | // Returns MULTRES (unscaled) or negated error code.
@@ -2128,29 +2463,34 @@ static void build_subroutines(BuildCtx *ctx)
2128 |.if JIT 2463 |.if JIT
2129 | // CRET1 = MULTRES or negated error code, BASE, PC and JGL set. 2464 | // CRET1 = MULTRES or negated error code, BASE, PC and JGL set.
2130 | lw L, SAVE_L 2465 | lw L, SAVE_L
2131 | addiu DISPATCH, JGL, -GG_DISP2G-32768 2466 | addiu DISPATCH, JGL, -GG_DISP2G-32768
2467 | sw BASE, L->base
2132 |1: 2468 |1:
2133 | bltz CRET1, >3 // Check for error from exit. 2469 | sltiu TMP0, CRET1, -LUA_ERRERR // Check for error from exit.
2134 |. lw LFUNC:TMP1, FRAME_FUNC(BASE) 2470 | beqz TMP0, >9
2135 | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 2471 |. lw LFUNC:RB, FRAME_FUNC(BASE)
2472 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
2136 | sll MULTRES, CRET1, 3 2473 | sll MULTRES, CRET1, 3
2137 | li TISNIL, LJ_TNIL 2474 | li TISNIL, LJ_TNIL
2475 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
2138 | sw MULTRES, SAVE_MULTRES 2476 | sw MULTRES, SAVE_MULTRES
2139 | mtc1 TMP3, TOBIT 2477 | .FPU mtc1 TMP3, TOBIT
2140 | lw TMP1, LFUNC:TMP1->pc 2478 | lw TMP1, LFUNC:RB->pc
2141 | sw r0, DISPATCH_GL(jit_L)(DISPATCH) 2479 | sw r0, DISPATCH_GL(jit_base)(DISPATCH)
2142 | lw KBASE, PC2PROTO(k)(TMP1) 2480 | lw KBASE, PC2PROTO(k)(TMP1)
2143 | cvt.d.s TOBIT, TOBIT 2481 | .FPU cvt.d.s TOBIT, TOBIT
2144 | // Modified copy of ins_next which handles function header dispatch, too. 2482 | // Modified copy of ins_next which handles function header dispatch, too.
2145 | lw INS, 0(PC) 2483 | lw INS, 0(PC)
2146 | addiu PC, PC, 4 2484 | addiu CRET1, CRET1, 17 // Static dispatch?
2147 | // Assumes TISNIL == ~LJ_VMST_INTERP == -1 2485 | // Assumes TISNIL == ~LJ_VMST_INTERP == -1
2148 | sw TISNIL, DISPATCH_GL(vmstate)(DISPATCH) 2486 | sw TISNIL, DISPATCH_GL(vmstate)(DISPATCH)
2487 | decode_RD8a RD, INS
2488 | beqz CRET1, >5
2489 |. addiu PC, PC, 4
2149 | decode_OP4a TMP1, INS 2490 | decode_OP4a TMP1, INS
2150 | decode_OP4b TMP1 2491 | decode_OP4b TMP1
2151 | sltiu TMP2, TMP1, BC_FUNCF*4 // Function header?
2152 | addu TMP0, DISPATCH, TMP1 2492 | addu TMP0, DISPATCH, TMP1
2153 | decode_RD8a RD, INS 2493 | sltiu TMP2, TMP1, BC_FUNCF*4
2154 | lw AT, 0(TMP0) 2494 | lw AT, 0(TMP0)
2155 | decode_RA8a RA, INS 2495 | decode_RA8a RA, INS
2156 | beqz TMP2, >2 2496 | beqz TMP2, >2
@@ -2158,13 +2498,46 @@ static void build_subroutines(BuildCtx *ctx)
2158 | jr AT 2498 | jr AT
2159 |. decode_RD8b RD 2499 |. decode_RD8b RD
2160 |2: 2500 |2:
2501 | sltiu TMP2, TMP1, (BC_FUNCC+2)*4 // Fast function?
2502 | bnez TMP2, >3
2503 |. lw TMP1, FRAME_PC(BASE)
2504 | // Check frame below fast function.
2505 | andi TMP0, TMP1, FRAME_TYPE
2506 | bnez TMP0, >3 // Trace stitching continuation?
2507 |. nop
2508 | // Otherwise set KBASE for Lua function below fast function.
2509 | lw TMP2, -4(TMP1)
2510 | decode_RA8a TMP0, TMP2
2511 | decode_RA8b TMP0
2512 | subu TMP1, BASE, TMP0
2513 | lw LFUNC:TMP2, -8+FRAME_FUNC(TMP1)
2514 | lw TMP1, LFUNC:TMP2->pc
2515 | lw KBASE, PC2PROTO(k)(TMP1)
2516 |3:
2161 | addiu RC, MULTRES, -8 2517 | addiu RC, MULTRES, -8
2162 | jr AT 2518 | jr AT
2163 |. addu RA, RA, BASE 2519 |. addu RA, RA, BASE
2164 | 2520 |
2165 |3: // Rethrow error from the right C frame. 2521 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2166 | load_got lj_err_run 2522 | lw TMP0, DISPATCH_J(trace)(DISPATCH)
2167 | call_intern lj_err_run // (lua_State *L) 2523 | decode_RD4b RD
2524 | addu TMP0, TMP0, RD
2525 | lw TRACE:TMP2, 0(TMP0)
2526 | lw INS, TRACE:TMP2->startins
2527 | decode_OP4a TMP1, INS
2528 | decode_OP4b TMP1
2529 | addu TMP0, DISPATCH, TMP1
2530 | decode_RD8a RD, INS
2531 | lw AT, GG_DISP2STATIC(TMP0)
2532 | decode_RA8a RA, INS
2533 | decode_RD8b RD
2534 | jr AT
2535 |. decode_RA8b RA
2536 |
2537 |9: // Rethrow error from the right C frame.
2538 | load_got lj_err_trace
2539 | sub CARG2, r0, CRET1
2540 | call_intern lj_err_trace // (lua_State *L, int errcode)
2168 |. move CARG1, L 2541 |. move CARG1, L
2169 |.endif 2542 |.endif
2170 | 2543 |
@@ -2172,8 +2545,9 @@ static void build_subroutines(BuildCtx *ctx)
2172 |//-- Math helper functions ---------------------------------------------- 2545 |//-- Math helper functions ----------------------------------------------
2173 |//----------------------------------------------------------------------- 2546 |//-----------------------------------------------------------------------
2174 | 2547 |
2548 |// Hard-float round to integer.
2175 |// Modifies AT, TMP0, FRET1, FRET2, f4. Keeps all others incl. FARG1. 2549 |// Modifies AT, TMP0, FRET1, FRET2, f4. Keeps all others incl. FARG1.
2176 |.macro vm_round, func 2550 |.macro vm_round_hf, func
2177 | lui TMP0, 0x4330 // Hiword of 2^52 (double). 2551 | lui TMP0, 0x4330 // Hiword of 2^52 (double).
2178 | mtc1 r0, f4 2552 | mtc1 r0, f4
2179 | mtc1 TMP0, f5 2553 | mtc1 TMP0, f5
@@ -2215,6 +2589,12 @@ static void build_subroutines(BuildCtx *ctx)
2215 |. mov.d FRET1, FARG1 2589 |. mov.d FRET1, FARG1
2216 |.endmacro 2590 |.endmacro
2217 | 2591 |
2592 |.macro vm_round, func
2593 |.if FPU
2594 | vm_round_hf, func
2595 |.endif
2596 |.endmacro
2597 |
2218 |->vm_floor: 2598 |->vm_floor:
2219 | vm_round floor 2599 | vm_round floor
2220 |->vm_ceil: 2600 |->vm_ceil:
@@ -2224,10 +2604,286 @@ static void build_subroutines(BuildCtx *ctx)
2224 | vm_round trunc 2604 | vm_round trunc
2225 |.endif 2605 |.endif
2226 | 2606 |
2607 |// Soft-float integer to number conversion.
2608 |.macro sfi2d, AHI, ALO
2609 |.if not FPU
2610 | beqz ALO, >9 // Handle zero first.
2611 |. sra TMP0, ALO, 31
2612 | xor TMP1, ALO, TMP0
2613 | subu TMP1, TMP1, TMP0 // Absolute value in TMP1.
2614 | clz AHI, TMP1
2615 | andi TMP0, TMP0, 0x800 // Mask sign bit.
2616 | li AT, 0x3ff+31-1
2617 | sllv TMP1, TMP1, AHI // Align mantissa left with leading 1.
2618 | subu AHI, AT, AHI // Exponent - 1 in AHI.
2619 | sll ALO, TMP1, 21
2620 | or AHI, AHI, TMP0 // Sign | Exponent.
2621 | srl TMP1, TMP1, 11
2622 | sll AHI, AHI, 20 // Align left.
2623 | jr ra
2624 |. addu AHI, AHI, TMP1 // Add mantissa, increment exponent.
2625 |9:
2626 | jr ra
2627 |. li AHI, 0
2628 |.endif
2629 |.endmacro
2630 |
2631 |// Input SFARG1LO. Output: SFARG1*. Temporaries: AT, TMP0, TMP1.
2632 |->vm_sfi2d_1:
2633 | sfi2d SFARG1HI, SFARG1LO
2634 |
2635 |// Input SFARG2LO. Output: SFARG2*. Temporaries: AT, TMP0, TMP1.
2636 |->vm_sfi2d_2:
2637 | sfi2d SFARG2HI, SFARG2LO
2638 |
2639 |// Soft-float comparison. Equivalent to c.eq.d.
2640 |// Input: SFARG*. Output: CRET1. Temporaries: AT, TMP0, TMP1.
2641 |->vm_sfcmpeq:
2642 |.if not FPU
2643 | sll AT, SFARG1HI, 1
2644 | sll TMP0, SFARG2HI, 1
2645 | or CRET1, SFARG1LO, SFARG2LO
2646 | or TMP1, AT, TMP0
2647 | or TMP1, TMP1, CRET1
2648 | beqz TMP1, >8 // Both args +-0: return 1.
2649 |. sltu CRET1, r0, SFARG1LO
2650 | lui TMP1, 0xffe0
2651 | addu AT, AT, CRET1
2652 | sltu CRET1, r0, SFARG2LO
2653 | sltu AT, TMP1, AT
2654 | addu TMP0, TMP0, CRET1
2655 | sltu TMP0, TMP1, TMP0
2656 | or TMP1, AT, TMP0
2657 | bnez TMP1, >9 // Either arg is NaN: return 0;
2658 |. xor TMP0, SFARG1HI, SFARG2HI
2659 | xor TMP1, SFARG1LO, SFARG2LO
2660 | or AT, TMP0, TMP1
2661 | jr ra
2662 |. sltiu CRET1, AT, 1 // Same values: return 1.
2663 |8:
2664 | jr ra
2665 |. li CRET1, 1
2666 |9:
2667 | jr ra
2668 |. li CRET1, 0
2669 |.endif
2670 |
2671 |// Soft-float comparison. Equivalent to c.ult.d and c.olt.d.
2672 |// Input: SFARG*. Output: CRET1. Temporaries: AT, TMP0, TMP1, CRET2.
2673 |->vm_sfcmpult:
2674 |.if not FPU
2675 | b >1
2676 |. li CRET2, 1
2677 |.endif
2678 |
2679 |->vm_sfcmpolt:
2680 |.if not FPU
2681 | li CRET2, 0
2682 |1:
2683 | sll AT, SFARG1HI, 1
2684 | sll TMP0, SFARG2HI, 1
2685 | or CRET1, SFARG1LO, SFARG2LO
2686 | or TMP1, AT, TMP0
2687 | or TMP1, TMP1, CRET1
2688 | beqz TMP1, >8 // Both args +-0: return 0.
2689 |. sltu CRET1, r0, SFARG1LO
2690 | lui TMP1, 0xffe0
2691 | addu AT, AT, CRET1
2692 | sltu CRET1, r0, SFARG2LO
2693 | sltu AT, TMP1, AT
2694 | addu TMP0, TMP0, CRET1
2695 | sltu TMP0, TMP1, TMP0
2696 | or TMP1, AT, TMP0
2697 | bnez TMP1, >9 // Either arg is NaN: return 0 or 1;
2698 |. and AT, SFARG1HI, SFARG2HI
2699 | bltz AT, >5 // Both args negative?
2700 |. nop
2701 | beq SFARG1HI, SFARG2HI, >8
2702 |. sltu CRET1, SFARG1LO, SFARG2LO
2703 | jr ra
2704 |. slt CRET1, SFARG1HI, SFARG2HI
2705 |5: // Swap conditions if both operands are negative.
2706 | beq SFARG1HI, SFARG2HI, >8
2707 |. sltu CRET1, SFARG2LO, SFARG1LO
2708 | jr ra
2709 |. slt CRET1, SFARG2HI, SFARG1HI
2710 |8:
2711 | jr ra
2712 |. nop
2713 |9:
2714 | jr ra
2715 |. move CRET1, CRET2
2716 |.endif
2717 |
2718 |->vm_sfcmpogt:
2719 |.if not FPU
2720 | sll AT, SFARG2HI, 1
2721 | sll TMP0, SFARG1HI, 1
2722 | or CRET1, SFARG2LO, SFARG1LO
2723 | or TMP1, AT, TMP0
2724 | or TMP1, TMP1, CRET1
2725 | beqz TMP1, >8 // Both args +-0: return 0.
2726 |. sltu CRET1, r0, SFARG2LO
2727 | lui TMP1, 0xffe0
2728 | addu AT, AT, CRET1
2729 | sltu CRET1, r0, SFARG1LO
2730 | sltu AT, TMP1, AT
2731 | addu TMP0, TMP0, CRET1
2732 | sltu TMP0, TMP1, TMP0
2733 | or TMP1, AT, TMP0
2734 | bnez TMP1, >9 // Either arg is NaN: return 0 or 1;
2735 |. and AT, SFARG2HI, SFARG1HI
2736 | bltz AT, >5 // Both args negative?
2737 |. nop
2738 | beq SFARG2HI, SFARG1HI, >8
2739 |. sltu CRET1, SFARG2LO, SFARG1LO
2740 | jr ra
2741 |. slt CRET1, SFARG2HI, SFARG1HI
2742 |5: // Swap conditions if both operands are negative.
2743 | beq SFARG2HI, SFARG1HI, >8
2744 |. sltu CRET1, SFARG1LO, SFARG2LO
2745 | jr ra
2746 |. slt CRET1, SFARG1HI, SFARG2HI
2747 |8:
2748 | jr ra
2749 |. nop
2750 |9:
2751 | jr ra
2752 |. li CRET1, 0
2753 |.endif
2754 |
2755 |// Soft-float comparison. Equivalent to c.ole.d a, b or c.ole.d b, a.
2756 |// Input: SFARG*, TMP3. Output: CRET1. Temporaries: AT, TMP0, TMP1.
2757 |->vm_sfcmpolex:
2758 |.if not FPU
2759 | sll AT, SFARG1HI, 1
2760 | sll TMP0, SFARG2HI, 1
2761 | or CRET1, SFARG1LO, SFARG2LO
2762 | or TMP1, AT, TMP0
2763 | or TMP1, TMP1, CRET1
2764 | beqz TMP1, >8 // Both args +-0: return 1.
2765 |. sltu CRET1, r0, SFARG1LO
2766 | lui TMP1, 0xffe0
2767 | addu AT, AT, CRET1
2768 | sltu CRET1, r0, SFARG2LO
2769 | sltu AT, TMP1, AT
2770 | addu TMP0, TMP0, CRET1
2771 | sltu TMP0, TMP1, TMP0
2772 | or TMP1, AT, TMP0
2773 | bnez TMP1, >9 // Either arg is NaN: return 0;
2774 |. and AT, SFARG1HI, SFARG2HI
2775 | xor AT, AT, TMP3
2776 | bltz AT, >5 // Both args negative?
2777 |. nop
2778 | beq SFARG1HI, SFARG2HI, >6
2779 |. sltu CRET1, SFARG2LO, SFARG1LO
2780 | jr ra
2781 |. slt CRET1, SFARG2HI, SFARG1HI
2782 |5: // Swap conditions if both operands are negative.
2783 | beq SFARG1HI, SFARG2HI, >6
2784 |. sltu CRET1, SFARG1LO, SFARG2LO
2785 | slt CRET1, SFARG1HI, SFARG2HI
2786 |6:
2787 | jr ra
2788 |. nop
2789 |8:
2790 | jr ra
2791 |. li CRET1, 1
2792 |9:
2793 | jr ra
2794 |. li CRET1, 0
2795 |.endif
2796 |
2797 |.macro sfmin_max, name, fpcall
2798 |->vm_sf .. name:
2799 |.if JIT and not FPU
2800 | move TMP2, ra
2801 | bal ->fpcall
2802 |. nop
2803 | move TMP0, CRET1
2804 | move SFRETHI, SFARG1HI
2805 | move SFRETLO, SFARG1LO
2806 | move ra, TMP2
2807 | movz SFRETHI, SFARG2HI, TMP0
2808 | jr ra
2809 |. movz SFRETLO, SFARG2LO, TMP0
2810 |.endif
2811 |.endmacro
2812 |
2813 | sfmin_max min, vm_sfcmpolt
2814 | sfmin_max max, vm_sfcmpogt
2815 |
2227 |//----------------------------------------------------------------------- 2816 |//-----------------------------------------------------------------------
2228 |//-- Miscellaneous functions -------------------------------------------- 2817 |//-- Miscellaneous functions --------------------------------------------
2229 |//----------------------------------------------------------------------- 2818 |//-----------------------------------------------------------------------
2230 | 2819 |
2820 |.define NEXT_TAB, TAB:CARG1
2821 |.define NEXT_IDX, CARG2
2822 |.define NEXT_ASIZE, CARG3
2823 |.define NEXT_NIL, CARG4
2824 |.define NEXT_TMP0, r12
2825 |.define NEXT_TMP1, r13
2826 |.define NEXT_TMP2, r14
2827 |.define NEXT_RES_VK, CRET1
2828 |.define NEXT_RES_IDX, CRET2
2829 |.define NEXT_RES_PTR, sp
2830 |.define NEXT_RES_VAL_I, 0(sp)
2831 |.define NEXT_RES_VAL_IT, 4(sp)
2832 |.define NEXT_RES_KEY_I, 8(sp)
2833 |.define NEXT_RES_KEY_IT, 12(sp)
2834 |
2835 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2836 |// Next idx returned in CRET2.
2837 |->vm_next:
2838 |.if JIT and ENDIAN_LE
2839 | lw NEXT_ASIZE, NEXT_TAB->asize
2840 | lw NEXT_TMP0, NEXT_TAB->array
2841 | li NEXT_NIL, LJ_TNIL
2842 |1: // Traverse array part.
2843 | sltu AT, NEXT_IDX, NEXT_ASIZE
2844 | sll NEXT_TMP1, NEXT_IDX, 3
2845 | beqz AT, >5
2846 |. addu NEXT_TMP1, NEXT_TMP0, NEXT_TMP1
2847 | lw NEXT_TMP2, 4(NEXT_TMP1)
2848 | sw NEXT_IDX, NEXT_RES_KEY_I
2849 | beq NEXT_TMP2, NEXT_NIL, <1
2850 |. addiu NEXT_IDX, NEXT_IDX, 1
2851 | lw NEXT_TMP0, 0(NEXT_TMP1)
2852 | li AT, LJ_TISNUM
2853 | sw NEXT_TMP2, NEXT_RES_VAL_IT
2854 | sw AT, NEXT_RES_KEY_IT
2855 | sw NEXT_TMP0, NEXT_RES_VAL_I
2856 | move NEXT_RES_VK, NEXT_RES_PTR
2857 | jr ra
2858 |. move NEXT_RES_IDX, NEXT_IDX
2859 |
2860 |5: // Traverse hash part.
2861 | subu NEXT_RES_IDX, NEXT_IDX, NEXT_ASIZE
2862 | lw NODE:NEXT_RES_VK, NEXT_TAB->node
2863 | sll NEXT_TMP2, NEXT_RES_IDX, 5
2864 | lw NEXT_TMP0, NEXT_TAB->hmask
2865 | sll AT, NEXT_RES_IDX, 3
2866 | subu AT, NEXT_TMP2, AT
2867 | addu NODE:NEXT_RES_VK, NODE:NEXT_RES_VK, AT
2868 |6:
2869 | sltu AT, NEXT_TMP0, NEXT_RES_IDX
2870 | bnez AT, >8
2871 |. nop
2872 | lw NEXT_TMP2, NODE:NEXT_RES_VK->val.it
2873 | bne NEXT_TMP2, NEXT_NIL, >9
2874 |. addiu NEXT_RES_IDX, NEXT_RES_IDX, 1
2875 | // Skip holes in hash part.
2876 | b <6
2877 |. addiu NODE:NEXT_RES_VK, NODE:NEXT_RES_VK, sizeof(Node)
2878 |
2879 |8: // End of iteration. Set the key to nil (not the value).
2880 | sw NEXT_NIL, NEXT_RES_KEY_IT
2881 | move NEXT_RES_VK, NEXT_RES_PTR
2882 |9:
2883 | jr ra
2884 |. addu NEXT_RES_IDX, NEXT_RES_IDX, NEXT_ASIZE
2885 |.endif
2886 |
2231 |//----------------------------------------------------------------------- 2887 |//-----------------------------------------------------------------------
2232 |//-- FFI helper functions ----------------------------------------------- 2888 |//-- FFI helper functions -----------------------------------------------
2233 |//----------------------------------------------------------------------- 2889 |//-----------------------------------------------------------------------
@@ -2243,10 +2899,10 @@ static void build_subroutines(BuildCtx *ctx)
2243 | sw r1, CTSTATE->cb.slot 2899 | sw r1, CTSTATE->cb.slot
2244 | sw CARG1, CTSTATE->cb.gpr[0] 2900 | sw CARG1, CTSTATE->cb.gpr[0]
2245 | sw CARG2, CTSTATE->cb.gpr[1] 2901 | sw CARG2, CTSTATE->cb.gpr[1]
2246 | sdc1 FARG1, CTSTATE->cb.fpr[0] 2902 | .FPU sdc1 FARG1, CTSTATE->cb.fpr[0]
2247 | sw CARG3, CTSTATE->cb.gpr[2] 2903 | sw CARG3, CTSTATE->cb.gpr[2]
2248 | sw CARG4, CTSTATE->cb.gpr[3] 2904 | sw CARG4, CTSTATE->cb.gpr[3]
2249 | sdc1 FARG2, CTSTATE->cb.fpr[1] 2905 | .FPU sdc1 FARG2, CTSTATE->cb.fpr[1]
2250 | addiu TMP0, sp, CFRAME_SPACE+16 2906 | addiu TMP0, sp, CFRAME_SPACE+16
2251 | sw TMP0, CTSTATE->cb.stack 2907 | sw TMP0, CTSTATE->cb.stack
2252 | sw r0, SAVE_PC // Any value outside of bytecode is ok. 2908 | sw r0, SAVE_PC // Any value outside of bytecode is ok.
@@ -2256,15 +2912,16 @@ static void build_subroutines(BuildCtx *ctx)
2256 | // Returns lua_State *. 2912 | // Returns lua_State *.
2257 | lw BASE, L:CRET1->base 2913 | lw BASE, L:CRET1->base
2258 | lw RC, L:CRET1->top 2914 | lw RC, L:CRET1->top
2915 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
2259 | move L, CRET1 2916 | move L, CRET1
2260 | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 2917 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
2261 | lw LFUNC:RB, FRAME_FUNC(BASE) 2918 | lw LFUNC:RB, FRAME_FUNC(BASE)
2262 | mtc1 TMP3, TOBIT 2919 | .FPU mtc1 TMP3, TOBIT
2263 | li_vmstate INTERP 2920 | li_vmstate INTERP
2264 | li TISNIL, LJ_TNIL 2921 | li TISNIL, LJ_TNIL
2265 | subu RC, RC, BASE 2922 | subu RC, RC, BASE
2266 | st_vmstate 2923 | st_vmstate
2267 | cvt.d.s TOBIT, TOBIT 2924 | .FPU cvt.d.s TOBIT, TOBIT
2268 | ins_callt 2925 | ins_callt
2269 |.endif 2926 |.endif
2270 | 2927 |
@@ -2278,11 +2935,11 @@ static void build_subroutines(BuildCtx *ctx)
2278 | move CARG2, RA 2935 | move CARG2, RA
2279 | call_intern lj_ccallback_leave // (CTState *cts, TValue *o) 2936 | call_intern lj_ccallback_leave // (CTState *cts, TValue *o)
2280 |. move CARG1, CTSTATE 2937 |. move CARG1, CTSTATE
2938 | .FPU ldc1 FRET1, CTSTATE->cb.fpr[0]
2281 | lw CRET1, CTSTATE->cb.gpr[0] 2939 | lw CRET1, CTSTATE->cb.gpr[0]
2282 | ldc1 FRET1, CTSTATE->cb.fpr[0] 2940 | .FPU ldc1 FRET2, CTSTATE->cb.fpr[1]
2283 | lw CRET2, CTSTATE->cb.gpr[1]
2284 | b ->vm_leave_unw 2941 | b ->vm_leave_unw
2285 |. ldc1 FRET2, CTSTATE->cb.fpr[1] 2942 |. lw CRET2, CTSTATE->cb.gpr[1]
2286 |.endif 2943 |.endif
2287 | 2944 |
2288 |->vm_ffi_call: // Call C function via FFI. 2945 |->vm_ffi_call: // Call C function via FFI.
@@ -2314,8 +2971,8 @@ static void build_subroutines(BuildCtx *ctx)
2314 | lw CARG2, CCSTATE->gpr[1] 2971 | lw CARG2, CCSTATE->gpr[1]
2315 | lw CARG3, CCSTATE->gpr[2] 2972 | lw CARG3, CCSTATE->gpr[2]
2316 | lw CARG4, CCSTATE->gpr[3] 2973 | lw CARG4, CCSTATE->gpr[3]
2317 | ldc1 FARG1, CCSTATE->fpr[0] 2974 | .FPU ldc1 FARG1, CCSTATE->fpr[0]
2318 | ldc1 FARG2, CCSTATE->fpr[1] 2975 | .FPU ldc1 FARG2, CCSTATE->fpr[1]
2319 | jalr CFUNCADDR 2976 | jalr CFUNCADDR
2320 |. lw CARG1, CCSTATE->gpr[0] // Do this last, since CCSTATE is CARG1. 2977 |. lw CARG1, CCSTATE->gpr[0] // Do this last, since CCSTATE is CARG1.
2321 | lw CCSTATE:TMP1, -12(r16) 2978 | lw CCSTATE:TMP1, -12(r16)
@@ -2323,8 +2980,13 @@ static void build_subroutines(BuildCtx *ctx)
2323 | lw ra, -4(r16) 2980 | lw ra, -4(r16)
2324 | sw CRET1, CCSTATE:TMP1->gpr[0] 2981 | sw CRET1, CCSTATE:TMP1->gpr[0]
2325 | sw CRET2, CCSTATE:TMP1->gpr[1] 2982 | sw CRET2, CCSTATE:TMP1->gpr[1]
2983 |.if FPU
2326 | sdc1 FRET1, CCSTATE:TMP1->fpr[0] 2984 | sdc1 FRET1, CCSTATE:TMP1->fpr[0]
2327 | sdc1 FRET2, CCSTATE:TMP1->fpr[1] 2985 | sdc1 FRET2, CCSTATE:TMP1->fpr[1]
2986 |.else
2987 | sw CARG1, CCSTATE:TMP1->gpr[2] // Soft-float: complex double .im part.
2988 | sw CARG2, CCSTATE:TMP1->gpr[3]
2989 |.endif
2328 | move sp, r16 2990 | move sp, r16
2329 | jr ra 2991 | jr ra
2330 |. move r16, TMP2 2992 |. move r16, TMP2
@@ -2348,82 +3010,143 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2348 3010
2349 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT: 3011 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2350 | // RA = src1*8, RD = src2*8, JMP with RD = target 3012 | // RA = src1*8, RD = src2*8, JMP with RD = target
2351 | addu CARG2, BASE, RA 3013 |.macro bc_comp, FRA, FRD, RAHI, RALO, RDHI, RDLO, movop, fmovop, fcomp, sfcomp
2352 | addu CARG3, BASE, RD 3014 | addu RA, BASE, RA
2353 | lw TMP0, HI(CARG2) 3015 | addu RD, BASE, RD
2354 | lw TMP1, HI(CARG3) 3016 | lw RAHI, HI(RA)
2355 | ldc1 f0, 0(CARG2) 3017 | lw RDHI, HI(RD)
2356 | ldc1 f2, 0(CARG3)
2357 | sltiu TMP0, TMP0, LJ_TISNUM
2358 | sltiu TMP1, TMP1, LJ_TISNUM
2359 | lhu TMP2, OFS_RD(PC) 3018 | lhu TMP2, OFS_RD(PC)
2360 | and TMP0, TMP0, TMP1
2361 | addiu PC, PC, 4 3019 | addiu PC, PC, 4
2362 | beqz TMP0, ->vmeta_comp 3020 | bne RAHI, TISNUM, >2
2363 |. lui TMP1, (-(BCBIAS_J*4 >> 16) & 65535) 3021 |. lw RALO, LO(RA)
2364 | decode_RD4b TMP2 3022 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
2365 | addu TMP2, TMP2, TMP1 3023 | lw RDLO, LO(RD)
2366 if (op == BC_ISLT || op == BC_ISGE) { 3024 | bne RDHI, TISNUM, >5
2367 | c.olt.d f0, f2 3025 |. decode_RD4b TMP2
2368 } else { 3026 | slt AT, SFARG1LO, SFARG2LO
2369 | c.ole.d f0, f2 3027 | addu TMP2, TMP2, TMP3
2370 } 3028 | movop TMP2, r0, AT
2371 if (op == BC_ISLT || op == BC_ISLE) {
2372 | movf TMP2, r0
2373 } else {
2374 | movt TMP2, r0
2375 }
2376 | addu PC, PC, TMP2
2377 |1: 3029 |1:
3030 | addu PC, PC, TMP2
2378 | ins_next 3031 | ins_next
3032 |
3033 |2: // RA is not an integer.
3034 | sltiu AT, RAHI, LJ_TISNUM
3035 | beqz AT, ->vmeta_comp
3036 |. lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3037 | sltiu AT, RDHI, LJ_TISNUM
3038 |.if FPU
3039 | ldc1 FRA, 0(RA)
3040 | ldc1 FRD, 0(RD)
3041 |.else
3042 | lw RDLO, LO(RD)
3043 |.endif
3044 | beqz AT, >4
3045 |. decode_RD4b TMP2
3046 |3: // RA and RD are both numbers.
3047 |.if FPU
3048 | fcomp f20, f22
3049 | addu TMP2, TMP2, TMP3
3050 | b <1
3051 |. fmovop TMP2, r0
3052 |.else
3053 | bal sfcomp
3054 |. addu TMP2, TMP2, TMP3
3055 | b <1
3056 |. movop TMP2, r0, CRET1
3057 |.endif
3058 |
3059 |4: // RA is a number, RD is not a number.
3060 | bne RDHI, TISNUM, ->vmeta_comp
3061 | // RA is a number, RD is an integer. Convert RD to a number.
3062 |.if FPU
3063 |. lwc1 FRD, LO(RD)
3064 | b <3
3065 |. cvt.d.w FRD, FRD
3066 |.else
3067 |. nop
3068 |.if "RDHI" == "SFARG1HI"
3069 | bal ->vm_sfi2d_1
3070 |.else
3071 | bal ->vm_sfi2d_2
3072 |.endif
3073 |. nop
3074 | b <3
3075 |. nop
3076 |.endif
3077 |
3078 |5: // RA is an integer, RD is not an integer
3079 | sltiu AT, RDHI, LJ_TISNUM
3080 | beqz AT, ->vmeta_comp
3081 | // RA is an integer, RD is a number. Convert RA to a number.
3082 |.if FPU
3083 |. mtc1 RALO, FRA
3084 | ldc1 FRD, 0(RD)
3085 | b <3
3086 | cvt.d.w FRA, FRA
3087 |.else
3088 |. nop
3089 |.if "RAHI" == "SFARG1HI"
3090 | bal ->vm_sfi2d_1
3091 |.else
3092 | bal ->vm_sfi2d_2
3093 |.endif
3094 |. nop
3095 | b <3
3096 |. nop
3097 |.endif
3098 |.endmacro
3099 |
3100 if (op == BC_ISLT) {
3101 | bc_comp f20, f22, SFARG1HI, SFARG1LO, SFARG2HI, SFARG2LO, movz, movf, c.olt.d, ->vm_sfcmpolt
3102 } else if (op == BC_ISGE) {
3103 | bc_comp f20, f22, SFARG1HI, SFARG1LO, SFARG2HI, SFARG2LO, movn, movt, c.olt.d, ->vm_sfcmpolt
3104 } else if (op == BC_ISLE) {
3105 | bc_comp f22, f20, SFARG2HI, SFARG2LO, SFARG1HI, SFARG1LO, movn, movt, c.ult.d, ->vm_sfcmpult
3106 } else {
3107 | bc_comp f22, f20, SFARG2HI, SFARG2LO, SFARG1HI, SFARG1LO, movz, movf, c.ult.d, ->vm_sfcmpult
3108 }
2379 break; 3109 break;
2380 3110
2381 case BC_ISEQV: case BC_ISNEV: 3111 case BC_ISEQV: case BC_ISNEV:
2382 vk = op == BC_ISEQV; 3112 vk = op == BC_ISEQV;
2383 | // RA = src1*8, RD = src2*8, JMP with RD = target 3113 | // RA = src1*8, RD = src2*8, JMP with RD = target
2384 | addu RA, BASE, RA 3114 | addu RA, BASE, RA
2385 | addiu PC, PC, 4 3115 | addiu PC, PC, 4
2386 | lw TMP0, HI(RA)
2387 | ldc1 f0, 0(RA)
2388 | addu RD, BASE, RD 3116 | addu RD, BASE, RD
3117 | lw SFARG1HI, HI(RA)
2389 | lhu TMP2, -4+OFS_RD(PC) 3118 | lhu TMP2, -4+OFS_RD(PC)
2390 | lw TMP1, HI(RD) 3119 | lw SFARG2HI, HI(RD)
2391 | ldc1 f2, 0(RD)
2392 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535) 3120 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
2393 | sltiu AT, TMP0, LJ_TISNUM 3121 | sltu AT, TISNUM, SFARG1HI
2394 | sltiu CARG1, TMP1, LJ_TISNUM 3122 | sltu TMP0, TISNUM, SFARG2HI
2395 | decode_RD4b TMP2 3123 | or AT, AT, TMP0
2396 | and AT, AT, CARG1
2397 | beqz AT, >5
2398 |. addu TMP2, TMP2, TMP3
2399 | c.eq.d f0, f2
2400 if (vk) { 3124 if (vk) {
2401 | movf TMP2, r0 3125 | beqz AT, ->BC_ISEQN_Z
2402 } else { 3126 } else {
2403 | movt TMP2, r0 3127 | beqz AT, ->BC_ISNEN_Z
2404 } 3128 }
2405 |1: 3129 |. decode_RD4b TMP2
2406 | addu PC, PC, TMP2 3130 | // Either or both types are not numbers.
2407 | ins_next 3131 | lw SFARG1LO, LO(RA)
2408 |5: // Either or both types are not numbers. 3132 | lw SFARG2LO, LO(RD)
2409 | lw CARG2, LO(RA) 3133 | addu TMP2, TMP2, TMP3
2410 | lw CARG3, LO(RD)
2411 |.if FFI 3134 |.if FFI
2412 | li TMP3, LJ_TCDATA 3135 | li TMP3, LJ_TCDATA
2413 | beq TMP0, TMP3, ->vmeta_equal_cd 3136 | beq SFARG1HI, TMP3, ->vmeta_equal_cd
2414 |.endif 3137 |.endif
2415 |. sltiu AT, TMP0, LJ_TISPRI // Not a primitive? 3138 |. sltiu AT, SFARG1HI, LJ_TISPRI // Not a primitive?
2416 |.if FFI 3139 |.if FFI
2417 | beq TMP1, TMP3, ->vmeta_equal_cd 3140 | beq SFARG2HI, TMP3, ->vmeta_equal_cd
2418 |.endif 3141 |.endif
2419 |. xor TMP3, CARG2, CARG3 // Same tv? 3142 |. xor TMP3, SFARG1LO, SFARG2LO // Same tv?
2420 | xor TMP1, TMP1, TMP0 // Same type? 3143 | xor SFARG2HI, SFARG2HI, SFARG1HI // Same type?
2421 | sltiu CARG1, TMP0, LJ_TISTABUD+1 // Table or userdata? 3144 | sltiu TMP0, SFARG1HI, LJ_TISTABUD+1 // Table or userdata?
2422 | movz TMP3, r0, AT // Ignore tv if primitive. 3145 | movz TMP3, r0, AT // Ignore tv if primitive.
2423 | movn CARG1, r0, TMP1 // Tab/ud and same type? 3146 | movn TMP0, r0, SFARG2HI // Tab/ud and same type?
2424 | or AT, TMP1, TMP3 // Same type && (pri||same tv). 3147 | or AT, SFARG2HI, TMP3 // Same type && (pri||same tv).
2425 | movz CARG1, r0, AT 3148 | movz TMP0, r0, AT
2426 | beqz CARG1, <1 // Done if not tab/ud or not same type or same tv. 3149 | beqz TMP0, >1 // Done if not tab/ud or not same type or same tv.
2427 if (vk) { 3150 if (vk) {
2428 |. movn TMP2, r0, AT 3151 |. movn TMP2, r0, AT
2429 } else { 3152 } else {
@@ -2431,15 +3154,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2431 } 3154 }
2432 | // Different tables or userdatas. Need to check __eq metamethod. 3155 | // Different tables or userdatas. Need to check __eq metamethod.
2433 | // Field metatable must be at same offset for GCtab and GCudata! 3156 | // Field metatable must be at same offset for GCtab and GCudata!
2434 | lw TAB:TMP1, TAB:CARG2->metatable 3157 | lw TAB:TMP1, TAB:SFARG1LO->metatable
2435 | beqz TAB:TMP1, <1 // No metatable? 3158 | beqz TAB:TMP1, >1 // No metatable?
2436 |. nop 3159 |. nop
2437 | lbu TMP1, TAB:TMP1->nomm 3160 | lbu TMP1, TAB:TMP1->nomm
2438 | andi TMP1, TMP1, 1<<MM_eq 3161 | andi TMP1, TMP1, 1<<MM_eq
2439 | bnez TMP1, <1 // Or 'no __eq' flag set? 3162 | bnez TMP1, >1 // Or 'no __eq' flag set?
2440 |. nop 3163 |. nop
2441 | b ->vmeta_equal // Handle __eq metamethod. 3164 | b ->vmeta_equal // Handle __eq metamethod.
2442 |. li CARG4, 1-vk // ne = 0 or 1. 3165 |. li TMP0, 1-vk // ne = 0 or 1.
3166 |1:
3167 | addu PC, PC, TMP2
3168 | ins_next
2443 break; 3169 break;
2444 3170
2445 case BC_ISEQS: case BC_ISNES: 3171 case BC_ISEQS: case BC_ISNES:
@@ -2476,38 +3202,124 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2476 vk = op == BC_ISEQN; 3202 vk = op == BC_ISEQN;
2477 | // RA = src*8, RD = num_const*8, JMP with RD = target 3203 | // RA = src*8, RD = num_const*8, JMP with RD = target
2478 | addu RA, BASE, RA 3204 | addu RA, BASE, RA
2479 | addiu PC, PC, 4 3205 | addu RD, KBASE, RD
2480 | lw TMP0, HI(RA) 3206 | lw SFARG1HI, HI(RA)
2481 | ldc1 f0, 0(RA) 3207 | lw SFARG2HI, HI(RD)
2482 | addu RD, KBASE, RD 3208 | lhu TMP2, OFS_RD(PC)
2483 | lhu TMP2, -4+OFS_RD(PC) 3209 | addiu PC, PC, 4
2484 | ldc1 f2, 0(RD)
2485 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535) 3210 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
2486 | sltiu AT, TMP0, LJ_TISNUM
2487 | decode_RD4b TMP2 3211 | decode_RD4b TMP2
2488 |.if FFI
2489 | beqz AT, >5
2490 |.else
2491 | beqz AT, >1
2492 |.endif
2493 |. addu TMP2, TMP2, TMP3
2494 | c.eq.d f0, f2
2495 if (vk) { 3212 if (vk) {
2496 | movf TMP2, r0 3213 |->BC_ISEQN_Z:
2497 | addu PC, PC, TMP2 3214 } else {
3215 |->BC_ISNEN_Z:
3216 }
3217 | bne SFARG1HI, TISNUM, >3
3218 |. lw SFARG1LO, LO(RA)
3219 | lw SFARG2LO, LO(RD)
3220 | addu TMP2, TMP2, TMP3
3221 | bne SFARG2HI, TISNUM, >6
3222 |. xor AT, SFARG1LO, SFARG2LO
3223 if (vk) {
3224 | movn TMP2, r0, AT
2498 |1: 3225 |1:
3226 | addu PC, PC, TMP2
3227 |2:
2499 } else { 3228 } else {
2500 | movt TMP2, r0 3229 | movz TMP2, r0, AT
2501 |1: 3230 |1:
3231 |2:
2502 | addu PC, PC, TMP2 3232 | addu PC, PC, TMP2
2503 } 3233 }
2504 | ins_next 3234 | ins_next
3235 |
3236 |3: // RA is not an integer.
3237 | sltiu AT, SFARG1HI, LJ_TISNUM
2505 |.if FFI 3238 |.if FFI
2506 |5: 3239 | beqz AT, >8
2507 | li AT, LJ_TCDATA 3240 |.else
2508 | beq TMP0, AT, ->vmeta_equal_cd 3241 | beqz AT, <2
3242 |.endif
3243 |. addu TMP2, TMP2, TMP3
3244 | sltiu AT, SFARG2HI, LJ_TISNUM
3245 |.if FPU
3246 | ldc1 f20, 0(RA)
3247 | ldc1 f22, 0(RD)
3248 |.endif
3249 | beqz AT, >5
3250 |. lw SFARG2LO, LO(RD)
3251 |4: // RA and RD are both numbers.
3252 |.if FPU
3253 | c.eq.d f20, f22
3254 | b <1
3255 if (vk) {
3256 |. movf TMP2, r0
3257 } else {
3258 |. movt TMP2, r0
3259 }
3260 |.else
3261 | bal ->vm_sfcmpeq
2509 |. nop 3262 |. nop
2510 | b <1 3263 | b <1
3264 if (vk) {
3265 |. movz TMP2, r0, CRET1
3266 } else {
3267 |. movn TMP2, r0, CRET1
3268 }
3269 |.endif
3270 |
3271 |5: // RA is a number, RD is not a number.
3272 |.if FFI
3273 | bne SFARG2HI, TISNUM, >9
3274 |.else
3275 | bne SFARG2HI, TISNUM, <2
3276 |.endif
3277 | // RA is a number, RD is an integer. Convert RD to a number.
3278 |.if FPU
3279 |. lwc1 f22, LO(RD)
3280 | b <4
3281 |. cvt.d.w f22, f22
3282 |.else
3283 |. nop
3284 | bal ->vm_sfi2d_2
3285 |. nop
3286 | b <4
3287 |. nop
3288 |.endif
3289 |
3290 |6: // RA is an integer, RD is not an integer
3291 | sltiu AT, SFARG2HI, LJ_TISNUM
3292 |.if FFI
3293 | beqz AT, >9
3294 |.else
3295 | beqz AT, <2
3296 |.endif
3297 | // RA is an integer, RD is a number. Convert RA to a number.
3298 |.if FPU
3299 |. mtc1 SFARG1LO, f20
3300 | ldc1 f22, 0(RD)
3301 | b <4
3302 | cvt.d.w f20, f20
3303 |.else
3304 |. nop
3305 | bal ->vm_sfi2d_1
3306 |. nop
3307 | b <4
3308 |. nop
3309 |.endif
3310 |
3311 |.if FFI
3312 |8:
3313 | li AT, LJ_TCDATA
3314 | bne SFARG1HI, AT, <2
3315 |. nop
3316 | b ->vmeta_equal_cd
3317 |. nop
3318 |9:
3319 | li AT, LJ_TCDATA
3320 | bne SFARG2HI, AT, <2
3321 |. nop
3322 | b ->vmeta_equal_cd
2511 |. nop 3323 |. nop
2512 |.endif 3324 |.endif
2513 break; 3325 break;
@@ -2559,7 +3371,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2559 | addu PC, PC, TMP2 3371 | addu PC, PC, TMP2
2560 } else { 3372 } else {
2561 | sltiu TMP0, TMP0, LJ_TISTRUECOND 3373 | sltiu TMP0, TMP0, LJ_TISTRUECOND
2562 | ldc1 f0, 0(RD) 3374 | lw SFRETHI, HI(RD)
3375 | lw SFRETLO, LO(RD)
2563 if (op == BC_ISTC) { 3376 if (op == BC_ISTC) {
2564 | beqz TMP0, >1 3377 | beqz TMP0, >1
2565 } else { 3378 } else {
@@ -2569,22 +3382,45 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2569 | decode_RD4b TMP2 3382 | decode_RD4b TMP2
2570 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535) 3383 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
2571 | addu TMP2, TMP2, TMP3 3384 | addu TMP2, TMP2, TMP3
2572 | sdc1 f0, 0(RA) 3385 | sw SFRETHI, HI(RA)
3386 | sw SFRETLO, LO(RA)
2573 | addu PC, PC, TMP2 3387 | addu PC, PC, TMP2
2574 |1: 3388 |1:
2575 } 3389 }
2576 | ins_next 3390 | ins_next
2577 break; 3391 break;
2578 3392
3393 case BC_ISTYPE:
3394 | // RA = src*8, RD = -type*8
3395 | addu TMP2, BASE, RA
3396 | srl TMP1, RD, 3
3397 | lw TMP0, HI(TMP2)
3398 | ins_next1
3399 | addu AT, TMP0, TMP1
3400 | bnez AT, ->vmeta_istype
3401 |. ins_next2
3402 break;
3403 case BC_ISNUM:
3404 | // RA = src*8, RD = -(TISNUM-1)*8
3405 | addu TMP2, BASE, RA
3406 | lw TMP0, HI(TMP2)
3407 | ins_next1
3408 | sltiu AT, TMP0, LJ_TISNUM
3409 | beqz AT, ->vmeta_istype
3410 |. ins_next2
3411 break;
3412
2579 /* -- Unary ops --------------------------------------------------------- */ 3413 /* -- Unary ops --------------------------------------------------------- */
2580 3414
2581 case BC_MOV: 3415 case BC_MOV:
2582 | // RA = dst*8, RD = src*8 3416 | // RA = dst*8, RD = src*8
2583 | addu RD, BASE, RD 3417 | addu RD, BASE, RD
2584 | addu RA, BASE, RA 3418 | addu RA, BASE, RA
2585 | ldc1 f0, 0(RD) 3419 | lw SFRETHI, HI(RD)
3420 | lw SFRETLO, LO(RD)
2586 | ins_next1 3421 | ins_next1
2587 | sdc1 f0, 0(RA) 3422 | sw SFRETHI, HI(RA)
3423 | sw SFRETLO, LO(RA)
2588 | ins_next2 3424 | ins_next2
2589 break; 3425 break;
2590 case BC_NOT: 3426 case BC_NOT:
@@ -2601,16 +3437,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2601 break; 3437 break;
2602 case BC_UNM: 3438 case BC_UNM:
2603 | // RA = dst*8, RD = src*8 3439 | // RA = dst*8, RD = src*8
2604 | addu CARG3, BASE, RD 3440 | addu RB, BASE, RD
3441 | lw SFARG1HI, HI(RB)
2605 | addu RA, BASE, RA 3442 | addu RA, BASE, RA
2606 | lw TMP0, HI(CARG3) 3443 | bne SFARG1HI, TISNUM, >2
2607 | ldc1 f0, 0(CARG3) 3444 |. lw SFARG1LO, LO(RB)
2608 | sltiu AT, TMP0, LJ_TISNUM 3445 | lui TMP1, 0x8000
2609 | beqz AT, ->vmeta_unm 3446 | beq SFARG1LO, TMP1, ->vmeta_unm // Meta handler deals with -2^31.
2610 |. neg.d f0, f0 3447 |. negu SFARG1LO, SFARG1LO
3448 |1:
2611 | ins_next1 3449 | ins_next1
2612 | sdc1 f0, 0(RA) 3450 | sw SFARG1HI, HI(RA)
3451 | sw SFARG1LO, LO(RA)
2613 | ins_next2 3452 | ins_next2
3453 |2:
3454 | sltiu AT, SFARG1HI, LJ_TISNUM
3455 | beqz AT, ->vmeta_unm
3456 |. lui TMP1, 0x8000
3457 | b <1
3458 |. xor SFARG1HI, SFARG1HI, TMP1
2614 break; 3459 break;
2615 case BC_LEN: 3460 case BC_LEN:
2616 | // RA = dst*8, RD = src*8 3461 | // RA = dst*8, RD = src*8
@@ -2621,12 +3466,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2621 | li AT, LJ_TSTR 3466 | li AT, LJ_TSTR
2622 | bne TMP0, AT, >2 3467 | bne TMP0, AT, >2
2623 |. li AT, LJ_TTAB 3468 |. li AT, LJ_TTAB
2624 | lw CRET1, STR:CARG1->len 3469 | lw CRET1, STR:CARG1->len
2625 |1: 3470 |1:
2626 | mtc1 CRET1, f0
2627 | cvt.d.w f0, f0
2628 | ins_next1 3471 | ins_next1
2629 | sdc1 f0, 0(RA) 3472 | sw TISNUM, HI(RA)
3473 | sw CRET1, LO(RA)
2630 | ins_next2 3474 | ins_next2
2631 |2: 3475 |2:
2632 | bne TMP0, AT, ->vmeta_len 3476 | bne TMP0, AT, ->vmeta_len
@@ -2657,104 +3501,232 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2657 3501
2658 /* -- Binary ops -------------------------------------------------------- */ 3502 /* -- Binary ops -------------------------------------------------------- */
2659 3503
2660 |.macro ins_arithpre 3504 |.macro fpmod, a, b, c
3505 | bal ->vm_floor // floor(b/c)
3506 |. div.d FARG1, b, c
3507 | mul.d a, FRET1, c
3508 | sub.d a, b, a // b - floor(b/c)*c
3509 |.endmacro
3510
3511 |.macro sfpmod
3512 | addiu sp, sp, -16
3513 |
3514 | load_got __divdf3
3515 | sw SFARG1HI, HI(sp)
3516 | sw SFARG1LO, LO(sp)
3517 | sw SFARG2HI, 8+HI(sp)
3518 | call_extern
3519 |. sw SFARG2LO, 8+LO(sp)
3520 |
3521 | load_got floor
3522 | move SFARG1HI, SFRETHI
3523 | call_extern
3524 |. move SFARG1LO, SFRETLO
3525 |
3526 | load_got __muldf3
3527 | move SFARG1HI, SFRETHI
3528 | move SFARG1LO, SFRETLO
3529 | lw SFARG2HI, 8+HI(sp)
3530 | call_extern
3531 |. lw SFARG2LO, 8+LO(sp)
3532 |
3533 | load_got __subdf3
3534 | lw SFARG1HI, HI(sp)
3535 | lw SFARG1LO, LO(sp)
3536 | move SFARG2HI, SFRETHI
3537 | call_extern
3538 |. move SFARG2LO, SFRETLO
3539 |
3540 | addiu sp, sp, 16
3541 |.endmacro
3542
3543 |.macro ins_arithpre, label
2661 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN); 3544 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2662 | decode_RB8a RB, INS
2663 | decode_RB8b RB
2664 | decode_RDtoRC8 RC, RD
2665 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8 3545 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
2666 ||switch (vk) { 3546 ||switch (vk) {
2667 ||case 0: 3547 ||case 0:
2668 | addu CARG3, BASE, RB 3548 | decode_RB8a RB, INS
2669 | addu CARG4, KBASE, RC 3549 | decode_RB8b RB
2670 | lw TMP1, HI(CARG3) 3550 | decode_RDtoRC8 RC, RD
2671 | ldc1 f20, 0(CARG3) 3551 | // RA = dst*8, RB = src1*8, RC = num_const*8
2672 | ldc1 f22, 0(CARG4) 3552 | addu RB, BASE, RB
2673 | sltiu AT, TMP1, LJ_TISNUM 3553 |.if "label" ~= "none"
3554 | b label
3555 |.endif
3556 |. addu RC, KBASE, RC
2674 || break; 3557 || break;
2675 ||case 1: 3558 ||case 1:
2676 | addu CARG4, BASE, RB 3559 | decode_RB8a RC, INS
2677 | addu CARG3, KBASE, RC 3560 | decode_RB8b RC
2678 | lw TMP1, HI(CARG4) 3561 | decode_RDtoRC8 RB, RD
2679 | ldc1 f22, 0(CARG4) 3562 | // RA = dst*8, RB = num_const*8, RC = src1*8
2680 | ldc1 f20, 0(CARG3) 3563 | addu RC, BASE, RC
2681 | sltiu AT, TMP1, LJ_TISNUM 3564 |.if "label" ~= "none"
3565 | b label
3566 |.endif
3567 |. addu RB, KBASE, RB
2682 || break; 3568 || break;
2683 ||default: 3569 ||default:
2684 | addu CARG3, BASE, RB 3570 | decode_RB8a RB, INS
2685 | addu CARG4, BASE, RC 3571 | decode_RB8b RB
2686 | lw TMP1, HI(CARG3) 3572 | decode_RDtoRC8 RC, RD
2687 | lw TMP2, HI(CARG4) 3573 | // RA = dst*8, RB = src1*8, RC = src2*8
2688 | ldc1 f20, 0(CARG3) 3574 | addu RB, BASE, RB
2689 | ldc1 f22, 0(CARG4) 3575 |.if "label" ~= "none"
2690 | sltiu AT, TMP1, LJ_TISNUM 3576 | b label
2691 | sltiu TMP0, TMP2, LJ_TISNUM 3577 |.endif
2692 | and AT, AT, TMP0 3578 |. addu RC, BASE, RC
2693 || break; 3579 || break;
2694 ||} 3580 ||}
2695 | beqz AT, ->vmeta_arith
2696 |. addu RA, BASE, RA
2697 |.endmacro 3581 |.endmacro
2698 | 3582 |
2699 |.macro fpmod, a, b, c 3583 |.macro ins_arith, intins, fpins, fpcall, label
2700 |->BC_MODVN_Z: 3584 | ins_arithpre none
2701 | bal ->vm_floor // floor(b/c)
2702 |. div.d FARG1, b, c
2703 | mul.d a, FRET1, c
2704 | sub.d a, b, a // b - floor(b/c)*c
2705 |.endmacro
2706 | 3585 |
2707 |.macro ins_arith, ins 3586 |.if "label" ~= "none"
2708 | ins_arithpre 3587 |label:
2709 |.if "ins" == "fpmod_" 3588 |.endif
2710 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway. 3589 |
2711 |. nop 3590 | lw SFARG1HI, HI(RB)
3591 | lw SFARG2HI, HI(RC)
3592 |
3593 |.if "intins" ~= "div"
3594 |
3595 | // Check for two integers.
3596 | lw SFARG1LO, LO(RB)
3597 | bne SFARG1HI, TISNUM, >5
3598 |. lw SFARG2LO, LO(RC)
3599 | bne SFARG2HI, TISNUM, >5
3600 |
3601 |.if "intins" == "addu"
3602 |. intins CRET1, SFARG1LO, SFARG2LO
3603 | xor TMP1, CRET1, SFARG1LO // ((y^a) & (y^b)) < 0: overflow.
3604 | xor TMP2, CRET1, SFARG2LO
3605 | and TMP1, TMP1, TMP2
3606 | bltz TMP1, ->vmeta_arith
3607 |. addu RA, BASE, RA
3608 |.elif "intins" == "subu"
3609 |. intins CRET1, SFARG1LO, SFARG2LO
3610 | xor TMP1, CRET1, SFARG1LO // ((y^a) & (a^b)) < 0: overflow.
3611 | xor TMP2, SFARG1LO, SFARG2LO
3612 | and TMP1, TMP1, TMP2
3613 | bltz TMP1, ->vmeta_arith
3614 |. addu RA, BASE, RA
3615 |.elif "intins" == "mult"
3616 |. intins SFARG1LO, SFARG2LO
3617 | mflo CRET1
3618 | mfhi TMP2
3619 | sra TMP1, CRET1, 31
3620 | bne TMP1, TMP2, ->vmeta_arith
3621 |. addu RA, BASE, RA
2712 |.else 3622 |.else
2713 | ins f0, f20, f22 3623 |. load_got lj_vm_modi
3624 | beqz SFARG2LO, ->vmeta_arith
3625 |. addu RA, BASE, RA
3626 |.if ENDIAN_BE
3627 | move CARG1, SFARG1LO
3628 |.endif
3629 | call_extern
3630 |. move CARG2, SFARG2LO
3631 |.endif
3632 |
2714 | ins_next1 3633 | ins_next1
2715 | sdc1 f0, 0(RA) 3634 | sw TISNUM, HI(RA)
3635 | sw CRET1, LO(RA)
3636 |3:
2716 | ins_next2 3637 | ins_next2
3638 |
3639 |.elif not FPU
3640 |
3641 | lw SFARG1LO, LO(RB)
3642 | lw SFARG2LO, LO(RC)
3643 |
2717 |.endif 3644 |.endif
3645 |
3646 |5: // Check for two numbers.
3647 | .FPU ldc1 f20, 0(RB)
3648 | sltiu AT, SFARG1HI, LJ_TISNUM
3649 | sltiu TMP0, SFARG2HI, LJ_TISNUM
3650 | .FPU ldc1 f22, 0(RC)
3651 | and AT, AT, TMP0
3652 | beqz AT, ->vmeta_arith
3653 |. addu RA, BASE, RA
3654 |
3655 |.if FPU
3656 | fpins FRET1, f20, f22
3657 |.elif "fpcall" == "sfpmod"
3658 | sfpmod
3659 |.else
3660 | load_got fpcall
3661 | call_extern
3662 |. nop
3663 |.endif
3664 |
3665 | ins_next1
3666 |.if not FPU
3667 | sw SFRETHI, HI(RA)
3668 |.endif
3669 |.if "intins" ~= "div"
3670 | b <3
3671 |.endif
3672 |.if FPU
3673 |. sdc1 FRET1, 0(RA)
3674 |.else
3675 |. sw SFRETLO, LO(RA)
3676 |.endif
3677 |.if "intins" == "div"
3678 | ins_next2
3679 |.endif
3680 |
2718 |.endmacro 3681 |.endmacro
2719 3682
2720 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV: 3683 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2721 | ins_arith add.d 3684 | ins_arith addu, add.d, __adddf3, none
2722 break; 3685 break;
2723 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV: 3686 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2724 | ins_arith sub.d 3687 | ins_arith subu, sub.d, __subdf3, none
2725 break; 3688 break;
2726 case BC_MULVN: case BC_MULNV: case BC_MULVV: 3689 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2727 | ins_arith mul.d 3690 | ins_arith mult, mul.d, __muldf3, none
3691 break;
3692 case BC_DIVVN:
3693 | ins_arith div, div.d, __divdf3, ->BC_DIVVN_Z
2728 break; 3694 break;
2729 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV: 3695 case BC_DIVNV: case BC_DIVVV:
2730 | ins_arith div.d 3696 | ins_arithpre ->BC_DIVVN_Z
2731 break; 3697 break;
2732 case BC_MODVN: 3698 case BC_MODVN:
2733 | ins_arith fpmod 3699 | ins_arith modi, fpmod, sfpmod, ->BC_MODVN_Z
2734 break; 3700 break;
2735 case BC_MODNV: case BC_MODVV: 3701 case BC_MODNV: case BC_MODVV:
2736 | ins_arith fpmod_ 3702 | ins_arithpre ->BC_MODVN_Z
2737 break; 3703 break;
2738 case BC_POW: 3704 case BC_POW:
2739 | decode_RB8a RB, INS 3705 | ins_arithpre none
2740 | decode_RB8b RB 3706 | lw SFARG1HI, HI(RB)
2741 | decode_RDtoRC8 RC, RD 3707 | lw SFARG2HI, HI(RC)
2742 | addu CARG3, BASE, RB 3708 | sltiu AT, SFARG1HI, LJ_TISNUM
2743 | addu CARG4, BASE, RC 3709 | sltiu TMP0, SFARG2HI, LJ_TISNUM
2744 | lw TMP1, HI(CARG3)
2745 | lw TMP2, HI(CARG4)
2746 | ldc1 FARG1, 0(CARG3)
2747 | ldc1 FARG2, 0(CARG4)
2748 | sltiu AT, TMP1, LJ_TISNUM
2749 | sltiu TMP0, TMP2, LJ_TISNUM
2750 | and AT, AT, TMP0 3710 | and AT, AT, TMP0
2751 | load_got pow 3711 | load_got pow
2752 | beqz AT, ->vmeta_arith 3712 | beqz AT, ->vmeta_arith
2753 |. addu RA, BASE, RA 3713 |. addu RA, BASE, RA
3714 |.if FPU
3715 | ldc1 FARG1, 0(RB)
3716 | ldc1 FARG2, 0(RC)
3717 |.else
3718 | lw SFARG1LO, LO(RB)
3719 | lw SFARG2LO, LO(RC)
3720 |.endif
2754 | call_extern 3721 | call_extern
2755 |. nop 3722 |. nop
2756 | ins_next1 3723 | ins_next1
3724 |.if FPU
2757 | sdc1 FRET1, 0(RA) 3725 | sdc1 FRET1, 0(RA)
3726 |.else
3727 | sw SFRETHI, HI(RA)
3728 | sw SFRETLO, LO(RA)
3729 |.endif
2758 | ins_next2 3730 | ins_next2
2759 break; 3731 break;
2760 3732
@@ -2777,10 +3749,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2777 | bnez CRET1, ->vmeta_binop 3749 | bnez CRET1, ->vmeta_binop
2778 |. lw BASE, L->base 3750 |. lw BASE, L->base
2779 | addu RB, BASE, MULTRES 3751 | addu RB, BASE, MULTRES
2780 | ldc1 f0, 0(RB) 3752 | lw SFRETHI, HI(RB)
3753 | lw SFRETLO, LO(RB)
2781 | addu RA, BASE, RA 3754 | addu RA, BASE, RA
2782 | ins_next1 3755 | ins_next1
2783 | sdc1 f0, 0(RA) // Copy result from RB to RA. 3756 | sw SFRETHI, HI(RA)
3757 | sw SFRETLO, LO(RA)
2784 | ins_next2 3758 | ins_next2
2785 break; 3759 break;
2786 3760
@@ -2815,20 +3789,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2815 case BC_KSHORT: 3789 case BC_KSHORT:
2816 | // RA = dst*8, RD = int16_literal*8 3790 | // RA = dst*8, RD = int16_literal*8
2817 | sra RD, INS, 16 3791 | sra RD, INS, 16
2818 | mtc1 RD, f0
2819 | addu RA, BASE, RA 3792 | addu RA, BASE, RA
2820 | cvt.d.w f0, f0
2821 | ins_next1 3793 | ins_next1
2822 | sdc1 f0, 0(RA) 3794 | sw TISNUM, HI(RA)
3795 | sw RD, LO(RA)
2823 | ins_next2 3796 | ins_next2
2824 break; 3797 break;
2825 case BC_KNUM: 3798 case BC_KNUM:
2826 | // RA = dst*8, RD = num_const*8 3799 | // RA = dst*8, RD = num_const*8
2827 | addu RD, KBASE, RD 3800 | addu RD, KBASE, RD
2828 | addu RA, BASE, RA 3801 | addu RA, BASE, RA
2829 | ldc1 f0, 0(RD) 3802 | lw SFRETHI, HI(RD)
3803 | lw SFRETLO, LO(RD)
2830 | ins_next1 3804 | ins_next1
2831 | sdc1 f0, 0(RA) 3805 | sw SFRETHI, HI(RA)
3806 | sw SFRETLO, LO(RA)
2832 | ins_next2 3807 | ins_next2
2833 break; 3808 break;
2834 case BC_KPRI: 3809 case BC_KPRI:
@@ -2864,9 +3839,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2864 | lw UPVAL:RB, LFUNC:RD->uvptr 3839 | lw UPVAL:RB, LFUNC:RD->uvptr
2865 | ins_next1 3840 | ins_next1
2866 | lw TMP1, UPVAL:RB->v 3841 | lw TMP1, UPVAL:RB->v
2867 | ldc1 f0, 0(TMP1) 3842 | lw SFRETHI, HI(TMP1)
3843 | lw SFRETLO, LO(TMP1)
2868 | addu RA, BASE, RA 3844 | addu RA, BASE, RA
2869 | sdc1 f0, 0(RA) 3845 | sw SFRETHI, HI(RA)
3846 | sw SFRETLO, LO(RA)
2870 | ins_next2 3847 | ins_next2
2871 break; 3848 break;
2872 case BC_USETV: 3849 case BC_USETV:
@@ -2875,26 +3852,27 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2875 | srl RA, RA, 1 3852 | srl RA, RA, 1
2876 | addu RD, BASE, RD 3853 | addu RD, BASE, RD
2877 | addu RA, RA, LFUNC:RB 3854 | addu RA, RA, LFUNC:RB
2878 | ldc1 f0, 0(RD)
2879 | lw UPVAL:RB, LFUNC:RA->uvptr 3855 | lw UPVAL:RB, LFUNC:RA->uvptr
3856 | lw SFRETHI, HI(RD)
3857 | lw SFRETLO, LO(RD)
2880 | lbu TMP3, UPVAL:RB->marked 3858 | lbu TMP3, UPVAL:RB->marked
2881 | lw CARG2, UPVAL:RB->v 3859 | lw CARG2, UPVAL:RB->v
2882 | andi TMP3, TMP3, LJ_GC_BLACK // isblack(uv) 3860 | andi TMP3, TMP3, LJ_GC_BLACK // isblack(uv)
2883 | lbu TMP0, UPVAL:RB->closed 3861 | lbu TMP0, UPVAL:RB->closed
2884 | lw TMP2, HI(RD) 3862 | sw SFRETHI, HI(CARG2)
2885 | sdc1 f0, 0(CARG2) 3863 | sw SFRETLO, LO(CARG2)
2886 | li AT, LJ_GC_BLACK|1 3864 | li AT, LJ_GC_BLACK|1
2887 | or TMP3, TMP3, TMP0 3865 | or TMP3, TMP3, TMP0
2888 | beq TMP3, AT, >2 // Upvalue is closed and black? 3866 | beq TMP3, AT, >2 // Upvalue is closed and black?
2889 |. addiu TMP2, TMP2, -(LJ_TNUMX+1) 3867 |. addiu TMP2, SFRETHI, -(LJ_TNUMX+1)
2890 |1: 3868 |1:
2891 | ins_next 3869 | ins_next
2892 | 3870 |
2893 |2: // Check if new value is collectable. 3871 |2: // Check if new value is collectable.
2894 | sltiu AT, TMP2, LJ_TISGCV - (LJ_TNUMX+1) 3872 | sltiu AT, TMP2, LJ_TISGCV - (LJ_TNUMX+1)
2895 | beqz AT, <1 // tvisgcv(v) 3873 | beqz AT, <1 // tvisgcv(v)
2896 |. lw TMP1, LO(RD) 3874 |. nop
2897 | lbu TMP3, GCOBJ:TMP1->gch.marked 3875 | lbu TMP3, GCOBJ:SFRETLO->gch.marked
2898 | andi TMP3, TMP3, LJ_GC_WHITES // iswhite(v) 3876 | andi TMP3, TMP3, LJ_GC_WHITES // iswhite(v)
2899 | beqz TMP3, <1 3877 | beqz TMP3, <1
2900 |. load_got lj_gc_barrieruv 3878 |. load_got lj_gc_barrieruv
@@ -2942,11 +3920,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2942 | srl RA, RA, 1 3920 | srl RA, RA, 1
2943 | addu RD, KBASE, RD 3921 | addu RD, KBASE, RD
2944 | addu RA, RA, LFUNC:RB 3922 | addu RA, RA, LFUNC:RB
2945 | ldc1 f0, 0(RD) 3923 | lw UPVAL:RB, LFUNC:RA->uvptr
2946 | lw UPVAL:RB, LFUNC:RA->uvptr 3924 | lw SFRETHI, HI(RD)
3925 | lw SFRETLO, LO(RD)
3926 | lw TMP1, UPVAL:RB->v
2947 | ins_next1 3927 | ins_next1
2948 | lw TMP1, UPVAL:RB->v 3928 | sw SFRETHI, HI(TMP1)
2949 | sdc1 f0, 0(TMP1) 3929 | sw SFRETLO, LO(TMP1)
2950 | ins_next2 3930 | ins_next2
2951 break; 3931 break;
2952 case BC_USETP: 3932 case BC_USETP:
@@ -2956,10 +3936,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2956 | srl TMP0, RD, 3 3936 | srl TMP0, RD, 3
2957 | addu RA, RA, LFUNC:RB 3937 | addu RA, RA, LFUNC:RB
2958 | not TMP0, TMP0 3938 | not TMP0, TMP0
2959 | lw UPVAL:RB, LFUNC:RA->uvptr 3939 | lw UPVAL:RB, LFUNC:RA->uvptr
2960 | ins_next1 3940 | ins_next1
2961 | lw TMP1, UPVAL:RB->v 3941 | lw TMP1, UPVAL:RB->v
2962 | sw TMP0, HI(TMP1) 3942 | sw TMP0, HI(TMP1)
2963 | ins_next2 3943 | ins_next2
2964 break; 3944 break;
2965 3945
@@ -2995,8 +3975,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2995 | li TMP0, LJ_TFUNC 3975 | li TMP0, LJ_TFUNC
2996 | ins_next1 3976 | ins_next1
2997 | addu RA, BASE, RA 3977 | addu RA, BASE, RA
2998 | sw TMP0, HI(RA)
2999 | sw LFUNC:CRET1, LO(RA) 3978 | sw LFUNC:CRET1, LO(RA)
3979 | sw TMP0, HI(RA)
3000 | ins_next2 3980 | ins_next2
3001 break; 3981 break;
3002 3982
@@ -3077,31 +4057,23 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3077 | lw TMP2, HI(CARG3) 4057 | lw TMP2, HI(CARG3)
3078 | lw TAB:RB, LO(CARG2) 4058 | lw TAB:RB, LO(CARG2)
3079 | li AT, LJ_TTAB 4059 | li AT, LJ_TTAB
3080 | ldc1 f0, 0(CARG3)
3081 | bne TMP1, AT, ->vmeta_tgetv 4060 | bne TMP1, AT, ->vmeta_tgetv
3082 |. addu RA, BASE, RA 4061 |. addu RA, BASE, RA
3083 | sltiu AT, TMP2, LJ_TISNUM 4062 | bne TMP2, TISNUM, >5
3084 | beqz AT, >5 4063 |. lw RC, LO(CARG3)
3085 |. li AT, LJ_TSTR 4064 | lw TMP0, TAB:RB->asize
3086 |
3087 | // Convert number key to integer, check for integerness and range.
3088 | cvt.w.d f2, f0
3089 | lw TMP0, TAB:RB->asize
3090 | mfc1 TMP2, f2
3091 | cvt.d.w f4, f2
3092 | lw TMP1, TAB:RB->array 4065 | lw TMP1, TAB:RB->array
3093 | c.eq.d f0, f4 4066 | sltu AT, RC, TMP0
3094 | sltu AT, TMP2, TMP0 4067 | sll TMP2, RC, 3
3095 | movf AT, r0
3096 | sll TMP2, TMP2, 3
3097 | beqz AT, ->vmeta_tgetv // Integer key and in array part? 4068 | beqz AT, ->vmeta_tgetv // Integer key and in array part?
3098 |. addu TMP2, TMP1, TMP2 4069 |. addu TMP2, TMP1, TMP2
3099 | lw TMP0, HI(TMP2) 4070 | lw SFRETHI, HI(TMP2)
3100 | beq TMP0, TISNIL, >2 4071 | beq SFRETHI, TISNIL, >2
3101 |. ldc1 f0, 0(TMP2) 4072 |. lw SFRETLO, LO(TMP2)
3102 |1: 4073 |1:
3103 | ins_next1 4074 | ins_next1
3104 | sdc1 f0, 0(RA) 4075 | sw SFRETHI, HI(RA)
4076 | sw SFRETLO, LO(RA)
3105 | ins_next2 4077 | ins_next2
3106 | 4078 |
3107 |2: // Check for __index if table value is nil. 4079 |2: // Check for __index if table value is nil.
@@ -3116,8 +4088,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3116 |. nop 4088 |. nop
3117 | 4089 |
3118 |5: 4090 |5:
4091 | li AT, LJ_TSTR
3119 | bne TMP2, AT, ->vmeta_tgetv 4092 | bne TMP2, AT, ->vmeta_tgetv
3120 |. lw STR:RC, LO(CARG3) 4093 |. nop
3121 | b ->BC_TGETS_Z // String key? 4094 | b ->BC_TGETS_Z // String key?
3122 |. nop 4095 |. nop
3123 break; 4096 break;
@@ -3138,9 +4111,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3138 |->BC_TGETS_Z: 4111 |->BC_TGETS_Z:
3139 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 4112 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8
3140 | lw TMP0, TAB:RB->hmask 4113 | lw TMP0, TAB:RB->hmask
3141 | lw TMP1, STR:RC->hash 4114 | lw TMP1, STR:RC->sid
3142 | lw NODE:TMP2, TAB:RB->node 4115 | lw NODE:TMP2, TAB:RB->node
3143 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 4116 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
3144 | sll TMP0, TMP1, 5 4117 | sll TMP0, TMP1, 5
3145 | sll TMP1, TMP1, 3 4118 | sll TMP1, TMP1, 3
3146 | subu TMP1, TMP0, TMP1 4119 | subu TMP1, TMP0, TMP1
@@ -3149,18 +4122,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3149 | lw CARG1, offsetof(Node, key)+HI(NODE:TMP2) 4122 | lw CARG1, offsetof(Node, key)+HI(NODE:TMP2)
3150 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2) 4123 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2)
3151 | lw NODE:TMP1, NODE:TMP2->next 4124 | lw NODE:TMP1, NODE:TMP2->next
3152 | lw CARG2, offsetof(Node, val)+HI(NODE:TMP2) 4125 | lw SFRETHI, offsetof(Node, val)+HI(NODE:TMP2)
3153 | addiu CARG1, CARG1, -LJ_TSTR 4126 | addiu CARG1, CARG1, -LJ_TSTR
3154 | xor TMP0, TMP0, STR:RC 4127 | xor TMP0, TMP0, STR:RC
3155 | or AT, CARG1, TMP0 4128 | or AT, CARG1, TMP0
3156 | bnez AT, >4 4129 | bnez AT, >4
3157 |. lw TAB:TMP3, TAB:RB->metatable 4130 |. lw TAB:TMP3, TAB:RB->metatable
3158 | beq CARG2, TISNIL, >5 // Key found, but nil value? 4131 | beq SFRETHI, TISNIL, >5 // Key found, but nil value?
3159 |. lw CARG1, offsetof(Node, val)+LO(NODE:TMP2) 4132 |. lw SFRETLO, offsetof(Node, val)+LO(NODE:TMP2)
3160 |3: 4133 |3:
3161 | ins_next1 4134 | ins_next1
3162 | sw CARG2, HI(RA) 4135 | sw SFRETHI, HI(RA)
3163 | sw CARG1, LO(RA) 4136 | sw SFRETLO, LO(RA)
3164 | ins_next2 4137 | ins_next2
3165 | 4138 |
3166 |4: // Follow hash chain. 4139 |4: // Follow hash chain.
@@ -3170,7 +4143,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3170 | 4143 |
3171 |5: // Check for __index if table value is nil. 4144 |5: // Check for __index if table value is nil.
3172 | beqz TAB:TMP3, <3 // No metatable: done. 4145 | beqz TAB:TMP3, <3 // No metatable: done.
3173 |. li CARG2, LJ_TNIL 4146 |. li SFRETHI, LJ_TNIL
3174 | lbu TMP0, TAB:TMP3->nomm 4147 | lbu TMP0, TAB:TMP3->nomm
3175 | andi TMP0, TMP0, 1<<MM_index 4148 | andi TMP0, TMP0, 1<<MM_index
3176 | bnez TMP0, <3 // 'no __index' flag set: done. 4149 | bnez TMP0, <3 // 'no __index' flag set: done.
@@ -3195,12 +4168,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3195 | sltu AT, TMP0, TMP1 4168 | sltu AT, TMP0, TMP1
3196 | beqz AT, ->vmeta_tgetb 4169 | beqz AT, ->vmeta_tgetb
3197 |. addu RC, TMP2, RC 4170 |. addu RC, TMP2, RC
3198 | lw TMP1, HI(RC) 4171 | lw SFRETHI, HI(RC)
3199 | beq TMP1, TISNIL, >5 4172 | beq SFRETHI, TISNIL, >5
3200 |. ldc1 f0, 0(RC) 4173 |. lw SFRETLO, LO(RC)
3201 |1: 4174 |1:
3202 | ins_next1 4175 | ins_next1
3203 | sdc1 f0, 0(RA) 4176 | sw SFRETHI, HI(RA)
4177 | sw SFRETLO, LO(RA)
3204 | ins_next2 4178 | ins_next2
3205 | 4179 |
3206 |5: // Check for __index if table value is nil. 4180 |5: // Check for __index if table value is nil.
@@ -3211,9 +4185,33 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3211 | andi TMP1, TMP1, 1<<MM_index 4185 | andi TMP1, TMP1, 1<<MM_index
3212 | bnez TMP1, <1 // 'no __index' flag set: done. 4186 | bnez TMP1, <1 // 'no __index' flag set: done.
3213 |. nop 4187 |. nop
3214 | b ->vmeta_tgetb // Caveat: preserve TMP0! 4188 | b ->vmeta_tgetb // Caveat: preserve TMP0 and CARG2!
3215 |. nop 4189 |. nop
3216 break; 4190 break;
4191 case BC_TGETR:
4192 | // RA = dst*8, RB = table*8, RC = key*8
4193 | decode_RB8a RB, INS
4194 | decode_RB8b RB
4195 | decode_RDtoRC8 RC, RD
4196 | addu RB, BASE, RB
4197 | addu RC, BASE, RC
4198 | lw TAB:CARG1, LO(RB)
4199 | lw CARG2, LO(RC)
4200 | addu RA, BASE, RA
4201 | lw TMP0, TAB:CARG1->asize
4202 | lw TMP1, TAB:CARG1->array
4203 | sltu AT, CARG2, TMP0
4204 | sll TMP2, CARG2, 3
4205 | beqz AT, ->vmeta_tgetr // In array part?
4206 |. addu CRET1, TMP1, TMP2
4207 | lw SFARG2HI, HI(CRET1)
4208 | lw SFARG2LO, LO(CRET1)
4209 |->BC_TGETR_Z:
4210 | ins_next1
4211 | sw SFARG2HI, HI(RA)
4212 | sw SFARG2LO, LO(RA)
4213 | ins_next2
4214 break;
3217 4215
3218 case BC_TSETV: 4216 case BC_TSETV:
3219 | // RA = src*8, RB = table*8, RC = key*8 4217 | // RA = src*8, RB = table*8, RC = key*8
@@ -3226,33 +4224,26 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3226 | lw TMP2, HI(CARG3) 4224 | lw TMP2, HI(CARG3)
3227 | lw TAB:RB, LO(CARG2) 4225 | lw TAB:RB, LO(CARG2)
3228 | li AT, LJ_TTAB 4226 | li AT, LJ_TTAB
3229 | ldc1 f0, 0(CARG3)
3230 | bne TMP1, AT, ->vmeta_tsetv 4227 | bne TMP1, AT, ->vmeta_tsetv
3231 |. addu RA, BASE, RA 4228 |. addu RA, BASE, RA
3232 | sltiu AT, TMP2, LJ_TISNUM 4229 | bne TMP2, TISNUM, >5
3233 | beqz AT, >5 4230 |. lw RC, LO(CARG3)
3234 |. li AT, LJ_TSTR 4231 | lw TMP0, TAB:RB->asize
3235 |
3236 | // Convert number key to integer, check for integerness and range.
3237 | cvt.w.d f2, f0
3238 | lw TMP0, TAB:RB->asize
3239 | mfc1 TMP2, f2
3240 | cvt.d.w f4, f2
3241 | lw TMP1, TAB:RB->array 4232 | lw TMP1, TAB:RB->array
3242 | c.eq.d f0, f4 4233 | sltu AT, RC, TMP0
3243 | sltu AT, TMP2, TMP0 4234 | sll TMP2, RC, 3
3244 | movf AT, r0
3245 | sll TMP2, TMP2, 3
3246 | beqz AT, ->vmeta_tsetv // Integer key and in array part? 4235 | beqz AT, ->vmeta_tsetv // Integer key and in array part?
3247 |. addu TMP1, TMP1, TMP2 4236 |. addu TMP1, TMP1, TMP2
3248 | lbu TMP3, TAB:RB->marked
3249 | lw TMP0, HI(TMP1) 4237 | lw TMP0, HI(TMP1)
4238 | lbu TMP3, TAB:RB->marked
4239 | lw SFRETHI, HI(RA)
3250 | beq TMP0, TISNIL, >3 4240 | beq TMP0, TISNIL, >3
3251 |. ldc1 f0, 0(RA) 4241 |. lw SFRETLO, LO(RA)
3252 |1: 4242 |1:
3253 | andi AT, TMP3, LJ_GC_BLACK // isblack(table) 4243 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
3254 | bnez AT, >7 4244 | sw SFRETHI, HI(TMP1)
3255 |. sdc1 f0, 0(TMP1) 4245 | bnez AT, >7
4246 |. sw SFRETLO, LO(TMP1)
3256 |2: 4247 |2:
3257 | ins_next 4248 | ins_next
3258 | 4249 |
@@ -3268,8 +4259,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3268 |. nop 4259 |. nop
3269 | 4260 |
3270 |5: 4261 |5:
4262 | li AT, LJ_TSTR
3271 | bne TMP2, AT, ->vmeta_tsetv 4263 | bne TMP2, AT, ->vmeta_tsetv
3272 |. lw STR:RC, LO(CARG3) 4264 |. nop
3273 | b ->BC_TSETS_Z // String key? 4265 | b ->BC_TSETS_Z // String key?
3274 |. nop 4266 |. nop
3275 | 4267 |
@@ -3293,15 +4285,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3293 |->BC_TSETS_Z: 4285 |->BC_TSETS_Z:
3294 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8 4286 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8
3295 | lw TMP0, TAB:RB->hmask 4287 | lw TMP0, TAB:RB->hmask
3296 | lw TMP1, STR:RC->hash 4288 | lw TMP1, STR:RC->sid
3297 | lw NODE:TMP2, TAB:RB->node 4289 | lw NODE:TMP2, TAB:RB->node
3298 | sb r0, TAB:RB->nomm // Clear metamethod cache. 4290 | sb r0, TAB:RB->nomm // Clear metamethod cache.
3299 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 4291 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
3300 | sll TMP0, TMP1, 5 4292 | sll TMP0, TMP1, 5
3301 | sll TMP1, TMP1, 3 4293 | sll TMP1, TMP1, 3
3302 | subu TMP1, TMP0, TMP1 4294 | subu TMP1, TMP0, TMP1
3303 | addu NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8) 4295 | addu NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
4296 |.if FPU
3304 | ldc1 f20, 0(RA) 4297 | ldc1 f20, 0(RA)
4298 |.else
4299 | lw SFRETHI, HI(RA)
4300 | lw SFRETLO, LO(RA)
4301 |.endif
3305 |1: 4302 |1:
3306 | lw CARG1, offsetof(Node, key)+HI(NODE:TMP2) 4303 | lw CARG1, offsetof(Node, key)+HI(NODE:TMP2)
3307 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2) 4304 | lw TMP0, offsetof(Node, key)+LO(NODE:TMP2)
@@ -3315,8 +4312,14 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3315 |. lw TAB:TMP0, TAB:RB->metatable 4312 |. lw TAB:TMP0, TAB:RB->metatable
3316 |2: 4313 |2:
3317 | andi AT, TMP3, LJ_GC_BLACK // isblack(table) 4314 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4315 |.if FPU
3318 | bnez AT, >7 4316 | bnez AT, >7
3319 |. sdc1 f20, NODE:TMP2->val 4317 |. sdc1 f20, NODE:TMP2->val
4318 |.else
4319 | sw SFRETHI, NODE:TMP2->val.u32.hi
4320 | bnez AT, >7
4321 |. sw SFRETLO, NODE:TMP2->val.u32.lo
4322 |.endif
3320 |3: 4323 |3:
3321 | ins_next 4324 | ins_next
3322 | 4325 |
@@ -3354,8 +4357,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3354 |. move CARG1, L 4357 |. move CARG1, L
3355 | // Returns TValue *. 4358 | // Returns TValue *.
3356 | lw BASE, L->base 4359 | lw BASE, L->base
4360 |.if FPU
3357 | b <3 // No 2nd write barrier needed. 4361 | b <3 // No 2nd write barrier needed.
3358 |. sdc1 f20, 0(CRET1) 4362 |. sdc1 f20, 0(CRET1)
4363 |.else
4364 | lw SFARG1HI, HI(RA)
4365 | lw SFARG1LO, LO(RA)
4366 | sw SFARG1HI, HI(CRET1)
4367 | b <3 // No 2nd write barrier needed.
4368 |. sw SFARG1LO, LO(CRET1)
4369 |.endif
3359 | 4370 |
3360 |7: // Possible table write barrier for the value. Skip valiswhite check. 4371 |7: // Possible table write barrier for the value. Skip valiswhite check.
3361 | barrierback TAB:RB, TMP3, TMP0, <3 4372 | barrierback TAB:RB, TMP3, TMP0, <3
@@ -3380,11 +4391,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3380 | lw TMP1, HI(RC) 4391 | lw TMP1, HI(RC)
3381 | lbu TMP3, TAB:RB->marked 4392 | lbu TMP3, TAB:RB->marked
3382 | beq TMP1, TISNIL, >5 4393 | beq TMP1, TISNIL, >5
3383 |. ldc1 f0, 0(RA)
3384 |1: 4394 |1:
4395 |. lw SFRETHI, HI(RA)
4396 | lw SFRETLO, LO(RA)
3385 | andi AT, TMP3, LJ_GC_BLACK // isblack(table) 4397 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4398 | sw SFRETHI, HI(RC)
3386 | bnez AT, >7 4399 | bnez AT, >7
3387 |. sdc1 f0, 0(RC) 4400 |. sw SFRETLO, LO(RC)
3388 |2: 4401 |2:
3389 | ins_next 4402 | ins_next
3390 | 4403 |
@@ -3396,12 +4409,43 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3396 | andi TMP1, TMP1, 1<<MM_newindex 4409 | andi TMP1, TMP1, 1<<MM_newindex
3397 | bnez TMP1, <1 // 'no __newindex' flag set: done. 4410 | bnez TMP1, <1 // 'no __newindex' flag set: done.
3398 |. nop 4411 |. nop
3399 | b ->vmeta_tsetb // Caveat: preserve TMP0! 4412 | b ->vmeta_tsetb // Caveat: preserve TMP0 and CARG2!
3400 |. nop 4413 |. nop
3401 | 4414 |
3402 |7: // Possible table write barrier for the value. Skip valiswhite check. 4415 |7: // Possible table write barrier for the value. Skip valiswhite check.
3403 | barrierback TAB:RB, TMP3, TMP0, <2 4416 | barrierback TAB:RB, TMP3, TMP0, <2
3404 break; 4417 break;
4418 case BC_TSETR:
4419 | // RA = dst*8, RB = table*8, RC = key*8
4420 | decode_RB8a RB, INS
4421 | decode_RB8b RB
4422 | decode_RDtoRC8 RC, RD
4423 | addu CARG1, BASE, RB
4424 | addu CARG3, BASE, RC
4425 | lw TAB:CARG2, LO(CARG1)
4426 | lw CARG3, LO(CARG3)
4427 | lbu TMP3, TAB:CARG2->marked
4428 | lw TMP0, TAB:CARG2->asize
4429 | lw TMP1, TAB:CARG2->array
4430 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4431 | bnez AT, >7
4432 |. addu RA, BASE, RA
4433 |2:
4434 | sltu AT, CARG3, TMP0
4435 | sll TMP2, CARG3, 3
4436 | beqz AT, ->vmeta_tsetr // In array part?
4437 |. addu CRET1, TMP1, TMP2
4438 |->BC_TSETR_Z:
4439 | lw SFARG1HI, HI(RA)
4440 | lw SFARG1LO, LO(RA)
4441 | ins_next1
4442 | sw SFARG1HI, HI(CRET1)
4443 | sw SFARG1LO, LO(CRET1)
4444 | ins_next2
4445 |
4446 |7: // Possible table write barrier for the value. Skip valiswhite check.
4447 | barrierback TAB:CARG2, TMP3, CRET1, <2
4448 break;
3405 4449
3406 case BC_TSETM: 4450 case BC_TSETM:
3407 | // RA = base*8 (table at base-1), RD = num_const*8 (start index) 4451 | // RA = base*8 (table at base-1), RD = num_const*8 (start index)
@@ -3424,10 +4468,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3424 | addu TMP1, TMP1, CARG1 4468 | addu TMP1, TMP1, CARG1
3425 | andi TMP0, TMP3, LJ_GC_BLACK // isblack(table) 4469 | andi TMP0, TMP3, LJ_GC_BLACK // isblack(table)
3426 |3: // Copy result slots to table. 4470 |3: // Copy result slots to table.
3427 | ldc1 f0, 0(RA) 4471 | lw SFRETHI, HI(RA)
4472 | lw SFRETLO, LO(RA)
3428 | addiu RA, RA, 8 4473 | addiu RA, RA, 8
3429 | sltu AT, RA, TMP2 4474 | sltu AT, RA, TMP2
3430 | sdc1 f0, 0(TMP1) 4475 | sw SFRETHI, HI(TMP1)
4476 | sw SFRETLO, LO(TMP1)
3431 | bnez AT, <3 4477 | bnez AT, <3
3432 |. addiu TMP1, TMP1, 8 4478 |. addiu TMP1, TMP1, 8
3433 | bnez TMP0, >7 4479 | bnez TMP0, >7
@@ -3502,10 +4548,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3502 | beqz NARGS8:RC, >3 4548 | beqz NARGS8:RC, >3
3503 |. move TMP3, NARGS8:RC 4549 |. move TMP3, NARGS8:RC
3504 |2: 4550 |2:
3505 | ldc1 f0, 0(RA) 4551 | lw SFRETHI, HI(RA)
4552 | lw SFRETLO, LO(RA)
3506 | addiu RA, RA, 8 4553 | addiu RA, RA, 8
3507 | addiu TMP3, TMP3, -8 4554 | addiu TMP3, TMP3, -8
3508 | sdc1 f0, 0(TMP2) 4555 | sw SFRETHI, HI(TMP2)
4556 | sw SFRETLO, LO(TMP2)
3509 | bnez TMP3, <2 4557 | bnez TMP3, <2
3510 |. addiu TMP2, TMP2, 8 4558 |. addiu TMP2, TMP2, 8
3511 |3: 4559 |3:
@@ -3542,12 +4590,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3542 | li AT, LJ_TFUNC 4590 | li AT, LJ_TFUNC
3543 | lw TMP1, -24+HI(BASE) 4591 | lw TMP1, -24+HI(BASE)
3544 | lw LFUNC:RB, -24+LO(BASE) 4592 | lw LFUNC:RB, -24+LO(BASE)
3545 | ldc1 f2, -8(BASE) 4593 | lw SFARG1HI, -16+HI(BASE)
3546 | ldc1 f0, -16(BASE) 4594 | lw SFARG1LO, -16+LO(BASE)
4595 | lw SFARG2HI, -8+HI(BASE)
4596 | lw SFARG2LO, -8+LO(BASE)
3547 | sw TMP1, HI(BASE) // Copy callable. 4597 | sw TMP1, HI(BASE) // Copy callable.
3548 | sw LFUNC:RB, LO(BASE) 4598 | sw LFUNC:RB, LO(BASE)
3549 | sdc1 f2, 16(BASE) // Copy control var. 4599 | sw SFARG1HI, 8+HI(BASE) // Copy state.
3550 | sdc1 f0, 8(BASE) // Copy state. 4600 | sw SFARG1LO, 8+LO(BASE)
4601 | sw SFARG2HI, 16+HI(BASE) // Copy control var.
4602 | sw SFARG2LO, 16+LO(BASE)
3551 | addiu BASE, BASE, 8 4603 | addiu BASE, BASE, 8
3552 | bne TMP1, AT, ->vmeta_call 4604 | bne TMP1, AT, ->vmeta_call
3553 |. li NARGS8:RC, 16 // Iterators get 2 arguments. 4605 |. li NARGS8:RC, 16 // Iterators get 2 arguments.
@@ -3555,10 +4607,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3555 break; 4607 break;
3556 4608
3557 case BC_ITERN: 4609 case BC_ITERN:
3558 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8) 4610 |.if JIT and ENDIAN_LE
3559 |.if JIT 4611 | hotloop
3560 | // NYI: add hotloop, record BC_ITERN.
3561 |.endif 4612 |.endif
4613 |->vm_IITERN:
4614 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8)
3562 | addu RA, BASE, RA 4615 | addu RA, BASE, RA
3563 | lw TAB:RB, -16+LO(RA) 4616 | lw TAB:RB, -16+LO(RA)
3564 | lw RC, -8+LO(RA) // Get index from control var. 4617 | lw RC, -8+LO(RA) // Get index from control var.
@@ -3570,20 +4623,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3570 | beqz AT, >5 // Index points after array part? 4623 | beqz AT, >5 // Index points after array part?
3571 |. sll TMP3, RC, 3 4624 |. sll TMP3, RC, 3
3572 | addu TMP3, TMP1, TMP3 4625 | addu TMP3, TMP1, TMP3
3573 | lw TMP2, HI(TMP3) 4626 | lw SFARG1HI, HI(TMP3)
3574 | ldc1 f0, 0(TMP3) 4627 | lw SFARG1LO, LO(TMP3)
3575 | mtc1 RC, f2
3576 | lhu RD, -4+OFS_RD(PC) 4628 | lhu RD, -4+OFS_RD(PC)
3577 | beq TMP2, TISNIL, <1 // Skip holes in array part. 4629 | sw TISNUM, HI(RA)
4630 | sw RC, LO(RA)
4631 | beq SFARG1HI, TISNIL, <1 // Skip holes in array part.
3578 |. addiu RC, RC, 1 4632 |. addiu RC, RC, 1
3579 | cvt.d.w f2, f2 4633 | sw SFARG1HI, 8+HI(RA)
4634 | sw SFARG1LO, 8+LO(RA)
3580 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535) 4635 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3581 | sdc1 f0, 8(RA)
3582 | decode_RD4b RD 4636 | decode_RD4b RD
3583 | addu RD, RD, TMP3 4637 | addu RD, RD, TMP3
3584 | sw RC, -8+LO(RA) // Update control var. 4638 | sw RC, -8+LO(RA) // Update control var.
3585 | addu PC, PC, RD 4639 | addu PC, PC, RD
3586 | sdc1 f2, 0(RA)
3587 |3: 4640 |3:
3588 | ins_next 4641 | ins_next
3589 | 4642 |
@@ -3598,18 +4651,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3598 | sll RB, RC, 3 4651 | sll RB, RC, 3
3599 | subu TMP3, TMP3, RB 4652 | subu TMP3, TMP3, RB
3600 | addu NODE:TMP3, TMP3, TMP2 4653 | addu NODE:TMP3, TMP3, TMP2
3601 | lw RB, HI(NODE:TMP3) 4654 | lw SFARG1HI, NODE:TMP3->val.u32.hi
3602 | ldc1 f0, 0(NODE:TMP3) 4655 | lw SFARG1LO, NODE:TMP3->val.u32.lo
3603 | lhu RD, -4+OFS_RD(PC) 4656 | lhu RD, -4+OFS_RD(PC)
3604 | beq RB, TISNIL, <6 // Skip holes in hash part. 4657 | beq SFARG1HI, TISNIL, <6 // Skip holes in hash part.
3605 |. addiu RC, RC, 1 4658 |. addiu RC, RC, 1
3606 | ldc1 f2, NODE:TMP3->key 4659 | lw SFARG2HI, NODE:TMP3->key.u32.hi
4660 | lw SFARG2LO, NODE:TMP3->key.u32.lo
3607 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535) 4661 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3608 | sdc1 f0, 8(RA) 4662 | sw SFARG1HI, 8+HI(RA)
4663 | sw SFARG1LO, 8+LO(RA)
3609 | addu RC, RC, TMP0 4664 | addu RC, RC, TMP0
3610 | decode_RD4b RD 4665 | decode_RD4b RD
3611 | addu RD, RD, TMP3 4666 | addu RD, RD, TMP3
3612 | sdc1 f2, 0(RA) 4667 | sw SFARG2HI, HI(RA)
4668 | sw SFARG2LO, LO(RA)
3613 | addu PC, PC, RD 4669 | addu PC, PC, RD
3614 | b <3 4670 | b <3
3615 |. sw RC, -8+LO(RA) // Update control var. 4671 |. sw RC, -8+LO(RA) // Update control var.
@@ -3634,9 +4690,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3634 | addiu CARG2, CARG2, -FF_next_N 4690 | addiu CARG2, CARG2, -FF_next_N
3635 | or CARG2, CARG2, CARG3 4691 | or CARG2, CARG2, CARG3
3636 | bnez CARG2, >5 4692 | bnez CARG2, >5
3637 |. lui TMP1, 0xfffe 4693 |. lui TMP1, (LJ_KEYINDEX >> 16)
3638 | addu PC, TMP0, TMP2 4694 | addu PC, TMP0, TMP2
3639 | ori TMP1, TMP1, 0x7fff 4695 | ori TMP1, TMP1, (LJ_KEYINDEX & 0xffff)
3640 | sw r0, -8+LO(RA) // Initialize control var. 4696 | sw r0, -8+LO(RA) // Initialize control var.
3641 | sw TMP1, -8+HI(RA) 4697 | sw TMP1, -8+HI(RA)
3642 |1: 4698 |1:
@@ -3645,9 +4701,28 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3645 | li TMP3, BC_JMP 4701 | li TMP3, BC_JMP
3646 | li TMP1, BC_ITERC 4702 | li TMP1, BC_ITERC
3647 | sb TMP3, -4+OFS_OP(PC) 4703 | sb TMP3, -4+OFS_OP(PC)
3648 | addu PC, TMP0, TMP2 4704 | addu PC, TMP0, TMP2
4705 |.if JIT
4706 | lb TMP0, OFS_OP(PC)
4707 | li AT, BC_ITERN
4708 | bne TMP0, AT, >6
4709 |. lhu TMP2, OFS_RD(PC)
4710 |.endif
3649 | b <1 4711 | b <1
3650 |. sb TMP1, OFS_OP(PC) 4712 |. sb TMP1, OFS_OP(PC)
4713 |.if JIT
4714 |6: // Unpatch JLOOP.
4715 | lw TMP0, DISPATCH_J(trace)(DISPATCH)
4716 | sll TMP2, TMP2, 2
4717 | addu TMP0, TMP0, TMP2
4718 | lw TRACE:TMP2, 0(TMP0)
4719 | lw TMP0, TRACE:TMP2->startins
4720 | li AT, -256
4721 | and TMP0, TMP0, AT
4722 | or TMP0, TMP0, TMP1
4723 | b <1
4724 |. sw TMP0, 0(PC)
4725 |.endif
3651 break; 4726 break;
3652 4727
3653 case BC_VARG: 4728 case BC_VARG:
@@ -3689,9 +4764,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3689 | bnez AT, >7 4764 | bnez AT, >7
3690 |. addiu MULTRES, TMP1, 8 4765 |. addiu MULTRES, TMP1, 8
3691 |6: 4766 |6:
3692 | ldc1 f0, 0(RC) 4767 | lw SFRETHI, HI(RC)
4768 | lw SFRETLO, LO(RC)
3693 | addiu RC, RC, 8 4769 | addiu RC, RC, 8
3694 | sdc1 f0, 0(RA) 4770 | sw SFRETHI, HI(RA)
4771 | sw SFRETLO, LO(RA)
3695 | sltu AT, RC, TMP3 4772 | sltu AT, RC, TMP3
3696 | bnez AT, <6 // More vararg slots? 4773 | bnez AT, <6 // More vararg slots?
3697 |. addiu RA, RA, 8 4774 |. addiu RA, RA, 8
@@ -3747,10 +4824,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3747 | beqz RC, >3 4824 | beqz RC, >3
3748 |. subu BASE, TMP2, TMP0 4825 |. subu BASE, TMP2, TMP0
3749 |2: 4826 |2:
3750 | ldc1 f0, 0(RA) 4827 | lw SFRETHI, HI(RA)
4828 | lw SFRETLO, LO(RA)
3751 | addiu RA, RA, 8 4829 | addiu RA, RA, 8
3752 | addiu RC, RC, -8 4830 | addiu RC, RC, -8
3753 | sdc1 f0, 0(TMP2) 4831 | sw SFRETHI, HI(TMP2)
4832 | sw SFRETLO, LO(TMP2)
3754 | bnez RC, <2 4833 | bnez RC, <2
3755 |. addiu TMP2, TMP2, 8 4834 |. addiu TMP2, TMP2, 8
3756 |3: 4835 |3:
@@ -3791,14 +4870,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3791 | lw INS, -4(PC) 4870 | lw INS, -4(PC)
3792 | addiu TMP2, BASE, -8 4871 | addiu TMP2, BASE, -8
3793 if (op == BC_RET1) { 4872 if (op == BC_RET1) {
3794 | ldc1 f0, 0(RA) 4873 | lw SFRETHI, HI(RA)
4874 | lw SFRETLO, LO(RA)
3795 } 4875 }
3796 | decode_RB8a RB, INS 4876 | decode_RB8a RB, INS
3797 | decode_RA8a RA, INS 4877 | decode_RA8a RA, INS
3798 | decode_RB8b RB 4878 | decode_RB8b RB
3799 | decode_RA8b RA 4879 | decode_RA8b RA
3800 if (op == BC_RET1) { 4880 if (op == BC_RET1) {
3801 | sdc1 f0, 0(TMP2) 4881 | sw SFRETHI, HI(TMP2)
4882 | sw SFRETLO, LO(TMP2)
3802 } 4883 }
3803 | subu BASE, TMP2, RA 4884 | subu BASE, TMP2, RA
3804 |5: 4885 |5:
@@ -3840,69 +4921,147 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3840 | // RA = base*8, RD = target (after end of loop or start of loop) 4921 | // RA = base*8, RD = target (after end of loop or start of loop)
3841 vk = (op == BC_IFORL || op == BC_JFORL); 4922 vk = (op == BC_IFORL || op == BC_JFORL);
3842 | addu RA, BASE, RA 4923 | addu RA, BASE, RA
3843 if (vk) { 4924 | lw SFARG1HI, FORL_IDX*8+HI(RA)
3844 | ldc1 f0, FORL_IDX*8(RA) 4925 | lw SFARG1LO, FORL_IDX*8+LO(RA)
3845 | ldc1 f4, FORL_STEP*8(RA)
3846 | ldc1 f2, FORL_STOP*8(RA)
3847 | lw TMP3, FORL_STEP*8+HI(RA)
3848 | add.d f0, f0, f4
3849 | sdc1 f0, FORL_IDX*8(RA)
3850 } else {
3851 | lw TMP1, FORL_IDX*8+HI(RA)
3852 | lw TMP3, FORL_STEP*8+HI(RA)
3853 | lw TMP2, FORL_STOP*8+HI(RA)
3854 | sltiu TMP1, TMP1, LJ_TISNUM
3855 | sltiu TMP0, TMP3, LJ_TISNUM
3856 | sltiu TMP2, TMP2, LJ_TISNUM
3857 | and TMP1, TMP1, TMP0
3858 | and TMP1, TMP1, TMP2
3859 | ldc1 f0, FORL_IDX*8(RA)
3860 | beqz TMP1, ->vmeta_for
3861 |. ldc1 f2, FORL_STOP*8(RA)
3862 }
3863 if (op != BC_JFORL) { 4926 if (op != BC_JFORL) {
3864 | srl RD, RD, 1 4927 | srl RD, RD, 1
3865 | lui TMP0, (-(BCBIAS_J*4 >> 16) & 65535) 4928 | lui TMP2, (-(BCBIAS_J*4 >> 16) & 65535)
4929 | addu TMP2, RD, TMP2
3866 } 4930 }
3867 | c.le.d 0, f0, f2 4931 if (!vk) {
3868 | c.le.d 1, f2, f0 4932 | lw SFARG2HI, FORL_STOP*8+HI(RA)
3869 | sdc1 f0, FORL_EXT*8(RA) 4933 | lw SFARG2LO, FORL_STOP*8+LO(RA)
4934 | bne SFARG1HI, TISNUM, >5
4935 |. lw SFRETHI, FORL_STEP*8+HI(RA)
4936 | xor AT, SFARG2HI, TISNUM
4937 | lw SFRETLO, FORL_STEP*8+LO(RA)
4938 | xor TMP0, SFRETHI, TISNUM
4939 | or AT, AT, TMP0
4940 | bnez AT, ->vmeta_for
4941 |. slt AT, SFRETLO, r0
4942 | slt CRET1, SFARG2LO, SFARG1LO
4943 | slt TMP1, SFARG1LO, SFARG2LO
4944 | movn CRET1, TMP1, AT
4945 } else {
4946 | bne SFARG1HI, TISNUM, >5
4947 |. lw SFARG2LO, FORL_STEP*8+LO(RA)
4948 | lw SFRETLO, FORL_STOP*8+LO(RA)
4949 | move TMP3, SFARG1LO
4950 | addu SFARG1LO, SFARG1LO, SFARG2LO
4951 | xor TMP0, SFARG1LO, TMP3
4952 | xor TMP1, SFARG1LO, SFARG2LO
4953 | and TMP0, TMP0, TMP1
4954 | slt TMP1, SFARG1LO, SFRETLO
4955 | slt CRET1, SFRETLO, SFARG1LO
4956 | slt AT, SFARG2LO, r0
4957 | slt TMP0, TMP0, r0 // ((y^a) & (y^b)) < 0: overflow.
4958 | movn CRET1, TMP1, AT
4959 | or CRET1, CRET1, TMP0
4960 }
4961 |1:
4962 if (op == BC_FORI) {
4963 | movz TMP2, r0, CRET1
4964 | addu PC, PC, TMP2
4965 } else if (op == BC_JFORI) {
4966 | addu PC, PC, TMP2
4967 | lhu RD, -4+OFS_RD(PC)
4968 } else if (op == BC_IFORL) {
4969 | movn TMP2, r0, CRET1
4970 | addu PC, PC, TMP2
4971 }
4972 if (vk) {
4973 | sw SFARG1HI, FORL_IDX*8+HI(RA)
4974 | sw SFARG1LO, FORL_IDX*8+LO(RA)
4975 }
4976 | ins_next1
4977 | sw SFARG1HI, FORL_EXT*8+HI(RA)
4978 | sw SFARG1LO, FORL_EXT*8+LO(RA)
4979 |2:
3870 if (op == BC_JFORI) { 4980 if (op == BC_JFORI) {
3871 | li TMP1, 1 4981 | beqz CRET1, =>BC_JLOOP
3872 | li TMP2, 1
3873 | addu TMP0, RD, TMP0
3874 | slt TMP3, TMP3, r0
3875 | movf TMP1, r0, 0
3876 | addu PC, PC, TMP0
3877 | movf TMP2, r0, 1
3878 | lhu RD, -4+OFS_RD(PC)
3879 | movn TMP1, TMP2, TMP3
3880 | bnez TMP1, =>BC_JLOOP
3881 |. decode_RD8b RD 4982 |. decode_RD8b RD
3882 } else if (op == BC_JFORL) { 4983 } else if (op == BC_JFORL) {
3883 | li TMP1, 1 4984 | beqz CRET1, =>BC_JLOOP
3884 | li TMP2, 1 4985 }
3885 | slt TMP3, TMP3, r0 4986 | ins_next2
3886 | movf TMP1, r0, 0 4987 |
3887 | movf TMP2, r0, 1 4988 |5: // FP loop.
3888 | movn TMP1, TMP2, TMP3 4989 |.if FPU
3889 | bnez TMP1, =>BC_JLOOP 4990 if (!vk) {
4991 | ldc1 f0, FORL_IDX*8(RA)
4992 | ldc1 f2, FORL_STOP*8(RA)
4993 | sltiu TMP0, SFARG1HI, LJ_TISNUM
4994 | sltiu TMP1, SFARG2HI, LJ_TISNUM
4995 | sltiu AT, SFRETHI, LJ_TISNUM
4996 | and TMP0, TMP0, TMP1
4997 | and AT, AT, TMP0
4998 | beqz AT, ->vmeta_for
4999 |. slt TMP3, SFRETHI, r0
5000 | c.ole.d 0, f0, f2
5001 | c.ole.d 1, f2, f0
5002 | li CRET1, 1
5003 | movt CRET1, r0, 0
5004 | movt AT, r0, 1
5005 | b <1
5006 |. movn CRET1, AT, TMP3
5007 } else {
5008 | ldc1 f0, FORL_IDX*8(RA)
5009 | ldc1 f4, FORL_STEP*8(RA)
5010 | ldc1 f2, FORL_STOP*8(RA)
5011 | lw SFARG2HI, FORL_STEP*8+HI(RA)
5012 | add.d f0, f0, f4
5013 | c.ole.d 0, f0, f2
5014 | c.ole.d 1, f2, f0
5015 | slt TMP3, SFARG2HI, r0
5016 | li CRET1, 1
5017 | li AT, 1
5018 | movt CRET1, r0, 0
5019 | movt AT, r0, 1
5020 | movn CRET1, AT, TMP3
5021 if (op == BC_IFORL) {
5022 | movn TMP2, r0, CRET1
5023 | addu PC, PC, TMP2
5024 }
5025 | sdc1 f0, FORL_IDX*8(RA)
5026 | ins_next1
5027 | b <2
5028 |. sdc1 f0, FORL_EXT*8(RA)
5029 }
5030 |.else
5031 if (!vk) {
5032 | sltiu TMP0, SFARG1HI, LJ_TISNUM
5033 | sltiu TMP1, SFARG2HI, LJ_TISNUM
5034 | sltiu AT, SFRETHI, LJ_TISNUM
5035 | and TMP0, TMP0, TMP1
5036 | and AT, AT, TMP0
5037 | beqz AT, ->vmeta_for
5038 |. nop
5039 | bal ->vm_sfcmpolex
5040 |. move TMP3, SFRETHI
5041 | b <1
3890 |. nop 5042 |. nop
3891 } else { 5043 } else {
3892 | addu TMP1, RD, TMP0 5044 | lw SFARG2HI, FORL_STEP*8+HI(RA)
3893 | slt TMP3, TMP3, r0 5045 | load_got __adddf3
3894 | move TMP2, TMP1 5046 | call_extern
3895 if (op == BC_FORI) { 5047 |. sw TMP2, ARG5
3896 | movt TMP1, r0, 0 5048 | lw SFARG2HI, FORL_STOP*8+HI(RA)
3897 | movt TMP2, r0, 1 5049 | lw SFARG2LO, FORL_STOP*8+LO(RA)
5050 | move SFARG1HI, SFRETHI
5051 | move SFARG1LO, SFRETLO
5052 | bal ->vm_sfcmpolex
5053 |. lw TMP3, FORL_STEP*8+HI(RA)
5054 if ( op == BC_JFORL ) {
5055 | lhu RD, -4+OFS_RD(PC)
5056 | lw TMP2, ARG5
5057 | b <1
5058 |. decode_RD8b RD
3898 } else { 5059 } else {
3899 | movf TMP1, r0, 0 5060 | b <1
3900 | movf TMP2, r0, 1 5061 |. lw TMP2, ARG5
3901 } 5062 }
3902 | movn TMP1, TMP2, TMP3
3903 | addu PC, PC, TMP1
3904 } 5063 }
3905 | ins_next 5064 |.endif
3906 break; 5065 break;
3907 5066
3908 case BC_ITERL: 5067 case BC_ITERL:
@@ -3961,8 +5120,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3961 | sw AT, DISPATCH_GL(vmstate)(DISPATCH) 5120 | sw AT, DISPATCH_GL(vmstate)(DISPATCH)
3962 | lw TRACE:TMP2, 0(TMP1) 5121 | lw TRACE:TMP2, 0(TMP1)
3963 | sw BASE, DISPATCH_GL(jit_base)(DISPATCH) 5122 | sw BASE, DISPATCH_GL(jit_base)(DISPATCH)
3964 | sw L, DISPATCH_GL(jit_L)(DISPATCH)
3965 | lw TMP2, TRACE:TMP2->mcode 5123 | lw TMP2, TRACE:TMP2->mcode
5124 | sw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
3966 | jr TMP2 5125 | jr TMP2
3967 |. addiu JGL, DISPATCH, GG_DISP2G+32768 5126 |. addiu JGL, DISPATCH, GG_DISP2G+32768
3968 |.endif 5127 |.endif
@@ -4088,6 +5247,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4088 | li_vmstate INTERP 5247 | li_vmstate INTERP
4089 | lw PC, FRAME_PC(BASE) // Fetch PC of caller. 5248 | lw PC, FRAME_PC(BASE) // Fetch PC of caller.
4090 | subu RA, TMP1, RD // RA = L->top - nresults*8 5249 | subu RA, TMP1, RD // RA = L->top - nresults*8
5250 | sw L, DISPATCH_GL(cur_L)(DISPATCH)
4091 | b ->vm_returnc 5251 | b ->vm_returnc
4092 |. st_vmstate 5252 |. st_vmstate
4093 break; 5253 break;
@@ -4150,8 +5310,10 @@ static void emit_asm_debug(BuildCtx *ctx)
4150 fcofs, CFRAME_SIZE); 5310 fcofs, CFRAME_SIZE);
4151 for (i = 23; i >= 16; i--) 5311 for (i = 23; i >= 16; i--)
4152 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 26-i); 5312 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 26-i);
5313#if !LJ_SOFTFP
4153 for (i = 30; i >= 20; i -= 2) 5314 for (i = 30; i >= 20; i -= 2)
4154 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+32+i, 42-i); 5315 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+32+i, 42-i);
5316#endif
4155 fprintf(ctx->fp, 5317 fprintf(ctx->fp,
4156 "\t.align 2\n" 5318 "\t.align 2\n"
4157 ".LEFDE0:\n\n"); 5319 ".LEFDE0:\n\n");
@@ -4203,8 +5365,10 @@ static void emit_asm_debug(BuildCtx *ctx)
4203 fcofs, CFRAME_SIZE); 5365 fcofs, CFRAME_SIZE);
4204 for (i = 23; i >= 16; i--) 5366 for (i = 23; i >= 16; i--)
4205 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 26-i); 5367 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 26-i);
5368#if !LJ_SOFTFP
4206 for (i = 30; i >= 20; i -= 2) 5369 for (i = 30; i >= 20; i -= 2)
4207 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+32+i, 42-i); 5370 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+32+i, 42-i);
5371#endif
4208 fprintf(ctx->fp, 5372 fprintf(ctx->fp,
4209 "\t.align 2\n" 5373 "\t.align 2\n"
4210 ".LEFDE2:\n\n"); 5374 ".LEFDE2:\n\n");
diff --git a/src/vm_mips64.dasc b/src/vm_mips64.dasc
new file mode 100644
index 00000000..f8e181ee
--- /dev/null
+++ b/src/vm_mips64.dasc
@@ -0,0 +1,5557 @@
1|// Low-level VM code for MIPS64 CPUs.
2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4|//
5|// Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com.
6|// Sponsored by Cisco Systems, Inc.
7|
8|.arch mips64
9|.section code_op, code_sub
10|
11|.actionlist build_actionlist
12|.globals GLOB_
13|.globalnames globnames
14|.externnames extnames
15|
16|// Note: The ragged indentation of the instructions is intentional.
17|// The starting columns indicate data dependencies.
18|
19|//-----------------------------------------------------------------------
20|
21|// Fixed register assignments for the interpreter.
22|// Don't use: r0 = 0, r26/r27 = reserved, r28 = gp, r29 = sp, r31 = ra
23|
24|.macro .FPU, a, b
25|.if FPU
26| a, b
27|.endif
28|.endmacro
29|
30|// The following must be C callee-save (but BASE is often refetched).
31|.define BASE, r16 // Base of current Lua stack frame.
32|.define KBASE, r17 // Constants of current Lua function.
33|.define PC, r18 // Next PC.
34|.define DISPATCH, r19 // Opcode dispatch table.
35|.define LREG, r20 // Register holding lua_State (also in SAVE_L).
36|.define MULTRES, r21 // Size of multi-result: (nresults+1)*8.
37|
38|.define JGL, r30 // On-trace: global_State + 32768.
39|
40|// Constants for type-comparisons, stores and conversions. C callee-save.
41|.define TISNIL, r30
42|.define TISNUM, r22
43|.if FPU
44|.define TOBIT, f30 // 2^52 + 2^51.
45|.endif
46|
47|// The following temporaries are not saved across C calls, except for RA.
48|.define RA, r23 // Callee-save.
49|.define RB, r8
50|.define RC, r9
51|.define RD, r10
52|.define INS, r11
53|
54|.define AT, r1 // Assembler temporary.
55|.define TMP0, r12
56|.define TMP1, r13
57|.define TMP2, r14
58|.define TMP3, r15
59|
60|// MIPS n64 calling convention.
61|.define CFUNCADDR, r25
62|.define CARG1, r4
63|.define CARG2, r5
64|.define CARG3, r6
65|.define CARG4, r7
66|.define CARG5, r8
67|.define CARG6, r9
68|.define CARG7, r10
69|.define CARG8, r11
70|
71|.define CRET1, r2
72|.define CRET2, r3
73|
74|.if FPU
75|.define FARG1, f12
76|.define FARG2, f13
77|.define FARG3, f14
78|.define FARG4, f15
79|.define FARG5, f16
80|.define FARG6, f17
81|.define FARG7, f18
82|.define FARG8, f19
83|
84|.define FRET1, f0
85|.define FRET2, f2
86|
87|.define FTMP0, f20
88|.define FTMP1, f21
89|.define FTMP2, f22
90|.endif
91|
92|// Stack layout while in interpreter. Must match with lj_frame.h.
93|.if FPU // MIPS64 hard-float.
94|
95|.define CFRAME_SPACE, 192 // Delta for sp.
96|
97|//----- 16 byte aligned, <-- sp entering interpreter
98|.define SAVE_ERRF, 188(sp) // 32 bit values.
99|.define SAVE_NRES, 184(sp)
100|.define SAVE_CFRAME, 176(sp) // 64 bit values.
101|.define SAVE_L, 168(sp)
102|.define SAVE_PC, 160(sp)
103|//----- 16 byte aligned
104|.define SAVE_GPR_, 80 // .. 80+10*8: 64 bit GPR saves.
105|.define SAVE_FPR_, 16 // .. 16+8*8: 64 bit FPR saves.
106|
107|.else // MIPS64 soft-float
108|
109|.define CFRAME_SPACE, 128 // Delta for sp.
110|
111|//----- 16 byte aligned, <-- sp entering interpreter
112|.define SAVE_ERRF, 124(sp) // 32 bit values.
113|.define SAVE_NRES, 120(sp)
114|.define SAVE_CFRAME, 112(sp) // 64 bit values.
115|.define SAVE_L, 104(sp)
116|.define SAVE_PC, 96(sp)
117|//----- 16 byte aligned
118|.define SAVE_GPR_, 16 // .. 16+10*8: 64 bit GPR saves.
119|
120|.endif
121|
122|.define TMPX, 8(sp) // Unused by interpreter, temp for JIT code.
123|.define TMPD, 0(sp)
124|//----- 16 byte aligned
125|
126|.define TMPD_OFS, 0
127|
128|.define SAVE_MULTRES, TMPD
129|
130|//-----------------------------------------------------------------------
131|
132|.macro saveregs
133| daddiu sp, sp, -CFRAME_SPACE
134| sd ra, SAVE_GPR_+9*8(sp)
135| sd r30, SAVE_GPR_+8*8(sp)
136| .FPU sdc1 f31, SAVE_FPR_+7*8(sp)
137| sd r23, SAVE_GPR_+7*8(sp)
138| .FPU sdc1 f30, SAVE_FPR_+6*8(sp)
139| sd r22, SAVE_GPR_+6*8(sp)
140| .FPU sdc1 f29, SAVE_FPR_+5*8(sp)
141| sd r21, SAVE_GPR_+5*8(sp)
142| .FPU sdc1 f28, SAVE_FPR_+4*8(sp)
143| sd r20, SAVE_GPR_+4*8(sp)
144| .FPU sdc1 f27, SAVE_FPR_+3*8(sp)
145| sd r19, SAVE_GPR_+3*8(sp)
146| .FPU sdc1 f26, SAVE_FPR_+2*8(sp)
147| sd r18, SAVE_GPR_+2*8(sp)
148| .FPU sdc1 f25, SAVE_FPR_+1*8(sp)
149| sd r17, SAVE_GPR_+1*8(sp)
150| .FPU sdc1 f24, SAVE_FPR_+0*8(sp)
151| sd r16, SAVE_GPR_+0*8(sp)
152|.endmacro
153|
154|.macro restoreregs_ret
155| ld ra, SAVE_GPR_+9*8(sp)
156| ld r30, SAVE_GPR_+8*8(sp)
157| ld r23, SAVE_GPR_+7*8(sp)
158| .FPU ldc1 f31, SAVE_FPR_+7*8(sp)
159| ld r22, SAVE_GPR_+6*8(sp)
160| .FPU ldc1 f30, SAVE_FPR_+6*8(sp)
161| ld r21, SAVE_GPR_+5*8(sp)
162| .FPU ldc1 f29, SAVE_FPR_+5*8(sp)
163| ld r20, SAVE_GPR_+4*8(sp)
164| .FPU ldc1 f28, SAVE_FPR_+4*8(sp)
165| ld r19, SAVE_GPR_+3*8(sp)
166| .FPU ldc1 f27, SAVE_FPR_+3*8(sp)
167| ld r18, SAVE_GPR_+2*8(sp)
168| .FPU ldc1 f26, SAVE_FPR_+2*8(sp)
169| ld r17, SAVE_GPR_+1*8(sp)
170| .FPU ldc1 f25, SAVE_FPR_+1*8(sp)
171| ld r16, SAVE_GPR_+0*8(sp)
172| .FPU ldc1 f24, SAVE_FPR_+0*8(sp)
173| jr ra
174| daddiu sp, sp, CFRAME_SPACE
175|.endmacro
176|
177|// Type definitions. Some of these are only used for documentation.
178|.type L, lua_State, LREG
179|.type GL, global_State
180|.type TVALUE, TValue
181|.type GCOBJ, GCobj
182|.type STR, GCstr
183|.type TAB, GCtab
184|.type LFUNC, GCfuncL
185|.type CFUNC, GCfuncC
186|.type PROTO, GCproto
187|.type UPVAL, GCupval
188|.type NODE, Node
189|.type NARGS8, int
190|.type TRACE, GCtrace
191|.type SBUF, SBuf
192|
193|//-----------------------------------------------------------------------
194|
195|// Trap for not-yet-implemented parts.
196|.macro NYI; .long 0xec1cf0f0; .endmacro
197|
198|// Macros to mark delay slots.
199|.macro ., a; a; .endmacro
200|.macro ., a,b; a,b; .endmacro
201|.macro ., a,b,c; a,b,c; .endmacro
202|.macro ., a,b,c,d; a,b,c,d; .endmacro
203|
204|.define FRAME_PC, -8
205|.define FRAME_FUNC, -16
206|
207|//-----------------------------------------------------------------------
208|
209|// Endian-specific defines.
210|.if ENDIAN_LE
211|.define HI, 4
212|.define LO, 0
213|.define OFS_RD, 2
214|.define OFS_RA, 1
215|.define OFS_OP, 0
216|.else
217|.define HI, 0
218|.define LO, 4
219|.define OFS_RD, 0
220|.define OFS_RA, 2
221|.define OFS_OP, 3
222|.endif
223|
224|// Instruction decode.
225|.macro decode_OP1, dst, ins; andi dst, ins, 0xff; .endmacro
226|.macro decode_OP8a, dst, ins; andi dst, ins, 0xff; .endmacro
227|.macro decode_OP8b, dst; sll dst, dst, 3; .endmacro
228|.macro decode_RC8a, dst, ins; srl dst, ins, 13; .endmacro
229|.macro decode_RC8b, dst; andi dst, dst, 0x7f8; .endmacro
230|.macro decode_RD4b, dst; sll dst, dst, 2; .endmacro
231|.macro decode_RA8a, dst, ins; srl dst, ins, 5; .endmacro
232|.macro decode_RA8b, dst; andi dst, dst, 0x7f8; .endmacro
233|.macro decode_RB8a, dst, ins; srl dst, ins, 21; .endmacro
234|.macro decode_RB8b, dst; andi dst, dst, 0x7f8; .endmacro
235|.macro decode_RD8a, dst, ins; srl dst, ins, 16; .endmacro
236|.macro decode_RD8b, dst; sll dst, dst, 3; .endmacro
237|.macro decode_RDtoRC8, dst, src; andi dst, src, 0x7f8; .endmacro
238|
239|// Instruction fetch.
240|.macro ins_NEXT1
241| lw INS, 0(PC)
242| daddiu PC, PC, 4
243|.endmacro
244|// Instruction decode+dispatch.
245|.macro ins_NEXT2
246| decode_OP8a TMP1, INS
247| decode_OP8b TMP1
248| daddu TMP0, DISPATCH, TMP1
249| decode_RD8a RD, INS
250| ld AT, 0(TMP0)
251| decode_RA8a RA, INS
252| decode_RD8b RD
253| jr AT
254| decode_RA8b RA
255|.endmacro
256|.macro ins_NEXT
257| ins_NEXT1
258| ins_NEXT2
259|.endmacro
260|
261|// Instruction footer.
262|.if 1
263| // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
264| .define ins_next, ins_NEXT
265| .define ins_next_, ins_NEXT
266| .define ins_next1, ins_NEXT1
267| .define ins_next2, ins_NEXT2
268|.else
269| // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
270| // Affects only certain kinds of benchmarks (and only with -j off).
271| .macro ins_next
272| b ->ins_next
273| .endmacro
274| .macro ins_next1
275| .endmacro
276| .macro ins_next2
277| b ->ins_next
278| .endmacro
279| .macro ins_next_
280| ->ins_next:
281| ins_NEXT
282| .endmacro
283|.endif
284|
285|// Call decode and dispatch.
286|.macro ins_callt
287| // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
288| ld PC, LFUNC:RB->pc
289| lw INS, 0(PC)
290| daddiu PC, PC, 4
291| decode_OP8a TMP1, INS
292| decode_RA8a RA, INS
293| decode_OP8b TMP1
294| decode_RA8b RA
295| daddu TMP0, DISPATCH, TMP1
296| ld TMP0, 0(TMP0)
297| jr TMP0
298| daddu RA, RA, BASE
299|.endmacro
300|
301|.macro ins_call
302| // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
303| sd PC, FRAME_PC(BASE)
304| ins_callt
305|.endmacro
306|
307|//-----------------------------------------------------------------------
308|
309|.macro branch_RD
310| srl TMP0, RD, 1
311| lui AT, (-(BCBIAS_J*4 >> 16) & 65535)
312| addu TMP0, TMP0, AT
313| daddu PC, PC, TMP0
314|.endmacro
315|
316|// Assumes DISPATCH is relative to GL.
317#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
318#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
319#define GG_DISP2GOT (GG_OFS(got) - GG_OFS(dispatch))
320#define DISPATCH_GOT(name) (GG_DISP2GOT + sizeof(void*)*LJ_GOT_##name)
321|
322#define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
323|
324|.macro load_got, func
325| ld CFUNCADDR, DISPATCH_GOT(func)(DISPATCH)
326|.endmacro
327|// Much faster. Sadly, there's no easy way to force the required code layout.
328|// .macro call_intern, func; bal extern func; .endmacro
329|.macro call_intern, func; jalr CFUNCADDR; .endmacro
330|.macro call_extern; jalr CFUNCADDR; .endmacro
331|.macro jmp_extern; jr CFUNCADDR; .endmacro
332|
333|.macro hotcheck, delta, target
334| dsrl TMP1, PC, 1
335| andi TMP1, TMP1, 126
336| daddu TMP1, TMP1, DISPATCH
337| lhu TMP2, GG_DISP2HOT(TMP1)
338| addiu TMP2, TMP2, -delta
339| bltz TMP2, target
340|. sh TMP2, GG_DISP2HOT(TMP1)
341|.endmacro
342|
343|.macro hotloop
344| hotcheck HOTCOUNT_LOOP, ->vm_hotloop
345|.endmacro
346|
347|.macro hotcall
348| hotcheck HOTCOUNT_CALL, ->vm_hotcall
349|.endmacro
350|
351|// Set current VM state. Uses TMP0.
352|.macro li_vmstate, st; li TMP0, ~LJ_VMST_..st; .endmacro
353|.macro st_vmstate; sw TMP0, DISPATCH_GL(vmstate)(DISPATCH); .endmacro
354|
355|// Move table write barrier back. Overwrites mark and tmp.
356|.macro barrierback, tab, mark, tmp, target
357| ld tmp, DISPATCH_GL(gc.grayagain)(DISPATCH)
358| andi mark, mark, ~LJ_GC_BLACK & 255 // black2gray(tab)
359| sd tab, DISPATCH_GL(gc.grayagain)(DISPATCH)
360| sb mark, tab->marked
361| b target
362|. sd tmp, tab->gclist
363|.endmacro
364|
365|// Clear type tag. Isolate lowest 14+32+1=47 bits of reg.
366|.macro cleartp, reg; dextm reg, reg, 0, 14; .endmacro
367|.macro cleartp, dst, reg; dextm dst, reg, 0, 14; .endmacro
368|
369|// Set type tag: Merge 17 type bits into bits [15+32=47, 31+32+1=64) of dst.
370|.macro settp, dst, tp; dinsu dst, tp, 15, 31; .endmacro
371|
372|// Extract (negative) type tag.
373|.macro gettp, dst, src; dsra dst, src, 47; .endmacro
374|
375|// Macros to check the TValue type and extract the GCobj. Branch on failure.
376|.macro checktp, reg, tp, target
377| gettp AT, reg
378| daddiu AT, AT, tp
379| bnez AT, target
380|. cleartp reg
381|.endmacro
382|.macro checktp, dst, reg, tp, target
383| gettp AT, reg
384| daddiu AT, AT, tp
385| bnez AT, target
386|. cleartp dst, reg
387|.endmacro
388|.macro checkstr, reg, target; checktp reg, -LJ_TSTR, target; .endmacro
389|.macro checktab, reg, target; checktp reg, -LJ_TTAB, target; .endmacro
390|.macro checkfunc, reg, target; checktp reg, -LJ_TFUNC, target; .endmacro
391|.macro checkint, reg, target // Caveat: has delay slot!
392| gettp AT, reg
393| bne AT, TISNUM, target
394|.endmacro
395|.macro checknum, reg, target // Caveat: has delay slot!
396| gettp AT, reg
397| sltiu AT, AT, LJ_TISNUM
398| beqz AT, target
399|.endmacro
400|
401|.macro mov_false, reg
402| lu reg, 0x8000
403| dsll reg, reg, 32
404| not reg, reg
405|.endmacro
406|.macro mov_true, reg
407| li reg, 0x0001
408| dsll reg, reg, 48
409| not reg, reg
410|.endmacro
411|
412|//-----------------------------------------------------------------------
413
414/* Generate subroutines used by opcodes and other parts of the VM. */
415/* The .code_sub section should be last to help static branch prediction. */
416static void build_subroutines(BuildCtx *ctx)
417{
418 |.code_sub
419 |
420 |//-----------------------------------------------------------------------
421 |//-- Return handling ----------------------------------------------------
422 |//-----------------------------------------------------------------------
423 |
424 |->vm_returnp:
425 | // See vm_return. Also: TMP2 = previous base.
426 | andi AT, PC, FRAME_P
427 | beqz AT, ->cont_dispatch
428 |
429 | // Return from pcall or xpcall fast func.
430 |. mov_true TMP1
431 | ld PC, FRAME_PC(TMP2) // Fetch PC of previous frame.
432 | move BASE, TMP2 // Restore caller base.
433 | // Prepending may overwrite the pcall frame, so do it at the end.
434 | sd TMP1, -8(RA) // Prepend true to results.
435 | daddiu RA, RA, -8
436 |
437 |->vm_returnc:
438 | addiu RD, RD, 8 // RD = (nresults+1)*8.
439 | andi TMP0, PC, FRAME_TYPE
440 | beqz RD, ->vm_unwind_c_eh
441 |. li CRET1, LUA_YIELD
442 | beqz TMP0, ->BC_RET_Z // Handle regular return to Lua.
443 |. move MULTRES, RD
444 |
445 |->vm_return:
446 | // BASE = base, RA = resultptr, RD/MULTRES = (nresults+1)*8, PC = return
447 | // TMP0 = PC & FRAME_TYPE
448 | li TMP2, -8
449 | xori AT, TMP0, FRAME_C
450 | and TMP2, PC, TMP2
451 | bnez AT, ->vm_returnp
452 | dsubu TMP2, BASE, TMP2 // TMP2 = previous base.
453 |
454 | addiu TMP1, RD, -8
455 | sd TMP2, L->base
456 | li_vmstate C
457 | lw TMP2, SAVE_NRES
458 | daddiu BASE, BASE, -16
459 | st_vmstate
460 | beqz TMP1, >2
461 |. sll TMP2, TMP2, 3
462 |1:
463 | addiu TMP1, TMP1, -8
464 | ld CRET1, 0(RA)
465 | daddiu RA, RA, 8
466 | sd CRET1, 0(BASE)
467 | bnez TMP1, <1
468 |. daddiu BASE, BASE, 8
469 |
470 |2:
471 | bne TMP2, RD, >6
472 |3:
473 |. sd BASE, L->top // Store new top.
474 |
475 |->vm_leave_cp:
476 | ld TMP0, SAVE_CFRAME // Restore previous C frame.
477 | move CRET1, r0 // Ok return status for vm_pcall.
478 | sd TMP0, L->cframe
479 |
480 |->vm_leave_unw:
481 | restoreregs_ret
482 |
483 |6:
484 | ld TMP1, L->maxstack
485 | slt AT, TMP2, RD
486 | bnez AT, >7 // Less results wanted?
487 | // More results wanted. Check stack size and fill up results with nil.
488 |. slt AT, BASE, TMP1
489 | beqz AT, >8
490 |. nop
491 | sd TISNIL, 0(BASE)
492 | addiu RD, RD, 8
493 | b <2
494 |. daddiu BASE, BASE, 8
495 |
496 |7: // Less results wanted.
497 | subu TMP0, RD, TMP2
498 | dsubu TMP0, BASE, TMP0 // Either keep top or shrink it.
499 |.if MIPSR6
500 | selnez TMP0, TMP0, TMP2 // LUA_MULTRET+1 case?
501 | seleqz BASE, BASE, TMP2
502 | b <3
503 |. or BASE, BASE, TMP0
504 |.else
505 | b <3
506 |. movn BASE, TMP0, TMP2 // LUA_MULTRET+1 case?
507 |.endif
508 |
509 |8: // Corner case: need to grow stack for filling up results.
510 | // This can happen if:
511 | // - A C function grows the stack (a lot).
512 | // - The GC shrinks the stack in between.
513 | // - A return back from a lua_call() with (high) nresults adjustment.
514 | load_got lj_state_growstack
515 | move MULTRES, RD
516 | srl CARG2, TMP2, 3
517 | call_intern lj_state_growstack // (lua_State *L, int n)
518 |. move CARG1, L
519 | lw TMP2, SAVE_NRES
520 | ld BASE, L->top // Need the (realloced) L->top in BASE.
521 | move RD, MULTRES
522 | b <2
523 |. sll TMP2, TMP2, 3
524 |
525 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
526 | // (void *cframe, int errcode)
527 | move sp, CARG1
528 | move CRET1, CARG2
529 |->vm_unwind_c_eh: // Landing pad for external unwinder.
530 | ld L, SAVE_L
531 | li TMP0, ~LJ_VMST_C
532 | ld GL:TMP1, L->glref
533 | b ->vm_leave_unw
534 |. sw TMP0, GL:TMP1->vmstate
535 |
536 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
537 | // (void *cframe)
538 | li AT, -4
539 | and sp, CARG1, AT
540 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
541 | ld L, SAVE_L
542 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
543 | li TISNIL, LJ_TNIL
544 | li TISNUM, LJ_TISNUM
545 | ld BASE, L->base
546 | ld DISPATCH, L->glref // Setup pointer to dispatch table.
547 | .FPU mtc1 TMP3, TOBIT
548 | mov_false TMP1
549 | li_vmstate INTERP
550 | ld PC, FRAME_PC(BASE) // Fetch PC of previous frame.
551 | .FPU cvt.d.s TOBIT, TOBIT
552 | daddiu RA, BASE, -8 // Results start at BASE-8.
553 | daddiu DISPATCH, DISPATCH, GG_G2DISP
554 | sd TMP1, 0(RA) // Prepend false to error message.
555 | st_vmstate
556 | b ->vm_returnc
557 |. li RD, 16 // 2 results: false + error message.
558 |
559 |->vm_unwind_stub: // Jump to exit stub from unwinder.
560 | jr CARG1
561 |. move ra, CARG2
562 |
563 |//-----------------------------------------------------------------------
564 |//-- Grow stack for calls -----------------------------------------------
565 |//-----------------------------------------------------------------------
566 |
567 |->vm_growstack_c: // Grow stack for C function.
568 | b >2
569 |. li CARG2, LUA_MINSTACK
570 |
571 |->vm_growstack_l: // Grow stack for Lua function.
572 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
573 | daddu RC, BASE, RC
574 | dsubu RA, RA, BASE
575 | sd BASE, L->base
576 | daddiu PC, PC, 4 // Must point after first instruction.
577 | sd RC, L->top
578 | srl CARG2, RA, 3
579 |2:
580 | // L->base = new base, L->top = top
581 | load_got lj_state_growstack
582 | sd PC, SAVE_PC
583 | call_intern lj_state_growstack // (lua_State *L, int n)
584 |. move CARG1, L
585 | ld BASE, L->base
586 | ld RC, L->top
587 | ld LFUNC:RB, FRAME_FUNC(BASE)
588 | dsubu RC, RC, BASE
589 | cleartp LFUNC:RB
590 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
591 | ins_callt // Just retry the call.
592 |
593 |//-----------------------------------------------------------------------
594 |//-- Entry points into the assembler VM ---------------------------------
595 |//-----------------------------------------------------------------------
596 |
597 |->vm_resume: // Setup C frame and resume thread.
598 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
599 | saveregs
600 | move L, CARG1
601 | ld DISPATCH, L->glref // Setup pointer to dispatch table.
602 | move BASE, CARG2
603 | lbu TMP1, L->status
604 | sd L, SAVE_L
605 | li PC, FRAME_CP
606 | daddiu TMP0, sp, CFRAME_RESUME
607 | daddiu DISPATCH, DISPATCH, GG_G2DISP
608 | sw r0, SAVE_NRES
609 | sw r0, SAVE_ERRF
610 | sd CARG1, SAVE_PC // Any value outside of bytecode is ok.
611 | sd r0, SAVE_CFRAME
612 | beqz TMP1, >3
613 |. sd TMP0, L->cframe
614 |
615 | // Resume after yield (like a return).
616 | sd L, DISPATCH_GL(cur_L)(DISPATCH)
617 | move RA, BASE
618 | ld BASE, L->base
619 | ld TMP1, L->top
620 | ld PC, FRAME_PC(BASE)
621 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
622 | dsubu RD, TMP1, BASE
623 | .FPU mtc1 TMP3, TOBIT
624 | sb r0, L->status
625 | .FPU cvt.d.s TOBIT, TOBIT
626 | li_vmstate INTERP
627 | daddiu RD, RD, 8
628 | st_vmstate
629 | move MULTRES, RD
630 | andi TMP0, PC, FRAME_TYPE
631 | li TISNIL, LJ_TNIL
632 | beqz TMP0, ->BC_RET_Z
633 |. li TISNUM, LJ_TISNUM
634 | b ->vm_return
635 |. nop
636 |
637 |->vm_pcall: // Setup protected C frame and enter VM.
638 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
639 | saveregs
640 | sw CARG4, SAVE_ERRF
641 | b >1
642 |. li PC, FRAME_CP
643 |
644 |->vm_call: // Setup C frame and enter VM.
645 | // (lua_State *L, TValue *base, int nres1)
646 | saveregs
647 | li PC, FRAME_C
648 |
649 |1: // Entry point for vm_pcall above (PC = ftype).
650 | ld TMP1, L:CARG1->cframe
651 | move L, CARG1
652 | sw CARG3, SAVE_NRES
653 | ld DISPATCH, L->glref // Setup pointer to dispatch table.
654 | sd CARG1, SAVE_L
655 | move BASE, CARG2
656 | daddiu DISPATCH, DISPATCH, GG_G2DISP
657 | sd CARG1, SAVE_PC // Any value outside of bytecode is ok.
658 | sd TMP1, SAVE_CFRAME
659 | sd sp, L->cframe // Add our C frame to cframe chain.
660 |
661 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
662 | sd L, DISPATCH_GL(cur_L)(DISPATCH)
663 | ld TMP2, L->base // TMP2 = old base (used in vmeta_call).
664 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
665 | ld TMP1, L->top
666 | .FPU mtc1 TMP3, TOBIT
667 | daddu PC, PC, BASE
668 | dsubu NARGS8:RC, TMP1, BASE
669 | li TISNUM, LJ_TISNUM
670 | dsubu PC, PC, TMP2 // PC = frame delta + frame type
671 | .FPU cvt.d.s TOBIT, TOBIT
672 | li_vmstate INTERP
673 | li TISNIL, LJ_TNIL
674 | st_vmstate
675 |
676 |->vm_call_dispatch:
677 | // TMP2 = old base, BASE = new base, RC = nargs*8, PC = caller PC
678 | ld LFUNC:RB, FRAME_FUNC(BASE)
679 | checkfunc LFUNC:RB, ->vmeta_call
680 |
681 |->vm_call_dispatch_f:
682 | ins_call
683 | // BASE = new base, RB = func, RC = nargs*8, PC = caller PC
684 |
685 |->vm_cpcall: // Setup protected C frame, call C.
686 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
687 | saveregs
688 | move L, CARG1
689 | ld TMP0, L:CARG1->stack
690 | sd CARG1, SAVE_L
691 | ld TMP1, L->top
692 | ld DISPATCH, L->glref // Setup pointer to dispatch table.
693 | sd CARG1, SAVE_PC // Any value outside of bytecode is ok.
694 | dsubu TMP0, TMP0, TMP1 // Compute -savestack(L, L->top).
695 | ld TMP1, L->cframe
696 | daddiu DISPATCH, DISPATCH, GG_G2DISP
697 | sw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame.
698 | sw r0, SAVE_ERRF // No error function.
699 | sd TMP1, SAVE_CFRAME
700 | sd sp, L->cframe // Add our C frame to cframe chain.
701 | sd L, DISPATCH_GL(cur_L)(DISPATCH)
702 | jalr CARG4 // (lua_State *L, lua_CFunction func, void *ud)
703 |. move CFUNCADDR, CARG4
704 | move BASE, CRET1
705 | bnez CRET1, <3 // Else continue with the call.
706 |. li PC, FRAME_CP
707 | b ->vm_leave_cp // No base? Just remove C frame.
708 |. nop
709 |
710 |//-----------------------------------------------------------------------
711 |//-- Metamethod handling ------------------------------------------------
712 |//-----------------------------------------------------------------------
713 |
714 |// The lj_meta_* functions (except for lj_meta_cat) don't reallocate the
715 |// stack, so BASE doesn't need to be reloaded across these calls.
716 |
717 |//-- Continuation dispatch ----------------------------------------------
718 |
719 |->cont_dispatch:
720 | // BASE = meta base, RA = resultptr, RD = (nresults+1)*8
721 | ld TMP0, -32(BASE) // Continuation.
722 | move RB, BASE
723 | move BASE, TMP2 // Restore caller BASE.
724 | ld LFUNC:TMP1, FRAME_FUNC(TMP2)
725 |.if FFI
726 | sltiu AT, TMP0, 2
727 |.endif
728 | ld PC, -24(RB) // Restore PC from [cont|PC].
729 | cleartp LFUNC:TMP1
730 | daddu TMP2, RA, RD
731 |.if FFI
732 | bnez AT, >1
733 |.endif
734 |. sd TISNIL, -8(TMP2) // Ensure one valid arg.
735 | ld TMP1, LFUNC:TMP1->pc
736 | // BASE = base, RA = resultptr, RB = meta base
737 | jr TMP0 // Jump to continuation.
738 |. ld KBASE, PC2PROTO(k)(TMP1)
739 |
740 |.if FFI
741 |1:
742 | bnez TMP0, ->cont_ffi_callback // cont = 1: return from FFI callback.
743 | // cont = 0: tailcall from C function.
744 |. daddiu TMP1, RB, -32
745 | b ->vm_call_tail
746 |. dsubu RC, TMP1, BASE
747 |.endif
748 |
749 |->cont_cat: // RA = resultptr, RB = meta base
750 | lw INS, -4(PC)
751 | daddiu CARG2, RB, -32
752 | ld CRET1, 0(RA)
753 | decode_RB8a MULTRES, INS
754 | decode_RA8a RA, INS
755 | decode_RB8b MULTRES
756 | decode_RA8b RA
757 | daddu TMP1, BASE, MULTRES
758 | sd BASE, L->base
759 | dsubu CARG3, CARG2, TMP1
760 | bne TMP1, CARG2, ->BC_CAT_Z
761 |. sd CRET1, 0(CARG2)
762 | daddu RA, BASE, RA
763 | b ->cont_nop
764 |. sd CRET1, 0(RA)
765 |
766 |//-- Table indexing metamethods -----------------------------------------
767 |
768 |->vmeta_tgets1:
769 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
770 | li TMP0, LJ_TSTR
771 | settp STR:RC, TMP0
772 | b >1
773 |. sd STR:RC, 0(CARG3)
774 |
775 |->vmeta_tgets:
776 | daddiu CARG2, DISPATCH, DISPATCH_GL(tmptv)
777 | li TMP0, LJ_TTAB
778 | li TMP1, LJ_TSTR
779 | settp TAB:RB, TMP0
780 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv2)
781 | sd TAB:RB, 0(CARG2)
782 | settp STR:RC, TMP1
783 | b >1
784 |. sd STR:RC, 0(CARG3)
785 |
786 |->vmeta_tgetb: // TMP0 = index
787 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
788 | settp TMP0, TISNUM
789 | sd TMP0, 0(CARG3)
790 |
791 |->vmeta_tgetv:
792 |1:
793 | load_got lj_meta_tget
794 | sd BASE, L->base
795 | sd PC, SAVE_PC
796 | call_intern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
797 |. move CARG1, L
798 | // Returns TValue * (finished) or NULL (metamethod).
799 | beqz CRET1, >3
800 |. daddiu TMP1, BASE, -FRAME_CONT
801 | ld CARG1, 0(CRET1)
802 | ins_next1
803 | sd CARG1, 0(RA)
804 | ins_next2
805 |
806 |3: // Call __index metamethod.
807 | // BASE = base, L->top = new base, stack = cont/func/t/k
808 | ld BASE, L->top
809 | sd PC, -24(BASE) // [cont|PC]
810 | dsubu PC, BASE, TMP1
811 | ld LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
812 | cleartp LFUNC:RB
813 | b ->vm_call_dispatch_f
814 |. li NARGS8:RC, 16 // 2 args for func(t, k).
815 |
816 |->vmeta_tgetr:
817 | load_got lj_tab_getinth
818 | call_intern lj_tab_getinth // (GCtab *t, int32_t key)
819 |. nop
820 | // Returns cTValue * or NULL.
821 | beqz CRET1, ->BC_TGETR_Z
822 |. move CARG2, TISNIL
823 | b ->BC_TGETR_Z
824 |. ld CARG2, 0(CRET1)
825 |
826 |//-----------------------------------------------------------------------
827 |
828 |->vmeta_tsets1:
829 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
830 | li TMP0, LJ_TSTR
831 | settp STR:RC, TMP0
832 | b >1
833 |. sd STR:RC, 0(CARG3)
834 |
835 |->vmeta_tsets:
836 | daddiu CARG2, DISPATCH, DISPATCH_GL(tmptv)
837 | li TMP0, LJ_TTAB
838 | li TMP1, LJ_TSTR
839 | settp TAB:RB, TMP0
840 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv2)
841 | sd TAB:RB, 0(CARG2)
842 | settp STR:RC, TMP1
843 | b >1
844 |. sd STR:RC, 0(CARG3)
845 |
846 |->vmeta_tsetb: // TMP0 = index
847 | daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
848 | settp TMP0, TISNUM
849 | sd TMP0, 0(CARG3)
850 |
851 |->vmeta_tsetv:
852 |1:
853 | load_got lj_meta_tset
854 | sd BASE, L->base
855 | sd PC, SAVE_PC
856 | call_intern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
857 |. move CARG1, L
858 | // Returns TValue * (finished) or NULL (metamethod).
859 | beqz CRET1, >3
860 |. ld CARG1, 0(RA)
861 | // NOBARRIER: lj_meta_tset ensures the table is not black.
862 | ins_next1
863 | sd CARG1, 0(CRET1)
864 | ins_next2
865 |
866 |3: // Call __newindex metamethod.
867 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
868 | daddiu TMP1, BASE, -FRAME_CONT
869 | ld BASE, L->top
870 | sd PC, -24(BASE) // [cont|PC]
871 | dsubu PC, BASE, TMP1
872 | ld LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
873 | cleartp LFUNC:RB
874 | sd CARG1, 16(BASE) // Copy value to third argument.
875 | b ->vm_call_dispatch_f
876 |. li NARGS8:RC, 24 // 3 args for func(t, k, v)
877 |
878 |->vmeta_tsetr:
879 | load_got lj_tab_setinth
880 | sd BASE, L->base
881 | sd PC, SAVE_PC
882 | call_intern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
883 |. move CARG1, L
884 | // Returns TValue *.
885 | b ->BC_TSETR_Z
886 |. nop
887 |
888 |//-- Comparison metamethods ---------------------------------------------
889 |
890 |->vmeta_comp:
891 | // RA/RD point to o1/o2.
892 | move CARG2, RA
893 | move CARG3, RD
894 | load_got lj_meta_comp
895 | daddiu PC, PC, -4
896 | sd BASE, L->base
897 | sd PC, SAVE_PC
898 | decode_OP1 CARG4, INS
899 | call_intern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
900 |. move CARG1, L
901 | // Returns 0/1 or TValue * (metamethod).
902 |3:
903 | sltiu AT, CRET1, 2
904 | beqz AT, ->vmeta_binop
905 | negu TMP2, CRET1
906 |4:
907 | lhu RD, OFS_RD(PC)
908 | daddiu PC, PC, 4
909 | lui TMP1, (-(BCBIAS_J*4 >> 16) & 65535)
910 | sll RD, RD, 2
911 | addu RD, RD, TMP1
912 | and RD, RD, TMP2
913 | daddu PC, PC, RD
914 |->cont_nop:
915 | ins_next
916 |
917 |->cont_ra: // RA = resultptr
918 | lbu TMP1, -4+OFS_RA(PC)
919 | ld CRET1, 0(RA)
920 | sll TMP1, TMP1, 3
921 | daddu TMP1, BASE, TMP1
922 | b ->cont_nop
923 |. sd CRET1, 0(TMP1)
924 |
925 |->cont_condt: // RA = resultptr
926 | ld TMP0, 0(RA)
927 | gettp TMP0, TMP0
928 | sltiu AT, TMP0, LJ_TISTRUECOND
929 | b <4
930 |. negu TMP2, AT // Branch if result is true.
931 |
932 |->cont_condf: // RA = resultptr
933 | ld TMP0, 0(RA)
934 | gettp TMP0, TMP0
935 | sltiu AT, TMP0, LJ_TISTRUECOND
936 | b <4
937 |. addiu TMP2, AT, -1 // Branch if result is false.
938 |
939 |->vmeta_equal:
940 | // CARG1/CARG2 point to o1/o2. TMP0 is set to 0/1.
941 | load_got lj_meta_equal
942 | cleartp LFUNC:CARG3, CARG2
943 | cleartp LFUNC:CARG2, CARG1
944 | move CARG4, TMP0
945 | daddiu PC, PC, -4
946 | sd BASE, L->base
947 | sd PC, SAVE_PC
948 | call_intern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
949 |. move CARG1, L
950 | // Returns 0/1 or TValue * (metamethod).
951 | b <3
952 |. nop
953 |
954 |->vmeta_equal_cd:
955 |.if FFI
956 | load_got lj_meta_equal_cd
957 | move CARG2, INS
958 | daddiu PC, PC, -4
959 | sd BASE, L->base
960 | sd PC, SAVE_PC
961 | call_intern lj_meta_equal_cd // (lua_State *L, BCIns op)
962 |. move CARG1, L
963 | // Returns 0/1 or TValue * (metamethod).
964 | b <3
965 |. nop
966 |.endif
967 |
968 |->vmeta_istype:
969 | load_got lj_meta_istype
970 | daddiu PC, PC, -4
971 | sd BASE, L->base
972 | srl CARG2, RA, 3
973 | srl CARG3, RD, 3
974 | sd PC, SAVE_PC
975 | call_intern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
976 |. move CARG1, L
977 | b ->cont_nop
978 |. nop
979 |
980 |//-- Arithmetic metamethods ---------------------------------------------
981 |
982 |->vmeta_unm:
983 | move RC, RB
984 |
985 |->vmeta_arith:
986 | load_got lj_meta_arith
987 | sd BASE, L->base
988 | move CARG2, RA
989 | sd PC, SAVE_PC
990 | move CARG3, RB
991 | move CARG4, RC
992 | decode_OP1 CARG5, INS // CARG5 == RB.
993 | call_intern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
994 |. move CARG1, L
995 | // Returns NULL (finished) or TValue * (metamethod).
996 | beqz CRET1, ->cont_nop
997 |. nop
998 |
999 | // Call metamethod for binary op.
1000 |->vmeta_binop:
1001 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
1002 | dsubu TMP1, CRET1, BASE
1003 | sd PC, -24(CRET1) // [cont|PC]
1004 | move TMP2, BASE
1005 | daddiu PC, TMP1, FRAME_CONT
1006 | move BASE, CRET1
1007 | b ->vm_call_dispatch
1008 |. li NARGS8:RC, 16 // 2 args for func(o1, o2).
1009 |
1010 |->vmeta_len:
1011 | // CARG2 already set by BC_LEN.
1012#if LJ_52
1013 | move MULTRES, CARG1
1014#endif
1015 | load_got lj_meta_len
1016 | sd BASE, L->base
1017 | sd PC, SAVE_PC
1018 | call_intern lj_meta_len // (lua_State *L, TValue *o)
1019 |. move CARG1, L
1020 | // Returns NULL (retry) or TValue * (metamethod base).
1021#if LJ_52
1022 | bnez CRET1, ->vmeta_binop // Binop call for compatibility.
1023 |. nop
1024 | b ->BC_LEN_Z
1025 |. move CARG1, MULTRES
1026#else
1027 | b ->vmeta_binop // Binop call for compatibility.
1028 |. nop
1029#endif
1030 |
1031 |//-- Call metamethod ----------------------------------------------------
1032 |
1033 |->vmeta_call: // Resolve and call __call metamethod.
1034 | // TMP2 = old base, BASE = new base, RC = nargs*8
1035 | load_got lj_meta_call
1036 | sd TMP2, L->base // This is the callers base!
1037 | daddiu CARG2, BASE, -16
1038 | sd PC, SAVE_PC
1039 | daddu CARG3, BASE, RC
1040 | move MULTRES, NARGS8:RC
1041 | call_intern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
1042 |. move CARG1, L
1043 | ld LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
1044 | daddiu NARGS8:RC, MULTRES, 8 // Got one more argument now.
1045 | cleartp LFUNC:RB
1046 | ins_call
1047 |
1048 |->vmeta_callt: // Resolve __call for BC_CALLT.
1049 | // BASE = old base, RA = new base, RC = nargs*8
1050 | load_got lj_meta_call
1051 | sd BASE, L->base
1052 | daddiu CARG2, RA, -16
1053 | sd PC, SAVE_PC
1054 | daddu CARG3, RA, RC
1055 | move MULTRES, NARGS8:RC
1056 | call_intern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
1057 |. move CARG1, L
1058 | ld RB, FRAME_FUNC(RA) // Guaranteed to be a function here.
1059 | ld TMP1, FRAME_PC(BASE)
1060 | daddiu NARGS8:RC, MULTRES, 8 // Got one more argument now.
1061 | b ->BC_CALLT_Z
1062 |. cleartp LFUNC:CARG3, RB
1063 |
1064 |//-- Argument coercion for 'for' statement ------------------------------
1065 |
1066 |->vmeta_for:
1067 | load_got lj_meta_for
1068 | sd BASE, L->base
1069 | move CARG2, RA
1070 | sd PC, SAVE_PC
1071 | move MULTRES, INS
1072 | call_intern lj_meta_for // (lua_State *L, TValue *base)
1073 |. move CARG1, L
1074 |.if JIT
1075 | decode_OP1 TMP0, MULTRES
1076 | li AT, BC_JFORI
1077 |.endif
1078 | decode_RA8a RA, MULTRES
1079 | decode_RD8a RD, MULTRES
1080 | decode_RA8b RA
1081 |.if JIT
1082 | beq TMP0, AT, =>BC_JFORI
1083 |. decode_RD8b RD
1084 | b =>BC_FORI
1085 |. nop
1086 |.else
1087 | b =>BC_FORI
1088 |. decode_RD8b RD
1089 |.endif
1090 |
1091 |//-----------------------------------------------------------------------
1092 |//-- Fast functions -----------------------------------------------------
1093 |//-----------------------------------------------------------------------
1094 |
1095 |.macro .ffunc, name
1096 |->ff_ .. name:
1097 |.endmacro
1098 |
1099 |.macro .ffunc_1, name
1100 |->ff_ .. name:
1101 | beqz NARGS8:RC, ->fff_fallback
1102 |. ld CARG1, 0(BASE)
1103 |.endmacro
1104 |
1105 |.macro .ffunc_2, name
1106 |->ff_ .. name:
1107 | sltiu AT, NARGS8:RC, 16
1108 | ld CARG1, 0(BASE)
1109 | bnez AT, ->fff_fallback
1110 |. ld CARG2, 8(BASE)
1111 |.endmacro
1112 |
1113 |.macro .ffunc_n, name // Caveat: has delay slot!
1114 |->ff_ .. name:
1115 | ld CARG1, 0(BASE)
1116 | beqz NARGS8:RC, ->fff_fallback
1117 | // Either ldc1 or the 1st instruction of checknum is in the delay slot.
1118 | .FPU ldc1 FARG1, 0(BASE)
1119 | checknum CARG1, ->fff_fallback
1120 |.endmacro
1121 |
1122 |.macro .ffunc_nn, name // Caveat: has delay slot!
1123 |->ff_ .. name:
1124 | ld CARG1, 0(BASE)
1125 | sltiu AT, NARGS8:RC, 16
1126 | ld CARG2, 8(BASE)
1127 | bnez AT, ->fff_fallback
1128 |. gettp TMP0, CARG1
1129 | gettp TMP1, CARG2
1130 | sltiu TMP0, TMP0, LJ_TISNUM
1131 | sltiu TMP1, TMP1, LJ_TISNUM
1132 | .FPU ldc1 FARG1, 0(BASE)
1133 | and TMP0, TMP0, TMP1
1134 | .FPU ldc1 FARG2, 8(BASE)
1135 | beqz TMP0, ->fff_fallback
1136 |.endmacro
1137 |
1138 |// Inlined GC threshold check. Caveat: uses TMP0 and TMP1 and has delay slot!
1139 |// MIPSR6: no delay slot, but a forbidden slot.
1140 |.macro ffgccheck
1141 | ld TMP0, DISPATCH_GL(gc.total)(DISPATCH)
1142 | ld TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
1143 | dsubu AT, TMP0, TMP1
1144 |.if MIPSR6
1145 | bgezalc AT, ->fff_gcstep
1146 |.else
1147 | bgezal AT, ->fff_gcstep
1148 |.endif
1149 |.endmacro
1150 |
1151 |//-- Base library: checks -----------------------------------------------
1152 |.ffunc_1 assert
1153 | gettp AT, CARG1
1154 | sltiu AT, AT, LJ_TISTRUECOND
1155 | beqz AT, ->fff_fallback
1156 |. daddiu RA, BASE, -16
1157 | ld PC, FRAME_PC(BASE)
1158 | addiu RD, NARGS8:RC, 8 // Compute (nresults+1)*8.
1159 | daddu TMP2, RA, RD
1160 | daddiu TMP1, BASE, 8
1161 | beq BASE, TMP2, ->fff_res // Done if exactly 1 argument.
1162 |. sd CARG1, 0(RA)
1163 |1:
1164 | ld CRET1, 0(TMP1)
1165 | sd CRET1, -16(TMP1)
1166 | bne TMP1, TMP2, <1
1167 |. daddiu TMP1, TMP1, 8
1168 | b ->fff_res
1169 |. nop
1170 |
1171 |.ffunc_1 type
1172 | gettp TMP0, CARG1
1173 | sltu TMP1, TISNUM, TMP0
1174 | not TMP2, TMP0
1175 | li TMP3, ~LJ_TISNUM
1176 |.if MIPSR6
1177 | selnez TMP2, TMP2, TMP1
1178 | seleqz TMP3, TMP3, TMP1
1179 | or TMP2, TMP2, TMP3
1180 |.else
1181 | movz TMP2, TMP3, TMP1
1182 |.endif
1183 | dsll TMP2, TMP2, 3
1184 | daddu TMP2, CFUNC:RB, TMP2
1185 | b ->fff_restv
1186 |. ld CARG1, CFUNC:TMP2->upvalue
1187 |
1188 |//-- Base library: getters and setters ---------------------------------
1189 |
1190 |.ffunc_1 getmetatable
1191 | gettp TMP2, CARG1
1192 | daddiu TMP0, TMP2, -LJ_TTAB
1193 | daddiu TMP1, TMP2, -LJ_TUDATA
1194 |.if MIPSR6
1195 | selnez TMP0, TMP1, TMP0
1196 |.else
1197 | movn TMP0, TMP1, TMP0
1198 |.endif
1199 | bnez TMP0, >6
1200 |. cleartp TAB:CARG1
1201 |1: // Field metatable must be at same offset for GCtab and GCudata!
1202 | ld TAB:RB, TAB:CARG1->metatable
1203 |2:
1204 | ld STR:RC, DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])(DISPATCH)
1205 | beqz TAB:RB, ->fff_restv
1206 |. li CARG1, LJ_TNIL
1207 | lw TMP0, TAB:RB->hmask
1208 | lw TMP1, STR:RC->sid
1209 | ld NODE:TMP2, TAB:RB->node
1210 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
1211 | dsll TMP0, TMP1, 5
1212 | dsll TMP1, TMP1, 3
1213 | dsubu TMP1, TMP0, TMP1
1214 | daddu NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
1215 | li CARG4, LJ_TSTR
1216 | settp STR:RC, CARG4 // Tagged key to look for.
1217 |3: // Rearranged logic, because we expect _not_ to find the key.
1218 | ld TMP0, NODE:TMP2->key
1219 | ld CARG1, NODE:TMP2->val
1220 | ld NODE:TMP2, NODE:TMP2->next
1221 | beq RC, TMP0, >5
1222 |. li AT, LJ_TTAB
1223 | bnez NODE:TMP2, <3
1224 |. nop
1225 |4:
1226 | move CARG1, RB
1227 | b ->fff_restv // Not found, keep default result.
1228 |. settp CARG1, AT
1229 |5:
1230 | bne CARG1, TISNIL, ->fff_restv
1231 |. nop
1232 | b <4 // Ditto for nil value.
1233 |. nop
1234 |
1235 |6:
1236 | sltiu AT, TMP2, LJ_TISNUM
1237 |.if MIPSR6
1238 | selnez TMP0, TISNUM, AT
1239 | seleqz AT, TMP2, AT
1240 | or TMP2, TMP0, AT
1241 |.else
1242 | movn TMP2, TISNUM, AT
1243 |.endif
1244 | dsll TMP2, TMP2, 3
1245 | dsubu TMP0, DISPATCH, TMP2
1246 | b <2
1247 |. ld TAB:RB, DISPATCH_GL(gcroot[GCROOT_BASEMT])-8(TMP0)
1248 |
1249 |.ffunc_2 setmetatable
1250 | // Fast path: no mt for table yet and not clearing the mt.
1251 | checktp TMP1, CARG1, -LJ_TTAB, ->fff_fallback
1252 | gettp TMP3, CARG2
1253 | ld TAB:TMP0, TAB:TMP1->metatable
1254 | lbu TMP2, TAB:TMP1->marked
1255 | daddiu AT, TMP3, -LJ_TTAB
1256 | cleartp TAB:CARG2
1257 | or AT, AT, TAB:TMP0
1258 | bnez AT, ->fff_fallback
1259 |. andi AT, TMP2, LJ_GC_BLACK // isblack(table)
1260 | beqz AT, ->fff_restv
1261 |. sd TAB:CARG2, TAB:TMP1->metatable
1262 | barrierback TAB:TMP1, TMP2, TMP0, ->fff_restv
1263 |
1264 |.ffunc rawget
1265 | ld CARG2, 0(BASE)
1266 | sltiu AT, NARGS8:RC, 16
1267 | load_got lj_tab_get
1268 | gettp TMP0, CARG2
1269 | cleartp CARG2
1270 | daddiu TMP0, TMP0, -LJ_TTAB
1271 | or AT, AT, TMP0
1272 | bnez AT, ->fff_fallback
1273 |. daddiu CARG3, BASE, 8
1274 | call_intern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1275 |. move CARG1, L
1276 | b ->fff_restv
1277 |. ld CARG1, 0(CRET1)
1278 |
1279 |//-- Base library: conversions ------------------------------------------
1280 |
1281 |.ffunc tonumber
1282 | // Only handles the number case inline (without a base argument).
1283 | ld CARG1, 0(BASE)
1284 | xori AT, NARGS8:RC, 8 // Exactly one number argument.
1285 | gettp TMP1, CARG1
1286 | sltu TMP0, TISNUM, TMP1
1287 | or AT, AT, TMP0
1288 | bnez AT, ->fff_fallback
1289 |. nop
1290 | b ->fff_restv
1291 |. nop
1292 |
1293 |.ffunc_1 tostring
1294 | // Only handles the string or number case inline.
1295 | gettp TMP0, CARG1
1296 | daddiu AT, TMP0, -LJ_TSTR
1297 | // A __tostring method in the string base metatable is ignored.
1298 | beqz AT, ->fff_restv // String key?
1299 | // Handle numbers inline, unless a number base metatable is present.
1300 |. ld TMP1, DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])(DISPATCH)
1301 | sltu TMP0, TISNUM, TMP0
1302 | or TMP0, TMP0, TMP1
1303 | bnez TMP0, ->fff_fallback
1304 |. sd BASE, L->base // Add frame since C call can throw.
1305 |.if MIPSR6
1306 | sd PC, SAVE_PC // Redundant (but a defined value).
1307 | ffgccheck
1308 |.else
1309 | ffgccheck
1310 |. sd PC, SAVE_PC // Redundant (but a defined value).
1311 |.endif
1312 | load_got lj_strfmt_number
1313 | move CARG1, L
1314 | call_intern lj_strfmt_number // (lua_State *L, cTValue *o)
1315 |. move CARG2, BASE
1316 | // Returns GCstr *.
1317 | li AT, LJ_TSTR
1318 | settp CRET1, AT
1319 | b ->fff_restv
1320 |. move CARG1, CRET1
1321 |
1322 |//-- Base library: iterators -------------------------------------------
1323 |
1324 |.ffunc_1 next
1325 | checktp CARG1, -LJ_TTAB, ->fff_fallback
1326 | daddu TMP2, BASE, NARGS8:RC
1327 | sd TISNIL, 0(TMP2) // Set missing 2nd arg to nil.
1328 | load_got lj_tab_next
1329 | ld PC, FRAME_PC(BASE)
1330 | daddiu CARG2, BASE, 8
1331 | call_intern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1332 |. daddiu CARG3, BASE, -16
1333 | // Returns 1=found, 0=end, -1=error.
1334 | daddiu RA, BASE, -16
1335 | bgtz CRET1, ->fff_res // Found key/value.
1336 |. li RD, (2+1)*8
1337 | beqz CRET1, ->fff_restv // End of traversal: return nil.
1338 |. move CARG1, TISNIL
1339 | ld CFUNC:RB, FRAME_FUNC(BASE)
1340 | cleartp CFUNC:RB
1341 | b ->fff_fallback // Invalid key.
1342 |. li RC, 2*8
1343 |
1344 |.ffunc_1 pairs
1345 | checktp TAB:TMP1, CARG1, -LJ_TTAB, ->fff_fallback
1346 | ld PC, FRAME_PC(BASE)
1347#if LJ_52
1348 | ld TAB:TMP2, TAB:TMP1->metatable
1349 | ld TMP0, CFUNC:RB->upvalue[0]
1350 | bnez TAB:TMP2, ->fff_fallback
1351#else
1352 | ld TMP0, CFUNC:RB->upvalue[0]
1353#endif
1354 |. daddiu RA, BASE, -16
1355 | sd TISNIL, 0(BASE)
1356 | sd CARG1, -8(BASE)
1357 | sd TMP0, 0(RA)
1358 | b ->fff_res
1359 |. li RD, (3+1)*8
1360 |
1361 |.ffunc_2 ipairs_aux
1362 | checktab CARG1, ->fff_fallback
1363 | checkint CARG2, ->fff_fallback
1364 |. lw TMP0, TAB:CARG1->asize
1365 | ld TMP1, TAB:CARG1->array
1366 | ld PC, FRAME_PC(BASE)
1367 | sextw TMP2, CARG2
1368 | addiu TMP2, TMP2, 1
1369 | sltu AT, TMP2, TMP0
1370 | daddiu RA, BASE, -16
1371 | zextw TMP0, TMP2
1372 | settp TMP0, TISNUM
1373 | beqz AT, >2 // Not in array part?
1374 |. sd TMP0, 0(RA)
1375 | dsll TMP3, TMP2, 3
1376 | daddu TMP3, TMP1, TMP3
1377 | ld TMP1, 0(TMP3)
1378 |1:
1379 | beq TMP1, TISNIL, ->fff_res // End of iteration, return 0 results.
1380 |. li RD, (0+1)*8
1381 | sd TMP1, -8(BASE)
1382 | b ->fff_res
1383 |. li RD, (2+1)*8
1384 |2: // Check for empty hash part first. Otherwise call C function.
1385 | lw TMP0, TAB:CARG1->hmask
1386 | load_got lj_tab_getinth
1387 | beqz TMP0, ->fff_res
1388 |. li RD, (0+1)*8
1389 | call_intern lj_tab_getinth // (GCtab *t, int32_t key)
1390 |. move CARG2, TMP2
1391 | // Returns cTValue * or NULL.
1392 | beqz CRET1, ->fff_res
1393 |. li RD, (0+1)*8
1394 | b <1
1395 |. ld TMP1, 0(CRET1)
1396 |
1397 |.ffunc_1 ipairs
1398 | checktp TAB:TMP1, CARG1, -LJ_TTAB, ->fff_fallback
1399 | ld PC, FRAME_PC(BASE)
1400#if LJ_52
1401 | ld TAB:TMP2, TAB:TMP1->metatable
1402 | ld CFUNC:TMP0, CFUNC:RB->upvalue[0]
1403 | bnez TAB:TMP2, ->fff_fallback
1404#else
1405 | ld TMP0, CFUNC:RB->upvalue[0]
1406#endif
1407 | daddiu RA, BASE, -16
1408 | dsll AT, TISNUM, 47
1409 | sd CARG1, -8(BASE)
1410 | sd AT, 0(BASE)
1411 | sd CFUNC:TMP0, 0(RA)
1412 | b ->fff_res
1413 |. li RD, (3+1)*8
1414 |
1415 |//-- Base library: catch errors ----------------------------------------
1416 |
1417 |.ffunc pcall
1418 | daddiu NARGS8:RC, NARGS8:RC, -8
1419 | lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1420 | bltz NARGS8:RC, ->fff_fallback
1421 |. move TMP2, BASE
1422 | daddiu BASE, BASE, 16
1423 | // Remember active hook before pcall.
1424 | srl TMP3, TMP3, HOOK_ACTIVE_SHIFT
1425 | andi TMP3, TMP3, 1
1426 | daddiu PC, TMP3, 16+FRAME_PCALL
1427 | beqz NARGS8:RC, ->vm_call_dispatch
1428 |1:
1429 |. daddu TMP0, BASE, NARGS8:RC
1430 |2:
1431 | ld TMP1, -16(TMP0)
1432 | sd TMP1, -8(TMP0)
1433 | daddiu TMP0, TMP0, -8
1434 | bne TMP0, BASE, <2
1435 |. nop
1436 | b ->vm_call_dispatch
1437 |. nop
1438 |
1439 |.ffunc xpcall
1440 | daddiu NARGS8:TMP0, NARGS8:RC, -16
1441 | ld CARG1, 0(BASE)
1442 | ld CARG2, 8(BASE)
1443 | bltz NARGS8:TMP0, ->fff_fallback
1444 |. lbu TMP1, DISPATCH_GL(hookmask)(DISPATCH)
1445 | gettp AT, CARG2
1446 | daddiu AT, AT, -LJ_TFUNC
1447 | bnez AT, ->fff_fallback // Traceback must be a function.
1448 |. move TMP2, BASE
1449 | move NARGS8:RC, NARGS8:TMP0
1450 | daddiu BASE, BASE, 24
1451 | // Remember active hook before pcall.
1452 | srl TMP3, TMP3, HOOK_ACTIVE_SHIFT
1453 | sd CARG2, 0(TMP2) // Swap function and traceback.
1454 | andi TMP3, TMP3, 1
1455 | sd CARG1, 8(TMP2)
1456 | beqz NARGS8:RC, ->vm_call_dispatch
1457 |. daddiu PC, TMP3, 24+FRAME_PCALL
1458 | b <1
1459 |. nop
1460 |
1461 |//-- Coroutine library --------------------------------------------------
1462 |
1463 |.macro coroutine_resume_wrap, resume
1464 |.if resume
1465 |.ffunc_1 coroutine_resume
1466 | checktp CARG1, CARG1, -LJ_TTHREAD, ->fff_fallback
1467 |.else
1468 |.ffunc coroutine_wrap_aux
1469 | ld L:CARG1, CFUNC:RB->upvalue[0].gcr
1470 | cleartp L:CARG1
1471 |.endif
1472 | lbu TMP0, L:CARG1->status
1473 | ld TMP1, L:CARG1->cframe
1474 | ld CARG2, L:CARG1->top
1475 | ld TMP2, L:CARG1->base
1476 | addiu AT, TMP0, -LUA_YIELD
1477 | daddu CARG3, CARG2, TMP0
1478 | daddiu TMP3, CARG2, 8
1479 |.if MIPSR6
1480 | seleqz CARG2, CARG2, AT
1481 | selnez TMP3, TMP3, AT
1482 | bgtz AT, ->fff_fallback // st > LUA_YIELD?
1483 |. or CARG2, TMP3, CARG2
1484 |.else
1485 | bgtz AT, ->fff_fallback // st > LUA_YIELD?
1486 |. movn CARG2, TMP3, AT
1487 |.endif
1488 | xor TMP2, TMP2, CARG3
1489 | bnez TMP1, ->fff_fallback // cframe != 0?
1490 |. or AT, TMP2, TMP0
1491 | ld TMP0, L:CARG1->maxstack
1492 | beqz AT, ->fff_fallback // base == top && st == 0?
1493 |. ld PC, FRAME_PC(BASE)
1494 | daddu TMP2, CARG2, NARGS8:RC
1495 | sltu AT, TMP0, TMP2
1496 | bnez AT, ->fff_fallback // Stack overflow?
1497 |. sd PC, SAVE_PC
1498 | sd BASE, L->base
1499 |1:
1500 |.if resume
1501 | daddiu BASE, BASE, 8 // Keep resumed thread in stack for GC.
1502 | daddiu NARGS8:RC, NARGS8:RC, -8
1503 | daddiu TMP2, TMP2, -8
1504 |.endif
1505 | sd TMP2, L:CARG1->top
1506 | daddu TMP1, BASE, NARGS8:RC
1507 | move CARG3, CARG2
1508 | sd BASE, L->top
1509 |2: // Move args to coroutine.
1510 | ld CRET1, 0(BASE)
1511 | sltu AT, BASE, TMP1
1512 | beqz AT, >3
1513 |. daddiu BASE, BASE, 8
1514 | sd CRET1, 0(CARG3)
1515 | b <2
1516 |. daddiu CARG3, CARG3, 8
1517 |3:
1518 | bal ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1519 |. move L:RA, L:CARG1
1520 | // Returns thread status.
1521 |4:
1522 | ld TMP2, L:RA->base
1523 | sltiu AT, CRET1, LUA_YIELD+1
1524 | ld TMP3, L:RA->top
1525 | li_vmstate INTERP
1526 | ld BASE, L->base
1527 | sd L, DISPATCH_GL(cur_L)(DISPATCH)
1528 | st_vmstate
1529 | beqz AT, >8
1530 |. dsubu RD, TMP3, TMP2
1531 | ld TMP0, L->maxstack
1532 | beqz RD, >6 // No results?
1533 |. daddu TMP1, BASE, RD
1534 | sltu AT, TMP0, TMP1
1535 | bnez AT, >9 // Need to grow stack?
1536 |. daddu TMP3, TMP2, RD
1537 | sd TMP2, L:RA->top // Clear coroutine stack.
1538 | move TMP1, BASE
1539 |5: // Move results from coroutine.
1540 | ld CRET1, 0(TMP2)
1541 | daddiu TMP2, TMP2, 8
1542 | sltu AT, TMP2, TMP3
1543 | sd CRET1, 0(TMP1)
1544 | bnez AT, <5
1545 |. daddiu TMP1, TMP1, 8
1546 |6:
1547 | andi TMP0, PC, FRAME_TYPE
1548 |.if resume
1549 | mov_true TMP1
1550 | daddiu RA, BASE, -8
1551 | sd TMP1, -8(BASE) // Prepend true to results.
1552 | daddiu RD, RD, 16
1553 |.else
1554 | move RA, BASE
1555 | daddiu RD, RD, 8
1556 |.endif
1557 |7:
1558 | sd PC, SAVE_PC
1559 | beqz TMP0, ->BC_RET_Z
1560 |. move MULTRES, RD
1561 | b ->vm_return
1562 |. nop
1563 |
1564 |8: // Coroutine returned with error (at co->top-1).
1565 |.if resume
1566 | daddiu TMP3, TMP3, -8
1567 | mov_false TMP1
1568 | ld CRET1, 0(TMP3)
1569 | sd TMP3, L:RA->top // Remove error from coroutine stack.
1570 | li RD, (2+1)*8
1571 | sd TMP1, -8(BASE) // Prepend false to results.
1572 | daddiu RA, BASE, -8
1573 | sd CRET1, 0(BASE) // Copy error message.
1574 | b <7
1575 |. andi TMP0, PC, FRAME_TYPE
1576 |.else
1577 | load_got lj_ffh_coroutine_wrap_err
1578 | move CARG2, L:RA
1579 | call_intern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1580 |. move CARG1, L
1581 |.endif
1582 |
1583 |9: // Handle stack expansion on return from yield.
1584 | load_got lj_state_growstack
1585 | srl CARG2, RD, 3
1586 | call_intern lj_state_growstack // (lua_State *L, int n)
1587 |. move CARG1, L
1588 | b <4
1589 |. li CRET1, 0
1590 |.endmacro
1591 |
1592 | coroutine_resume_wrap 1 // coroutine.resume
1593 | coroutine_resume_wrap 0 // coroutine.wrap
1594 |
1595 |.ffunc coroutine_yield
1596 | ld TMP0, L->cframe
1597 | daddu TMP1, BASE, NARGS8:RC
1598 | sd BASE, L->base
1599 | andi TMP0, TMP0, CFRAME_RESUME
1600 | sd TMP1, L->top
1601 | beqz TMP0, ->fff_fallback
1602 |. li CRET1, LUA_YIELD
1603 | sd r0, L->cframe
1604 | b ->vm_leave_unw
1605 |. sb CRET1, L->status
1606 |
1607 |//-- Math library -------------------------------------------------------
1608 |
1609 |.ffunc_1 math_abs
1610 | gettp CARG2, CARG1
1611 | daddiu AT, CARG2, -LJ_TISNUM
1612 | bnez AT, >1
1613 |. sextw TMP1, CARG1
1614 | sra TMP0, TMP1, 31 // Extract sign.
1615 | xor TMP1, TMP1, TMP0
1616 | dsubu CARG1, TMP1, TMP0
1617 | dsll TMP3, CARG1, 32
1618 | bgez TMP3, ->fff_restv
1619 |. settp CARG1, TISNUM
1620 | li CARG1, 0x41e0 // 2^31 as a double.
1621 | b ->fff_restv
1622 |. dsll CARG1, CARG1, 48
1623 |1:
1624 | sltiu AT, CARG2, LJ_TISNUM
1625 | beqz AT, ->fff_fallback
1626 |. dextm CARG1, CARG1, 0, 30
1627 |// fallthrough
1628 |
1629 |->fff_restv:
1630 | // CARG1 = TValue result.
1631 | ld PC, FRAME_PC(BASE)
1632 | daddiu RA, BASE, -16
1633 | sd CARG1, -16(BASE)
1634 |->fff_res1:
1635 | // RA = results, PC = return.
1636 | li RD, (1+1)*8
1637 |->fff_res:
1638 | // RA = results, RD = (nresults+1)*8, PC = return.
1639 | andi TMP0, PC, FRAME_TYPE
1640 | bnez TMP0, ->vm_return
1641 |. move MULTRES, RD
1642 | lw INS, -4(PC)
1643 | decode_RB8a RB, INS
1644 | decode_RB8b RB
1645 |5:
1646 | sltu AT, RD, RB
1647 | bnez AT, >6 // More results expected?
1648 |. decode_RA8a TMP0, INS
1649 | decode_RA8b TMP0
1650 | ins_next1
1651 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1652 | dsubu BASE, RA, TMP0
1653 | ins_next2
1654 |
1655 |6: // Fill up results with nil.
1656 | daddu TMP1, RA, RD
1657 | daddiu RD, RD, 8
1658 | b <5
1659 |. sd TISNIL, -8(TMP1)
1660 |
1661 |.macro math_extern, func
1662 | .ffunc_n math_ .. func
1663 | load_got func
1664 | call_extern
1665 |. nop
1666 | b ->fff_resn
1667 |. nop
1668 |.endmacro
1669 |
1670 |.macro math_extern2, func
1671 | .ffunc_nn math_ .. func
1672 |. load_got func
1673 | call_extern
1674 |. nop
1675 | b ->fff_resn
1676 |. nop
1677 |.endmacro
1678 |
1679 |// TODO: Return integer type if result is integer (own sf implementation).
1680 |.macro math_round, func
1681 |->ff_math_ .. func:
1682 | ld CARG1, 0(BASE)
1683 | beqz NARGS8:RC, ->fff_fallback
1684 |. gettp TMP0, CARG1
1685 | beq TMP0, TISNUM, ->fff_restv
1686 |. sltu AT, TMP0, TISNUM
1687 | beqz AT, ->fff_fallback
1688 |.if FPU
1689 |. ldc1 FARG1, 0(BASE)
1690 | bal ->vm_ .. func
1691 |. nop
1692 |.else
1693 |. load_got func
1694 | call_extern
1695 |. nop
1696 |.endif
1697 | b ->fff_resn
1698 |. nop
1699 |.endmacro
1700 |
1701 | math_round floor
1702 | math_round ceil
1703 |
1704 |.ffunc math_log
1705 | li AT, 8
1706 | bne NARGS8:RC, AT, ->fff_fallback // Exactly 1 argument.
1707 |. ld CARG1, 0(BASE)
1708 | checknum CARG1, ->fff_fallback
1709 |. load_got log
1710 |.if FPU
1711 | call_extern
1712 |. ldc1 FARG1, 0(BASE)
1713 |.else
1714 | call_extern
1715 |. nop
1716 |.endif
1717 | b ->fff_resn
1718 |. nop
1719 |
1720 | math_extern log10
1721 | math_extern exp
1722 | math_extern sin
1723 | math_extern cos
1724 | math_extern tan
1725 | math_extern asin
1726 | math_extern acos
1727 | math_extern atan
1728 | math_extern sinh
1729 | math_extern cosh
1730 | math_extern tanh
1731 | math_extern2 pow
1732 | math_extern2 atan2
1733 | math_extern2 fmod
1734 |
1735 |.if FPU
1736 |.ffunc_n math_sqrt
1737 |. sqrt.d FRET1, FARG1
1738 |// fallthrough to ->fff_resn
1739 |.else
1740 | math_extern sqrt
1741 |.endif
1742 |
1743 |->fff_resn:
1744 | ld PC, FRAME_PC(BASE)
1745 | daddiu RA, BASE, -16
1746 | b ->fff_res1
1747 |.if FPU
1748 |. sdc1 FRET1, 0(RA)
1749 |.else
1750 |. sd CRET1, 0(RA)
1751 |.endif
1752 |
1753 |
1754 |.ffunc_2 math_ldexp
1755 | checknum CARG1, ->fff_fallback
1756 | checkint CARG2, ->fff_fallback
1757 |. load_got ldexp
1758 | .FPU ldc1 FARG1, 0(BASE)
1759 | call_extern
1760 |. lw CARG2, 8+LO(BASE)
1761 | b ->fff_resn
1762 |. nop
1763 |
1764 |.ffunc_n math_frexp
1765 | load_got frexp
1766 | ld PC, FRAME_PC(BASE)
1767 | call_extern
1768 |. daddiu CARG2, DISPATCH, DISPATCH_GL(tmptv)
1769 | lw TMP1, DISPATCH_GL(tmptv)(DISPATCH)
1770 | daddiu RA, BASE, -16
1771 |.if FPU
1772 | mtc1 TMP1, FARG2
1773 | sdc1 FRET1, 0(RA)
1774 | cvt.d.w FARG2, FARG2
1775 | sdc1 FARG2, 8(RA)
1776 |.else
1777 | sd CRET1, 0(RA)
1778 | zextw TMP1, TMP1
1779 | settp TMP1, TISNUM
1780 | sd TMP1, 8(RA)
1781 |.endif
1782 | b ->fff_res
1783 |. li RD, (2+1)*8
1784 |
1785 |.ffunc_n math_modf
1786 | load_got modf
1787 | ld PC, FRAME_PC(BASE)
1788 | call_extern
1789 |. daddiu CARG2, BASE, -16
1790 | daddiu RA, BASE, -16
1791 |.if FPU
1792 | sdc1 FRET1, -8(BASE)
1793 |.else
1794 | sd CRET1, -8(BASE)
1795 |.endif
1796 | b ->fff_res
1797 |. li RD, (2+1)*8
1798 |
1799 |.macro math_minmax, name, intins, intinsc, fpins
1800 | .ffunc_1 name
1801 | daddu TMP3, BASE, NARGS8:RC
1802 | checkint CARG1, >5
1803 |. daddiu TMP2, BASE, 8
1804 |1: // Handle integers.
1805 | beq TMP2, TMP3, ->fff_restv
1806 |. ld CARG2, 0(TMP2)
1807 | checkint CARG2, >3
1808 |. sextw CARG1, CARG1
1809 | lw CARG2, LO(TMP2)
1810 |. slt AT, CARG1, CARG2
1811 |.if MIPSR6
1812 | intins TMP1, CARG2, AT
1813 | intinsc CARG1, CARG1, AT
1814 | or CARG1, CARG1, TMP1
1815 |.else
1816 | intins CARG1, CARG2, AT
1817 |.endif
1818 | daddiu TMP2, TMP2, 8
1819 | zextw CARG1, CARG1
1820 | b <1
1821 |. settp CARG1, TISNUM
1822 |
1823 |3: // Convert intermediate result to number and continue with number loop.
1824 | checknum CARG2, ->fff_fallback
1825 |.if FPU
1826 |. mtc1 CARG1, FRET1
1827 | cvt.d.w FRET1, FRET1
1828 | b >7
1829 |. ldc1 FARG1, 0(TMP2)
1830 |.else
1831 |. nop
1832 | bal ->vm_sfi2d_1
1833 |. nop
1834 | b >7
1835 |. nop
1836 |.endif
1837 |
1838 |5:
1839 | .FPU ldc1 FRET1, 0(BASE)
1840 | checknum CARG1, ->fff_fallback
1841 |6: // Handle numbers.
1842 |. ld CARG2, 0(TMP2)
1843 | beq TMP2, TMP3, ->fff_resn
1844 |.if FPU
1845 | ldc1 FARG1, 0(TMP2)
1846 |.else
1847 | move CRET1, CARG1
1848 |.endif
1849 | checknum CARG2, >8
1850 |. nop
1851 |7:
1852 |.if FPU
1853 |.if MIPSR6
1854 | fpins FRET1, FRET1, FARG1
1855 |.else
1856 |.if fpins // ismax
1857 | c.olt.d FARG1, FRET1
1858 |.else
1859 | c.olt.d FRET1, FARG1
1860 |.endif
1861 | movf.d FRET1, FARG1
1862 |.endif
1863 |.else
1864 |.if fpins // ismax
1865 | bal ->vm_sfcmpogt
1866 |.else
1867 | bal ->vm_sfcmpolt
1868 |.endif
1869 |. nop
1870 |.if MIPSR6
1871 | seleqz AT, CARG2, CRET1
1872 | selnez CARG1, CARG1, CRET1
1873 | or CARG1, CARG1, AT
1874 |.else
1875 | movz CARG1, CARG2, CRET1
1876 |.endif
1877 |.endif
1878 | b <6
1879 |. daddiu TMP2, TMP2, 8
1880 |
1881 |8: // Convert integer to number and continue with number loop.
1882 | checkint CARG2, ->fff_fallback
1883 |.if FPU
1884 |. lwc1 FARG1, LO(TMP2)
1885 | b <7
1886 |. cvt.d.w FARG1, FARG1
1887 |.else
1888 |. lw CARG2, LO(TMP2)
1889 | bal ->vm_sfi2d_2
1890 |. nop
1891 | b <7
1892 |. nop
1893 |.endif
1894 |
1895 |.endmacro
1896 |
1897 |.if MIPSR6
1898 | math_minmax math_min, seleqz, selnez, min.d
1899 | math_minmax math_max, selnez, seleqz, max.d
1900 |.else
1901 | math_minmax math_min, movz, _, 0
1902 | math_minmax math_max, movn, _, 1
1903 |.endif
1904 |
1905 |//-- String library -----------------------------------------------------
1906 |
1907 |.ffunc string_byte // Only handle the 1-arg case here.
1908 | ld CARG1, 0(BASE)
1909 | gettp TMP0, CARG1
1910 | xori AT, NARGS8:RC, 8
1911 | daddiu TMP0, TMP0, -LJ_TSTR
1912 | or AT, AT, TMP0
1913 | bnez AT, ->fff_fallback // Need exactly 1 string argument.
1914 |. cleartp STR:CARG1
1915 | lw TMP0, STR:CARG1->len
1916 | daddiu RA, BASE, -16
1917 | ld PC, FRAME_PC(BASE)
1918 | sltu RD, r0, TMP0
1919 | lbu TMP1, STR:CARG1[1] // Access is always ok (NUL at end).
1920 | addiu RD, RD, 1
1921 | sll RD, RD, 3 // RD = ((str->len != 0)+1)*8
1922 | settp TMP1, TISNUM
1923 | b ->fff_res
1924 |. sd TMP1, 0(RA)
1925 |
1926 |.ffunc string_char // Only handle the 1-arg case here.
1927 | ffgccheck
1928 |.if not MIPSR6
1929 |. nop
1930 |.endif
1931 | ld CARG1, 0(BASE)
1932 | gettp TMP0, CARG1
1933 | xori AT, NARGS8:RC, 8 // Exactly 1 argument.
1934 | daddiu TMP0, TMP0, -LJ_TISNUM // Integer.
1935 | li TMP1, 255
1936 | sextw CARG1, CARG1
1937 | or AT, AT, TMP0
1938 | sltu TMP1, TMP1, CARG1 // !(255 < n).
1939 | or AT, AT, TMP1
1940 | bnez AT, ->fff_fallback
1941 |. li CARG3, 1
1942 | daddiu CARG2, sp, TMPD_OFS
1943 | sb CARG1, TMPD
1944 |->fff_newstr:
1945 | load_got lj_str_new
1946 | sd BASE, L->base
1947 | sd PC, SAVE_PC
1948 | call_intern lj_str_new // (lua_State *L, char *str, size_t l)
1949 |. move CARG1, L
1950 | // Returns GCstr *.
1951 | ld BASE, L->base
1952 |->fff_resstr:
1953 | li AT, LJ_TSTR
1954 | settp CRET1, AT
1955 | b ->fff_restv
1956 |. move CARG1, CRET1
1957 |
1958 |.ffunc string_sub
1959 | ffgccheck
1960 |.if not MIPSR6
1961 |. nop
1962 |.endif
1963 | addiu AT, NARGS8:RC, -16
1964 | ld TMP0, 0(BASE)
1965 | bltz AT, ->fff_fallback
1966 |. gettp TMP3, TMP0
1967 | cleartp STR:CARG1, TMP0
1968 | ld CARG2, 8(BASE)
1969 | beqz AT, >1
1970 |. li CARG4, -1
1971 | ld CARG3, 16(BASE)
1972 | checkint CARG3, ->fff_fallback
1973 |. sextw CARG4, CARG3
1974 |1:
1975 | checkint CARG2, ->fff_fallback
1976 |. li AT, LJ_TSTR
1977 | bne TMP3, AT, ->fff_fallback
1978 |. sextw CARG3, CARG2
1979 | lw CARG2, STR:CARG1->len
1980 | // STR:CARG1 = str, CARG2 = str->len, CARG3 = start, CARG4 = end
1981 | slt AT, CARG4, r0
1982 | addiu TMP0, CARG2, 1
1983 | addu TMP1, CARG4, TMP0
1984 | slt TMP3, CARG3, r0
1985 |.if MIPSR6
1986 | seleqz CARG4, CARG4, AT
1987 | selnez TMP1, TMP1, AT
1988 | or CARG4, TMP1, CARG4 // if (end < 0) end += len+1
1989 |.else
1990 | movn CARG4, TMP1, AT // if (end < 0) end += len+1
1991 |.endif
1992 | addu TMP1, CARG3, TMP0
1993 |.if MIPSR6
1994 | selnez TMP1, TMP1, TMP3
1995 | seleqz CARG3, CARG3, TMP3
1996 | or CARG3, TMP1, CARG3 // if (start < 0) start += len+1
1997 | li TMP2, 1
1998 | slt AT, CARG4, r0
1999 | slt TMP3, r0, CARG3
2000 | seleqz CARG4, CARG4, AT // if (end < 0) end = 0
2001 | selnez CARG3, CARG3, TMP3
2002 | seleqz TMP2, TMP2, TMP3
2003 | or CARG3, TMP2, CARG3 // if (start < 1) start = 1
2004 | slt AT, CARG2, CARG4
2005 | seleqz CARG4, CARG4, AT
2006 | selnez CARG2, CARG2, AT
2007 | or CARG4, CARG2, CARG4 // if (end > len) end = len
2008 |.else
2009 | movn CARG3, TMP1, TMP3 // if (start < 0) start += len+1
2010 | li TMP2, 1
2011 | slt AT, CARG4, r0
2012 | slt TMP3, r0, CARG3
2013 | movn CARG4, r0, AT // if (end < 0) end = 0
2014 | movz CARG3, TMP2, TMP3 // if (start < 1) start = 1
2015 | slt AT, CARG2, CARG4
2016 | movn CARG4, CARG2, AT // if (end > len) end = len
2017 |.endif
2018 | daddu CARG2, STR:CARG1, CARG3
2019 | subu CARG3, CARG4, CARG3 // len = end - start
2020 | daddiu CARG2, CARG2, sizeof(GCstr)-1
2021 | bgez CARG3, ->fff_newstr
2022 |. addiu CARG3, CARG3, 1 // len++
2023 |->fff_emptystr: // Return empty string.
2024 | li AT, LJ_TSTR
2025 | daddiu STR:CARG1, DISPATCH, DISPATCH_GL(strempty)
2026 | b ->fff_restv
2027 |. settp CARG1, AT
2028 |
2029 |.macro ffstring_op, name
2030 | .ffunc string_ .. name
2031 | ffgccheck
2032 |. nop
2033 | beqz NARGS8:RC, ->fff_fallback
2034 |. ld CARG2, 0(BASE)
2035 | checkstr STR:CARG2, ->fff_fallback
2036 | daddiu SBUF:CARG1, DISPATCH, DISPATCH_GL(tmpbuf)
2037 | load_got lj_buf_putstr_ .. name
2038 | ld TMP0, SBUF:CARG1->b
2039 | sd L, SBUF:CARG1->L
2040 | sd BASE, L->base
2041 | sd TMP0, SBUF:CARG1->w
2042 | call_intern extern lj_buf_putstr_ .. name
2043 |. sd PC, SAVE_PC
2044 | load_got lj_buf_tostr
2045 | call_intern lj_buf_tostr
2046 |. move SBUF:CARG1, SBUF:CRET1
2047 | b ->fff_resstr
2048 |. ld BASE, L->base
2049 |.endmacro
2050 |
2051 |ffstring_op reverse
2052 |ffstring_op lower
2053 |ffstring_op upper
2054 |
2055 |//-- Bit library --------------------------------------------------------
2056 |
2057 |->vm_tobit_fb:
2058 | beqz TMP1, ->fff_fallback
2059 |.if FPU
2060 |. ldc1 FARG1, 0(BASE)
2061 | add.d FARG1, FARG1, TOBIT
2062 | mfc1 CRET1, FARG1
2063 | jr ra
2064 |. zextw CRET1, CRET1
2065 |.else
2066 |// FP number to bit conversion for soft-float.
2067 |->vm_tobit:
2068 | dsll TMP0, CARG1, 1
2069 | li CARG3, 1076
2070 | dsrl AT, TMP0, 53
2071 | dsubu CARG3, CARG3, AT
2072 | sltiu AT, CARG3, 54
2073 | beqz AT, >1
2074 |. dextm TMP0, TMP0, 0, 20
2075 | dinsu TMP0, AT, 21, 21
2076 | slt AT, CARG1, r0
2077 | dsrlv CRET1, TMP0, CARG3
2078 | dsubu TMP0, r0, CRET1
2079 |.if MIPSR6
2080 | selnez TMP0, TMP0, AT
2081 | seleqz CRET1, CRET1, AT
2082 | or CRET1, CRET1, TMP0
2083 |.else
2084 | movn CRET1, TMP0, AT
2085 |.endif
2086 | jr ra
2087 |. zextw CRET1, CRET1
2088 |1:
2089 | jr ra
2090 |. move CRET1, r0
2091 |
2092 |// FP number to int conversion with a check for soft-float.
2093 |// Modifies CARG1, CRET1, CRET2, TMP0, AT.
2094 |->vm_tointg:
2095 |.if JIT
2096 | dsll CRET2, CARG1, 1
2097 | beqz CRET2, >2
2098 |. li TMP0, 1076
2099 | dsrl AT, CRET2, 53
2100 | dsubu TMP0, TMP0, AT
2101 | sltiu AT, TMP0, 54
2102 | beqz AT, >1
2103 |. dextm CRET2, CRET2, 0, 20
2104 | dinsu CRET2, AT, 21, 21
2105 | slt AT, CARG1, r0
2106 | dsrlv CRET1, CRET2, TMP0
2107 | dsubu CARG1, r0, CRET1
2108 |.if MIPSR6
2109 | seleqz CRET1, CRET1, AT
2110 | selnez CARG1, CARG1, AT
2111 | or CRET1, CRET1, CARG1
2112 |.else
2113 | movn CRET1, CARG1, AT
2114 |.endif
2115 | li CARG1, 64
2116 | subu TMP0, CARG1, TMP0
2117 | dsllv CRET2, CRET2, TMP0 // Integer check.
2118 | sextw AT, CRET1
2119 | xor AT, CRET1, AT // Range check.
2120 |.if MIPSR6
2121 | seleqz AT, AT, CRET2
2122 | selnez CRET2, CRET2, CRET2
2123 | jr ra
2124 |. or CRET2, AT, CRET2
2125 |.else
2126 | jr ra
2127 |. movz CRET2, AT, CRET2
2128 |.endif
2129 |1:
2130 | jr ra
2131 |. li CRET2, 1
2132 |2:
2133 | jr ra
2134 |. move CRET1, r0
2135 |.endif
2136 |.endif
2137 |
2138 |.macro .ffunc_bit, name
2139 | .ffunc_1 bit_..name
2140 | gettp TMP0, CARG1
2141 | beq TMP0, TISNUM, >6
2142 |. zextw CRET1, CARG1
2143 | bal ->vm_tobit_fb
2144 |. sltiu TMP1, TMP0, LJ_TISNUM
2145 |6:
2146 |.endmacro
2147 |
2148 |.macro .ffunc_bit_op, name, bins
2149 | .ffunc_bit name
2150 | daddiu TMP2, BASE, 8
2151 | daddu TMP3, BASE, NARGS8:RC
2152 |1:
2153 | beq TMP2, TMP3, ->fff_resi
2154 |. ld CARG1, 0(TMP2)
2155 | gettp TMP0, CARG1
2156 |.if FPU
2157 | bne TMP0, TISNUM, >2
2158 |. daddiu TMP2, TMP2, 8
2159 | zextw CARG1, CARG1
2160 | b <1
2161 |. bins CRET1, CRET1, CARG1
2162 |2:
2163 | ldc1 FARG1, -8(TMP2)
2164 | sltiu AT, TMP0, LJ_TISNUM
2165 | beqz AT, ->fff_fallback
2166 |. add.d FARG1, FARG1, TOBIT
2167 | mfc1 CARG1, FARG1
2168 | zextw CARG1, CARG1
2169 | b <1
2170 |. bins CRET1, CRET1, CARG1
2171 |.else
2172 | beq TMP0, TISNUM, >2
2173 |. move CRET2, CRET1
2174 | bal ->vm_tobit_fb
2175 |. sltiu TMP1, TMP0, LJ_TISNUM
2176 | move CARG1, CRET2
2177 |2:
2178 | zextw CARG1, CARG1
2179 | bins CRET1, CRET1, CARG1
2180 | b <1
2181 |. daddiu TMP2, TMP2, 8
2182 |.endif
2183 |.endmacro
2184 |
2185 |.ffunc_bit_op band, and
2186 |.ffunc_bit_op bor, or
2187 |.ffunc_bit_op bxor, xor
2188 |
2189 |.ffunc_bit bswap
2190 | dsrl TMP0, CRET1, 8
2191 | dsrl TMP1, CRET1, 24
2192 | andi TMP2, TMP0, 0xff00
2193 | dins TMP1, CRET1, 24, 31
2194 | dins TMP2, TMP0, 16, 23
2195 | b ->fff_resi
2196 |. or CRET1, TMP1, TMP2
2197 |
2198 |.ffunc_bit bnot
2199 | not CRET1, CRET1
2200 | b ->fff_resi
2201 |. zextw CRET1, CRET1
2202 |
2203 |.macro .ffunc_bit_sh, name, shins, shmod
2204 | .ffunc_2 bit_..name
2205 | gettp TMP0, CARG1
2206 | beq TMP0, TISNUM, >1
2207 |. nop
2208 | bal ->vm_tobit_fb
2209 |. sltiu TMP1, TMP0, LJ_TISNUM
2210 | move CARG1, CRET1
2211 |1:
2212 | gettp TMP0, CARG2
2213 | bne TMP0, TISNUM, ->fff_fallback
2214 |. zextw CARG2, CARG2
2215 | sextw CARG1, CARG1
2216 |.if shmod == 1
2217 | negu CARG2, CARG2
2218 |.endif
2219 | shins CRET1, CARG1, CARG2
2220 | b ->fff_resi
2221 |. zextw CRET1, CRET1
2222 |.endmacro
2223 |
2224 |.ffunc_bit_sh lshift, sllv, 0
2225 |.ffunc_bit_sh rshift, srlv, 0
2226 |.ffunc_bit_sh arshift, srav, 0
2227 |.ffunc_bit_sh rol, rotrv, 1
2228 |.ffunc_bit_sh ror, rotrv, 0
2229 |
2230 |.ffunc_bit tobit
2231 |->fff_resi:
2232 | ld PC, FRAME_PC(BASE)
2233 | daddiu RA, BASE, -16
2234 | settp CRET1, TISNUM
2235 | b ->fff_res1
2236 |. sd CRET1, -16(BASE)
2237 |
2238 |//-----------------------------------------------------------------------
2239 |->fff_fallback: // Call fast function fallback handler.
2240 | // BASE = new base, RB = CFUNC, RC = nargs*8
2241 | ld TMP3, CFUNC:RB->f
2242 | daddu TMP1, BASE, NARGS8:RC
2243 | ld PC, FRAME_PC(BASE) // Fallback may overwrite PC.
2244 | daddiu TMP0, TMP1, 8*LUA_MINSTACK
2245 | ld TMP2, L->maxstack
2246 | sd PC, SAVE_PC // Redundant (but a defined value).
2247 | sltu AT, TMP2, TMP0
2248 | sd BASE, L->base
2249 | sd TMP1, L->top
2250 | bnez AT, >5 // Need to grow stack.
2251 |. move CFUNCADDR, TMP3
2252 | jalr TMP3 // (lua_State *L)
2253 |. move CARG1, L
2254 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
2255 | ld BASE, L->base
2256 | sll RD, CRET1, 3
2257 | bgtz CRET1, ->fff_res // Returned nresults+1?
2258 |. daddiu RA, BASE, -16
2259 |1: // Returned 0 or -1: retry fast path.
2260 | ld LFUNC:RB, FRAME_FUNC(BASE)
2261 | ld TMP0, L->top
2262 | cleartp LFUNC:RB
2263 | bnez CRET1, ->vm_call_tail // Returned -1?
2264 |. dsubu NARGS8:RC, TMP0, BASE
2265 | ins_callt // Returned 0: retry fast path.
2266 |
2267 |// Reconstruct previous base for vmeta_call during tailcall.
2268 |->vm_call_tail:
2269 | andi TMP0, PC, FRAME_TYPE
2270 | li AT, -4
2271 | bnez TMP0, >3
2272 |. and TMP1, PC, AT
2273 | lbu TMP1, OFS_RA(PC)
2274 | sll TMP1, TMP1, 3
2275 | addiu TMP1, TMP1, 16
2276 |3:
2277 | b ->vm_call_dispatch // Resolve again for tailcall.
2278 |. dsubu TMP2, BASE, TMP1
2279 |
2280 |5: // Grow stack for fallback handler.
2281 | load_got lj_state_growstack
2282 | li CARG2, LUA_MINSTACK
2283 | call_intern lj_state_growstack // (lua_State *L, int n)
2284 |. move CARG1, L
2285 | ld BASE, L->base
2286 | b <1
2287 |. li CRET1, 0 // Force retry.
2288 |
2289 |->fff_gcstep: // Call GC step function.
2290 | // BASE = new base, RC = nargs*8
2291 | move MULTRES, ra
2292 | load_got lj_gc_step
2293 | sd BASE, L->base
2294 | daddu TMP0, BASE, NARGS8:RC
2295 | sd PC, SAVE_PC // Redundant (but a defined value).
2296 | sd TMP0, L->top
2297 | call_intern lj_gc_step // (lua_State *L)
2298 |. move CARG1, L
2299 | ld BASE, L->base
2300 | move ra, MULTRES
2301 | ld TMP0, L->top
2302 | ld CFUNC:RB, FRAME_FUNC(BASE)
2303 | cleartp CFUNC:RB
2304 | jr ra
2305 |. dsubu NARGS8:RC, TMP0, BASE
2306 |
2307 |//-----------------------------------------------------------------------
2308 |//-- Special dispatch targets -------------------------------------------
2309 |//-----------------------------------------------------------------------
2310 |
2311 |->vm_record: // Dispatch target for recording phase.
2312 |.if JIT
2313 | lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2314 | andi AT, TMP3, HOOK_VMEVENT // No recording while in vmevent.
2315 | bnez AT, >5
2316 | // Decrement the hookcount for consistency, but always do the call.
2317 |. lw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2318 | andi AT, TMP3, HOOK_ACTIVE
2319 | bnez AT, >1
2320 |. addiu TMP2, TMP2, -1
2321 | andi AT, TMP3, LUA_MASKLINE|LUA_MASKCOUNT
2322 | beqz AT, >1
2323 |. nop
2324 | b >1
2325 |. sw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2326 |.endif
2327 |
2328 |->vm_rethook: // Dispatch target for return hooks.
2329 | lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2330 | andi AT, TMP3, HOOK_ACTIVE // Hook already active?
2331 | beqz AT, >1
2332 |5: // Re-dispatch to static ins.
2333 |. ld AT, GG_DISP2STATIC(TMP0) // Assumes TMP0 holds DISPATCH+OP*4.
2334 | jr AT
2335 |. nop
2336 |
2337 |->vm_inshook: // Dispatch target for instr/line hooks.
2338 | lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2339 | lw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2340 | andi AT, TMP3, HOOK_ACTIVE // Hook already active?
2341 | bnez AT, <5
2342 |. andi AT, TMP3, LUA_MASKLINE|LUA_MASKCOUNT
2343 | beqz AT, <5
2344 |. addiu TMP2, TMP2, -1
2345 | beqz TMP2, >1
2346 |. sw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2347 | andi AT, TMP3, LUA_MASKLINE
2348 | beqz AT, <5
2349 |1:
2350 |. load_got lj_dispatch_ins
2351 | sw MULTRES, SAVE_MULTRES
2352 | move CARG2, PC
2353 | sd BASE, L->base
2354 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
2355 | call_intern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
2356 |. move CARG1, L
2357 |3:
2358 | ld BASE, L->base
2359 |4: // Re-dispatch to static ins.
2360 | lw INS, -4(PC)
2361 | decode_OP8a TMP1, INS
2362 | decode_OP8b TMP1
2363 | daddu TMP0, DISPATCH, TMP1
2364 | decode_RD8a RD, INS
2365 | ld AT, GG_DISP2STATIC(TMP0)
2366 | decode_RA8a RA, INS
2367 | decode_RD8b RD
2368 | jr AT
2369 | decode_RA8b RA
2370 |
2371 |->cont_hook: // Continue from hook yield.
2372 | daddiu PC, PC, 4
2373 | b <4
2374 |. lw MULTRES, -24+LO(RB) // Restore MULTRES for *M ins.
2375 |
2376 |->vm_hotloop: // Hot loop counter underflow.
2377 |.if JIT
2378 | ld LFUNC:TMP1, FRAME_FUNC(BASE)
2379 | daddiu CARG1, DISPATCH, GG_DISP2J
2380 | cleartp LFUNC:TMP1
2381 | sd PC, SAVE_PC
2382 | ld TMP1, LFUNC:TMP1->pc
2383 | move CARG2, PC
2384 | sd L, DISPATCH_J(L)(DISPATCH)
2385 | lbu TMP1, PC2PROTO(framesize)(TMP1)
2386 | load_got lj_trace_hot
2387 | sd BASE, L->base
2388 | dsll TMP1, TMP1, 3
2389 | daddu TMP1, BASE, TMP1
2390 | call_intern lj_trace_hot // (jit_State *J, const BCIns *pc)
2391 |. sd TMP1, L->top
2392 | b <3
2393 |. nop
2394 |.endif
2395 |
2396 |
2397 |->vm_callhook: // Dispatch target for call hooks.
2398 |.if JIT
2399 | b >1
2400 |.endif
2401 |. move CARG2, PC
2402 |
2403 |->vm_hotcall: // Hot call counter underflow.
2404 |.if JIT
2405 | ori CARG2, PC, 1
2406 |1:
2407 |.endif
2408 | load_got lj_dispatch_call
2409 | daddu TMP0, BASE, RC
2410 | sd PC, SAVE_PC
2411 | sd BASE, L->base
2412 | dsubu RA, RA, BASE
2413 | sd TMP0, L->top
2414 | call_intern lj_dispatch_call // (lua_State *L, const BCIns *pc)
2415 |. move CARG1, L
2416 | // Returns ASMFunction.
2417 | ld BASE, L->base
2418 | ld TMP0, L->top
2419 | sd r0, SAVE_PC // Invalidate for subsequent line hook.
2420 | dsubu NARGS8:RC, TMP0, BASE
2421 | daddu RA, BASE, RA
2422 | ld LFUNC:RB, FRAME_FUNC(BASE)
2423 | cleartp LFUNC:RB
2424 | jr CRET1
2425 |. lw INS, -4(PC)
2426 |
2427 |->cont_stitch: // Trace stitching.
2428 |.if JIT
2429 | // RA = resultptr, RB = meta base
2430 | lw INS, -4(PC)
2431 | ld TRACE:TMP2, -40(RB) // Save previous trace.
2432 | decode_RA8a RC, INS
2433 | daddiu AT, MULTRES, -8
2434 | cleartp TRACE:TMP2
2435 | decode_RA8b RC
2436 | beqz AT, >2
2437 |. daddu RC, BASE, RC // Call base.
2438 |1: // Move results down.
2439 | ld CARG1, 0(RA)
2440 | daddiu AT, AT, -8
2441 | daddiu RA, RA, 8
2442 | sd CARG1, 0(RC)
2443 | bnez AT, <1
2444 |. daddiu RC, RC, 8
2445 |2:
2446 | decode_RA8a RA, INS
2447 | decode_RB8a RB, INS
2448 | decode_RA8b RA
2449 | decode_RB8b RB
2450 | daddu RA, RA, RB
2451 | daddu RA, BASE, RA
2452 |3:
2453 | sltu AT, RC, RA
2454 | bnez AT, >9 // More results wanted?
2455 |. nop
2456 |
2457 | lhu TMP3, TRACE:TMP2->traceno
2458 | lhu RD, TRACE:TMP2->link
2459 | beq RD, TMP3, ->cont_nop // Blacklisted.
2460 |. load_got lj_dispatch_stitch
2461 | bnez RD, =>BC_JLOOP // Jump to stitched trace.
2462 |. sll RD, RD, 3
2463 |
2464 | // Stitch a new trace to the previous trace.
2465 | sw TMP3, DISPATCH_J(exitno)(DISPATCH)
2466 | sd L, DISPATCH_J(L)(DISPATCH)
2467 | sd BASE, L->base
2468 | daddiu CARG1, DISPATCH, GG_DISP2J
2469 | call_intern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
2470 |. move CARG2, PC
2471 | b ->cont_nop
2472 |. ld BASE, L->base
2473 |
2474 |9:
2475 | sd TISNIL, 0(RC)
2476 | b <3
2477 |. daddiu RC, RC, 8
2478 |.endif
2479 |
2480 |->vm_profhook: // Dispatch target for profiler hook.
2481#if LJ_HASPROFILE
2482 | load_got lj_dispatch_profile
2483 | sw MULTRES, SAVE_MULTRES
2484 | move CARG2, PC
2485 | sd BASE, L->base
2486 | call_intern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
2487 |. move CARG1, L
2488 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2489 | daddiu PC, PC, -4
2490 | b ->cont_nop
2491 |. ld BASE, L->base
2492#endif
2493 |
2494 |//-----------------------------------------------------------------------
2495 |//-- Trace exit handler -------------------------------------------------
2496 |//-----------------------------------------------------------------------
2497 |
2498 |.macro savex_, a, b
2499 |.if FPU
2500 | sdc1 f..a, a*8(sp)
2501 | sdc1 f..b, b*8(sp)
2502 | sd r..a, 32*8+a*8(sp)
2503 | sd r..b, 32*8+b*8(sp)
2504 |.else
2505 | sd r..a, a*8(sp)
2506 | sd r..b, b*8(sp)
2507 |.endif
2508 |.endmacro
2509 |
2510 |->vm_exit_handler:
2511 |.if JIT
2512 |.if FPU
2513 | daddiu sp, sp, -(32*8+32*8)
2514 |.else
2515 | daddiu sp, sp, -(32*8)
2516 |.endif
2517 | savex_ 0, 1
2518 | savex_ 2, 3
2519 | savex_ 4, 5
2520 | savex_ 6, 7
2521 | savex_ 8, 9
2522 | savex_ 10, 11
2523 | savex_ 12, 13
2524 | savex_ 14, 15
2525 | savex_ 16, 17
2526 | savex_ 18, 19
2527 | savex_ 20, 21
2528 | savex_ 22, 23
2529 | savex_ 24, 25
2530 | savex_ 26, 27
2531 | savex_ 28, 30
2532 |.if FPU
2533 | sdc1 f29, 29*8(sp)
2534 | sdc1 f31, 31*8(sp)
2535 | sd r0, 32*8+31*8(sp) // Clear RID_TMP.
2536 | daddiu TMP2, sp, 32*8+32*8 // Recompute original value of sp.
2537 | sd TMP2, 32*8+29*8(sp) // Store sp in RID_SP
2538 |.else
2539 | sd r0, 31*8(sp) // Clear RID_TMP.
2540 | daddiu TMP2, sp, 32*8 // Recompute original value of sp.
2541 | sd TMP2, 29*8(sp) // Store sp in RID_SP
2542 |.endif
2543 | li_vmstate EXIT
2544 | daddiu DISPATCH, JGL, -GG_DISP2G-32768
2545 | lw TMP1, 0(TMP2) // Load exit number.
2546 | st_vmstate
2547 | ld L, DISPATCH_GL(cur_L)(DISPATCH)
2548 | ld BASE, DISPATCH_GL(jit_base)(DISPATCH)
2549 | load_got lj_trace_exit
2550 | sd L, DISPATCH_J(L)(DISPATCH)
2551 | sw ra, DISPATCH_J(parent)(DISPATCH) // Store trace number.
2552 | sd BASE, L->base
2553 | sw TMP1, DISPATCH_J(exitno)(DISPATCH) // Store exit number.
2554 | daddiu CARG1, DISPATCH, GG_DISP2J
2555 | sd r0, DISPATCH_GL(jit_base)(DISPATCH)
2556 | call_intern lj_trace_exit // (jit_State *J, ExitState *ex)
2557 |. move CARG2, sp
2558 | // Returns MULTRES (unscaled) or negated error code.
2559 | ld TMP1, L->cframe
2560 | li AT, -4
2561 | ld BASE, L->base
2562 | and sp, TMP1, AT
2563 | ld PC, SAVE_PC // Get SAVE_PC.
2564 | b >1
2565 |. sd L, SAVE_L // Set SAVE_L (on-trace resume/yield).
2566 |.endif
2567 |->vm_exit_interp:
2568 |.if JIT
2569 | // CRET1 = MULTRES or negated error code, BASE, PC and JGL set.
2570 | ld L, SAVE_L
2571 | daddiu DISPATCH, JGL, -GG_DISP2G-32768
2572 | sd BASE, L->base
2573 |1:
2574 | sltiu TMP0, CRET1, -LUA_ERRERR // Check for error from exit.
2575 | beqz TMP0, >9
2576 |. ld LFUNC:RB, FRAME_FUNC(BASE)
2577 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
2578 | dsll MULTRES, CRET1, 3
2579 | cleartp LFUNC:RB
2580 | sw MULTRES, SAVE_MULTRES
2581 | li TISNIL, LJ_TNIL
2582 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
2583 | .FPU mtc1 TMP3, TOBIT
2584 | ld TMP1, LFUNC:RB->pc
2585 | sd r0, DISPATCH_GL(jit_base)(DISPATCH)
2586 | ld KBASE, PC2PROTO(k)(TMP1)
2587 | .FPU cvt.d.s TOBIT, TOBIT
2588 | // Modified copy of ins_next which handles function header dispatch, too.
2589 | lw INS, 0(PC)
2590 | addiu CRET1, CRET1, 17 // Static dispatch?
2591 | // Assumes TISNIL == ~LJ_VMST_INTERP == -1
2592 | sw TISNIL, DISPATCH_GL(vmstate)(DISPATCH)
2593 | decode_RD8a RD, INS
2594 | beqz CRET1, >5
2595 |. daddiu PC, PC, 4
2596 | decode_OP8a TMP1, INS
2597 | decode_OP8b TMP1
2598 | daddu TMP0, DISPATCH, TMP1
2599 | sltiu TMP2, TMP1, BC_FUNCF*8
2600 | ld AT, 0(TMP0)
2601 | decode_RA8a RA, INS
2602 | beqz TMP2, >2
2603 |. decode_RA8b RA
2604 | jr AT
2605 |. decode_RD8b RD
2606 |2:
2607 | sltiu TMP2, TMP1, (BC_FUNCC+2)*8 // Fast function?
2608 | bnez TMP2, >3
2609 |. ld TMP1, FRAME_PC(BASE)
2610 | // Check frame below fast function.
2611 | andi TMP0, TMP1, FRAME_TYPE
2612 | bnez TMP0, >3 // Trace stitching continuation?
2613 |. nop
2614 | // Otherwise set KBASE for Lua function below fast function.
2615 | lw TMP2, -4(TMP1)
2616 | decode_RA8a TMP0, TMP2
2617 | decode_RA8b TMP0
2618 | dsubu TMP1, BASE, TMP0
2619 | ld LFUNC:TMP2, -32(TMP1)
2620 | cleartp LFUNC:TMP2
2621 | ld TMP1, LFUNC:TMP2->pc
2622 | ld KBASE, PC2PROTO(k)(TMP1)
2623 |3:
2624 | daddiu RC, MULTRES, -8
2625 | jr AT
2626 |. daddu RA, RA, BASE
2627 |
2628 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2629 | ld TMP0, DISPATCH_J(trace)(DISPATCH)
2630 | decode_RD8b RD
2631 | daddu TMP0, TMP0, RD
2632 | ld TRACE:TMP2, 0(TMP0)
2633 | lw INS, TRACE:TMP2->startins
2634 | decode_OP8a TMP1, INS
2635 | decode_OP8b TMP1
2636 | daddu TMP0, DISPATCH, TMP1
2637 | decode_RD8a RD, INS
2638 | ld AT, GG_DISP2STATIC(TMP0)
2639 | decode_RA8a RA, INS
2640 | decode_RD8b RD
2641 | jr AT
2642 |. decode_RA8b RA
2643 |
2644 |9: // Rethrow error from the right C frame.
2645 | load_got lj_err_trace
2646 | sub CARG2, r0, CRET1
2647 | call_intern lj_err_trace // (lua_State *L, int errcode)
2648 |. move CARG1, L
2649 |.endif
2650 |
2651 |//-----------------------------------------------------------------------
2652 |//-- Math helper functions ----------------------------------------------
2653 |//-----------------------------------------------------------------------
2654 |
2655 |// Hard-float round to integer.
2656 |// Modifies AT, TMP0, FRET1, FRET2, f4. Keeps all others incl. FARG1.
2657 |// MIPSR6: Modifies FTMP1, too.
2658 |.macro vm_round_hf, func
2659 | lui TMP0, 0x4330 // Hiword of 2^52 (double).
2660 | dsll TMP0, TMP0, 32
2661 | dmtc1 TMP0, f4
2662 | abs.d FRET2, FARG1 // |x|
2663 | dmfc1 AT, FARG1
2664 |.if MIPSR6
2665 | cmp.lt.d FTMP1, FRET2, f4
2666 | add.d FRET1, FRET2, f4 // (|x| + 2^52) - 2^52
2667 | bc1eqz FTMP1, >1 // Truncate only if |x| < 2^52.
2668 |.else
2669 | c.olt.d 0, FRET2, f4
2670 | add.d FRET1, FRET2, f4 // (|x| + 2^52) - 2^52
2671 | bc1f 0, >1 // Truncate only if |x| < 2^52.
2672 |.endif
2673 |. sub.d FRET1, FRET1, f4
2674 | slt AT, AT, r0
2675 |.if "func" == "ceil"
2676 | lui TMP0, 0xbff0 // Hiword of -1 (double). Preserves -0.
2677 |.else
2678 | lui TMP0, 0x3ff0 // Hiword of +1 (double).
2679 |.endif
2680 |.if "func" == "trunc"
2681 | dsll TMP0, TMP0, 32
2682 | dmtc1 TMP0, f4
2683 |.if MIPSR6
2684 | cmp.lt.d FTMP1, FRET2, FRET1 // |x| < result?
2685 | sub.d FRET2, FRET1, f4
2686 | sel.d FTMP1, FRET1, FRET2 // If yes, subtract +1.
2687 | dmtc1 AT, FRET1
2688 | neg.d FRET2, FTMP1
2689 | jr ra
2690 |. sel.d FRET1, FTMP1, FRET2 // Merge sign bit back in.
2691 |.else
2692 | c.olt.d 0, FRET2, FRET1 // |x| < result?
2693 | sub.d FRET2, FRET1, f4
2694 | movt.d FRET1, FRET2, 0 // If yes, subtract +1.
2695 | neg.d FRET2, FRET1
2696 | jr ra
2697 |. movn.d FRET1, FRET2, AT // Merge sign bit back in.
2698 |.endif
2699 |.else
2700 | neg.d FRET2, FRET1
2701 | dsll TMP0, TMP0, 32
2702 | dmtc1 TMP0, f4
2703 |.if MIPSR6
2704 | dmtc1 AT, FTMP1
2705 | sel.d FTMP1, FRET1, FRET2
2706 |.if "func" == "ceil"
2707 | cmp.lt.d FRET1, FTMP1, FARG1 // x > result?
2708 |.else
2709 | cmp.lt.d FRET1, FARG1, FTMP1 // x < result?
2710 |.endif
2711 | sub.d FRET2, FTMP1, f4 // If yes, subtract +-1.
2712 | jr ra
2713 |. sel.d FRET1, FTMP1, FRET2
2714 |.else
2715 | movn.d FRET1, FRET2, AT // Merge sign bit back in.
2716 |.if "func" == "ceil"
2717 | c.olt.d 0, FRET1, FARG1 // x > result?
2718 |.else
2719 | c.olt.d 0, FARG1, FRET1 // x < result?
2720 |.endif
2721 | sub.d FRET2, FRET1, f4 // If yes, subtract +-1.
2722 | jr ra
2723 |. movt.d FRET1, FRET2, 0
2724 |.endif
2725 |.endif
2726 |1:
2727 | jr ra
2728 |. mov.d FRET1, FARG1
2729 |.endmacro
2730 |
2731 |.macro vm_round, func
2732 |.if FPU
2733 | vm_round_hf, func
2734 |.endif
2735 |.endmacro
2736 |
2737 |->vm_floor:
2738 | vm_round floor
2739 |->vm_ceil:
2740 | vm_round ceil
2741 |->vm_trunc:
2742 |.if JIT
2743 | vm_round trunc
2744 |.endif
2745 |
2746 |// Soft-float integer to number conversion.
2747 |.macro sfi2d, ARG
2748 |.if not FPU
2749 | beqz ARG, >9 // Handle zero first.
2750 |. sra TMP0, ARG, 31
2751 | xor TMP1, ARG, TMP0
2752 | dsubu TMP1, TMP1, TMP0 // Absolute value in TMP1.
2753 | dclz ARG, TMP1
2754 | addiu ARG, ARG, -11
2755 | li AT, 0x3ff+63-11-1
2756 | dsllv TMP1, TMP1, ARG // Align mantissa left with leading 1.
2757 | subu ARG, AT, ARG // Exponent - 1.
2758 | ins ARG, TMP0, 11, 11 // Sign | Exponent.
2759 | dsll ARG, ARG, 52 // Align left.
2760 | jr ra
2761 |. daddu ARG, ARG, TMP1 // Add mantissa, increment exponent.
2762 |9:
2763 | jr ra
2764 |. nop
2765 |.endif
2766 |.endmacro
2767 |
2768 |// Input CARG1. Output: CARG1. Temporaries: AT, TMP0, TMP1.
2769 |->vm_sfi2d_1:
2770 | sfi2d CARG1
2771 |
2772 |// Input CARG2. Output: CARG2. Temporaries: AT, TMP0, TMP1.
2773 |->vm_sfi2d_2:
2774 | sfi2d CARG2
2775 |
2776 |// Soft-float comparison. Equivalent to c.eq.d.
2777 |// Input: CARG*. Output: CRET1. Temporaries: AT, TMP0, TMP1.
2778 |->vm_sfcmpeq:
2779 |.if not FPU
2780 | dsll AT, CARG1, 1
2781 | dsll TMP0, CARG2, 1
2782 | or TMP1, AT, TMP0
2783 | beqz TMP1, >8 // Both args +-0: return 1.
2784 |. lui TMP1, 0xffe0
2785 | dsll TMP1, TMP1, 32
2786 | sltu AT, TMP1, AT
2787 | sltu TMP0, TMP1, TMP0
2788 | or TMP1, AT, TMP0
2789 | bnez TMP1, >9 // Either arg is NaN: return 0;
2790 |. xor AT, CARG1, CARG2
2791 | jr ra
2792 |. sltiu CRET1, AT, 1 // Same values: return 1.
2793 |8:
2794 | jr ra
2795 |. li CRET1, 1
2796 |9:
2797 | jr ra
2798 |. li CRET1, 0
2799 |.endif
2800 |
2801 |// Soft-float comparison. Equivalent to c.ult.d and c.olt.d.
2802 |// Input: CARG1, CARG2. Output: CRET1. Temporaries: AT, TMP0, TMP1, CRET2.
2803 |->vm_sfcmpult:
2804 |.if not FPU
2805 | b >1
2806 |. li CRET2, 1
2807 |.endif
2808 |
2809 |->vm_sfcmpolt:
2810 |.if not FPU
2811 | li CRET2, 0
2812 |1:
2813 | dsll AT, CARG1, 1
2814 | dsll TMP0, CARG2, 1
2815 | or TMP1, AT, TMP0
2816 | beqz TMP1, >8 // Both args +-0: return 0.
2817 |. lui TMP1, 0xffe0
2818 | dsll TMP1, TMP1, 32
2819 | sltu AT, TMP1, AT
2820 | sltu TMP0, TMP1, TMP0
2821 | or TMP1, AT, TMP0
2822 | bnez TMP1, >9 // Either arg is NaN: return 0 or 1;
2823 |. and AT, CARG1, CARG2
2824 | bltz AT, >5 // Both args negative?
2825 |. nop
2826 | jr ra
2827 |. slt CRET1, CARG1, CARG2
2828 |5: // Swap conditions if both operands are negative.
2829 | jr ra
2830 |. slt CRET1, CARG2, CARG1
2831 |8:
2832 | jr ra
2833 |. li CRET1, 0
2834 |9:
2835 | jr ra
2836 |. move CRET1, CRET2
2837 |.endif
2838 |
2839 |->vm_sfcmpogt:
2840 |.if not FPU
2841 | dsll AT, CARG2, 1
2842 | dsll TMP0, CARG1, 1
2843 | or TMP1, AT, TMP0
2844 | beqz TMP1, >8 // Both args +-0: return 0.
2845 |. lui TMP1, 0xffe0
2846 | dsll TMP1, TMP1, 32
2847 | sltu AT, TMP1, AT
2848 | sltu TMP0, TMP1, TMP0
2849 | or TMP1, AT, TMP0
2850 | bnez TMP1, >9 // Either arg is NaN: return 0 or 1;
2851 |. and AT, CARG2, CARG1
2852 | bltz AT, >5 // Both args negative?
2853 |. nop
2854 | jr ra
2855 |. slt CRET1, CARG2, CARG1
2856 |5: // Swap conditions if both operands are negative.
2857 | jr ra
2858 |. slt CRET1, CARG1, CARG2
2859 |8:
2860 | jr ra
2861 |. li CRET1, 0
2862 |9:
2863 | jr ra
2864 |. li CRET1, 0
2865 |.endif
2866 |
2867 |// Soft-float comparison. Equivalent to c.ole.d a, b or c.ole.d b, a.
2868 |// Input: CARG1, CARG2, TMP3. Output: CRET1. Temporaries: AT, TMP0, TMP1.
2869 |->vm_sfcmpolex:
2870 |.if not FPU
2871 | dsll AT, CARG1, 1
2872 | dsll TMP0, CARG2, 1
2873 | or TMP1, AT, TMP0
2874 | beqz TMP1, >8 // Both args +-0: return 1.
2875 |. lui TMP1, 0xffe0
2876 | dsll TMP1, TMP1, 32
2877 | sltu AT, TMP1, AT
2878 | sltu TMP0, TMP1, TMP0
2879 | or TMP1, AT, TMP0
2880 | bnez TMP1, >9 // Either arg is NaN: return 0;
2881 |. and AT, CARG1, CARG2
2882 | xor AT, AT, TMP3
2883 | bltz AT, >5 // Both args negative?
2884 |. nop
2885 | jr ra
2886 |. slt CRET1, CARG2, CARG1
2887 |5: // Swap conditions if both operands are negative.
2888 | jr ra
2889 |. slt CRET1, CARG1, CARG2
2890 |8:
2891 | jr ra
2892 |. li CRET1, 1
2893 |9:
2894 | jr ra
2895 |. li CRET1, 0
2896 |.endif
2897 |
2898 |.macro sfmin_max, name, fpcall
2899 |->vm_sf .. name:
2900 |.if JIT and not FPU
2901 | move TMP2, ra
2902 | bal ->fpcall
2903 |. nop
2904 | move ra, TMP2
2905 | move TMP0, CRET1
2906 | move CRET1, CARG1
2907 |.if MIPSR6
2908 | selnez CRET1, CRET1, TMP0
2909 | seleqz TMP0, CARG2, TMP0
2910 | jr ra
2911 |. or CRET1, CRET1, TMP0
2912 |.else
2913 | jr ra
2914 |. movz CRET1, CARG2, TMP0
2915 |.endif
2916 |.endif
2917 |.endmacro
2918 |
2919 | sfmin_max min, vm_sfcmpolt
2920 | sfmin_max max, vm_sfcmpogt
2921 |
2922 |//-----------------------------------------------------------------------
2923 |//-- Miscellaneous functions --------------------------------------------
2924 |//-----------------------------------------------------------------------
2925 |
2926 |.define NEXT_TAB, TAB:CARG1
2927 |.define NEXT_IDX, CARG2
2928 |.define NEXT_ASIZE, CARG3
2929 |.define NEXT_NIL, CARG4
2930 |.define NEXT_TMP0, r12
2931 |.define NEXT_TMP1, r13
2932 |.define NEXT_TMP2, r14
2933 |.define NEXT_RES_VK, CRET1
2934 |.define NEXT_RES_IDX, CRET2
2935 |.define NEXT_RES_PTR, sp
2936 |.define NEXT_RES_VAL, 0(sp)
2937 |.define NEXT_RES_KEY, 8(sp)
2938 |
2939 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2940 |// Next idx returned in CRET2.
2941 |->vm_next:
2942 |.if JIT and ENDIAN_LE
2943 | lw NEXT_ASIZE, NEXT_TAB->asize
2944 | ld NEXT_TMP0, NEXT_TAB->array
2945 | li NEXT_NIL, LJ_TNIL
2946 |1: // Traverse array part.
2947 | sltu AT, NEXT_IDX, NEXT_ASIZE
2948 | sll NEXT_TMP1, NEXT_IDX, 3
2949 | beqz AT, >5
2950 |. daddu NEXT_TMP1, NEXT_TMP0, NEXT_TMP1
2951 | li AT, LJ_TISNUM
2952 | ld NEXT_TMP2, 0(NEXT_TMP1)
2953 | dsll AT, AT, 47
2954 | or NEXT_TMP1, NEXT_IDX, AT
2955 | beq NEXT_TMP2, NEXT_NIL, <1
2956 |. addiu NEXT_IDX, NEXT_IDX, 1
2957 | sd NEXT_TMP2, NEXT_RES_VAL
2958 | sd NEXT_TMP1, NEXT_RES_KEY
2959 | move NEXT_RES_VK, NEXT_RES_PTR
2960 | jr ra
2961 |. move NEXT_RES_IDX, NEXT_IDX
2962 |
2963 |5: // Traverse hash part.
2964 | subu NEXT_RES_IDX, NEXT_IDX, NEXT_ASIZE
2965 | ld NODE:NEXT_RES_VK, NEXT_TAB->node
2966 | sll NEXT_TMP2, NEXT_RES_IDX, 5
2967 | lw NEXT_TMP0, NEXT_TAB->hmask
2968 | sll AT, NEXT_RES_IDX, 3
2969 | subu AT, NEXT_TMP2, AT
2970 | daddu NODE:NEXT_RES_VK, NODE:NEXT_RES_VK, AT
2971 |6:
2972 | sltu AT, NEXT_TMP0, NEXT_RES_IDX
2973 | bnez AT, >8
2974 |. nop
2975 | ld NEXT_TMP2, NODE:NEXT_RES_VK->val
2976 | bne NEXT_TMP2, NEXT_NIL, >9
2977 |. addiu NEXT_RES_IDX, NEXT_RES_IDX, 1
2978 | // Skip holes in hash part.
2979 | b <6
2980 |. daddiu NODE:NEXT_RES_VK, NODE:NEXT_RES_VK, sizeof(Node)
2981 |
2982 |8: // End of iteration. Set the key to nil (not the value).
2983 | sd NEXT_NIL, NEXT_RES_KEY
2984 | move NEXT_RES_VK, NEXT_RES_PTR
2985 |9:
2986 | jr ra
2987 |. addu NEXT_RES_IDX, NEXT_RES_IDX, NEXT_ASIZE
2988 |.endif
2989 |
2990 |//-----------------------------------------------------------------------
2991 |//-- FFI helper functions -----------------------------------------------
2992 |//-----------------------------------------------------------------------
2993 |
2994 |// Handler for callback functions. Callback slot number in r1, g in r2.
2995 |->vm_ffi_callback:
2996 |.if FFI
2997 |.type CTSTATE, CTState, PC
2998 | saveregs
2999 | ld CTSTATE, GL:r2->ctype_state
3000 | daddiu DISPATCH, r2, GG_G2DISP
3001 | load_got lj_ccallback_enter
3002 | sw r1, CTSTATE->cb.slot
3003 | sd CARG1, CTSTATE->cb.gpr[0]
3004 | .FPU sdc1 FARG1, CTSTATE->cb.fpr[0]
3005 | sd CARG2, CTSTATE->cb.gpr[1]
3006 | .FPU sdc1 FARG2, CTSTATE->cb.fpr[1]
3007 | sd CARG3, CTSTATE->cb.gpr[2]
3008 | .FPU sdc1 FARG3, CTSTATE->cb.fpr[2]
3009 | sd CARG4, CTSTATE->cb.gpr[3]
3010 | .FPU sdc1 FARG4, CTSTATE->cb.fpr[3]
3011 | sd CARG5, CTSTATE->cb.gpr[4]
3012 | .FPU sdc1 FARG5, CTSTATE->cb.fpr[4]
3013 | sd CARG6, CTSTATE->cb.gpr[5]
3014 | .FPU sdc1 FARG6, CTSTATE->cb.fpr[5]
3015 | sd CARG7, CTSTATE->cb.gpr[6]
3016 | .FPU sdc1 FARG7, CTSTATE->cb.fpr[6]
3017 | sd CARG8, CTSTATE->cb.gpr[7]
3018 | .FPU sdc1 FARG8, CTSTATE->cb.fpr[7]
3019 | daddiu TMP0, sp, CFRAME_SPACE
3020 | sd TMP0, CTSTATE->cb.stack
3021 | sd r0, SAVE_PC // Any value outside of bytecode is ok.
3022 | move CARG2, sp
3023 | call_intern lj_ccallback_enter // (CTState *cts, void *cf)
3024 |. move CARG1, CTSTATE
3025 | // Returns lua_State *.
3026 | ld BASE, L:CRET1->base
3027 | ld RC, L:CRET1->top
3028 | move L, CRET1
3029 | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
3030 | ld LFUNC:RB, FRAME_FUNC(BASE)
3031 | .FPU mtc1 TMP3, TOBIT
3032 | li TISNIL, LJ_TNIL
3033 | li TISNUM, LJ_TISNUM
3034 | li_vmstate INTERP
3035 | subu RC, RC, BASE
3036 | cleartp LFUNC:RB
3037 | st_vmstate
3038 | .FPU cvt.d.s TOBIT, TOBIT
3039 | ins_callt
3040 |.endif
3041 |
3042 |->cont_ffi_callback: // Return from FFI callback.
3043 |.if FFI
3044 | load_got lj_ccallback_leave
3045 | ld CTSTATE, DISPATCH_GL(ctype_state)(DISPATCH)
3046 | sd BASE, L->base
3047 | sd RB, L->top
3048 | sd L, CTSTATE->L
3049 | move CARG2, RA
3050 | call_intern lj_ccallback_leave // (CTState *cts, TValue *o)
3051 |. move CARG1, CTSTATE
3052 | .FPU ldc1 FRET1, CTSTATE->cb.fpr[0]
3053 | ld CRET1, CTSTATE->cb.gpr[0]
3054 | .FPU ldc1 FRET2, CTSTATE->cb.fpr[1]
3055 | b ->vm_leave_unw
3056 |. ld CRET2, CTSTATE->cb.gpr[1]
3057 |.endif
3058 |
3059 |->vm_ffi_call: // Call C function via FFI.
3060 | // Caveat: needs special frame unwinding, see below.
3061 |.if FFI
3062 | .type CCSTATE, CCallState, CARG1
3063 | lw TMP1, CCSTATE->spadj
3064 | lbu CARG2, CCSTATE->nsp
3065 | move TMP2, sp
3066 | dsubu sp, sp, TMP1
3067 | sd ra, -8(TMP2)
3068 | sll CARG2, CARG2, 3
3069 | sd r16, -16(TMP2)
3070 | sd CCSTATE, -24(TMP2)
3071 | move r16, TMP2
3072 | daddiu TMP1, CCSTATE, offsetof(CCallState, stack)
3073 | move TMP2, sp
3074 | beqz CARG2, >2
3075 |. daddu TMP3, TMP1, CARG2
3076 |1:
3077 | ld TMP0, 0(TMP1)
3078 | daddiu TMP1, TMP1, 8
3079 | sltu AT, TMP1, TMP3
3080 | sd TMP0, 0(TMP2)
3081 | bnez AT, <1
3082 |. daddiu TMP2, TMP2, 8
3083 |2:
3084 | ld CFUNCADDR, CCSTATE->func
3085 | .FPU ldc1 FARG1, CCSTATE->gpr[0]
3086 | ld CARG2, CCSTATE->gpr[1]
3087 | .FPU ldc1 FARG2, CCSTATE->gpr[1]
3088 | ld CARG3, CCSTATE->gpr[2]
3089 | .FPU ldc1 FARG3, CCSTATE->gpr[2]
3090 | ld CARG4, CCSTATE->gpr[3]
3091 | .FPU ldc1 FARG4, CCSTATE->gpr[3]
3092 | ld CARG5, CCSTATE->gpr[4]
3093 | .FPU ldc1 FARG5, CCSTATE->gpr[4]
3094 | ld CARG6, CCSTATE->gpr[5]
3095 | .FPU ldc1 FARG6, CCSTATE->gpr[5]
3096 | ld CARG7, CCSTATE->gpr[6]
3097 | .FPU ldc1 FARG7, CCSTATE->gpr[6]
3098 | ld CARG8, CCSTATE->gpr[7]
3099 | .FPU ldc1 FARG8, CCSTATE->gpr[7]
3100 | jalr CFUNCADDR
3101 |. ld CARG1, CCSTATE->gpr[0] // Do this last, since CCSTATE is CARG1.
3102 | ld CCSTATE:TMP1, -24(r16)
3103 | ld TMP2, -16(r16)
3104 | ld ra, -8(r16)
3105 | sd CRET1, CCSTATE:TMP1->gpr[0]
3106 | sd CRET2, CCSTATE:TMP1->gpr[1]
3107 |.if FPU
3108 | sdc1 FRET1, CCSTATE:TMP1->fpr[0]
3109 | sdc1 FRET2, CCSTATE:TMP1->fpr[1]
3110 |.else
3111 | sd CARG1, CCSTATE:TMP1->gpr[2] // 2nd FP struct field for soft-float.
3112 |.endif
3113 | move sp, r16
3114 | jr ra
3115 |. move r16, TMP2
3116 |.endif
3117 |// Note: vm_ffi_call must be the last function in this object file!
3118 |
3119 |//-----------------------------------------------------------------------
3120}
3121
3122/* Generate the code for a single instruction. */
3123static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3124{
3125 int vk = 0;
3126 |=>defop:
3127
3128 switch (op) {
3129
3130 /* -- Comparison ops ---------------------------------------------------- */
3131
3132 /* Remember: all ops branch for a true comparison, fall through otherwise. */
3133
3134 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
3135 | // RA = src1*8, RD = src2*8, JMP with RD = target
3136 |.macro bc_comp, FRA, FRD, ARGRA, ARGRD, movop, fmovop, fcomp, sfcomp
3137 | daddu RA, BASE, RA
3138 | daddu RD, BASE, RD
3139 | ld ARGRA, 0(RA)
3140 | ld ARGRD, 0(RD)
3141 | lhu TMP2, OFS_RD(PC)
3142 | gettp CARG3, ARGRA
3143 | gettp CARG4, ARGRD
3144 | bne CARG3, TISNUM, >2
3145 |. daddiu PC, PC, 4
3146 | bne CARG4, TISNUM, >5
3147 |. decode_RD4b TMP2
3148 | sextw ARGRA, ARGRA
3149 | sextw ARGRD, ARGRD
3150 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3151 | slt AT, CARG1, CARG2
3152 | addu TMP2, TMP2, TMP3
3153 |.if MIPSR6
3154 | movop TMP2, TMP2, AT
3155 |.else
3156 | movop TMP2, r0, AT
3157 |.endif
3158 |1:
3159 | daddu PC, PC, TMP2
3160 | ins_next
3161 |
3162 |2: // RA is not an integer.
3163 | sltiu AT, CARG3, LJ_TISNUM
3164 | beqz AT, ->vmeta_comp
3165 |. lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3166 | sltiu AT, CARG4, LJ_TISNUM
3167 | beqz AT, >4
3168 |. decode_RD4b TMP2
3169 |.if FPU
3170 | ldc1 FRA, 0(RA)
3171 | ldc1 FRD, 0(RD)
3172 |.endif
3173 |3: // RA and RD are both numbers.
3174 |.if FPU
3175 |.if MIPSR6
3176 | fcomp FTMP0, FTMP0, FTMP2
3177 | addu TMP2, TMP2, TMP3
3178 | mfc1 TMP3, FTMP0
3179 | b <1
3180 |. fmovop TMP2, TMP2, TMP3
3181 |.else
3182 | fcomp FTMP0, FTMP2
3183 | addu TMP2, TMP2, TMP3
3184 | b <1
3185 |. fmovop TMP2, r0
3186 |.endif
3187 |.else
3188 | bal sfcomp
3189 |. addu TMP2, TMP2, TMP3
3190 | b <1
3191 |.if MIPSR6
3192 |. movop TMP2, TMP2, CRET1
3193 |.else
3194 |. movop TMP2, r0, CRET1
3195 |.endif
3196 |.endif
3197 |
3198 |4: // RA is a number, RD is not a number.
3199 | bne CARG4, TISNUM, ->vmeta_comp
3200 | // RA is a number, RD is an integer. Convert RD to a number.
3201 |.if FPU
3202 |. lwc1 FRD, LO(RD)
3203 | ldc1 FRA, 0(RA)
3204 | b <3
3205 |. cvt.d.w FRD, FRD
3206 |.else
3207 |.if "ARGRD" == "CARG1"
3208 |. sextw CARG1, CARG1
3209 | bal ->vm_sfi2d_1
3210 |. nop
3211 |.else
3212 |. sextw CARG2, CARG2
3213 | bal ->vm_sfi2d_2
3214 |. nop
3215 |.endif
3216 | b <3
3217 |. nop
3218 |.endif
3219 |
3220 |5: // RA is an integer, RD is not an integer
3221 | sltiu AT, CARG4, LJ_TISNUM
3222 | beqz AT, ->vmeta_comp
3223 |. lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3224 | // RA is an integer, RD is a number. Convert RA to a number.
3225 |.if FPU
3226 | lwc1 FRA, LO(RA)
3227 | ldc1 FRD, 0(RD)
3228 | b <3
3229 | cvt.d.w FRA, FRA
3230 |.else
3231 |.if "ARGRA" == "CARG1"
3232 | bal ->vm_sfi2d_1
3233 |. sextw CARG1, CARG1
3234 |.else
3235 | bal ->vm_sfi2d_2
3236 |. sextw CARG2, CARG2
3237 |.endif
3238 | b <3
3239 |. nop
3240 |.endif
3241 |.endmacro
3242 |
3243 |.if MIPSR6
3244 if (op == BC_ISLT) {
3245 | bc_comp FTMP0, FTMP2, CARG1, CARG2, selnez, selnez, cmp.lt.d, ->vm_sfcmpolt
3246 } else if (op == BC_ISGE) {
3247 | bc_comp FTMP0, FTMP2, CARG1, CARG2, seleqz, seleqz, cmp.lt.d, ->vm_sfcmpolt
3248 } else if (op == BC_ISLE) {
3249 | bc_comp FTMP2, FTMP0, CARG2, CARG1, seleqz, seleqz, cmp.ult.d, ->vm_sfcmpult
3250 } else {
3251 | bc_comp FTMP2, FTMP0, CARG2, CARG1, selnez, selnez, cmp.ult.d, ->vm_sfcmpult
3252 }
3253 |.else
3254 if (op == BC_ISLT) {
3255 | bc_comp FTMP0, FTMP2, CARG1, CARG2, movz, movf, c.olt.d, ->vm_sfcmpolt
3256 } else if (op == BC_ISGE) {
3257 | bc_comp FTMP0, FTMP2, CARG1, CARG2, movn, movt, c.olt.d, ->vm_sfcmpolt
3258 } else if (op == BC_ISLE) {
3259 | bc_comp FTMP2, FTMP0, CARG2, CARG1, movn, movt, c.ult.d, ->vm_sfcmpult
3260 } else {
3261 | bc_comp FTMP2, FTMP0, CARG2, CARG1, movz, movf, c.ult.d, ->vm_sfcmpult
3262 }
3263 |.endif
3264 break;
3265
3266 case BC_ISEQV: case BC_ISNEV:
3267 vk = op == BC_ISEQV;
3268 | // RA = src1*8, RD = src2*8, JMP with RD = target
3269 | daddu RA, BASE, RA
3270 | daddiu PC, PC, 4
3271 | daddu RD, BASE, RD
3272 | ld CARG1, 0(RA)
3273 | lhu TMP2, -4+OFS_RD(PC)
3274 | ld CARG2, 0(RD)
3275 | gettp CARG3, CARG1
3276 | gettp CARG4, CARG2
3277 | sltu AT, TISNUM, CARG3
3278 | sltu TMP1, TISNUM, CARG4
3279 | or AT, AT, TMP1
3280 if (vk) {
3281 | beqz AT, ->BC_ISEQN_Z
3282 } else {
3283 | beqz AT, ->BC_ISNEN_Z
3284 }
3285 | // Either or both types are not numbers.
3286 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3287 |.if FFI
3288 |. li AT, LJ_TCDATA
3289 | beq CARG3, AT, ->vmeta_equal_cd
3290 |.endif
3291 | decode_RD4b TMP2
3292 |.if FFI
3293 | beq CARG4, AT, ->vmeta_equal_cd
3294 |. nop
3295 |.endif
3296 | bne CARG1, CARG2, >2
3297 |. addu TMP2, TMP2, TMP3
3298 | // Tag and value are equal.
3299 if (vk) {
3300 |->BC_ISEQV_Z:
3301 | daddu PC, PC, TMP2
3302 }
3303 |1:
3304 | ins_next
3305 |
3306 |2: // Check if the tags are the same and it's a table or userdata.
3307 | xor AT, CARG3, CARG4 // Same type?
3308 | sltiu TMP0, CARG3, LJ_TISTABUD+1 // Table or userdata?
3309 |.if MIPSR6
3310 | seleqz TMP0, TMP0, AT
3311 |.else
3312 | movn TMP0, r0, AT
3313 |.endif
3314 if (vk) {
3315 | beqz TMP0, <1
3316 } else {
3317 | beqz TMP0, ->BC_ISEQV_Z // Reuse code from opposite instruction.
3318 }
3319 | // Different tables or userdatas. Need to check __eq metamethod.
3320 | // Field metatable must be at same offset for GCtab and GCudata!
3321 |. cleartp TAB:TMP1, CARG1
3322 | ld TAB:TMP3, TAB:TMP1->metatable
3323 if (vk) {
3324 | beqz TAB:TMP3, <1 // No metatable?
3325 |. nop
3326 | lbu TMP3, TAB:TMP3->nomm
3327 | andi TMP3, TMP3, 1<<MM_eq
3328 | bnez TMP3, >1 // Or 'no __eq' flag set?
3329 } else {
3330 | beqz TAB:TMP3,->BC_ISEQV_Z // No metatable?
3331 |. nop
3332 | lbu TMP3, TAB:TMP3->nomm
3333 | andi TMP3, TMP3, 1<<MM_eq
3334 | bnez TMP3, ->BC_ISEQV_Z // Or 'no __eq' flag set?
3335 }
3336 |. nop
3337 | b ->vmeta_equal // Handle __eq metamethod.
3338 |. li TMP0, 1-vk // ne = 0 or 1.
3339 break;
3340
3341 case BC_ISEQS: case BC_ISNES:
3342 vk = op == BC_ISEQS;
3343 | // RA = src*8, RD = str_const*8 (~), JMP with RD = target
3344 | daddu RA, BASE, RA
3345 | daddiu PC, PC, 4
3346 | ld CARG1, 0(RA)
3347 | dsubu RD, KBASE, RD
3348 | lhu TMP2, -4+OFS_RD(PC)
3349 | ld CARG2, -8(RD) // KBASE-8-str_const*8
3350 |.if FFI
3351 | gettp TMP0, CARG1
3352 | li AT, LJ_TCDATA
3353 |.endif
3354 | li TMP1, LJ_TSTR
3355 | decode_RD4b TMP2
3356 |.if FFI
3357 | beq TMP0, AT, ->vmeta_equal_cd
3358 |.endif
3359 |. settp CARG2, TMP1
3360 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3361 | xor TMP1, CARG1, CARG2
3362 | addu TMP2, TMP2, TMP3
3363 |.if MIPSR6
3364 if (vk) {
3365 | seleqz TMP2, TMP2, TMP1
3366 } else {
3367 | selnez TMP2, TMP2, TMP1
3368 }
3369 |.else
3370 if (vk) {
3371 | movn TMP2, r0, TMP1
3372 } else {
3373 | movz TMP2, r0, TMP1
3374 }
3375 |.endif
3376 | daddu PC, PC, TMP2
3377 | ins_next
3378 break;
3379
3380 case BC_ISEQN: case BC_ISNEN:
3381 vk = op == BC_ISEQN;
3382 | // RA = src*8, RD = num_const*8, JMP with RD = target
3383 | daddu RA, BASE, RA
3384 | daddu RD, KBASE, RD
3385 | ld CARG1, 0(RA)
3386 | ld CARG2, 0(RD)
3387 | lhu TMP2, OFS_RD(PC)
3388 | gettp CARG3, CARG1
3389 | gettp CARG4, CARG2
3390 | daddiu PC, PC, 4
3391 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3392 if (vk) {
3393 |->BC_ISEQN_Z:
3394 } else {
3395 |->BC_ISNEN_Z:
3396 }
3397 | bne CARG3, TISNUM, >3
3398 |. decode_RD4b TMP2
3399 | bne CARG4, TISNUM, >6
3400 |. addu TMP2, TMP2, TMP3
3401 | xor AT, CARG1, CARG2
3402 |.if MIPSR6
3403 if (vk) {
3404 | seleqz TMP2, TMP2, AT
3405 |1:
3406 | daddu PC, PC, TMP2
3407 |2:
3408 } else {
3409 | selnez TMP2, TMP2, AT
3410 |1:
3411 |2:
3412 | daddu PC, PC, TMP2
3413 }
3414 |.else
3415 if (vk) {
3416 | movn TMP2, r0, AT
3417 |1:
3418 | daddu PC, PC, TMP2
3419 |2:
3420 } else {
3421 | movz TMP2, r0, AT
3422 |1:
3423 |2:
3424 | daddu PC, PC, TMP2
3425 }
3426 |.endif
3427 | ins_next
3428 |
3429 |3: // RA is not an integer.
3430 | sltu AT, CARG3, TISNUM
3431 |.if FFI
3432 | beqz AT, >8
3433 |.else
3434 | beqz AT, <2
3435 |.endif
3436 |. addu TMP2, TMP2, TMP3
3437 | sltu AT, CARG4, TISNUM
3438 |.if FPU
3439 | ldc1 FTMP0, 0(RA)
3440 | ldc1 FTMP2, 0(RD)
3441 |.endif
3442 | beqz AT, >5
3443 |. nop
3444 |4: // RA and RD are both numbers.
3445 |.if FPU
3446 |.if MIPSR6
3447 | cmp.eq.d FTMP0, FTMP0, FTMP2
3448 | dmfc1 TMP1, FTMP0
3449 | b <1
3450 if (vk) {
3451 |. selnez TMP2, TMP2, TMP1
3452 } else {
3453 |. seleqz TMP2, TMP2, TMP1
3454 }
3455 |.else
3456 | c.eq.d FTMP0, FTMP2
3457 | b <1
3458 if (vk) {
3459 |. movf TMP2, r0
3460 } else {
3461 |. movt TMP2, r0
3462 }
3463 |.endif
3464 |.else
3465 | bal ->vm_sfcmpeq
3466 |. nop
3467 | b <1
3468 |.if MIPSR6
3469 if (vk) {
3470 |. selnez TMP2, TMP2, CRET1
3471 } else {
3472 |. seleqz TMP2, TMP2, CRET1
3473 }
3474 |.else
3475 if (vk) {
3476 |. movz TMP2, r0, CRET1
3477 } else {
3478 |. movn TMP2, r0, CRET1
3479 }
3480 |.endif
3481 |.endif
3482 |
3483 |5: // RA is a number, RD is not a number.
3484 |.if FFI
3485 | bne CARG4, TISNUM, >9
3486 |.else
3487 | bne CARG4, TISNUM, <2
3488 |.endif
3489 | // RA is a number, RD is an integer. Convert RD to a number.
3490 |.if FPU
3491 |. lwc1 FTMP2, LO(RD)
3492 | b <4
3493 |. cvt.d.w FTMP2, FTMP2
3494 |.else
3495 |. sextw CARG2, CARG2
3496 | bal ->vm_sfi2d_2
3497 |. nop
3498 | b <4
3499 |. nop
3500 |.endif
3501 |
3502 |6: // RA is an integer, RD is not an integer
3503 | sltu AT, CARG4, TISNUM
3504 |.if FFI
3505 | beqz AT, >9
3506 |.else
3507 | beqz AT, <2
3508 |.endif
3509 | // RA is an integer, RD is a number. Convert RA to a number.
3510 |.if FPU
3511 |. lwc1 FTMP0, LO(RA)
3512 | ldc1 FTMP2, 0(RD)
3513 | b <4
3514 | cvt.d.w FTMP0, FTMP0
3515 |.else
3516 |. sextw CARG1, CARG1
3517 | bal ->vm_sfi2d_1
3518 |. nop
3519 | b <4
3520 |. nop
3521 |.endif
3522 |
3523 |.if FFI
3524 |8:
3525 | li AT, LJ_TCDATA
3526 | bne CARG3, AT, <2
3527 |. nop
3528 | b ->vmeta_equal_cd
3529 |. nop
3530 |9:
3531 | li AT, LJ_TCDATA
3532 | bne CARG4, AT, <2
3533 |. nop
3534 | b ->vmeta_equal_cd
3535 |. nop
3536 |.endif
3537 break;
3538
3539 case BC_ISEQP: case BC_ISNEP:
3540 vk = op == BC_ISEQP;
3541 | // RA = src*8, RD = primitive_type*8 (~), JMP with RD = target
3542 | daddu RA, BASE, RA
3543 | srl TMP1, RD, 3
3544 | ld TMP0, 0(RA)
3545 | lhu TMP2, OFS_RD(PC)
3546 | not TMP1, TMP1
3547 | gettp TMP0, TMP0
3548 | daddiu PC, PC, 4
3549 |.if FFI
3550 | li AT, LJ_TCDATA
3551 | beq TMP0, AT, ->vmeta_equal_cd
3552 |.endif
3553 |. xor TMP0, TMP0, TMP1
3554 | decode_RD4b TMP2
3555 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3556 | addu TMP2, TMP2, TMP3
3557 |.if MIPSR6
3558 if (vk) {
3559 | seleqz TMP2, TMP2, TMP0
3560 } else {
3561 | selnez TMP2, TMP2, TMP0
3562 }
3563 |.else
3564 if (vk) {
3565 | movn TMP2, r0, TMP0
3566 } else {
3567 | movz TMP2, r0, TMP0
3568 }
3569 |.endif
3570 | daddu PC, PC, TMP2
3571 | ins_next
3572 break;
3573
3574 /* -- Unary test and copy ops ------------------------------------------- */
3575
3576 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
3577 | // RA = dst*8 or unused, RD = src*8, JMP with RD = target
3578 | daddu RD, BASE, RD
3579 | lhu TMP2, OFS_RD(PC)
3580 | ld TMP0, 0(RD)
3581 | daddiu PC, PC, 4
3582 | gettp TMP0, TMP0
3583 | sltiu TMP0, TMP0, LJ_TISTRUECOND
3584 if (op == BC_IST || op == BC_ISF) {
3585 | decode_RD4b TMP2
3586 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3587 | addu TMP2, TMP2, TMP3
3588 |.if MIPSR6
3589 if (op == BC_IST) {
3590 | selnez TMP2, TMP2, TMP0;
3591 } else {
3592 | seleqz TMP2, TMP2, TMP0;
3593 }
3594 |.else
3595 if (op == BC_IST) {
3596 | movz TMP2, r0, TMP0
3597 } else {
3598 | movn TMP2, r0, TMP0
3599 }
3600 |.endif
3601 | daddu PC, PC, TMP2
3602 } else {
3603 | ld CRET1, 0(RD)
3604 if (op == BC_ISTC) {
3605 | beqz TMP0, >1
3606 } else {
3607 | bnez TMP0, >1
3608 }
3609 |. daddu RA, BASE, RA
3610 | decode_RD4b TMP2
3611 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
3612 | addu TMP2, TMP2, TMP3
3613 | sd CRET1, 0(RA)
3614 | daddu PC, PC, TMP2
3615 |1:
3616 }
3617 | ins_next
3618 break;
3619
3620 case BC_ISTYPE:
3621 | // RA = src*8, RD = -type*8
3622 | daddu TMP2, BASE, RA
3623 | srl TMP1, RD, 3
3624 | ld TMP0, 0(TMP2)
3625 | ins_next1
3626 | gettp TMP0, TMP0
3627 | daddu AT, TMP0, TMP1
3628 | bnez AT, ->vmeta_istype
3629 |. ins_next2
3630 break;
3631 case BC_ISNUM:
3632 | // RA = src*8, RD = -(TISNUM-1)*8
3633 | daddu TMP2, BASE, RA
3634 | ld TMP0, 0(TMP2)
3635 | ins_next1
3636 | checknum TMP0, ->vmeta_istype
3637 |. ins_next2
3638 break;
3639
3640 /* -- Unary ops --------------------------------------------------------- */
3641
3642 case BC_MOV:
3643 | // RA = dst*8, RD = src*8
3644 | daddu RD, BASE, RD
3645 | daddu RA, BASE, RA
3646 | ld CRET1, 0(RD)
3647 | ins_next1
3648 | sd CRET1, 0(RA)
3649 | ins_next2
3650 break;
3651 case BC_NOT:
3652 | // RA = dst*8, RD = src*8
3653 | daddu RD, BASE, RD
3654 | daddu RA, BASE, RA
3655 | ld TMP0, 0(RD)
3656 | li AT, LJ_TTRUE
3657 | gettp TMP0, TMP0
3658 | sltu TMP0, AT, TMP0
3659 | addiu TMP0, TMP0, 1
3660 | dsll TMP0, TMP0, 47
3661 | not TMP0, TMP0
3662 | ins_next1
3663 | sd TMP0, 0(RA)
3664 | ins_next2
3665 break;
3666 case BC_UNM:
3667 | // RA = dst*8, RD = src*8
3668 | daddu RB, BASE, RD
3669 | ld CARG1, 0(RB)
3670 | daddu RA, BASE, RA
3671 | gettp CARG3, CARG1
3672 | bne CARG3, TISNUM, >2
3673 |. lui TMP1, 0x8000
3674 | sextw CARG1, CARG1
3675 | beq CARG1, TMP1, ->vmeta_unm // Meta handler deals with -2^31.
3676 |. negu CARG1, CARG1
3677 | zextw CARG1, CARG1
3678 | settp CARG1, TISNUM
3679 |1:
3680 | ins_next1
3681 | sd CARG1, 0(RA)
3682 | ins_next2
3683 |2:
3684 | sltiu AT, CARG3, LJ_TISNUM
3685 | beqz AT, ->vmeta_unm
3686 |. dsll TMP1, TMP1, 32
3687 | b <1
3688 |. xor CARG1, CARG1, TMP1
3689 break;
3690 case BC_LEN:
3691 | // RA = dst*8, RD = src*8
3692 | daddu CARG2, BASE, RD
3693 | daddu RA, BASE, RA
3694 | ld TMP0, 0(CARG2)
3695 | gettp TMP1, TMP0
3696 | daddiu AT, TMP1, -LJ_TSTR
3697 | bnez AT, >2
3698 |. cleartp STR:CARG1, TMP0
3699 | lw CRET1, STR:CARG1->len
3700 |1:
3701 | settp CRET1, TISNUM
3702 | ins_next1
3703 | sd CRET1, 0(RA)
3704 | ins_next2
3705 |2:
3706 | daddiu AT, TMP1, -LJ_TTAB
3707 | bnez AT, ->vmeta_len
3708 |. nop
3709#if LJ_52
3710 | ld TAB:TMP2, TAB:CARG1->metatable
3711 | bnez TAB:TMP2, >9
3712 |. nop
3713 |3:
3714#endif
3715 |->BC_LEN_Z:
3716 | load_got lj_tab_len
3717 | call_intern lj_tab_len // (GCtab *t)
3718 |. nop
3719 | // Returns uint32_t (but less than 2^31).
3720 | b <1
3721 |. nop
3722#if LJ_52
3723 |9:
3724 | lbu TMP0, TAB:TMP2->nomm
3725 | andi TMP0, TMP0, 1<<MM_len
3726 | bnez TMP0, <3 // 'no __len' flag set: done.
3727 |. nop
3728 | b ->vmeta_len
3729 |. nop
3730#endif
3731 break;
3732
3733 /* -- Binary ops -------------------------------------------------------- */
3734
3735 |.macro fpmod, a, b, c
3736 | bal ->vm_floor // floor(b/c)
3737 |. div.d FARG1, b, c
3738 | mul.d a, FRET1, c
3739 | sub.d a, b, a // b - floor(b/c)*c
3740 |.endmacro
3741
3742 |.macro sfpmod
3743 | daddiu sp, sp, -16
3744 |
3745 | load_got __divdf3
3746 | sd CARG1, 0(sp)
3747 | call_extern
3748 |. sd CARG2, 8(sp)
3749 |
3750 | load_got floor
3751 | call_extern
3752 |. move CARG1, CRET1
3753 |
3754 | load_got __muldf3
3755 | move CARG1, CRET1
3756 | call_extern
3757 |. ld CARG2, 8(sp)
3758 |
3759 | load_got __subdf3
3760 | ld CARG1, 0(sp)
3761 | call_extern
3762 |. move CARG2, CRET1
3763 |
3764 | daddiu sp, sp, 16
3765 |.endmacro
3766
3767 |.macro ins_arithpre, label
3768 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3769 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
3770 ||switch (vk) {
3771 ||case 0:
3772 | decode_RB8a RB, INS
3773 | decode_RB8b RB
3774 | decode_RDtoRC8 RC, RD
3775 | // RA = dst*8, RB = src1*8, RC = num_const*8
3776 | daddu RB, BASE, RB
3777 |.if "label" ~= "none"
3778 | b label
3779 |.endif
3780 |. daddu RC, KBASE, RC
3781 || break;
3782 ||case 1:
3783 | decode_RB8a RC, INS
3784 | decode_RB8b RC
3785 | decode_RDtoRC8 RB, RD
3786 | // RA = dst*8, RB = num_const*8, RC = src1*8
3787 | daddu RC, BASE, RC
3788 |.if "label" ~= "none"
3789 | b label
3790 |.endif
3791 |. daddu RB, KBASE, RB
3792 || break;
3793 ||default:
3794 | decode_RB8a RB, INS
3795 | decode_RB8b RB
3796 | decode_RDtoRC8 RC, RD
3797 | // RA = dst*8, RB = src1*8, RC = src2*8
3798 | daddu RB, BASE, RB
3799 |.if "label" ~= "none"
3800 | b label
3801 |.endif
3802 |. daddu RC, BASE, RC
3803 || break;
3804 ||}
3805 |.endmacro
3806 |
3807 |.macro ins_arith, intins, fpins, fpcall, label
3808 | ins_arithpre none
3809 |
3810 |.if "label" ~= "none"
3811 |label:
3812 |.endif
3813 |
3814 |// Used in 5.
3815 | ld CARG1, 0(RB)
3816 | ld CARG2, 0(RC)
3817 | gettp TMP0, CARG1
3818 | gettp TMP1, CARG2
3819 |
3820 |.if "intins" ~= "div"
3821 |
3822 | // Check for two integers.
3823 | sextw CARG3, CARG1
3824 | bne TMP0, TISNUM, >5
3825 |. sextw CARG4, CARG2
3826 | bne TMP1, TISNUM, >5
3827 |
3828 |.if "intins" == "addu"
3829 |. intins CRET1, CARG3, CARG4
3830 | xor TMP1, CRET1, CARG3 // ((y^a) & (y^b)) < 0: overflow.
3831 | xor TMP2, CRET1, CARG4
3832 | and TMP1, TMP1, TMP2
3833 | bltz TMP1, ->vmeta_arith
3834 |. daddu RA, BASE, RA
3835 |.elif "intins" == "subu"
3836 |. intins CRET1, CARG3, CARG4
3837 | xor TMP1, CRET1, CARG3 // ((y^a) & (a^b)) < 0: overflow.
3838 | xor TMP2, CARG3, CARG4
3839 | and TMP1, TMP1, TMP2
3840 | bltz TMP1, ->vmeta_arith
3841 |. daddu RA, BASE, RA
3842 |.elif "intins" == "mult"
3843 |.if MIPSR6
3844 |. nop
3845 | mul CRET1, CARG3, CARG4
3846 | muh TMP2, CARG3, CARG4
3847 |.else
3848 |. intins CARG3, CARG4
3849 | mflo CRET1
3850 | mfhi TMP2
3851 |.endif
3852 | sra TMP1, CRET1, 31
3853 | bne TMP1, TMP2, ->vmeta_arith
3854 |. daddu RA, BASE, RA
3855 |.else
3856 |. load_got lj_vm_modi
3857 | beqz CARG4, ->vmeta_arith
3858 |. daddu RA, BASE, RA
3859 | move CARG1, CARG3
3860 | call_extern
3861 |. move CARG2, CARG4
3862 |.endif
3863 |
3864 | zextw CRET1, CRET1
3865 | settp CRET1, TISNUM
3866 | ins_next1
3867 | sd CRET1, 0(RA)
3868 |3:
3869 | ins_next2
3870 |
3871 |.endif
3872 |
3873 |5: // Check for two numbers.
3874 | .FPU ldc1 FTMP0, 0(RB)
3875 | sltu AT, TMP0, TISNUM
3876 | sltu TMP0, TMP1, TISNUM
3877 | .FPU ldc1 FTMP2, 0(RC)
3878 | and AT, AT, TMP0
3879 | beqz AT, ->vmeta_arith
3880 |. daddu RA, BASE, RA
3881 |
3882 |.if FPU
3883 | fpins FRET1, FTMP0, FTMP2
3884 |.elif "fpcall" == "sfpmod"
3885 | sfpmod
3886 |.else
3887 | load_got fpcall
3888 | call_extern
3889 |. nop
3890 |.endif
3891 |
3892 | ins_next1
3893 |.if "intins" ~= "div"
3894 | b <3
3895 |.endif
3896 |.if FPU
3897 |. sdc1 FRET1, 0(RA)
3898 |.else
3899 |. sd CRET1, 0(RA)
3900 |.endif
3901 |.if "intins" == "div"
3902 | ins_next2
3903 |.endif
3904 |
3905 |.endmacro
3906
3907 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
3908 | ins_arith addu, add.d, __adddf3, none
3909 break;
3910 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
3911 | ins_arith subu, sub.d, __subdf3, none
3912 break;
3913 case BC_MULVN: case BC_MULNV: case BC_MULVV:
3914 | ins_arith mult, mul.d, __muldf3, none
3915 break;
3916 case BC_DIVVN:
3917 | ins_arith div, div.d, __divdf3, ->BC_DIVVN_Z
3918 break;
3919 case BC_DIVNV: case BC_DIVVV:
3920 | ins_arithpre ->BC_DIVVN_Z
3921 break;
3922 case BC_MODVN:
3923 | ins_arith modi, fpmod, sfpmod, ->BC_MODVN_Z
3924 break;
3925 case BC_MODNV: case BC_MODVV:
3926 | ins_arithpre ->BC_MODVN_Z
3927 break;
3928 case BC_POW:
3929 | ins_arithpre none
3930 | ld CARG1, 0(RB)
3931 | ld CARG2, 0(RC)
3932 | gettp TMP0, CARG1
3933 | gettp TMP1, CARG2
3934 | sltiu TMP0, TMP0, LJ_TISNUM
3935 | sltiu TMP1, TMP1, LJ_TISNUM
3936 | and AT, TMP0, TMP1
3937 | load_got pow
3938 | beqz AT, ->vmeta_arith
3939 |. daddu RA, BASE, RA
3940 |.if FPU
3941 | ldc1 FARG1, 0(RB)
3942 | ldc1 FARG2, 0(RC)
3943 |.endif
3944 | call_extern
3945 |. nop
3946 | ins_next1
3947 |.if FPU
3948 | sdc1 FRET1, 0(RA)
3949 |.else
3950 | sd CRET1, 0(RA)
3951 |.endif
3952 | ins_next2
3953 break;
3954
3955 case BC_CAT:
3956 | // RA = dst*8, RB = src_start*8, RC = src_end*8
3957 | decode_RB8a RB, INS
3958 | decode_RB8b RB
3959 | decode_RDtoRC8 RC, RD
3960 | dsubu CARG3, RC, RB
3961 | sd BASE, L->base
3962 | daddu CARG2, BASE, RC
3963 | move MULTRES, RB
3964 |->BC_CAT_Z:
3965 | load_got lj_meta_cat
3966 | srl CARG3, CARG3, 3
3967 | sd PC, SAVE_PC
3968 | call_intern lj_meta_cat // (lua_State *L, TValue *top, int left)
3969 |. move CARG1, L
3970 | // Returns NULL (finished) or TValue * (metamethod).
3971 | bnez CRET1, ->vmeta_binop
3972 |. ld BASE, L->base
3973 | daddu RB, BASE, MULTRES
3974 | ld CRET1, 0(RB)
3975 | daddu RA, BASE, RA
3976 | ins_next1
3977 | sd CRET1, 0(RA)
3978 | ins_next2
3979 break;
3980
3981 /* -- Constant ops ------------------------------------------------------ */
3982
3983 case BC_KSTR:
3984 | // RA = dst*8, RD = str_const*8 (~)
3985 | dsubu TMP1, KBASE, RD
3986 | ins_next1
3987 | li TMP2, LJ_TSTR
3988 | ld TMP0, -8(TMP1) // KBASE-8-str_const*8
3989 | daddu RA, BASE, RA
3990 | settp TMP0, TMP2
3991 | sd TMP0, 0(RA)
3992 | ins_next2
3993 break;
3994 case BC_KCDATA:
3995 |.if FFI
3996 | // RA = dst*8, RD = cdata_const*8 (~)
3997 | dsubu TMP1, KBASE, RD
3998 | ins_next1
3999 | ld TMP0, -8(TMP1) // KBASE-8-cdata_const*8
4000 | li TMP2, LJ_TCDATA
4001 | daddu RA, BASE, RA
4002 | settp TMP0, TMP2
4003 | sd TMP0, 0(RA)
4004 | ins_next2
4005 |.endif
4006 break;
4007 case BC_KSHORT:
4008 | // RA = dst*8, RD = int16_literal*8
4009 | sra RD, INS, 16
4010 | daddu RA, BASE, RA
4011 | zextw RD, RD
4012 | ins_next1
4013 | settp RD, TISNUM
4014 | sd RD, 0(RA)
4015 | ins_next2
4016 break;
4017 case BC_KNUM:
4018 | // RA = dst*8, RD = num_const*8
4019 | daddu RD, KBASE, RD
4020 | daddu RA, BASE, RA
4021 | ld CRET1, 0(RD)
4022 | ins_next1
4023 | sd CRET1, 0(RA)
4024 | ins_next2
4025 break;
4026 case BC_KPRI:
4027 | // RA = dst*8, RD = primitive_type*8 (~)
4028 | daddu RA, BASE, RA
4029 | dsll TMP0, RD, 44
4030 | not TMP0, TMP0
4031 | ins_next1
4032 | sd TMP0, 0(RA)
4033 | ins_next2
4034 break;
4035 case BC_KNIL:
4036 | // RA = base*8, RD = end*8
4037 | daddu RA, BASE, RA
4038 | sd TISNIL, 0(RA)
4039 | daddiu RA, RA, 8
4040 | daddu RD, BASE, RD
4041 |1:
4042 | sd TISNIL, 0(RA)
4043 | slt AT, RA, RD
4044 | bnez AT, <1
4045 |. daddiu RA, RA, 8
4046 | ins_next_
4047 break;
4048
4049 /* -- Upvalue and function ops ------------------------------------------ */
4050
4051 case BC_UGET:
4052 | // RA = dst*8, RD = uvnum*8
4053 | ld LFUNC:RB, FRAME_FUNC(BASE)
4054 | daddu RA, BASE, RA
4055 | cleartp LFUNC:RB
4056 | daddu RD, RD, LFUNC:RB
4057 | ld UPVAL:RB, LFUNC:RD->uvptr
4058 | ins_next1
4059 | ld TMP1, UPVAL:RB->v
4060 | ld CRET1, 0(TMP1)
4061 | sd CRET1, 0(RA)
4062 | ins_next2
4063 break;
4064 case BC_USETV:
4065 | // RA = uvnum*8, RD = src*8
4066 | ld LFUNC:RB, FRAME_FUNC(BASE)
4067 | daddu RD, BASE, RD
4068 | cleartp LFUNC:RB
4069 | daddu RA, RA, LFUNC:RB
4070 | ld UPVAL:RB, LFUNC:RA->uvptr
4071 | ld CRET1, 0(RD)
4072 | lbu TMP3, UPVAL:RB->marked
4073 | ld CARG2, UPVAL:RB->v
4074 | andi TMP3, TMP3, LJ_GC_BLACK // isblack(uv)
4075 | lbu TMP0, UPVAL:RB->closed
4076 | gettp TMP2, CRET1
4077 | sd CRET1, 0(CARG2)
4078 | li AT, LJ_GC_BLACK|1
4079 | or TMP3, TMP3, TMP0
4080 | beq TMP3, AT, >2 // Upvalue is closed and black?
4081 |. daddiu TMP2, TMP2, -(LJ_TNUMX+1)
4082 |1:
4083 | ins_next
4084 |
4085 |2: // Check if new value is collectable.
4086 | sltiu AT, TMP2, LJ_TISGCV - (LJ_TNUMX+1)
4087 | beqz AT, <1 // tvisgcv(v)
4088 |. cleartp GCOBJ:CRET1, CRET1
4089 | lbu TMP3, GCOBJ:CRET1->gch.marked
4090 | andi TMP3, TMP3, LJ_GC_WHITES // iswhite(v)
4091 | beqz TMP3, <1
4092 |. load_got lj_gc_barrieruv
4093 | // Crossed a write barrier. Move the barrier forward.
4094 | call_intern lj_gc_barrieruv // (global_State *g, TValue *tv)
4095 |. daddiu CARG1, DISPATCH, GG_DISP2G
4096 | b <1
4097 |. nop
4098 break;
4099 case BC_USETS:
4100 | // RA = uvnum*8, RD = str_const*8 (~)
4101 | ld LFUNC:RB, FRAME_FUNC(BASE)
4102 | dsubu TMP1, KBASE, RD
4103 | cleartp LFUNC:RB
4104 | daddu RA, RA, LFUNC:RB
4105 | ld UPVAL:RB, LFUNC:RA->uvptr
4106 | ld STR:TMP1, -8(TMP1) // KBASE-8-str_const*8
4107 | lbu TMP2, UPVAL:RB->marked
4108 | ld CARG2, UPVAL:RB->v
4109 | lbu TMP3, STR:TMP1->marked
4110 | andi AT, TMP2, LJ_GC_BLACK // isblack(uv)
4111 | lbu TMP2, UPVAL:RB->closed
4112 | li TMP0, LJ_TSTR
4113 | settp TMP1, TMP0
4114 | bnez AT, >2
4115 |. sd TMP1, 0(CARG2)
4116 |1:
4117 | ins_next
4118 |
4119 |2: // Check if string is white and ensure upvalue is closed.
4120 | beqz TMP2, <1
4121 |. andi AT, TMP3, LJ_GC_WHITES // iswhite(str)
4122 | beqz AT, <1
4123 |. load_got lj_gc_barrieruv
4124 | // Crossed a write barrier. Move the barrier forward.
4125 | call_intern lj_gc_barrieruv // (global_State *g, TValue *tv)
4126 |. daddiu CARG1, DISPATCH, GG_DISP2G
4127 | b <1
4128 |. nop
4129 break;
4130 case BC_USETN:
4131 | // RA = uvnum*8, RD = num_const*8
4132 | ld LFUNC:RB, FRAME_FUNC(BASE)
4133 | daddu RD, KBASE, RD
4134 | cleartp LFUNC:RB
4135 | daddu RA, RA, LFUNC:RB
4136 | ld UPVAL:RB, LFUNC:RA->uvptr
4137 | ld CRET1, 0(RD)
4138 | ld TMP1, UPVAL:RB->v
4139 | ins_next1
4140 | sd CRET1, 0(TMP1)
4141 | ins_next2
4142 break;
4143 case BC_USETP:
4144 | // RA = uvnum*8, RD = primitive_type*8 (~)
4145 | ld LFUNC:RB, FRAME_FUNC(BASE)
4146 | dsll TMP0, RD, 44
4147 | cleartp LFUNC:RB
4148 | daddu RA, RA, LFUNC:RB
4149 | not TMP0, TMP0
4150 | ld UPVAL:RB, LFUNC:RA->uvptr
4151 | ins_next1
4152 | ld TMP1, UPVAL:RB->v
4153 | sd TMP0, 0(TMP1)
4154 | ins_next2
4155 break;
4156
4157 case BC_UCLO:
4158 | // RA = level*8, RD = target
4159 | ld TMP2, L->openupval
4160 | branch_RD // Do this first since RD is not saved.
4161 | load_got lj_func_closeuv
4162 | sd BASE, L->base
4163 | beqz TMP2, >1
4164 |. move CARG1, L
4165 | call_intern lj_func_closeuv // (lua_State *L, TValue *level)
4166 |. daddu CARG2, BASE, RA
4167 | ld BASE, L->base
4168 |1:
4169 | ins_next
4170 break;
4171
4172 case BC_FNEW:
4173 | // RA = dst*8, RD = proto_const*8 (~) (holding function prototype)
4174 | load_got lj_func_newL_gc
4175 | dsubu TMP1, KBASE, RD
4176 | ld CARG3, FRAME_FUNC(BASE)
4177 | ld CARG2, -8(TMP1) // KBASE-8-tab_const*8
4178 | sd BASE, L->base
4179 | sd PC, SAVE_PC
4180 | cleartp CARG3
4181 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
4182 | call_intern lj_func_newL_gc
4183 |. move CARG1, L
4184 | // Returns GCfuncL *.
4185 | li TMP0, LJ_TFUNC
4186 | ld BASE, L->base
4187 | ins_next1
4188 | settp CRET1, TMP0
4189 | daddu RA, BASE, RA
4190 | sd CRET1, 0(RA)
4191 | ins_next2
4192 break;
4193
4194 /* -- Table ops --------------------------------------------------------- */
4195
4196 case BC_TNEW:
4197 case BC_TDUP:
4198 | // RA = dst*8, RD = (hbits|asize)*8 | tab_const*8 (~)
4199 | ld TMP0, DISPATCH_GL(gc.total)(DISPATCH)
4200 | ld TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
4201 | sd BASE, L->base
4202 | sd PC, SAVE_PC
4203 | sltu AT, TMP0, TMP1
4204 | beqz AT, >5
4205 |1:
4206 if (op == BC_TNEW) {
4207 | load_got lj_tab_new
4208 | srl CARG2, RD, 3
4209 | andi CARG2, CARG2, 0x7ff
4210 | li TMP0, 0x801
4211 | addiu AT, CARG2, -0x7ff
4212 | srl CARG3, RD, 14
4213 |.if MIPSR6
4214 | seleqz TMP0, TMP0, AT
4215 | selnez CARG2, CARG2, AT
4216 | or CARG2, CARG2, TMP0
4217 |.else
4218 | movz CARG2, TMP0, AT
4219 |.endif
4220 | // (lua_State *L, int32_t asize, uint32_t hbits)
4221 | call_intern lj_tab_new
4222 |. move CARG1, L
4223 | // Returns Table *.
4224 } else {
4225 | load_got lj_tab_dup
4226 | dsubu TMP1, KBASE, RD
4227 | move CARG1, L
4228 | call_intern lj_tab_dup // (lua_State *L, Table *kt)
4229 |. ld CARG2, -8(TMP1) // KBASE-8-str_const*8
4230 | // Returns Table *.
4231 }
4232 | li TMP0, LJ_TTAB
4233 | ld BASE, L->base
4234 | ins_next1
4235 | daddu RA, BASE, RA
4236 | settp CRET1, TMP0
4237 | sd CRET1, 0(RA)
4238 | ins_next2
4239 |5:
4240 | load_got lj_gc_step_fixtop
4241 | move MULTRES, RD
4242 | call_intern lj_gc_step_fixtop // (lua_State *L)
4243 |. move CARG1, L
4244 | b <1
4245 |. move RD, MULTRES
4246 break;
4247
4248 case BC_GGET:
4249 | // RA = dst*8, RD = str_const*8 (~)
4250 case BC_GSET:
4251 | // RA = src*8, RD = str_const*8 (~)
4252 | ld LFUNC:TMP2, FRAME_FUNC(BASE)
4253 | dsubu TMP1, KBASE, RD
4254 | ld STR:RC, -8(TMP1) // KBASE-8-str_const*8
4255 | cleartp LFUNC:TMP2
4256 | ld TAB:RB, LFUNC:TMP2->env
4257 if (op == BC_GGET) {
4258 | b ->BC_TGETS_Z
4259 } else {
4260 | b ->BC_TSETS_Z
4261 }
4262 |. daddu RA, BASE, RA
4263 break;
4264
4265 case BC_TGETV:
4266 | // RA = dst*8, RB = table*8, RC = key*8
4267 | decode_RB8a RB, INS
4268 | decode_RB8b RB
4269 | decode_RDtoRC8 RC, RD
4270 | daddu CARG2, BASE, RB
4271 | daddu CARG3, BASE, RC
4272 | ld TAB:RB, 0(CARG2)
4273 | ld TMP2, 0(CARG3)
4274 | daddu RA, BASE, RA
4275 | checktab TAB:RB, ->vmeta_tgetv
4276 | gettp TMP3, TMP2
4277 | bne TMP3, TISNUM, >5 // Integer key?
4278 |. lw TMP0, TAB:RB->asize
4279 | sextw TMP2, TMP2
4280 | ld TMP1, TAB:RB->array
4281 | sltu AT, TMP2, TMP0
4282 | sll TMP2, TMP2, 3
4283 | beqz AT, ->vmeta_tgetv // Integer key and in array part?
4284 |. daddu TMP2, TMP1, TMP2
4285 | ld AT, 0(TMP2)
4286 | beq AT, TISNIL, >2
4287 |. ld CRET1, 0(TMP2)
4288 |1:
4289 | ins_next1
4290 | sd CRET1, 0(RA)
4291 | ins_next2
4292 |
4293 |2: // Check for __index if table value is nil.
4294 | ld TAB:TMP2, TAB:RB->metatable
4295 | beqz TAB:TMP2, <1 // No metatable: done.
4296 |. nop
4297 | lbu TMP0, TAB:TMP2->nomm
4298 | andi TMP0, TMP0, 1<<MM_index
4299 | bnez TMP0, <1 // 'no __index' flag set: done.
4300 |. nop
4301 | b ->vmeta_tgetv
4302 |. nop
4303 |
4304 |5:
4305 | li AT, LJ_TSTR
4306 | bne TMP3, AT, ->vmeta_tgetv
4307 |. cleartp RC, TMP2
4308 | b ->BC_TGETS_Z // String key?
4309 |. nop
4310 break;
4311 case BC_TGETS:
4312 | // RA = dst*8, RB = table*8, RC = str_const*8 (~)
4313 | decode_RB8a RB, INS
4314 | decode_RB8b RB
4315 | decode_RC8a RC, INS
4316 | daddu CARG2, BASE, RB
4317 | decode_RC8b RC
4318 | ld TAB:RB, 0(CARG2)
4319 | dsubu CARG3, KBASE, RC
4320 | daddu RA, BASE, RA
4321 | ld STR:RC, -8(CARG3) // KBASE-8-str_const*8
4322 | checktab TAB:RB, ->vmeta_tgets1
4323 |->BC_TGETS_Z:
4324 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8
4325 | lw TMP0, TAB:RB->hmask
4326 | lw TMP1, STR:RC->sid
4327 | ld NODE:TMP2, TAB:RB->node
4328 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
4329 | sll TMP0, TMP1, 5
4330 | sll TMP1, TMP1, 3
4331 | subu TMP1, TMP0, TMP1
4332 | li TMP3, LJ_TSTR
4333 | daddu NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
4334 | settp STR:RC, TMP3 // Tagged key to look for.
4335 |1:
4336 | ld CARG1, NODE:TMP2->key
4337 | ld CRET1, NODE:TMP2->val
4338 | ld NODE:TMP1, NODE:TMP2->next
4339 | bne CARG1, RC, >4
4340 |. ld TAB:TMP3, TAB:RB->metatable
4341 | beq CRET1, TISNIL, >5 // Key found, but nil value?
4342 |. nop
4343 |3:
4344 | ins_next1
4345 | sd CRET1, 0(RA)
4346 | ins_next2
4347 |
4348 |4: // Follow hash chain.
4349 | bnez NODE:TMP1, <1
4350 |. move NODE:TMP2, NODE:TMP1
4351 | // End of hash chain: key not found, nil result.
4352 |
4353 |5: // Check for __index if table value is nil.
4354 | beqz TAB:TMP3, <3 // No metatable: done.
4355 |. move CRET1, TISNIL
4356 | lbu TMP0, TAB:TMP3->nomm
4357 | andi TMP0, TMP0, 1<<MM_index
4358 | bnez TMP0, <3 // 'no __index' flag set: done.
4359 |. nop
4360 | b ->vmeta_tgets
4361 |. nop
4362 break;
4363 case BC_TGETB:
4364 | // RA = dst*8, RB = table*8, RC = index*8
4365 | decode_RB8a RB, INS
4366 | decode_RB8b RB
4367 | daddu CARG2, BASE, RB
4368 | decode_RDtoRC8 RC, RD
4369 | ld TAB:RB, 0(CARG2)
4370 | daddu RA, BASE, RA
4371 | srl TMP0, RC, 3
4372 | checktab TAB:RB, ->vmeta_tgetb
4373 | lw TMP1, TAB:RB->asize
4374 | ld TMP2, TAB:RB->array
4375 | sltu AT, TMP0, TMP1
4376 | beqz AT, ->vmeta_tgetb
4377 |. daddu RC, TMP2, RC
4378 | ld AT, 0(RC)
4379 | beq AT, TISNIL, >5
4380 |. ld CRET1, 0(RC)
4381 |1:
4382 | ins_next1
4383 | sd CRET1, 0(RA)
4384 | ins_next2
4385 |
4386 |5: // Check for __index if table value is nil.
4387 | ld TAB:TMP2, TAB:RB->metatable
4388 | beqz TAB:TMP2, <1 // No metatable: done.
4389 |. nop
4390 | lbu TMP1, TAB:TMP2->nomm
4391 | andi TMP1, TMP1, 1<<MM_index
4392 | bnez TMP1, <1 // 'no __index' flag set: done.
4393 |. nop
4394 | b ->vmeta_tgetb // Caveat: preserve TMP0 and CARG2!
4395 |. nop
4396 break;
4397 case BC_TGETR:
4398 | // RA = dst*8, RB = table*8, RC = key*8
4399 | decode_RB8a RB, INS
4400 | decode_RB8b RB
4401 | decode_RDtoRC8 RC, RD
4402 | daddu RB, BASE, RB
4403 | daddu RC, BASE, RC
4404 | ld TAB:CARG1, 0(RB)
4405 | lw CARG2, LO(RC)
4406 | daddu RA, BASE, RA
4407 | cleartp TAB:CARG1
4408 | lw TMP0, TAB:CARG1->asize
4409 | ld TMP1, TAB:CARG1->array
4410 | sltu AT, CARG2, TMP0
4411 | sll TMP2, CARG2, 3
4412 | beqz AT, ->vmeta_tgetr // In array part?
4413 |. daddu CRET1, TMP1, TMP2
4414 | ld CARG2, 0(CRET1)
4415 |->BC_TGETR_Z:
4416 | ins_next1
4417 | sd CARG2, 0(RA)
4418 | ins_next2
4419 break;
4420
4421 case BC_TSETV:
4422 | // RA = src*8, RB = table*8, RC = key*8
4423 | decode_RB8a RB, INS
4424 | decode_RB8b RB
4425 | decode_RDtoRC8 RC, RD
4426 | daddu CARG2, BASE, RB
4427 | daddu CARG3, BASE, RC
4428 | ld RB, 0(CARG2)
4429 | ld TMP2, 0(CARG3)
4430 | daddu RA, BASE, RA
4431 | checktab RB, ->vmeta_tsetv
4432 | checkint TMP2, >5
4433 |. sextw RC, TMP2
4434 | lw TMP0, TAB:RB->asize
4435 | ld TMP1, TAB:RB->array
4436 | sltu AT, RC, TMP0
4437 | sll TMP2, RC, 3
4438 | beqz AT, ->vmeta_tsetv // Integer key and in array part?
4439 |. daddu TMP1, TMP1, TMP2
4440 | ld TMP0, 0(TMP1)
4441 | lbu TMP3, TAB:RB->marked
4442 | beq TMP0, TISNIL, >3
4443 |. ld CRET1, 0(RA)
4444 |1:
4445 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4446 | bnez AT, >7
4447 |. sd CRET1, 0(TMP1)
4448 |2:
4449 | ins_next
4450 |
4451 |3: // Check for __newindex if previous value is nil.
4452 | ld TAB:TMP2, TAB:RB->metatable
4453 | beqz TAB:TMP2, <1 // No metatable: done.
4454 |. nop
4455 | lbu TMP2, TAB:TMP2->nomm
4456 | andi TMP2, TMP2, 1<<MM_newindex
4457 | bnez TMP2, <1 // 'no __newindex' flag set: done.
4458 |. nop
4459 | b ->vmeta_tsetv
4460 |. nop
4461 |
4462 |5:
4463 | gettp AT, TMP2
4464 | daddiu AT, AT, -LJ_TSTR
4465 | bnez AT, ->vmeta_tsetv
4466 |. nop
4467 | b ->BC_TSETS_Z // String key?
4468 |. cleartp STR:RC, TMP2
4469 |
4470 |7: // Possible table write barrier for the value. Skip valiswhite check.
4471 | barrierback TAB:RB, TMP3, TMP0, <2
4472 break;
4473 case BC_TSETS:
4474 | // RA = src*8, RB = table*8, RC = str_const*8 (~)
4475 | decode_RB8a RB, INS
4476 | decode_RB8b RB
4477 | daddu CARG2, BASE, RB
4478 | decode_RC8a RC, INS
4479 | ld TAB:RB, 0(CARG2)
4480 | decode_RC8b RC
4481 | dsubu CARG3, KBASE, RC
4482 | ld RC, -8(CARG3) // KBASE-8-str_const*8
4483 | daddu RA, BASE, RA
4484 | cleartp STR:RC
4485 | checktab TAB:RB, ->vmeta_tsets1
4486 |->BC_TSETS_Z:
4487 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8
4488 | lw TMP0, TAB:RB->hmask
4489 | lw TMP1, STR:RC->sid
4490 | ld NODE:TMP2, TAB:RB->node
4491 | sb r0, TAB:RB->nomm // Clear metamethod cache.
4492 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
4493 | sll TMP0, TMP1, 5
4494 | sll TMP1, TMP1, 3
4495 | subu TMP1, TMP0, TMP1
4496 | li TMP3, LJ_TSTR
4497 | daddu NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
4498 | settp STR:RC, TMP3 // Tagged key to look for.
4499 |.if FPU
4500 | ldc1 FTMP0, 0(RA)
4501 |.else
4502 | ld CRET1, 0(RA)
4503 |.endif
4504 |1:
4505 | ld TMP0, NODE:TMP2->key
4506 | ld CARG2, NODE:TMP2->val
4507 | ld NODE:TMP1, NODE:TMP2->next
4508 | bne TMP0, RC, >5
4509 |. lbu TMP3, TAB:RB->marked
4510 | beq CARG2, TISNIL, >4 // Key found, but nil value?
4511 |. ld TAB:TMP0, TAB:RB->metatable
4512 |2:
4513 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4514 | bnez AT, >7
4515 |.if FPU
4516 |. sdc1 FTMP0, NODE:TMP2->val
4517 |.else
4518 |. sd CRET1, NODE:TMP2->val
4519 |.endif
4520 |3:
4521 | ins_next
4522 |
4523 |4: // Check for __newindex if previous value is nil.
4524 | beqz TAB:TMP0, <2 // No metatable: done.
4525 |. nop
4526 | lbu TMP0, TAB:TMP0->nomm
4527 | andi TMP0, TMP0, 1<<MM_newindex
4528 | bnez TMP0, <2 // 'no __newindex' flag set: done.
4529 |. nop
4530 | b ->vmeta_tsets
4531 |. nop
4532 |
4533 |5: // Follow hash chain.
4534 | bnez NODE:TMP1, <1
4535 |. move NODE:TMP2, NODE:TMP1
4536 | // End of hash chain: key not found, add a new one
4537 |
4538 | // But check for __newindex first.
4539 | ld TAB:TMP2, TAB:RB->metatable
4540 | beqz TAB:TMP2, >6 // No metatable: continue.
4541 |. daddiu CARG3, DISPATCH, DISPATCH_GL(tmptv)
4542 | lbu TMP0, TAB:TMP2->nomm
4543 | andi TMP0, TMP0, 1<<MM_newindex
4544 | beqz TMP0, ->vmeta_tsets // 'no __newindex' flag NOT set: check.
4545 |6:
4546 | load_got lj_tab_newkey
4547 | sd RC, 0(CARG3)
4548 | sd BASE, L->base
4549 | move CARG2, TAB:RB
4550 | sd PC, SAVE_PC
4551 | call_intern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k
4552 |. move CARG1, L
4553 | // Returns TValue *.
4554 | ld BASE, L->base
4555 |.if FPU
4556 | b <3 // No 2nd write barrier needed.
4557 |. sdc1 FTMP0, 0(CRET1)
4558 |.else
4559 | ld CARG1, 0(RA)
4560 | b <3 // No 2nd write barrier needed.
4561 |. sd CARG1, 0(CRET1)
4562 |.endif
4563 |
4564 |7: // Possible table write barrier for the value. Skip valiswhite check.
4565 | barrierback TAB:RB, TMP3, TMP0, <3
4566 break;
4567 case BC_TSETB:
4568 | // RA = src*8, RB = table*8, RC = index*8
4569 | decode_RB8a RB, INS
4570 | decode_RB8b RB
4571 | daddu CARG2, BASE, RB
4572 | decode_RDtoRC8 RC, RD
4573 | ld TAB:RB, 0(CARG2)
4574 | daddu RA, BASE, RA
4575 | srl TMP0, RC, 3
4576 | checktab RB, ->vmeta_tsetb
4577 | lw TMP1, TAB:RB->asize
4578 | ld TMP2, TAB:RB->array
4579 | sltu AT, TMP0, TMP1
4580 | beqz AT, ->vmeta_tsetb
4581 |. daddu RC, TMP2, RC
4582 | ld TMP1, 0(RC)
4583 | lbu TMP3, TAB:RB->marked
4584 | beq TMP1, TISNIL, >5
4585 |1:
4586 |. ld CRET1, 0(RA)
4587 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4588 | bnez AT, >7
4589 |. sd CRET1, 0(RC)
4590 |2:
4591 | ins_next
4592 |
4593 |5: // Check for __newindex if previous value is nil.
4594 | ld TAB:TMP2, TAB:RB->metatable
4595 | beqz TAB:TMP2, <1 // No metatable: done.
4596 |. nop
4597 | lbu TMP1, TAB:TMP2->nomm
4598 | andi TMP1, TMP1, 1<<MM_newindex
4599 | bnez TMP1, <1 // 'no __newindex' flag set: done.
4600 |. nop
4601 | b ->vmeta_tsetb // Caveat: preserve TMP0 and CARG2!
4602 |. nop
4603 |
4604 |7: // Possible table write barrier for the value. Skip valiswhite check.
4605 | barrierback TAB:RB, TMP3, TMP0, <2
4606 break;
4607 case BC_TSETR:
4608 | // RA = dst*8, RB = table*8, RC = key*8
4609 | decode_RB8a RB, INS
4610 | decode_RB8b RB
4611 | decode_RDtoRC8 RC, RD
4612 | daddu CARG1, BASE, RB
4613 | daddu CARG3, BASE, RC
4614 | ld TAB:CARG2, 0(CARG1)
4615 | lw CARG3, LO(CARG3)
4616 | cleartp TAB:CARG2
4617 | lbu TMP3, TAB:CARG2->marked
4618 | lw TMP0, TAB:CARG2->asize
4619 | ld TMP1, TAB:CARG2->array
4620 | andi AT, TMP3, LJ_GC_BLACK // isblack(table)
4621 | bnez AT, >7
4622 |. daddu RA, BASE, RA
4623 |2:
4624 | sltu AT, CARG3, TMP0
4625 | sll TMP2, CARG3, 3
4626 | beqz AT, ->vmeta_tsetr // In array part?
4627 |. daddu CRET1, TMP1, TMP2
4628 |->BC_TSETR_Z:
4629 | ld CARG1, 0(RA)
4630 | ins_next1
4631 | sd CARG1, 0(CRET1)
4632 | ins_next2
4633 |
4634 |7: // Possible table write barrier for the value. Skip valiswhite check.
4635 | barrierback TAB:CARG2, TMP3, CRET1, <2
4636 break;
4637
4638 case BC_TSETM:
4639 | // RA = base*8 (table at base-1), RD = num_const*8 (start index)
4640 | daddu RA, BASE, RA
4641 |1:
4642 | daddu TMP3, KBASE, RD
4643 | ld TAB:CARG2, -8(RA) // Guaranteed to be a table.
4644 | addiu TMP0, MULTRES, -8
4645 | lw TMP3, LO(TMP3) // Integer constant is in lo-word.
4646 | beqz TMP0, >4 // Nothing to copy?
4647 |. srl CARG3, TMP0, 3
4648 | cleartp CARG2
4649 | addu CARG3, CARG3, TMP3
4650 | lw TMP2, TAB:CARG2->asize
4651 | sll TMP1, TMP3, 3
4652 | lbu TMP3, TAB:CARG2->marked
4653 | ld CARG1, TAB:CARG2->array
4654 | sltu AT, TMP2, CARG3
4655 | bnez AT, >5
4656 |. daddu TMP2, RA, TMP0
4657 | daddu TMP1, TMP1, CARG1
4658 | andi TMP0, TMP3, LJ_GC_BLACK // isblack(table)
4659 |3: // Copy result slots to table.
4660 | ld CRET1, 0(RA)
4661 | daddiu RA, RA, 8
4662 | sltu AT, RA, TMP2
4663 | sd CRET1, 0(TMP1)
4664 | bnez AT, <3
4665 |. daddiu TMP1, TMP1, 8
4666 | bnez TMP0, >7
4667 |. nop
4668 |4:
4669 | ins_next
4670 |
4671 |5: // Need to resize array part.
4672 | load_got lj_tab_reasize
4673 | sd BASE, L->base
4674 | sd PC, SAVE_PC
4675 | move BASE, RD
4676 | call_intern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
4677 |. move CARG1, L
4678 | // Must not reallocate the stack.
4679 | move RD, BASE
4680 | b <1
4681 |. ld BASE, L->base // Reload BASE for lack of a saved register.
4682 |
4683 |7: // Possible table write barrier for any value. Skip valiswhite check.
4684 | barrierback TAB:CARG2, TMP3, TMP0, <4
4685 break;
4686
4687 /* -- Calls and vararg handling ----------------------------------------- */
4688
4689 case BC_CALLM:
4690 | // RA = base*8, (RB = (nresults+1)*8,) RC = extra_nargs*8
4691 | decode_RDtoRC8 NARGS8:RC, RD
4692 | b ->BC_CALL_Z
4693 |. addu NARGS8:RC, NARGS8:RC, MULTRES
4694 break;
4695 case BC_CALL:
4696 | // RA = base*8, (RB = (nresults+1)*8,) RC = (nargs+1)*8
4697 | decode_RDtoRC8 NARGS8:RC, RD
4698 |->BC_CALL_Z:
4699 | move TMP2, BASE
4700 | daddu BASE, BASE, RA
4701 | ld LFUNC:RB, 0(BASE)
4702 | daddiu BASE, BASE, 16
4703 | addiu NARGS8:RC, NARGS8:RC, -8
4704 | checkfunc RB, ->vmeta_call
4705 | ins_call
4706 break;
4707
4708 case BC_CALLMT:
4709 | // RA = base*8, (RB = 0,) RC = extra_nargs*8
4710 | addu NARGS8:RD, NARGS8:RD, MULTRES // BC_CALLT gets RC from RD.
4711 | // Fall through. Assumes BC_CALLT follows.
4712 break;
4713 case BC_CALLT:
4714 | // RA = base*8, (RB = 0,) RC = (nargs+1)*8
4715 | daddu RA, BASE, RA
4716 | ld RB, 0(RA)
4717 | move NARGS8:RC, RD
4718 | ld TMP1, FRAME_PC(BASE)
4719 | daddiu RA, RA, 16
4720 | addiu NARGS8:RC, NARGS8:RC, -8
4721 | checktp CARG3, RB, -LJ_TFUNC, ->vmeta_callt
4722 |->BC_CALLT_Z:
4723 | andi TMP0, TMP1, FRAME_TYPE // Caveat: preserve TMP0 until the 'or'.
4724 | lbu TMP3, LFUNC:CARG3->ffid
4725 | bnez TMP0, >7
4726 |. xori TMP2, TMP1, FRAME_VARG
4727 |1:
4728 | sd RB, FRAME_FUNC(BASE) // Copy function down, but keep PC.
4729 | sltiu AT, TMP3, 2 // (> FF_C) Calling a fast function?
4730 | move TMP2, BASE
4731 | move RB, CARG3
4732 | beqz NARGS8:RC, >3
4733 |. move TMP3, NARGS8:RC
4734 |2:
4735 | ld CRET1, 0(RA)
4736 | daddiu RA, RA, 8
4737 | addiu TMP3, TMP3, -8
4738 | sd CRET1, 0(TMP2)
4739 | bnez TMP3, <2
4740 |. daddiu TMP2, TMP2, 8
4741 |3:
4742 | or TMP0, TMP0, AT
4743 | beqz TMP0, >5
4744 |. nop
4745 |4:
4746 | ins_callt
4747 |
4748 |5: // Tailcall to a fast function with a Lua frame below.
4749 | lw INS, -4(TMP1)
4750 | decode_RA8a RA, INS
4751 | decode_RA8b RA
4752 | dsubu TMP1, BASE, RA
4753 | ld TMP1, -32(TMP1)
4754 | cleartp LFUNC:TMP1
4755 | ld TMP1, LFUNC:TMP1->pc
4756 | b <4
4757 |. ld KBASE, PC2PROTO(k)(TMP1) // Need to prepare KBASE.
4758 |
4759 |7: // Tailcall from a vararg function.
4760 | andi AT, TMP2, FRAME_TYPEP
4761 | bnez AT, <1 // Vararg frame below?
4762 |. dsubu TMP2, BASE, TMP2 // Relocate BASE down.
4763 | move BASE, TMP2
4764 | ld TMP1, FRAME_PC(TMP2)
4765 | b <1
4766 |. andi TMP0, TMP1, FRAME_TYPE
4767 break;
4768
4769 case BC_ITERC:
4770 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 ((2+1)*8))
4771 | move TMP2, BASE // Save old BASE fir vmeta_call.
4772 | daddu BASE, BASE, RA
4773 | ld RB, -24(BASE)
4774 | ld CARG1, -16(BASE)
4775 | ld CARG2, -8(BASE)
4776 | li NARGS8:RC, 16 // Iterators get 2 arguments.
4777 | sd RB, 0(BASE) // Copy callable.
4778 | sd CARG1, 16(BASE) // Copy state.
4779 | sd CARG2, 24(BASE) // Copy control var.
4780 | daddiu BASE, BASE, 16
4781 | checkfunc RB, ->vmeta_call
4782 | ins_call
4783 break;
4784
4785 case BC_ITERN:
4786 |.if JIT and ENDIAN_LE
4787 | hotloop
4788 |.endif
4789 |->vm_IITERN:
4790 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8)
4791 | daddu RA, BASE, RA
4792 | ld TAB:RB, -16(RA)
4793 | lw RC, -8+LO(RA) // Get index from control var.
4794 | cleartp TAB:RB
4795 | daddiu PC, PC, 4
4796 | lw TMP0, TAB:RB->asize
4797 | ld TMP1, TAB:RB->array
4798 | dsll CARG3, TISNUM, 47
4799 |1: // Traverse array part.
4800 | sltu AT, RC, TMP0
4801 | beqz AT, >5 // Index points after array part?
4802 |. sll TMP3, RC, 3
4803 | daddu TMP3, TMP1, TMP3
4804 | ld CARG1, 0(TMP3)
4805 | lhu RD, -4+OFS_RD(PC)
4806 | or TMP2, RC, CARG3
4807 | beq CARG1, TISNIL, <1 // Skip holes in array part.
4808 |. addiu RC, RC, 1
4809 | sd TMP2, 0(RA)
4810 | sd CARG1, 8(RA)
4811 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
4812 | decode_RD4b RD
4813 | daddu RD, RD, TMP3
4814 | sw RC, -8+LO(RA) // Update control var.
4815 | daddu PC, PC, RD
4816 |3:
4817 | ins_next
4818 |
4819 |5: // Traverse hash part.
4820 | lw TMP1, TAB:RB->hmask
4821 | subu RC, RC, TMP0
4822 | ld TMP2, TAB:RB->node
4823 |6:
4824 | sltu AT, TMP1, RC // End of iteration? Branch to ITERL+1.
4825 | bnez AT, <3
4826 |. sll TMP3, RC, 5
4827 | sll RB, RC, 3
4828 | subu TMP3, TMP3, RB
4829 | daddu NODE:TMP3, TMP3, TMP2
4830 | ld CARG1, 0(NODE:TMP3)
4831 | lhu RD, -4+OFS_RD(PC)
4832 | beq CARG1, TISNIL, <6 // Skip holes in hash part.
4833 |. addiu RC, RC, 1
4834 | ld CARG2, NODE:TMP3->key
4835 | lui TMP3, (-(BCBIAS_J*4 >> 16) & 65535)
4836 | sd CARG1, 8(RA)
4837 | addu RC, RC, TMP0
4838 | decode_RD4b RD
4839 | addu RD, RD, TMP3
4840 | sd CARG2, 0(RA)
4841 | daddu PC, PC, RD
4842 | b <3
4843 |. sw RC, -8+LO(RA) // Update control var.
4844 break;
4845
4846 case BC_ISNEXT:
4847 | // RA = base*8, RD = target (points to ITERN)
4848 | daddu RA, BASE, RA
4849 | srl TMP0, RD, 1
4850 | ld CFUNC:CARG1, -24(RA)
4851 | daddu TMP0, PC, TMP0
4852 | ld CARG2, -16(RA)
4853 | ld CARG3, -8(RA)
4854 | lui TMP2, (-(BCBIAS_J*4 >> 16) & 65535)
4855 | checkfunc CFUNC:CARG1, >5
4856 | gettp CARG2, CARG2
4857 | daddiu CARG2, CARG2, -LJ_TTAB
4858 | lbu TMP1, CFUNC:CARG1->ffid
4859 | daddiu CARG3, CARG3, -LJ_TNIL
4860 | or AT, CARG2, CARG3
4861 | daddiu TMP1, TMP1, -FF_next_N
4862 | or AT, AT, TMP1
4863 | bnez AT, >5
4864 |. lui TMP1, (LJ_KEYINDEX >> 16)
4865 | daddu PC, TMP0, TMP2
4866 | ori TMP1, TMP1, (LJ_KEYINDEX & 0xffff)
4867 | dsll TMP1, TMP1, 32
4868 | sd TMP1, -8(RA)
4869 |1:
4870 | ins_next
4871 |5: // Despecialize bytecode if any of the checks fail.
4872 | li TMP3, BC_JMP
4873 | li TMP1, BC_ITERC
4874 | sb TMP3, -4+OFS_OP(PC)
4875 | daddu PC, TMP0, TMP2
4876 |.if JIT
4877 | lb TMP0, OFS_OP(PC)
4878 | li AT, BC_ITERN
4879 | bne TMP0, AT, >6
4880 |. lhu TMP2, OFS_RD(PC)
4881 |.endif
4882 | b <1
4883 |. sb TMP1, OFS_OP(PC)
4884 |.if JIT
4885 |6: // Unpatch JLOOP.
4886 | ld TMP0, DISPATCH_J(trace)(DISPATCH)
4887 | sll TMP2, TMP2, 3
4888 | daddu TMP0, TMP0, TMP2
4889 | ld TRACE:TMP2, 0(TMP0)
4890 | lw TMP0, TRACE:TMP2->startins
4891 | li AT, -256
4892 | and TMP0, TMP0, AT
4893 | or TMP0, TMP0, TMP1
4894 | b <1
4895 |. sw TMP0, 0(PC)
4896 |.endif
4897 break;
4898
4899 case BC_VARG:
4900 | // RA = base*8, RB = (nresults+1)*8, RC = numparams*8
4901 | ld TMP0, FRAME_PC(BASE)
4902 | decode_RDtoRC8 RC, RD
4903 | decode_RB8a RB, INS
4904 | daddu RC, BASE, RC
4905 | decode_RB8b RB
4906 | daddu RA, BASE, RA
4907 | daddiu RC, RC, FRAME_VARG
4908 | daddu TMP2, RA, RB
4909 | daddiu TMP3, BASE, -16 // TMP3 = vtop
4910 | dsubu RC, RC, TMP0 // RC = vbase
4911 | // Note: RC may now be even _above_ BASE if nargs was < numparams.
4912 | beqz RB, >5 // Copy all varargs?
4913 |. dsubu TMP1, TMP3, RC
4914 | daddiu TMP2, TMP2, -16
4915 |1: // Copy vararg slots to destination slots.
4916 | ld CARG1, 0(RC)
4917 | sltu AT, RC, TMP3
4918 | daddiu RC, RC, 8
4919 |.if MIPSR6
4920 | selnez CARG1, CARG1, AT
4921 | seleqz AT, TISNIL, AT
4922 | or CARG1, CARG1, AT
4923 |.else
4924 | movz CARG1, TISNIL, AT
4925 |.endif
4926 | sd CARG1, 0(RA)
4927 | sltu AT, RA, TMP2
4928 | bnez AT, <1
4929 |. daddiu RA, RA, 8
4930 |3:
4931 | ins_next
4932 |
4933 |5: // Copy all varargs.
4934 | ld TMP0, L->maxstack
4935 | blez TMP1, <3 // No vararg slots?
4936 |. li MULTRES, 8 // MULTRES = (0+1)*8
4937 | daddu TMP2, RA, TMP1
4938 | sltu AT, TMP0, TMP2
4939 | bnez AT, >7
4940 |. daddiu MULTRES, TMP1, 8
4941 |6:
4942 | ld CRET1, 0(RC)
4943 | daddiu RC, RC, 8
4944 | sd CRET1, 0(RA)
4945 | sltu AT, RC, TMP3
4946 | bnez AT, <6 // More vararg slots?
4947 |. daddiu RA, RA, 8
4948 | b <3
4949 |. nop
4950 |
4951 |7: // Grow stack for varargs.
4952 | load_got lj_state_growstack
4953 | sd RA, L->top
4954 | dsubu RA, RA, BASE
4955 | sd BASE, L->base
4956 | dsubu BASE, RC, BASE // Need delta, because BASE may change.
4957 | sd PC, SAVE_PC
4958 | srl CARG2, TMP1, 3
4959 | call_intern lj_state_growstack // (lua_State *L, int n)
4960 |. move CARG1, L
4961 | move RC, BASE
4962 | ld BASE, L->base
4963 | daddu RA, BASE, RA
4964 | daddu RC, BASE, RC
4965 | b <6
4966 |. daddiu TMP3, BASE, -16
4967 break;
4968
4969 /* -- Returns ----------------------------------------------------------- */
4970
4971 case BC_RETM:
4972 | // RA = results*8, RD = extra_nresults*8
4973 | addu RD, RD, MULTRES // MULTRES >= 8, so RD >= 8.
4974 | // Fall through. Assumes BC_RET follows.
4975 break;
4976
4977 case BC_RET:
4978 | // RA = results*8, RD = (nresults+1)*8
4979 | ld PC, FRAME_PC(BASE)
4980 | daddu RA, BASE, RA
4981 | move MULTRES, RD
4982 |1:
4983 | andi TMP0, PC, FRAME_TYPE
4984 | bnez TMP0, ->BC_RETV_Z
4985 |. xori TMP1, PC, FRAME_VARG
4986 |
4987 |->BC_RET_Z:
4988 | // BASE = base, RA = resultptr, RD = (nresults+1)*8, PC = return
4989 | lw INS, -4(PC)
4990 | daddiu TMP2, BASE, -16
4991 | daddiu RC, RD, -8
4992 | decode_RA8a TMP0, INS
4993 | decode_RB8a RB, INS
4994 | decode_RA8b TMP0
4995 | decode_RB8b RB
4996 | daddu TMP3, TMP2, RB
4997 | beqz RC, >3
4998 |. dsubu BASE, TMP2, TMP0
4999 |2:
5000 | ld CRET1, 0(RA)
5001 | daddiu RA, RA, 8
5002 | daddiu RC, RC, -8
5003 | sd CRET1, 0(TMP2)
5004 | bnez RC, <2
5005 |. daddiu TMP2, TMP2, 8
5006 |3:
5007 | daddiu TMP3, TMP3, -8
5008 |5:
5009 | sltu AT, TMP2, TMP3
5010 | bnez AT, >6
5011 |. ld LFUNC:TMP1, FRAME_FUNC(BASE)
5012 | ins_next1
5013 | cleartp LFUNC:TMP1
5014 | ld TMP1, LFUNC:TMP1->pc
5015 | ld KBASE, PC2PROTO(k)(TMP1)
5016 | ins_next2
5017 |
5018 |6: // Fill up results with nil.
5019 | sd TISNIL, 0(TMP2)
5020 | b <5
5021 |. daddiu TMP2, TMP2, 8
5022 |
5023 |->BC_RETV_Z: // Non-standard return case.
5024 | andi TMP2, TMP1, FRAME_TYPEP
5025 | bnez TMP2, ->vm_return
5026 |. nop
5027 | // Return from vararg function: relocate BASE down.
5028 | dsubu BASE, BASE, TMP1
5029 | b <1
5030 |. ld PC, FRAME_PC(BASE)
5031 break;
5032
5033 case BC_RET0: case BC_RET1:
5034 | // RA = results*8, RD = (nresults+1)*8
5035 | ld PC, FRAME_PC(BASE)
5036 | daddu RA, BASE, RA
5037 | move MULTRES, RD
5038 | andi TMP0, PC, FRAME_TYPE
5039 | bnez TMP0, ->BC_RETV_Z
5040 |. xori TMP1, PC, FRAME_VARG
5041 | lw INS, -4(PC)
5042 | daddiu TMP2, BASE, -16
5043 if (op == BC_RET1) {
5044 | ld CRET1, 0(RA)
5045 }
5046 | decode_RB8a RB, INS
5047 | decode_RA8a RA, INS
5048 | decode_RB8b RB
5049 | decode_RA8b RA
5050 | dsubu BASE, TMP2, RA
5051 if (op == BC_RET1) {
5052 | sd CRET1, 0(TMP2)
5053 }
5054 |5:
5055 | sltu AT, RD, RB
5056 | bnez AT, >6
5057 |. ld TMP1, FRAME_FUNC(BASE)
5058 | ins_next1
5059 | cleartp LFUNC:TMP1
5060 | ld TMP1, LFUNC:TMP1->pc
5061 | ld KBASE, PC2PROTO(k)(TMP1)
5062 | ins_next2
5063 |
5064 |6: // Fill up results with nil.
5065 | daddiu TMP2, TMP2, 8
5066 | daddiu RD, RD, 8
5067 | b <5
5068 if (op == BC_RET1) {
5069 |. sd TISNIL, 0(TMP2)
5070 } else {
5071 |. sd TISNIL, -8(TMP2)
5072 }
5073 break;
5074
5075 /* -- Loops and branches ------------------------------------------------ */
5076
5077 case BC_FORL:
5078 |.if JIT
5079 | hotloop
5080 |.endif
5081 | // Fall through. Assumes BC_IFORL follows.
5082 break;
5083
5084 case BC_JFORI:
5085 case BC_JFORL:
5086#if !LJ_HASJIT
5087 break;
5088#endif
5089 case BC_FORI:
5090 case BC_IFORL:
5091 | // RA = base*8, RD = target (after end of loop or start of loop)
5092 vk = (op == BC_IFORL || op == BC_JFORL);
5093 | daddu RA, BASE, RA
5094 | ld CARG1, FORL_IDX*8(RA) // IDX CARG1 - CARG3 type
5095 | gettp CARG3, CARG1
5096 if (op != BC_JFORL) {
5097 | srl RD, RD, 1
5098 | lui TMP2, (-(BCBIAS_J*4 >> 16) & 65535)
5099 | daddu TMP2, RD, TMP2
5100 }
5101 if (!vk) {
5102 | ld CARG2, FORL_STOP*8(RA) // STOP CARG2 - CARG4 type
5103 | ld CRET1, FORL_STEP*8(RA) // STEP CRET1 - CRET2 type
5104 | gettp CARG4, CARG2
5105 | bne CARG3, TISNUM, >5
5106 |. gettp CRET2, CRET1
5107 | bne CARG4, TISNUM, ->vmeta_for
5108 |. sextw CARG3, CARG1
5109 | bne CRET2, TISNUM, ->vmeta_for
5110 |. sextw CARG2, CARG2
5111 | dext AT, CRET1, 31, 0
5112 | slt CRET1, CARG2, CARG3
5113 | slt TMP1, CARG3, CARG2
5114 |.if MIPSR6
5115 | selnez TMP1, TMP1, AT
5116 | seleqz CRET1, CRET1, AT
5117 | or CRET1, CRET1, TMP1
5118 |.else
5119 | movn CRET1, TMP1, AT
5120 |.endif
5121 } else {
5122 | bne CARG3, TISNUM, >5
5123 |. ld CARG2, FORL_STEP*8(RA) // STEP CARG2 - CARG4 type
5124 | ld CRET1, FORL_STOP*8(RA) // STOP CRET1 - CRET2 type
5125 | sextw TMP3, CARG1
5126 | sextw CARG2, CARG2
5127 | sextw CRET1, CRET1
5128 | addu CARG1, TMP3, CARG2
5129 | xor TMP0, CARG1, TMP3
5130 | xor TMP1, CARG1, CARG2
5131 | and TMP0, TMP0, TMP1
5132 | slt TMP1, CARG1, CRET1
5133 | slt CRET1, CRET1, CARG1
5134 | slt AT, CARG2, r0
5135 | slt TMP0, TMP0, r0 // ((y^a) & (y^b)) < 0: overflow.
5136 |.if MIPSR6
5137 | selnez TMP1, TMP1, AT
5138 | seleqz CRET1, CRET1, AT
5139 | or CRET1, CRET1, TMP1
5140 |.else
5141 | movn CRET1, TMP1, AT
5142 |.endif
5143 | or CRET1, CRET1, TMP0
5144 | zextw CARG1, CARG1
5145 | settp CARG1, TISNUM
5146 }
5147 |1:
5148 if (op == BC_FORI) {
5149 |.if MIPSR6
5150 | selnez TMP2, TMP2, CRET1
5151 |.else
5152 | movz TMP2, r0, CRET1
5153 |.endif
5154 | daddu PC, PC, TMP2
5155 } else if (op == BC_JFORI) {
5156 | daddu PC, PC, TMP2
5157 | lhu RD, -4+OFS_RD(PC)
5158 } else if (op == BC_IFORL) {
5159 |.if MIPSR6
5160 | seleqz TMP2, TMP2, CRET1
5161 |.else
5162 | movn TMP2, r0, CRET1
5163 |.endif
5164 | daddu PC, PC, TMP2
5165 }
5166 if (vk) {
5167 | sd CARG1, FORL_IDX*8(RA)
5168 }
5169 | ins_next1
5170 | sd CARG1, FORL_EXT*8(RA)
5171 |2:
5172 if (op == BC_JFORI) {
5173 | beqz CRET1, =>BC_JLOOP
5174 |. decode_RD8b RD
5175 } else if (op == BC_JFORL) {
5176 | beqz CRET1, =>BC_JLOOP
5177 }
5178 | ins_next2
5179 |
5180 |5: // FP loop.
5181 |.if FPU
5182 if (!vk) {
5183 | ldc1 f0, FORL_IDX*8(RA)
5184 | ldc1 f2, FORL_STOP*8(RA)
5185 | sltiu TMP0, CARG3, LJ_TISNUM
5186 | sltiu TMP1, CARG4, LJ_TISNUM
5187 | sltiu AT, CRET2, LJ_TISNUM
5188 | ld TMP3, FORL_STEP*8(RA)
5189 | and TMP0, TMP0, TMP1
5190 | and AT, AT, TMP0
5191 | beqz AT, ->vmeta_for
5192 |. slt TMP3, TMP3, r0
5193 |.if MIPSR6
5194 | dmtc1 TMP3, FTMP2
5195 | cmp.lt.d FTMP0, f0, f2
5196 | cmp.lt.d FTMP1, f2, f0
5197 | sel.d FTMP2, FTMP1, FTMP0
5198 | b <1
5199 |. dmfc1 CRET1, FTMP2
5200 |.else
5201 | c.ole.d 0, f0, f2
5202 | c.ole.d 1, f2, f0
5203 | li CRET1, 1
5204 | movt CRET1, r0, 0
5205 | movt AT, r0, 1
5206 | b <1
5207 |. movn CRET1, AT, TMP3
5208 |.endif
5209 } else {
5210 | ldc1 f0, FORL_IDX*8(RA)
5211 | ldc1 f4, FORL_STEP*8(RA)
5212 | ldc1 f2, FORL_STOP*8(RA)
5213 | ld TMP3, FORL_STEP*8(RA)
5214 | add.d f0, f0, f4
5215 |.if MIPSR6
5216 | slt TMP3, TMP3, r0
5217 | dmtc1 TMP3, FTMP2
5218 | cmp.lt.d FTMP0, f0, f2
5219 | cmp.lt.d FTMP1, f2, f0
5220 | sel.d FTMP2, FTMP1, FTMP0
5221 | dmfc1 CRET1, FTMP2
5222 if (op == BC_IFORL) {
5223 | seleqz TMP2, TMP2, CRET1
5224 | daddu PC, PC, TMP2
5225 }
5226 |.else
5227 | c.ole.d 0, f0, f2
5228 | c.ole.d 1, f2, f0
5229 | slt TMP3, TMP3, r0
5230 | li CRET1, 1
5231 | li AT, 1
5232 | movt CRET1, r0, 0
5233 | movt AT, r0, 1
5234 | movn CRET1, AT, TMP3
5235 if (op == BC_IFORL) {
5236 | movn TMP2, r0, CRET1
5237 | daddu PC, PC, TMP2
5238 }
5239 |.endif
5240 | sdc1 f0, FORL_IDX*8(RA)
5241 | ins_next1
5242 | b <2
5243 |. sdc1 f0, FORL_EXT*8(RA)
5244 }
5245 |.else
5246 if (!vk) {
5247 | sltiu TMP0, CARG3, LJ_TISNUM
5248 | sltiu TMP1, CARG4, LJ_TISNUM
5249 | sltiu AT, CRET2, LJ_TISNUM
5250 | and TMP0, TMP0, TMP1
5251 | and AT, AT, TMP0
5252 | beqz AT, ->vmeta_for
5253 |. nop
5254 | bal ->vm_sfcmpolex
5255 |. lw TMP3, FORL_STEP*8+HI(RA)
5256 | b <1
5257 |. nop
5258 } else {
5259 | load_got __adddf3
5260 | call_extern
5261 |. sw TMP2, TMPD
5262 | ld CARG2, FORL_STOP*8(RA)
5263 | move CARG1, CRET1
5264 if ( op == BC_JFORL ) {
5265 | lhu RD, -4+OFS_RD(PC)
5266 | decode_RD8b RD
5267 }
5268 | bal ->vm_sfcmpolex
5269 |. lw TMP3, FORL_STEP*8+HI(RA)
5270 | b <1
5271 |. lw TMP2, TMPD
5272 }
5273 |.endif
5274 break;
5275
5276 case BC_ITERL:
5277 |.if JIT
5278 | hotloop
5279 |.endif
5280 | // Fall through. Assumes BC_IITERL follows.
5281 break;
5282
5283 case BC_JITERL:
5284#if !LJ_HASJIT
5285 break;
5286#endif
5287 case BC_IITERL:
5288 | // RA = base*8, RD = target
5289 | daddu RA, BASE, RA
5290 | ld TMP1, 0(RA)
5291 | beq TMP1, TISNIL, >1 // Stop if iterator returned nil.
5292 |. nop
5293 if (op == BC_JITERL) {
5294 | b =>BC_JLOOP
5295 |. sd TMP1, -8(RA)
5296 } else {
5297 | branch_RD // Otherwise save control var + branch.
5298 | sd TMP1, -8(RA)
5299 }
5300 |1:
5301 | ins_next
5302 break;
5303
5304 case BC_LOOP:
5305 | // RA = base*8, RD = target (loop extent)
5306 | // Note: RA/RD is only used by trace recorder to determine scope/extent
5307 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
5308 |.if JIT
5309 | hotloop
5310 |.endif
5311 | // Fall through. Assumes BC_ILOOP follows.
5312 break;
5313
5314 case BC_ILOOP:
5315 | // RA = base*8, RD = target (loop extent)
5316 | ins_next
5317 break;
5318
5319 case BC_JLOOP:
5320 |.if JIT
5321 | // RA = base*8 (ignored), RD = traceno*8
5322 | ld TMP1, DISPATCH_J(trace)(DISPATCH)
5323 | li AT, 0
5324 | daddu TMP1, TMP1, RD
5325 | // Traces on MIPS don't store the trace number, so use 0.
5326 | sd AT, DISPATCH_GL(vmstate)(DISPATCH)
5327 | ld TRACE:TMP2, 0(TMP1)
5328 | sd BASE, DISPATCH_GL(jit_base)(DISPATCH)
5329 | ld TMP2, TRACE:TMP2->mcode
5330 | sd L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
5331 | jr TMP2
5332 |. daddiu JGL, DISPATCH, GG_DISP2G+32768
5333 |.endif
5334 break;
5335
5336 case BC_JMP:
5337 | // RA = base*8 (only used by trace recorder), RD = target
5338 | branch_RD
5339 | ins_next
5340 break;
5341
5342 /* -- Function headers -------------------------------------------------- */
5343
5344 case BC_FUNCF:
5345 |.if JIT
5346 | hotcall
5347 |.endif
5348 case BC_FUNCV: /* NYI: compiled vararg functions. */
5349 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
5350 break;
5351
5352 case BC_JFUNCF:
5353#if !LJ_HASJIT
5354 break;
5355#endif
5356 case BC_IFUNCF:
5357 | // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
5358 | ld TMP2, L->maxstack
5359 | lbu TMP1, -4+PC2PROTO(numparams)(PC)
5360 | ld KBASE, -4+PC2PROTO(k)(PC)
5361 | sltu AT, TMP2, RA
5362 | bnez AT, ->vm_growstack_l
5363 |. sll TMP1, TMP1, 3
5364 if (op != BC_JFUNCF) {
5365 | ins_next1
5366 }
5367 |2:
5368 | sltu AT, NARGS8:RC, TMP1 // Check for missing parameters.
5369 | bnez AT, >3
5370 |. daddu AT, BASE, NARGS8:RC
5371 if (op == BC_JFUNCF) {
5372 | decode_RD8a RD, INS
5373 | b =>BC_JLOOP
5374 |. decode_RD8b RD
5375 } else {
5376 | ins_next2
5377 }
5378 |
5379 |3: // Clear missing parameters.
5380 | sd TISNIL, 0(AT)
5381 | b <2
5382 |. addiu NARGS8:RC, NARGS8:RC, 8
5383 break;
5384
5385 case BC_JFUNCV:
5386#if !LJ_HASJIT
5387 break;
5388#endif
5389 | NYI // NYI: compiled vararg functions
5390 break; /* NYI: compiled vararg functions. */
5391
5392 case BC_IFUNCV:
5393 | // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
5394 | li TMP0, LJ_TFUNC
5395 | daddu TMP1, BASE, RC
5396 | ld TMP2, L->maxstack
5397 | settp LFUNC:RB, TMP0
5398 | daddu TMP0, RA, RC
5399 | sd LFUNC:RB, 0(TMP1) // Store (tagged) copy of LFUNC.
5400 | daddiu TMP3, RC, 16+FRAME_VARG
5401 | sltu AT, TMP0, TMP2
5402 | ld KBASE, -4+PC2PROTO(k)(PC)
5403 | beqz AT, ->vm_growstack_l
5404 |. sd TMP3, 8(TMP1) // Store delta + FRAME_VARG.
5405 | lbu TMP2, -4+PC2PROTO(numparams)(PC)
5406 | move RA, BASE
5407 | move RC, TMP1
5408 | ins_next1
5409 | beqz TMP2, >3
5410 |. daddiu BASE, TMP1, 16
5411 |1:
5412 | ld TMP0, 0(RA)
5413 | sltu AT, RA, RC // Less args than parameters?
5414 | move CARG1, TMP0
5415 |.if MIPSR6
5416 | selnez TMP0, TMP0, AT
5417 | seleqz TMP3, TISNIL, AT
5418 | or TMP0, TMP0, TMP3
5419 | seleqz TMP3, CARG1, AT
5420 | selnez CARG1, TISNIL, AT
5421 | or CARG1, CARG1, TMP3
5422 |.else
5423 | movz TMP0, TISNIL, AT // Clear missing parameters.
5424 | movn CARG1, TISNIL, AT // Clear old fixarg slot (help the GC).
5425 |.endif
5426 | addiu TMP2, TMP2, -1
5427 | sd TMP0, 16(TMP1)
5428 | daddiu TMP1, TMP1, 8
5429 | sd CARG1, 0(RA)
5430 | bnez TMP2, <1
5431 |. daddiu RA, RA, 8
5432 |3:
5433 | ins_next2
5434 break;
5435
5436 case BC_FUNCC:
5437 case BC_FUNCCW:
5438 | // BASE = new base, RA = BASE+framesize*8, RB = CFUNC, RC = nargs*8
5439 if (op == BC_FUNCC) {
5440 | ld CFUNCADDR, CFUNC:RB->f
5441 } else {
5442 | ld CFUNCADDR, DISPATCH_GL(wrapf)(DISPATCH)
5443 }
5444 | daddu TMP1, RA, NARGS8:RC
5445 | ld TMP2, L->maxstack
5446 | daddu RC, BASE, NARGS8:RC
5447 | sd BASE, L->base
5448 | sltu AT, TMP2, TMP1
5449 | sd RC, L->top
5450 | li_vmstate C
5451 if (op == BC_FUNCCW) {
5452 | ld CARG2, CFUNC:RB->f
5453 }
5454 | bnez AT, ->vm_growstack_c // Need to grow stack.
5455 |. move CARG1, L
5456 | jalr CFUNCADDR // (lua_State *L [, lua_CFunction f])
5457 |. st_vmstate
5458 | // Returns nresults.
5459 | ld BASE, L->base
5460 | sll RD, CRET1, 3
5461 | ld TMP1, L->top
5462 | li_vmstate INTERP
5463 | ld PC, FRAME_PC(BASE) // Fetch PC of caller.
5464 | dsubu RA, TMP1, RD // RA = L->top - nresults*8
5465 | sd L, DISPATCH_GL(cur_L)(DISPATCH)
5466 | b ->vm_returnc
5467 |. st_vmstate
5468 break;
5469
5470 /* ---------------------------------------------------------------------- */
5471
5472 default:
5473 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
5474 exit(2);
5475 break;
5476 }
5477}
5478
5479static int build_backend(BuildCtx *ctx)
5480{
5481 int op;
5482
5483 dasm_growpc(Dst, BC__MAX);
5484
5485 build_subroutines(ctx);
5486
5487 |.code_op
5488 for (op = 0; op < BC__MAX; op++)
5489 build_ins(ctx, (BCOp)op, op);
5490
5491 return BC__MAX;
5492}
5493
5494/* Emit pseudo frame-info for all assembler functions. */
5495static void emit_asm_debug(BuildCtx *ctx)
5496{
5497 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
5498 int i;
5499 switch (ctx->mode) {
5500 case BUILD_elfasm:
5501 fprintf(ctx->fp, "\t.section .debug_frame,\"\",@progbits\n");
5502 fprintf(ctx->fp,
5503 ".Lframe0:\n"
5504 "\t.4byte .LECIE0-.LSCIE0\n"
5505 ".LSCIE0:\n"
5506 "\t.4byte 0xffffffff\n"
5507 "\t.byte 0x1\n"
5508 "\t.string \"\"\n"
5509 "\t.uleb128 0x1\n"
5510 "\t.sleb128 -4\n"
5511 "\t.byte 31\n"
5512 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 0\n"
5513 "\t.align 2\n"
5514 ".LECIE0:\n\n");
5515 fprintf(ctx->fp,
5516 ".LSFDE0:\n"
5517 "\t.4byte .LEFDE0-.LASFDE0\n"
5518 ".LASFDE0:\n"
5519 "\t.4byte .Lframe0\n"
5520 "\t.8byte .Lbegin\n"
5521 "\t.8byte %d\n"
5522 "\t.byte 0xe\n\t.uleb128 %d\n"
5523 "\t.byte 0x9f\n\t.sleb128 2*5\n"
5524 "\t.byte 0x9e\n\t.sleb128 2*6\n",
5525 fcofs, CFRAME_SIZE);
5526 for (i = 23; i >= 16; i--)
5527 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 2*(30-i));
5528#if !LJ_SOFTFP
5529 for (i = 31; i >= 24; i--)
5530 fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+32+i, 2*(46-i));
5531#endif
5532 fprintf(ctx->fp,
5533 "\t.align 2\n"
5534 ".LEFDE0:\n\n");
5535#if LJ_HASFFI
5536 fprintf(ctx->fp,
5537 ".LSFDE1:\n"
5538 "\t.4byte .LEFDE1-.LASFDE1\n"
5539 ".LASFDE1:\n"
5540 "\t.4byte .Lframe0\n"
5541 "\t.4byte lj_vm_ffi_call\n"
5542 "\t.4byte %d\n"
5543 "\t.byte 0x9f\n\t.uleb128 2*1\n"
5544 "\t.byte 0x90\n\t.uleb128 2*2\n"
5545 "\t.byte 0xd\n\t.uleb128 0x10\n"
5546 "\t.align 2\n"
5547 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
5548#endif
5549#if !LJ_NO_UNWIND
5550 /* NYI */
5551#endif
5552 break;
5553 default:
5554 break;
5555 }
5556}
5557
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index 61ebbb04..73a70a00 100644
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -1,4 +1,4 @@
1|// Low-level VM code for PowerPC CPUs. 1|// Low-level VM code for PowerPC 32 bit or 32on64 bit mode.
2|// Bytecode interpreter, fast functions and helper functions. 2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h 3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4| 4|
@@ -18,7 +18,6 @@
18|// DynASM defines used by the PPC port: 18|// DynASM defines used by the PPC port:
19|// 19|//
20|// P64 64 bit pointers (only for GPR64 testing). 20|// P64 64 bit pointers (only for GPR64 testing).
21|// Note: a full PPC64 _LP64 port is not planned.
22|// GPR64 64 bit registers (but possibly 32 bit pointers, e.g. PS3). 21|// GPR64 64 bit registers (but possibly 32 bit pointers, e.g. PS3).
23|// Affects reg saves, stack layout, carry/overflow/dot flags etc. 22|// Affects reg saves, stack layout, carry/overflow/dot flags etc.
24|// FRAME32 Use 32 bit frame layout, even with GPR64 (Xbox 360). 23|// FRAME32 Use 32 bit frame layout, even with GPR64 (Xbox 360).
@@ -103,6 +102,18 @@
103|// Fixed register assignments for the interpreter. 102|// Fixed register assignments for the interpreter.
104|// Don't use: r1 = sp, r2 and r13 = reserved (TOC, TLS or SDATA) 103|// Don't use: r1 = sp, r2 and r13 = reserved (TOC, TLS or SDATA)
105| 104|
105|.macro .FPU, a, b
106|.if FPU
107| a, b
108|.endif
109|.endmacro
110|
111|.macro .FPU, a, b, c
112|.if FPU
113| a, b, c
114|.endif
115|.endmacro
116|
106|// The following must be C callee-save (but BASE is often refetched). 117|// The following must be C callee-save (but BASE is often refetched).
107|.define BASE, r14 // Base of current Lua stack frame. 118|.define BASE, r14 // Base of current Lua stack frame.
108|.define KBASE, r15 // Constants of current Lua function. 119|.define KBASE, r15 // Constants of current Lua function.
@@ -116,8 +127,10 @@
116|.define TISNUM, r22 127|.define TISNUM, r22
117|.define TISNIL, r23 128|.define TISNIL, r23
118|.define ZERO, r24 129|.define ZERO, r24
130|.if FPU
119|.define TOBIT, f30 // 2^52 + 2^51. 131|.define TOBIT, f30 // 2^52 + 2^51.
120|.define TONUM, f31 // 2^52 + 2^51 + 2^31. 132|.define TONUM, f31 // 2^52 + 2^51 + 2^31.
133|.endif
121| 134|
122|// The following temporaries are not saved across C calls, except for RA. 135|// The following temporaries are not saved across C calls, except for RA.
123|.define RA, r20 // Callee-save. 136|.define RA, r20 // Callee-save.
@@ -133,6 +146,7 @@
133| 146|
134|// Saved temporaries. 147|// Saved temporaries.
135|.define SAVE0, r21 148|.define SAVE0, r21
149|.define SAVE1, r25
136| 150|
137|// Calling conventions. 151|// Calling conventions.
138|.define CARG1, r3 152|.define CARG1, r3
@@ -141,8 +155,10 @@
141|.define CARG4, r6 // Overlaps TMP3. 155|.define CARG4, r6 // Overlaps TMP3.
142|.define CARG5, r7 // Overlaps INS. 156|.define CARG5, r7 // Overlaps INS.
143| 157|
158|.if FPU
144|.define FARG1, f1 159|.define FARG1, f1
145|.define FARG2, f2 160|.define FARG2, f2
161|.endif
146| 162|
147|.define CRET1, r3 163|.define CRET1, r3
148|.define CRET2, r4 164|.define CRET2, r4
@@ -213,10 +229,16 @@
213|.endif 229|.endif
214|.else 230|.else
215| 231|
232|.if FPU
216|.define SAVE_LR, 276(sp) 233|.define SAVE_LR, 276(sp)
217|.define CFRAME_SPACE, 272 // Delta for sp. 234|.define CFRAME_SPACE, 272 // Delta for sp.
218|// Back chain for sp: 272(sp) <-- sp entering interpreter 235|// Back chain for sp: 272(sp) <-- sp entering interpreter
219|.define SAVE_FPR_, 128 // .. 128+18*8: 64 bit FPR saves. 236|.define SAVE_FPR_, 128 // .. 128+18*8: 64 bit FPR saves.
237|.else
238|.define SAVE_LR, 132(sp)
239|.define CFRAME_SPACE, 128 // Delta for sp.
240|// Back chain for sp: 128(sp) <-- sp entering interpreter
241|.endif
220|.define SAVE_GPR_, 56 // .. 56+18*4: 32 bit GPR saves. 242|.define SAVE_GPR_, 56 // .. 56+18*4: 32 bit GPR saves.
221|.define SAVE_CR, 52(sp) // 32 bit CR save. 243|.define SAVE_CR, 52(sp) // 32 bit CR save.
222|.define SAVE_ERRF, 48(sp) // 32 bit C frame info. 244|.define SAVE_ERRF, 48(sp) // 32 bit C frame info.
@@ -226,16 +248,25 @@
226|.define SAVE_PC, 32(sp) 248|.define SAVE_PC, 32(sp)
227|.define SAVE_MULTRES, 28(sp) 249|.define SAVE_MULTRES, 28(sp)
228|.define UNUSED1, 24(sp) 250|.define UNUSED1, 24(sp)
251|.if FPU
229|.define TMPD_LO, 20(sp) 252|.define TMPD_LO, 20(sp)
230|.define TMPD_HI, 16(sp) 253|.define TMPD_HI, 16(sp)
231|.define TONUM_LO, 12(sp) 254|.define TONUM_LO, 12(sp)
232|.define TONUM_HI, 8(sp) 255|.define TONUM_HI, 8(sp)
256|.else
257|.define SFSAVE_4, 20(sp)
258|.define SFSAVE_3, 16(sp)
259|.define SFSAVE_2, 12(sp)
260|.define SFSAVE_1, 8(sp)
261|.endif
233|// Next frame lr: 4(sp) 262|// Next frame lr: 4(sp)
234|// Back chain for sp: 0(sp) <-- sp while in interpreter 263|// Back chain for sp: 0(sp) <-- sp while in interpreter
235| 264|
265|.if FPU
236|.define TMPD_BLO, 23(sp) 266|.define TMPD_BLO, 23(sp)
237|.define TMPD, TMPD_HI 267|.define TMPD, TMPD_HI
238|.define TONUM_D, TONUM_HI 268|.define TONUM_D, TONUM_HI
269|.endif
239| 270|
240|.endif 271|.endif
241| 272|
@@ -245,7 +276,7 @@
245|.else 276|.else
246| stw r..reg, SAVE_GPR_+(reg-14)*4(sp) 277| stw r..reg, SAVE_GPR_+(reg-14)*4(sp)
247|.endif 278|.endif
248| stfd f..reg, SAVE_FPR_+(reg-14)*8(sp) 279| .FPU stfd f..reg, SAVE_FPR_+(reg-14)*8(sp)
249|.endmacro 280|.endmacro
250|.macro rest_, reg 281|.macro rest_, reg
251|.if GPR64 282|.if GPR64
@@ -253,7 +284,7 @@
253|.else 284|.else
254| lwz r..reg, SAVE_GPR_+(reg-14)*4(sp) 285| lwz r..reg, SAVE_GPR_+(reg-14)*4(sp)
255|.endif 286|.endif
256| lfd f..reg, SAVE_FPR_+(reg-14)*8(sp) 287| .FPU lfd f..reg, SAVE_FPR_+(reg-14)*8(sp)
257|.endmacro 288|.endmacro
258| 289|
259|.macro saveregs 290|.macro saveregs
@@ -316,19 +347,14 @@
316|.type NODE, Node 347|.type NODE, Node
317|.type NARGS8, int 348|.type NARGS8, int
318|.type TRACE, GCtrace 349|.type TRACE, GCtrace
350|.type SBUF, SBuf
319| 351|
320|//----------------------------------------------------------------------- 352|//-----------------------------------------------------------------------
321| 353|
322|// These basic macros should really be part of DynASM.
323|.macro srwi, rx, ry, n; rlwinm rx, ry, 32-n, n, 31; .endmacro
324|.macro slwi, rx, ry, n; rlwinm rx, ry, n, 0, 31-n; .endmacro
325|.macro rotlwi, rx, ry, n; rlwinm rx, ry, n, 0, 31; .endmacro
326|.macro rotlw, rx, ry, rn; rlwnm rx, ry, rn, 0, 31; .endmacro
327|.macro subi, rx, ry, i; addi rx, ry, -i; .endmacro
328|
329|// Trap for not-yet-implemented parts. 354|// Trap for not-yet-implemented parts.
330|.macro NYI; tw 4, sp, sp; .endmacro 355|.macro NYI; tw 4, sp, sp; .endmacro
331| 356|
357|.if FPU
332|// int/FP conversions. 358|// int/FP conversions.
333|.macro tonum_i, freg, reg 359|.macro tonum_i, freg, reg
334| xoris reg, reg, 0x8000 360| xoris reg, reg, 0x8000
@@ -352,6 +378,7 @@
352|.macro toint, reg, freg 378|.macro toint, reg, freg
353| toint reg, freg, freg 379| toint reg, freg, freg
354|.endmacro 380|.endmacro
381|.endif
355| 382|
356|//----------------------------------------------------------------------- 383|//-----------------------------------------------------------------------
357| 384|
@@ -539,9 +566,19 @@ static void build_subroutines(BuildCtx *ctx)
539 | beq >2 566 | beq >2
540 |1: 567 |1:
541 | addic. TMP1, TMP1, -8 568 | addic. TMP1, TMP1, -8
569 |.if FPU
542 | lfd f0, 0(RA) 570 | lfd f0, 0(RA)
571 |.else
572 | lwz CARG1, 0(RA)
573 | lwz CARG2, 4(RA)
574 |.endif
543 | addi RA, RA, 8 575 | addi RA, RA, 8
576 |.if FPU
544 | stfd f0, 0(BASE) 577 | stfd f0, 0(BASE)
578 |.else
579 | stw CARG1, 0(BASE)
580 | stw CARG2, 4(BASE)
581 |.endif
545 | addi BASE, BASE, 8 582 | addi BASE, BASE, 8
546 | bney <1 583 | bney <1
547 | 584 |
@@ -619,23 +656,23 @@ static void build_subroutines(BuildCtx *ctx)
619 | .toc ld TOCREG, SAVE_TOC 656 | .toc ld TOCREG, SAVE_TOC
620 | li TISNUM, LJ_TISNUM // Setup type comparison constants. 657 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
621 | lp BASE, L->base 658 | lp BASE, L->base
622 | lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 659 | .FPU lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
623 | lwz DISPATCH, L->glref // Setup pointer to dispatch table. 660 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
624 | li ZERO, 0 661 | li ZERO, 0
625 | stw TMP3, TMPD 662 | .FPU stw TMP3, TMPD
626 | li TMP1, LJ_TFALSE 663 | li TMP1, LJ_TFALSE
627 | ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float). 664 | .FPU ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float).
628 | li TISNIL, LJ_TNIL 665 | li TISNIL, LJ_TNIL
629 | li_vmstate INTERP 666 | li_vmstate INTERP
630 | lfs TOBIT, TMPD 667 | .FPU lfs TOBIT, TMPD
631 | lwz PC, FRAME_PC(BASE) // Fetch PC of previous frame. 668 | lwz PC, FRAME_PC(BASE) // Fetch PC of previous frame.
632 | la RA, -8(BASE) // Results start at BASE-8. 669 | la RA, -8(BASE) // Results start at BASE-8.
633 | stw TMP3, TMPD 670 | .FPU stw TMP3, TMPD
634 | addi DISPATCH, DISPATCH, GG_G2DISP 671 | addi DISPATCH, DISPATCH, GG_G2DISP
635 | stw TMP1, 0(RA) // Prepend false to error message. 672 | stw TMP1, 0(RA) // Prepend false to error message.
636 | li RD, 16 // 2 results: false + error message. 673 | li RD, 16 // 2 results: false + error message.
637 | st_vmstate 674 | st_vmstate
638 | lfs TONUM, TMPD 675 | .FPU lfs TONUM, TMPD
639 | b ->vm_returnc 676 | b ->vm_returnc
640 | 677 |
641 |//----------------------------------------------------------------------- 678 |//-----------------------------------------------------------------------
@@ -684,33 +721,34 @@ static void build_subroutines(BuildCtx *ctx)
684 | stw CARG3, SAVE_NRES 721 | stw CARG3, SAVE_NRES
685 | cmplwi TMP1, 0 722 | cmplwi TMP1, 0
686 | stw CARG3, SAVE_ERRF 723 | stw CARG3, SAVE_ERRF
687 | stp TMP0, L->cframe
688 | stp CARG3, SAVE_CFRAME 724 | stp CARG3, SAVE_CFRAME
689 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok. 725 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
726 | stp TMP0, L->cframe
690 | beq >3 727 | beq >3
691 | 728 |
692 | // Resume after yield (like a return). 729 | // Resume after yield (like a return).
730 | stw L, DISPATCH_GL(cur_L)(DISPATCH)
693 | mr RA, BASE 731 | mr RA, BASE
694 | lp BASE, L->base 732 | lp BASE, L->base
695 | li TISNUM, LJ_TISNUM // Setup type comparison constants. 733 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
696 | lp TMP1, L->top 734 | lp TMP1, L->top
697 | lwz PC, FRAME_PC(BASE) 735 | lwz PC, FRAME_PC(BASE)
698 | lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 736 | .FPU lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
699 | stb CARG3, L->status 737 | stb CARG3, L->status
700 | stw TMP3, TMPD 738 | .FPU stw TMP3, TMPD
701 | ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float). 739 | .FPU ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float).
702 | lfs TOBIT, TMPD 740 | .FPU lfs TOBIT, TMPD
703 | sub RD, TMP1, BASE 741 | sub RD, TMP1, BASE
704 | stw TMP3, TMPD 742 | .FPU stw TMP3, TMPD
705 | lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double) 743 | .FPU lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double)
706 | addi RD, RD, 8 744 | addi RD, RD, 8
707 | stw TMP0, TONUM_HI 745 | .FPU stw TMP0, TONUM_HI
708 | li_vmstate INTERP 746 | li_vmstate INTERP
709 | li ZERO, 0 747 | li ZERO, 0
710 | st_vmstate 748 | st_vmstate
711 | andix. TMP0, PC, FRAME_TYPE 749 | andix. TMP0, PC, FRAME_TYPE
712 | mr MULTRES, RD 750 | mr MULTRES, RD
713 | lfs TONUM, TMPD 751 | .FPU lfs TONUM, TMPD
714 | li TISNIL, LJ_TNIL 752 | li TISNIL, LJ_TNIL
715 | beq ->BC_RET_Z 753 | beq ->BC_RET_Z
716 | b ->vm_return 754 | b ->vm_return
@@ -729,33 +767,34 @@ static void build_subroutines(BuildCtx *ctx)
729 | 767 |
730 |1: // Entry point for vm_pcall above (PC = ftype). 768 |1: // Entry point for vm_pcall above (PC = ftype).
731 | lp TMP1, L:CARG1->cframe 769 | lp TMP1, L:CARG1->cframe
732 | stw CARG3, SAVE_NRES
733 | mr L, CARG1 770 | mr L, CARG1
734 | stw CARG1, SAVE_L 771 | stw CARG3, SAVE_NRES
735 | mr BASE, CARG2
736 | stp sp, L->cframe // Add our C frame to cframe chain.
737 | lwz DISPATCH, L->glref // Setup pointer to dispatch table. 772 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
773 | stw CARG1, SAVE_L
774 | mr BASE, CARG2
775 | addi DISPATCH, DISPATCH, GG_G2DISP
738 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok. 776 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
739 | stp TMP1, SAVE_CFRAME 777 | stp TMP1, SAVE_CFRAME
740 | addi DISPATCH, DISPATCH, GG_G2DISP 778 | stp sp, L->cframe // Add our C frame to cframe chain.
741 | 779 |
742 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype). 780 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
781 | stw L, DISPATCH_GL(cur_L)(DISPATCH)
743 | lp TMP2, L->base // TMP2 = old base (used in vmeta_call). 782 | lp TMP2, L->base // TMP2 = old base (used in vmeta_call).
744 | li TISNUM, LJ_TISNUM // Setup type comparison constants. 783 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
745 | lp TMP1, L->top 784 | lp TMP1, L->top
746 | lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 785 | .FPU lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
747 | add PC, PC, BASE 786 | add PC, PC, BASE
748 | stw TMP3, TMPD 787 | .FPU stw TMP3, TMPD
749 | li ZERO, 0 788 | li ZERO, 0
750 | ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float). 789 | .FPU ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float).
751 | lfs TOBIT, TMPD 790 | .FPU lfs TOBIT, TMPD
752 | sub PC, PC, TMP2 // PC = frame delta + frame type 791 | sub PC, PC, TMP2 // PC = frame delta + frame type
753 | stw TMP3, TMPD 792 | .FPU stw TMP3, TMPD
754 | lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double) 793 | .FPU lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double)
755 | sub NARGS8:RC, TMP1, BASE 794 | sub NARGS8:RC, TMP1, BASE
756 | stw TMP0, TONUM_HI 795 | .FPU stw TMP0, TONUM_HI
757 | li_vmstate INTERP 796 | li_vmstate INTERP
758 | lfs TONUM, TMPD 797 | .FPU lfs TONUM, TMPD
759 | li TISNIL, LJ_TNIL 798 | li TISNIL, LJ_TNIL
760 | st_vmstate 799 | st_vmstate
761 | 800 |
@@ -776,15 +815,18 @@ static void build_subroutines(BuildCtx *ctx)
776 | lwz TMP0, L:CARG1->stack 815 | lwz TMP0, L:CARG1->stack
777 | stw CARG1, SAVE_L 816 | stw CARG1, SAVE_L
778 | lp TMP1, L->top 817 | lp TMP1, L->top
818 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
779 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok. 819 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
780 | sub TMP0, TMP0, TMP1 // Compute -savestack(L, L->top). 820 | sub TMP0, TMP0, TMP1 // Compute -savestack(L, L->top).
781 | lp TMP1, L->cframe 821 | lp TMP1, L->cframe
782 | stp sp, L->cframe // Add our C frame to cframe chain. 822 | addi DISPATCH, DISPATCH, GG_G2DISP
783 | .toc lp CARG4, 0(CARG4) 823 | .toc lp CARG4, 0(CARG4)
784 | li TMP2, 0 824 | li TMP2, 0
785 | stw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame. 825 | stw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame.
786 | stw TMP2, SAVE_ERRF // No error function. 826 | stw TMP2, SAVE_ERRF // No error function.
787 | stp TMP1, SAVE_CFRAME 827 | stp TMP1, SAVE_CFRAME
828 | stp sp, L->cframe // Add our C frame to cframe chain.
829 | stw L, DISPATCH_GL(cur_L)(DISPATCH)
788 | mtctr CARG4 830 | mtctr CARG4
789 | bctrl // (lua_State *L, lua_CFunction func, void *ud) 831 | bctrl // (lua_State *L, lua_CFunction func, void *ud)
790 |.if PPE 832 |.if PPE
@@ -793,9 +835,7 @@ static void build_subroutines(BuildCtx *ctx)
793 |.else 835 |.else
794 | mr. BASE, CRET1 836 | mr. BASE, CRET1
795 |.endif 837 |.endif
796 | lwz DISPATCH, L->glref // Setup pointer to dispatch table. 838 | li PC, FRAME_CP
797 | li PC, FRAME_CP
798 | addi DISPATCH, DISPATCH, GG_G2DISP
799 | bne <3 // Else continue with the call. 839 | bne <3 // Else continue with the call.
800 | b ->vm_leave_cp // No base? Just remove C frame. 840 | b ->vm_leave_cp // No base? Just remove C frame.
801 | 841 |
@@ -842,15 +882,30 @@ static void build_subroutines(BuildCtx *ctx)
842 | lwz INS, -4(PC) 882 | lwz INS, -4(PC)
843 | subi CARG2, RB, 16 883 | subi CARG2, RB, 16
844 | decode_RB8 SAVE0, INS 884 | decode_RB8 SAVE0, INS
885 |.if FPU
845 | lfd f0, 0(RA) 886 | lfd f0, 0(RA)
887 |.else
888 | lwz TMP2, 0(RA)
889 | lwz TMP3, 4(RA)
890 |.endif
846 | add TMP1, BASE, SAVE0 891 | add TMP1, BASE, SAVE0
847 | stp BASE, L->base 892 | stp BASE, L->base
848 | cmplw TMP1, CARG2 893 | cmplw TMP1, CARG2
849 | sub CARG3, CARG2, TMP1 894 | sub CARG3, CARG2, TMP1
850 | decode_RA8 RA, INS 895 | decode_RA8 RA, INS
896 |.if FPU
851 | stfd f0, 0(CARG2) 897 | stfd f0, 0(CARG2)
898 |.else
899 | stw TMP2, 0(CARG2)
900 | stw TMP3, 4(CARG2)
901 |.endif
852 | bney ->BC_CAT_Z 902 | bney ->BC_CAT_Z
903 |.if FPU
853 | stfdx f0, BASE, RA 904 | stfdx f0, BASE, RA
905 |.else
906 | stwux TMP2, RA, BASE
907 | stw TMP3, 4(RA)
908 |.endif
854 | b ->cont_nop 909 | b ->cont_nop
855 | 910 |
856 |//-- Table indexing metamethods ----------------------------------------- 911 |//-- Table indexing metamethods -----------------------------------------
@@ -903,9 +958,19 @@ static void build_subroutines(BuildCtx *ctx)
903 | // Returns TValue * (finished) or NULL (metamethod). 958 | // Returns TValue * (finished) or NULL (metamethod).
904 | cmplwi CRET1, 0 959 | cmplwi CRET1, 0
905 | beq >3 960 | beq >3
961 |.if FPU
906 | lfd f0, 0(CRET1) 962 | lfd f0, 0(CRET1)
963 |.else
964 | lwz TMP0, 0(CRET1)
965 | lwz TMP1, 4(CRET1)
966 |.endif
907 | ins_next1 967 | ins_next1
968 |.if FPU
908 | stfdx f0, BASE, RA 969 | stfdx f0, BASE, RA
970 |.else
971 | stwux TMP0, RA, BASE
972 | stw TMP1, 4(RA)
973 |.endif
909 | ins_next2 974 | ins_next2
910 | 975 |
911 |3: // Call __index metamethod. 976 |3: // Call __index metamethod.
@@ -918,6 +983,22 @@ static void build_subroutines(BuildCtx *ctx)
918 | li NARGS8:RC, 16 // 2 args for func(t, k). 983 | li NARGS8:RC, 16 // 2 args for func(t, k).
919 | b ->vm_call_dispatch_f 984 | b ->vm_call_dispatch_f
920 | 985 |
986 |->vmeta_tgetr:
987 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
988 | // Returns cTValue * or NULL.
989 | cmplwi CRET1, 0
990 | beq >1
991 |.if FPU
992 | lfd f14, 0(CRET1)
993 |.else
994 | lwz SAVE0, 0(CRET1)
995 | lwz SAVE1, 4(CRET1)
996 |.endif
997 | b ->BC_TGETR_Z
998 |1:
999 | stwx TISNIL, BASE, RA
1000 | b ->cont_nop
1001 |
921 |//----------------------------------------------------------------------- 1002 |//-----------------------------------------------------------------------
922 | 1003 |
923 |->vmeta_tsets1: 1004 |->vmeta_tsets1:
@@ -967,11 +1048,21 @@ static void build_subroutines(BuildCtx *ctx)
967 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k) 1048 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
968 | // Returns TValue * (finished) or NULL (metamethod). 1049 | // Returns TValue * (finished) or NULL (metamethod).
969 | cmplwi CRET1, 0 1050 | cmplwi CRET1, 0
1051 |.if FPU
970 | lfdx f0, BASE, RA 1052 | lfdx f0, BASE, RA
1053 |.else
1054 | lwzux TMP2, RA, BASE
1055 | lwz TMP3, 4(RA)
1056 |.endif
971 | beq >3 1057 | beq >3
972 | // NOBARRIER: lj_meta_tset ensures the table is not black. 1058 | // NOBARRIER: lj_meta_tset ensures the table is not black.
973 | ins_next1 1059 | ins_next1
1060 |.if FPU
974 | stfd f0, 0(CRET1) 1061 | stfd f0, 0(CRET1)
1062 |.else
1063 | stw TMP2, 0(CRET1)
1064 | stw TMP3, 4(CRET1)
1065 |.endif
975 | ins_next2 1066 | ins_next2
976 | 1067 |
977 |3: // Call __newindex metamethod. 1068 |3: // Call __newindex metamethod.
@@ -982,9 +1073,28 @@ static void build_subroutines(BuildCtx *ctx)
982 | add PC, TMP1, BASE 1073 | add PC, TMP1, BASE
983 | lwz LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here. 1074 | lwz LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
984 | li NARGS8:RC, 24 // 3 args for func(t, k, v) 1075 | li NARGS8:RC, 24 // 3 args for func(t, k, v)
1076 |.if FPU
985 | stfd f0, 16(BASE) // Copy value to third argument. 1077 | stfd f0, 16(BASE) // Copy value to third argument.
1078 |.else
1079 | stw TMP2, 16(BASE)
1080 | stw TMP3, 20(BASE)
1081 |.endif
986 | b ->vm_call_dispatch_f 1082 | b ->vm_call_dispatch_f
987 | 1083 |
1084 |->vmeta_tsetr:
1085 | stp BASE, L->base
1086 | mr CARG1, L
1087 | stw PC, SAVE_PC
1088 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
1089 | // Returns TValue *.
1090 |.if FPU
1091 | stfd f14, 0(CRET1)
1092 |.else
1093 | stw SAVE0, 0(CRET1)
1094 | stw SAVE1, 4(CRET1)
1095 |.endif
1096 | b ->cont_nop
1097 |
988 |//-- Comparison metamethods --------------------------------------------- 1098 |//-- Comparison metamethods ---------------------------------------------
989 | 1099 |
990 |->vmeta_comp: 1100 |->vmeta_comp:
@@ -1021,9 +1131,19 @@ static void build_subroutines(BuildCtx *ctx)
1021 | 1131 |
1022 |->cont_ra: // RA = resultptr 1132 |->cont_ra: // RA = resultptr
1023 | lwz INS, -4(PC) 1133 | lwz INS, -4(PC)
1134 |.if FPU
1024 | lfd f0, 0(RA) 1135 | lfd f0, 0(RA)
1136 |.else
1137 | lwz CARG1, 0(RA)
1138 | lwz CARG2, 4(RA)
1139 |.endif
1025 | decode_RA8 TMP1, INS 1140 | decode_RA8 TMP1, INS
1141 |.if FPU
1026 | stfdx f0, BASE, TMP1 1142 | stfdx f0, BASE, TMP1
1143 |.else
1144 | stwux CARG1, TMP1, BASE
1145 | stw CARG2, 4(TMP1)
1146 |.endif
1027 | b ->cont_nop 1147 | b ->cont_nop
1028 | 1148 |
1029 |->cont_condt: // RA = resultptr 1149 |->cont_condt: // RA = resultptr
@@ -1063,6 +1183,16 @@ static void build_subroutines(BuildCtx *ctx)
1063 | b <3 1183 | b <3
1064 |.endif 1184 |.endif
1065 | 1185 |
1186 |->vmeta_istype:
1187 | subi PC, PC, 4
1188 | stp BASE, L->base
1189 | srwi CARG2, RA, 3
1190 | mr CARG1, L
1191 | srwi CARG3, RD, 3
1192 | stw PC, SAVE_PC
1193 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
1194 | b ->cont_nop
1195 |
1066 |//-- Arithmetic metamethods --------------------------------------------- 1196 |//-- Arithmetic metamethods ---------------------------------------------
1067 | 1197 |
1068 |->vmeta_arith_nv: 1198 |->vmeta_arith_nv:
@@ -1219,22 +1349,32 @@ static void build_subroutines(BuildCtx *ctx)
1219 |.macro .ffunc_n, name 1349 |.macro .ffunc_n, name
1220 |->ff_ .. name: 1350 |->ff_ .. name:
1221 | cmplwi NARGS8:RC, 8 1351 | cmplwi NARGS8:RC, 8
1222 | lwz CARG3, 0(BASE) 1352 | lwz CARG1, 0(BASE)
1353 |.if FPU
1223 | lfd FARG1, 0(BASE) 1354 | lfd FARG1, 0(BASE)
1355 |.else
1356 | lwz CARG2, 4(BASE)
1357 |.endif
1224 | blt ->fff_fallback 1358 | blt ->fff_fallback
1225 | checknum CARG3; bge ->fff_fallback 1359 | checknum CARG1; bge ->fff_fallback
1226 |.endmacro 1360 |.endmacro
1227 | 1361 |
1228 |.macro .ffunc_nn, name 1362 |.macro .ffunc_nn, name
1229 |->ff_ .. name: 1363 |->ff_ .. name:
1230 | cmplwi NARGS8:RC, 16 1364 | cmplwi NARGS8:RC, 16
1231 | lwz CARG3, 0(BASE) 1365 | lwz CARG1, 0(BASE)
1366 |.if FPU
1232 | lfd FARG1, 0(BASE) 1367 | lfd FARG1, 0(BASE)
1233 | lwz CARG4, 8(BASE) 1368 | lwz CARG3, 8(BASE)
1234 | lfd FARG2, 8(BASE) 1369 | lfd FARG2, 8(BASE)
1370 |.else
1371 | lwz CARG2, 4(BASE)
1372 | lwz CARG3, 8(BASE)
1373 | lwz CARG4, 12(BASE)
1374 |.endif
1235 | blt ->fff_fallback 1375 | blt ->fff_fallback
1376 | checknum CARG1; bge ->fff_fallback
1236 | checknum CARG3; bge ->fff_fallback 1377 | checknum CARG3; bge ->fff_fallback
1237 | checknum CARG4; bge ->fff_fallback
1238 |.endmacro 1378 |.endmacro
1239 | 1379 |
1240 |// Inlined GC threshold check. Caveat: uses TMP0 and TMP1. 1380 |// Inlined GC threshold check. Caveat: uses TMP0 and TMP1.
@@ -1255,14 +1395,21 @@ static void build_subroutines(BuildCtx *ctx)
1255 | bge cr1, ->fff_fallback 1395 | bge cr1, ->fff_fallback
1256 | stw CARG3, 0(RA) 1396 | stw CARG3, 0(RA)
1257 | addi RD, NARGS8:RC, 8 // Compute (nresults+1)*8. 1397 | addi RD, NARGS8:RC, 8 // Compute (nresults+1)*8.
1398 | addi TMP1, BASE, 8
1399 | add TMP2, RA, NARGS8:RC
1258 | stw CARG1, 4(RA) 1400 | stw CARG1, 4(RA)
1259 | beq ->fff_res // Done if exactly 1 argument. 1401 | beq ->fff_res // Done if exactly 1 argument.
1260 | li TMP1, 8
1261 | subi RC, RC, 8
1262 |1: 1402 |1:
1263 | cmplw TMP1, RC 1403 | cmplw TMP1, TMP2
1264 | lfdx f0, BASE, TMP1 1404 |.if FPU
1265 | stfdx f0, RA, TMP1 1405 | lfd f0, 0(TMP1)
1406 | stfd f0, 0(TMP1)
1407 |.else
1408 | lwz CARG1, 0(TMP1)
1409 | lwz CARG2, 4(TMP1)
1410 | stw CARG1, -8(TMP1)
1411 | stw CARG2, -4(TMP1)
1412 |.endif
1266 | addi TMP1, TMP1, 8 1413 | addi TMP1, TMP1, 8
1267 | bney <1 1414 | bney <1
1268 | b ->fff_res 1415 | b ->fff_res
@@ -1277,8 +1424,14 @@ static void build_subroutines(BuildCtx *ctx)
1277 | orc TMP1, TMP2, TMP0 1424 | orc TMP1, TMP2, TMP0
1278 | addi TMP1, TMP1, ~LJ_TISNUM+1 1425 | addi TMP1, TMP1, ~LJ_TISNUM+1
1279 | slwi TMP1, TMP1, 3 1426 | slwi TMP1, TMP1, 3
1427 |.if FPU
1280 | la TMP2, CFUNC:RB->upvalue 1428 | la TMP2, CFUNC:RB->upvalue
1281 | lfdx FARG1, TMP2, TMP1 1429 | lfdx FARG1, TMP2, TMP1
1430 |.else
1431 | add TMP1, CFUNC:RB, TMP1
1432 | lwz CARG1, CFUNC:TMP1->upvalue[0].u32.hi
1433 | lwz CARG2, CFUNC:TMP1->upvalue[0].u32.lo
1434 |.endif
1282 | b ->fff_resn 1435 | b ->fff_resn
1283 | 1436 |
1284 |//-- Base library: getters and setters --------------------------------- 1437 |//-- Base library: getters and setters ---------------------------------
@@ -1294,9 +1447,9 @@ static void build_subroutines(BuildCtx *ctx)
1294 | beq ->fff_restv 1447 | beq ->fff_restv
1295 | lwz TMP0, TAB:CARG1->hmask 1448 | lwz TMP0, TAB:CARG1->hmask
1296 | li CARG3, LJ_TTAB // Use metatable as default result. 1449 | li CARG3, LJ_TTAB // Use metatable as default result.
1297 | lwz TMP1, STR:RC->hash 1450 | lwz TMP1, STR:RC->sid
1298 | lwz NODE:TMP2, TAB:CARG1->node 1451 | lwz NODE:TMP2, TAB:CARG1->node
1299 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 1452 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
1300 | slwi TMP0, TMP1, 5 1453 | slwi TMP0, TMP1, 5
1301 | slwi TMP1, TMP1, 3 1454 | slwi TMP1, TMP1, 3
1302 | sub TMP1, TMP0, TMP1 1455 | sub TMP1, TMP0, TMP1
@@ -1356,7 +1509,12 @@ static void build_subroutines(BuildCtx *ctx)
1356 | mr CARG1, L 1509 | mr CARG1, L
1357 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key) 1510 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1358 | // Returns cTValue *. 1511 | // Returns cTValue *.
1512 |.if FPU
1359 | lfd FARG1, 0(CRET1) 1513 | lfd FARG1, 0(CRET1)
1514 |.else
1515 | lwz CARG2, 4(CRET1)
1516 | lwz CARG1, 0(CRET1) // Caveat: CARG1 == CRET1.
1517 |.endif
1360 | b ->fff_resn 1518 | b ->fff_resn
1361 | 1519 |
1362 |//-- Base library: conversions ------------------------------------------ 1520 |//-- Base library: conversions ------------------------------------------
@@ -1365,7 +1523,11 @@ static void build_subroutines(BuildCtx *ctx)
1365 | // Only handles the number case inline (without a base argument). 1523 | // Only handles the number case inline (without a base argument).
1366 | cmplwi NARGS8:RC, 8 1524 | cmplwi NARGS8:RC, 8
1367 | lwz CARG1, 0(BASE) 1525 | lwz CARG1, 0(BASE)
1526 |.if FPU
1368 | lfd FARG1, 0(BASE) 1527 | lfd FARG1, 0(BASE)
1528 |.else
1529 | lwz CARG2, 4(BASE)
1530 |.endif
1369 | bne ->fff_fallback // Exactly one argument. 1531 | bne ->fff_fallback // Exactly one argument.
1370 | checknum CARG1; bgt ->fff_fallback 1532 | checknum CARG1; bgt ->fff_fallback
1371 | b ->fff_resn 1533 | b ->fff_resn
@@ -1387,9 +1549,9 @@ static void build_subroutines(BuildCtx *ctx)
1387 | mr CARG1, L 1549 | mr CARG1, L
1388 | mr CARG2, BASE 1550 | mr CARG2, BASE
1389 |.if DUALNUM 1551 |.if DUALNUM
1390 | bl extern lj_str_fromnumber // (lua_State *L, cTValue *o) 1552 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1391 |.else 1553 |.else
1392 | bl extern lj_str_fromnum // (lua_State *L, lua_Number *np) 1554 | bl extern lj_strfmt_num // (lua_State *L, lua_Number *np)
1393 |.endif 1555 |.endif
1394 | // Returns GCstr *. 1556 | // Returns GCstr *.
1395 | li CARG3, LJ_TSTR 1557 | li CARG3, LJ_TSTR
@@ -1397,32 +1559,24 @@ static void build_subroutines(BuildCtx *ctx)
1397 | 1559 |
1398 |//-- Base library: iterators ------------------------------------------- 1560 |//-- Base library: iterators -------------------------------------------
1399 | 1561 |
1400 |.ffunc next 1562 |.ffunc_1 next
1401 | cmplwi NARGS8:RC, 8
1402 | lwz CARG1, 0(BASE)
1403 | lwz TAB:CARG2, 4(BASE)
1404 | blt ->fff_fallback
1405 | stwx TISNIL, BASE, NARGS8:RC // Set missing 2nd arg to nil. 1563 | stwx TISNIL, BASE, NARGS8:RC // Set missing 2nd arg to nil.
1406 | checktab CARG1 1564 | checktab CARG3
1407 | lwz PC, FRAME_PC(BASE) 1565 | lwz PC, FRAME_PC(BASE)
1408 | bne ->fff_fallback 1566 | bne ->fff_fallback
1409 | stp BASE, L->base // Add frame since C call can throw. 1567 | la CARG2, 8(BASE)
1410 | mr CARG1, L 1568 | la CARG3, -8(BASE)
1411 | stp BASE, L->top // Dummy frame length is ok. 1569 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1412 | la CARG3, 8(BASE) 1570 | // Returns 1=found, 0=end, -1=error.
1413 | stw PC, SAVE_PC 1571 | cmpwi CRET1, 0
1414 | bl extern lj_tab_next // (lua_State *L, GCtab *t, TValue *key)
1415 | // Returns 0 at end of traversal.
1416 | cmplwi CRET1, 0
1417 | li CARG3, LJ_TNIL
1418 | beq ->fff_restv // End of traversal: return nil.
1419 | lfd f0, 8(BASE) // Copy key and value to results.
1420 | la RA, -8(BASE) 1572 | la RA, -8(BASE)
1421 | lfd f1, 16(BASE)
1422 | stfd f0, 0(RA)
1423 | li RD, (2+1)*8 1573 | li RD, (2+1)*8
1424 | stfd f1, 8(RA) 1574 | bgt ->fff_res // Found key/value.
1425 | b ->fff_res 1575 | li CARG3, LJ_TNIL
1576 | beq ->fff_restv // End of traversal: return nil.
1577 | lwz CFUNC:RB, FRAME_FUNC(BASE)
1578 | li NARGS8:RC, 2*8
1579 | b ->fff_fallback // Invalid key.
1426 | 1580 |
1427 |.ffunc_1 pairs 1581 |.ffunc_1 pairs
1428 | checktab CARG3 1582 | checktab CARG3
@@ -1430,17 +1584,32 @@ static void build_subroutines(BuildCtx *ctx)
1430 | bne ->fff_fallback 1584 | bne ->fff_fallback
1431#if LJ_52 1585#if LJ_52
1432 | lwz TAB:TMP2, TAB:CARG1->metatable 1586 | lwz TAB:TMP2, TAB:CARG1->metatable
1587 |.if FPU
1433 | lfd f0, CFUNC:RB->upvalue[0] 1588 | lfd f0, CFUNC:RB->upvalue[0]
1589 |.else
1590 | lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1591 | lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1592 |.endif
1434 | cmplwi TAB:TMP2, 0 1593 | cmplwi TAB:TMP2, 0
1435 | la RA, -8(BASE) 1594 | la RA, -8(BASE)
1436 | bne ->fff_fallback 1595 | bne ->fff_fallback
1437#else 1596#else
1597 |.if FPU
1438 | lfd f0, CFUNC:RB->upvalue[0] 1598 | lfd f0, CFUNC:RB->upvalue[0]
1599 |.else
1600 | lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1601 | lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1602 |.endif
1439 | la RA, -8(BASE) 1603 | la RA, -8(BASE)
1440#endif 1604#endif
1441 | stw TISNIL, 8(BASE) 1605 | stw TISNIL, 8(BASE)
1442 | li RD, (3+1)*8 1606 | li RD, (3+1)*8
1607 |.if FPU
1443 | stfd f0, 0(RA) 1608 | stfd f0, 0(RA)
1609 |.else
1610 | stw TMP0, 0(RA)
1611 | stw TMP1, 4(RA)
1612 |.endif
1444 | b ->fff_res 1613 | b ->fff_res
1445 | 1614 |
1446 |.ffunc ipairs_aux 1615 |.ffunc ipairs_aux
@@ -1486,14 +1655,24 @@ static void build_subroutines(BuildCtx *ctx)
1486 | stfd FARG2, 0(RA) 1655 | stfd FARG2, 0(RA)
1487 |.endif 1656 |.endif
1488 | ble >2 // Not in array part? 1657 | ble >2 // Not in array part?
1658 |.if FPU
1489 | lwzx TMP2, TMP1, TMP3 1659 | lwzx TMP2, TMP1, TMP3
1490 | lfdx f0, TMP1, TMP3 1660 | lfdx f0, TMP1, TMP3
1661 |.else
1662 | lwzux TMP2, TMP1, TMP3
1663 | lwz TMP3, 4(TMP1)
1664 |.endif
1491 |1: 1665 |1:
1492 | checknil TMP2 1666 | checknil TMP2
1493 | li RD, (0+1)*8 1667 | li RD, (0+1)*8
1494 | beq ->fff_res // End of iteration, return 0 results. 1668 | beq ->fff_res // End of iteration, return 0 results.
1495 | li RD, (2+1)*8 1669 | li RD, (2+1)*8
1670 |.if FPU
1496 | stfd f0, 8(RA) 1671 | stfd f0, 8(RA)
1672 |.else
1673 | stw TMP2, 8(RA)
1674 | stw TMP3, 12(RA)
1675 |.endif
1497 | b ->fff_res 1676 | b ->fff_res
1498 |2: // Check for empty hash part first. Otherwise call C function. 1677 |2: // Check for empty hash part first. Otherwise call C function.
1499 | lwz TMP0, TAB:CARG1->hmask 1678 | lwz TMP0, TAB:CARG1->hmask
@@ -1507,7 +1686,11 @@ static void build_subroutines(BuildCtx *ctx)
1507 | li RD, (0+1)*8 1686 | li RD, (0+1)*8
1508 | beq ->fff_res 1687 | beq ->fff_res
1509 | lwz TMP2, 0(CRET1) 1688 | lwz TMP2, 0(CRET1)
1689 |.if FPU
1510 | lfd f0, 0(CRET1) 1690 | lfd f0, 0(CRET1)
1691 |.else
1692 | lwz TMP3, 4(CRET1)
1693 |.endif
1511 | b <1 1694 | b <1
1512 | 1695 |
1513 |.ffunc_1 ipairs 1696 |.ffunc_1 ipairs
@@ -1516,12 +1699,22 @@ static void build_subroutines(BuildCtx *ctx)
1516 | bne ->fff_fallback 1699 | bne ->fff_fallback
1517#if LJ_52 1700#if LJ_52
1518 | lwz TAB:TMP2, TAB:CARG1->metatable 1701 | lwz TAB:TMP2, TAB:CARG1->metatable
1702 |.if FPU
1519 | lfd f0, CFUNC:RB->upvalue[0] 1703 | lfd f0, CFUNC:RB->upvalue[0]
1704 |.else
1705 | lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1706 | lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1707 |.endif
1520 | cmplwi TAB:TMP2, 0 1708 | cmplwi TAB:TMP2, 0
1521 | la RA, -8(BASE) 1709 | la RA, -8(BASE)
1522 | bne ->fff_fallback 1710 | bne ->fff_fallback
1523#else 1711#else
1712 |.if FPU
1524 | lfd f0, CFUNC:RB->upvalue[0] 1713 | lfd f0, CFUNC:RB->upvalue[0]
1714 |.else
1715 | lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1716 | lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1717 |.endif
1525 | la RA, -8(BASE) 1718 | la RA, -8(BASE)
1526#endif 1719#endif
1527 |.if DUALNUM 1720 |.if DUALNUM
@@ -1531,7 +1724,12 @@ static void build_subroutines(BuildCtx *ctx)
1531 |.endif 1724 |.endif
1532 | stw ZERO, 12(BASE) 1725 | stw ZERO, 12(BASE)
1533 | li RD, (3+1)*8 1726 | li RD, (3+1)*8
1727 |.if FPU
1534 | stfd f0, 0(RA) 1728 | stfd f0, 0(RA)
1729 |.else
1730 | stw TMP0, 0(RA)
1731 | stw TMP1, 4(RA)
1732 |.endif
1535 | b ->fff_res 1733 | b ->fff_res
1536 | 1734 |
1537 |//-- Base library: catch errors ---------------------------------------- 1735 |//-- Base library: catch errors ----------------------------------------
@@ -1550,19 +1748,32 @@ static void build_subroutines(BuildCtx *ctx)
1550 | 1748 |
1551 |.ffunc xpcall 1749 |.ffunc xpcall
1552 | cmplwi NARGS8:RC, 16 1750 | cmplwi NARGS8:RC, 16
1553 | lwz CARG4, 8(BASE) 1751 | lwz CARG3, 8(BASE)
1752 |.if FPU
1554 | lfd FARG2, 8(BASE) 1753 | lfd FARG2, 8(BASE)
1555 | lfd FARG1, 0(BASE) 1754 | lfd FARG1, 0(BASE)
1755 |.else
1756 | lwz CARG1, 0(BASE)
1757 | lwz CARG2, 4(BASE)
1758 | lwz CARG4, 12(BASE)
1759 |.endif
1556 | blt ->fff_fallback 1760 | blt ->fff_fallback
1557 | lbz TMP1, DISPATCH_GL(hookmask)(DISPATCH) 1761 | lbz TMP1, DISPATCH_GL(hookmask)(DISPATCH)
1558 | mr TMP2, BASE 1762 | mr TMP2, BASE
1559 | checkfunc CARG4; bne ->fff_fallback // Traceback must be a function. 1763 | checkfunc CARG3; bne ->fff_fallback // Traceback must be a function.
1560 | la BASE, 16(BASE) 1764 | la BASE, 16(BASE)
1561 | // Remember active hook before pcall. 1765 | // Remember active hook before pcall.
1562 | rlwinm TMP1, TMP1, 32-HOOK_ACTIVE_SHIFT, 31, 31 1766 | rlwinm TMP1, TMP1, 32-HOOK_ACTIVE_SHIFT, 31, 31
1767 |.if FPU
1563 | stfd FARG2, 0(TMP2) // Swap function and traceback. 1768 | stfd FARG2, 0(TMP2) // Swap function and traceback.
1564 | subi NARGS8:RC, NARGS8:RC, 16
1565 | stfd FARG1, 8(TMP2) 1769 | stfd FARG1, 8(TMP2)
1770 |.else
1771 | stw CARG3, 0(TMP2)
1772 | stw CARG4, 4(TMP2)
1773 | stw CARG1, 8(TMP2)
1774 | stw CARG2, 12(TMP2)
1775 |.endif
1776 | subi NARGS8:RC, NARGS8:RC, 16
1566 | addi PC, TMP1, 16+FRAME_PCALL 1777 | addi PC, TMP1, 16+FRAME_PCALL
1567 | b ->vm_call_dispatch 1778 | b ->vm_call_dispatch
1568 | 1779 |
@@ -1605,9 +1816,21 @@ static void build_subroutines(BuildCtx *ctx)
1605 | stp BASE, L->top 1816 | stp BASE, L->top
1606 |2: // Move args to coroutine. 1817 |2: // Move args to coroutine.
1607 | cmpw TMP1, NARGS8:RC 1818 | cmpw TMP1, NARGS8:RC
1819 |.if FPU
1608 | lfdx f0, BASE, TMP1 1820 | lfdx f0, BASE, TMP1
1821 |.else
1822 | add CARG3, BASE, TMP1
1823 | lwz TMP2, 0(CARG3)
1824 | lwz TMP3, 4(CARG3)
1825 |.endif
1609 | beq >3 1826 | beq >3
1827 |.if FPU
1610 | stfdx f0, CARG2, TMP1 1828 | stfdx f0, CARG2, TMP1
1829 |.else
1830 | add CARG3, CARG2, TMP1
1831 | stw TMP2, 0(CARG3)
1832 | stw TMP3, 4(CARG3)
1833 |.endif
1611 | addi TMP1, TMP1, 8 1834 | addi TMP1, TMP1, 8
1612 | b <2 1835 | b <2
1613 |3: 1836 |3:
@@ -1622,6 +1845,7 @@ static void build_subroutines(BuildCtx *ctx)
1622 | lp TMP3, L:SAVE0->top 1845 | lp TMP3, L:SAVE0->top
1623 | li_vmstate INTERP 1846 | li_vmstate INTERP
1624 | lp BASE, L->base 1847 | lp BASE, L->base
1848 | stw L, DISPATCH_GL(cur_L)(DISPATCH)
1625 | st_vmstate 1849 | st_vmstate
1626 | bgt >8 1850 | bgt >8
1627 | sub RD, TMP3, TMP2 1851 | sub RD, TMP3, TMP2
@@ -1637,8 +1861,17 @@ static void build_subroutines(BuildCtx *ctx)
1637 | stp TMP2, L:SAVE0->top // Clear coroutine stack. 1861 | stp TMP2, L:SAVE0->top // Clear coroutine stack.
1638 |5: // Move results from coroutine. 1862 |5: // Move results from coroutine.
1639 | cmplw TMP1, TMP3 1863 | cmplw TMP1, TMP3
1864 |.if FPU
1640 | lfdx f0, TMP2, TMP1 1865 | lfdx f0, TMP2, TMP1
1641 | stfdx f0, BASE, TMP1 1866 | stfdx f0, BASE, TMP1
1867 |.else
1868 | add CARG3, TMP2, TMP1
1869 | lwz CARG1, 0(CARG3)
1870 | lwz CARG2, 4(CARG3)
1871 | add CARG3, BASE, TMP1
1872 | stw CARG1, 0(CARG3)
1873 | stw CARG2, 4(CARG3)
1874 |.endif
1642 | addi TMP1, TMP1, 8 1875 | addi TMP1, TMP1, 8
1643 | bne <5 1876 | bne <5
1644 |6: 1877 |6:
@@ -1663,12 +1896,22 @@ static void build_subroutines(BuildCtx *ctx)
1663 | andix. TMP0, PC, FRAME_TYPE 1896 | andix. TMP0, PC, FRAME_TYPE
1664 | la TMP3, -8(TMP3) 1897 | la TMP3, -8(TMP3)
1665 | li TMP1, LJ_TFALSE 1898 | li TMP1, LJ_TFALSE
1899 |.if FPU
1666 | lfd f0, 0(TMP3) 1900 | lfd f0, 0(TMP3)
1901 |.else
1902 | lwz CARG1, 0(TMP3)
1903 | lwz CARG2, 4(TMP3)
1904 |.endif
1667 | stp TMP3, L:SAVE0->top // Remove error from coroutine stack. 1905 | stp TMP3, L:SAVE0->top // Remove error from coroutine stack.
1668 | li RD, (2+1)*8 1906 | li RD, (2+1)*8
1669 | stw TMP1, -8(BASE) // Prepend false to results. 1907 | stw TMP1, -8(BASE) // Prepend false to results.
1670 | la RA, -8(BASE) 1908 | la RA, -8(BASE)
1909 |.if FPU
1671 | stfd f0, 0(BASE) // Copy error message. 1910 | stfd f0, 0(BASE) // Copy error message.
1911 |.else
1912 | stw CARG1, 0(BASE) // Copy error message.
1913 | stw CARG2, 4(BASE)
1914 |.endif
1672 | b <7 1915 | b <7
1673 |.else 1916 |.else
1674 | mr CARG1, L 1917 | mr CARG1, L
@@ -1847,7 +2090,12 @@ static void build_subroutines(BuildCtx *ctx)
1847 | lus CARG1, 0x8000 // -(2^31). 2090 | lus CARG1, 0x8000 // -(2^31).
1848 | beqy ->fff_resi 2091 | beqy ->fff_resi
1849 |5: 2092 |5:
2093 |.if FPU
1850 | lfd FARG1, 0(BASE) 2094 | lfd FARG1, 0(BASE)
2095 |.else
2096 | lwz CARG1, 0(BASE)
2097 | lwz CARG2, 4(BASE)
2098 |.endif
1851 | blex func 2099 | blex func
1852 | b ->fff_resn 2100 | b ->fff_resn
1853 |.endmacro 2101 |.endmacro
@@ -1871,10 +2119,14 @@ static void build_subroutines(BuildCtx *ctx)
1871 | 2119 |
1872 |.ffunc math_log 2120 |.ffunc math_log
1873 | cmplwi NARGS8:RC, 8 2121 | cmplwi NARGS8:RC, 8
1874 | lwz CARG3, 0(BASE) 2122 | lwz CARG1, 0(BASE)
1875 | lfd FARG1, 0(BASE)
1876 | bne ->fff_fallback // Need exactly 1 argument. 2123 | bne ->fff_fallback // Need exactly 1 argument.
1877 | checknum CARG3; bge ->fff_fallback 2124 | checknum CARG1; bge ->fff_fallback
2125 |.if FPU
2126 | lfd FARG1, 0(BASE)
2127 |.else
2128 | lwz CARG2, 4(BASE)
2129 |.endif
1878 | blex log 2130 | blex log
1879 | b ->fff_resn 2131 | b ->fff_resn
1880 | 2132 |
@@ -1893,26 +2145,27 @@ static void build_subroutines(BuildCtx *ctx)
1893 | math_extern2 atan2 2145 | math_extern2 atan2
1894 | math_extern2 fmod 2146 | math_extern2 fmod
1895 | 2147 |
1896 |->ff_math_deg:
1897 |.ffunc_n math_rad
1898 | lfd FARG2, CFUNC:RB->upvalue[0]
1899 | fmul FARG1, FARG1, FARG2
1900 | b ->fff_resn
1901 |
1902 |.if DUALNUM 2148 |.if DUALNUM
1903 |.ffunc math_ldexp 2149 |.ffunc math_ldexp
1904 | cmplwi NARGS8:RC, 16 2150 | cmplwi NARGS8:RC, 16
1905 | lwz CARG3, 0(BASE) 2151 | lwz TMP0, 0(BASE)
2152 |.if FPU
1906 | lfd FARG1, 0(BASE) 2153 | lfd FARG1, 0(BASE)
1907 | lwz CARG4, 8(BASE) 2154 |.else
2155 | lwz CARG1, 0(BASE)
2156 | lwz CARG2, 4(BASE)
2157 |.endif
2158 | lwz TMP1, 8(BASE)
1908 |.if GPR64 2159 |.if GPR64
1909 | lwz CARG2, 12(BASE) 2160 | lwz CARG2, 12(BASE)
1910 |.else 2161 |.elif FPU
1911 | lwz CARG1, 12(BASE) 2162 | lwz CARG1, 12(BASE)
2163 |.else
2164 | lwz CARG3, 12(BASE)
1912 |.endif 2165 |.endif
1913 | blt ->fff_fallback 2166 | blt ->fff_fallback
1914 | checknum CARG3; bge ->fff_fallback 2167 | checknum TMP0; bge ->fff_fallback
1915 | checknum CARG4; bne ->fff_fallback 2168 | checknum TMP1; bne ->fff_fallback
1916 |.else 2169 |.else
1917 |.ffunc_nn math_ldexp 2170 |.ffunc_nn math_ldexp
1918 |.if GPR64 2171 |.if GPR64
@@ -1927,8 +2180,10 @@ static void build_subroutines(BuildCtx *ctx)
1927 |.ffunc_n math_frexp 2180 |.ffunc_n math_frexp
1928 |.if GPR64 2181 |.if GPR64
1929 | la CARG2, DISPATCH_GL(tmptv)(DISPATCH) 2182 | la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
1930 |.else 2183 |.elif FPU
1931 | la CARG1, DISPATCH_GL(tmptv)(DISPATCH) 2184 | la CARG1, DISPATCH_GL(tmptv)(DISPATCH)
2185 |.else
2186 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
1932 |.endif 2187 |.endif
1933 | lwz PC, FRAME_PC(BASE) 2188 | lwz PC, FRAME_PC(BASE)
1934 | blex frexp 2189 | blex frexp
@@ -1937,7 +2192,12 @@ static void build_subroutines(BuildCtx *ctx)
1937 |.if not DUALNUM 2192 |.if not DUALNUM
1938 | tonum_i FARG2, TMP1 2193 | tonum_i FARG2, TMP1
1939 |.endif 2194 |.endif
2195 |.if FPU
1940 | stfd FARG1, 0(RA) 2196 | stfd FARG1, 0(RA)
2197 |.else
2198 | stw CRET1, 0(RA)
2199 | stw CRET2, 4(RA)
2200 |.endif
1941 | li RD, (2+1)*8 2201 | li RD, (2+1)*8
1942 |.if DUALNUM 2202 |.if DUALNUM
1943 | stw TISNUM, 8(RA) 2203 | stw TISNUM, 8(RA)
@@ -1950,13 +2210,20 @@ static void build_subroutines(BuildCtx *ctx)
1950 |.ffunc_n math_modf 2210 |.ffunc_n math_modf
1951 |.if GPR64 2211 |.if GPR64
1952 | la CARG2, -8(BASE) 2212 | la CARG2, -8(BASE)
1953 |.else 2213 |.elif FPU
1954 | la CARG1, -8(BASE) 2214 | la CARG1, -8(BASE)
2215 |.else
2216 | la CARG3, -8(BASE)
1955 |.endif 2217 |.endif
1956 | lwz PC, FRAME_PC(BASE) 2218 | lwz PC, FRAME_PC(BASE)
1957 | blex modf 2219 | blex modf
1958 | la RA, -8(BASE) 2220 | la RA, -8(BASE)
2221 |.if FPU
1959 | stfd FARG1, 0(BASE) 2222 | stfd FARG1, 0(BASE)
2223 |.else
2224 | stw CRET1, 0(BASE)
2225 | stw CRET2, 4(BASE)
2226 |.endif
1960 | li RD, (2+1)*8 2227 | li RD, (2+1)*8
1961 | b ->fff_res 2228 | b ->fff_res
1962 | 2229 |
@@ -1964,13 +2231,13 @@ static void build_subroutines(BuildCtx *ctx)
1964 |.if DUALNUM 2231 |.if DUALNUM
1965 | .ffunc_1 name 2232 | .ffunc_1 name
1966 | checknum CARG3 2233 | checknum CARG3
1967 | addi TMP1, BASE, 8 2234 | addi SAVE0, BASE, 8
1968 | add TMP2, BASE, NARGS8:RC 2235 | add SAVE1, BASE, NARGS8:RC
1969 | bne >4 2236 | bne >4
1970 |1: // Handle integers. 2237 |1: // Handle integers.
1971 | lwz CARG4, 0(TMP1) 2238 | lwz CARG4, 0(SAVE0)
1972 | cmplw cr1, TMP1, TMP2 2239 | cmplw cr1, SAVE0, SAVE1
1973 | lwz CARG2, 4(TMP1) 2240 | lwz CARG2, 4(SAVE0)
1974 | bge cr1, ->fff_resi 2241 | bge cr1, ->fff_resi
1975 | checknum CARG4 2242 | checknum CARG4
1976 | xoris TMP0, CARG1, 0x8000 2243 | xoris TMP0, CARG1, 0x8000
@@ -1987,36 +2254,76 @@ static void build_subroutines(BuildCtx *ctx)
1987 |.if GPR64 2254 |.if GPR64
1988 | rldicl CARG1, CARG1, 0, 32 2255 | rldicl CARG1, CARG1, 0, 32
1989 |.endif 2256 |.endif
1990 | addi TMP1, TMP1, 8 2257 | addi SAVE0, SAVE0, 8
1991 | b <1 2258 | b <1
1992 |3: 2259 |3:
1993 | bge ->fff_fallback 2260 | bge ->fff_fallback
1994 | // Convert intermediate result to number and continue below. 2261 | // Convert intermediate result to number and continue below.
2262 |.if FPU
1995 | tonum_i FARG1, CARG1 2263 | tonum_i FARG1, CARG1
1996 | lfd FARG2, 0(TMP1) 2264 | lfd FARG2, 0(SAVE0)
2265 |.else
2266 | mr CARG2, CARG1
2267 | bl ->vm_sfi2d_1
2268 | lwz CARG3, 0(SAVE0)
2269 | lwz CARG4, 4(SAVE0)
2270 |.endif
1997 | b >6 2271 | b >6
1998 |4: 2272 |4:
2273 |.if FPU
1999 | lfd FARG1, 0(BASE) 2274 | lfd FARG1, 0(BASE)
2275 |.else
2276 | lwz CARG1, 0(BASE)
2277 | lwz CARG2, 4(BASE)
2278 |.endif
2000 | bge ->fff_fallback 2279 | bge ->fff_fallback
2001 |5: // Handle numbers. 2280 |5: // Handle numbers.
2002 | lwz CARG4, 0(TMP1) 2281 | lwz CARG3, 0(SAVE0)
2003 | cmplw cr1, TMP1, TMP2 2282 | cmplw cr1, SAVE0, SAVE1
2004 | lfd FARG2, 0(TMP1) 2283 |.if FPU
2284 | lfd FARG2, 0(SAVE0)
2285 |.else
2286 | lwz CARG4, 4(SAVE0)
2287 |.endif
2005 | bge cr1, ->fff_resn 2288 | bge cr1, ->fff_resn
2006 | checknum CARG4; bge >7 2289 | checknum CARG3; bge >7
2007 |6: 2290 |6:
2008 | fsub f0, FARG1, FARG2 2291 | addi SAVE0, SAVE0, 8
2009 | addi TMP1, TMP1, 8 2292 |.if FPU
2010 |.if ismax 2293 |.if ismax
2294 | fsub f0, FARG1, FARG2
2295 |.else
2296 | fsub f0, FARG2, FARG1
2297 |.endif
2011 | fsel FARG1, f0, FARG1, FARG2 2298 | fsel FARG1, f0, FARG1, FARG2
2012 |.else 2299 |.else
2013 | fsel FARG1, f0, FARG2, FARG1 2300 | stw CARG1, SFSAVE_1
2301 | stw CARG2, SFSAVE_2
2302 | stw CARG3, SFSAVE_3
2303 | stw CARG4, SFSAVE_4
2304 | blex __ledf2
2305 | cmpwi CRET1, 0
2306 |.if ismax
2307 | blt >8
2308 |.else
2309 | bge >8
2310 |.endif
2311 | lwz CARG1, SFSAVE_1
2312 | lwz CARG2, SFSAVE_2
2313 | b <5
2314 |8:
2315 | lwz CARG1, SFSAVE_3
2316 | lwz CARG2, SFSAVE_4
2014 |.endif 2317 |.endif
2015 | b <5 2318 | b <5
2016 |7: // Convert integer to number and continue above. 2319 |7: // Convert integer to number and continue above.
2017 | lwz CARG2, 4(TMP1) 2320 | lwz CARG3, 4(SAVE0)
2018 | bne ->fff_fallback 2321 | bne ->fff_fallback
2019 | tonum_i FARG2, CARG2 2322 |.if FPU
2323 | tonum_i FARG2, CARG3
2324 |.else
2325 | bl ->vm_sfi2d_2
2326 |.endif
2020 | b <6 2327 | b <6
2021 |.else 2328 |.else
2022 | .ffunc_n name 2329 | .ffunc_n name
@@ -2028,13 +2335,13 @@ static void build_subroutines(BuildCtx *ctx)
2028 | checknum CARG2 2335 | checknum CARG2
2029 | bge cr1, ->fff_resn 2336 | bge cr1, ->fff_resn
2030 | bge ->fff_fallback 2337 | bge ->fff_fallback
2031 | fsub f0, FARG1, FARG2
2032 | addi TMP1, TMP1, 8
2033 |.if ismax 2338 |.if ismax
2034 | fsel FARG1, f0, FARG1, FARG2 2339 | fsub f0, FARG1, FARG2
2035 |.else 2340 |.else
2036 | fsel FARG1, f0, FARG2, FARG1 2341 | fsub f0, FARG2, FARG1
2037 |.endif 2342 |.endif
2343 | addi TMP1, TMP1, 8
2344 | fsel FARG1, f0, FARG1, FARG2
2038 | b <1 2345 | b <1
2039 |.endif 2346 |.endif
2040 |.endmacro 2347 |.endmacro
@@ -2044,11 +2351,6 @@ static void build_subroutines(BuildCtx *ctx)
2044 | 2351 |
2045 |//-- String library ----------------------------------------------------- 2352 |//-- String library -----------------------------------------------------
2046 | 2353 |
2047 |.ffunc_1 string_len
2048 | checkstr CARG3; bne ->fff_fallback
2049 | lwz CRET1, STR:CARG1->len
2050 | b ->fff_resi
2051 |
2052 |.ffunc string_byte // Only handle the 1-arg case here. 2354 |.ffunc string_byte // Only handle the 1-arg case here.
2053 | cmplwi NARGS8:RC, 8 2355 | cmplwi NARGS8:RC, 8
2054 | lwz CARG3, 0(BASE) 2356 | lwz CARG3, 0(BASE)
@@ -2103,6 +2405,7 @@ static void build_subroutines(BuildCtx *ctx)
2103 | stp BASE, L->base 2405 | stp BASE, L->base
2104 | stw PC, SAVE_PC 2406 | stw PC, SAVE_PC
2105 | bl extern lj_str_new // (lua_State *L, char *str, size_t l) 2407 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
2408 |->fff_resstr:
2106 | // Returns GCstr *. 2409 | // Returns GCstr *.
2107 | lp BASE, L->base 2410 | lp BASE, L->base
2108 | li CARG3, LJ_TSTR 2411 | li CARG3, LJ_TSTR
@@ -2180,114 +2483,29 @@ static void build_subroutines(BuildCtx *ctx)
2180 | addi TMP1, TMP1, 1 // start = 1 + (start ? start+len : 0) 2483 | addi TMP1, TMP1, 1 // start = 1 + (start ? start+len : 0)
2181 | b <3 2484 | b <3
2182 | 2485 |
2183 |.ffunc string_rep // Only handle the 1-char case inline. 2486 |.macro ffstring_op, name
2184 | ffgccheck 2487 | .ffunc string_ .. name
2185 | cmplwi NARGS8:RC, 16
2186 | lwz TMP0, 0(BASE)
2187 | lwz STR:CARG1, 4(BASE)
2188 | lwz CARG4, 8(BASE)
2189 |.if DUALNUM
2190 | lwz CARG3, 12(BASE)
2191 |.else
2192 | lfd FARG2, 8(BASE)
2193 |.endif
2194 | bne ->fff_fallback // Exactly 2 arguments.
2195 | checkstr TMP0; bne ->fff_fallback
2196 |.if DUALNUM
2197 | checknum CARG4; bne ->fff_fallback
2198 |.else
2199 | checknum CARG4; bge ->fff_fallback
2200 | toint CARG3, FARG2
2201 |.endif
2202 | lwz TMP0, STR:CARG1->len
2203 | cmpwi CARG3, 0
2204 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
2205 | ble >2 // Count <= 0? (or non-int)
2206 | cmplwi TMP0, 1
2207 | subi TMP2, CARG3, 1
2208 | blt >2 // Zero length string?
2209 | cmplw cr1, TMP1, CARG3
2210 | bne ->fff_fallback // Fallback for > 1-char strings.
2211 | lbz TMP0, STR:CARG1[1]
2212 | lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
2213 | blt cr1, ->fff_fallback
2214 |1: // Fill buffer with char. Yes, this is suboptimal code (do you care?).
2215 | cmplwi TMP2, 0
2216 | stbx TMP0, CARG2, TMP2
2217 | subi TMP2, TMP2, 1
2218 | bne <1
2219 | b ->fff_newstr
2220 |2: // Return empty string.
2221 | la STR:CARG1, DISPATCH_GL(strempty)(DISPATCH)
2222 | li CARG3, LJ_TSTR
2223 | b ->fff_restv
2224 |
2225 |.ffunc string_reverse
2226 | ffgccheck
2227 | cmplwi NARGS8:RC, 8
2228 | lwz CARG3, 0(BASE)
2229 | lwz STR:CARG1, 4(BASE)
2230 | blt ->fff_fallback
2231 | checkstr CARG3
2232 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
2233 | bne ->fff_fallback
2234 | lwz CARG3, STR:CARG1->len
2235 | la CARG1, #STR(STR:CARG1)
2236 | lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
2237 | li TMP2, 0
2238 | cmplw TMP1, CARG3
2239 | subi TMP3, CARG3, 1
2240 | blt ->fff_fallback
2241 |1: // Reverse string copy.
2242 | cmpwi TMP3, 0
2243 | lbzx TMP1, CARG1, TMP2
2244 | blty ->fff_newstr
2245 | stbx TMP1, CARG2, TMP3
2246 | subi TMP3, TMP3, 1
2247 | addi TMP2, TMP2, 1
2248 | b <1
2249 |
2250 |.macro ffstring_case, name, lo
2251 | .ffunc name
2252 | ffgccheck 2488 | ffgccheck
2253 | cmplwi NARGS8:RC, 8 2489 | cmplwi NARGS8:RC, 8
2254 | lwz CARG3, 0(BASE) 2490 | lwz CARG3, 0(BASE)
2255 | lwz STR:CARG1, 4(BASE) 2491 | lwz STR:CARG2, 4(BASE)
2256 | blt ->fff_fallback 2492 | blt ->fff_fallback
2257 | checkstr CARG3 2493 | checkstr CARG3
2258 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH) 2494 | la SBUF:CARG1, DISPATCH_GL(tmpbuf)(DISPATCH)
2259 | bne ->fff_fallback 2495 | bne ->fff_fallback
2260 | lwz CARG3, STR:CARG1->len 2496 | lwz TMP0, SBUF:CARG1->b
2261 | la CARG1, #STR(STR:CARG1) 2497 | stw L, SBUF:CARG1->L
2262 | lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH) 2498 | stp BASE, L->base
2263 | cmplw TMP1, CARG3 2499 | stw PC, SAVE_PC
2264 | li TMP2, 0 2500 | stw TMP0, SBUF:CARG1->w
2265 | blt ->fff_fallback 2501 | bl extern lj_buf_putstr_ .. name
2266 |1: // ASCII case conversion. 2502 | bl extern lj_buf_tostr
2267 | cmplw TMP2, CARG3 2503 | b ->fff_resstr
2268 | lbzx TMP1, CARG1, TMP2
2269 | bgey ->fff_newstr
2270 | subi TMP0, TMP1, lo
2271 | xori TMP3, TMP1, 0x20
2272 | addic TMP0, TMP0, -26
2273 | subfe TMP3, TMP3, TMP3
2274 | rlwinm TMP3, TMP3, 0, 26, 26 // x &= 0x20.
2275 | xor TMP1, TMP1, TMP3
2276 | stbx TMP1, CARG2, TMP2
2277 | addi TMP2, TMP2, 1
2278 | b <1
2279 |.endmacro 2504 |.endmacro
2280 | 2505 |
2281 |ffstring_case string_lower, 65 2506 |ffstring_op reverse
2282 |ffstring_case string_upper, 97 2507 |ffstring_op lower
2283 | 2508 |ffstring_op upper
2284 |//-- Table library ------------------------------------------------------
2285 |
2286 |.ffunc_1 table_getn
2287 | checktab CARG3; bne ->fff_fallback
2288 | bl extern lj_tab_len // (GCtab *t)
2289 | // Returns uint32_t (but less than 2^31).
2290 | b ->fff_resi
2291 | 2509 |
2292 |//-- Bit library -------------------------------------------------------- 2510 |//-- Bit library --------------------------------------------------------
2293 | 2511 |
@@ -2305,28 +2523,37 @@ static void build_subroutines(BuildCtx *ctx)
2305 | 2523 |
2306 |.macro .ffunc_bit_op, name, ins 2524 |.macro .ffunc_bit_op, name, ins
2307 | .ffunc_bit name 2525 | .ffunc_bit name
2308 | addi TMP1, BASE, 8 2526 | addi SAVE0, BASE, 8
2309 | add TMP2, BASE, NARGS8:RC 2527 | add SAVE1, BASE, NARGS8:RC
2310 |1: 2528 |1:
2311 | lwz CARG4, 0(TMP1) 2529 | lwz CARG4, 0(SAVE0)
2312 | cmplw cr1, TMP1, TMP2 2530 | cmplw cr1, SAVE0, SAVE1
2313 |.if DUALNUM 2531 |.if DUALNUM
2314 | lwz CARG2, 4(TMP1) 2532 | lwz CARG2, 4(SAVE0)
2315 |.else 2533 |.else
2316 | lfd FARG1, 0(TMP1) 2534 | lfd FARG1, 0(SAVE0)
2317 |.endif 2535 |.endif
2318 | bgey cr1, ->fff_resi 2536 | bgey cr1, ->fff_resi
2319 | checknum CARG4 2537 | checknum CARG4
2320 |.if DUALNUM 2538 |.if DUALNUM
2539 |.if FPU
2321 | bnel ->fff_bitop_fb 2540 | bnel ->fff_bitop_fb
2322 |.else 2541 |.else
2542 | beq >3
2543 | stw CARG1, SFSAVE_1
2544 | bl ->fff_bitop_fb
2545 | mr CARG2, CARG1
2546 | lwz CARG1, SFSAVE_1
2547 |3:
2548 |.endif
2549 |.else
2323 | fadd FARG1, FARG1, TOBIT 2550 | fadd FARG1, FARG1, TOBIT
2324 | bge ->fff_fallback 2551 | bge ->fff_fallback
2325 | stfd FARG1, TMPD 2552 | stfd FARG1, TMPD
2326 | lwz CARG2, TMPD_LO 2553 | lwz CARG2, TMPD_LO
2327 |.endif 2554 |.endif
2328 | ins CARG1, CARG1, CARG2 2555 | ins CARG1, CARG1, CARG2
2329 | addi TMP1, TMP1, 8 2556 | addi SAVE0, SAVE0, 8
2330 | b <1 2557 | b <1
2331 |.endmacro 2558 |.endmacro
2332 | 2559 |
@@ -2348,7 +2575,14 @@ static void build_subroutines(BuildCtx *ctx)
2348 |.macro .ffunc_bit_sh, name, ins, shmod 2575 |.macro .ffunc_bit_sh, name, ins, shmod
2349 |.if DUALNUM 2576 |.if DUALNUM
2350 | .ffunc_2 bit_..name 2577 | .ffunc_2 bit_..name
2578 |.if FPU
2351 | checknum CARG3; bnel ->fff_tobit_fb 2579 | checknum CARG3; bnel ->fff_tobit_fb
2580 |.else
2581 | checknum CARG3; beq >1
2582 | bl ->fff_tobit_fb
2583 | lwz CARG2, 12(BASE) // Conversion polluted CARG2.
2584 |1:
2585 |.endif
2352 | // Note: no inline conversion from number for 2nd argument! 2586 | // Note: no inline conversion from number for 2nd argument!
2353 | checknum CARG4; bne ->fff_fallback 2587 | checknum CARG4; bne ->fff_fallback
2354 |.else 2588 |.else
@@ -2385,27 +2619,77 @@ static void build_subroutines(BuildCtx *ctx)
2385 |->fff_resn: 2619 |->fff_resn:
2386 | lwz PC, FRAME_PC(BASE) 2620 | lwz PC, FRAME_PC(BASE)
2387 | la RA, -8(BASE) 2621 | la RA, -8(BASE)
2622 |.if FPU
2388 | stfd FARG1, -8(BASE) 2623 | stfd FARG1, -8(BASE)
2624 |.else
2625 | stw CARG1, -8(BASE)
2626 | stw CARG2, -4(BASE)
2627 |.endif
2389 | b ->fff_res1 2628 | b ->fff_res1
2390 | 2629 |
2391 |// Fallback FP number to bit conversion. 2630 |// Fallback FP number to bit conversion.
2392 |->fff_tobit_fb: 2631 |->fff_tobit_fb:
2393 |.if DUALNUM 2632 |.if DUALNUM
2633 |.if FPU
2394 | lfd FARG1, 0(BASE) 2634 | lfd FARG1, 0(BASE)
2395 | bgt ->fff_fallback 2635 | bgt ->fff_fallback
2396 | fadd FARG1, FARG1, TOBIT 2636 | fadd FARG1, FARG1, TOBIT
2397 | stfd FARG1, TMPD 2637 | stfd FARG1, TMPD
2398 | lwz CARG1, TMPD_LO 2638 | lwz CARG1, TMPD_LO
2399 | blr 2639 | blr
2640 |.else
2641 | bgt ->fff_fallback
2642 | mr CARG2, CARG1
2643 | mr CARG1, CARG3
2644 |// Modifies: CARG1, CARG2, TMP0, TMP1, TMP2.
2645 |->vm_tobit:
2646 | slwi TMP2, CARG1, 1
2647 | addis TMP2, TMP2, 0x0020
2648 | cmpwi TMP2, 0
2649 | bge >2
2650 | li TMP1, 0x3e0
2651 | srawi TMP2, TMP2, 21
2652 | not TMP1, TMP1
2653 | sub. TMP2, TMP1, TMP2
2654 | cmpwi cr7, CARG1, 0
2655 | blt >1
2656 | slwi TMP1, CARG1, 11
2657 | srwi TMP0, CARG2, 21
2658 | oris TMP1, TMP1, 0x8000
2659 | or TMP1, TMP1, TMP0
2660 | srw CARG1, TMP1, TMP2
2661 | bclr 4, 28 // Return if cr7[lt] == 0, no hint.
2662 | neg CARG1, CARG1
2663 | blr
2664 |1:
2665 | addi TMP2, TMP2, 21
2666 | srw TMP1, CARG2, TMP2
2667 | slwi CARG2, CARG1, 12
2668 | subfic TMP2, TMP2, 20
2669 | slw TMP0, CARG2, TMP2
2670 | or CARG1, TMP1, TMP0
2671 | bclr 4, 28 // Return if cr7[lt] == 0, no hint.
2672 | neg CARG1, CARG1
2673 | blr
2674 |2:
2675 | li CARG1, 0
2676 | blr
2677 |.endif
2400 |.endif 2678 |.endif
2401 |->fff_bitop_fb: 2679 |->fff_bitop_fb:
2402 |.if DUALNUM 2680 |.if DUALNUM
2403 | lfd FARG1, 0(TMP1) 2681 |.if FPU
2682 | lfd FARG1, 0(SAVE0)
2404 | bgt ->fff_fallback 2683 | bgt ->fff_fallback
2405 | fadd FARG1, FARG1, TOBIT 2684 | fadd FARG1, FARG1, TOBIT
2406 | stfd FARG1, TMPD 2685 | stfd FARG1, TMPD
2407 | lwz CARG2, TMPD_LO 2686 | lwz CARG2, TMPD_LO
2408 | blr 2687 | blr
2688 |.else
2689 | bgt ->fff_fallback
2690 | mr CARG1, CARG4
2691 | b ->vm_tobit
2692 |.endif
2409 |.endif 2693 |.endif
2410 | 2694 |
2411 |//----------------------------------------------------------------------- 2695 |//-----------------------------------------------------------------------
@@ -2589,15 +2873,88 @@ static void build_subroutines(BuildCtx *ctx)
2589 | mtctr CRET1 2873 | mtctr CRET1
2590 | bctr 2874 | bctr
2591 | 2875 |
2876 |->cont_stitch: // Trace stitching.
2877 |.if JIT
2878 | // RA = resultptr, RB = meta base
2879 | lwz INS, -4(PC)
2880 | lwz TRACE:TMP2, -20(RB) // Save previous trace.
2881 | addic. TMP1, MULTRES, -8
2882 | decode_RA8 RC, INS // Call base.
2883 | beq >2
2884 |1: // Move results down.
2885 |.if FPU
2886 | lfd f0, 0(RA)
2887 |.else
2888 | lwz CARG1, 0(RA)
2889 | lwz CARG2, 4(RA)
2890 |.endif
2891 | addic. TMP1, TMP1, -8
2892 | addi RA, RA, 8
2893 |.if FPU
2894 | stfdx f0, BASE, RC
2895 |.else
2896 | add CARG3, BASE, RC
2897 | stw CARG1, 0(CARG3)
2898 | stw CARG2, 4(CARG3)
2899 |.endif
2900 | addi RC, RC, 8
2901 | bne <1
2902 |2:
2903 | decode_RA8 RA, INS
2904 | decode_RB8 RB, INS
2905 | add RA, RA, RB
2906 |3:
2907 | cmplw RA, RC
2908 | bgt >9 // More results wanted?
2909 |
2910 | lhz TMP3, TRACE:TMP2->traceno
2911 | lhz RD, TRACE:TMP2->link
2912 | cmpw RD, TMP3
2913 | cmpwi cr1, RD, 0
2914 | beq ->cont_nop // Blacklisted.
2915 | slwi RD, RD, 3
2916 | bne cr1, =>BC_JLOOP // Jump to stitched trace.
2917 |
2918 | // Stitch a new trace to the previous trace.
2919 | stw TMP3, DISPATCH_J(exitno)(DISPATCH)
2920 | stp L, DISPATCH_J(L)(DISPATCH)
2921 | stp BASE, L->base
2922 | addi CARG1, DISPATCH, GG_DISP2J
2923 | mr CARG2, PC
2924 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
2925 | lp BASE, L->base
2926 | b ->cont_nop
2927 |
2928 |9:
2929 | stwx TISNIL, BASE, RC
2930 | addi RC, RC, 8
2931 | b <3
2932 |.endif
2933 |
2934 |->vm_profhook: // Dispatch target for profiler hook.
2935#if LJ_HASPROFILE
2936 | mr CARG1, L
2937 | stw MULTRES, SAVE_MULTRES
2938 | mr CARG2, PC
2939 | stp BASE, L->base
2940 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
2941 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2942 | lp BASE, L->base
2943 | subi PC, PC, 4
2944 | b ->cont_nop
2945#endif
2946 |
2592 |//----------------------------------------------------------------------- 2947 |//-----------------------------------------------------------------------
2593 |//-- Trace exit handler ------------------------------------------------- 2948 |//-- Trace exit handler -------------------------------------------------
2594 |//----------------------------------------------------------------------- 2949 |//-----------------------------------------------------------------------
2595 | 2950 |
2596 |.macro savex_, a, b, c, d 2951 |.macro savex_, a, b, c, d
2952 |.if FPU
2597 | stfd f..a, 16+a*8(sp) 2953 | stfd f..a, 16+a*8(sp)
2598 | stfd f..b, 16+b*8(sp) 2954 | stfd f..b, 16+b*8(sp)
2599 | stfd f..c, 16+c*8(sp) 2955 | stfd f..c, 16+c*8(sp)
2600 | stfd f..d, 16+d*8(sp) 2956 | stfd f..d, 16+d*8(sp)
2957 |.endif
2601 |.endmacro 2958 |.endmacro
2602 | 2959 |
2603 |->vm_exit_handler: 2960 |->vm_exit_handler:
@@ -2623,16 +2980,16 @@ static void build_subroutines(BuildCtx *ctx)
2623 | savex_ 20,21,22,23 2980 | savex_ 20,21,22,23
2624 | lhz CARG4, 2(CARG3) // Load trace number. 2981 | lhz CARG4, 2(CARG3) // Load trace number.
2625 | savex_ 24,25,26,27 2982 | savex_ 24,25,26,27
2626 | lwz L, DISPATCH_GL(jit_L)(DISPATCH) 2983 | lwz L, DISPATCH_GL(cur_L)(DISPATCH)
2627 | savex_ 28,29,30,31 2984 | savex_ 28,29,30,31
2628 | sub CARG3, TMP0, CARG3 // Compute exit number. 2985 | sub CARG3, TMP0, CARG3 // Compute exit number.
2629 | lp BASE, DISPATCH_GL(jit_base)(DISPATCH) 2986 | lp BASE, DISPATCH_GL(jit_base)(DISPATCH)
2630 | srwi CARG3, CARG3, 2 2987 | srwi CARG3, CARG3, 2
2631 | stw L, DISPATCH_J(L)(DISPATCH) 2988 | stp L, DISPATCH_J(L)(DISPATCH)
2632 | subi CARG3, CARG3, 2 2989 | subi CARG3, CARG3, 2
2633 | stw TMP1, DISPATCH_GL(jit_L)(DISPATCH)
2634 | stw CARG4, DISPATCH_J(parent)(DISPATCH)
2635 | stp BASE, L->base 2990 | stp BASE, L->base
2991 | stw CARG4, DISPATCH_J(parent)(DISPATCH)
2992 | stw TMP1, DISPATCH_GL(jit_base)(DISPATCH)
2636 | addi CARG1, DISPATCH, GG_DISP2J 2993 | addi CARG1, DISPATCH, GG_DISP2J
2637 | stw CARG3, DISPATCH_J(exitno)(DISPATCH) 2994 | stw CARG3, DISPATCH_J(exitno)(DISPATCH)
2638 | addi CARG2, sp, 16 2995 | addi CARG2, sp, 16
@@ -2656,33 +3013,37 @@ static void build_subroutines(BuildCtx *ctx)
2656 | // CARG1 = MULTRES or negated error code, BASE, PC and JGL set. 3013 | // CARG1 = MULTRES or negated error code, BASE, PC and JGL set.
2657 | lwz L, SAVE_L 3014 | lwz L, SAVE_L
2658 | addi DISPATCH, JGL, -GG_DISP2G-32768 3015 | addi DISPATCH, JGL, -GG_DISP2G-32768
3016 | stp BASE, L->base
2659 |1: 3017 |1:
2660 | cmpwi CARG1, 0 3018 | li TMP2, -LUA_ERRERR
2661 | blt >3 // Check for error from exit. 3019 | cmplw CARG1, TMP2
2662 | lwz LFUNC:TMP1, FRAME_FUNC(BASE) 3020 | bge >9 // Check for error from exit.
3021 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2663 | slwi MULTRES, CARG1, 3 3022 | slwi MULTRES, CARG1, 3
2664 | li TMP2, 0 3023 | li TMP2, 0
2665 | stw MULTRES, SAVE_MULTRES 3024 | stw MULTRES, SAVE_MULTRES
2666 | lwz TMP1, LFUNC:TMP1->pc 3025 | lwz TMP1, LFUNC:RB->pc
2667 | stw TMP2, DISPATCH_GL(jit_L)(DISPATCH) 3026 | stw TMP2, DISPATCH_GL(jit_base)(DISPATCH)
2668 | lwz KBASE, PC2PROTO(k)(TMP1) 3027 | lwz KBASE, PC2PROTO(k)(TMP1)
2669 | // Setup type comparison constants. 3028 | // Setup type comparison constants.
2670 | li TISNUM, LJ_TISNUM 3029 | li TISNUM, LJ_TISNUM
2671 | lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 3030 | .FPU lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
2672 | stw TMP3, TMPD 3031 | .FPU stw TMP3, TMPD
2673 | li ZERO, 0 3032 | li ZERO, 0
2674 | ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float). 3033 | .FPU ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float).
2675 | lfs TOBIT, TMPD 3034 | .FPU lfs TOBIT, TMPD
2676 | stw TMP3, TMPD 3035 | .FPU stw TMP3, TMPD
2677 | lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double) 3036 | .FPU lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double)
2678 | li TISNIL, LJ_TNIL 3037 | li TISNIL, LJ_TNIL
2679 | stw TMP0, TONUM_HI 3038 | .FPU stw TMP0, TONUM_HI
2680 | lfs TONUM, TMPD 3039 | .FPU lfs TONUM, TMPD
2681 | // Modified copy of ins_next which handles function header dispatch, too. 3040 | // Modified copy of ins_next which handles function header dispatch, too.
2682 | lwz INS, 0(PC) 3041 | lwz INS, 0(PC)
2683 | addi PC, PC, 4 3042 | addi PC, PC, 4
2684 | // Assumes TISNIL == ~LJ_VMST_INTERP == -1. 3043 | // Assumes TISNIL == ~LJ_VMST_INTERP == -1.
2685 | stw TISNIL, DISPATCH_GL(vmstate)(DISPATCH) 3044 | stw TISNIL, DISPATCH_GL(vmstate)(DISPATCH)
3045 | cmpwi CARG1, -17 // Static dispatch?
3046 | beq >5
2686 | decode_OPP TMP1, INS 3047 | decode_OPP TMP1, INS
2687 | decode_RA8 RA, INS 3048 | decode_RA8 RA, INS
2688 | lpx TMP0, DISPATCH, TMP1 3049 | lpx TMP0, DISPATCH, TMP1
@@ -2694,20 +3055,78 @@ static void build_subroutines(BuildCtx *ctx)
2694 | decode_RC8 RC, INS 3055 | decode_RC8 RC, INS
2695 | bctr 3056 | bctr
2696 |2: 3057 |2:
3058 | cmplwi TMP1, (BC_FUNCC+2)*4 // Fast function?
3059 | blt >3
3060 | // Check frame below fast function.
3061 | lwz TMP1, FRAME_PC(BASE)
3062 | andix. TMP0, TMP1, FRAME_TYPE
3063 | bney >3 // Trace stitching continuation?
3064 | // Otherwise set KBASE for Lua function below fast function.
3065 | lwz TMP2, -4(TMP1)
3066 | decode_RA8 TMP0, TMP2
3067 | sub TMP1, BASE, TMP0
3068 | lwz LFUNC:TMP2, -12(TMP1)
3069 | lwz TMP1, LFUNC:TMP2->pc
3070 | lwz KBASE, PC2PROTO(k)(TMP1)
3071 |3:
2697 | subi RC, MULTRES, 8 3072 | subi RC, MULTRES, 8
2698 | add RA, RA, BASE 3073 | add RA, RA, BASE
2699 | bctr 3074 | bctr
2700 | 3075 |
2701 |3: // Rethrow error from the right C frame. 3076 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
3077 | lwz TMP1, DISPATCH_J(trace)(DISPATCH)
3078 | decode_RD4 RD, INS
3079 | lwzx TRACE:TMP1, TMP1, RD
3080 | lwz INS, TRACE:TMP1->startins
3081 | decode_OPP TMP1, INS
3082 | addi TMP1, TMP1, GG_DISP2STATIC
3083 | lpx TMP0, DISPATCH, TMP1
3084 | mtctr TMP0
3085 | decode_RB8 RB, INS
3086 | decode_RD8 RD, INS
3087 | decode_RA8 RA, INS
3088 | decode_RC8 RC, INS
3089 | bctr
3090 |
3091 |9: // Rethrow error from the right C frame.
3092 | neg CARG2, CARG1
2702 | mr CARG1, L 3093 | mr CARG1, L
2703 | bl extern lj_err_run // (lua_State *L) 3094 | bl extern lj_err_trace // (lua_State *L, int errcode)
2704 |.endif 3095 |.endif
2705 | 3096 |
2706 |//----------------------------------------------------------------------- 3097 |//-----------------------------------------------------------------------
2707 |//-- Math helper functions ---------------------------------------------- 3098 |//-- Math helper functions ----------------------------------------------
2708 |//----------------------------------------------------------------------- 3099 |//-----------------------------------------------------------------------
2709 | 3100 |
2710 |// NYI: Use internal implementations of floor, ceil, trunc. 3101 |// NYI: Use internal implementations of floor, ceil, trunc, sfcmp.
3102 |
3103 |.macro sfi2d, AHI, ALO
3104 |.if not FPU
3105 | mr. AHI, ALO
3106 | bclr 12, 2 // Handle zero first.
3107 | srawi TMP0, ALO, 31
3108 | xor TMP1, ALO, TMP0
3109 | sub TMP1, TMP1, TMP0 // Absolute value in TMP1.
3110 | cntlzw AHI, TMP1
3111 | andix. TMP0, TMP0, 0x800 // Mask sign bit.
3112 | slw TMP1, TMP1, AHI // Align mantissa left with leading 1.
3113 | subfic AHI, AHI, 0x3ff+31-1 // Exponent -1 in AHI.
3114 | slwi ALO, TMP1, 21
3115 | or AHI, AHI, TMP0 // Sign | Exponent.
3116 | srwi TMP1, TMP1, 11
3117 | slwi AHI, AHI, 20 // Align left.
3118 | add AHI, AHI, TMP1 // Add mantissa, increment exponent.
3119 | blr
3120 |.endif
3121 |.endmacro
3122 |
3123 |// Input: CARG2. Output: CARG1, CARG2. Temporaries: TMP0, TMP1.
3124 |->vm_sfi2d_1:
3125 | sfi2d CARG1, CARG2
3126 |
3127 |// Input: CARG4. Output: CARG3, CARG4. Temporaries: TMP0, TMP1.
3128 |->vm_sfi2d_2:
3129 | sfi2d CARG3, CARG4
2711 | 3130 |
2712 |->vm_modi: 3131 |->vm_modi:
2713 | divwo. TMP0, CARG1, CARG2 3132 | divwo. TMP0, CARG1, CARG2
@@ -2762,6 +3181,11 @@ static void build_subroutines(BuildCtx *ctx)
2762 | blr 3181 | blr
2763 |.endif 3182 |.endif
2764 | 3183 |
3184 |->vm_next:
3185 |.if JIT
3186 | NYI // On big-endian.
3187 |.endif
3188 |
2765 |//----------------------------------------------------------------------- 3189 |//-----------------------------------------------------------------------
2766 |//-- FFI helper functions ----------------------------------------------- 3190 |//-- FFI helper functions -----------------------------------------------
2767 |//----------------------------------------------------------------------- 3191 |//-----------------------------------------------------------------------
@@ -2775,21 +3199,21 @@ static void build_subroutines(BuildCtx *ctx)
2775 | addi DISPATCH, r12, GG_G2DISP 3199 | addi DISPATCH, r12, GG_G2DISP
2776 | stw r11, CTSTATE->cb.slot 3200 | stw r11, CTSTATE->cb.slot
2777 | stw r3, CTSTATE->cb.gpr[0] 3201 | stw r3, CTSTATE->cb.gpr[0]
2778 | stfd f1, CTSTATE->cb.fpr[0] 3202 | .FPU stfd f1, CTSTATE->cb.fpr[0]
2779 | stw r4, CTSTATE->cb.gpr[1] 3203 | stw r4, CTSTATE->cb.gpr[1]
2780 | stfd f2, CTSTATE->cb.fpr[1] 3204 | .FPU stfd f2, CTSTATE->cb.fpr[1]
2781 | stw r5, CTSTATE->cb.gpr[2] 3205 | stw r5, CTSTATE->cb.gpr[2]
2782 | stfd f3, CTSTATE->cb.fpr[2] 3206 | .FPU stfd f3, CTSTATE->cb.fpr[2]
2783 | stw r6, CTSTATE->cb.gpr[3] 3207 | stw r6, CTSTATE->cb.gpr[3]
2784 | stfd f4, CTSTATE->cb.fpr[3] 3208 | .FPU stfd f4, CTSTATE->cb.fpr[3]
2785 | stw r7, CTSTATE->cb.gpr[4] 3209 | stw r7, CTSTATE->cb.gpr[4]
2786 | stfd f5, CTSTATE->cb.fpr[4] 3210 | .FPU stfd f5, CTSTATE->cb.fpr[4]
2787 | stw r8, CTSTATE->cb.gpr[5] 3211 | stw r8, CTSTATE->cb.gpr[5]
2788 | stfd f6, CTSTATE->cb.fpr[5] 3212 | .FPU stfd f6, CTSTATE->cb.fpr[5]
2789 | stw r9, CTSTATE->cb.gpr[6] 3213 | stw r9, CTSTATE->cb.gpr[6]
2790 | stfd f7, CTSTATE->cb.fpr[6] 3214 | .FPU stfd f7, CTSTATE->cb.fpr[6]
2791 | stw r10, CTSTATE->cb.gpr[7] 3215 | stw r10, CTSTATE->cb.gpr[7]
2792 | stfd f8, CTSTATE->cb.fpr[7] 3216 | .FPU stfd f8, CTSTATE->cb.fpr[7]
2793 | addi TMP0, sp, CFRAME_SPACE+8 3217 | addi TMP0, sp, CFRAME_SPACE+8
2794 | stw TMP0, CTSTATE->cb.stack 3218 | stw TMP0, CTSTATE->cb.stack
2795 | mr CARG1, CTSTATE 3219 | mr CARG1, CTSTATE
@@ -2800,21 +3224,21 @@ static void build_subroutines(BuildCtx *ctx)
2800 | lp BASE, L:CRET1->base 3224 | lp BASE, L:CRET1->base
2801 | li TISNUM, LJ_TISNUM // Setup type comparison constants. 3225 | li TISNUM, LJ_TISNUM // Setup type comparison constants.
2802 | lp RC, L:CRET1->top 3226 | lp RC, L:CRET1->top
2803 | lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). 3227 | .FPU lus TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
2804 | li ZERO, 0 3228 | li ZERO, 0
2805 | mr L, CRET1 3229 | mr L, CRET1
2806 | stw TMP3, TMPD 3230 | .FPU stw TMP3, TMPD
2807 | lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double) 3231 | .FPU lus TMP0, 0x4338 // Hiword of 2^52 + 2^51 (double)
2808 | lwz LFUNC:RB, FRAME_FUNC(BASE) 3232 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2809 | ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float). 3233 | .FPU ori TMP3, TMP3, 0x0004 // TONUM = 2^52 + 2^51 + 2^31 (float).
2810 | stw TMP0, TONUM_HI 3234 | .FPU stw TMP0, TONUM_HI
2811 | li TISNIL, LJ_TNIL 3235 | li TISNIL, LJ_TNIL
2812 | li_vmstate INTERP 3236 | li_vmstate INTERP
2813 | lfs TOBIT, TMPD 3237 | .FPU lfs TOBIT, TMPD
2814 | stw TMP3, TMPD 3238 | .FPU stw TMP3, TMPD
2815 | sub RC, RC, BASE 3239 | sub RC, RC, BASE
2816 | st_vmstate 3240 | st_vmstate
2817 | lfs TONUM, TMPD 3241 | .FPU lfs TONUM, TMPD
2818 | ins_callt 3242 | ins_callt
2819 |.endif 3243 |.endif
2820 | 3244 |
@@ -2828,7 +3252,7 @@ static void build_subroutines(BuildCtx *ctx)
2828 | mr CARG2, RA 3252 | mr CARG2, RA
2829 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o) 3253 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
2830 | lwz CRET1, CTSTATE->cb.gpr[0] 3254 | lwz CRET1, CTSTATE->cb.gpr[0]
2831 | lfd FARG1, CTSTATE->cb.fpr[0] 3255 | .FPU lfd FARG1, CTSTATE->cb.fpr[0]
2832 | lwz CRET2, CTSTATE->cb.gpr[1] 3256 | lwz CRET2, CTSTATE->cb.gpr[1]
2833 | b ->vm_leave_unw 3257 | b ->vm_leave_unw
2834 |.endif 3258 |.endif
@@ -2862,14 +3286,14 @@ static void build_subroutines(BuildCtx *ctx)
2862 | bge <1 3286 | bge <1
2863 |2: 3287 |2:
2864 | bney cr1, >3 3288 | bney cr1, >3
2865 | lfd f1, CCSTATE->fpr[0] 3289 | .FPU lfd f1, CCSTATE->fpr[0]
2866 | lfd f2, CCSTATE->fpr[1] 3290 | .FPU lfd f2, CCSTATE->fpr[1]
2867 | lfd f3, CCSTATE->fpr[2] 3291 | .FPU lfd f3, CCSTATE->fpr[2]
2868 | lfd f4, CCSTATE->fpr[3] 3292 | .FPU lfd f4, CCSTATE->fpr[3]
2869 | lfd f5, CCSTATE->fpr[4] 3293 | .FPU lfd f5, CCSTATE->fpr[4]
2870 | lfd f6, CCSTATE->fpr[5] 3294 | .FPU lfd f6, CCSTATE->fpr[5]
2871 | lfd f7, CCSTATE->fpr[6] 3295 | .FPU lfd f7, CCSTATE->fpr[6]
2872 | lfd f8, CCSTATE->fpr[7] 3296 | .FPU lfd f8, CCSTATE->fpr[7]
2873 |3: 3297 |3:
2874 | lp TMP0, CCSTATE->func 3298 | lp TMP0, CCSTATE->func
2875 | lwz CARG2, CCSTATE->gpr[1] 3299 | lwz CARG2, CCSTATE->gpr[1]
@@ -2886,7 +3310,7 @@ static void build_subroutines(BuildCtx *ctx)
2886 | lwz TMP2, -4(r14) 3310 | lwz TMP2, -4(r14)
2887 | lwz TMP0, 4(r14) 3311 | lwz TMP0, 4(r14)
2888 | stw CARG1, CCSTATE:TMP1->gpr[0] 3312 | stw CARG1, CCSTATE:TMP1->gpr[0]
2889 | stfd FARG1, CCSTATE:TMP1->fpr[0] 3313 | .FPU stfd FARG1, CCSTATE:TMP1->fpr[0]
2890 | stw CARG2, CCSTATE:TMP1->gpr[1] 3314 | stw CARG2, CCSTATE:TMP1->gpr[1]
2891 | mtlr TMP0 3315 | mtlr TMP0
2892 | stw CARG3, CCSTATE:TMP1->gpr[2] 3316 | stw CARG3, CCSTATE:TMP1->gpr[2]
@@ -2915,19 +3339,19 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2915 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT: 3339 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2916 | // RA = src1*8, RD = src2*8, JMP with RD = target 3340 | // RA = src1*8, RD = src2*8, JMP with RD = target
2917 |.if DUALNUM 3341 |.if DUALNUM
2918 | lwzux TMP0, RA, BASE 3342 | lwzux CARG1, RA, BASE
2919 | addi PC, PC, 4 3343 | addi PC, PC, 4
2920 | lwz CARG2, 4(RA) 3344 | lwz CARG2, 4(RA)
2921 | lwzux TMP1, RD, BASE 3345 | lwzux CARG3, RD, BASE
2922 | lwz TMP2, -4(PC) 3346 | lwz TMP2, -4(PC)
2923 | checknum cr0, TMP0 3347 | checknum cr0, CARG1
2924 | lwz CARG3, 4(RD) 3348 | lwz CARG4, 4(RD)
2925 | decode_RD4 TMP2, TMP2 3349 | decode_RD4 TMP2, TMP2
2926 | checknum cr1, TMP1 3350 | checknum cr1, CARG3
2927 | addis TMP2, TMP2, -(BCBIAS_J*4 >> 16) 3351 | addis SAVE0, TMP2, -(BCBIAS_J*4 >> 16)
2928 | bne cr0, >7 3352 | bne cr0, >7
2929 | bne cr1, >8 3353 | bne cr1, >8
2930 | cmpw CARG2, CARG3 3354 | cmpw CARG2, CARG4
2931 if (op == BC_ISLT) { 3355 if (op == BC_ISLT) {
2932 | bge >2 3356 | bge >2
2933 } else if (op == BC_ISGE) { 3357 } else if (op == BC_ISGE) {
@@ -2938,28 +3362,41 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2938 | ble >2 3362 | ble >2
2939 } 3363 }
2940 |1: 3364 |1:
2941 | add PC, PC, TMP2 3365 | add PC, PC, SAVE0
2942 |2: 3366 |2:
2943 | ins_next 3367 | ins_next
2944 | 3368 |
2945 |7: // RA is not an integer. 3369 |7: // RA is not an integer.
2946 | bgt cr0, ->vmeta_comp 3370 | bgt cr0, ->vmeta_comp
2947 | // RA is a number. 3371 | // RA is a number.
2948 | lfd f0, 0(RA) 3372 | .FPU lfd f0, 0(RA)
2949 | bgt cr1, ->vmeta_comp 3373 | bgt cr1, ->vmeta_comp
2950 | blt cr1, >4 3374 | blt cr1, >4
2951 | // RA is a number, RD is an integer. 3375 | // RA is a number, RD is an integer.
2952 | tonum_i f1, CARG3 3376 |.if FPU
3377 | tonum_i f1, CARG4
3378 |.else
3379 | bl ->vm_sfi2d_2
3380 |.endif
2953 | b >5 3381 | b >5
2954 | 3382 |
2955 |8: // RA is an integer, RD is not an integer. 3383 |8: // RA is an integer, RD is not an integer.
2956 | bgt cr1, ->vmeta_comp 3384 | bgt cr1, ->vmeta_comp
2957 | // RA is an integer, RD is a number. 3385 | // RA is an integer, RD is a number.
3386 |.if FPU
2958 | tonum_i f0, CARG2 3387 | tonum_i f0, CARG2
3388 |.else
3389 | bl ->vm_sfi2d_1
3390 |.endif
2959 |4: 3391 |4:
2960 | lfd f1, 0(RD) 3392 | .FPU lfd f1, 0(RD)
2961 |5: 3393 |5:
3394 |.if FPU
2962 | fcmpu cr0, f0, f1 3395 | fcmpu cr0, f0, f1
3396 |.else
3397 | blex __ledf2
3398 | cmpwi CRET1, 0
3399 |.endif
2963 if (op == BC_ISLT) { 3400 if (op == BC_ISLT) {
2964 | bge <2 3401 | bge <2
2965 } else if (op == BC_ISGE) { 3402 } else if (op == BC_ISGE) {
@@ -3007,42 +3444,42 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3007 vk = op == BC_ISEQV; 3444 vk = op == BC_ISEQV;
3008 | // RA = src1*8, RD = src2*8, JMP with RD = target 3445 | // RA = src1*8, RD = src2*8, JMP with RD = target
3009 |.if DUALNUM 3446 |.if DUALNUM
3010 | lwzux TMP0, RA, BASE 3447 | lwzux CARG1, RA, BASE
3011 | addi PC, PC, 4 3448 | addi PC, PC, 4
3012 | lwz CARG2, 4(RA) 3449 | lwz CARG2, 4(RA)
3013 | lwzux TMP1, RD, BASE 3450 | lwzux CARG3, RD, BASE
3014 | checknum cr0, TMP0 3451 | checknum cr0, CARG1
3015 | lwz TMP2, -4(PC) 3452 | lwz SAVE0, -4(PC)
3016 | checknum cr1, TMP1 3453 | checknum cr1, CARG3
3017 | decode_RD4 TMP2, TMP2 3454 | decode_RD4 SAVE0, SAVE0
3018 | lwz CARG3, 4(RD) 3455 | lwz CARG4, 4(RD)
3019 | cror 4*cr7+gt, 4*cr0+gt, 4*cr1+gt 3456 | cror 4*cr7+gt, 4*cr0+gt, 4*cr1+gt
3020 | addis TMP2, TMP2, -(BCBIAS_J*4 >> 16) 3457 | addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3021 if (vk) { 3458 if (vk) {
3022 | ble cr7, ->BC_ISEQN_Z 3459 | ble cr7, ->BC_ISEQN_Z
3023 } else { 3460 } else {
3024 | ble cr7, ->BC_ISNEN_Z 3461 | ble cr7, ->BC_ISNEN_Z
3025 } 3462 }
3026 |.else 3463 |.else
3027 | lwzux TMP0, RA, BASE 3464 | lwzux CARG1, RA, BASE
3028 | lwz TMP2, 0(PC) 3465 | lwz SAVE0, 0(PC)
3029 | lfd f0, 0(RA) 3466 | lfd f0, 0(RA)
3030 | addi PC, PC, 4 3467 | addi PC, PC, 4
3031 | lwzux TMP1, RD, BASE 3468 | lwzux CARG3, RD, BASE
3032 | checknum cr0, TMP0 3469 | checknum cr0, CARG1
3033 | decode_RD4 TMP2, TMP2 3470 | decode_RD4 SAVE0, SAVE0
3034 | lfd f1, 0(RD) 3471 | lfd f1, 0(RD)
3035 | checknum cr1, TMP1 3472 | checknum cr1, CARG3
3036 | addis TMP2, TMP2, -(BCBIAS_J*4 >> 16) 3473 | addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3037 | bge cr0, >5 3474 | bge cr0, >5
3038 | bge cr1, >5 3475 | bge cr1, >5
3039 | fcmpu cr0, f0, f1 3476 | fcmpu cr0, f0, f1
3040 if (vk) { 3477 if (vk) {
3041 | bne >1 3478 | bne >1
3042 | add PC, PC, TMP2 3479 | add PC, PC, SAVE0
3043 } else { 3480 } else {
3044 | beq >1 3481 | beq >1
3045 | add PC, PC, TMP2 3482 | add PC, PC, SAVE0
3046 } 3483 }
3047 |1: 3484 |1:
3048 | ins_next 3485 | ins_next
@@ -3050,36 +3487,36 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3050 |5: // Either or both types are not numbers. 3487 |5: // Either or both types are not numbers.
3051 |.if not DUALNUM 3488 |.if not DUALNUM
3052 | lwz CARG2, 4(RA) 3489 | lwz CARG2, 4(RA)
3053 | lwz CARG3, 4(RD) 3490 | lwz CARG4, 4(RD)
3054 |.endif 3491 |.endif
3055 |.if FFI 3492 |.if FFI
3056 | cmpwi cr7, TMP0, LJ_TCDATA 3493 | cmpwi cr7, CARG1, LJ_TCDATA
3057 | cmpwi cr5, TMP1, LJ_TCDATA 3494 | cmpwi cr5, CARG3, LJ_TCDATA
3058 |.endif 3495 |.endif
3059 | not TMP3, TMP0 3496 | not TMP2, CARG1
3060 | cmplw TMP0, TMP1 3497 | cmplw CARG1, CARG3
3061 | cmplwi cr1, TMP3, ~LJ_TISPRI // Primitive? 3498 | cmplwi cr1, TMP2, ~LJ_TISPRI // Primitive?
3062 |.if FFI 3499 |.if FFI
3063 | cror 4*cr7+eq, 4*cr7+eq, 4*cr5+eq 3500 | cror 4*cr7+eq, 4*cr7+eq, 4*cr5+eq
3064 |.endif 3501 |.endif
3065 | cmplwi cr6, TMP3, ~LJ_TISTABUD // Table or userdata? 3502 | cmplwi cr6, TMP2, ~LJ_TISTABUD // Table or userdata?
3066 |.if FFI 3503 |.if FFI
3067 | beq cr7, ->vmeta_equal_cd 3504 | beq cr7, ->vmeta_equal_cd
3068 |.endif 3505 |.endif
3069 | cmplw cr5, CARG2, CARG3 3506 | cmplw cr5, CARG2, CARG4
3070 | crandc 4*cr0+gt, 4*cr0+eq, 4*cr1+gt // 2: Same type and primitive. 3507 | crandc 4*cr0+gt, 4*cr0+eq, 4*cr1+gt // 2: Same type and primitive.
3071 | crorc 4*cr0+lt, 4*cr5+eq, 4*cr0+eq // 1: Same tv or different type. 3508 | crorc 4*cr0+lt, 4*cr5+eq, 4*cr0+eq // 1: Same tv or different type.
3072 | crand 4*cr0+eq, 4*cr0+eq, 4*cr5+eq // 0: Same type and same tv. 3509 | crand 4*cr0+eq, 4*cr0+eq, 4*cr5+eq // 0: Same type and same tv.
3073 | mr SAVE0, PC 3510 | mr SAVE1, PC
3074 | cror 4*cr0+eq, 4*cr0+eq, 4*cr0+gt // 0 or 2. 3511 | cror 4*cr0+eq, 4*cr0+eq, 4*cr0+gt // 0 or 2.
3075 | cror 4*cr0+lt, 4*cr0+lt, 4*cr0+gt // 1 or 2. 3512 | cror 4*cr0+lt, 4*cr0+lt, 4*cr0+gt // 1 or 2.
3076 if (vk) { 3513 if (vk) {
3077 | bne cr0, >6 3514 | bne cr0, >6
3078 | add PC, PC, TMP2 3515 | add PC, PC, SAVE0
3079 |6: 3516 |6:
3080 } else { 3517 } else {
3081 | beq cr0, >6 3518 | beq cr0, >6
3082 | add PC, PC, TMP2 3519 | add PC, PC, SAVE0
3083 |6: 3520 |6:
3084 } 3521 }
3085 |.if DUALNUM 3522 |.if DUALNUM
@@ -3094,6 +3531,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3094 | 3531 |
3095 | // Different tables or userdatas. Need to check __eq metamethod. 3532 | // Different tables or userdatas. Need to check __eq metamethod.
3096 | // Field metatable must be at same offset for GCtab and GCudata! 3533 | // Field metatable must be at same offset for GCtab and GCudata!
3534 | mr CARG3, CARG4
3097 | lwz TAB:TMP2, TAB:CARG2->metatable 3535 | lwz TAB:TMP2, TAB:CARG2->metatable
3098 | li CARG4, 1-vk // ne = 0 or 1. 3536 | li CARG4, 1-vk // ne = 0 or 1.
3099 | cmplwi TAB:TMP2, 0 3537 | cmplwi TAB:TMP2, 0
@@ -3101,7 +3539,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3101 | lbz TMP2, TAB:TMP2->nomm 3539 | lbz TMP2, TAB:TMP2->nomm
3102 | andix. TMP2, TMP2, 1<<MM_eq 3540 | andix. TMP2, TMP2, 1<<MM_eq
3103 | bne <1 // Or 'no __eq' flag set? 3541 | bne <1 // Or 'no __eq' flag set?
3104 | mr PC, SAVE0 // Restore old PC. 3542 | mr PC, SAVE1 // Restore old PC.
3105 | b ->vmeta_equal // Handle __eq metamethod. 3543 | b ->vmeta_equal // Handle __eq metamethod.
3106 break; 3544 break;
3107 3545
@@ -3142,16 +3580,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3142 vk = op == BC_ISEQN; 3580 vk = op == BC_ISEQN;
3143 | // RA = src*8, RD = num_const*8, JMP with RD = target 3581 | // RA = src*8, RD = num_const*8, JMP with RD = target
3144 |.if DUALNUM 3582 |.if DUALNUM
3145 | lwzux TMP0, RA, BASE 3583 | lwzux CARG1, RA, BASE
3146 | addi PC, PC, 4 3584 | addi PC, PC, 4
3147 | lwz CARG2, 4(RA) 3585 | lwz CARG2, 4(RA)
3148 | lwzux TMP1, RD, KBASE 3586 | lwzux CARG3, RD, KBASE
3149 | checknum cr0, TMP0 3587 | checknum cr0, CARG1
3150 | lwz TMP2, -4(PC) 3588 | lwz SAVE0, -4(PC)
3151 | checknum cr1, TMP1 3589 | checknum cr1, CARG3
3152 | decode_RD4 TMP2, TMP2 3590 | decode_RD4 SAVE0, SAVE0
3153 | lwz CARG3, 4(RD) 3591 | lwz CARG4, 4(RD)
3154 | addis TMP2, TMP2, -(BCBIAS_J*4 >> 16) 3592 | addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3155 if (vk) { 3593 if (vk) {
3156 |->BC_ISEQN_Z: 3594 |->BC_ISEQN_Z:
3157 } else { 3595 } else {
@@ -3159,7 +3597,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3159 } 3597 }
3160 | bne cr0, >7 3598 | bne cr0, >7
3161 | bne cr1, >8 3599 | bne cr1, >8
3162 | cmpw CARG2, CARG3 3600 | cmpw CARG2, CARG4
3163 |4: 3601 |4:
3164 |.else 3602 |.else
3165 if (vk) { 3603 if (vk) {
@@ -3167,20 +3605,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3167 } else { 3605 } else {
3168 |->BC_ISNEN_Z: // Dummy label. 3606 |->BC_ISNEN_Z: // Dummy label.
3169 } 3607 }
3170 | lwzx TMP0, BASE, RA 3608 | lwzx CARG1, BASE, RA
3171 | addi PC, PC, 4 3609 | addi PC, PC, 4
3172 | lfdx f0, BASE, RA 3610 | lfdx f0, BASE, RA
3173 | lwz TMP2, -4(PC) 3611 | lwz SAVE0, -4(PC)
3174 | lfdx f1, KBASE, RD 3612 | lfdx f1, KBASE, RD
3175 | decode_RD4 TMP2, TMP2 3613 | decode_RD4 SAVE0, SAVE0
3176 | checknum TMP0 3614 | checknum CARG1
3177 | addis TMP2, TMP2, -(BCBIAS_J*4 >> 16) 3615 | addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3178 | bge >3 3616 | bge >3
3179 | fcmpu cr0, f0, f1 3617 | fcmpu cr0, f0, f1
3180 |.endif 3618 |.endif
3181 if (vk) { 3619 if (vk) {
3182 | bne >1 3620 | bne >1
3183 | add PC, PC, TMP2 3621 | add PC, PC, SAVE0
3184 |1: 3622 |1:
3185 |.if not FFI 3623 |.if not FFI
3186 |3: 3624 |3:
@@ -3191,13 +3629,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3191 |.if not FFI 3629 |.if not FFI
3192 |3: 3630 |3:
3193 |.endif 3631 |.endif
3194 | add PC, PC, TMP2 3632 | add PC, PC, SAVE0
3195 |2: 3633 |2:
3196 } 3634 }
3197 | ins_next 3635 | ins_next
3198 |.if FFI 3636 |.if FFI
3199 |3: 3637 |3:
3200 | cmpwi TMP0, LJ_TCDATA 3638 | cmpwi CARG1, LJ_TCDATA
3201 | beq ->vmeta_equal_cd 3639 | beq ->vmeta_equal_cd
3202 | b <1 3640 | b <1
3203 |.endif 3641 |.endif
@@ -3205,18 +3643,31 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3205 |7: // RA is not an integer. 3643 |7: // RA is not an integer.
3206 | bge cr0, <3 3644 | bge cr0, <3
3207 | // RA is a number. 3645 | // RA is a number.
3208 | lfd f0, 0(RA) 3646 | .FPU lfd f0, 0(RA)
3209 | blt cr1, >1 3647 | blt cr1, >1
3210 | // RA is a number, RD is an integer. 3648 | // RA is a number, RD is an integer.
3211 | tonum_i f1, CARG3 3649 |.if FPU
3650 | tonum_i f1, CARG4
3651 |.else
3652 | bl ->vm_sfi2d_2
3653 |.endif
3212 | b >2 3654 | b >2
3213 | 3655 |
3214 |8: // RA is an integer, RD is a number. 3656 |8: // RA is an integer, RD is a number.
3657 |.if FPU
3215 | tonum_i f0, CARG2 3658 | tonum_i f0, CARG2
3659 |.else
3660 | bl ->vm_sfi2d_1
3661 |.endif
3216 |1: 3662 |1:
3217 | lfd f1, 0(RD) 3663 | .FPU lfd f1, 0(RD)
3218 |2: 3664 |2:
3665 |.if FPU
3219 | fcmpu cr0, f0, f1 3666 | fcmpu cr0, f0, f1
3667 |.else
3668 | blex __ledf2
3669 | cmpwi CRET1, 0
3670 |.endif
3220 | b <4 3671 | b <4
3221 |.endif 3672 |.endif
3222 break; 3673 break;
@@ -3271,7 +3722,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3271 | add PC, PC, TMP2 3722 | add PC, PC, TMP2
3272 } else { 3723 } else {
3273 | li TMP1, LJ_TFALSE 3724 | li TMP1, LJ_TFALSE
3725 |.if FPU
3274 | lfdx f0, BASE, RD 3726 | lfdx f0, BASE, RD
3727 |.else
3728 | lwzux CARG1, RD, BASE
3729 | lwz CARG2, 4(RD)
3730 |.endif
3275 | cmplw TMP0, TMP1 3731 | cmplw TMP0, TMP1
3276 if (op == BC_ISTC) { 3732 if (op == BC_ISTC) {
3277 | bge >1 3733 | bge >1
@@ -3280,20 +3736,55 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3280 } 3736 }
3281 | addis PC, PC, -(BCBIAS_J*4 >> 16) 3737 | addis PC, PC, -(BCBIAS_J*4 >> 16)
3282 | decode_RD4 TMP2, INS 3738 | decode_RD4 TMP2, INS
3739 |.if FPU
3283 | stfdx f0, BASE, RA 3740 | stfdx f0, BASE, RA
3741 |.else
3742 | stwux CARG1, RA, BASE
3743 | stw CARG2, 4(RA)
3744 |.endif
3284 | add PC, PC, TMP2 3745 | add PC, PC, TMP2
3285 |1: 3746 |1:
3286 } 3747 }
3287 | ins_next 3748 | ins_next
3288 break; 3749 break;
3289 3750
3751 case BC_ISTYPE:
3752 | // RA = src*8, RD = -type*8
3753 | lwzx TMP0, BASE, RA
3754 | srwi TMP1, RD, 3
3755 | ins_next1
3756 |.if not PPE and not GPR64
3757 | add. TMP0, TMP0, TMP1
3758 |.else
3759 | neg TMP1, TMP1
3760 | cmpw TMP0, TMP1
3761 |.endif
3762 | bne ->vmeta_istype
3763 | ins_next2
3764 break;
3765 case BC_ISNUM:
3766 | // RA = src*8, RD = -(TISNUM-1)*8
3767 | lwzx TMP0, BASE, RA
3768 | ins_next1
3769 | checknum TMP0
3770 | bge ->vmeta_istype
3771 | ins_next2
3772 break;
3773
3290 /* -- Unary ops --------------------------------------------------------- */ 3774 /* -- Unary ops --------------------------------------------------------- */
3291 3775
3292 case BC_MOV: 3776 case BC_MOV:
3293 | // RA = dst*8, RD = src*8 3777 | // RA = dst*8, RD = src*8
3294 | ins_next1 3778 | ins_next1
3779 |.if FPU
3295 | lfdx f0, BASE, RD 3780 | lfdx f0, BASE, RD
3296 | stfdx f0, BASE, RA 3781 | stfdx f0, BASE, RA
3782 |.else
3783 | lwzux TMP0, RD, BASE
3784 | lwz TMP1, 4(RD)
3785 | stwux TMP0, RA, BASE
3786 | stw TMP1, 4(RA)
3787 |.endif
3297 | ins_next2 3788 | ins_next2
3298 break; 3789 break;
3299 case BC_NOT: 3790 case BC_NOT:
@@ -3395,44 +3886,65 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3395 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN); 3886 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3396 ||switch (vk) { 3887 ||switch (vk) {
3397 ||case 0: 3888 ||case 0:
3398 | lwzx TMP1, BASE, RB 3889 | lwzx CARG1, BASE, RB
3399 | .if DUALNUM 3890 | .if DUALNUM
3400 | lwzx TMP2, KBASE, RC 3891 | lwzx CARG3, KBASE, RC
3401 | .endif 3892 | .endif
3893 | .if FPU
3402 | lfdx f14, BASE, RB 3894 | lfdx f14, BASE, RB
3403 | lfdx f15, KBASE, RC 3895 | lfdx f15, KBASE, RC
3896 | .else
3897 | add TMP1, BASE, RB
3898 | add TMP2, KBASE, RC
3899 | lwz CARG2, 4(TMP1)
3900 | lwz CARG4, 4(TMP2)
3901 | .endif
3404 | .if DUALNUM 3902 | .if DUALNUM
3405 | checknum cr0, TMP1 3903 | checknum cr0, CARG1
3406 | checknum cr1, TMP2 3904 | checknum cr1, CARG3
3407 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt 3905 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3408 | bge ->vmeta_arith_vn 3906 | bge ->vmeta_arith_vn
3409 | .else 3907 | .else
3410 | checknum TMP1; bge ->vmeta_arith_vn 3908 | checknum CARG1; bge ->vmeta_arith_vn
3411 | .endif 3909 | .endif
3412 || break; 3910 || break;
3413 ||case 1: 3911 ||case 1:
3414 | lwzx TMP1, BASE, RB 3912 | lwzx CARG1, BASE, RB
3415 | .if DUALNUM 3913 | .if DUALNUM
3416 | lwzx TMP2, KBASE, RC 3914 | lwzx CARG3, KBASE, RC
3417 | .endif 3915 | .endif
3916 | .if FPU
3418 | lfdx f15, BASE, RB 3917 | lfdx f15, BASE, RB
3419 | lfdx f14, KBASE, RC 3918 | lfdx f14, KBASE, RC
3919 | .else
3920 | add TMP1, BASE, RB
3921 | add TMP2, KBASE, RC
3922 | lwz CARG2, 4(TMP1)
3923 | lwz CARG4, 4(TMP2)
3924 | .endif
3420 | .if DUALNUM 3925 | .if DUALNUM
3421 | checknum cr0, TMP1 3926 | checknum cr0, CARG1
3422 | checknum cr1, TMP2 3927 | checknum cr1, CARG3
3423 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt 3928 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3424 | bge ->vmeta_arith_nv 3929 | bge ->vmeta_arith_nv
3425 | .else 3930 | .else
3426 | checknum TMP1; bge ->vmeta_arith_nv 3931 | checknum CARG1; bge ->vmeta_arith_nv
3427 | .endif 3932 | .endif
3428 || break; 3933 || break;
3429 ||default: 3934 ||default:
3430 | lwzx TMP1, BASE, RB 3935 | lwzx CARG1, BASE, RB
3431 | lwzx TMP2, BASE, RC 3936 | lwzx CARG3, BASE, RC
3937 | .if FPU
3432 | lfdx f14, BASE, RB 3938 | lfdx f14, BASE, RB
3433 | lfdx f15, BASE, RC 3939 | lfdx f15, BASE, RC
3434 | checknum cr0, TMP1 3940 | .else
3435 | checknum cr1, TMP2 3941 | add TMP1, BASE, RB
3942 | add TMP2, BASE, RC
3943 | lwz CARG2, 4(TMP1)
3944 | lwz CARG4, 4(TMP2)
3945 | .endif
3946 | checknum cr0, CARG1
3947 | checknum cr1, CARG3
3436 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt 3948 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3437 | bge ->vmeta_arith_vv 3949 | bge ->vmeta_arith_vv
3438 || break; 3950 || break;
@@ -3466,48 +3978,78 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3466 | fsub a, b, a // b - floor(b/c)*c 3978 | fsub a, b, a // b - floor(b/c)*c
3467 |.endmacro 3979 |.endmacro
3468 | 3980 |
3981 |.macro sfpmod
3982 |->BC_MODVN_Z:
3983 | stw CARG1, SFSAVE_1
3984 | stw CARG2, SFSAVE_2
3985 | mr SAVE0, CARG3
3986 | mr SAVE1, CARG4
3987 | blex __divdf3
3988 | blex floor
3989 | mr CARG3, SAVE0
3990 | mr CARG4, SAVE1
3991 | blex __muldf3
3992 | mr CARG3, CRET1
3993 | mr CARG4, CRET2
3994 | lwz CARG1, SFSAVE_1
3995 | lwz CARG2, SFSAVE_2
3996 | blex __subdf3
3997 |.endmacro
3998 |
3469 |.macro ins_arithfp, fpins 3999 |.macro ins_arithfp, fpins
3470 | ins_arithpre 4000 | ins_arithpre
3471 |.if "fpins" == "fpmod_" 4001 |.if "fpins" == "fpmod_"
3472 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway. 4002 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway.
3473 |.else 4003 |.elif FPU
3474 | fpins f0, f14, f15 4004 | fpins f0, f14, f15
3475 | ins_next1 4005 | ins_next1
3476 | stfdx f0, BASE, RA 4006 | stfdx f0, BASE, RA
3477 | ins_next2 4007 | ins_next2
4008 |.else
4009 | blex __divdf3 // Only soft-float div uses this macro.
4010 | ins_next1
4011 | stwux CRET1, RA, BASE
4012 | stw CRET2, 4(RA)
4013 | ins_next2
3478 |.endif 4014 |.endif
3479 |.endmacro 4015 |.endmacro
3480 | 4016 |
3481 |.macro ins_arithdn, intins, fpins 4017 |.macro ins_arithdn, intins, fpins, fpcall
3482 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8 4018 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
3483 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN); 4019 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3484 ||switch (vk) { 4020 ||switch (vk) {
3485 ||case 0: 4021 ||case 0:
3486 | lwzux TMP1, RB, BASE 4022 | lwzux CARG1, RB, BASE
3487 | lwzux TMP2, RC, KBASE 4023 | lwzux CARG3, RC, KBASE
3488 | lwz CARG1, 4(RB) 4024 | lwz CARG2, 4(RB)
3489 | checknum cr0, TMP1 4025 | checknum cr0, CARG1
3490 | lwz CARG2, 4(RC) 4026 | lwz CARG4, 4(RC)
4027 | checknum cr1, CARG3
3491 || break; 4028 || break;
3492 ||case 1: 4029 ||case 1:
3493 | lwzux TMP1, RB, BASE 4030 | lwzux CARG3, RB, BASE
3494 | lwzux TMP2, RC, KBASE 4031 | lwzux CARG1, RC, KBASE
3495 | lwz CARG2, 4(RB) 4032 | lwz CARG4, 4(RB)
3496 | checknum cr0, TMP1 4033 | checknum cr0, CARG3
3497 | lwz CARG1, 4(RC) 4034 | lwz CARG2, 4(RC)
4035 | checknum cr1, CARG1
3498 || break; 4036 || break;
3499 ||default: 4037 ||default:
3500 | lwzux TMP1, RB, BASE 4038 | lwzux CARG1, RB, BASE
3501 | lwzux TMP2, RC, BASE 4039 | lwzux CARG3, RC, BASE
3502 | lwz CARG1, 4(RB) 4040 | lwz CARG2, 4(RB)
3503 | checknum cr0, TMP1 4041 | checknum cr0, CARG1
3504 | lwz CARG2, 4(RC) 4042 | lwz CARG4, 4(RC)
4043 | checknum cr1, CARG3
3505 || break; 4044 || break;
3506 ||} 4045 ||}
3507 | checknum cr1, TMP2
3508 | bne >5 4046 | bne >5
3509 | bne cr1, >5 4047 | bne cr1, >5
3510 | intins CARG1, CARG1, CARG2 4048 |.if "intins" == "intmod"
4049 | mr CARG1, CARG2
4050 | mr CARG2, CARG4
4051 |.endif
4052 | intins CARG1, CARG2, CARG4
3511 | bso >4 4053 | bso >4
3512 |1: 4054 |1:
3513 | ins_next1 4055 | ins_next1
@@ -3519,29 +4061,40 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3519 | checkov TMP0, <1 // Ignore unrelated overflow. 4061 | checkov TMP0, <1 // Ignore unrelated overflow.
3520 | ins_arithfallback b 4062 | ins_arithfallback b
3521 |5: // FP variant. 4063 |5: // FP variant.
4064 |.if FPU
3522 ||if (vk == 1) { 4065 ||if (vk == 1) {
3523 | lfd f15, 0(RB) 4066 | lfd f15, 0(RB)
3524 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3525 | lfd f14, 0(RC) 4067 | lfd f14, 0(RC)
3526 ||} else { 4068 ||} else {
3527 | lfd f14, 0(RB) 4069 | lfd f14, 0(RB)
3528 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3529 | lfd f15, 0(RC) 4070 | lfd f15, 0(RC)
3530 ||} 4071 ||}
4072 |.endif
4073 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3531 | ins_arithfallback bge 4074 | ins_arithfallback bge
3532 |.if "fpins" == "fpmod_" 4075 |.if "fpins" == "fpmod_"
3533 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway. 4076 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway.
3534 |.else 4077 |.else
4078 |.if FPU
3535 | fpins f0, f14, f15 4079 | fpins f0, f14, f15
3536 | ins_next1
3537 | stfdx f0, BASE, RA 4080 | stfdx f0, BASE, RA
4081 |.else
4082 |.if "fpcall" == "sfpmod"
4083 | sfpmod
4084 |.else
4085 | blex fpcall
4086 |.endif
4087 | stwux CRET1, RA, BASE
4088 | stw CRET2, 4(RA)
4089 |.endif
4090 | ins_next1
3538 | b <2 4091 | b <2
3539 |.endif 4092 |.endif
3540 |.endmacro 4093 |.endmacro
3541 | 4094 |
3542 |.macro ins_arith, intins, fpins 4095 |.macro ins_arith, intins, fpins, fpcall
3543 |.if DUALNUM 4096 |.if DUALNUM
3544 | ins_arithdn intins, fpins 4097 | ins_arithdn intins, fpins, fpcall
3545 |.else 4098 |.else
3546 | ins_arithfp fpins 4099 | ins_arithfp fpins
3547 |.endif 4100 |.endif
@@ -3556,9 +4109,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3556 | addo. TMP0, TMP0, TMP1 4109 | addo. TMP0, TMP0, TMP1
3557 | add y, a, b 4110 | add y, a, b
3558 |.endmacro 4111 |.endmacro
3559 | ins_arith addo32., fadd 4112 | ins_arith addo32., fadd, __adddf3
3560 |.else 4113 |.else
3561 | ins_arith addo., fadd 4114 | ins_arith addo., fadd, __adddf3
3562 |.endif 4115 |.endif
3563 break; 4116 break;
3564 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV: 4117 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
@@ -3570,36 +4123,48 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3570 | subo. TMP0, TMP0, TMP1 4123 | subo. TMP0, TMP0, TMP1
3571 | sub y, a, b 4124 | sub y, a, b
3572 |.endmacro 4125 |.endmacro
3573 | ins_arith subo32., fsub 4126 | ins_arith subo32., fsub, __subdf3
3574 |.else 4127 |.else
3575 | ins_arith subo., fsub 4128 | ins_arith subo., fsub, __subdf3
3576 |.endif 4129 |.endif
3577 break; 4130 break;
3578 case BC_MULVN: case BC_MULNV: case BC_MULVV: 4131 case BC_MULVN: case BC_MULNV: case BC_MULVV:
3579 | ins_arith mullwo., fmul 4132 | ins_arith mullwo., fmul, __muldf3
3580 break; 4133 break;
3581 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV: 4134 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
3582 | ins_arithfp fdiv 4135 | ins_arithfp fdiv
3583 break; 4136 break;
3584 case BC_MODVN: 4137 case BC_MODVN:
3585 | ins_arith intmod, fpmod 4138 | ins_arith intmod, fpmod, sfpmod
3586 break; 4139 break;
3587 case BC_MODNV: case BC_MODVV: 4140 case BC_MODNV: case BC_MODVV:
3588 | ins_arith intmod, fpmod_ 4141 | ins_arith intmod, fpmod_, sfpmod
3589 break; 4142 break;
3590 case BC_POW: 4143 case BC_POW:
3591 | // NYI: (partial) integer arithmetic. 4144 | // NYI: (partial) integer arithmetic.
3592 | lwzx TMP1, BASE, RB 4145 | lwzx CARG1, BASE, RB
4146 | lwzx CARG3, BASE, RC
4147 |.if FPU
3593 | lfdx FARG1, BASE, RB 4148 | lfdx FARG1, BASE, RB
3594 | lwzx TMP2, BASE, RC
3595 | lfdx FARG2, BASE, RC 4149 | lfdx FARG2, BASE, RC
3596 | checknum cr0, TMP1 4150 |.else
3597 | checknum cr1, TMP2 4151 | add TMP1, BASE, RB
4152 | add TMP2, BASE, RC
4153 | lwz CARG2, 4(TMP1)
4154 | lwz CARG4, 4(TMP2)
4155 |.endif
4156 | checknum cr0, CARG1
4157 | checknum cr1, CARG3
3598 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt 4158 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3599 | bge ->vmeta_arith_vv 4159 | bge ->vmeta_arith_vv
3600 | blex pow 4160 | blex pow
3601 | ins_next1 4161 | ins_next1
4162 |.if FPU
3602 | stfdx FARG1, BASE, RA 4163 | stfdx FARG1, BASE, RA
4164 |.else
4165 | stwux CARG1, RA, BASE
4166 | stw CARG2, 4(RA)
4167 |.endif
3603 | ins_next2 4168 | ins_next2
3604 break; 4169 break;
3605 4170
@@ -3619,8 +4184,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3619 | lp BASE, L->base 4184 | lp BASE, L->base
3620 | bne ->vmeta_binop 4185 | bne ->vmeta_binop
3621 | ins_next1 4186 | ins_next1
4187 |.if FPU
3622 | lfdx f0, BASE, SAVE0 // Copy result from RB to RA. 4188 | lfdx f0, BASE, SAVE0 // Copy result from RB to RA.
3623 | stfdx f0, BASE, RA 4189 | stfdx f0, BASE, RA
4190 |.else
4191 | lwzux TMP0, SAVE0, BASE
4192 | lwz TMP1, 4(SAVE0)
4193 | stwux TMP0, RA, BASE
4194 | stw TMP1, 4(RA)
4195 |.endif
3624 | ins_next2 4196 | ins_next2
3625 break; 4197 break;
3626 4198
@@ -3683,8 +4255,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3683 case BC_KNUM: 4255 case BC_KNUM:
3684 | // RA = dst*8, RD = num_const*8 4256 | // RA = dst*8, RD = num_const*8
3685 | ins_next1 4257 | ins_next1
4258 |.if FPU
3686 | lfdx f0, KBASE, RD 4259 | lfdx f0, KBASE, RD
3687 | stfdx f0, BASE, RA 4260 | stfdx f0, BASE, RA
4261 |.else
4262 | lwzux TMP0, RD, KBASE
4263 | lwz TMP1, 4(RD)
4264 | stwux TMP0, RA, BASE
4265 | stw TMP1, 4(RA)
4266 |.endif
3688 | ins_next2 4267 | ins_next2
3689 break; 4268 break;
3690 case BC_KPRI: 4269 case BC_KPRI:
@@ -3717,8 +4296,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3717 | lwzx UPVAL:RB, LFUNC:RB, RD 4296 | lwzx UPVAL:RB, LFUNC:RB, RD
3718 | ins_next1 4297 | ins_next1
3719 | lwz TMP1, UPVAL:RB->v 4298 | lwz TMP1, UPVAL:RB->v
4299 |.if FPU
3720 | lfd f0, 0(TMP1) 4300 | lfd f0, 0(TMP1)
3721 | stfdx f0, BASE, RA 4301 | stfdx f0, BASE, RA
4302 |.else
4303 | lwz TMP2, 0(TMP1)
4304 | lwz TMP3, 4(TMP1)
4305 | stwux TMP2, RA, BASE
4306 | stw TMP3, 4(RA)
4307 |.endif
3722 | ins_next2 4308 | ins_next2
3723 break; 4309 break;
3724 case BC_USETV: 4310 case BC_USETV:
@@ -3726,14 +4312,24 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3726 | lwz LFUNC:RB, FRAME_FUNC(BASE) 4312 | lwz LFUNC:RB, FRAME_FUNC(BASE)
3727 | srwi RA, RA, 1 4313 | srwi RA, RA, 1
3728 | addi RA, RA, offsetof(GCfuncL, uvptr) 4314 | addi RA, RA, offsetof(GCfuncL, uvptr)
4315 |.if FPU
3729 | lfdux f0, RD, BASE 4316 | lfdux f0, RD, BASE
4317 |.else
4318 | lwzux CARG1, RD, BASE
4319 | lwz CARG3, 4(RD)
4320 |.endif
3730 | lwzx UPVAL:RB, LFUNC:RB, RA 4321 | lwzx UPVAL:RB, LFUNC:RB, RA
3731 | lbz TMP3, UPVAL:RB->marked 4322 | lbz TMP3, UPVAL:RB->marked
3732 | lwz CARG2, UPVAL:RB->v 4323 | lwz CARG2, UPVAL:RB->v
3733 | andix. TMP3, TMP3, LJ_GC_BLACK // isblack(uv) 4324 | andix. TMP3, TMP3, LJ_GC_BLACK // isblack(uv)
3734 | lbz TMP0, UPVAL:RB->closed 4325 | lbz TMP0, UPVAL:RB->closed
3735 | lwz TMP2, 0(RD) 4326 | lwz TMP2, 0(RD)
4327 |.if FPU
3736 | stfd f0, 0(CARG2) 4328 | stfd f0, 0(CARG2)
4329 |.else
4330 | stw CARG1, 0(CARG2)
4331 | stw CARG3, 4(CARG2)
4332 |.endif
3737 | cmplwi cr1, TMP0, 0 4333 | cmplwi cr1, TMP0, 0
3738 | lwz TMP1, 4(RD) 4334 | lwz TMP1, 4(RD)
3739 | cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq 4335 | cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
@@ -3789,11 +4385,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3789 | lwz LFUNC:RB, FRAME_FUNC(BASE) 4385 | lwz LFUNC:RB, FRAME_FUNC(BASE)
3790 | srwi RA, RA, 1 4386 | srwi RA, RA, 1
3791 | addi RA, RA, offsetof(GCfuncL, uvptr) 4387 | addi RA, RA, offsetof(GCfuncL, uvptr)
4388 |.if FPU
3792 | lfdx f0, KBASE, RD 4389 | lfdx f0, KBASE, RD
4390 |.else
4391 | lwzux TMP2, RD, KBASE
4392 | lwz TMP3, 4(RD)
4393 |.endif
3793 | lwzx UPVAL:RB, LFUNC:RB, RA 4394 | lwzx UPVAL:RB, LFUNC:RB, RA
3794 | ins_next1 4395 | ins_next1
3795 | lwz TMP1, UPVAL:RB->v 4396 | lwz TMP1, UPVAL:RB->v
4397 |.if FPU
3796 | stfd f0, 0(TMP1) 4398 | stfd f0, 0(TMP1)
4399 |.else
4400 | stw TMP2, 0(TMP1)
4401 | stw TMP3, 4(TMP1)
4402 |.endif
3797 | ins_next2 4403 | ins_next2
3798 break; 4404 break;
3799 case BC_USETP: 4405 case BC_USETP:
@@ -3941,11 +4547,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3941 |.endif 4547 |.endif
3942 | ble ->vmeta_tgetv // Integer key and in array part? 4548 | ble ->vmeta_tgetv // Integer key and in array part?
3943 | lwzx TMP0, TMP1, TMP2 4549 | lwzx TMP0, TMP1, TMP2
4550 |.if FPU
3944 | lfdx f14, TMP1, TMP2 4551 | lfdx f14, TMP1, TMP2
4552 |.else
4553 | lwzux SAVE0, TMP1, TMP2
4554 | lwz SAVE1, 4(TMP1)
4555 |.endif
3945 | checknil TMP0; beq >2 4556 | checknil TMP0; beq >2
3946 |1: 4557 |1:
3947 | ins_next1 4558 | ins_next1
4559 |.if FPU
3948 | stfdx f14, BASE, RA 4560 | stfdx f14, BASE, RA
4561 |.else
4562 | stwux SAVE0, RA, BASE
4563 | stw SAVE1, 4(RA)
4564 |.endif
3949 | ins_next2 4565 | ins_next2
3950 | 4566 |
3951 |2: // Check for __index if table value is nil. 4567 |2: // Check for __index if table value is nil.
@@ -3976,9 +4592,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3976 |->BC_TGETS_Z: 4592 |->BC_TGETS_Z:
3977 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 4593 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8
3978 | lwz TMP0, TAB:RB->hmask 4594 | lwz TMP0, TAB:RB->hmask
3979 | lwz TMP1, STR:RC->hash 4595 | lwz TMP1, STR:RC->sid
3980 | lwz NODE:TMP2, TAB:RB->node 4596 | lwz NODE:TMP2, TAB:RB->node
3981 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 4597 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
3982 | slwi TMP0, TMP1, 5 4598 | slwi TMP0, TMP1, 5
3983 | slwi TMP1, TMP1, 3 4599 | slwi TMP1, TMP1, 3
3984 | sub TMP1, TMP0, TMP1 4600 | sub TMP1, TMP0, TMP1
@@ -4021,12 +4637,22 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4021 | lwz TMP1, TAB:RB->asize 4637 | lwz TMP1, TAB:RB->asize
4022 | lwz TMP2, TAB:RB->array 4638 | lwz TMP2, TAB:RB->array
4023 | cmplw TMP0, TMP1; bge ->vmeta_tgetb 4639 | cmplw TMP0, TMP1; bge ->vmeta_tgetb
4640 |.if FPU
4024 | lwzx TMP1, TMP2, RC 4641 | lwzx TMP1, TMP2, RC
4025 | lfdx f0, TMP2, RC 4642 | lfdx f0, TMP2, RC
4643 |.else
4644 | lwzux TMP1, TMP2, RC
4645 | lwz TMP3, 4(TMP2)
4646 |.endif
4026 | checknil TMP1; beq >5 4647 | checknil TMP1; beq >5
4027 |1: 4648 |1:
4028 | ins_next1 4649 | ins_next1
4650 |.if FPU
4029 | stfdx f0, BASE, RA 4651 | stfdx f0, BASE, RA
4652 |.else
4653 | stwux TMP1, RA, BASE
4654 | stw TMP3, 4(RA)
4655 |.endif
4030 | ins_next2 4656 | ins_next2
4031 | 4657 |
4032 |5: // Check for __index if table value is nil. 4658 |5: // Check for __index if table value is nil.
@@ -4038,6 +4664,40 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4038 | bne <1 // 'no __index' flag set: done. 4664 | bne <1 // 'no __index' flag set: done.
4039 | b ->vmeta_tgetb // Caveat: preserve TMP0! 4665 | b ->vmeta_tgetb // Caveat: preserve TMP0!
4040 break; 4666 break;
4667 case BC_TGETR:
4668 | // RA = dst*8, RB = table*8, RC = key*8
4669 | add RB, BASE, RB
4670 | lwz TAB:CARG1, 4(RB)
4671 |.if DUALNUM
4672 | add RC, BASE, RC
4673 | lwz TMP0, TAB:CARG1->asize
4674 | lwz CARG2, 4(RC)
4675 | lwz TMP1, TAB:CARG1->array
4676 |.else
4677 | lfdx f0, BASE, RC
4678 | lwz TMP0, TAB:CARG1->asize
4679 | toint CARG2, f0
4680 | lwz TMP1, TAB:CARG1->array
4681 |.endif
4682 | cmplw TMP0, CARG2
4683 | slwi TMP2, CARG2, 3
4684 | ble ->vmeta_tgetr // In array part?
4685 |.if FPU
4686 | lfdx f14, TMP1, TMP2
4687 |.else
4688 | lwzux SAVE0, TMP2, TMP1
4689 | lwz SAVE1, 4(TMP2)
4690 |.endif
4691 |->BC_TGETR_Z:
4692 | ins_next1
4693 |.if FPU
4694 | stfdx f14, BASE, RA
4695 |.else
4696 | stwux SAVE0, RA, BASE
4697 | stw SAVE1, 4(RA)
4698 |.endif
4699 | ins_next2
4700 break;
4041 4701
4042 case BC_TSETV: 4702 case BC_TSETV:
4043 | // RA = src*8, RB = table*8, RC = key*8 4703 | // RA = src*8, RB = table*8, RC = key*8
@@ -4076,11 +4736,22 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4076 | ble ->vmeta_tsetv // Integer key and in array part? 4736 | ble ->vmeta_tsetv // Integer key and in array part?
4077 | lwzx TMP2, TMP1, TMP0 4737 | lwzx TMP2, TMP1, TMP0
4078 | lbz TMP3, TAB:RB->marked 4738 | lbz TMP3, TAB:RB->marked
4739 |.if FPU
4079 | lfdx f14, BASE, RA 4740 | lfdx f14, BASE, RA
4741 |.else
4742 | add SAVE1, BASE, RA
4743 | lwz SAVE0, 0(SAVE1)
4744 | lwz SAVE1, 4(SAVE1)
4745 |.endif
4080 | checknil TMP2; beq >3 4746 | checknil TMP2; beq >3
4081 |1: 4747 |1:
4082 | andix. TMP2, TMP3, LJ_GC_BLACK // isblack(table) 4748 | andix. TMP2, TMP3, LJ_GC_BLACK // isblack(table)
4749 |.if FPU
4083 | stfdx f14, TMP1, TMP0 4750 | stfdx f14, TMP1, TMP0
4751 |.else
4752 | stwux SAVE0, TMP1, TMP0
4753 | stw SAVE1, 4(TMP1)
4754 |.endif
4084 | bne >7 4755 | bne >7
4085 |2: 4756 |2:
4086 | ins_next 4757 | ins_next
@@ -4117,11 +4788,17 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4117 |->BC_TSETS_Z: 4788 |->BC_TSETS_Z:
4118 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8 4789 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8
4119 | lwz TMP0, TAB:RB->hmask 4790 | lwz TMP0, TAB:RB->hmask
4120 | lwz TMP1, STR:RC->hash 4791 | lwz TMP1, STR:RC->sid
4121 | lwz NODE:TMP2, TAB:RB->node 4792 | lwz NODE:TMP2, TAB:RB->node
4122 | stb ZERO, TAB:RB->nomm // Clear metamethod cache. 4793 | stb ZERO, TAB:RB->nomm // Clear metamethod cache.
4123 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask 4794 | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask
4795 |.if FPU
4124 | lfdx f14, BASE, RA 4796 | lfdx f14, BASE, RA
4797 |.else
4798 | add CARG2, BASE, RA
4799 | lwz SAVE0, 0(CARG2)
4800 | lwz SAVE1, 4(CARG2)
4801 |.endif
4125 | slwi TMP0, TMP1, 5 4802 | slwi TMP0, TMP1, 5
4126 | slwi TMP1, TMP1, 3 4803 | slwi TMP1, TMP1, 3
4127 | sub TMP1, TMP0, TMP1 4804 | sub TMP1, TMP0, TMP1
@@ -4137,7 +4814,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4137 | checknil CARG2; beq >4 // Key found, but nil value? 4814 | checknil CARG2; beq >4 // Key found, but nil value?
4138 |2: 4815 |2:
4139 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table) 4816 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
4817 |.if FPU
4140 | stfd f14, NODE:TMP2->val 4818 | stfd f14, NODE:TMP2->val
4819 |.else
4820 | stw SAVE0, NODE:TMP2->val.u32.hi
4821 | stw SAVE1, NODE:TMP2->val.u32.lo
4822 |.endif
4141 | bne >7 4823 | bne >7
4142 |3: 4824 |3:
4143 | ins_next 4825 | ins_next
@@ -4176,7 +4858,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4176 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k) 4858 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
4177 | // Returns TValue *. 4859 | // Returns TValue *.
4178 | lp BASE, L->base 4860 | lp BASE, L->base
4861 |.if FPU
4179 | stfd f14, 0(CRET1) 4862 | stfd f14, 0(CRET1)
4863 |.else
4864 | stw SAVE0, 0(CRET1)
4865 | stw SAVE1, 4(CRET1)
4866 |.endif
4180 | b <3 // No 2nd write barrier needed. 4867 | b <3 // No 2nd write barrier needed.
4181 | 4868 |
4182 |7: // Possible table write barrier for the value. Skip valiswhite check. 4869 |7: // Possible table write barrier for the value. Skip valiswhite check.
@@ -4193,13 +4880,24 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4193 | lwz TMP2, TAB:RB->array 4880 | lwz TMP2, TAB:RB->array
4194 | lbz TMP3, TAB:RB->marked 4881 | lbz TMP3, TAB:RB->marked
4195 | cmplw TMP0, TMP1 4882 | cmplw TMP0, TMP1
4883 |.if FPU
4196 | lfdx f14, BASE, RA 4884 | lfdx f14, BASE, RA
4885 |.else
4886 | add CARG2, BASE, RA
4887 | lwz SAVE0, 0(CARG2)
4888 | lwz SAVE1, 4(CARG2)
4889 |.endif
4197 | bge ->vmeta_tsetb 4890 | bge ->vmeta_tsetb
4198 | lwzx TMP1, TMP2, RC 4891 | lwzx TMP1, TMP2, RC
4199 | checknil TMP1; beq >5 4892 | checknil TMP1; beq >5
4200 |1: 4893 |1:
4201 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table) 4894 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
4895 |.if FPU
4202 | stfdx f14, TMP2, RC 4896 | stfdx f14, TMP2, RC
4897 |.else
4898 | stwux SAVE0, RC, TMP2
4899 | stw SAVE1, 4(RC)
4900 |.endif
4203 | bne >7 4901 | bne >7
4204 |2: 4902 |2:
4205 | ins_next 4903 | ins_next
@@ -4217,6 +4915,49 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4217 | barrierback TAB:RB, TMP3, TMP0 4915 | barrierback TAB:RB, TMP3, TMP0
4218 | b <2 4916 | b <2
4219 break; 4917 break;
4918 case BC_TSETR:
4919 | // RA = dst*8, RB = table*8, RC = key*8
4920 | add RB, BASE, RB
4921 | lwz TAB:CARG2, 4(RB)
4922 |.if DUALNUM
4923 | add RC, BASE, RC
4924 | lbz TMP3, TAB:CARG2->marked
4925 | lwz TMP0, TAB:CARG2->asize
4926 | lwz CARG3, 4(RC)
4927 | lwz TMP1, TAB:CARG2->array
4928 |.else
4929 | lfdx f0, BASE, RC
4930 | lbz TMP3, TAB:CARG2->marked
4931 | lwz TMP0, TAB:CARG2->asize
4932 | toint CARG3, f0
4933 | lwz TMP1, TAB:CARG2->array
4934 |.endif
4935 | andix. TMP2, TMP3, LJ_GC_BLACK // isblack(table)
4936 | bne >7
4937 |2:
4938 | cmplw TMP0, CARG3
4939 | slwi TMP2, CARG3, 3
4940 |.if FPU
4941 | lfdx f14, BASE, RA
4942 |.else
4943 | lwzux SAVE0, RA, BASE
4944 | lwz SAVE1, 4(RA)
4945 |.endif
4946 | ble ->vmeta_tsetr // In array part?
4947 | ins_next1
4948 |.if FPU
4949 | stfdx f14, TMP1, TMP2
4950 |.else
4951 | stwux SAVE0, TMP1, TMP2
4952 | stw SAVE1, 4(TMP1)
4953 |.endif
4954 | ins_next2
4955 |
4956 |7: // Possible table write barrier for the value. Skip valiswhite check.
4957 | barrierback TAB:CARG2, TMP3, TMP2
4958 | b <2
4959 break;
4960
4220 4961
4221 case BC_TSETM: 4962 case BC_TSETM:
4222 | // RA = base*8 (table at base-1), RD = num_const*8 (start index) 4963 | // RA = base*8 (table at base-1), RD = num_const*8 (start index)
@@ -4239,10 +4980,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4239 | add TMP1, TMP1, TMP0 4980 | add TMP1, TMP1, TMP0
4240 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table) 4981 | andix. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
4241 |3: // Copy result slots to table. 4982 |3: // Copy result slots to table.
4983 |.if FPU
4242 | lfd f0, 0(RA) 4984 | lfd f0, 0(RA)
4985 |.else
4986 | lwz SAVE0, 0(RA)
4987 | lwz SAVE1, 4(RA)
4988 |.endif
4243 | addi RA, RA, 8 4989 | addi RA, RA, 8
4244 | cmpw cr1, RA, TMP2 4990 | cmpw cr1, RA, TMP2
4991 |.if FPU
4245 | stfd f0, 0(TMP1) 4992 | stfd f0, 0(TMP1)
4993 |.else
4994 | stw SAVE0, 0(TMP1)
4995 | stw SAVE1, 4(TMP1)
4996 |.endif
4246 | addi TMP1, TMP1, 8 4997 | addi TMP1, TMP1, 8
4247 | blt cr1, <3 4998 | blt cr1, <3
4248 | bne >7 4999 | bne >7
@@ -4309,9 +5060,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4309 | beq cr1, >3 5060 | beq cr1, >3
4310 |2: 5061 |2:
4311 | addi TMP3, TMP2, 8 5062 | addi TMP3, TMP2, 8
5063 |.if FPU
4312 | lfdx f0, RA, TMP2 5064 | lfdx f0, RA, TMP2
5065 |.else
5066 | add CARG3, RA, TMP2
5067 | lwz CARG1, 0(CARG3)
5068 | lwz CARG2, 4(CARG3)
5069 |.endif
4313 | cmplw cr1, TMP3, NARGS8:RC 5070 | cmplw cr1, TMP3, NARGS8:RC
5071 |.if FPU
4314 | stfdx f0, BASE, TMP2 5072 | stfdx f0, BASE, TMP2
5073 |.else
5074 | stwux CARG1, TMP2, BASE
5075 | stw CARG2, 4(TMP2)
5076 |.endif
4315 | mr TMP2, TMP3 5077 | mr TMP2, TMP3
4316 | bne cr1, <2 5078 | bne cr1, <2
4317 |3: 5079 |3:
@@ -4344,14 +5106,28 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4344 | add BASE, BASE, RA 5106 | add BASE, BASE, RA
4345 | lwz TMP1, -24(BASE) 5107 | lwz TMP1, -24(BASE)
4346 | lwz LFUNC:RB, -20(BASE) 5108 | lwz LFUNC:RB, -20(BASE)
5109 |.if FPU
4347 | lfd f1, -8(BASE) 5110 | lfd f1, -8(BASE)
4348 | lfd f0, -16(BASE) 5111 | lfd f0, -16(BASE)
5112 |.else
5113 | lwz CARG1, -8(BASE)
5114 | lwz CARG2, -4(BASE)
5115 | lwz CARG3, -16(BASE)
5116 | lwz CARG4, -12(BASE)
5117 |.endif
4349 | stw TMP1, 0(BASE) // Copy callable. 5118 | stw TMP1, 0(BASE) // Copy callable.
4350 | stw LFUNC:RB, 4(BASE) 5119 | stw LFUNC:RB, 4(BASE)
4351 | checkfunc TMP1 5120 | checkfunc TMP1
4352 | stfd f1, 16(BASE) // Copy control var.
4353 | li NARGS8:RC, 16 // Iterators get 2 arguments. 5121 | li NARGS8:RC, 16 // Iterators get 2 arguments.
5122 |.if FPU
5123 | stfd f1, 16(BASE) // Copy control var.
4354 | stfdu f0, 8(BASE) // Copy state. 5124 | stfdu f0, 8(BASE) // Copy state.
5125 |.else
5126 | stw CARG1, 16(BASE) // Copy control var.
5127 | stw CARG2, 20(BASE)
5128 | stwu CARG3, 8(BASE) // Copy state.
5129 | stw CARG4, 4(BASE)
5130 |.endif
4355 | bne ->vmeta_call 5131 | bne ->vmeta_call
4356 | ins_call 5132 | ins_call
4357 break; 5133 break;
@@ -4359,8 +5135,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4359 case BC_ITERN: 5135 case BC_ITERN:
4360 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8) 5136 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8)
4361 |.if JIT 5137 |.if JIT
4362 | // NYI: add hotloop, record BC_ITERN. 5138 | // NYI on big-endian
4363 |.endif 5139 |.endif
5140 |->vm_IITERN:
4364 | add RA, BASE, RA 5141 | add RA, BASE, RA
4365 | lwz TAB:RB, -12(RA) 5142 | lwz TAB:RB, -12(RA)
4366 | lwz RC, -4(RA) // Get index from control var. 5143 | lwz RC, -4(RA) // Get index from control var.
@@ -4372,7 +5149,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4372 | slwi TMP3, RC, 3 5149 | slwi TMP3, RC, 3
4373 | bge >5 // Index points after array part? 5150 | bge >5 // Index points after array part?
4374 | lwzx TMP2, TMP1, TMP3 5151 | lwzx TMP2, TMP1, TMP3
5152 |.if FPU
4375 | lfdx f0, TMP1, TMP3 5153 | lfdx f0, TMP1, TMP3
5154 |.else
5155 | lwzux CARG1, TMP3, TMP1
5156 | lwz CARG2, 4(TMP3)
5157 |.endif
4376 | checknil TMP2 5158 | checknil TMP2
4377 | lwz INS, -4(PC) 5159 | lwz INS, -4(PC)
4378 | beq >4 5160 | beq >4
@@ -4384,7 +5166,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4384 |.endif 5166 |.endif
4385 | addi RC, RC, 1 5167 | addi RC, RC, 1
4386 | addis TMP3, PC, -(BCBIAS_J*4 >> 16) 5168 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
5169 |.if FPU
4387 | stfd f0, 8(RA) 5170 | stfd f0, 8(RA)
5171 |.else
5172 | stw CARG1, 8(RA)
5173 | stw CARG2, 12(RA)
5174 |.endif
4388 | decode_RD4 TMP1, INS 5175 | decode_RD4 TMP1, INS
4389 | stw RC, -4(RA) // Update control var. 5176 | stw RC, -4(RA) // Update control var.
4390 | add PC, TMP1, TMP3 5177 | add PC, TMP1, TMP3
@@ -4409,17 +5196,38 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4409 | slwi RB, RC, 3 5196 | slwi RB, RC, 3
4410 | sub TMP3, TMP3, RB 5197 | sub TMP3, TMP3, RB
4411 | lwzx RB, TMP2, TMP3 5198 | lwzx RB, TMP2, TMP3
5199 |.if FPU
4412 | lfdx f0, TMP2, TMP3 5200 | lfdx f0, TMP2, TMP3
5201 |.else
5202 | add CARG3, TMP2, TMP3
5203 | lwz CARG1, 0(CARG3)
5204 | lwz CARG2, 4(CARG3)
5205 |.endif
4413 | add NODE:TMP3, TMP2, TMP3 5206 | add NODE:TMP3, TMP2, TMP3
4414 | checknil RB 5207 | checknil RB
4415 | lwz INS, -4(PC) 5208 | lwz INS, -4(PC)
4416 | beq >7 5209 | beq >7
5210 |.if FPU
4417 | lfd f1, NODE:TMP3->key 5211 | lfd f1, NODE:TMP3->key
5212 |.else
5213 | lwz CARG3, NODE:TMP3->key.u32.hi
5214 | lwz CARG4, NODE:TMP3->key.u32.lo
5215 |.endif
4418 | addis TMP2, PC, -(BCBIAS_J*4 >> 16) 5216 | addis TMP2, PC, -(BCBIAS_J*4 >> 16)
5217 |.if FPU
4419 | stfd f0, 8(RA) 5218 | stfd f0, 8(RA)
5219 |.else
5220 | stw CARG1, 8(RA)
5221 | stw CARG2, 12(RA)
5222 |.endif
4420 | add RC, RC, TMP0 5223 | add RC, RC, TMP0
4421 | decode_RD4 TMP1, INS 5224 | decode_RD4 TMP1, INS
5225 |.if FPU
4422 | stfd f1, 0(RA) 5226 | stfd f1, 0(RA)
5227 |.else
5228 | stw CARG3, 0(RA)
5229 | stw CARG4, 4(RA)
5230 |.endif
4423 | addi RC, RC, 1 5231 | addi RC, RC, 1
4424 | add PC, TMP1, TMP2 5232 | add PC, TMP1, TMP2
4425 | stw RC, -4(RA) // Update control var. 5233 | stw RC, -4(RA) // Update control var.
@@ -4448,8 +5256,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4448 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq 5256 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq
4449 | add TMP3, PC, TMP0 5257 | add TMP3, PC, TMP0
4450 | bne cr0, >5 5258 | bne cr0, >5
4451 | lus TMP1, 0xfffe 5259 | lus TMP1, (LJ_KEYINDEX >> 16)
4452 | ori TMP1, TMP1, 0x7fff 5260 | ori TMP1, TMP1, (LJ_KEYINDEX & 0xffff)
4453 | stw ZERO, -4(RA) // Initialize control var. 5261 | stw ZERO, -4(RA) // Initialize control var.
4454 | stw TMP1, -8(RA) 5262 | stw TMP1, -8(RA)
4455 | addis PC, TMP3, -(BCBIAS_J*4 >> 16) 5263 | addis PC, TMP3, -(BCBIAS_J*4 >> 16)
@@ -4460,6 +5268,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4460 | li TMP1, BC_ITERC 5268 | li TMP1, BC_ITERC
4461 | stb TMP0, -1(PC) 5269 | stb TMP0, -1(PC)
4462 | addis PC, TMP3, -(BCBIAS_J*4 >> 16) 5270 | addis PC, TMP3, -(BCBIAS_J*4 >> 16)
5271 | // NYI on big-endian: unpatch JLOOP.
4463 | stb TMP1, 3(PC) 5272 | stb TMP1, 3(PC)
4464 | b <1 5273 | b <1
4465 break; 5274 break;
@@ -4485,9 +5294,19 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4485 | subi TMP2, TMP2, 16 5294 | subi TMP2, TMP2, 16
4486 | ble >2 // No vararg slots? 5295 | ble >2 // No vararg slots?
4487 |1: // Copy vararg slots to destination slots. 5296 |1: // Copy vararg slots to destination slots.
5297 |.if FPU
4488 | lfd f0, 0(RC) 5298 | lfd f0, 0(RC)
5299 |.else
5300 | lwz CARG1, 0(RC)
5301 | lwz CARG2, 4(RC)
5302 |.endif
4489 | addi RC, RC, 8 5303 | addi RC, RC, 8
5304 |.if FPU
4490 | stfd f0, 0(RA) 5305 | stfd f0, 0(RA)
5306 |.else
5307 | stw CARG1, 0(RA)
5308 | stw CARG2, 4(RA)
5309 |.endif
4491 | cmplw RA, TMP2 5310 | cmplw RA, TMP2
4492 | cmplw cr1, RC, TMP3 5311 | cmplw cr1, RC, TMP3
4493 | bge >3 // All destination slots filled? 5312 | bge >3 // All destination slots filled?
@@ -4510,9 +5329,19 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4510 | addi MULTRES, TMP1, 8 5329 | addi MULTRES, TMP1, 8
4511 | bgt >7 5330 | bgt >7
4512 |6: 5331 |6:
5332 |.if FPU
4513 | lfd f0, 0(RC) 5333 | lfd f0, 0(RC)
5334 |.else
5335 | lwz CARG1, 0(RC)
5336 | lwz CARG2, 4(RC)
5337 |.endif
4514 | addi RC, RC, 8 5338 | addi RC, RC, 8
5339 |.if FPU
4515 | stfd f0, 0(RA) 5340 | stfd f0, 0(RA)
5341 |.else
5342 | stw CARG1, 0(RA)
5343 | stw CARG2, 4(RA)
5344 |.endif
4516 | cmplw RC, TMP3 5345 | cmplw RC, TMP3
4517 | addi RA, RA, 8 5346 | addi RA, RA, 8
4518 | blt <6 // More vararg slots? 5347 | blt <6 // More vararg slots?
@@ -4563,14 +5392,38 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4563 | li TMP1, 0 5392 | li TMP1, 0
4564 |2: 5393 |2:
4565 | addi TMP3, TMP1, 8 5394 | addi TMP3, TMP1, 8
5395 |.if FPU
4566 | lfdx f0, RA, TMP1 5396 | lfdx f0, RA, TMP1
5397 |.else
5398 | add CARG3, RA, TMP1
5399 | lwz CARG1, 0(CARG3)
5400 | lwz CARG2, 4(CARG3)
5401 |.endif
4567 | cmpw TMP3, RC 5402 | cmpw TMP3, RC
5403 |.if FPU
4568 | stfdx f0, TMP2, TMP1 5404 | stfdx f0, TMP2, TMP1
5405 |.else
5406 | add CARG3, TMP2, TMP1
5407 | stw CARG1, 0(CARG3)
5408 | stw CARG2, 4(CARG3)
5409 |.endif
4569 | beq >3 5410 | beq >3
4570 | addi TMP1, TMP3, 8 5411 | addi TMP1, TMP3, 8
5412 |.if FPU
4571 | lfdx f1, RA, TMP3 5413 | lfdx f1, RA, TMP3
5414 |.else
5415 | add CARG3, RA, TMP3
5416 | lwz CARG1, 0(CARG3)
5417 | lwz CARG2, 4(CARG3)
5418 |.endif
4572 | cmpw TMP1, RC 5419 | cmpw TMP1, RC
5420 |.if FPU
4573 | stfdx f1, TMP2, TMP3 5421 | stfdx f1, TMP2, TMP3
5422 |.else
5423 | add CARG3, TMP2, TMP3
5424 | stw CARG1, 0(CARG3)
5425 | stw CARG2, 4(CARG3)
5426 |.endif
4574 | bne <2 5427 | bne <2
4575 |3: 5428 |3:
4576 |5: 5429 |5:
@@ -4612,8 +5465,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4612 | subi TMP2, BASE, 8 5465 | subi TMP2, BASE, 8
4613 | decode_RB8 RB, INS 5466 | decode_RB8 RB, INS
4614 if (op == BC_RET1) { 5467 if (op == BC_RET1) {
5468 |.if FPU
4615 | lfd f0, 0(RA) 5469 | lfd f0, 0(RA)
4616 | stfd f0, 0(TMP2) 5470 | stfd f0, 0(TMP2)
5471 |.else
5472 | lwz CARG1, 0(RA)
5473 | lwz CARG2, 4(RA)
5474 | stw CARG1, 0(TMP2)
5475 | stw CARG2, 4(TMP2)
5476 |.endif
4617 } 5477 }
4618 |5: 5478 |5:
4619 | cmplw RB, RD 5479 | cmplw RB, RD
@@ -4674,11 +5534,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4674 |4: 5534 |4:
4675 | stw CARG1, FORL_IDX*8+4(RA) 5535 | stw CARG1, FORL_IDX*8+4(RA)
4676 } else { 5536 } else {
4677 | lwz TMP3, FORL_STEP*8(RA) 5537 | lwz SAVE0, FORL_STEP*8(RA)
4678 | lwz CARG3, FORL_STEP*8+4(RA) 5538 | lwz CARG3, FORL_STEP*8+4(RA)
4679 | lwz TMP2, FORL_STOP*8(RA) 5539 | lwz TMP2, FORL_STOP*8(RA)
4680 | lwz CARG2, FORL_STOP*8+4(RA) 5540 | lwz CARG2, FORL_STOP*8+4(RA)
4681 | cmplw cr7, TMP3, TISNUM 5541 | cmplw cr7, SAVE0, TISNUM
4682 | cmplw cr1, TMP2, TISNUM 5542 | cmplw cr1, TMP2, TISNUM
4683 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq 5543 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq
4684 | crand 4*cr0+eq, 4*cr0+eq, 4*cr1+eq 5544 | crand 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
@@ -4721,41 +5581,80 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4721 if (vk) { 5581 if (vk) {
4722 |.if DUALNUM 5582 |.if DUALNUM
4723 |9: // FP loop. 5583 |9: // FP loop.
5584 |.if FPU
4724 | lfd f1, FORL_IDX*8(RA) 5585 | lfd f1, FORL_IDX*8(RA)
4725 |.else 5586 |.else
5587 | lwz CARG1, FORL_IDX*8(RA)
5588 | lwz CARG2, FORL_IDX*8+4(RA)
5589 |.endif
5590 |.else
4726 | lfdux f1, RA, BASE 5591 | lfdux f1, RA, BASE
4727 |.endif 5592 |.endif
5593 |.if FPU
4728 | lfd f3, FORL_STEP*8(RA) 5594 | lfd f3, FORL_STEP*8(RA)
4729 | lfd f2, FORL_STOP*8(RA) 5595 | lfd f2, FORL_STOP*8(RA)
4730 | lwz TMP3, FORL_STEP*8(RA)
4731 | fadd f1, f1, f3 5596 | fadd f1, f1, f3
4732 | stfd f1, FORL_IDX*8(RA) 5597 | stfd f1, FORL_IDX*8(RA)
5598 |.else
5599 | lwz CARG3, FORL_STEP*8(RA)
5600 | lwz CARG4, FORL_STEP*8+4(RA)
5601 | mr SAVE1, RD
5602 | blex __adddf3
5603 | mr RD, SAVE1
5604 | stw CRET1, FORL_IDX*8(RA)
5605 | stw CRET2, FORL_IDX*8+4(RA)
5606 | lwz CARG3, FORL_STOP*8(RA)
5607 | lwz CARG4, FORL_STOP*8+4(RA)
5608 |.endif
5609 | lwz SAVE0, FORL_STEP*8(RA)
4733 } else { 5610 } else {
4734 |.if DUALNUM 5611 |.if DUALNUM
4735 |9: // FP loop. 5612 |9: // FP loop.
4736 |.else 5613 |.else
4737 | lwzux TMP1, RA, BASE 5614 | lwzux TMP1, RA, BASE
4738 | lwz TMP3, FORL_STEP*8(RA) 5615 | lwz SAVE0, FORL_STEP*8(RA)
4739 | lwz TMP2, FORL_STOP*8(RA) 5616 | lwz TMP2, FORL_STOP*8(RA)
4740 | cmplw cr0, TMP1, TISNUM 5617 | cmplw cr0, TMP1, TISNUM
4741 | cmplw cr7, TMP3, TISNUM 5618 | cmplw cr7, SAVE0, TISNUM
4742 | cmplw cr1, TMP2, TISNUM 5619 | cmplw cr1, TMP2, TISNUM
4743 |.endif 5620 |.endif
5621 |.if FPU
4744 | lfd f1, FORL_IDX*8(RA) 5622 | lfd f1, FORL_IDX*8(RA)
5623 |.else
5624 | lwz CARG1, FORL_IDX*8(RA)
5625 | lwz CARG2, FORL_IDX*8+4(RA)
5626 |.endif
4745 | crand 4*cr0+lt, 4*cr0+lt, 4*cr7+lt 5627 | crand 4*cr0+lt, 4*cr0+lt, 4*cr7+lt
4746 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt 5628 | crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
5629 |.if FPU
4747 | lfd f2, FORL_STOP*8(RA) 5630 | lfd f2, FORL_STOP*8(RA)
5631 |.else
5632 | lwz CARG3, FORL_STOP*8(RA)
5633 | lwz CARG4, FORL_STOP*8+4(RA)
5634 |.endif
4748 | bge ->vmeta_for 5635 | bge ->vmeta_for
4749 } 5636 }
4750 | cmpwi cr6, TMP3, 0 5637 | cmpwi cr6, SAVE0, 0
4751 if (op != BC_JFORL) { 5638 if (op != BC_JFORL) {
4752 | srwi RD, RD, 1 5639 | srwi RD, RD, 1
4753 } 5640 }
5641 |.if FPU
4754 | stfd f1, FORL_EXT*8(RA) 5642 | stfd f1, FORL_EXT*8(RA)
5643 |.else
5644 | stw CARG1, FORL_EXT*8(RA)
5645 | stw CARG2, FORL_EXT*8+4(RA)
5646 |.endif
4755 if (op != BC_JFORL) { 5647 if (op != BC_JFORL) {
4756 | add RD, PC, RD 5648 | add RD, PC, RD
4757 } 5649 }
5650 |.if FPU
4758 | fcmpu cr0, f1, f2 5651 | fcmpu cr0, f1, f2
5652 |.else
5653 | mr SAVE1, RD
5654 | blex __ledf2
5655 | cmpwi CRET1, 0
5656 | mr RD, SAVE1
5657 |.endif
4759 if (op == BC_JFORI) { 5658 if (op == BC_JFORI) {
4760 | addis PC, RD, -(BCBIAS_J*4 >> 16) 5659 | addis PC, RD, -(BCBIAS_J*4 >> 16)
4761 } 5660 }
@@ -4858,8 +5757,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4858 | lp TMP2, TRACE:TMP2->mcode 5757 | lp TMP2, TRACE:TMP2->mcode
4859 | stw BASE, DISPATCH_GL(jit_base)(DISPATCH) 5758 | stw BASE, DISPATCH_GL(jit_base)(DISPATCH)
4860 | mtctr TMP2 5759 | mtctr TMP2
4861 | stw L, DISPATCH_GL(jit_L)(DISPATCH)
4862 | addi JGL, DISPATCH, GG_DISP2G+32768 5760 | addi JGL, DISPATCH, GG_DISP2G+32768
5761 | stw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
4863 | bctr 5762 | bctr
4864 |.endif 5763 |.endif
4865 break; 5764 break;
@@ -4994,6 +5893,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4994 | lp TMP1, L->top 5893 | lp TMP1, L->top
4995 | li_vmstate INTERP 5894 | li_vmstate INTERP
4996 | lwz PC, FRAME_PC(BASE) // Fetch PC of caller. 5895 | lwz PC, FRAME_PC(BASE) // Fetch PC of caller.
5896 | stw L, DISPATCH_GL(cur_L)(DISPATCH)
4997 | sub RA, TMP1, RD // RA = L->top - nresults*8 5897 | sub RA, TMP1, RD // RA = L->top - nresults*8
4998 | st_vmstate 5898 | st_vmstate
4999 | b ->vm_returnc 5899 | b ->vm_returnc
diff --git a/src/vm_ppcspe.dasc b/src/vm_ppcspe.dasc
deleted file mode 100644
index c4a44191..00000000
--- a/src/vm_ppcspe.dasc
+++ /dev/null
@@ -1,3691 +0,0 @@
1|// Low-level VM code for PowerPC/e500 CPUs.
2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4|
5|.arch ppc
6|.section code_op, code_sub
7|
8|.actionlist build_actionlist
9|.globals GLOB_
10|.globalnames globnames
11|.externnames extnames
12|
13|// Note: The ragged indentation of the instructions is intentional.
14|// The starting columns indicate data dependencies.
15|
16|//-----------------------------------------------------------------------
17|
18|// Fixed register assignments for the interpreter.
19|// Don't use: r1 = sp, r2 and r13 = reserved and/or small data area ptr
20|
21|// The following must be C callee-save (but BASE is often refetched).
22|.define BASE, r14 // Base of current Lua stack frame.
23|.define KBASE, r15 // Constants of current Lua function.
24|.define PC, r16 // Next PC.
25|.define DISPATCH, r17 // Opcode dispatch table.
26|.define LREG, r18 // Register holding lua_State (also in SAVE_L).
27|.define MULTRES, r19 // Size of multi-result: (nresults+1)*8.
28|
29|// Constants for vectorized type-comparisons (hi+low GPR). C callee-save.
30|.define TISNUM, r22
31|.define TISSTR, r23
32|.define TISTAB, r24
33|.define TISFUNC, r25
34|.define TISNIL, r26
35|.define TOBIT, r27
36|.define ZERO, TOBIT // Zero in lo word.
37|
38|// The following temporaries are not saved across C calls, except for RA.
39|.define RA, r20 // Callee-save.
40|.define RB, r10
41|.define RC, r11
42|.define RD, r12
43|.define INS, r7 // Overlaps CARG5.
44|
45|.define TMP0, r0
46|.define TMP1, r8
47|.define TMP2, r9
48|.define TMP3, r6 // Overlaps CARG4.
49|
50|// Saved temporaries.
51|.define SAVE0, r21
52|
53|// Calling conventions.
54|.define CARG1, r3
55|.define CARG2, r4
56|.define CARG3, r5
57|.define CARG4, r6 // Overlaps TMP3.
58|.define CARG5, r7 // Overlaps INS.
59|
60|.define CRET1, r3
61|.define CRET2, r4
62|
63|// Stack layout while in interpreter. Must match with lj_frame.h.
64|.define SAVE_LR, 188(sp)
65|.define CFRAME_SPACE, 184 // Delta for sp.
66|// Back chain for sp: 184(sp) <-- sp entering interpreter
67|.define SAVE_r31, 176(sp) // 64 bit register saves.
68|.define SAVE_r30, 168(sp)
69|.define SAVE_r29, 160(sp)
70|.define SAVE_r28, 152(sp)
71|.define SAVE_r27, 144(sp)
72|.define SAVE_r26, 136(sp)
73|.define SAVE_r25, 128(sp)
74|.define SAVE_r24, 120(sp)
75|.define SAVE_r23, 112(sp)
76|.define SAVE_r22, 104(sp)
77|.define SAVE_r21, 96(sp)
78|.define SAVE_r20, 88(sp)
79|.define SAVE_r19, 80(sp)
80|.define SAVE_r18, 72(sp)
81|.define SAVE_r17, 64(sp)
82|.define SAVE_r16, 56(sp)
83|.define SAVE_r15, 48(sp)
84|.define SAVE_r14, 40(sp)
85|.define SAVE_CR, 36(sp)
86|.define UNUSED1, 32(sp)
87|.define SAVE_ERRF, 28(sp) // 32 bit C frame info.
88|.define SAVE_NRES, 24(sp)
89|.define SAVE_CFRAME, 20(sp)
90|.define SAVE_L, 16(sp)
91|.define SAVE_PC, 12(sp)
92|.define SAVE_MULTRES, 8(sp)
93|// Next frame lr: 4(sp)
94|// Back chain for sp: 0(sp) <-- sp while in interpreter
95|
96|.macro save_, reg; evstdd reg, SAVE_..reg; .endmacro
97|.macro rest_, reg; evldd reg, SAVE_..reg; .endmacro
98|
99|.macro saveregs
100| stwu sp, -CFRAME_SPACE(sp)
101| save_ r14; save_ r15; save_ r16; save_ r17; save_ r18; save_ r19
102| mflr r0; mfcr r12
103| save_ r20; save_ r21; save_ r22; save_ r23; save_ r24; save_ r25
104| stw r0, SAVE_LR; stw r12, SAVE_CR
105| save_ r26; save_ r27; save_ r28; save_ r29; save_ r30; save_ r31
106|.endmacro
107|
108|.macro restoreregs
109| lwz r0, SAVE_LR; lwz r12, SAVE_CR
110| rest_ r14; rest_ r15; rest_ r16; rest_ r17; rest_ r18; rest_ r19
111| mtlr r0; mtcrf 0x38, r12
112| rest_ r20; rest_ r21; rest_ r22; rest_ r23; rest_ r24; rest_ r25
113| rest_ r26; rest_ r27; rest_ r28; rest_ r29; rest_ r30; rest_ r31
114| addi sp, sp, CFRAME_SPACE
115|.endmacro
116|
117|// Type definitions. Some of these are only used for documentation.
118|.type L, lua_State, LREG
119|.type GL, global_State
120|.type TVALUE, TValue
121|.type GCOBJ, GCobj
122|.type STR, GCstr
123|.type TAB, GCtab
124|.type LFUNC, GCfuncL
125|.type CFUNC, GCfuncC
126|.type PROTO, GCproto
127|.type UPVAL, GCupval
128|.type NODE, Node
129|.type NARGS8, int
130|.type TRACE, GCtrace
131|
132|//-----------------------------------------------------------------------
133|
134|// These basic macros should really be part of DynASM.
135|.macro srwi, rx, ry, n; rlwinm rx, ry, 32-n, n, 31; .endmacro
136|.macro slwi, rx, ry, n; rlwinm rx, ry, n, 0, 31-n; .endmacro
137|.macro rotlwi, rx, ry, n; rlwinm rx, ry, n, 0, 31; .endmacro
138|.macro rotlw, rx, ry, rn; rlwnm rx, ry, rn, 0, 31; .endmacro
139|.macro subi, rx, ry, i; addi rx, ry, -i; .endmacro
140|
141|// Trap for not-yet-implemented parts.
142|.macro NYI; tw 4, sp, sp; .endmacro
143|
144|//-----------------------------------------------------------------------
145|
146|// Access to frame relative to BASE.
147|.define FRAME_PC, -8
148|.define FRAME_FUNC, -4
149|
150|// Instruction decode.
151|.macro decode_OP4, dst, ins; rlwinm dst, ins, 2, 22, 29; .endmacro
152|.macro decode_RA8, dst, ins; rlwinm dst, ins, 27, 21, 28; .endmacro
153|.macro decode_RB8, dst, ins; rlwinm dst, ins, 11, 21, 28; .endmacro
154|.macro decode_RC8, dst, ins; rlwinm dst, ins, 19, 21, 28; .endmacro
155|.macro decode_RD8, dst, ins; rlwinm dst, ins, 19, 13, 28; .endmacro
156|
157|.macro decode_OP1, dst, ins; rlwinm dst, ins, 0, 24, 31; .endmacro
158|.macro decode_RD4, dst, ins; rlwinm dst, ins, 18, 14, 29; .endmacro
159|
160|// Instruction fetch.
161|.macro ins_NEXT1
162| lwz INS, 0(PC)
163| addi PC, PC, 4
164|.endmacro
165|// Instruction decode+dispatch.
166|.macro ins_NEXT2
167| decode_OP4 TMP1, INS
168| decode_RB8 RB, INS
169| decode_RD8 RD, INS
170| lwzx TMP0, DISPATCH, TMP1
171| decode_RA8 RA, INS
172| decode_RC8 RC, INS
173| mtctr TMP0
174| bctr
175|.endmacro
176|.macro ins_NEXT
177| ins_NEXT1
178| ins_NEXT2
179|.endmacro
180|
181|// Instruction footer.
182|.if 1
183| // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
184| .define ins_next, ins_NEXT
185| .define ins_next_, ins_NEXT
186| .define ins_next1, ins_NEXT1
187| .define ins_next2, ins_NEXT2
188|.else
189| // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
190| // Affects only certain kinds of benchmarks (and only with -j off).
191| .macro ins_next
192| b ->ins_next
193| .endmacro
194| .macro ins_next1
195| .endmacro
196| .macro ins_next2
197| b ->ins_next
198| .endmacro
199| .macro ins_next_
200| ->ins_next:
201| ins_NEXT
202| .endmacro
203|.endif
204|
205|// Call decode and dispatch.
206|.macro ins_callt
207| // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
208| lwz PC, LFUNC:RB->pc
209| lwz INS, 0(PC)
210| addi PC, PC, 4
211| decode_OP4 TMP1, INS
212| decode_RA8 RA, INS
213| lwzx TMP0, DISPATCH, TMP1
214| add RA, RA, BASE
215| mtctr TMP0
216| bctr
217|.endmacro
218|
219|.macro ins_call
220| // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
221| stw PC, FRAME_PC(BASE)
222| ins_callt
223|.endmacro
224|
225|//-----------------------------------------------------------------------
226|
227|// Macros to test operand types.
228|.macro checknum, reg; evcmpltu reg, TISNUM; .endmacro
229|.macro checkstr, reg; evcmpeq reg, TISSTR; .endmacro
230|.macro checktab, reg; evcmpeq reg, TISTAB; .endmacro
231|.macro checkfunc, reg; evcmpeq reg, TISFUNC; .endmacro
232|.macro checknil, reg; evcmpeq reg, TISNIL; .endmacro
233|.macro checkok, label; blt label; .endmacro
234|.macro checkfail, label; bge label; .endmacro
235|.macro checkanyfail, label; bns label; .endmacro
236|.macro checkallok, label; bso label; .endmacro
237|
238|.macro branch_RD
239| srwi TMP0, RD, 1
240| add PC, PC, TMP0
241| addis PC, PC, -(BCBIAS_J*4 >> 16)
242|.endmacro
243|
244|// Assumes DISPATCH is relative to GL.
245#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
246#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
247|
248#define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
249|
250|.macro hotloop
251| NYI
252|.endmacro
253|
254|.macro hotcall
255| NYI
256|.endmacro
257|
258|// Set current VM state. Uses TMP0.
259|.macro li_vmstate, st; li TMP0, ~LJ_VMST_..st; .endmacro
260|.macro st_vmstate; stw TMP0, DISPATCH_GL(vmstate)(DISPATCH); .endmacro
261|
262|// Move table write barrier back. Overwrites mark and tmp.
263|.macro barrierback, tab, mark, tmp
264| lwz tmp, DISPATCH_GL(gc.grayagain)(DISPATCH)
265| // Assumes LJ_GC_BLACK is 0x04.
266| rlwinm mark, mark, 0, 30, 28 // black2gray(tab)
267| stw tab, DISPATCH_GL(gc.grayagain)(DISPATCH)
268| stb mark, tab->marked
269| stw tmp, tab->gclist
270|.endmacro
271|
272|//-----------------------------------------------------------------------
273
274/* Generate subroutines used by opcodes and other parts of the VM. */
275/* The .code_sub section should be last to help static branch prediction. */
276static void build_subroutines(BuildCtx *ctx)
277{
278 |.code_sub
279 |
280 |//-----------------------------------------------------------------------
281 |//-- Return handling ----------------------------------------------------
282 |//-----------------------------------------------------------------------
283 |
284 |->vm_returnp:
285 | // See vm_return. Also: TMP2 = previous base.
286 | andi. TMP0, PC, FRAME_P
287 | evsplati TMP1, LJ_TTRUE
288 | beq ->cont_dispatch
289 |
290 | // Return from pcall or xpcall fast func.
291 | lwz PC, FRAME_PC(TMP2) // Fetch PC of previous frame.
292 | mr BASE, TMP2 // Restore caller base.
293 | // Prepending may overwrite the pcall frame, so do it at the end.
294 | stwu TMP1, FRAME_PC(RA) // Prepend true to results.
295 |
296 |->vm_returnc:
297 | addi RD, RD, 8 // RD = (nresults+1)*8.
298 | andi. TMP0, PC, FRAME_TYPE
299 | cmpwi cr1, RD, 0
300 | li CRET1, LUA_YIELD
301 | beq cr1, ->vm_unwind_c_eh
302 | mr MULTRES, RD
303 | beq ->BC_RET_Z // Handle regular return to Lua.
304 |
305 |->vm_return:
306 | // BASE = base, RA = resultptr, RD/MULTRES = (nresults+1)*8, PC = return
307 | // TMP0 = PC & FRAME_TYPE
308 | cmpwi TMP0, FRAME_C
309 | rlwinm TMP2, PC, 0, 0, 28
310 | li_vmstate C
311 | sub TMP2, BASE, TMP2 // TMP2 = previous base.
312 | bne ->vm_returnp
313 |
314 | addic. TMP1, RD, -8
315 | stw TMP2, L->base
316 | lwz TMP2, SAVE_NRES
317 | subi BASE, BASE, 8
318 | st_vmstate
319 | slwi TMP2, TMP2, 3
320 | beq >2
321 |1:
322 | addic. TMP1, TMP1, -8
323 | evldd TMP0, 0(RA)
324 | addi RA, RA, 8
325 | evstdd TMP0, 0(BASE)
326 | addi BASE, BASE, 8
327 | bne <1
328 |
329 |2:
330 | cmpw TMP2, RD // More/less results wanted?
331 | bne >6
332 |3:
333 | stw BASE, L->top // Store new top.
334 |
335 |->vm_leave_cp:
336 | lwz TMP0, SAVE_CFRAME // Restore previous C frame.
337 | li CRET1, 0 // Ok return status for vm_pcall.
338 | stw TMP0, L->cframe
339 |
340 |->vm_leave_unw:
341 | restoreregs
342 | blr
343 |
344 |6:
345 | ble >7 // Less results wanted?
346 | // More results wanted. Check stack size and fill up results with nil.
347 | lwz TMP1, L->maxstack
348 | cmplw BASE, TMP1
349 | bge >8
350 | evstdd TISNIL, 0(BASE)
351 | addi RD, RD, 8
352 | addi BASE, BASE, 8
353 | b <2
354 |
355 |7: // Less results wanted.
356 | sub TMP0, RD, TMP2
357 | cmpwi TMP2, 0 // LUA_MULTRET+1 case?
358 | sub TMP0, BASE, TMP0 // Subtract the difference.
359 | iseleq BASE, BASE, TMP0 // Either keep top or shrink it.
360 | b <3
361 |
362 |8: // Corner case: need to grow stack for filling up results.
363 | // This can happen if:
364 | // - A C function grows the stack (a lot).
365 | // - The GC shrinks the stack in between.
366 | // - A return back from a lua_call() with (high) nresults adjustment.
367 | stw BASE, L->top // Save current top held in BASE (yes).
368 | mr SAVE0, RD
369 | mr CARG2, TMP2
370 | mr CARG1, L
371 | bl extern lj_state_growstack // (lua_State *L, int n)
372 | lwz TMP2, SAVE_NRES
373 | mr RD, SAVE0
374 | slwi TMP2, TMP2, 3
375 | lwz BASE, L->top // Need the (realloced) L->top in BASE.
376 | b <2
377 |
378 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
379 | // (void *cframe, int errcode)
380 | mr sp, CARG1
381 | mr CRET1, CARG2
382 |->vm_unwind_c_eh: // Landing pad for external unwinder.
383 | lwz L, SAVE_L
384 | li TMP0, ~LJ_VMST_C
385 | lwz GL:TMP1, L->glref
386 | stw TMP0, GL:TMP1->vmstate
387 | b ->vm_leave_unw
388 |
389 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
390 | // (void *cframe)
391 | rlwinm sp, CARG1, 0, 0, 29
392 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
393 | lwz L, SAVE_L
394 | evsplati TISNUM, LJ_TISNUM+1 // Setup type comparison constants.
395 | evsplati TISFUNC, LJ_TFUNC
396 | lus TOBIT, 0x4338
397 | evsplati TISTAB, LJ_TTAB
398 | li TMP0, 0
399 | lwz BASE, L->base
400 | evmergelo TOBIT, TOBIT, TMP0
401 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
402 | evsplati TISSTR, LJ_TSTR
403 | li TMP1, LJ_TFALSE
404 | evsplati TISNIL, LJ_TNIL
405 | li_vmstate INTERP
406 | lwz PC, FRAME_PC(BASE) // Fetch PC of previous frame.
407 | la RA, -8(BASE) // Results start at BASE-8.
408 | addi DISPATCH, DISPATCH, GG_G2DISP
409 | stw TMP1, 0(RA) // Prepend false to error message.
410 | li RD, 16 // 2 results: false + error message.
411 | st_vmstate
412 | b ->vm_returnc
413 |
414 |//-----------------------------------------------------------------------
415 |//-- Grow stack for calls -----------------------------------------------
416 |//-----------------------------------------------------------------------
417 |
418 |->vm_growstack_c: // Grow stack for C function.
419 | li CARG2, LUA_MINSTACK
420 | b >2
421 |
422 |->vm_growstack_l: // Grow stack for Lua function.
423 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
424 | add RC, BASE, RC
425 | sub RA, RA, BASE
426 | stw BASE, L->base
427 | addi PC, PC, 4 // Must point after first instruction.
428 | stw RC, L->top
429 | srwi CARG2, RA, 3
430 |2:
431 | // L->base = new base, L->top = top
432 | stw PC, SAVE_PC
433 | mr CARG1, L
434 | bl extern lj_state_growstack // (lua_State *L, int n)
435 | lwz BASE, L->base
436 | lwz RC, L->top
437 | lwz LFUNC:RB, FRAME_FUNC(BASE)
438 | sub RC, RC, BASE
439 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
440 | ins_callt // Just retry the call.
441 |
442 |//-----------------------------------------------------------------------
443 |//-- Entry points into the assembler VM ---------------------------------
444 |//-----------------------------------------------------------------------
445 |
446 |->vm_resume: // Setup C frame and resume thread.
447 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
448 | saveregs
449 | mr L, CARG1
450 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
451 | mr BASE, CARG2
452 | lbz TMP1, L->status
453 | stw L, SAVE_L
454 | li PC, FRAME_CP
455 | addi TMP0, sp, CFRAME_RESUME
456 | addi DISPATCH, DISPATCH, GG_G2DISP
457 | stw CARG3, SAVE_NRES
458 | cmplwi TMP1, 0
459 | stw CARG3, SAVE_ERRF
460 | stw TMP0, L->cframe
461 | stw CARG3, SAVE_CFRAME
462 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
463 | beq >3
464 |
465 | // Resume after yield (like a return).
466 | mr RA, BASE
467 | lwz BASE, L->base
468 | evsplati TISNUM, LJ_TISNUM+1 // Setup type comparison constants.
469 | lwz TMP1, L->top
470 | evsplati TISFUNC, LJ_TFUNC
471 | lus TOBIT, 0x4338
472 | evsplati TISTAB, LJ_TTAB
473 | lwz PC, FRAME_PC(BASE)
474 | li TMP2, 0
475 | evsplati TISSTR, LJ_TSTR
476 | sub RD, TMP1, BASE
477 | evmergelo TOBIT, TOBIT, TMP2
478 | stb CARG3, L->status
479 | andi. TMP0, PC, FRAME_TYPE
480 | li_vmstate INTERP
481 | addi RD, RD, 8
482 | evsplati TISNIL, LJ_TNIL
483 | mr MULTRES, RD
484 | st_vmstate
485 | beq ->BC_RET_Z
486 | b ->vm_return
487 |
488 |->vm_pcall: // Setup protected C frame and enter VM.
489 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
490 | saveregs
491 | li PC, FRAME_CP
492 | stw CARG4, SAVE_ERRF
493 | b >1
494 |
495 |->vm_call: // Setup C frame and enter VM.
496 | // (lua_State *L, TValue *base, int nres1)
497 | saveregs
498 | li PC, FRAME_C
499 |
500 |1: // Entry point for vm_pcall above (PC = ftype).
501 | lwz TMP1, L:CARG1->cframe
502 | stw CARG3, SAVE_NRES
503 | mr L, CARG1
504 | stw CARG1, SAVE_L
505 | mr BASE, CARG2
506 | stw sp, L->cframe // Add our C frame to cframe chain.
507 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
508 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
509 | stw TMP1, SAVE_CFRAME
510 | addi DISPATCH, DISPATCH, GG_G2DISP
511 |
512 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
513 | lwz TMP2, L->base // TMP2 = old base (used in vmeta_call).
514 | evsplati TISNUM, LJ_TISNUM+1 // Setup type comparison constants.
515 | lwz TMP1, L->top
516 | evsplati TISFUNC, LJ_TFUNC
517 | add PC, PC, BASE
518 | evsplati TISTAB, LJ_TTAB
519 | lus TOBIT, 0x4338
520 | li TMP0, 0
521 | sub PC, PC, TMP2 // PC = frame delta + frame type
522 | evsplati TISSTR, LJ_TSTR
523 | sub NARGS8:RC, TMP1, BASE
524 | evmergelo TOBIT, TOBIT, TMP0
525 | li_vmstate INTERP
526 | evsplati TISNIL, LJ_TNIL
527 | st_vmstate
528 |
529 |->vm_call_dispatch:
530 | // TMP2 = old base, BASE = new base, RC = nargs*8, PC = caller PC
531 | li TMP0, -8
532 | evlddx LFUNC:RB, BASE, TMP0
533 | checkfunc LFUNC:RB
534 | checkfail ->vmeta_call
535 |
536 |->vm_call_dispatch_f:
537 | ins_call
538 | // BASE = new base, RB = func, RC = nargs*8, PC = caller PC
539 |
540 |->vm_cpcall: // Setup protected C frame, call C.
541 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
542 | saveregs
543 | mr L, CARG1
544 | lwz TMP0, L:CARG1->stack
545 | stw CARG1, SAVE_L
546 | lwz TMP1, L->top
547 | stw CARG1, SAVE_PC // Any value outside of bytecode is ok.
548 | sub TMP0, TMP0, TMP1 // Compute -savestack(L, L->top).
549 | lwz TMP1, L->cframe
550 | stw sp, L->cframe // Add our C frame to cframe chain.
551 | li TMP2, 0
552 | stw TMP0, SAVE_NRES // Neg. delta means cframe w/o frame.
553 | stw TMP2, SAVE_ERRF // No error function.
554 | stw TMP1, SAVE_CFRAME
555 | mtctr CARG4
556 | bctrl // (lua_State *L, lua_CFunction func, void *ud)
557 | mr. BASE, CRET1
558 | lwz DISPATCH, L->glref // Setup pointer to dispatch table.
559 | li PC, FRAME_CP
560 | addi DISPATCH, DISPATCH, GG_G2DISP
561 | bne <3 // Else continue with the call.
562 | b ->vm_leave_cp // No base? Just remove C frame.
563 |
564 |//-----------------------------------------------------------------------
565 |//-- Metamethod handling ------------------------------------------------
566 |//-----------------------------------------------------------------------
567 |
568 |// The lj_meta_* functions (except for lj_meta_cat) don't reallocate the
569 |// stack, so BASE doesn't need to be reloaded across these calls.
570 |
571 |//-- Continuation dispatch ----------------------------------------------
572 |
573 |->cont_dispatch:
574 | // BASE = meta base, RA = resultptr, RD = (nresults+1)*8
575 | lwz TMP0, -12(BASE) // Continuation.
576 | mr RB, BASE
577 | mr BASE, TMP2 // Restore caller BASE.
578 | lwz LFUNC:TMP1, FRAME_FUNC(TMP2)
579 | cmplwi TMP0, 0
580 | lwz PC, -16(RB) // Restore PC from [cont|PC].
581 | beq >1
582 | subi TMP2, RD, 8
583 | lwz TMP1, LFUNC:TMP1->pc
584 | evstddx TISNIL, RA, TMP2 // Ensure one valid arg.
585 | lwz KBASE, PC2PROTO(k)(TMP1)
586 | // BASE = base, RA = resultptr, RB = meta base
587 | mtctr TMP0
588 | bctr // Jump to continuation.
589 |
590 |1: // Tail call from C function.
591 | subi TMP1, RB, 16
592 | sub RC, TMP1, BASE
593 | b ->vm_call_tail
594 |
595 |->cont_cat: // RA = resultptr, RB = meta base
596 | lwz INS, -4(PC)
597 | subi CARG2, RB, 16
598 | decode_RB8 SAVE0, INS
599 | evldd TMP0, 0(RA)
600 | add TMP1, BASE, SAVE0
601 | stw BASE, L->base
602 | cmplw TMP1, CARG2
603 | sub CARG3, CARG2, TMP1
604 | decode_RA8 RA, INS
605 | evstdd TMP0, 0(CARG2)
606 | bne ->BC_CAT_Z
607 | evstddx TMP0, BASE, RA
608 | b ->cont_nop
609 |
610 |//-- Table indexing metamethods -----------------------------------------
611 |
612 |->vmeta_tgets1:
613 | evmergelo STR:RC, TISSTR, STR:RC
614 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
615 | decode_RB8 RB, INS
616 | evstdd STR:RC, 0(CARG3)
617 | add CARG2, BASE, RB
618 | b >1
619 |
620 |->vmeta_tgets:
621 | evmergelo TAB:RB, TISTAB, TAB:RB
622 | la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
623 | evmergelo STR:RC, TISSTR, STR:RC
624 | evstdd TAB:RB, 0(CARG2)
625 | la CARG3, DISPATCH_GL(tmptv2)(DISPATCH)
626 | evstdd STR:RC, 0(CARG3)
627 | b >1
628 |
629 |->vmeta_tgetb: // TMP0 = index
630 | efdcfsi TMP0, TMP0
631 | decode_RB8 RB, INS
632 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
633 | add CARG2, BASE, RB
634 | evstdd TMP0, 0(CARG3)
635 | b >1
636 |
637 |->vmeta_tgetv:
638 | decode_RB8 RB, INS
639 | decode_RC8 RC, INS
640 | add CARG2, BASE, RB
641 | add CARG3, BASE, RC
642 |1:
643 | stw BASE, L->base
644 | mr CARG1, L
645 | stw PC, SAVE_PC
646 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
647 | // Returns TValue * (finished) or NULL (metamethod).
648 | cmplwi CRET1, 0
649 | beq >3
650 | evldd TMP0, 0(CRET1)
651 | evstddx TMP0, BASE, RA
652 | ins_next
653 |
654 |3: // Call __index metamethod.
655 | // BASE = base, L->top = new base, stack = cont/func/t/k
656 | subfic TMP1, BASE, FRAME_CONT
657 | lwz BASE, L->top
658 | stw PC, -16(BASE) // [cont|PC]
659 | add PC, TMP1, BASE
660 | lwz LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
661 | li NARGS8:RC, 16 // 2 args for func(t, k).
662 | b ->vm_call_dispatch_f
663 |
664 |//-----------------------------------------------------------------------
665 |
666 |->vmeta_tsets1:
667 | evmergelo STR:RC, TISSTR, STR:RC
668 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
669 | decode_RB8 RB, INS
670 | evstdd STR:RC, 0(CARG3)
671 | add CARG2, BASE, RB
672 | b >1
673 |
674 |->vmeta_tsets:
675 | evmergelo TAB:RB, TISTAB, TAB:RB
676 | la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
677 | evmergelo STR:RC, TISSTR, STR:RC
678 | evstdd TAB:RB, 0(CARG2)
679 | la CARG3, DISPATCH_GL(tmptv2)(DISPATCH)
680 | evstdd STR:RC, 0(CARG3)
681 | b >1
682 |
683 |->vmeta_tsetb: // TMP0 = index
684 | efdcfsi TMP0, TMP0
685 | decode_RB8 RB, INS
686 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
687 | add CARG2, BASE, RB
688 | evstdd TMP0, 0(CARG3)
689 | b >1
690 |
691 |->vmeta_tsetv:
692 | decode_RB8 RB, INS
693 | decode_RC8 RC, INS
694 | add CARG2, BASE, RB
695 | add CARG3, BASE, RC
696 |1:
697 | stw BASE, L->base
698 | mr CARG1, L
699 | stw PC, SAVE_PC
700 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
701 | // Returns TValue * (finished) or NULL (metamethod).
702 | cmplwi CRET1, 0
703 | evlddx TMP0, BASE, RA
704 | beq >3
705 | // NOBARRIER: lj_meta_tset ensures the table is not black.
706 | evstdd TMP0, 0(CRET1)
707 | ins_next
708 |
709 |3: // Call __newindex metamethod.
710 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
711 | subfic TMP1, BASE, FRAME_CONT
712 | lwz BASE, L->top
713 | stw PC, -16(BASE) // [cont|PC]
714 | add PC, TMP1, BASE
715 | lwz LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
716 | li NARGS8:RC, 24 // 3 args for func(t, k, v)
717 | evstdd TMP0, 16(BASE) // Copy value to third argument.
718 | b ->vm_call_dispatch_f
719 |
720 |//-- Comparison metamethods ---------------------------------------------
721 |
722 |->vmeta_comp:
723 | mr CARG1, L
724 | subi PC, PC, 4
725 | add CARG2, BASE, RA
726 | stw PC, SAVE_PC
727 | add CARG3, BASE, RD
728 | stw BASE, L->base
729 | decode_OP1 CARG4, INS
730 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
731 | // Returns 0/1 or TValue * (metamethod).
732 |3:
733 | cmplwi CRET1, 1
734 | bgt ->vmeta_binop
735 |4:
736 | lwz INS, 0(PC)
737 | addi PC, PC, 4
738 | decode_RD4 TMP2, INS
739 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
740 | add TMP2, TMP2, TMP3
741 | isellt PC, PC, TMP2
742 |->cont_nop:
743 | ins_next
744 |
745 |->cont_ra: // RA = resultptr
746 | lwz INS, -4(PC)
747 | evldd TMP0, 0(RA)
748 | decode_RA8 TMP1, INS
749 | evstddx TMP0, BASE, TMP1
750 | b ->cont_nop
751 |
752 |->cont_condt: // RA = resultptr
753 | lwz TMP0, 0(RA)
754 | li TMP1, LJ_TTRUE
755 | cmplw TMP1, TMP0 // Branch if result is true.
756 | b <4
757 |
758 |->cont_condf: // RA = resultptr
759 | lwz TMP0, 0(RA)
760 | li TMP1, LJ_TFALSE
761 | cmplw TMP0, TMP1 // Branch if result is false.
762 | b <4
763 |
764 |->vmeta_equal:
765 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
766 | subi PC, PC, 4
767 | stw BASE, L->base
768 | mr CARG1, L
769 | stw PC, SAVE_PC
770 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
771 | // Returns 0/1 or TValue * (metamethod).
772 | b <3
773 |
774 |//-- Arithmetic metamethods ---------------------------------------------
775 |
776 |->vmeta_arith_vn:
777 | add CARG3, BASE, RB
778 | add CARG4, KBASE, RC
779 | b >1
780 |
781 |->vmeta_arith_nv:
782 | add CARG3, KBASE, RC
783 | add CARG4, BASE, RB
784 | b >1
785 |
786 |->vmeta_unm:
787 | add CARG3, BASE, RD
788 | mr CARG4, CARG3
789 | b >1
790 |
791 |->vmeta_arith_vv:
792 | add CARG3, BASE, RB
793 | add CARG4, BASE, RC
794 |1:
795 | add CARG2, BASE, RA
796 | stw BASE, L->base
797 | mr CARG1, L
798 | stw PC, SAVE_PC
799 | decode_OP1 CARG5, INS // Caveat: CARG5 overlaps INS.
800 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
801 | // Returns NULL (finished) or TValue * (metamethod).
802 | cmplwi CRET1, 0
803 | beq ->cont_nop
804 |
805 | // Call metamethod for binary op.
806 |->vmeta_binop:
807 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
808 | sub TMP1, CRET1, BASE
809 | stw PC, -16(CRET1) // [cont|PC]
810 | mr TMP2, BASE
811 | addi PC, TMP1, FRAME_CONT
812 | mr BASE, CRET1
813 | li NARGS8:RC, 16 // 2 args for func(o1, o2).
814 | b ->vm_call_dispatch
815 |
816 |->vmeta_len:
817#if LJ_52
818 | mr SAVE0, CARG1
819#endif
820 | add CARG2, BASE, RD
821 | stw BASE, L->base
822 | mr CARG1, L
823 | stw PC, SAVE_PC
824 | bl extern lj_meta_len // (lua_State *L, TValue *o)
825 | // Returns NULL (retry) or TValue * (metamethod base).
826#if LJ_52
827 | cmplwi CRET1, 0
828 | bne ->vmeta_binop // Binop call for compatibility.
829 | mr CARG1, SAVE0
830 | b ->BC_LEN_Z
831#else
832 | b ->vmeta_binop // Binop call for compatibility.
833#endif
834 |
835 |//-- Call metamethod ----------------------------------------------------
836 |
837 |->vmeta_call: // Resolve and call __call metamethod.
838 | // TMP2 = old base, BASE = new base, RC = nargs*8
839 | mr CARG1, L
840 | stw TMP2, L->base // This is the callers base!
841 | subi CARG2, BASE, 8
842 | stw PC, SAVE_PC
843 | add CARG3, BASE, RC
844 | mr SAVE0, NARGS8:RC
845 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
846 | lwz LFUNC:RB, FRAME_FUNC(BASE) // Guaranteed to be a function here.
847 | addi NARGS8:RC, SAVE0, 8 // Got one more argument now.
848 | ins_call
849 |
850 |->vmeta_callt: // Resolve __call for BC_CALLT.
851 | // BASE = old base, RA = new base, RC = nargs*8
852 | mr CARG1, L
853 | stw BASE, L->base
854 | subi CARG2, RA, 8
855 | stw PC, SAVE_PC
856 | add CARG3, RA, RC
857 | mr SAVE0, NARGS8:RC
858 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
859 | lwz TMP1, FRAME_PC(BASE)
860 | addi NARGS8:RC, SAVE0, 8 // Got one more argument now.
861 | lwz LFUNC:RB, FRAME_FUNC(RA) // Guaranteed to be a function here.
862 | b ->BC_CALLT_Z
863 |
864 |//-- Argument coercion for 'for' statement ------------------------------
865 |
866 |->vmeta_for:
867 | mr CARG1, L
868 | stw BASE, L->base
869 | mr CARG2, RA
870 | stw PC, SAVE_PC
871 | mr SAVE0, INS
872 | bl extern lj_meta_for // (lua_State *L, TValue *base)
873 |.if JIT
874 | decode_OP1 TMP0, SAVE0
875 |.endif
876 | decode_RA8 RA, SAVE0
877 |.if JIT
878 | cmpwi TMP0, BC_JFORI
879 |.endif
880 | decode_RD8 RD, SAVE0
881 |.if JIT
882 | beq =>BC_JFORI
883 |.endif
884 | b =>BC_FORI
885 |
886 |//-----------------------------------------------------------------------
887 |//-- Fast functions -----------------------------------------------------
888 |//-----------------------------------------------------------------------
889 |
890 |.macro .ffunc, name
891 |->ff_ .. name:
892 |.endmacro
893 |
894 |.macro .ffunc_1, name
895 |->ff_ .. name:
896 | cmplwi NARGS8:RC, 8
897 | evldd CARG1, 0(BASE)
898 | blt ->fff_fallback
899 |.endmacro
900 |
901 |.macro .ffunc_2, name
902 |->ff_ .. name:
903 | cmplwi NARGS8:RC, 16
904 | evldd CARG1, 0(BASE)
905 | evldd CARG2, 8(BASE)
906 | blt ->fff_fallback
907 |.endmacro
908 |
909 |.macro .ffunc_n, name
910 | .ffunc_1 name
911 | checknum CARG1
912 | checkfail ->fff_fallback
913 |.endmacro
914 |
915 |.macro .ffunc_nn, name
916 | .ffunc_2 name
917 | evmergehi TMP0, CARG1, CARG2
918 | checknum TMP0
919 | checkanyfail ->fff_fallback
920 |.endmacro
921 |
922 |// Inlined GC threshold check. Caveat: uses TMP0 and TMP1.
923 |.macro ffgccheck
924 | lwz TMP0, DISPATCH_GL(gc.total)(DISPATCH)
925 | lwz TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
926 | cmplw TMP0, TMP1
927 | bgel ->fff_gcstep
928 |.endmacro
929 |
930 |//-- Base library: checks -----------------------------------------------
931 |
932 |.ffunc assert
933 | cmplwi NARGS8:RC, 8
934 | evldd TMP0, 0(BASE)
935 | blt ->fff_fallback
936 | evaddw TMP1, TISNIL, TISNIL // Synthesize LJ_TFALSE.
937 | la RA, -8(BASE)
938 | evcmpltu cr1, TMP0, TMP1
939 | lwz PC, FRAME_PC(BASE)
940 | bge cr1, ->fff_fallback
941 | evstdd TMP0, 0(RA)
942 | addi RD, NARGS8:RC, 8 // Compute (nresults+1)*8.
943 | beq ->fff_res // Done if exactly 1 argument.
944 | li TMP1, 8
945 | subi RC, RC, 8
946 |1:
947 | cmplw TMP1, RC
948 | evlddx TMP0, BASE, TMP1
949 | evstddx TMP0, RA, TMP1
950 | addi TMP1, TMP1, 8
951 | bne <1
952 | b ->fff_res
953 |
954 |.ffunc type
955 | cmplwi NARGS8:RC, 8
956 | lwz CARG1, 0(BASE)
957 | blt ->fff_fallback
958 | li TMP2, ~LJ_TNUMX
959 | cmplw CARG1, TISNUM
960 | not TMP1, CARG1
961 | isellt TMP1, TMP2, TMP1
962 | slwi TMP1, TMP1, 3
963 | la TMP2, CFUNC:RB->upvalue
964 | evlddx STR:CRET1, TMP2, TMP1
965 | b ->fff_restv
966 |
967 |//-- Base library: getters and setters ---------------------------------
968 |
969 |.ffunc_1 getmetatable
970 | checktab CARG1
971 | evmergehi TMP1, CARG1, CARG1
972 | checkfail >6
973 |1: // Field metatable must be at same offset for GCtab and GCudata!
974 | lwz TAB:RB, TAB:CARG1->metatable
975 |2:
976 | evmr CRET1, TISNIL
977 | cmplwi TAB:RB, 0
978 | lwz STR:RC, DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])(DISPATCH)
979 | beq ->fff_restv
980 | lwz TMP0, TAB:RB->hmask
981 | evmergelo CRET1, TISTAB, TAB:RB // Use metatable as default result.
982 | lwz TMP1, STR:RC->hash
983 | lwz NODE:TMP2, TAB:RB->node
984 | evmergelo STR:RC, TISSTR, STR:RC
985 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask
986 | slwi TMP0, TMP1, 5
987 | slwi TMP1, TMP1, 3
988 | sub TMP1, TMP0, TMP1
989 | add NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
990 |3: // Rearranged logic, because we expect _not_ to find the key.
991 | evldd TMP0, NODE:TMP2->key
992 | evldd TMP1, NODE:TMP2->val
993 | evcmpeq TMP0, STR:RC
994 | lwz NODE:TMP2, NODE:TMP2->next
995 | checkallok >5
996 | cmplwi NODE:TMP2, 0
997 | beq ->fff_restv // Not found, keep default result.
998 | b <3
999 |5:
1000 | checknil TMP1
1001 | checkok ->fff_restv // Ditto for nil value.
1002 | evmr CRET1, TMP1 // Return value of mt.__metatable.
1003 | b ->fff_restv
1004 |
1005 |6:
1006 | cmpwi TMP1, LJ_TUDATA
1007 | not TMP1, TMP1
1008 | beq <1
1009 | checknum CARG1
1010 | slwi TMP1, TMP1, 2
1011 | li TMP2, 4*~LJ_TNUMX
1012 | isellt TMP1, TMP2, TMP1
1013 | la TMP2, DISPATCH_GL(gcroot[GCROOT_BASEMT])(DISPATCH)
1014 | lwzx TAB:RB, TMP2, TMP1
1015 | b <2
1016 |
1017 |.ffunc_2 setmetatable
1018 | // Fast path: no mt for table yet and not clearing the mt.
1019 | evmergehi TMP0, TAB:CARG1, TAB:CARG2
1020 | checktab TMP0
1021 | checkanyfail ->fff_fallback
1022 | lwz TAB:TMP1, TAB:CARG1->metatable
1023 | cmplwi TAB:TMP1, 0
1024 | lbz TMP3, TAB:CARG1->marked
1025 | bne ->fff_fallback
1026 | andi. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
1027 | stw TAB:CARG2, TAB:CARG1->metatable
1028 | beq ->fff_restv
1029 | barrierback TAB:CARG1, TMP3, TMP0
1030 | b ->fff_restv
1031 |
1032 |.ffunc rawget
1033 | cmplwi NARGS8:RC, 16
1034 | evldd CARG2, 0(BASE)
1035 | blt ->fff_fallback
1036 | checktab CARG2
1037 | la CARG3, 8(BASE)
1038 | checkfail ->fff_fallback
1039 | mr CARG1, L
1040 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1041 | // Returns cTValue *.
1042 | evldd CRET1, 0(CRET1)
1043 | b ->fff_restv
1044 |
1045 |//-- Base library: conversions ------------------------------------------
1046 |
1047 |.ffunc tonumber
1048 | // Only handles the number case inline (without a base argument).
1049 | cmplwi NARGS8:RC, 8
1050 | evldd CARG1, 0(BASE)
1051 | bne ->fff_fallback // Exactly one argument.
1052 | checknum CARG1
1053 | checkok ->fff_restv
1054 | b ->fff_fallback
1055 |
1056 |.ffunc_1 tostring
1057 | // Only handles the string or number case inline.
1058 | checkstr CARG1
1059 | // A __tostring method in the string base metatable is ignored.
1060 | checkok ->fff_restv // String key?
1061 | // Handle numbers inline, unless a number base metatable is present.
1062 | lwz TMP0, DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])(DISPATCH)
1063 | checknum CARG1
1064 | cmplwi cr1, TMP0, 0
1065 | stw BASE, L->base // Add frame since C call can throw.
1066 | crand 4*cr0+eq, 4*cr0+lt, 4*cr1+eq
1067 | stw PC, SAVE_PC // Redundant (but a defined value).
1068 | bne ->fff_fallback
1069 | ffgccheck
1070 | mr CARG1, L
1071 | mr CARG2, BASE
1072 | bl extern lj_str_fromnum // (lua_State *L, lua_Number *np)
1073 | // Returns GCstr *.
1074 | evmergelo STR:CRET1, TISSTR, STR:CRET1
1075 | b ->fff_restv
1076 |
1077 |//-- Base library: iterators -------------------------------------------
1078 |
1079 |.ffunc next
1080 | cmplwi NARGS8:RC, 8
1081 | evldd CARG2, 0(BASE)
1082 | blt ->fff_fallback
1083 | evstddx TISNIL, BASE, NARGS8:RC // Set missing 2nd arg to nil.
1084 | checktab TAB:CARG2
1085 | lwz PC, FRAME_PC(BASE)
1086 | checkfail ->fff_fallback
1087 | stw BASE, L->base // Add frame since C call can throw.
1088 | mr CARG1, L
1089 | stw BASE, L->top // Dummy frame length is ok.
1090 | la CARG3, 8(BASE)
1091 | stw PC, SAVE_PC
1092 | bl extern lj_tab_next // (lua_State *L, GCtab *t, TValue *key)
1093 | // Returns 0 at end of traversal.
1094 | cmplwi CRET1, 0
1095 | evmr CRET1, TISNIL
1096 | beq ->fff_restv // End of traversal: return nil.
1097 | evldd TMP0, 8(BASE) // Copy key and value to results.
1098 | la RA, -8(BASE)
1099 | evldd TMP1, 16(BASE)
1100 | evstdd TMP0, 0(RA)
1101 | li RD, (2+1)*8
1102 | evstdd TMP1, 8(RA)
1103 | b ->fff_res
1104 |
1105 |.ffunc_1 pairs
1106 | checktab TAB:CARG1
1107 | lwz PC, FRAME_PC(BASE)
1108 | checkfail ->fff_fallback
1109#if LJ_52
1110 | lwz TAB:TMP2, TAB:CARG1->metatable
1111 | evldd CFUNC:TMP0, CFUNC:RB->upvalue[0]
1112 | cmplwi TAB:TMP2, 0
1113 | la RA, -8(BASE)
1114 | bne ->fff_fallback
1115#else
1116 | evldd CFUNC:TMP0, CFUNC:RB->upvalue[0]
1117 | la RA, -8(BASE)
1118#endif
1119 | evstdd TISNIL, 8(BASE)
1120 | li RD, (3+1)*8
1121 | evstdd CFUNC:TMP0, 0(RA)
1122 | b ->fff_res
1123 |
1124 |.ffunc_2 ipairs_aux
1125 | checktab TAB:CARG1
1126 | lwz PC, FRAME_PC(BASE)
1127 | checkfail ->fff_fallback
1128 | checknum CARG2
1129 | lus TMP3, 0x3ff0
1130 | checkfail ->fff_fallback
1131 | efdctsi TMP2, CARG2
1132 | lwz TMP0, TAB:CARG1->asize
1133 | evmergelo TMP3, TMP3, ZERO
1134 | lwz TMP1, TAB:CARG1->array
1135 | efdadd CARG2, CARG2, TMP3
1136 | addi TMP2, TMP2, 1
1137 | la RA, -8(BASE)
1138 | cmplw TMP0, TMP2
1139 | slwi TMP3, TMP2, 3
1140 | evstdd CARG2, 0(RA)
1141 | ble >2 // Not in array part?
1142 | evlddx TMP1, TMP1, TMP3
1143 |1:
1144 | checknil TMP1
1145 | li RD, (0+1)*8
1146 | checkok ->fff_res // End of iteration, return 0 results.
1147 | li RD, (2+1)*8
1148 | evstdd TMP1, 8(RA)
1149 | b ->fff_res
1150 |2: // Check for empty hash part first. Otherwise call C function.
1151 | lwz TMP0, TAB:CARG1->hmask
1152 | cmplwi TMP0, 0
1153 | li RD, (0+1)*8
1154 | beq ->fff_res
1155 | mr CARG2, TMP2
1156 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1157 | // Returns cTValue * or NULL.
1158 | cmplwi CRET1, 0
1159 | li RD, (0+1)*8
1160 | beq ->fff_res
1161 | evldd TMP1, 0(CRET1)
1162 | b <1
1163 |
1164 |.ffunc_1 ipairs
1165 | checktab TAB:CARG1
1166 | lwz PC, FRAME_PC(BASE)
1167 | checkfail ->fff_fallback
1168#if LJ_52
1169 | lwz TAB:TMP2, TAB:CARG1->metatable
1170 | evldd CFUNC:TMP0, CFUNC:RB->upvalue[0]
1171 | cmplwi TAB:TMP2, 0
1172 | la RA, -8(BASE)
1173 | bne ->fff_fallback
1174#else
1175 | evldd CFUNC:TMP0, CFUNC:RB->upvalue[0]
1176 | la RA, -8(BASE)
1177#endif
1178 | evsplati TMP1, 0
1179 | li RD, (3+1)*8
1180 | evstdd TMP1, 8(BASE)
1181 | evstdd CFUNC:TMP0, 0(RA)
1182 | b ->fff_res
1183 |
1184 |//-- Base library: catch errors ----------------------------------------
1185 |
1186 |.ffunc pcall
1187 | cmplwi NARGS8:RC, 8
1188 | lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1189 | blt ->fff_fallback
1190 | mr TMP2, BASE
1191 | la BASE, 8(BASE)
1192 | // Remember active hook before pcall.
1193 | rlwinm TMP3, TMP3, 32-HOOK_ACTIVE_SHIFT, 31, 31
1194 | subi NARGS8:RC, NARGS8:RC, 8
1195 | addi PC, TMP3, 8+FRAME_PCALL
1196 | b ->vm_call_dispatch
1197 |
1198 |.ffunc_2 xpcall
1199 | lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1200 | mr TMP2, BASE
1201 | checkfunc CARG2 // Traceback must be a function.
1202 | checkfail ->fff_fallback
1203 | la BASE, 16(BASE)
1204 | // Remember active hook before pcall.
1205 | rlwinm TMP3, TMP3, 32-HOOK_ACTIVE_SHIFT, 31, 31
1206 | evstdd CARG2, 0(TMP2) // Swap function and traceback.
1207 | subi NARGS8:RC, NARGS8:RC, 16
1208 | evstdd CARG1, 8(TMP2)
1209 | addi PC, TMP3, 16+FRAME_PCALL
1210 | b ->vm_call_dispatch
1211 |
1212 |//-- Coroutine library --------------------------------------------------
1213 |
1214 |.macro coroutine_resume_wrap, resume
1215 |.if resume
1216 |.ffunc_1 coroutine_resume
1217 | evmergehi TMP0, L:CARG1, L:CARG1
1218 |.else
1219 |.ffunc coroutine_wrap_aux
1220 | lwz L:CARG1, CFUNC:RB->upvalue[0].gcr
1221 |.endif
1222 |.if resume
1223 | cmpwi TMP0, LJ_TTHREAD
1224 | bne ->fff_fallback
1225 |.endif
1226 | lbz TMP0, L:CARG1->status
1227 | lwz TMP1, L:CARG1->cframe
1228 | lwz CARG2, L:CARG1->top
1229 | cmplwi cr0, TMP0, LUA_YIELD
1230 | lwz TMP2, L:CARG1->base
1231 | cmplwi cr1, TMP1, 0
1232 | lwz TMP0, L:CARG1->maxstack
1233 | cmplw cr7, CARG2, TMP2
1234 | lwz PC, FRAME_PC(BASE)
1235 | crorc 4*cr6+lt, 4*cr0+gt, 4*cr1+eq // st>LUA_YIELD || cframe!=0
1236 | add TMP2, CARG2, NARGS8:RC
1237 | crandc 4*cr6+gt, 4*cr7+eq, 4*cr0+eq // base==top && st!=LUA_YIELD
1238 | cmplw cr1, TMP2, TMP0
1239 | cror 4*cr6+lt, 4*cr6+lt, 4*cr6+gt
1240 | stw PC, SAVE_PC
1241 | cror 4*cr6+lt, 4*cr6+lt, 4*cr1+gt // cond1 || cond2 || stackov
1242 | stw BASE, L->base
1243 | blt cr6, ->fff_fallback
1244 |1:
1245 |.if resume
1246 | addi BASE, BASE, 8 // Keep resumed thread in stack for GC.
1247 | subi NARGS8:RC, NARGS8:RC, 8
1248 | subi TMP2, TMP2, 8
1249 |.endif
1250 | stw TMP2, L:CARG1->top
1251 | li TMP1, 0
1252 | stw BASE, L->top
1253 |2: // Move args to coroutine.
1254 | cmpw TMP1, NARGS8:RC
1255 | evlddx TMP0, BASE, TMP1
1256 | beq >3
1257 | evstddx TMP0, CARG2, TMP1
1258 | addi TMP1, TMP1, 8
1259 | b <2
1260 |3:
1261 | li CARG3, 0
1262 | mr L:SAVE0, L:CARG1
1263 | li CARG4, 0
1264 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1265 | // Returns thread status.
1266 |4:
1267 | lwz TMP2, L:SAVE0->base
1268 | cmplwi CRET1, LUA_YIELD
1269 | lwz TMP3, L:SAVE0->top
1270 | li_vmstate INTERP
1271 | lwz BASE, L->base
1272 | st_vmstate
1273 | bgt >8
1274 | sub RD, TMP3, TMP2
1275 | lwz TMP0, L->maxstack
1276 | cmplwi RD, 0
1277 | add TMP1, BASE, RD
1278 | beq >6 // No results?
1279 | cmplw TMP1, TMP0
1280 | li TMP1, 0
1281 | bgt >9 // Need to grow stack?
1282 |
1283 | subi TMP3, RD, 8
1284 | stw TMP2, L:SAVE0->top // Clear coroutine stack.
1285 |5: // Move results from coroutine.
1286 | cmplw TMP1, TMP3
1287 | evlddx TMP0, TMP2, TMP1
1288 | evstddx TMP0, BASE, TMP1
1289 | addi TMP1, TMP1, 8
1290 | bne <5
1291 |6:
1292 | andi. TMP0, PC, FRAME_TYPE
1293 |.if resume
1294 | li TMP1, LJ_TTRUE
1295 | la RA, -8(BASE)
1296 | stw TMP1, -8(BASE) // Prepend true to results.
1297 | addi RD, RD, 16
1298 |.else
1299 | mr RA, BASE
1300 | addi RD, RD, 8
1301 |.endif
1302 |7:
1303 | stw PC, SAVE_PC
1304 | mr MULTRES, RD
1305 | beq ->BC_RET_Z
1306 | b ->vm_return
1307 |
1308 |8: // Coroutine returned with error (at co->top-1).
1309 |.if resume
1310 | andi. TMP0, PC, FRAME_TYPE
1311 | la TMP3, -8(TMP3)
1312 | li TMP1, LJ_TFALSE
1313 | evldd TMP0, 0(TMP3)
1314 | stw TMP3, L:SAVE0->top // Remove error from coroutine stack.
1315 | li RD, (2+1)*8
1316 | stw TMP1, -8(BASE) // Prepend false to results.
1317 | la RA, -8(BASE)
1318 | evstdd TMP0, 0(BASE) // Copy error message.
1319 | b <7
1320 |.else
1321 | mr CARG1, L
1322 | mr CARG2, L:SAVE0
1323 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1324 |.endif
1325 |
1326 |9: // Handle stack expansion on return from yield.
1327 | mr CARG1, L
1328 | srwi CARG2, RD, 3
1329 | bl extern lj_state_growstack // (lua_State *L, int n)
1330 | li CRET1, 0
1331 | b <4
1332 |.endmacro
1333 |
1334 | coroutine_resume_wrap 1 // coroutine.resume
1335 | coroutine_resume_wrap 0 // coroutine.wrap
1336 |
1337 |.ffunc coroutine_yield
1338 | lwz TMP0, L->cframe
1339 | add TMP1, BASE, NARGS8:RC
1340 | stw BASE, L->base
1341 | andi. TMP0, TMP0, CFRAME_RESUME
1342 | stw TMP1, L->top
1343 | li CRET1, LUA_YIELD
1344 | beq ->fff_fallback
1345 | stw ZERO, L->cframe
1346 | stb CRET1, L->status
1347 | b ->vm_leave_unw
1348 |
1349 |//-- Math library -------------------------------------------------------
1350 |
1351 |.ffunc_n math_abs
1352 | efdabs CRET1, CARG1
1353 | // Fallthrough.
1354 |
1355 |->fff_restv:
1356 | // CRET1 = TValue result.
1357 | lwz PC, FRAME_PC(BASE)
1358 | la RA, -8(BASE)
1359 | evstdd CRET1, 0(RA)
1360 |->fff_res1:
1361 | // RA = results, PC = return.
1362 | li RD, (1+1)*8
1363 |->fff_res:
1364 | // RA = results, RD = (nresults+1)*8, PC = return.
1365 | andi. TMP0, PC, FRAME_TYPE
1366 | mr MULTRES, RD
1367 | bne ->vm_return
1368 | lwz INS, -4(PC)
1369 | decode_RB8 RB, INS
1370 |5:
1371 | cmplw RB, RD // More results expected?
1372 | decode_RA8 TMP0, INS
1373 | bgt >6
1374 | ins_next1
1375 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1376 | sub BASE, RA, TMP0
1377 | ins_next2
1378 |
1379 |6: // Fill up results with nil.
1380 | subi TMP1, RD, 8
1381 | addi RD, RD, 8
1382 | evstddx TISNIL, RA, TMP1
1383 | b <5
1384 |
1385 |.macro math_extern, func
1386 | .ffunc math_ .. func
1387 | cmplwi NARGS8:RC, 8
1388 | evldd CARG2, 0(BASE)
1389 | blt ->fff_fallback
1390 | checknum CARG2
1391 | evmergehi CARG1, CARG2, CARG2
1392 | checkfail ->fff_fallback
1393 | bl extern func@plt
1394 | evmergelo CRET1, CRET1, CRET2
1395 | b ->fff_restv
1396 |.endmacro
1397 |
1398 |.macro math_extern2, func
1399 | .ffunc math_ .. func
1400 | cmplwi NARGS8:RC, 16
1401 | evldd CARG2, 0(BASE)
1402 | evldd CARG4, 8(BASE)
1403 | blt ->fff_fallback
1404 | evmergehi CARG1, CARG4, CARG2
1405 | checknum CARG1
1406 | evmergehi CARG3, CARG4, CARG4
1407 | checkanyfail ->fff_fallback
1408 | bl extern func@plt
1409 | evmergelo CRET1, CRET1, CRET2
1410 | b ->fff_restv
1411 |.endmacro
1412 |
1413 |.macro math_round, func
1414 | .ffunc math_ .. func
1415 | cmplwi NARGS8:RC, 8
1416 | evldd CARG2, 0(BASE)
1417 | blt ->fff_fallback
1418 | checknum CARG2
1419 | evmergehi CARG1, CARG2, CARG2
1420 | checkfail ->fff_fallback
1421 | lwz PC, FRAME_PC(BASE)
1422 | bl ->vm_..func.._hilo;
1423 | la RA, -8(BASE)
1424 | evstdd CRET2, 0(RA)
1425 | b ->fff_res1
1426 |.endmacro
1427 |
1428 | math_round floor
1429 | math_round ceil
1430 |
1431 | math_extern sqrt
1432 |
1433 |.ffunc math_log
1434 | cmplwi NARGS8:RC, 8
1435 | evldd CARG2, 0(BASE)
1436 | bne ->fff_fallback // Need exactly 1 argument.
1437 | checknum CARG2
1438 | evmergehi CARG1, CARG2, CARG2
1439 | checkfail ->fff_fallback
1440 | bl extern log@plt
1441 | evmergelo CRET1, CRET1, CRET2
1442 | b ->fff_restv
1443 |
1444 | math_extern log10
1445 | math_extern exp
1446 | math_extern sin
1447 | math_extern cos
1448 | math_extern tan
1449 | math_extern asin
1450 | math_extern acos
1451 | math_extern atan
1452 | math_extern sinh
1453 | math_extern cosh
1454 | math_extern tanh
1455 | math_extern2 pow
1456 | math_extern2 atan2
1457 | math_extern2 fmod
1458 |
1459 |->ff_math_deg:
1460 |.ffunc_n math_rad
1461 | evldd CARG2, CFUNC:RB->upvalue[0]
1462 | efdmul CRET1, CARG1, CARG2
1463 | b ->fff_restv
1464 |
1465 |.ffunc math_ldexp
1466 | cmplwi NARGS8:RC, 16
1467 | evldd CARG2, 0(BASE)
1468 | evldd CARG4, 8(BASE)
1469 | blt ->fff_fallback
1470 | evmergehi CARG1, CARG4, CARG2
1471 | checknum CARG1
1472 | checkanyfail ->fff_fallback
1473 | efdctsi CARG3, CARG4
1474 | bl extern ldexp@plt
1475 | evmergelo CRET1, CRET1, CRET2
1476 | b ->fff_restv
1477 |
1478 |.ffunc math_frexp
1479 | cmplwi NARGS8:RC, 8
1480 | evldd CARG2, 0(BASE)
1481 | blt ->fff_fallback
1482 | checknum CARG2
1483 | evmergehi CARG1, CARG2, CARG2
1484 | checkfail ->fff_fallback
1485 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
1486 | lwz PC, FRAME_PC(BASE)
1487 | bl extern frexp@plt
1488 | lwz TMP1, DISPATCH_GL(tmptv)(DISPATCH)
1489 | evmergelo CRET1, CRET1, CRET2
1490 | efdcfsi CRET2, TMP1
1491 | la RA, -8(BASE)
1492 | evstdd CRET1, 0(RA)
1493 | li RD, (2+1)*8
1494 | evstdd CRET2, 8(RA)
1495 | b ->fff_res
1496 |
1497 |.ffunc math_modf
1498 | cmplwi NARGS8:RC, 8
1499 | evldd CARG2, 0(BASE)
1500 | blt ->fff_fallback
1501 | checknum CARG2
1502 | evmergehi CARG1, CARG2, CARG2
1503 | checkfail ->fff_fallback
1504 | la CARG3, -8(BASE)
1505 | lwz PC, FRAME_PC(BASE)
1506 | bl extern modf@plt
1507 | evmergelo CRET1, CRET1, CRET2
1508 | la RA, -8(BASE)
1509 | evstdd CRET1, 0(BASE)
1510 | li RD, (2+1)*8
1511 | b ->fff_res
1512 |
1513 |.macro math_minmax, name, cmpop
1514 | .ffunc_1 name
1515 | checknum CARG1
1516 | li TMP1, 8
1517 | checkfail ->fff_fallback
1518 |1:
1519 | evlddx CARG2, BASE, TMP1
1520 | cmplw cr1, TMP1, NARGS8:RC
1521 | checknum CARG2
1522 | bge cr1, ->fff_restv // Ok, since CRET1 = CARG1.
1523 | checkfail ->fff_fallback
1524 | cmpop CARG2, CARG1
1525 | addi TMP1, TMP1, 8
1526 | crmove 4*cr0+lt, 4*cr0+gt
1527 | evsel CARG1, CARG2, CARG1
1528 | b <1
1529 |.endmacro
1530 |
1531 | math_minmax math_min, efdtstlt
1532 | math_minmax math_max, efdtstgt
1533 |
1534 |//-- String library -----------------------------------------------------
1535 |
1536 |.ffunc_1 string_len
1537 | checkstr STR:CARG1
1538 | checkfail ->fff_fallback
1539 | lwz TMP0, STR:CARG1->len
1540 | efdcfsi CRET1, TMP0
1541 | b ->fff_restv
1542 |
1543 |.ffunc string_byte // Only handle the 1-arg case here.
1544 | cmplwi NARGS8:RC, 8
1545 | evldd STR:CARG1, 0(BASE)
1546 | bne ->fff_fallback // Need exactly 1 argument.
1547 | checkstr STR:CARG1
1548 | la RA, -8(BASE)
1549 | checkfail ->fff_fallback
1550 | lwz TMP0, STR:CARG1->len
1551 | li RD, (0+1)*8
1552 | lbz TMP1, STR:CARG1[1] // Access is always ok (NUL at end).
1553 | li TMP2, (1+1)*8
1554 | cmplwi TMP0, 0
1555 | lwz PC, FRAME_PC(BASE)
1556 | efdcfsi CRET1, TMP1
1557 | iseleq RD, RD, TMP2
1558 | evstdd CRET1, 0(RA)
1559 | b ->fff_res
1560 |
1561 |.ffunc string_char // Only handle the 1-arg case here.
1562 | ffgccheck
1563 | cmplwi NARGS8:RC, 8
1564 | evldd CARG1, 0(BASE)
1565 | bne ->fff_fallback // Exactly 1 argument.
1566 | checknum CARG1
1567 | la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
1568 | checkfail ->fff_fallback
1569 | efdctsiz TMP0, CARG1
1570 | li CARG3, 1
1571 | cmplwi TMP0, 255
1572 | stb TMP0, 0(CARG2)
1573 | bgt ->fff_fallback
1574 |->fff_newstr:
1575 | mr CARG1, L
1576 | stw BASE, L->base
1577 | stw PC, SAVE_PC
1578 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1579 | // Returns GCstr *.
1580 | lwz BASE, L->base
1581 | evmergelo STR:CRET1, TISSTR, STR:CRET1
1582 | b ->fff_restv
1583 |
1584 |.ffunc string_sub
1585 | ffgccheck
1586 | cmplwi NARGS8:RC, 16
1587 | evldd CARG3, 16(BASE)
1588 | evldd STR:CARG1, 0(BASE)
1589 | blt ->fff_fallback
1590 | evldd CARG2, 8(BASE)
1591 | li TMP2, -1
1592 | beq >1
1593 | checknum CARG3
1594 | checkfail ->fff_fallback
1595 | efdctsiz TMP2, CARG3
1596 |1:
1597 | checknum CARG2
1598 | checkfail ->fff_fallback
1599 | checkstr STR:CARG1
1600 | efdctsiz TMP1, CARG2
1601 | checkfail ->fff_fallback
1602 | lwz TMP0, STR:CARG1->len
1603 | cmplw TMP0, TMP2 // len < end? (unsigned compare)
1604 | add TMP3, TMP2, TMP0
1605 | blt >5
1606 |2:
1607 | cmpwi TMP1, 0 // start <= 0?
1608 | add TMP3, TMP1, TMP0
1609 | ble >7
1610 |3:
1611 | sub. CARG3, TMP2, TMP1
1612 | addi CARG2, STR:CARG1, #STR-1
1613 | addi CARG3, CARG3, 1
1614 | add CARG2, CARG2, TMP1
1615 | isellt CARG3, r0, CARG3
1616 | b ->fff_newstr
1617 |
1618 |5: // Negative end or overflow.
1619 | cmpw TMP0, TMP2
1620 | addi TMP3, TMP3, 1
1621 | iselgt TMP2, TMP3, TMP0 // end = end > len ? len : end+len+1
1622 | b <2
1623 |
1624 |7: // Negative start or underflow.
1625 | cmpwi cr1, TMP3, 0
1626 | iseleq TMP1, r0, TMP3
1627 | isel TMP1, r0, TMP1, 4*cr1+lt
1628 | addi TMP1, TMP1, 1 // start = 1 + (start ? start+len : 0)
1629 | b <3
1630 |
1631 |.ffunc string_rep // Only handle the 1-char case inline.
1632 | ffgccheck
1633 | cmplwi NARGS8:RC, 16
1634 | evldd CARG1, 0(BASE)
1635 | evldd CARG2, 8(BASE)
1636 | bne ->fff_fallback // Exactly 2 arguments.
1637 | checknum CARG2
1638 | checkfail ->fff_fallback
1639 | checkstr STR:CARG1
1640 | efdctsiz CARG3, CARG2
1641 | checkfail ->fff_fallback
1642 | lwz TMP0, STR:CARG1->len
1643 | cmpwi CARG3, 0
1644 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
1645 | ble >2 // Count <= 0? (or non-int)
1646 | cmplwi TMP0, 1
1647 | subi TMP2, CARG3, 1
1648 | blt >2 // Zero length string?
1649 | cmplw cr1, TMP1, CARG3
1650 | bne ->fff_fallback // Fallback for > 1-char strings.
1651 | lbz TMP0, STR:CARG1[1]
1652 | lwz CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
1653 | blt cr1, ->fff_fallback
1654 |1: // Fill buffer with char. Yes, this is suboptimal code (do you care?).
1655 | cmplwi TMP2, 0
1656 | stbx TMP0, CARG2, TMP2
1657 | subi TMP2, TMP2, 1
1658 | bne <1
1659 | b ->fff_newstr
1660 |2: // Return empty string.
1661 | la STR:CRET1, DISPATCH_GL(strempty)(DISPATCH)
1662 | evmergelo CRET1, TISSTR, STR:CRET1
1663 | b ->fff_restv
1664 |
1665 |.ffunc string_reverse
1666 | ffgccheck
1667 | cmplwi NARGS8:RC, 8
1668 | evldd CARG1, 0(BASE)
1669 | blt ->fff_fallback
1670 | checkstr STR:CARG1
1671 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
1672 | checkfail ->fff_fallback
1673 | lwz CARG3, STR:CARG1->len
1674 | la CARG1, #STR(STR:CARG1)
1675 | lwz CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
1676 | li TMP2, 0
1677 | cmplw TMP1, CARG3
1678 | subi TMP3, CARG3, 1
1679 | blt ->fff_fallback
1680 |1: // Reverse string copy.
1681 | cmpwi TMP3, 0
1682 | lbzx TMP1, CARG1, TMP2
1683 | blt ->fff_newstr
1684 | stbx TMP1, CARG2, TMP3
1685 | subi TMP3, TMP3, 1
1686 | addi TMP2, TMP2, 1
1687 | b <1
1688 |
1689 |.macro ffstring_case, name, lo
1690 | .ffunc name
1691 | ffgccheck
1692 | cmplwi NARGS8:RC, 8
1693 | evldd CARG1, 0(BASE)
1694 | blt ->fff_fallback
1695 | checkstr STR:CARG1
1696 | lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
1697 | checkfail ->fff_fallback
1698 | lwz CARG3, STR:CARG1->len
1699 | la CARG1, #STR(STR:CARG1)
1700 | lwz CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
1701 | cmplw TMP1, CARG3
1702 | li TMP2, 0
1703 | blt ->fff_fallback
1704 |1: // ASCII case conversion.
1705 | cmplw TMP2, CARG3
1706 | lbzx TMP1, CARG1, TMP2
1707 | bge ->fff_newstr
1708 | subi TMP0, TMP1, lo
1709 | xori TMP3, TMP1, 0x20
1710 | cmplwi TMP0, 26
1711 | isellt TMP1, TMP3, TMP1
1712 | stbx TMP1, CARG2, TMP2
1713 | addi TMP2, TMP2, 1
1714 | b <1
1715 |.endmacro
1716 |
1717 |ffstring_case string_lower, 65
1718 |ffstring_case string_upper, 97
1719 |
1720 |//-- Table library ------------------------------------------------------
1721 |
1722 |.ffunc_1 table_getn
1723 | checktab CARG1
1724 | checkfail ->fff_fallback
1725 | bl extern lj_tab_len // (GCtab *t)
1726 | // Returns uint32_t (but less than 2^31).
1727 | efdcfsi CRET1, CRET1
1728 | b ->fff_restv
1729 |
1730 |//-- Bit library --------------------------------------------------------
1731 |
1732 |.macro .ffunc_bit, name
1733 | .ffunc_n bit_..name
1734 | efdadd CARG1, CARG1, TOBIT
1735 |.endmacro
1736 |
1737 |.ffunc_bit tobit
1738 |->fff_resbit:
1739 | efdcfsi CRET1, CARG1
1740 | b ->fff_restv
1741 |
1742 |.macro .ffunc_bit_op, name, ins
1743 | .ffunc_bit name
1744 | li TMP1, 8
1745 |1:
1746 | evlddx CARG2, BASE, TMP1
1747 | cmplw cr1, TMP1, NARGS8:RC
1748 | checknum CARG2
1749 | bge cr1, ->fff_resbit
1750 | checkfail ->fff_fallback
1751 | efdadd CARG2, CARG2, TOBIT
1752 | ins CARG1, CARG1, CARG2
1753 | addi TMP1, TMP1, 8
1754 | b <1
1755 |.endmacro
1756 |
1757 |.ffunc_bit_op band, and
1758 |.ffunc_bit_op bor, or
1759 |.ffunc_bit_op bxor, xor
1760 |
1761 |.ffunc_bit bswap
1762 | rotlwi TMP0, CARG1, 8
1763 | rlwimi TMP0, CARG1, 24, 0, 7
1764 | rlwimi TMP0, CARG1, 24, 16, 23
1765 | efdcfsi CRET1, TMP0
1766 | b ->fff_restv
1767 |
1768 |.ffunc_bit bnot
1769 | not TMP0, CARG1
1770 | efdcfsi CRET1, TMP0
1771 | b ->fff_restv
1772 |
1773 |.macro .ffunc_bit_sh, name, ins, shmod
1774 | .ffunc_nn bit_..name
1775 | efdadd CARG2, CARG2, TOBIT
1776 | efdadd CARG1, CARG1, TOBIT
1777 |.if shmod == 1
1778 | rlwinm CARG2, CARG2, 0, 27, 31
1779 |.elif shmod == 2
1780 | neg CARG2, CARG2
1781 |.endif
1782 | ins TMP0, CARG1, CARG2
1783 | efdcfsi CRET1, TMP0
1784 | b ->fff_restv
1785 |.endmacro
1786 |
1787 |.ffunc_bit_sh lshift, slw, 1
1788 |.ffunc_bit_sh rshift, srw, 1
1789 |.ffunc_bit_sh arshift, sraw, 1
1790 |.ffunc_bit_sh rol, rotlw, 0
1791 |.ffunc_bit_sh ror, rotlw, 2
1792 |
1793 |//-----------------------------------------------------------------------
1794 |
1795 |->fff_fallback: // Call fast function fallback handler.
1796 | // BASE = new base, RB = CFUNC, RC = nargs*8
1797 | lwz TMP3, CFUNC:RB->f
1798 | add TMP1, BASE, NARGS8:RC
1799 | lwz PC, FRAME_PC(BASE) // Fallback may overwrite PC.
1800 | addi TMP0, TMP1, 8*LUA_MINSTACK
1801 | lwz TMP2, L->maxstack
1802 | stw PC, SAVE_PC // Redundant (but a defined value).
1803 | cmplw TMP0, TMP2
1804 | stw BASE, L->base
1805 | stw TMP1, L->top
1806 | mr CARG1, L
1807 | bgt >5 // Need to grow stack.
1808 | mtctr TMP3
1809 | bctrl // (lua_State *L)
1810 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1811 | lwz BASE, L->base
1812 | cmpwi CRET1, 0
1813 | slwi RD, CRET1, 3
1814 | la RA, -8(BASE)
1815 | bgt ->fff_res // Returned nresults+1?
1816 |1: // Returned 0 or -1: retry fast path.
1817 | lwz TMP0, L->top
1818 | lwz LFUNC:RB, FRAME_FUNC(BASE)
1819 | sub NARGS8:RC, TMP0, BASE
1820 | bne ->vm_call_tail // Returned -1?
1821 | ins_callt // Returned 0: retry fast path.
1822 |
1823 |// Reconstruct previous base for vmeta_call during tailcall.
1824 |->vm_call_tail:
1825 | andi. TMP0, PC, FRAME_TYPE
1826 | rlwinm TMP1, PC, 0, 0, 28
1827 | bne >3
1828 | lwz INS, -4(PC)
1829 | decode_RA8 TMP1, INS
1830 | addi TMP1, TMP1, 8
1831 |3:
1832 | sub TMP2, BASE, TMP1
1833 | b ->vm_call_dispatch // Resolve again for tailcall.
1834 |
1835 |5: // Grow stack for fallback handler.
1836 | li CARG2, LUA_MINSTACK
1837 | bl extern lj_state_growstack // (lua_State *L, int n)
1838 | lwz BASE, L->base
1839 | cmpw TMP0, TMP0 // Set 4*cr0+eq to force retry.
1840 | b <1
1841 |
1842 |->fff_gcstep: // Call GC step function.
1843 | // BASE = new base, RC = nargs*8
1844 | mflr SAVE0
1845 | stw BASE, L->base
1846 | add TMP0, BASE, NARGS8:RC
1847 | stw PC, SAVE_PC // Redundant (but a defined value).
1848 | stw TMP0, L->top
1849 | mr CARG1, L
1850 | bl extern lj_gc_step // (lua_State *L)
1851 | lwz BASE, L->base
1852 | mtlr SAVE0
1853 | lwz TMP0, L->top
1854 | sub NARGS8:RC, TMP0, BASE
1855 | lwz CFUNC:RB, FRAME_FUNC(BASE)
1856 | blr
1857 |
1858 |//-----------------------------------------------------------------------
1859 |//-- Special dispatch targets -------------------------------------------
1860 |//-----------------------------------------------------------------------
1861 |
1862 |->vm_record: // Dispatch target for recording phase.
1863 |.if JIT
1864 | NYI
1865 |.endif
1866 |
1867 |->vm_rethook: // Dispatch target for return hooks.
1868 | lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1869 | andi. TMP0, TMP3, HOOK_ACTIVE // Hook already active?
1870 | beq >1
1871 |5: // Re-dispatch to static ins.
1872 | addi TMP1, TMP1, GG_DISP2STATIC // Assumes decode_OP4 TMP1, INS.
1873 | lwzx TMP0, DISPATCH, TMP1
1874 | mtctr TMP0
1875 | bctr
1876 |
1877 |->vm_inshook: // Dispatch target for instr/line hooks.
1878 | lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1879 | lwz TMP2, DISPATCH_GL(hookcount)(DISPATCH)
1880 | andi. TMP0, TMP3, HOOK_ACTIVE // Hook already active?
1881 | rlwinm TMP0, TMP3, 31-LUA_HOOKLINE, 31, 0
1882 | bne <5
1883 |
1884 | cmpwi cr1, TMP0, 0
1885 | addic. TMP2, TMP2, -1
1886 | beq cr1, <5
1887 | stw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
1888 | beq >1
1889 | bge cr1, <5
1890 |1:
1891 | mr CARG1, L
1892 | stw MULTRES, SAVE_MULTRES
1893 | mr CARG2, PC
1894 | stw BASE, L->base
1895 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1896 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1897 |3:
1898 | lwz BASE, L->base
1899 |4: // Re-dispatch to static ins.
1900 | lwz INS, -4(PC)
1901 | decode_OP4 TMP1, INS
1902 | decode_RB8 RB, INS
1903 | addi TMP1, TMP1, GG_DISP2STATIC
1904 | decode_RD8 RD, INS
1905 | lwzx TMP0, DISPATCH, TMP1
1906 | decode_RA8 RA, INS
1907 | decode_RC8 RC, INS
1908 | mtctr TMP0
1909 | bctr
1910 |
1911 |->cont_hook: // Continue from hook yield.
1912 | addi PC, PC, 4
1913 | lwz MULTRES, -20(RB) // Restore MULTRES for *M ins.
1914 | b <4
1915 |
1916 |->vm_hotloop: // Hot loop counter underflow.
1917 |.if JIT
1918 | NYI
1919 |.endif
1920 |
1921 |->vm_callhook: // Dispatch target for call hooks.
1922 | mr CARG2, PC
1923 |.if JIT
1924 | b >1
1925 |.endif
1926 |
1927 |->vm_hotcall: // Hot call counter underflow.
1928 |.if JIT
1929 | ori CARG2, PC, 1
1930 |1:
1931 |.endif
1932 | add TMP0, BASE, RC
1933 | stw PC, SAVE_PC
1934 | mr CARG1, L
1935 | stw BASE, L->base
1936 | sub RA, RA, BASE
1937 | stw TMP0, L->top
1938 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1939 | // Returns ASMFunction.
1940 | lwz BASE, L->base
1941 | lwz TMP0, L->top
1942 | stw ZERO, SAVE_PC // Invalidate for subsequent line hook.
1943 | sub NARGS8:RC, TMP0, BASE
1944 | add RA, BASE, RA
1945 | lwz LFUNC:RB, FRAME_FUNC(BASE)
1946 | mtctr CRET1
1947 | bctr
1948 |
1949 |//-----------------------------------------------------------------------
1950 |//-- Trace exit handler -------------------------------------------------
1951 |//-----------------------------------------------------------------------
1952 |
1953 |->vm_exit_handler:
1954 |.if JIT
1955 | NYI
1956 |.endif
1957 |->vm_exit_interp:
1958 |.if JIT
1959 | NYI
1960 |.endif
1961 |
1962 |//-----------------------------------------------------------------------
1963 |//-- Math helper functions ----------------------------------------------
1964 |//-----------------------------------------------------------------------
1965 |
1966 |// FP value rounding. Called by math.floor/math.ceil fast functions
1967 |// and from JIT code.
1968 |//
1969 |// This can be inlined if the CPU has the frin/friz/frip/frim instructions.
1970 |// The alternative hard-float approaches have a deep dependency chain.
1971 |// The resulting latency is at least 3x-7x the double-precision FP latency
1972 |// (e500v2: 6cy, e600: 5cy, Cell: 10cy) or around 20-70 cycles.
1973 |//
1974 |// The soft-float approach is tedious, but much faster (e500v2: ~11cy/~6cy).
1975 |// However it relies on a fast way to transfer the FP value to GPRs
1976 |// (e500v2: 0cy for lo-word, 1cy for hi-word).
1977 |//
1978 |.macro vm_round, name, mode
1979 | // Used temporaries: TMP0, TMP1, TMP2, TMP3.
1980 |->name.._efd: // Input: CARG2, output: CRET2
1981 | evmergehi CARG1, CARG2, CARG2
1982 |->name.._hilo:
1983 | // Input: CARG1 (hi), CARG2 (hi, lo), output: CRET2
1984 | rlwinm TMP2, CARG1, 12, 21, 31
1985 | addic. TMP2, TMP2, -1023 // exp = exponent(x) - 1023
1986 | li TMP1, -1
1987 | cmplwi cr1, TMP2, 51 // 0 <= exp <= 51?
1988 | subfic TMP0, TMP2, 52
1989 | bgt cr1, >1
1990 | lus TMP3, 0xfff0
1991 | slw TMP0, TMP1, TMP0 // lomask = -1 << (52-exp)
1992 | sraw TMP1, TMP3, TMP2 // himask = (int32_t)0xfff00000 >> exp
1993 |.if mode == 2 // trunc(x):
1994 | evmergelo TMP0, TMP1, TMP0
1995 | evand CRET2, CARG2, TMP0 // hi &= himask, lo &= lomask
1996 |.else
1997 | andc TMP2, CARG2, TMP0
1998 | andc TMP3, CARG1, TMP1
1999 | or TMP2, TMP2, TMP3 // ztest = (hi&~himask) | (lo&~lomask)
2000 | srawi TMP3, CARG1, 31 // signmask = (int32_t)hi >> 31
2001 |.if mode == 0 // floor(x):
2002 | and. TMP2, TMP2, TMP3 // iszero = ((ztest & signmask) == 0)
2003 |.else // ceil(x):
2004 | andc. TMP2, TMP2, TMP3 // iszero = ((ztest & ~signmask) == 0)
2005 |.endif
2006 | and CARG2, CARG2, TMP0 // lo &= lomask
2007 | and CARG1, CARG1, TMP1 // hi &= himask
2008 | subc TMP0, CARG2, TMP0
2009 | iseleq TMP0, CARG2, TMP0 // lo = iszero ? lo : lo-lomask
2010 | sube TMP1, CARG1, TMP1
2011 | iseleq TMP1, CARG1, TMP1 // hi = iszero ? hi : hi-himask+carry
2012 | evmergelo CRET2, TMP1, TMP0
2013 |.endif
2014 | blr
2015 |1:
2016 | bgtlr // Already done if >=2^52, +-inf or nan.
2017 |.if mode == 2 // trunc(x):
2018 | rlwinm TMP1, CARG1, 0, 0, 0 // hi = sign(x)
2019 | li TMP0, 0
2020 | evmergelo CRET2, TMP1, TMP0
2021 |.else
2022 | rlwinm TMP2, CARG1, 0, 1, 31
2023 | srawi TMP0, CARG1, 31 // signmask = (int32_t)hi >> 31
2024 | or TMP2, TMP2, CARG2 // ztest = abs(hi) | lo
2025 | lus TMP1, 0x3ff0
2026 |.if mode == 0 // floor(x):
2027 | and. TMP2, TMP2, TMP0 // iszero = ((ztest & signmask) == 0)
2028 |.else // ceil(x):
2029 | andc. TMP2, TMP2, TMP0 // iszero = ((ztest & ~signmask) == 0)
2030 |.endif
2031 | li TMP0, 0
2032 | iseleq TMP1, r0, TMP1
2033 | rlwimi CARG1, TMP1, 0, 1, 31 // hi = sign(x) | (iszero ? 0.0 : 1.0)
2034 | evmergelo CRET2, CARG1, TMP0
2035 |.endif
2036 | blr
2037 |.endmacro
2038 |
2039 |->vm_floor:
2040 | mflr CARG3
2041 | evmergelo CARG2, CARG1, CARG2
2042 | bl ->vm_floor_hilo
2043 | mtlr CARG3
2044 | evmergehi CRET1, CRET2, CRET2
2045 | blr
2046 |
2047 | vm_round vm_floor, 0
2048 | vm_round vm_ceil, 1
2049 |.if JIT
2050 | vm_round vm_trunc, 2
2051 |.else
2052 |->vm_trunc_efd:
2053 |->vm_trunc_hilo:
2054 |.endif
2055 |
2056 |//-----------------------------------------------------------------------
2057 |//-- Miscellaneous functions --------------------------------------------
2058 |//-----------------------------------------------------------------------
2059 |
2060 |//-----------------------------------------------------------------------
2061 |//-- FFI helper functions -----------------------------------------------
2062 |//-----------------------------------------------------------------------
2063 |
2064 |->vm_ffi_call:
2065 |.if FFI
2066 | NYI
2067 |.endif
2068 |
2069 |//-----------------------------------------------------------------------
2070}
2071
2072/* Generate the code for a single instruction. */
2073static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2074{
2075 int vk = 0;
2076 |=>defop:
2077
2078 switch (op) {
2079
2080 /* -- Comparison ops ---------------------------------------------------- */
2081
2082 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2083
2084 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2085 | // RA = src1*8, RD = src2*8, JMP with RD = target
2086 | evlddx TMP0, BASE, RA
2087 | addi PC, PC, 4
2088 | evlddx TMP1, BASE, RD
2089 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2090 | lwz TMP2, -4(PC)
2091 | evmergehi RB, TMP0, TMP1
2092 | decode_RD4 TMP2, TMP2
2093 | checknum RB
2094 | add TMP2, TMP2, TMP3
2095 | checkanyfail ->vmeta_comp
2096 | efdcmplt TMP0, TMP1
2097 if (op == BC_ISLE || op == BC_ISGT) {
2098 | efdcmpeq cr1, TMP0, TMP1
2099 | cror 4*cr0+gt, 4*cr0+gt, 4*cr1+gt
2100 }
2101 if (op == BC_ISLT || op == BC_ISLE) {
2102 | iselgt PC, TMP2, PC
2103 } else {
2104 | iselgt PC, PC, TMP2
2105 }
2106 | ins_next
2107 break;
2108
2109 case BC_ISEQV: case BC_ISNEV:
2110 vk = op == BC_ISEQV;
2111 | // RA = src1*8, RD = src2*8, JMP with RD = target
2112 | evlddx CARG2, BASE, RA
2113 | addi PC, PC, 4
2114 | evlddx CARG3, BASE, RD
2115 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2116 | lwz TMP2, -4(PC)
2117 | evmergehi RB, CARG2, CARG3
2118 | decode_RD4 TMP2, TMP2
2119 | checknum RB
2120 | add TMP2, TMP2, TMP3
2121 | checkanyfail >5
2122 | efdcmpeq CARG2, CARG3
2123 if (vk) {
2124 | iselgt PC, TMP2, PC
2125 } else {
2126 | iselgt PC, PC, TMP2
2127 }
2128 |1:
2129 | ins_next
2130 |
2131 |5: // Either or both types are not numbers.
2132 | evcmpeq CARG2, CARG3
2133 | not TMP3, RB
2134 | cmplwi cr1, TMP3, ~LJ_TISPRI // Primitive?
2135 | crorc 4*cr7+lt, 4*cr0+so, 4*cr0+lt // 1: Same tv or different type.
2136 | cmplwi cr6, TMP3, ~LJ_TISTABUD // Table or userdata?
2137 | crandc 4*cr7+gt, 4*cr0+lt, 4*cr1+gt // 2: Same type and primitive.
2138 | mr SAVE0, PC
2139 if (vk) {
2140 | isel PC, TMP2, PC, 4*cr7+gt
2141 } else {
2142 | isel TMP2, PC, TMP2, 4*cr7+gt
2143 }
2144 | cror 4*cr7+lt, 4*cr7+lt, 4*cr7+gt // 1 or 2.
2145 if (vk) {
2146 | isel PC, TMP2, PC, 4*cr0+so
2147 } else {
2148 | isel PC, PC, TMP2, 4*cr0+so
2149 }
2150 | blt cr7, <1 // Done if 1 or 2.
2151 | blt cr6, <1 // Done if not tab/ud.
2152 |
2153 | // Different tables or userdatas. Need to check __eq metamethod.
2154 | // Field metatable must be at same offset for GCtab and GCudata!
2155 | lwz TAB:TMP2, TAB:CARG2->metatable
2156 | li CARG4, 1-vk // ne = 0 or 1.
2157 | cmplwi TAB:TMP2, 0
2158 | beq <1 // No metatable?
2159 | lbz TMP2, TAB:TMP2->nomm
2160 | andi. TMP2, TMP2, 1<<MM_eq
2161 | bne <1 // Or 'no __eq' flag set?
2162 | mr PC, SAVE0 // Restore old PC.
2163 | b ->vmeta_equal // Handle __eq metamethod.
2164 break;
2165
2166 case BC_ISEQS: case BC_ISNES:
2167 vk = op == BC_ISEQS;
2168 | // RA = src*8, RD = str_const*8 (~), JMP with RD = target
2169 | evlddx TMP0, BASE, RA
2170 | srwi RD, RD, 1
2171 | lwz INS, 0(PC)
2172 | subfic RD, RD, -4
2173 | addi PC, PC, 4
2174 | lwzx STR:TMP1, KBASE, RD // KBASE-4-str_const*4
2175 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2176 | decode_RD4 TMP2, INS
2177 | evmergelo STR:TMP1, TISSTR, STR:TMP1
2178 | add TMP2, TMP2, TMP3
2179 | evcmpeq TMP0, STR:TMP1
2180 if (vk) {
2181 | isel PC, TMP2, PC, 4*cr0+so
2182 } else {
2183 | isel PC, PC, TMP2, 4*cr0+so
2184 }
2185 | ins_next
2186 break;
2187
2188 case BC_ISEQN: case BC_ISNEN:
2189 vk = op == BC_ISEQN;
2190 | // RA = src*8, RD = num_const*8, JMP with RD = target
2191 | evlddx TMP0, BASE, RA
2192 | addi PC, PC, 4
2193 | evlddx TMP1, KBASE, RD
2194 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2195 | lwz INS, -4(PC)
2196 | checknum TMP0
2197 | checkfail >5
2198 | efdcmpeq TMP0, TMP1
2199 |1:
2200 | decode_RD4 TMP2, INS
2201 | add TMP2, TMP2, TMP3
2202 if (vk) {
2203 | iselgt PC, TMP2, PC
2204 |5:
2205 } else {
2206 | iselgt PC, PC, TMP2
2207 }
2208 |3:
2209 | ins_next
2210 if (!vk) {
2211 |5:
2212 | decode_RD4 TMP2, INS
2213 | add PC, TMP2, TMP3
2214 | b <3
2215 }
2216 break;
2217
2218 case BC_ISEQP: case BC_ISNEP:
2219 vk = op == BC_ISEQP;
2220 | // RA = src*8, RD = primitive_type*8 (~), JMP with RD = target
2221 | lwzx TMP0, BASE, RA
2222 | srwi TMP1, RD, 3
2223 | lwz INS, 0(PC)
2224 | addi PC, PC, 4
2225 | not TMP1, TMP1
2226 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2227 | cmplw TMP0, TMP1
2228 | decode_RD4 TMP2, INS
2229 | add TMP2, TMP2, TMP3
2230 if (vk) {
2231 | iseleq PC, TMP2, PC
2232 } else {
2233 | iseleq PC, PC, TMP2
2234 }
2235 | ins_next
2236 break;
2237
2238 /* -- Unary test and copy ops ------------------------------------------- */
2239
2240 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2241 | // RA = dst*8 or unused, RD = src*8, JMP with RD = target
2242 | evlddx TMP0, BASE, RD
2243 | evaddw TMP1, TISNIL, TISNIL // Synthesize LJ_TFALSE.
2244 | lwz INS, 0(PC)
2245 | evcmpltu TMP0, TMP1
2246 | addi PC, PC, 4
2247 if (op == BC_IST || op == BC_ISF) {
2248 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
2249 | decode_RD4 TMP2, INS
2250 | add TMP2, TMP2, TMP3
2251 if (op == BC_IST) {
2252 | isellt PC, TMP2, PC
2253 } else {
2254 | isellt PC, PC, TMP2
2255 }
2256 } else {
2257 if (op == BC_ISTC) {
2258 | checkfail >1
2259 } else {
2260 | checkok >1
2261 }
2262 | addis PC, PC, -(BCBIAS_J*4 >> 16)
2263 | decode_RD4 TMP2, INS
2264 | evstddx TMP0, BASE, RA
2265 | add PC, PC, TMP2
2266 |1:
2267 }
2268 | ins_next
2269 break;
2270
2271 /* -- Unary ops --------------------------------------------------------- */
2272
2273 case BC_MOV:
2274 | // RA = dst*8, RD = src*8
2275 | ins_next1
2276 | evlddx TMP0, BASE, RD
2277 | evstddx TMP0, BASE, RA
2278 | ins_next2
2279 break;
2280 case BC_NOT:
2281 | // RA = dst*8, RD = src*8
2282 | ins_next1
2283 | lwzx TMP0, BASE, RD
2284 | subfic TMP1, TMP0, LJ_TTRUE
2285 | adde TMP0, TMP0, TMP1
2286 | stwx TMP0, BASE, RA
2287 | ins_next2
2288 break;
2289 case BC_UNM:
2290 | // RA = dst*8, RD = src*8
2291 | evlddx TMP0, BASE, RD
2292 | checknum TMP0
2293 | checkfail ->vmeta_unm
2294 | efdneg TMP0, TMP0
2295 | ins_next1
2296 | evstddx TMP0, BASE, RA
2297 | ins_next2
2298 break;
2299 case BC_LEN:
2300 | // RA = dst*8, RD = src*8
2301 | evlddx CARG1, BASE, RD
2302 | checkstr CARG1
2303 | checkfail >2
2304 | lwz CRET1, STR:CARG1->len
2305 |1:
2306 | ins_next1
2307 | efdcfsi TMP0, CRET1
2308 | evstddx TMP0, BASE, RA
2309 | ins_next2
2310 |2:
2311 | checktab CARG1
2312 | checkfail ->vmeta_len
2313#if LJ_52
2314 | lwz TAB:TMP2, TAB:CARG1->metatable
2315 | cmplwi TAB:TMP2, 0
2316 | bne >9
2317 |3:
2318#endif
2319 |->BC_LEN_Z:
2320 | bl extern lj_tab_len // (GCtab *t)
2321 | // Returns uint32_t (but less than 2^31).
2322 | b <1
2323#if LJ_52
2324 |9:
2325 | lbz TMP0, TAB:TMP2->nomm
2326 | andi. TMP0, TMP0, 1<<MM_len
2327 | bne <3 // 'no __len' flag set: done.
2328 | b ->vmeta_len
2329#endif
2330 break;
2331
2332 /* -- Binary ops -------------------------------------------------------- */
2333
2334 |.macro ins_arithpre, t0, t1
2335 | // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
2336 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2337 ||switch (vk) {
2338 ||case 0:
2339 | evlddx t0, BASE, RB
2340 | checknum t0
2341 | evlddx t1, KBASE, RC
2342 | checkfail ->vmeta_arith_vn
2343 || break;
2344 ||case 1:
2345 | evlddx t1, BASE, RB
2346 | checknum t1
2347 | evlddx t0, KBASE, RC
2348 | checkfail ->vmeta_arith_nv
2349 || break;
2350 ||default:
2351 | evlddx t0, BASE, RB
2352 | evlddx t1, BASE, RC
2353 | evmergehi TMP2, t0, t1
2354 | checknum TMP2
2355 | checkanyfail ->vmeta_arith_vv
2356 || break;
2357 ||}
2358 |.endmacro
2359 |
2360 |.macro ins_arith, ins
2361 | ins_arithpre TMP0, TMP1
2362 | ins_next1
2363 | ins TMP0, TMP0, TMP1
2364 | evstddx TMP0, BASE, RA
2365 | ins_next2
2366 |.endmacro
2367
2368 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2369 | ins_arith efdadd
2370 break;
2371 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2372 | ins_arith efdsub
2373 break;
2374 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2375 | ins_arith efdmul
2376 break;
2377 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2378 | ins_arith efddiv
2379 break;
2380 case BC_MODVN:
2381 | ins_arithpre RD, SAVE0
2382 |->BC_MODVN_Z:
2383 | efddiv CARG2, RD, SAVE0
2384 | bl ->vm_floor_efd // floor(b/c)
2385 | efdmul TMP0, CRET2, SAVE0
2386 | ins_next1
2387 | efdsub TMP0, RD, TMP0 // b - floor(b/c)*c
2388 | evstddx TMP0, BASE, RA
2389 | ins_next2
2390 break;
2391 case BC_MODNV: case BC_MODVV:
2392 | ins_arithpre RD, SAVE0
2393 | b ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway.
2394 break;
2395 case BC_POW:
2396 | evlddx CARG2, BASE, RB
2397 | evlddx CARG4, BASE, RC
2398 | evmergehi CARG1, CARG4, CARG2
2399 | checknum CARG1
2400 | evmergehi CARG3, CARG4, CARG4
2401 | checkanyfail ->vmeta_arith_vv
2402 | bl extern pow@plt
2403 | evmergelo CRET2, CRET1, CRET2
2404 | evstddx CRET2, BASE, RA
2405 | ins_next
2406 break;
2407
2408 case BC_CAT:
2409 | // RA = dst*8, RB = src_start*8, RC = src_end*8
2410 | sub CARG3, RC, RB
2411 | stw BASE, L->base
2412 | add CARG2, BASE, RC
2413 | mr SAVE0, RB
2414 |->BC_CAT_Z:
2415 | stw PC, SAVE_PC
2416 | mr CARG1, L
2417 | srwi CARG3, CARG3, 3
2418 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2419 | // Returns NULL (finished) or TValue * (metamethod).
2420 | cmplwi CRET1, 0
2421 | lwz BASE, L->base
2422 | bne ->vmeta_binop
2423 | evlddx TMP0, BASE, SAVE0 // Copy result from RB to RA.
2424 | evstddx TMP0, BASE, RA
2425 | ins_next
2426 break;
2427
2428 /* -- Constant ops ------------------------------------------------------ */
2429
2430 case BC_KSTR:
2431 | // RA = dst*8, RD = str_const*8 (~)
2432 | ins_next1
2433 | srwi TMP1, RD, 1
2434 | subfic TMP1, TMP1, -4
2435 | lwzx TMP0, KBASE, TMP1 // KBASE-4-str_const*4
2436 | evmergelo TMP0, TISSTR, TMP0
2437 | evstddx TMP0, BASE, RA
2438 | ins_next2
2439 break;
2440 case BC_KCDATA:
2441 |.if FFI
2442 | // RA = dst*8, RD = cdata_const*8 (~)
2443 | ins_next1
2444 | srwi TMP1, RD, 1
2445 | subfic TMP1, TMP1, -4
2446 | lwzx TMP0, KBASE, TMP1 // KBASE-4-cdata_const*4
2447 | li TMP2, LJ_TCDATA
2448 | evmergelo TMP0, TMP2, TMP0
2449 | evstddx TMP0, BASE, RA
2450 | ins_next2
2451 |.endif
2452 break;
2453 case BC_KSHORT:
2454 | // RA = dst*8, RD = int16_literal*8
2455 | srwi TMP1, RD, 3
2456 | extsh TMP1, TMP1
2457 | ins_next1
2458 | efdcfsi TMP0, TMP1
2459 | evstddx TMP0, BASE, RA
2460 | ins_next2
2461 break;
2462 case BC_KNUM:
2463 | // RA = dst*8, RD = num_const*8
2464 | evlddx TMP0, KBASE, RD
2465 | ins_next1
2466 | evstddx TMP0, BASE, RA
2467 | ins_next2
2468 break;
2469 case BC_KPRI:
2470 | // RA = dst*8, RD = primitive_type*8 (~)
2471 | srwi TMP1, RD, 3
2472 | not TMP0, TMP1
2473 | ins_next1
2474 | stwx TMP0, BASE, RA
2475 | ins_next2
2476 break;
2477 case BC_KNIL:
2478 | // RA = base*8, RD = end*8
2479 | evstddx TISNIL, BASE, RA
2480 | addi RA, RA, 8
2481 |1:
2482 | evstddx TISNIL, BASE, RA
2483 | cmpw RA, RD
2484 | addi RA, RA, 8
2485 | blt <1
2486 | ins_next_
2487 break;
2488
2489 /* -- Upvalue and function ops ------------------------------------------ */
2490
2491 case BC_UGET:
2492 | // RA = dst*8, RD = uvnum*8
2493 | ins_next1
2494 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2495 | srwi RD, RD, 1
2496 | addi RD, RD, offsetof(GCfuncL, uvptr)
2497 | lwzx UPVAL:RB, LFUNC:RB, RD
2498 | lwz TMP1, UPVAL:RB->v
2499 | evldd TMP0, 0(TMP1)
2500 | evstddx TMP0, BASE, RA
2501 | ins_next2
2502 break;
2503 case BC_USETV:
2504 | // RA = uvnum*8, RD = src*8
2505 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2506 | srwi RA, RA, 1
2507 | addi RA, RA, offsetof(GCfuncL, uvptr)
2508 | evlddx TMP1, BASE, RD
2509 | lwzx UPVAL:RB, LFUNC:RB, RA
2510 | lbz TMP3, UPVAL:RB->marked
2511 | lwz CARG2, UPVAL:RB->v
2512 | andi. TMP3, TMP3, LJ_GC_BLACK // isblack(uv)
2513 | lbz TMP0, UPVAL:RB->closed
2514 | evmergehi TMP2, TMP1, TMP1
2515 | evstdd TMP1, 0(CARG2)
2516 | cmplwi cr1, TMP0, 0
2517 | cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
2518 | subi TMP2, TMP2, (LJ_TISNUM+1)
2519 | bne >2 // Upvalue is closed and black?
2520 |1:
2521 | ins_next
2522 |
2523 |2: // Check if new value is collectable.
2524 | cmplwi TMP2, LJ_TISGCV - (LJ_TISNUM+1)
2525 | bge <1 // tvisgcv(v)
2526 | lbz TMP3, GCOBJ:TMP1->gch.marked
2527 | andi. TMP3, TMP3, LJ_GC_WHITES // iswhite(v)
2528 | la CARG1, GG_DISP2G(DISPATCH)
2529 | // Crossed a write barrier. Move the barrier forward.
2530 | beq <1
2531 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2532 | b <1
2533 break;
2534 case BC_USETS:
2535 | // RA = uvnum*8, RD = str_const*8 (~)
2536 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2537 | srwi TMP1, RD, 1
2538 | srwi RA, RA, 1
2539 | subfic TMP1, TMP1, -4
2540 | addi RA, RA, offsetof(GCfuncL, uvptr)
2541 | lwzx STR:TMP1, KBASE, TMP1 // KBASE-4-str_const*4
2542 | lwzx UPVAL:RB, LFUNC:RB, RA
2543 | evmergelo STR:TMP1, TISSTR, STR:TMP1
2544 | lbz TMP3, UPVAL:RB->marked
2545 | lwz CARG2, UPVAL:RB->v
2546 | andi. TMP3, TMP3, LJ_GC_BLACK // isblack(uv)
2547 | lbz TMP3, STR:TMP1->marked
2548 | lbz TMP2, UPVAL:RB->closed
2549 | evstdd STR:TMP1, 0(CARG2)
2550 | bne >2
2551 |1:
2552 | ins_next
2553 |
2554 |2: // Check if string is white and ensure upvalue is closed.
2555 | andi. TMP3, TMP3, LJ_GC_WHITES // iswhite(str)
2556 | cmplwi cr1, TMP2, 0
2557 | cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
2558 | la CARG1, GG_DISP2G(DISPATCH)
2559 | // Crossed a write barrier. Move the barrier forward.
2560 | beq <1
2561 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2562 | b <1
2563 break;
2564 case BC_USETN:
2565 | // RA = uvnum*8, RD = num_const*8
2566 | ins_next1
2567 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2568 | srwi RA, RA, 1
2569 | addi RA, RA, offsetof(GCfuncL, uvptr)
2570 | evlddx TMP0, KBASE, RD
2571 | lwzx UPVAL:RB, LFUNC:RB, RA
2572 | lwz TMP1, UPVAL:RB->v
2573 | evstdd TMP0, 0(TMP1)
2574 | ins_next2
2575 break;
2576 case BC_USETP:
2577 | // RA = uvnum*8, RD = primitive_type*8 (~)
2578 | ins_next1
2579 | lwz LFUNC:RB, FRAME_FUNC(BASE)
2580 | srwi RA, RA, 1
2581 | addi RA, RA, offsetof(GCfuncL, uvptr)
2582 | srwi TMP0, RD, 3
2583 | lwzx UPVAL:RB, LFUNC:RB, RA
2584 | not TMP0, TMP0
2585 | lwz TMP1, UPVAL:RB->v
2586 | stw TMP0, 0(TMP1)
2587 | ins_next2
2588 break;
2589
2590 case BC_UCLO:
2591 | // RA = level*8, RD = target
2592 | lwz TMP1, L->openupval
2593 | branch_RD // Do this first since RD is not saved.
2594 | stw BASE, L->base
2595 | cmplwi TMP1, 0
2596 | mr CARG1, L
2597 | beq >1
2598 | add CARG2, BASE, RA
2599 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2600 | lwz BASE, L->base
2601 |1:
2602 | ins_next
2603 break;
2604
2605 case BC_FNEW:
2606 | // RA = dst*8, RD = proto_const*8 (~) (holding function prototype)
2607 | srwi TMP1, RD, 1
2608 | stw BASE, L->base
2609 | subfic TMP1, TMP1, -4
2610 | stw PC, SAVE_PC
2611 | lwzx CARG2, KBASE, TMP1 // KBASE-4-tab_const*4
2612 | mr CARG1, L
2613 | lwz CARG3, FRAME_FUNC(BASE)
2614 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2615 | bl extern lj_func_newL_gc
2616 | // Returns GCfuncL *.
2617 | lwz BASE, L->base
2618 | evmergelo LFUNC:CRET1, TISFUNC, LFUNC:CRET1
2619 | evstddx LFUNC:CRET1, BASE, RA
2620 | ins_next
2621 break;
2622
2623 /* -- Table ops --------------------------------------------------------- */
2624
2625 case BC_TNEW:
2626 case BC_TDUP:
2627 | // RA = dst*8, RD = (hbits|asize)*8 | tab_const*8 (~)
2628 | lwz TMP0, DISPATCH_GL(gc.total)(DISPATCH)
2629 | mr CARG1, L
2630 | lwz TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
2631 | stw BASE, L->base
2632 | cmplw TMP0, TMP1
2633 | stw PC, SAVE_PC
2634 | bge >5
2635 |1:
2636 if (op == BC_TNEW) {
2637 | rlwinm CARG2, RD, 29, 21, 31
2638 | rlwinm CARG3, RD, 18, 27, 31
2639 | cmpwi CARG2, 0x7ff
2640 | li TMP1, 0x801
2641 | iseleq CARG2, TMP1, CARG2
2642 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2643 | // Returns Table *.
2644 } else {
2645 | srwi TMP1, RD, 1
2646 | subfic TMP1, TMP1, -4
2647 | lwzx CARG2, KBASE, TMP1 // KBASE-4-tab_const*4
2648 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2649 | // Returns Table *.
2650 }
2651 | lwz BASE, L->base
2652 | evmergelo TAB:CRET1, TISTAB, TAB:CRET1
2653 | evstddx TAB:CRET1, BASE, RA
2654 | ins_next
2655 |5:
2656 | mr SAVE0, RD
2657 | bl extern lj_gc_step_fixtop // (lua_State *L)
2658 | mr RD, SAVE0
2659 | mr CARG1, L
2660 | b <1
2661 break;
2662
2663 case BC_GGET:
2664 | // RA = dst*8, RD = str_const*8 (~)
2665 case BC_GSET:
2666 | // RA = src*8, RD = str_const*8 (~)
2667 | lwz LFUNC:TMP2, FRAME_FUNC(BASE)
2668 | srwi TMP1, RD, 1
2669 | lwz TAB:RB, LFUNC:TMP2->env
2670 | subfic TMP1, TMP1, -4
2671 | lwzx STR:RC, KBASE, TMP1 // KBASE-4-str_const*4
2672 if (op == BC_GGET) {
2673 | b ->BC_TGETS_Z
2674 } else {
2675 | b ->BC_TSETS_Z
2676 }
2677 break;
2678
2679 case BC_TGETV:
2680 | // RA = dst*8, RB = table*8, RC = key*8
2681 | evlddx TAB:RB, BASE, RB
2682 | evlddx RC, BASE, RC
2683 | checktab TAB:RB
2684 | checkfail ->vmeta_tgetv
2685 | checknum RC
2686 | checkfail >5
2687 | // Convert number key to integer
2688 | efdctsi TMP2, RC
2689 | lwz TMP0, TAB:RB->asize
2690 | efdcfsi TMP1, TMP2
2691 | cmplw cr0, TMP0, TMP2
2692 | efdcmpeq cr1, RC, TMP1
2693 | lwz TMP1, TAB:RB->array
2694 | crand 4*cr0+gt, 4*cr0+gt, 4*cr1+gt
2695 | slwi TMP2, TMP2, 3
2696 | ble ->vmeta_tgetv // Integer key and in array part?
2697 | evlddx TMP1, TMP1, TMP2
2698 | checknil TMP1
2699 | checkok >2
2700 |1:
2701 | evstddx TMP1, BASE, RA
2702 | ins_next
2703 |
2704 |2: // Check for __index if table value is nil.
2705 | lwz TAB:TMP2, TAB:RB->metatable
2706 | cmplwi TAB:TMP2, 0
2707 | beq <1 // No metatable: done.
2708 | lbz TMP0, TAB:TMP2->nomm
2709 | andi. TMP0, TMP0, 1<<MM_index
2710 | bne <1 // 'no __index' flag set: done.
2711 | b ->vmeta_tgetv
2712 |
2713 |5:
2714 | checkstr STR:RC // String key?
2715 | checkok ->BC_TGETS_Z
2716 | b ->vmeta_tgetv
2717 break;
2718 case BC_TGETS:
2719 | // RA = dst*8, RB = table*8, RC = str_const*8 (~)
2720 | evlddx TAB:RB, BASE, RB
2721 | srwi TMP1, RC, 1
2722 | checktab TAB:RB
2723 | subfic TMP1, TMP1, -4
2724 | lwzx STR:RC, KBASE, TMP1 // KBASE-4-str_const*4
2725 | checkfail ->vmeta_tgets1
2726 |->BC_TGETS_Z:
2727 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8
2728 | lwz TMP0, TAB:RB->hmask
2729 | lwz TMP1, STR:RC->hash
2730 | lwz NODE:TMP2, TAB:RB->node
2731 | evmergelo STR:RC, TISSTR, STR:RC
2732 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask
2733 | slwi TMP0, TMP1, 5
2734 | slwi TMP1, TMP1, 3
2735 | sub TMP1, TMP0, TMP1
2736 | add NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
2737 |1:
2738 | evldd TMP0, NODE:TMP2->key
2739 | evldd TMP1, NODE:TMP2->val
2740 | evcmpeq TMP0, STR:RC
2741 | checkanyfail >4
2742 | checknil TMP1
2743 | checkok >5 // Key found, but nil value?
2744 |3:
2745 | evstddx TMP1, BASE, RA
2746 | ins_next
2747 |
2748 |4: // Follow hash chain.
2749 | lwz NODE:TMP2, NODE:TMP2->next
2750 | cmplwi NODE:TMP2, 0
2751 | bne <1
2752 | // End of hash chain: key not found, nil result.
2753 | evmr TMP1, TISNIL
2754 |
2755 |5: // Check for __index if table value is nil.
2756 | lwz TAB:TMP2, TAB:RB->metatable
2757 | cmplwi TAB:TMP2, 0
2758 | beq <3 // No metatable: done.
2759 | lbz TMP0, TAB:TMP2->nomm
2760 | andi. TMP0, TMP0, 1<<MM_index
2761 | bne <3 // 'no __index' flag set: done.
2762 | b ->vmeta_tgets
2763 break;
2764 case BC_TGETB:
2765 | // RA = dst*8, RB = table*8, RC = index*8
2766 | evlddx TAB:RB, BASE, RB
2767 | srwi TMP0, RC, 3
2768 | checktab TAB:RB
2769 | checkfail ->vmeta_tgetb
2770 | lwz TMP1, TAB:RB->asize
2771 | lwz TMP2, TAB:RB->array
2772 | cmplw TMP0, TMP1
2773 | bge ->vmeta_tgetb
2774 | evlddx TMP1, TMP2, RC
2775 | checknil TMP1
2776 | checkok >5
2777 |1:
2778 | ins_next1
2779 | evstddx TMP1, BASE, RA
2780 | ins_next2
2781 |
2782 |5: // Check for __index if table value is nil.
2783 | lwz TAB:TMP2, TAB:RB->metatable
2784 | cmplwi TAB:TMP2, 0
2785 | beq <1 // No metatable: done.
2786 | lbz TMP2, TAB:TMP2->nomm
2787 | andi. TMP2, TMP2, 1<<MM_index
2788 | bne <1 // 'no __index' flag set: done.
2789 | b ->vmeta_tgetb // Caveat: preserve TMP0!
2790 break;
2791
2792 case BC_TSETV:
2793 | // RA = src*8, RB = table*8, RC = key*8
2794 | evlddx TAB:RB, BASE, RB
2795 | evlddx RC, BASE, RC
2796 | checktab TAB:RB
2797 | checkfail ->vmeta_tsetv
2798 | checknum RC
2799 | checkfail >5
2800 | // Convert number key to integer
2801 | efdctsi TMP2, RC
2802 | evlddx SAVE0, BASE, RA
2803 | lwz TMP0, TAB:RB->asize
2804 | efdcfsi TMP1, TMP2
2805 | cmplw cr0, TMP0, TMP2
2806 | efdcmpeq cr1, RC, TMP1
2807 | lwz TMP1, TAB:RB->array
2808 | crand 4*cr0+gt, 4*cr0+gt, 4*cr1+gt
2809 | slwi TMP0, TMP2, 3
2810 | ble ->vmeta_tsetv // Integer key and in array part?
2811 | lbz TMP3, TAB:RB->marked
2812 | evlddx TMP2, TMP1, TMP0
2813 | checknil TMP2
2814 | checkok >3
2815 |1:
2816 | andi. TMP2, TMP3, LJ_GC_BLACK // isblack(table)
2817 | evstddx SAVE0, TMP1, TMP0
2818 | bne >7
2819 |2:
2820 | ins_next
2821 |
2822 |3: // Check for __newindex if previous value is nil.
2823 | lwz TAB:TMP2, TAB:RB->metatable
2824 | cmplwi TAB:TMP2, 0
2825 | beq <1 // No metatable: done.
2826 | lbz TMP2, TAB:TMP2->nomm
2827 | andi. TMP2, TMP2, 1<<MM_newindex
2828 | bne <1 // 'no __newindex' flag set: done.
2829 | b ->vmeta_tsetv
2830 |
2831 |5:
2832 | checkstr STR:RC // String key?
2833 | checkok ->BC_TSETS_Z
2834 | b ->vmeta_tsetv
2835 |
2836 |7: // Possible table write barrier for the value. Skip valiswhite check.
2837 | barrierback TAB:RB, TMP3, TMP0
2838 | b <2
2839 break;
2840 case BC_TSETS:
2841 | // RA = src*8, RB = table*8, RC = str_const*8 (~)
2842 | evlddx TAB:RB, BASE, RB
2843 | srwi TMP1, RC, 1
2844 | checktab TAB:RB
2845 | subfic TMP1, TMP1, -4
2846 | lwzx STR:RC, KBASE, TMP1 // KBASE-4-str_const*4
2847 | checkfail ->vmeta_tsets1
2848 |->BC_TSETS_Z:
2849 | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8
2850 | lwz TMP0, TAB:RB->hmask
2851 | lwz TMP1, STR:RC->hash
2852 | lwz NODE:TMP2, TAB:RB->node
2853 | evmergelo STR:RC, TISSTR, STR:RC
2854 | stb ZERO, TAB:RB->nomm // Clear metamethod cache.
2855 | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask
2856 | evlddx SAVE0, BASE, RA
2857 | slwi TMP0, TMP1, 5
2858 | slwi TMP1, TMP1, 3
2859 | sub TMP1, TMP0, TMP1
2860 | lbz TMP3, TAB:RB->marked
2861 | add NODE:TMP2, NODE:TMP2, TMP1 // node = tab->node + (idx*32-idx*8)
2862 |1:
2863 | evldd TMP0, NODE:TMP2->key
2864 | evldd TMP1, NODE:TMP2->val
2865 | evcmpeq TMP0, STR:RC
2866 | checkanyfail >5
2867 | checknil TMP1
2868 | checkok >4 // Key found, but nil value?
2869 |2:
2870 | andi. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
2871 | evstdd SAVE0, NODE:TMP2->val
2872 | bne >7
2873 |3:
2874 | ins_next
2875 |
2876 |4: // Check for __newindex if previous value is nil.
2877 | lwz TAB:TMP1, TAB:RB->metatable
2878 | cmplwi TAB:TMP1, 0
2879 | beq <2 // No metatable: done.
2880 | lbz TMP0, TAB:TMP1->nomm
2881 | andi. TMP0, TMP0, 1<<MM_newindex
2882 | bne <2 // 'no __newindex' flag set: done.
2883 | b ->vmeta_tsets
2884 |
2885 |5: // Follow hash chain.
2886 | lwz NODE:TMP2, NODE:TMP2->next
2887 | cmplwi NODE:TMP2, 0
2888 | bne <1
2889 | // End of hash chain: key not found, add a new one.
2890 |
2891 | // But check for __newindex first.
2892 | lwz TAB:TMP1, TAB:RB->metatable
2893 | la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
2894 | stw PC, SAVE_PC
2895 | mr CARG1, L
2896 | cmplwi TAB:TMP1, 0
2897 | stw BASE, L->base
2898 | beq >6 // No metatable: continue.
2899 | lbz TMP0, TAB:TMP1->nomm
2900 | andi. TMP0, TMP0, 1<<MM_newindex
2901 | beq ->vmeta_tsets // 'no __newindex' flag NOT set: check.
2902 |6:
2903 | mr CARG2, TAB:RB
2904 | evstdd STR:RC, 0(CARG3)
2905 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
2906 | // Returns TValue *.
2907 | lwz BASE, L->base
2908 | evstdd SAVE0, 0(CRET1)
2909 | b <3 // No 2nd write barrier needed.
2910 |
2911 |7: // Possible table write barrier for the value. Skip valiswhite check.
2912 | barrierback TAB:RB, TMP3, TMP0
2913 | b <3
2914 break;
2915 case BC_TSETB:
2916 | // RA = src*8, RB = table*8, RC = index*8
2917 | evlddx TAB:RB, BASE, RB
2918 | srwi TMP0, RC, 3
2919 | checktab TAB:RB
2920 | checkfail ->vmeta_tsetb
2921 | lwz TMP1, TAB:RB->asize
2922 | lwz TMP2, TAB:RB->array
2923 | lbz TMP3, TAB:RB->marked
2924 | cmplw TMP0, TMP1
2925 | evlddx SAVE0, BASE, RA
2926 | bge ->vmeta_tsetb
2927 | evlddx TMP1, TMP2, RC
2928 | checknil TMP1
2929 | checkok >5
2930 |1:
2931 | andi. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
2932 | evstddx SAVE0, TMP2, RC
2933 | bne >7
2934 |2:
2935 | ins_next
2936 |
2937 |5: // Check for __newindex if previous value is nil.
2938 | lwz TAB:TMP1, TAB:RB->metatable
2939 | cmplwi TAB:TMP1, 0
2940 | beq <1 // No metatable: done.
2941 | lbz TMP1, TAB:TMP1->nomm
2942 | andi. TMP1, TMP1, 1<<MM_newindex
2943 | bne <1 // 'no __newindex' flag set: done.
2944 | b ->vmeta_tsetb // Caveat: preserve TMP0!
2945 |
2946 |7: // Possible table write barrier for the value. Skip valiswhite check.
2947 | barrierback TAB:RB, TMP3, TMP0
2948 | b <2
2949 break;
2950
2951 case BC_TSETM:
2952 | // RA = base*8 (table at base-1), RD = num_const*8 (start index)
2953 | add RA, BASE, RA
2954 |1:
2955 | add TMP3, KBASE, RD
2956 | lwz TAB:CARG2, -4(RA) // Guaranteed to be a table.
2957 | addic. TMP0, MULTRES, -8
2958 | lwz TMP3, 4(TMP3) // Integer constant is in lo-word.
2959 | srwi CARG3, TMP0, 3
2960 | beq >4 // Nothing to copy?
2961 | add CARG3, CARG3, TMP3
2962 | lwz TMP2, TAB:CARG2->asize
2963 | slwi TMP1, TMP3, 3
2964 | lbz TMP3, TAB:CARG2->marked
2965 | cmplw CARG3, TMP2
2966 | add TMP2, RA, TMP0
2967 | lwz TMP0, TAB:CARG2->array
2968 | bgt >5
2969 | add TMP1, TMP1, TMP0
2970 | andi. TMP0, TMP3, LJ_GC_BLACK // isblack(table)
2971 |3: // Copy result slots to table.
2972 | evldd TMP0, 0(RA)
2973 | addi RA, RA, 8
2974 | cmpw cr1, RA, TMP2
2975 | evstdd TMP0, 0(TMP1)
2976 | addi TMP1, TMP1, 8
2977 | blt cr1, <3
2978 | bne >7
2979 |4:
2980 | ins_next
2981 |
2982 |5: // Need to resize array part.
2983 | stw BASE, L->base
2984 | mr CARG1, L
2985 | stw PC, SAVE_PC
2986 | mr SAVE0, RD
2987 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
2988 | // Must not reallocate the stack.
2989 | mr RD, SAVE0
2990 | b <1
2991 |
2992 |7: // Possible table write barrier for any value. Skip valiswhite check.
2993 | barrierback TAB:CARG2, TMP3, TMP0
2994 | b <4
2995 break;
2996
2997 /* -- Calls and vararg handling ----------------------------------------- */
2998
2999 case BC_CALLM:
3000 | // RA = base*8, (RB = (nresults+1)*8,) RC = extra_nargs*8
3001 | add NARGS8:RC, NARGS8:RC, MULTRES
3002 | // Fall through. Assumes BC_CALL follows.
3003 break;
3004 case BC_CALL:
3005 | // RA = base*8, (RB = (nresults+1)*8,) RC = (nargs+1)*8
3006 | evlddx LFUNC:RB, BASE, RA
3007 | mr TMP2, BASE
3008 | add BASE, BASE, RA
3009 | subi NARGS8:RC, NARGS8:RC, 8
3010 | checkfunc LFUNC:RB
3011 | addi BASE, BASE, 8
3012 | checkfail ->vmeta_call
3013 | ins_call
3014 break;
3015
3016 case BC_CALLMT:
3017 | // RA = base*8, (RB = 0,) RC = extra_nargs*8
3018 | add NARGS8:RC, NARGS8:RC, MULTRES
3019 | // Fall through. Assumes BC_CALLT follows.
3020 break;
3021 case BC_CALLT:
3022 | // RA = base*8, (RB = 0,) RC = (nargs+1)*8
3023 | evlddx LFUNC:RB, BASE, RA
3024 | add RA, BASE, RA
3025 | lwz TMP1, FRAME_PC(BASE)
3026 | subi NARGS8:RC, NARGS8:RC, 8
3027 | checkfunc LFUNC:RB
3028 | addi RA, RA, 8
3029 | checkfail ->vmeta_callt
3030 |->BC_CALLT_Z:
3031 | andi. TMP0, TMP1, FRAME_TYPE // Caveat: preserve cr0 until the crand.
3032 | lbz TMP3, LFUNC:RB->ffid
3033 | xori TMP2, TMP1, FRAME_VARG
3034 | cmplwi cr1, NARGS8:RC, 0
3035 | bne >7
3036 |1:
3037 | stw LFUNC:RB, FRAME_FUNC(BASE) // Copy function down, but keep PC.
3038 | li TMP2, 0
3039 | cmplwi cr7, TMP3, 1 // (> FF_C) Calling a fast function?
3040 | beq cr1, >3
3041 |2:
3042 | addi TMP3, TMP2, 8
3043 | evlddx TMP0, RA, TMP2
3044 | cmplw cr1, TMP3, NARGS8:RC
3045 | evstddx TMP0, BASE, TMP2
3046 | mr TMP2, TMP3
3047 | bne cr1, <2
3048 |3:
3049 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+gt
3050 | beq >5
3051 |4:
3052 | ins_callt
3053 |
3054 |5: // Tailcall to a fast function with a Lua frame below.
3055 | lwz INS, -4(TMP1)
3056 | decode_RA8 RA, INS
3057 | sub TMP1, BASE, RA
3058 | lwz LFUNC:TMP1, FRAME_FUNC-8(TMP1)
3059 | lwz TMP1, LFUNC:TMP1->pc
3060 | lwz KBASE, PC2PROTO(k)(TMP1) // Need to prepare KBASE.
3061 | b <4
3062 |
3063 |7: // Tailcall from a vararg function.
3064 | andi. TMP0, TMP2, FRAME_TYPEP
3065 | bne <1 // Vararg frame below?
3066 | sub BASE, BASE, TMP2 // Relocate BASE down.
3067 | lwz TMP1, FRAME_PC(BASE)
3068 | andi. TMP0, TMP1, FRAME_TYPE
3069 | b <1
3070 break;
3071
3072 case BC_ITERC:
3073 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 ((2+1)*8))
3074 | subi RA, RA, 24 // evldd doesn't support neg. offsets.
3075 | mr TMP2, BASE
3076 | evlddx LFUNC:RB, BASE, RA
3077 | add BASE, BASE, RA
3078 | evldd TMP0, 8(BASE)
3079 | evldd TMP1, 16(BASE)
3080 | evstdd LFUNC:RB, 24(BASE) // Copy callable.
3081 | checkfunc LFUNC:RB
3082 | evstdd TMP0, 32(BASE) // Copy state.
3083 | li NARGS8:RC, 16 // Iterators get 2 arguments.
3084 | evstdd TMP1, 40(BASE) // Copy control var.
3085 | addi BASE, BASE, 32
3086 | checkfail ->vmeta_call
3087 | ins_call
3088 break;
3089
3090 case BC_ITERN:
3091 | // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8)
3092 |.if JIT
3093 | // NYI: add hotloop, record BC_ITERN.
3094 |.endif
3095 | add RA, BASE, RA
3096 | lwz TAB:RB, -12(RA)
3097 | lwz RC, -4(RA) // Get index from control var.
3098 | lwz TMP0, TAB:RB->asize
3099 | lwz TMP1, TAB:RB->array
3100 | addi PC, PC, 4
3101 |1: // Traverse array part.
3102 | cmplw RC, TMP0
3103 | slwi TMP3, RC, 3
3104 | bge >5 // Index points after array part?
3105 | evlddx TMP2, TMP1, TMP3
3106 | checknil TMP2
3107 | lwz INS, -4(PC)
3108 | checkok >4
3109 | efdcfsi TMP0, RC
3110 | addi RC, RC, 1
3111 | addis TMP3, PC, -(BCBIAS_J*4 >> 16)
3112 | evstdd TMP2, 8(RA)
3113 | decode_RD4 TMP1, INS
3114 | stw RC, -4(RA) // Update control var.
3115 | add PC, TMP1, TMP3
3116 | evstdd TMP0, 0(RA)
3117 |3:
3118 | ins_next
3119 |
3120 |4: // Skip holes in array part.
3121 | addi RC, RC, 1
3122 | b <1
3123 |
3124 |5: // Traverse hash part.
3125 | lwz TMP1, TAB:RB->hmask
3126 | sub RC, RC, TMP0
3127 | lwz TMP2, TAB:RB->node
3128 |6:
3129 | cmplw RC, TMP1 // End of iteration? Branch to ITERL+1.
3130 | slwi TMP3, RC, 5
3131 | bgt <3
3132 | slwi RB, RC, 3
3133 | sub TMP3, TMP3, RB
3134 | evlddx RB, TMP2, TMP3
3135 | add NODE:TMP3, TMP2, TMP3
3136 | checknil RB
3137 | lwz INS, -4(PC)
3138 | checkok >7
3139 | evldd TMP3, NODE:TMP3->key
3140 | addis TMP2, PC, -(BCBIAS_J*4 >> 16)
3141 | evstdd RB, 8(RA)
3142 | add RC, RC, TMP0
3143 | decode_RD4 TMP1, INS
3144 | evstdd TMP3, 0(RA)
3145 | addi RC, RC, 1
3146 | add PC, TMP1, TMP2
3147 | stw RC, -4(RA) // Update control var.
3148 | b <3
3149 |
3150 |7: // Skip holes in hash part.
3151 | addi RC, RC, 1
3152 | b <6
3153 break;
3154
3155 case BC_ISNEXT:
3156 | // RA = base*8, RD = target (points to ITERN)
3157 | add RA, BASE, RA
3158 | li TMP2, -24
3159 | evlddx CFUNC:TMP1, RA, TMP2
3160 | lwz TMP2, -16(RA)
3161 | lwz TMP3, -8(RA)
3162 | evmergehi TMP0, CFUNC:TMP1, CFUNC:TMP1
3163 | cmpwi cr0, TMP2, LJ_TTAB
3164 | cmpwi cr1, TMP0, LJ_TFUNC
3165 | cmpwi cr6, TMP3, LJ_TNIL
3166 | bne cr1, >5
3167 | lbz TMP1, CFUNC:TMP1->ffid
3168 | crand 4*cr0+eq, 4*cr0+eq, 4*cr6+eq
3169 | cmpwi cr7, TMP1, FF_next_N
3170 | srwi TMP0, RD, 1
3171 | crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq
3172 | add TMP3, PC, TMP0
3173 | bne cr0, >5
3174 | lus TMP1, 0xfffe
3175 | ori TMP1, TMP1, 0x7fff
3176 | stw ZERO, -4(RA) // Initialize control var.
3177 | stw TMP1, -8(RA)
3178 | addis PC, TMP3, -(BCBIAS_J*4 >> 16)
3179 |1:
3180 | ins_next
3181 |5: // Despecialize bytecode if any of the checks fail.
3182 | li TMP0, BC_JMP
3183 | li TMP1, BC_ITERC
3184 | stb TMP0, -1(PC)
3185 | addis PC, TMP3, -(BCBIAS_J*4 >> 16)
3186 | stb TMP1, 3(PC)
3187 | b <1
3188 break;
3189
3190 case BC_VARG:
3191 | // RA = base*8, RB = (nresults+1)*8, RC = numparams*8
3192 | lwz TMP0, FRAME_PC(BASE)
3193 | add RC, BASE, RC
3194 | add RA, BASE, RA
3195 | addi RC, RC, FRAME_VARG
3196 | add TMP2, RA, RB
3197 | subi TMP3, BASE, 8 // TMP3 = vtop
3198 | sub RC, RC, TMP0 // RC = vbase
3199 | // Note: RC may now be even _above_ BASE if nargs was < numparams.
3200 | cmplwi cr1, RB, 0
3201 | sub. TMP1, TMP3, RC
3202 | beq cr1, >5 // Copy all varargs?
3203 | subi TMP2, TMP2, 16
3204 | ble >2 // No vararg slots?
3205 |1: // Copy vararg slots to destination slots.
3206 | evldd TMP0, 0(RC)
3207 | addi RC, RC, 8
3208 | evstdd TMP0, 0(RA)
3209 | cmplw RA, TMP2
3210 | cmplw cr1, RC, TMP3
3211 | bge >3 // All destination slots filled?
3212 | addi RA, RA, 8
3213 | blt cr1, <1 // More vararg slots?
3214 |2: // Fill up remainder with nil.
3215 | evstdd TISNIL, 0(RA)
3216 | cmplw RA, TMP2
3217 | addi RA, RA, 8
3218 | blt <2
3219 |3:
3220 | ins_next
3221 |
3222 |5: // Copy all varargs.
3223 | lwz TMP0, L->maxstack
3224 | li MULTRES, 8 // MULTRES = (0+1)*8
3225 | ble <3 // No vararg slots?
3226 | add TMP2, RA, TMP1
3227 | cmplw TMP2, TMP0
3228 | addi MULTRES, TMP1, 8
3229 | bgt >7
3230 |6:
3231 | evldd TMP0, 0(RC)
3232 | addi RC, RC, 8
3233 | evstdd TMP0, 0(RA)
3234 | cmplw RC, TMP3
3235 | addi RA, RA, 8
3236 | blt <6 // More vararg slots?
3237 | b <3
3238 |
3239 |7: // Grow stack for varargs.
3240 | mr CARG1, L
3241 | stw RA, L->top
3242 | sub SAVE0, RC, BASE // Need delta, because BASE may change.
3243 | stw BASE, L->base
3244 | sub RA, RA, BASE
3245 | stw PC, SAVE_PC
3246 | srwi CARG2, TMP1, 3
3247 | bl extern lj_state_growstack // (lua_State *L, int n)
3248 | lwz BASE, L->base
3249 | add RA, BASE, RA
3250 | add RC, BASE, SAVE0
3251 | subi TMP3, BASE, 8
3252 | b <6
3253 break;
3254
3255 /* -- Returns ----------------------------------------------------------- */
3256
3257 case BC_RETM:
3258 | // RA = results*8, RD = extra_nresults*8
3259 | add RD, RD, MULTRES // MULTRES >= 8, so RD >= 8.
3260 | // Fall through. Assumes BC_RET follows.
3261 break;
3262
3263 case BC_RET:
3264 | // RA = results*8, RD = (nresults+1)*8
3265 | lwz PC, FRAME_PC(BASE)
3266 | add RA, BASE, RA
3267 | mr MULTRES, RD
3268 |1:
3269 | andi. TMP0, PC, FRAME_TYPE
3270 | xori TMP1, PC, FRAME_VARG
3271 | bne ->BC_RETV_Z
3272 |
3273 |->BC_RET_Z:
3274 | // BASE = base, RA = resultptr, RD = (nresults+1)*8, PC = return
3275 | lwz INS, -4(PC)
3276 | cmpwi RD, 8
3277 | subi TMP2, BASE, 8
3278 | subi RC, RD, 8
3279 | decode_RB8 RB, INS
3280 | beq >3
3281 | li TMP1, 0
3282 |2:
3283 | addi TMP3, TMP1, 8
3284 | evlddx TMP0, RA, TMP1
3285 | cmpw TMP3, RC
3286 | evstddx TMP0, TMP2, TMP1
3287 | beq >3
3288 | addi TMP1, TMP3, 8
3289 | evlddx TMP0, RA, TMP3
3290 | cmpw TMP1, RC
3291 | evstddx TMP0, TMP2, TMP3
3292 | bne <2
3293 |3:
3294 |5:
3295 | cmplw RB, RD
3296 | decode_RA8 RA, INS
3297 | bgt >6
3298 | sub BASE, TMP2, RA
3299 | lwz LFUNC:TMP1, FRAME_FUNC(BASE)
3300 | ins_next1
3301 | lwz TMP1, LFUNC:TMP1->pc
3302 | lwz KBASE, PC2PROTO(k)(TMP1)
3303 | ins_next2
3304 |
3305 |6: // Fill up results with nil.
3306 | subi TMP1, RD, 8
3307 | addi RD, RD, 8
3308 | evstddx TISNIL, TMP2, TMP1
3309 | b <5
3310 |
3311 |->BC_RETV_Z: // Non-standard return case.
3312 | andi. TMP2, TMP1, FRAME_TYPEP
3313 | bne ->vm_return
3314 | // Return from vararg function: relocate BASE down.
3315 | sub BASE, BASE, TMP1
3316 | lwz PC, FRAME_PC(BASE)
3317 | b <1
3318 break;
3319
3320 case BC_RET0: case BC_RET1:
3321 | // RA = results*8, RD = (nresults+1)*8
3322 | lwz PC, FRAME_PC(BASE)
3323 | add RA, BASE, RA
3324 | mr MULTRES, RD
3325 | andi. TMP0, PC, FRAME_TYPE
3326 | xori TMP1, PC, FRAME_VARG
3327 | bne ->BC_RETV_Z
3328 |
3329 | lwz INS, -4(PC)
3330 | subi TMP2, BASE, 8
3331 | decode_RB8 RB, INS
3332 if (op == BC_RET1) {
3333 | evldd TMP0, 0(RA)
3334 | evstdd TMP0, 0(TMP2)
3335 }
3336 |5:
3337 | cmplw RB, RD
3338 | decode_RA8 RA, INS
3339 | bgt >6
3340 | sub BASE, TMP2, RA
3341 | lwz LFUNC:TMP1, FRAME_FUNC(BASE)
3342 | ins_next1
3343 | lwz TMP1, LFUNC:TMP1->pc
3344 | lwz KBASE, PC2PROTO(k)(TMP1)
3345 | ins_next2
3346 |
3347 |6: // Fill up results with nil.
3348 | subi TMP1, RD, 8
3349 | addi RD, RD, 8
3350 | evstddx TISNIL, TMP2, TMP1
3351 | b <5
3352 break;
3353
3354 /* -- Loops and branches ------------------------------------------------ */
3355
3356 case BC_FORL:
3357 |.if JIT
3358 | hotloop
3359 |.endif
3360 | // Fall through. Assumes BC_IFORL follows.
3361 break;
3362
3363 case BC_JFORI:
3364 case BC_JFORL:
3365#if !LJ_HASJIT
3366 break;
3367#endif
3368 case BC_FORI:
3369 case BC_IFORL:
3370 | // RA = base*8, RD = target (after end of loop or start of loop)
3371 vk = (op == BC_IFORL || op == BC_JFORL);
3372 | add RA, BASE, RA
3373 | evldd TMP1, FORL_IDX*8(RA)
3374 | evldd TMP3, FORL_STEP*8(RA)
3375 | evldd TMP2, FORL_STOP*8(RA)
3376 if (!vk) {
3377 | evcmpgtu cr0, TMP1, TISNUM
3378 | evcmpgtu cr7, TMP3, TISNUM
3379 | evcmpgtu cr1, TMP2, TISNUM
3380 | cror 4*cr0+lt, 4*cr0+lt, 4*cr7+lt
3381 | cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
3382 | blt ->vmeta_for
3383 }
3384 if (vk) {
3385 | efdadd TMP1, TMP1, TMP3
3386 | evstdd TMP1, FORL_IDX*8(RA)
3387 }
3388 | evcmpgts TMP3, TISNIL
3389 | evstdd TMP1, FORL_EXT*8(RA)
3390 | bge >2
3391 | efdcmpgt TMP1, TMP2
3392 |1:
3393 if (op != BC_JFORL) {
3394 | srwi RD, RD, 1
3395 | add RD, PC, RD
3396 if (op == BC_JFORI) {
3397 | addis PC, RD, -(BCBIAS_J*4 >> 16)
3398 } else {
3399 | addis RD, RD, -(BCBIAS_J*4 >> 16)
3400 }
3401 }
3402 if (op == BC_FORI) {
3403 | iselgt PC, RD, PC
3404 } else if (op == BC_IFORL) {
3405 | iselgt PC, PC, RD
3406 } else {
3407 | ble =>BC_JLOOP
3408 }
3409 | ins_next
3410 |2:
3411 | efdcmpgt TMP2, TMP1
3412 | b <1
3413 break;
3414
3415 case BC_ITERL:
3416 |.if JIT
3417 | hotloop
3418 |.endif
3419 | // Fall through. Assumes BC_IITERL follows.
3420 break;
3421
3422 case BC_JITERL:
3423#if !LJ_HASJIT
3424 break;
3425#endif
3426 case BC_IITERL:
3427 | // RA = base*8, RD = target
3428 | evlddx TMP1, BASE, RA
3429 | subi RA, RA, 8
3430 | checknil TMP1
3431 | checkok >1 // Stop if iterator returned nil.
3432 if (op == BC_JITERL) {
3433 | NYI
3434 } else {
3435 | branch_RD // Otherwise save control var + branch.
3436 | evstddx TMP1, BASE, RA
3437 }
3438 |1:
3439 | ins_next
3440 break;
3441
3442 case BC_LOOP:
3443 | // RA = base*8, RD = target (loop extent)
3444 | // Note: RA/RD is only used by trace recorder to determine scope/extent
3445 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3446 |.if JIT
3447 | hotloop
3448 |.endif
3449 | // Fall through. Assumes BC_ILOOP follows.
3450 break;
3451
3452 case BC_ILOOP:
3453 | // RA = base*8, RD = target (loop extent)
3454 | ins_next
3455 break;
3456
3457 case BC_JLOOP:
3458 |.if JIT
3459 | NYI
3460 |.endif
3461 break;
3462
3463 case BC_JMP:
3464 | // RA = base*8 (only used by trace recorder), RD = target
3465 | branch_RD
3466 | ins_next
3467 break;
3468
3469 /* -- Function headers -------------------------------------------------- */
3470
3471 case BC_FUNCF:
3472 |.if JIT
3473 | hotcall
3474 |.endif
3475 case BC_FUNCV: /* NYI: compiled vararg functions. */
3476 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3477 break;
3478
3479 case BC_JFUNCF:
3480#if !LJ_HASJIT
3481 break;
3482#endif
3483 case BC_IFUNCF:
3484 | // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
3485 | lwz TMP2, L->maxstack
3486 | lbz TMP1, -4+PC2PROTO(numparams)(PC)
3487 | lwz KBASE, -4+PC2PROTO(k)(PC)
3488 | cmplw RA, TMP2
3489 | slwi TMP1, TMP1, 3
3490 | bgt ->vm_growstack_l
3491 | ins_next1
3492 |2:
3493 | cmplw NARGS8:RC, TMP1 // Check for missing parameters.
3494 | ble >3
3495 if (op == BC_JFUNCF) {
3496 | NYI
3497 } else {
3498 | ins_next2
3499 }
3500 |
3501 |3: // Clear missing parameters.
3502 | evstddx TISNIL, BASE, NARGS8:RC
3503 | addi NARGS8:RC, NARGS8:RC, 8
3504 | b <2
3505 break;
3506
3507 case BC_JFUNCV:
3508#if !LJ_HASJIT
3509 break;
3510#endif
3511 | NYI // NYI: compiled vararg functions
3512 break; /* NYI: compiled vararg functions. */
3513
3514 case BC_IFUNCV:
3515 | // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
3516 | lwz TMP2, L->maxstack
3517 | add TMP1, BASE, RC
3518 | add TMP0, RA, RC
3519 | stw LFUNC:RB, 4(TMP1) // Store copy of LFUNC.
3520 | addi TMP3, RC, 8+FRAME_VARG
3521 | lwz KBASE, -4+PC2PROTO(k)(PC)
3522 | cmplw TMP0, TMP2
3523 | stw TMP3, 0(TMP1) // Store delta + FRAME_VARG.
3524 | bge ->vm_growstack_l
3525 | lbz TMP2, -4+PC2PROTO(numparams)(PC)
3526 | mr RA, BASE
3527 | mr RC, TMP1
3528 | ins_next1
3529 | cmpwi TMP2, 0
3530 | addi BASE, TMP1, 8
3531 | beq >3
3532 |1:
3533 | cmplw RA, RC // Less args than parameters?
3534 | evldd TMP0, 0(RA)
3535 | bge >4
3536 | evstdd TISNIL, 0(RA) // Clear old fixarg slot (help the GC).
3537 | addi RA, RA, 8
3538 |2:
3539 | addic. TMP2, TMP2, -1
3540 | evstdd TMP0, 8(TMP1)
3541 | addi TMP1, TMP1, 8
3542 | bne <1
3543 |3:
3544 | ins_next2
3545 |
3546 |4: // Clear missing parameters.
3547 | evmr TMP0, TISNIL
3548 | b <2
3549 break;
3550
3551 case BC_FUNCC:
3552 case BC_FUNCCW:
3553 | // BASE = new base, RA = BASE+framesize*8, RB = CFUNC, RC = nargs*8
3554 if (op == BC_FUNCC) {
3555 | lwz TMP3, CFUNC:RB->f
3556 } else {
3557 | lwz TMP3, DISPATCH_GL(wrapf)(DISPATCH)
3558 }
3559 | add TMP1, RA, NARGS8:RC
3560 | lwz TMP2, L->maxstack
3561 | add RC, BASE, NARGS8:RC
3562 | stw BASE, L->base
3563 | cmplw TMP1, TMP2
3564 | stw RC, L->top
3565 | li_vmstate C
3566 | mtctr TMP3
3567 if (op == BC_FUNCCW) {
3568 | lwz CARG2, CFUNC:RB->f
3569 }
3570 | mr CARG1, L
3571 | bgt ->vm_growstack_c // Need to grow stack.
3572 | st_vmstate
3573 | bctrl // (lua_State *L [, lua_CFunction f])
3574 | // Returns nresults.
3575 | lwz TMP1, L->top
3576 | slwi RD, CRET1, 3
3577 | lwz BASE, L->base
3578 | li_vmstate INTERP
3579 | lwz PC, FRAME_PC(BASE) // Fetch PC of caller.
3580 | sub RA, TMP1, RD // RA = L->top - nresults*8
3581 | st_vmstate
3582 | b ->vm_returnc
3583 break;
3584
3585 /* ---------------------------------------------------------------------- */
3586
3587 default:
3588 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3589 exit(2);
3590 break;
3591 }
3592}
3593
3594static int build_backend(BuildCtx *ctx)
3595{
3596 int op;
3597
3598 dasm_growpc(Dst, BC__MAX);
3599
3600 build_subroutines(ctx);
3601
3602 |.code_op
3603 for (op = 0; op < BC__MAX; op++)
3604 build_ins(ctx, (BCOp)op, op);
3605
3606 return BC__MAX;
3607}
3608
3609/* Emit pseudo frame-info for all assembler functions. */
3610static void emit_asm_debug(BuildCtx *ctx)
3611{
3612 int i;
3613 switch (ctx->mode) {
3614 case BUILD_elfasm:
3615 fprintf(ctx->fp, "\t.section .debug_frame,\"\",@progbits\n");
3616 fprintf(ctx->fp,
3617 ".Lframe0:\n"
3618 "\t.long .LECIE0-.LSCIE0\n"
3619 ".LSCIE0:\n"
3620 "\t.long 0xffffffff\n"
3621 "\t.byte 0x1\n"
3622 "\t.string \"\"\n"
3623 "\t.uleb128 0x1\n"
3624 "\t.sleb128 -4\n"
3625 "\t.byte 65\n"
3626 "\t.byte 0xc\n\t.uleb128 1\n\t.uleb128 0\n"
3627 "\t.align 2\n"
3628 ".LECIE0:\n\n");
3629 fprintf(ctx->fp,
3630 ".LSFDE0:\n"
3631 "\t.long .LEFDE0-.LASFDE0\n"
3632 ".LASFDE0:\n"
3633 "\t.long .Lframe0\n"
3634 "\t.long .Lbegin\n"
3635 "\t.long %d\n"
3636 "\t.byte 0xe\n\t.uleb128 %d\n"
3637 "\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
3638 "\t.byte 0x5\n\t.uleb128 70\n\t.sleb128 37\n",
3639 (int)ctx->codesz, CFRAME_SIZE);
3640 for (i = 14; i <= 31; i++)
3641 fprintf(ctx->fp,
3642 "\t.byte %d\n\t.uleb128 %d\n"
3643 "\t.byte 5\n\t.uleb128 %d\n\t.uleb128 %d\n",
3644 0x80+i, 1+2*(31-i), 1200+i, 2+2*(31-i));
3645 fprintf(ctx->fp,
3646 "\t.align 2\n"
3647 ".LEFDE0:\n\n");
3648 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@progbits\n");
3649 fprintf(ctx->fp,
3650 ".Lframe1:\n"
3651 "\t.long .LECIE1-.LSCIE1\n"
3652 ".LSCIE1:\n"
3653 "\t.long 0\n"
3654 "\t.byte 0x1\n"
3655 "\t.string \"zPR\"\n"
3656 "\t.uleb128 0x1\n"
3657 "\t.sleb128 -4\n"
3658 "\t.byte 65\n"
3659 "\t.uleb128 6\n" /* augmentation length */
3660 "\t.byte 0x1b\n" /* pcrel|sdata4 */
3661 "\t.long lj_err_unwind_dwarf-.\n"
3662 "\t.byte 0x1b\n" /* pcrel|sdata4 */
3663 "\t.byte 0xc\n\t.uleb128 1\n\t.uleb128 0\n"
3664 "\t.align 2\n"
3665 ".LECIE1:\n\n");
3666 fprintf(ctx->fp,
3667 ".LSFDE1:\n"
3668 "\t.long .LEFDE1-.LASFDE1\n"
3669 ".LASFDE1:\n"
3670 "\t.long .LASFDE1-.Lframe1\n"
3671 "\t.long .Lbegin-.\n"
3672 "\t.long %d\n"
3673 "\t.uleb128 0\n" /* augmentation length */
3674 "\t.byte 0xe\n\t.uleb128 %d\n"
3675 "\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
3676 "\t.byte 0x5\n\t.uleb128 70\n\t.sleb128 37\n",
3677 (int)ctx->codesz, CFRAME_SIZE);
3678 for (i = 14; i <= 31; i++)
3679 fprintf(ctx->fp,
3680 "\t.byte %d\n\t.uleb128 %d\n"
3681 "\t.byte 5\n\t.uleb128 %d\n\t.uleb128 %d\n",
3682 0x80+i, 1+2*(31-i), 1200+i, 2+2*(31-i));
3683 fprintf(ctx->fp,
3684 "\t.align 2\n"
3685 ".LEFDE1:\n\n");
3686 break;
3687 default:
3688 break;
3689 }
3690}
3691
diff --git a/src/vm_x64.dasc b/src/vm_x64.dasc
new file mode 100644
index 00000000..a8649b4e
--- /dev/null
+++ b/src/vm_x64.dasc
@@ -0,0 +1,4946 @@
1|// Low-level VM code for x64 CPUs in LJ_GC64 mode.
2|// Bytecode interpreter, fast functions and helper functions.
3|// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4|
5|.arch x64
6|.section code_op, code_sub
7|
8|.actionlist build_actionlist
9|.globals GLOB_
10|.globalnames globnames
11|.externnames extnames
12|
13|//-----------------------------------------------------------------------
14|
15|.if WIN
16|.define X64WIN, 1 // Windows/x64 calling conventions.
17|.endif
18|
19|// Fixed register assignments for the interpreter.
20|// This is very fragile and has many dependencies. Caveat emptor.
21|.define BASE, rdx // Not C callee-save, refetched anyway.
22|.if X64WIN
23|.define KBASE, rdi // Must be C callee-save.
24|.define PC, rsi // Must be C callee-save.
25|.define DISPATCH, rbx // Must be C callee-save.
26|.define KBASEd, edi
27|.define PCd, esi
28|.define DISPATCHd, ebx
29|.else
30|.define KBASE, r15 // Must be C callee-save.
31|.define PC, rbx // Must be C callee-save.
32|.define DISPATCH, r14 // Must be C callee-save.
33|.define KBASEd, r15d
34|.define PCd, ebx
35|.define DISPATCHd, r14d
36|.endif
37|
38|.define RA, rcx
39|.define RAd, ecx
40|.define RAH, ch
41|.define RAL, cl
42|.define RB, rbp // Must be rbp (C callee-save).
43|.define RBd, ebp
44|.define RC, rax // Must be rax.
45|.define RCd, eax
46|.define RCW, ax
47|.define RCH, ah
48|.define RCL, al
49|.define OP, RBd
50|.define RD, RC
51|.define RDd, RCd
52|.define RDW, RCW
53|.define RDL, RCL
54|.define TMPR, r10
55|.define TMPRd, r10d
56|.define ITYPE, r11
57|.define ITYPEd, r11d
58|
59|.if X64WIN
60|.define CARG1, rcx // x64/WIN64 C call arguments.
61|.define CARG2, rdx
62|.define CARG3, r8
63|.define CARG4, r9
64|.define CARG1d, ecx
65|.define CARG2d, edx
66|.define CARG3d, r8d
67|.define CARG4d, r9d
68|.else
69|.define CARG1, rdi // x64/POSIX C call arguments.
70|.define CARG2, rsi
71|.define CARG3, rdx
72|.define CARG4, rcx
73|.define CARG5, r8
74|.define CARG6, r9
75|.define CARG1d, edi
76|.define CARG2d, esi
77|.define CARG3d, edx
78|.define CARG4d, ecx
79|.define CARG5d, r8d
80|.define CARG6d, r9d
81|.endif
82|
83|// Type definitions. Some of these are only used for documentation.
84|.type L, lua_State
85|.type GL, global_State
86|.type TVALUE, TValue
87|.type GCOBJ, GCobj
88|.type STR, GCstr
89|.type TAB, GCtab
90|.type LFUNC, GCfuncL
91|.type CFUNC, GCfuncC
92|.type PROTO, GCproto
93|.type UPVAL, GCupval
94|.type NODE, Node
95|.type NARGS, int
96|.type TRACE, GCtrace
97|.type SBUF, SBuf
98|
99|// Stack layout while in interpreter. Must match with lj_frame.h.
100|//-----------------------------------------------------------------------
101|.if X64WIN // x64/Windows stack layout
102|
103|.define CFRAME_SPACE, aword*5 // Delta for rsp (see <--).
104|.macro saveregs_
105| push rdi; push rsi; push rbx
106| sub rsp, CFRAME_SPACE
107|.endmacro
108|.macro saveregs
109| push rbp; saveregs_
110|.endmacro
111|.macro restoreregs
112| add rsp, CFRAME_SPACE
113| pop rbx; pop rsi; pop rdi; pop rbp
114|.endmacro
115|
116|.define SAVE_CFRAME, aword [rsp+aword*13]
117|.define SAVE_PC, aword [rsp+aword*12]
118|.define SAVE_L, aword [rsp+aword*11]
119|.define SAVE_ERRF, dword [rsp+dword*21]
120|.define SAVE_NRES, dword [rsp+dword*20]
121|//----- 16 byte aligned, ^^^ 32 byte register save area, owned by interpreter
122|.define SAVE_RET, aword [rsp+aword*9] //<-- rsp entering interpreter.
123|.define SAVE_R4, aword [rsp+aword*8]
124|.define SAVE_R3, aword [rsp+aword*7]
125|.define SAVE_R2, aword [rsp+aword*6]
126|.define SAVE_R1, aword [rsp+aword*5] //<-- rsp after register saves.
127|.define ARG5, aword [rsp+aword*4]
128|.define CSAVE_4, aword [rsp+aword*3]
129|.define CSAVE_3, aword [rsp+aword*2]
130|.define CSAVE_2, aword [rsp+aword*1]
131|.define CSAVE_1, aword [rsp] //<-- rsp while in interpreter.
132|//----- 16 byte aligned, ^^^ 32 byte register save area, owned by callee
133|
134|.define ARG5d, dword [rsp+dword*8]
135|.define TMP1, ARG5 // TMP1 overlaps ARG5
136|.define TMP1d, ARG5d
137|.define TMP1hi, dword [rsp+dword*9]
138|.define MULTRES, TMP1d // MULTRES overlaps TMP1d.
139|
140|//-----------------------------------------------------------------------
141|.else // x64/POSIX stack layout
142|
143|.define CFRAME_SPACE, aword*5 // Delta for rsp (see <--).
144|.macro saveregs_
145| push rbx; push r15; push r14
146|.if NO_UNWIND
147| push r13; push r12
148|.endif
149| sub rsp, CFRAME_SPACE
150|.endmacro
151|.macro saveregs
152| push rbp; saveregs_
153|.endmacro
154|.macro restoreregs
155| add rsp, CFRAME_SPACE
156|.if NO_UNWIND
157| pop r12; pop r13
158|.endif
159| pop r14; pop r15; pop rbx; pop rbp
160|.endmacro
161|
162|//----- 16 byte aligned,
163|.if NO_UNWIND
164|.define SAVE_RET, aword [rsp+aword*11] //<-- rsp entering interpreter.
165|.define SAVE_R4, aword [rsp+aword*10]
166|.define SAVE_R3, aword [rsp+aword*9]
167|.define SAVE_R2, aword [rsp+aword*8]
168|.define SAVE_R1, aword [rsp+aword*7]
169|.define SAVE_RU2, aword [rsp+aword*6]
170|.define SAVE_RU1, aword [rsp+aword*5] //<-- rsp after register saves.
171|.else
172|.define SAVE_RET, aword [rsp+aword*9] //<-- rsp entering interpreter.
173|.define SAVE_R4, aword [rsp+aword*8]
174|.define SAVE_R3, aword [rsp+aword*7]
175|.define SAVE_R2, aword [rsp+aword*6]
176|.define SAVE_R1, aword [rsp+aword*5] //<-- rsp after register saves.
177|.endif
178|.define SAVE_CFRAME, aword [rsp+aword*4]
179|.define SAVE_PC, aword [rsp+aword*3]
180|.define SAVE_L, aword [rsp+aword*2]
181|.define SAVE_ERRF, dword [rsp+dword*3]
182|.define SAVE_NRES, dword [rsp+dword*2]
183|.define TMP1, aword [rsp] //<-- rsp while in interpreter.
184|//----- 16 byte aligned
185|
186|.define TMP1d, dword [rsp]
187|.define TMP1hi, dword [rsp+dword*1]
188|.define MULTRES, TMP1d // MULTRES overlaps TMP1d.
189|
190|.endif
191|
192|//-----------------------------------------------------------------------
193|
194|// Instruction headers.
195|.macro ins_A; .endmacro
196|.macro ins_AD; .endmacro
197|.macro ins_AJ; .endmacro
198|.macro ins_ABC; movzx RBd, RCH; movzx RCd, RCL; .endmacro
199|.macro ins_AB_; movzx RBd, RCH; .endmacro
200|.macro ins_A_C; movzx RCd, RCL; .endmacro
201|.macro ins_AND; not RD; .endmacro
202|
203|// Instruction decode+dispatch. Carefully tuned (nope, lodsd is not faster).
204|.macro ins_NEXT
205| mov RCd, [PC]
206| movzx RAd, RCH
207| movzx OP, RCL
208| add PC, 4
209| shr RCd, 16
210| jmp aword [DISPATCH+OP*8]
211|.endmacro
212|
213|// Instruction footer.
214|.if 1
215| // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
216| .define ins_next, ins_NEXT
217| .define ins_next_, ins_NEXT
218|.else
219| // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
220| // Affects only certain kinds of benchmarks (and only with -j off).
221| // Around 10%-30% slower on Core2, a lot more slower on P4.
222| .macro ins_next
223| jmp ->ins_next
224| .endmacro
225| .macro ins_next_
226| ->ins_next:
227| ins_NEXT
228| .endmacro
229|.endif
230|
231|// Call decode and dispatch.
232|.macro ins_callt
233| // BASE = new base, RB = LFUNC, RD = nargs+1, [BASE-8] = PC
234| mov PC, LFUNC:RB->pc
235| mov RAd, [PC]
236| movzx OP, RAL
237| movzx RAd, RAH
238| add PC, 4
239| jmp aword [DISPATCH+OP*8]
240|.endmacro
241|
242|.macro ins_call
243| // BASE = new base, RB = LFUNC, RD = nargs+1
244| mov [BASE-8], PC
245| ins_callt
246|.endmacro
247|
248|//-----------------------------------------------------------------------
249|
250|// Macros to clear or set tags.
251|.macro cleartp, reg; shl reg, 17; shr reg, 17; .endmacro
252|.macro settp, reg, tp
253| mov64 ITYPE, ((uint64_t)tp<<47)
254| or reg, ITYPE
255|.endmacro
256|.macro settp, dst, reg, tp
257| mov64 dst, ((uint64_t)tp<<47)
258| or dst, reg
259|.endmacro
260|.macro setint, reg
261| settp reg, LJ_TISNUM
262|.endmacro
263|.macro setint, dst, reg
264| settp dst, reg, LJ_TISNUM
265|.endmacro
266|
267|// Macros to test operand types.
268|.macro checktp_nc, reg, tp, target
269| mov ITYPE, reg
270| sar ITYPE, 47
271| cmp ITYPEd, tp
272| jne target
273|.endmacro
274|.macro checktp, reg, tp, target
275| mov ITYPE, reg
276| cleartp reg
277| sar ITYPE, 47
278| cmp ITYPEd, tp
279| jne target
280|.endmacro
281|.macro checktptp, src, tp, target
282| mov ITYPE, src
283| sar ITYPE, 47
284| cmp ITYPEd, tp
285| jne target
286|.endmacro
287|.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
288|.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
289|.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
290|
291|.macro checknumx, reg, target, jump
292| mov ITYPE, reg
293| sar ITYPE, 47
294| cmp ITYPEd, LJ_TISNUM
295| jump target
296|.endmacro
297|.macro checkint, reg, target; checknumx reg, target, jne; .endmacro
298|.macro checkinttp, src, target; checknumx src, target, jne; .endmacro
299|.macro checknum, reg, target; checknumx reg, target, jae; .endmacro
300|.macro checknumtp, src, target; checknumx src, target, jae; .endmacro
301|.macro checknumber, src, target; checknumx src, target, ja; .endmacro
302|
303|.macro mov_false, reg; mov64 reg, (int64_t)~((uint64_t)1<<47); .endmacro
304|.macro mov_true, reg; mov64 reg, (int64_t)~((uint64_t)2<<47); .endmacro
305|
306|// These operands must be used with movzx.
307|.define PC_OP, byte [PC-4]
308|.define PC_RA, byte [PC-3]
309|.define PC_RB, byte [PC-1]
310|.define PC_RC, byte [PC-2]
311|.define PC_RD, word [PC-2]
312|
313|.macro branchPC, reg
314| lea PC, [PC+reg*4-BCBIAS_J*4]
315|.endmacro
316|
317|// Assumes DISPATCH is relative to GL.
318#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
319#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
320|
321#define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
322|
323|// Decrement hashed hotcount and trigger trace recorder if zero.
324|.macro hotloop, reg
325| mov reg, PCd
326| shr reg, 1
327| and reg, HOTCOUNT_PCMASK
328| sub word [DISPATCH+reg+GG_DISP2HOT], HOTCOUNT_LOOP
329| jb ->vm_hotloop
330|.endmacro
331|
332|.macro hotcall, reg
333| mov reg, PCd
334| shr reg, 1
335| and reg, HOTCOUNT_PCMASK
336| sub word [DISPATCH+reg+GG_DISP2HOT], HOTCOUNT_CALL
337| jb ->vm_hotcall
338|.endmacro
339|
340|// Set current VM state.
341|.macro set_vmstate, st
342| mov dword [DISPATCH+DISPATCH_GL(vmstate)], ~LJ_VMST_..st
343|.endmacro
344|
345|.macro fpop1; fstp st1; .endmacro
346|
347|// Synthesize SSE FP constants.
348|.macro sseconst_abs, reg, tmp // Synthesize abs mask.
349| mov64 tmp, U64x(7fffffff,ffffffff); movd reg, tmp
350|.endmacro
351|
352|.macro sseconst_hi, reg, tmp, val // Synthesize hi-32 bit const.
353| mov64 tmp, U64x(val,00000000); movd reg, tmp
354|.endmacro
355|
356|.macro sseconst_sign, reg, tmp // Synthesize sign mask.
357| sseconst_hi reg, tmp, 80000000
358|.endmacro
359|.macro sseconst_1, reg, tmp // Synthesize 1.0.
360| sseconst_hi reg, tmp, 3ff00000
361|.endmacro
362|.macro sseconst_2p52, reg, tmp // Synthesize 2^52.
363| sseconst_hi reg, tmp, 43300000
364|.endmacro
365|.macro sseconst_tobit, reg, tmp // Synthesize 2^52 + 2^51.
366| sseconst_hi reg, tmp, 43380000
367|.endmacro
368|
369|// Move table write barrier back. Overwrites reg.
370|.macro barrierback, tab, reg
371| and byte tab->marked, (uint8_t)~LJ_GC_BLACK // black2gray(tab)
372| mov reg, [DISPATCH+DISPATCH_GL(gc.grayagain)]
373| mov [DISPATCH+DISPATCH_GL(gc.grayagain)], tab
374| mov tab->gclist, reg
375|.endmacro
376|
377|//-----------------------------------------------------------------------
378
379/* Generate subroutines used by opcodes and other parts of the VM. */
380/* The .code_sub section should be last to help static branch prediction. */
381static void build_subroutines(BuildCtx *ctx)
382{
383 |.code_sub
384 |
385 |//-----------------------------------------------------------------------
386 |//-- Return handling ----------------------------------------------------
387 |//-----------------------------------------------------------------------
388 |
389 |->vm_returnp:
390 | test PCd, FRAME_P
391 | jz ->cont_dispatch
392 |
393 | // Return from pcall or xpcall fast func.
394 | and PC, -8
395 | sub BASE, PC // Restore caller base.
396 | lea RA, [RA+PC-8] // Rebase RA and prepend one result.
397 | mov PC, [BASE-8] // Fetch PC of previous frame.
398 | // Prepending may overwrite the pcall frame, so do it at the end.
399 | mov_true ITYPE
400 | mov aword [BASE+RA], ITYPE // Prepend true to results.
401 |
402 |->vm_returnc:
403 | add RDd, 1 // RD = nresults+1
404 | jz ->vm_unwind_yield
405 | mov MULTRES, RDd
406 | test PC, FRAME_TYPE
407 | jz ->BC_RET_Z // Handle regular return to Lua.
408 |
409 |->vm_return:
410 | // BASE = base, RA = resultofs, RD = nresults+1 (= MULTRES), PC = return
411 | xor PC, FRAME_C
412 | test PCd, FRAME_TYPE
413 | jnz ->vm_returnp
414 |
415 | // Return to C.
416 | set_vmstate C
417 | and PC, -8
418 | sub PC, BASE
419 | neg PC // Previous base = BASE - delta.
420 |
421 | sub RDd, 1
422 | jz >2
423 |1: // Move results down.
424 | mov RB, [BASE+RA]
425 | mov [BASE-16], RB
426 | add BASE, 8
427 | sub RDd, 1
428 | jnz <1
429 |2:
430 | mov L:RB, SAVE_L
431 | mov L:RB->base, PC
432 |3:
433 | mov RDd, MULTRES
434 | mov RAd, SAVE_NRES // RA = wanted nresults+1
435 |4:
436 | cmp RAd, RDd
437 | jne >6 // More/less results wanted?
438 |5:
439 | sub BASE, 16
440 | mov L:RB->top, BASE
441 |
442 |->vm_leave_cp:
443 | mov RA, SAVE_CFRAME // Restore previous C frame.
444 | mov L:RB->cframe, RA
445 | xor eax, eax // Ok return status for vm_pcall.
446 |
447 |->vm_leave_unw:
448 | restoreregs
449 | ret
450 |
451 |6:
452 | jb >7 // Less results wanted?
453 | // More results wanted. Check stack size and fill up results with nil.
454 | cmp BASE, L:RB->maxstack
455 | ja >8
456 | mov aword [BASE-16], LJ_TNIL
457 | add BASE, 8
458 | add RDd, 1
459 | jmp <4
460 |
461 |7: // Less results wanted.
462 | test RAd, RAd
463 | jz <5 // But check for LUA_MULTRET+1.
464 | sub RA, RD // Negative result!
465 | lea BASE, [BASE+RA*8] // Correct top.
466 | jmp <5
467 |
468 |8: // Corner case: need to grow stack for filling up results.
469 | // This can happen if:
470 | // - A C function grows the stack (a lot).
471 | // - The GC shrinks the stack in between.
472 | // - A return back from a lua_call() with (high) nresults adjustment.
473 | mov L:RB->top, BASE // Save current top held in BASE (yes).
474 | mov MULTRES, RDd // Need to fill only remainder with nil.
475 | mov CARG2d, RAd
476 | mov CARG1, L:RB
477 | call extern lj_state_growstack // (lua_State *L, int n)
478 | mov BASE, L:RB->top // Need the (realloced) L->top in BASE.
479 | jmp <3
480 |
481 |->vm_unwind_yield:
482 | mov al, LUA_YIELD
483 | jmp ->vm_unwind_c_eh
484 |
485 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
486 | // (void *cframe, int errcode)
487 | mov eax, CARG2d // Error return status for vm_pcall.
488 | mov rsp, CARG1
489 |->vm_unwind_c_eh: // Landing pad for external unwinder.
490 | mov L:RB, SAVE_L
491 | mov GL:RB, L:RB->glref
492 | mov dword GL:RB->vmstate, ~LJ_VMST_C
493 | jmp ->vm_leave_unw
494 |
495 |->vm_unwind_rethrow:
496 |.if not X64WIN
497 | mov CARG1, SAVE_L
498 | mov CARG2d, eax
499 | restoreregs
500 | jmp extern lj_err_throw // (lua_State *L, int errcode)
501 |.endif
502 |
503 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
504 | // (void *cframe)
505 | and CARG1, CFRAME_RAWMASK
506 | mov rsp, CARG1
507 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
508 | mov L:RB, SAVE_L
509 | mov RDd, 1+1 // Really 1+2 results, incr. later.
510 | mov BASE, L:RB->base
511 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
512 | add DISPATCH, GG_G2DISP
513 | mov PC, [BASE-8] // Fetch PC of previous frame.
514 | mov_false RA
515 | mov RB, [BASE]
516 | mov [BASE-16], RA // Prepend false to error message.
517 | mov [BASE-8], RB
518 | mov RA, -16 // Results start at BASE+RA = BASE-16.
519 | set_vmstate INTERP
520 | jmp ->vm_returnc // Increments RD/MULTRES and returns.
521 |
522 |//-----------------------------------------------------------------------
523 |//-- Grow stack for calls -----------------------------------------------
524 |//-----------------------------------------------------------------------
525 |
526 |->vm_growstack_c: // Grow stack for C function.
527 | mov CARG2d, LUA_MINSTACK
528 | jmp >2
529 |
530 |->vm_growstack_v: // Grow stack for vararg Lua function.
531 | sub RD, 16 // LJ_FR2
532 | jmp >1
533 |
534 |->vm_growstack_f: // Grow stack for fixarg Lua function.
535 | // BASE = new base, RD = nargs+1, RB = L, PC = first PC
536 | lea RD, [BASE+NARGS:RD*8-8]
537 |1:
538 | movzx RAd, byte [PC-4+PC2PROTO(framesize)]
539 | add PC, 4 // Must point after first instruction.
540 | mov L:RB->base, BASE
541 | mov L:RB->top, RD
542 | mov SAVE_PC, PC
543 | mov CARG2, RA
544 |2:
545 | // RB = L, L->base = new base, L->top = top
546 | mov CARG1, L:RB
547 | call extern lj_state_growstack // (lua_State *L, int n)
548 | mov BASE, L:RB->base
549 | mov RD, L:RB->top
550 | mov LFUNC:RB, [BASE-16]
551 | cleartp LFUNC:RB
552 | sub RD, BASE
553 | shr RDd, 3
554 | add NARGS:RDd, 1
555 | // BASE = new base, RB = LFUNC, RD = nargs+1
556 | ins_callt // Just retry the call.
557 |
558 |//-----------------------------------------------------------------------
559 |//-- Entry points into the assembler VM ---------------------------------
560 |//-----------------------------------------------------------------------
561 |
562 |->vm_resume: // Setup C frame and resume thread.
563 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
564 | saveregs
565 | mov L:RB, CARG1 // Caveat: CARG1 may be RA.
566 | mov SAVE_L, CARG1
567 | mov RA, CARG2
568 | mov PCd, FRAME_CP
569 | xor RDd, RDd
570 | lea KBASE, [esp+CFRAME_RESUME]
571 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
572 | add DISPATCH, GG_G2DISP
573 | mov SAVE_PC, RD // Any value outside of bytecode is ok.
574 | mov SAVE_CFRAME, RD
575 | mov SAVE_NRES, RDd
576 | mov SAVE_ERRF, RDd
577 | mov L:RB->cframe, KBASE
578 | cmp byte L:RB->status, RDL
579 | je >2 // Initial resume (like a call).
580 |
581 | // Resume after yield (like a return).
582 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
583 | set_vmstate INTERP
584 | mov byte L:RB->status, RDL
585 | mov BASE, L:RB->base
586 | mov RD, L:RB->top
587 | sub RD, RA
588 | shr RDd, 3
589 | add RDd, 1 // RD = nresults+1
590 | sub RA, BASE // RA = resultofs
591 | mov PC, [BASE-8]
592 | mov MULTRES, RDd
593 | test PCd, FRAME_TYPE
594 | jz ->BC_RET_Z
595 | jmp ->vm_return
596 |
597 |->vm_pcall: // Setup protected C frame and enter VM.
598 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
599 | saveregs
600 | mov PCd, FRAME_CP
601 | mov SAVE_ERRF, CARG4d
602 | jmp >1
603 |
604 |->vm_call: // Setup C frame and enter VM.
605 | // (lua_State *L, TValue *base, int nres1)
606 | saveregs
607 | mov PCd, FRAME_C
608 |
609 |1: // Entry point for vm_pcall above (PC = ftype).
610 | mov SAVE_NRES, CARG3d
611 | mov L:RB, CARG1 // Caveat: CARG1 may be RA.
612 | mov SAVE_L, CARG1
613 | mov RA, CARG2
614 |
615 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
616 | mov KBASE, L:RB->cframe // Add our C frame to cframe chain.
617 | mov SAVE_CFRAME, KBASE
618 | mov SAVE_PC, L:RB // Any value outside of bytecode is ok.
619 | add DISPATCH, GG_G2DISP
620 | mov L:RB->cframe, rsp
621 |
622 |2: // Entry point for vm_resume/vm_cpcall (RA = base, RB = L, PC = ftype).
623 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
624 | set_vmstate INTERP
625 | mov BASE, L:RB->base // BASE = old base (used in vmeta_call).
626 | add PC, RA
627 | sub PC, BASE // PC = frame delta + frame type
628 |
629 | mov RD, L:RB->top
630 | sub RD, RA
631 | shr NARGS:RDd, 3
632 | add NARGS:RDd, 1 // RD = nargs+1
633 |
634 |->vm_call_dispatch:
635 | mov LFUNC:RB, [RA-16]
636 | checkfunc LFUNC:RB, ->vmeta_call // Ensure KBASE defined and != BASE.
637 |
638 |->vm_call_dispatch_f:
639 | mov BASE, RA
640 | ins_call
641 | // BASE = new base, RB = func, RD = nargs+1, PC = caller PC
642 |
643 |->vm_cpcall: // Setup protected C frame, call C.
644 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
645 | saveregs
646 | mov L:RB, CARG1 // Caveat: CARG1 may be RA.
647 | mov SAVE_L, CARG1
648 | mov SAVE_PC, L:RB // Any value outside of bytecode is ok.
649 |
650 | mov KBASE, L:RB->stack // Compute -savestack(L, L->top).
651 | sub KBASE, L:RB->top
652 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
653 | mov SAVE_ERRF, 0 // No error function.
654 | mov SAVE_NRES, KBASEd // Neg. delta means cframe w/o frame.
655 | add DISPATCH, GG_G2DISP
656 | // Handler may change cframe_nres(L->cframe) or cframe_errfunc(L->cframe).
657 |
658 | mov KBASE, L:RB->cframe // Add our C frame to cframe chain.
659 | mov SAVE_CFRAME, KBASE
660 | mov L:RB->cframe, rsp
661 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
662 |
663 | call CARG4 // (lua_State *L, lua_CFunction func, void *ud)
664 | // TValue * (new base) or NULL returned in eax (RC).
665 | test RC, RC
666 | jz ->vm_leave_cp // No base? Just remove C frame.
667 | mov RA, RC
668 | mov PCd, FRAME_CP
669 | jmp <2 // Else continue with the call.
670 |
671 |//-----------------------------------------------------------------------
672 |//-- Metamethod handling ------------------------------------------------
673 |//-----------------------------------------------------------------------
674 |
675 |//-- Continuation dispatch ----------------------------------------------
676 |
677 |->cont_dispatch:
678 | // BASE = meta base, RA = resultofs, RD = nresults+1 (also in MULTRES)
679 | add RA, BASE
680 | and PC, -8
681 | mov RB, BASE
682 | sub BASE, PC // Restore caller BASE.
683 | mov aword [RA+RD*8-8], LJ_TNIL // Ensure one valid arg.
684 | mov RC, RA // ... in [RC]
685 | mov PC, [RB-24] // Restore PC from [cont|PC].
686 | mov RA, qword [RB-32] // May be negative on WIN64 with debug.
687 |.if FFI
688 | cmp RA, 1
689 | jbe >1
690 |.endif
691 | mov LFUNC:KBASE, [BASE-16]
692 | cleartp LFUNC:KBASE
693 | mov KBASE, LFUNC:KBASE->pc
694 | mov KBASE, [KBASE+PC2PROTO(k)]
695 | // BASE = base, RC = result, RB = meta base
696 | jmp RA // Jump to continuation.
697 |
698 |.if FFI
699 |1:
700 | je ->cont_ffi_callback // cont = 1: return from FFI callback.
701 | // cont = 0: Tail call from C function.
702 | sub RB, BASE
703 | shr RBd, 3
704 | lea RDd, [RBd-3]
705 | jmp ->vm_call_tail
706 |.endif
707 |
708 |->cont_cat: // BASE = base, RC = result, RB = mbase
709 | movzx RAd, PC_RB
710 | sub RB, 32
711 | lea RA, [BASE+RA*8]
712 | sub RA, RB
713 | je ->cont_ra
714 | neg RA
715 | shr RAd, 3
716 |.if X64WIN
717 | mov CARG3d, RAd
718 | mov L:CARG1, SAVE_L
719 | mov L:CARG1->base, BASE
720 | mov RC, [RC]
721 | mov [RB], RC
722 | mov CARG2, RB
723 |.else
724 | mov L:CARG1, SAVE_L
725 | mov L:CARG1->base, BASE
726 | mov CARG3d, RAd
727 | mov RA, [RC]
728 | mov [RB], RA
729 | mov CARG2, RB
730 |.endif
731 | jmp ->BC_CAT_Z
732 |
733 |//-- Table indexing metamethods -----------------------------------------
734 |
735 |->vmeta_tgets:
736 | settp STR:RC, LJ_TSTR // STR:RC = GCstr *
737 | mov TMP1, STR:RC
738 | lea RC, TMP1
739 | cmp PC_OP, BC_GGET
740 | jne >1
741 | settp TAB:RA, TAB:RB, LJ_TTAB // TAB:RB = GCtab *
742 | lea RB, [DISPATCH+DISPATCH_GL(tmptv)] // Store fn->l.env in g->tmptv.
743 | mov [RB], TAB:RA
744 | jmp >2
745 |
746 |->vmeta_tgetb:
747 | movzx RCd, PC_RC
748 |.if DUALNUM
749 | setint RC
750 | mov TMP1, RC
751 |.else
752 | cvtsi2sd xmm0, RCd
753 | movsd TMP1, xmm0
754 |.endif
755 | lea RC, TMP1
756 | jmp >1
757 |
758 |->vmeta_tgetv:
759 | movzx RCd, PC_RC // Reload TValue *k from RC.
760 | lea RC, [BASE+RC*8]
761 |1:
762 | movzx RBd, PC_RB // Reload TValue *t from RB.
763 | lea RB, [BASE+RB*8]
764 |2:
765 | mov L:CARG1, SAVE_L
766 | mov L:CARG1->base, BASE // Caveat: CARG2/CARG3 may be BASE.
767 | mov CARG2, RB
768 | mov CARG3, RC
769 | mov L:RB, L:CARG1
770 | mov SAVE_PC, PC
771 | call extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
772 | // TValue * (finished) or NULL (metamethod) returned in eax (RC).
773 | mov BASE, L:RB->base
774 | test RC, RC
775 | jz >3
776 |->cont_ra: // BASE = base, RC = result
777 | movzx RAd, PC_RA
778 | mov RB, [RC]
779 | mov [BASE+RA*8], RB
780 | ins_next
781 |
782 |3: // Call __index metamethod.
783 | // BASE = base, L->top = new base, stack = cont/func/t/k
784 | mov RA, L:RB->top
785 | mov [RA-24], PC // [cont|PC]
786 | lea PC, [RA+FRAME_CONT]
787 | sub PC, BASE
788 | mov LFUNC:RB, [RA-16] // Guaranteed to be a function here.
789 | mov NARGS:RDd, 2+1 // 2 args for func(t, k).
790 | cleartp LFUNC:RB
791 | jmp ->vm_call_dispatch_f
792 |
793 |->vmeta_tgetr:
794 | mov CARG1, TAB:RB
795 | mov RB, BASE // Save BASE.
796 | mov CARG2d, RCd // Caveat: CARG2 == BASE
797 | call extern lj_tab_getinth // (GCtab *t, int32_t key)
798 | // cTValue * or NULL returned in eax (RC).
799 | movzx RAd, PC_RA
800 | mov BASE, RB // Restore BASE.
801 | test RC, RC
802 | jnz ->BC_TGETR_Z
803 | mov ITYPE, LJ_TNIL
804 | jmp ->BC_TGETR2_Z
805 |
806 |//-----------------------------------------------------------------------
807 |
808 |->vmeta_tsets:
809 | settp STR:RC, LJ_TSTR // STR:RC = GCstr *
810 | mov TMP1, STR:RC
811 | lea RC, TMP1
812 | cmp PC_OP, BC_GSET
813 | jne >1
814 | settp TAB:RA, TAB:RB, LJ_TTAB // TAB:RB = GCtab *
815 | lea RB, [DISPATCH+DISPATCH_GL(tmptv)] // Store fn->l.env in g->tmptv.
816 | mov [RB], TAB:RA
817 | jmp >2
818 |
819 |->vmeta_tsetb:
820 | movzx RCd, PC_RC
821 |.if DUALNUM
822 | setint RC
823 | mov TMP1, RC
824 |.else
825 | cvtsi2sd xmm0, RCd
826 | movsd TMP1, xmm0
827 |.endif
828 | lea RC, TMP1
829 | jmp >1
830 |
831 |->vmeta_tsetv:
832 | movzx RCd, PC_RC // Reload TValue *k from RC.
833 | lea RC, [BASE+RC*8]
834 |1:
835 | movzx RBd, PC_RB // Reload TValue *t from RB.
836 | lea RB, [BASE+RB*8]
837 |2:
838 | mov L:CARG1, SAVE_L
839 | mov L:CARG1->base, BASE // Caveat: CARG2/CARG3 may be BASE.
840 | mov CARG2, RB
841 | mov CARG3, RC
842 | mov L:RB, L:CARG1
843 | mov SAVE_PC, PC
844 | call extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
845 | // TValue * (finished) or NULL (metamethod) returned in eax (RC).
846 | mov BASE, L:RB->base
847 | test RC, RC
848 | jz >3
849 | // NOBARRIER: lj_meta_tset ensures the table is not black.
850 | movzx RAd, PC_RA
851 | mov RB, [BASE+RA*8]
852 | mov [RC], RB
853 |->cont_nop: // BASE = base, (RC = result)
854 | ins_next
855 |
856 |3: // Call __newindex metamethod.
857 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
858 | mov RA, L:RB->top
859 | mov [RA-24], PC // [cont|PC]
860 | movzx RCd, PC_RA
861 | // Copy value to third argument.
862 | mov RB, [BASE+RC*8]
863 | mov [RA+16], RB
864 | lea PC, [RA+FRAME_CONT]
865 | sub PC, BASE
866 | mov LFUNC:RB, [RA-16] // Guaranteed to be a function here.
867 | mov NARGS:RDd, 3+1 // 3 args for func(t, k, v).
868 | cleartp LFUNC:RB
869 | jmp ->vm_call_dispatch_f
870 |
871 |->vmeta_tsetr:
872 |.if X64WIN
873 | mov L:CARG1, SAVE_L
874 | mov CARG3d, RCd
875 | mov L:CARG1->base, BASE
876 | xchg CARG2, TAB:RB // Caveat: CARG2 == BASE.
877 |.else
878 | mov L:CARG1, SAVE_L
879 | mov CARG2, TAB:RB
880 | mov L:CARG1->base, BASE
881 | mov RB, BASE // Save BASE.
882 | mov CARG3d, RCd // Caveat: CARG3 == BASE.
883 |.endif
884 | mov SAVE_PC, PC
885 | call extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
886 | // TValue * returned in eax (RC).
887 | movzx RAd, PC_RA
888 | mov BASE, RB // Restore BASE.
889 | jmp ->BC_TSETR_Z
890 |
891 |//-- Comparison metamethods ---------------------------------------------
892 |
893 |->vmeta_comp:
894 | movzx RDd, PC_RD
895 | movzx RAd, PC_RA
896 | mov L:RB, SAVE_L
897 | mov L:RB->base, BASE // Caveat: CARG2/CARG3 == BASE.
898 |.if X64WIN
899 | lea CARG3, [BASE+RD*8]
900 | lea CARG2, [BASE+RA*8]
901 |.else
902 | lea CARG2, [BASE+RA*8]
903 | lea CARG3, [BASE+RD*8]
904 |.endif
905 | mov CARG1, L:RB // Caveat: CARG1/CARG4 == RA.
906 | movzx CARG4d, PC_OP
907 | mov SAVE_PC, PC
908 | call extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
909 | // 0/1 or TValue * (metamethod) returned in eax (RC).
910 |3:
911 | mov BASE, L:RB->base
912 | cmp RC, 1
913 | ja ->vmeta_binop
914 |4:
915 | lea PC, [PC+4]
916 | jb >6
917 |5:
918 | movzx RDd, PC_RD
919 | branchPC RD
920 |6:
921 | ins_next
922 |
923 |->cont_condt: // BASE = base, RC = result
924 | add PC, 4
925 | mov ITYPE, [RC]
926 | sar ITYPE, 47
927 | cmp ITYPEd, LJ_TISTRUECOND // Branch if result is true.
928 | jb <5
929 | jmp <6
930 |
931 |->cont_condf: // BASE = base, RC = result
932 | mov ITYPE, [RC]
933 | sar ITYPE, 47
934 | cmp ITYPEd, LJ_TISTRUECOND // Branch if result is false.
935 | jmp <4
936 |
937 |->vmeta_equal:
938 | cleartp TAB:RD
939 | sub PC, 4
940 |.if X64WIN
941 | mov CARG3, RD
942 | mov CARG4d, RBd
943 | mov L:RB, SAVE_L
944 | mov L:RB->base, BASE // Caveat: CARG2 == BASE.
945 | mov CARG2, RA
946 | mov CARG1, L:RB // Caveat: CARG1 == RA.
947 |.else
948 | mov CARG2, RA
949 | mov CARG4d, RBd // Caveat: CARG4 == RA.
950 | mov L:RB, SAVE_L
951 | mov L:RB->base, BASE // Caveat: CARG3 == BASE.
952 | mov CARG3, RD
953 | mov CARG1, L:RB
954 |.endif
955 | mov SAVE_PC, PC
956 | call extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
957 | // 0/1 or TValue * (metamethod) returned in eax (RC).
958 | jmp <3
959 |
960 |->vmeta_equal_cd:
961 |.if FFI
962 | sub PC, 4
963 | mov L:RB, SAVE_L
964 | mov L:RB->base, BASE
965 | mov CARG1, L:RB
966 | mov CARG2d, dword [PC-4]
967 | mov SAVE_PC, PC
968 | call extern lj_meta_equal_cd // (lua_State *L, BCIns ins)
969 | // 0/1 or TValue * (metamethod) returned in eax (RC).
970 | jmp <3
971 |.endif
972 |
973 |->vmeta_istype:
974 | mov L:RB, SAVE_L
975 | mov L:RB->base, BASE // Caveat: CARG2/CARG3 may be BASE.
976 | mov CARG2d, RAd
977 | mov CARG3d, RDd
978 | mov L:CARG1, L:RB
979 | mov SAVE_PC, PC
980 | call extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
981 | mov BASE, L:RB->base
982 | jmp <6
983 |
984 |//-- Arithmetic metamethods ---------------------------------------------
985 |
986 |->vmeta_arith_vno:
987 |.if DUALNUM
988 | movzx RBd, PC_RB
989 | movzx RCd, PC_RC
990 |.endif
991 |->vmeta_arith_vn:
992 | lea RC, [KBASE+RC*8]
993 | jmp >1
994 |
995 |->vmeta_arith_nvo:
996 |.if DUALNUM
997 | movzx RBd, PC_RB
998 | movzx RCd, PC_RC
999 |.endif
1000 |->vmeta_arith_nv:
1001 | lea TMPR, [KBASE+RC*8]
1002 | lea RC, [BASE+RB*8]
1003 | mov RB, TMPR
1004 | jmp >2
1005 |
1006 |->vmeta_unm:
1007 | lea RC, [BASE+RD*8]
1008 | mov RB, RC
1009 | jmp >2
1010 |
1011 |->vmeta_arith_vvo:
1012 |.if DUALNUM
1013 | movzx RBd, PC_RB
1014 | movzx RCd, PC_RC
1015 |.endif
1016 |->vmeta_arith_vv:
1017 | lea RC, [BASE+RC*8]
1018 |1:
1019 | lea RB, [BASE+RB*8]
1020 |2:
1021 | lea RA, [BASE+RA*8]
1022 |.if X64WIN
1023 | mov CARG3, RB
1024 | mov CARG4, RC
1025 | movzx RCd, PC_OP
1026 | mov ARG5d, RCd
1027 | mov L:RB, SAVE_L
1028 | mov L:RB->base, BASE // Caveat: CARG2 == BASE.
1029 | mov CARG2, RA
1030 | mov CARG1, L:RB // Caveat: CARG1 == RA.
1031 |.else
1032 | movzx CARG5d, PC_OP
1033 | mov CARG2, RA
1034 | mov CARG4, RC // Caveat: CARG4 == RA.
1035 | mov L:CARG1, SAVE_L
1036 | mov L:CARG1->base, BASE // Caveat: CARG3 == BASE.
1037 | mov CARG3, RB
1038 | mov L:RB, L:CARG1
1039 |.endif
1040 | mov SAVE_PC, PC
1041 | call extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
1042 | // NULL (finished) or TValue * (metamethod) returned in eax (RC).
1043 | mov BASE, L:RB->base
1044 | test RC, RC
1045 | jz ->cont_nop
1046 |
1047 | // Call metamethod for binary op.
1048 |->vmeta_binop:
1049 | // BASE = base, RC = new base, stack = cont/func/o1/o2
1050 | mov RA, RC
1051 | sub RC, BASE
1052 | mov [RA-24], PC // [cont|PC]
1053 | lea PC, [RC+FRAME_CONT]
1054 | mov NARGS:RDd, 2+1 // 2 args for func(o1, o2).
1055 | jmp ->vm_call_dispatch
1056 |
1057 |->vmeta_len:
1058 | movzx RDd, PC_RD
1059 | mov L:RB, SAVE_L
1060 | mov L:RB->base, BASE
1061 | lea CARG2, [BASE+RD*8] // Caveat: CARG2 == BASE
1062 | mov L:CARG1, L:RB
1063 | mov SAVE_PC, PC
1064 | call extern lj_meta_len // (lua_State *L, TValue *o)
1065 | // NULL (retry) or TValue * (metamethod) returned in eax (RC).
1066 | mov BASE, L:RB->base
1067#if LJ_52
1068 | test RC, RC
1069 | jne ->vmeta_binop // Binop call for compatibility.
1070 | movzx RDd, PC_RD
1071 | mov TAB:CARG1, [BASE+RD*8]
1072 | cleartp TAB:CARG1
1073 | jmp ->BC_LEN_Z
1074#else
1075 | jmp ->vmeta_binop // Binop call for compatibility.
1076#endif
1077 |
1078 |//-- Call metamethod ----------------------------------------------------
1079 |
1080 |->vmeta_call_ra:
1081 | lea RA, [BASE+RA*8+16]
1082 |->vmeta_call: // Resolve and call __call metamethod.
1083 | // BASE = old base, RA = new base, RC = nargs+1, PC = return
1084 | mov TMP1d, NARGS:RDd // Save RA, RC for us.
1085 | mov RB, RA
1086 |.if X64WIN
1087 | mov L:TMPR, SAVE_L
1088 | mov L:TMPR->base, BASE // Caveat: CARG2 is BASE.
1089 | lea CARG2, [RA-16]
1090 | lea CARG3, [RA+NARGS:RD*8-8]
1091 | mov CARG1, L:TMPR // Caveat: CARG1 is RA.
1092 |.else
1093 | mov L:CARG1, SAVE_L
1094 | mov L:CARG1->base, BASE // Caveat: CARG3 is BASE.
1095 | lea CARG2, [RA-16]
1096 | lea CARG3, [RA+NARGS:RD*8-8]
1097 |.endif
1098 | mov SAVE_PC, PC
1099 | call extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
1100 | mov RA, RB
1101 | mov L:RB, SAVE_L
1102 | mov BASE, L:RB->base
1103 | mov NARGS:RDd, TMP1d
1104 | mov LFUNC:RB, [RA-16]
1105 | add NARGS:RDd, 1
1106 | // This is fragile. L->base must not move, KBASE must always be defined.
1107 | cmp KBASE, BASE // Continue with CALLT if flag set.
1108 | je ->BC_CALLT_Z
1109 | cleartp LFUNC:RB
1110 | mov BASE, RA
1111 | ins_call // Otherwise call resolved metamethod.
1112 |
1113 |//-- Argument coercion for 'for' statement ------------------------------
1114 |
1115 |->vmeta_for:
1116 | mov L:RB, SAVE_L
1117 | mov L:RB->base, BASE
1118 | mov CARG2, RA // Caveat: CARG2 == BASE
1119 | mov L:CARG1, L:RB // Caveat: CARG1 == RA
1120 | mov SAVE_PC, PC
1121 | call extern lj_meta_for // (lua_State *L, TValue *base)
1122 | mov BASE, L:RB->base
1123 | mov RCd, [PC-4]
1124 | movzx RAd, RCH
1125 | movzx OP, RCL
1126 | shr RCd, 16
1127 | jmp aword [DISPATCH+OP*8+GG_DISP2STATIC] // Retry FORI or JFORI.
1128 |
1129 |//-----------------------------------------------------------------------
1130 |//-- Fast functions -----------------------------------------------------
1131 |//-----------------------------------------------------------------------
1132 |
1133 |.macro .ffunc, name
1134 |->ff_ .. name:
1135 |.endmacro
1136 |
1137 |.macro .ffunc_1, name
1138 |->ff_ .. name:
1139 | cmp NARGS:RDd, 1+1; jb ->fff_fallback
1140 |.endmacro
1141 |
1142 |.macro .ffunc_2, name
1143 |->ff_ .. name:
1144 | cmp NARGS:RDd, 2+1; jb ->fff_fallback
1145 |.endmacro
1146 |
1147 |.macro .ffunc_n, name, op
1148 | .ffunc_1 name
1149 | checknumtp [BASE], ->fff_fallback
1150 | op xmm0, qword [BASE]
1151 |.endmacro
1152 |
1153 |.macro .ffunc_n, name
1154 | .ffunc_n name, movsd
1155 |.endmacro
1156 |
1157 |.macro .ffunc_nn, name
1158 | .ffunc_2 name
1159 | checknumtp [BASE], ->fff_fallback
1160 | checknumtp [BASE+8], ->fff_fallback
1161 | movsd xmm0, qword [BASE]
1162 | movsd xmm1, qword [BASE+8]
1163 |.endmacro
1164 |
1165 |// Inlined GC threshold check. Caveat: uses label 1.
1166 |.macro ffgccheck
1167 | mov RB, [DISPATCH+DISPATCH_GL(gc.total)]
1168 | cmp RB, [DISPATCH+DISPATCH_GL(gc.threshold)]
1169 | jb >1
1170 | call ->fff_gcstep
1171 |1:
1172 |.endmacro
1173 |
1174 |//-- Base library: checks -----------------------------------------------
1175 |
1176 |.ffunc_1 assert
1177 | mov ITYPE, [BASE]
1178 | mov RB, ITYPE
1179 | sar ITYPE, 47
1180 | cmp ITYPEd, LJ_TISTRUECOND; jae ->fff_fallback
1181 | mov PC, [BASE-8]
1182 | mov MULTRES, RDd
1183 | mov RB, [BASE]
1184 | mov [BASE-16], RB
1185 | sub RDd, 2
1186 | jz >2
1187 | mov RA, BASE
1188 |1:
1189 | add RA, 8
1190 | mov RB, [RA]
1191 | mov [RA-16], RB
1192 | sub RDd, 1
1193 | jnz <1
1194 |2:
1195 | mov RDd, MULTRES
1196 | jmp ->fff_res_
1197 |
1198 |.ffunc_1 type
1199 | mov RC, [BASE]
1200 | sar RC, 47
1201 | mov RBd, LJ_TISNUM
1202 | cmp RCd, RBd
1203 | cmovb RCd, RBd
1204 | not RCd
1205 |2:
1206 | mov CFUNC:RB, [BASE-16]
1207 | cleartp CFUNC:RB
1208 | mov STR:RC, [CFUNC:RB+RC*8+((char *)(&((GCfuncC *)0)->upvalue))]
1209 | mov PC, [BASE-8]
1210 | settp STR:RC, LJ_TSTR
1211 | mov [BASE-16], STR:RC
1212 | jmp ->fff_res1
1213 |
1214 |//-- Base library: getters and setters ---------------------------------
1215 |
1216 |.ffunc_1 getmetatable
1217 | mov TAB:RB, [BASE]
1218 | mov PC, [BASE-8]
1219 | checktab TAB:RB, >6
1220 |1: // Field metatable must be at same offset for GCtab and GCudata!
1221 | mov TAB:RB, TAB:RB->metatable
1222 |2:
1223 | test TAB:RB, TAB:RB
1224 | mov aword [BASE-16], LJ_TNIL
1225 | jz ->fff_res1
1226 | settp TAB:RC, TAB:RB, LJ_TTAB
1227 | mov [BASE-16], TAB:RC // Store metatable as default result.
1228 | mov STR:RC, [DISPATCH+DISPATCH_GL(gcroot)+8*(GCROOT_MMNAME+MM_metatable)]
1229 | mov RAd, TAB:RB->hmask
1230 | and RAd, STR:RC->sid
1231 | settp STR:RC, LJ_TSTR
1232 | imul RAd, #NODE
1233 | add NODE:RA, TAB:RB->node
1234 |3: // Rearranged logic, because we expect _not_ to find the key.
1235 | cmp NODE:RA->key, STR:RC
1236 | je >5
1237 |4:
1238 | mov NODE:RA, NODE:RA->next
1239 | test NODE:RA, NODE:RA
1240 | jnz <3
1241 | jmp ->fff_res1 // Not found, keep default result.
1242 |5:
1243 | mov RB, NODE:RA->val
1244 | cmp RB, LJ_TNIL; je ->fff_res1 // Ditto for nil value.
1245 | mov [BASE-16], RB // Return value of mt.__metatable.
1246 | jmp ->fff_res1
1247 |
1248 |6:
1249 | cmp ITYPEd, LJ_TUDATA; je <1
1250 | cmp ITYPEd, LJ_TISNUM; ja >7
1251 | mov ITYPEd, LJ_TISNUM
1252 |7:
1253 | not ITYPEd
1254 | mov TAB:RB, [DISPATCH+ITYPE*8+DISPATCH_GL(gcroot[GCROOT_BASEMT])]
1255 | jmp <2
1256 |
1257 |.ffunc_2 setmetatable
1258 | mov TAB:RB, [BASE]
1259 | mov TAB:TMPR, TAB:RB
1260 | checktab TAB:RB, ->fff_fallback
1261 | // Fast path: no mt for table yet and not clearing the mt.
1262 | cmp aword TAB:RB->metatable, 0; jne ->fff_fallback
1263 | mov TAB:RA, [BASE+8]
1264 | checktab TAB:RA, ->fff_fallback
1265 | mov TAB:RB->metatable, TAB:RA
1266 | mov PC, [BASE-8]
1267 | mov [BASE-16], TAB:TMPR // Return original table.
1268 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
1269 | jz >1
1270 | // Possible write barrier. Table is black, but skip iswhite(mt) check.
1271 | barrierback TAB:RB, RC
1272 |1:
1273 | jmp ->fff_res1
1274 |
1275 |.ffunc_2 rawget
1276 |.if X64WIN
1277 | mov TAB:RA, [BASE]
1278 | checktab TAB:RA, ->fff_fallback
1279 | mov RB, BASE // Save BASE.
1280 | lea CARG3, [BASE+8]
1281 | mov CARG2, TAB:RA // Caveat: CARG2 == BASE.
1282 | mov CARG1, SAVE_L
1283 |.else
1284 | mov TAB:CARG2, [BASE]
1285 | checktab TAB:CARG2, ->fff_fallback
1286 | mov RB, BASE // Save BASE.
1287 | lea CARG3, [BASE+8] // Caveat: CARG3 == BASE.
1288 | mov CARG1, SAVE_L
1289 |.endif
1290 | call extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1291 | // cTValue * returned in eax (RD).
1292 | mov BASE, RB // Restore BASE.
1293 | // Copy table slot.
1294 | mov RB, [RD]
1295 | mov PC, [BASE-8]
1296 | mov [BASE-16], RB
1297 | jmp ->fff_res1
1298 |
1299 |//-- Base library: conversions ------------------------------------------
1300 |
1301 |.ffunc tonumber
1302 | // Only handles the number case inline (without a base argument).
1303 | cmp NARGS:RDd, 1+1; jne ->fff_fallback // Exactly one argument.
1304 | mov RB, [BASE]
1305 | checknumber RB, ->fff_fallback
1306 | mov PC, [BASE-8]
1307 | mov [BASE-16], RB
1308 | jmp ->fff_res1
1309 |
1310 |.ffunc_1 tostring
1311 | // Only handles the string or number case inline.
1312 | mov PC, [BASE-8]
1313 | mov STR:RB, [BASE]
1314 | checktp_nc STR:RB, LJ_TSTR, >3
1315 | // A __tostring method in the string base metatable is ignored.
1316 |2:
1317 | mov [BASE-16], STR:RB
1318 | jmp ->fff_res1
1319 |3: // Handle numbers inline, unless a number base metatable is present.
1320 | cmp ITYPEd, LJ_TISNUM; ja ->fff_fallback_1
1321 | cmp aword [DISPATCH+DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])], 0
1322 | jne ->fff_fallback
1323 | ffgccheck // Caveat: uses label 1.
1324 | mov L:RB, SAVE_L
1325 | mov L:RB->base, BASE // Add frame since C call can throw.
1326 | mov SAVE_PC, PC // Redundant (but a defined value).
1327 |.if not X64WIN
1328 | mov CARG2, BASE // Otherwise: CARG2 == BASE
1329 |.endif
1330 | mov L:CARG1, L:RB
1331 |.if DUALNUM
1332 | call extern lj_strfmt_number // (lua_State *L, cTValue *o)
1333 |.else
1334 | call extern lj_strfmt_num // (lua_State *L, lua_Number *np)
1335 |.endif
1336 | // GCstr returned in eax (RD).
1337 | mov BASE, L:RB->base
1338 | settp STR:RB, RD, LJ_TSTR
1339 | jmp <2
1340 |
1341 |//-- Base library: iterators -------------------------------------------
1342 |
1343 |.ffunc_1 next
1344 | je >2 // Missing 2nd arg?
1345 |1:
1346 | mov CARG1, [BASE]
1347 | mov PC, [BASE-8]
1348 | checktab CARG1, ->fff_fallback
1349 | mov RB, BASE // Save BASE.
1350 |.if X64WIN
1351 | lea CARG3, [BASE-16]
1352 | lea CARG2, [BASE+8] // Caveat: CARG2 == BASE.
1353 |.else
1354 | lea CARG2, [BASE+8]
1355 | lea CARG3, [BASE-16] // Caveat: CARG3 == BASE.
1356 |.endif
1357 | call extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1358 | // 1=found, 0=end, -1=error returned in eax (RD).
1359 | mov BASE, RB // Restore BASE.
1360 | test RDd, RDd; jg ->fff_res2 // Found key/value.
1361 | js ->fff_fallback_2 // Invalid key.
1362 | // End of traversal: return nil.
1363 | mov aword [BASE-16], LJ_TNIL
1364 | jmp ->fff_res1
1365 |2: // Set missing 2nd arg to nil.
1366 | mov aword [BASE+8], LJ_TNIL
1367 | jmp <1
1368 |
1369 |.ffunc_1 pairs
1370 | mov TAB:RB, [BASE]
1371 | mov TMPR, TAB:RB
1372 | checktab TAB:RB, ->fff_fallback
1373#if LJ_52
1374 | cmp aword TAB:RB->metatable, 0; jne ->fff_fallback
1375#endif
1376 | mov CFUNC:RD, [BASE-16]
1377 | cleartp CFUNC:RD
1378 | mov CFUNC:RD, CFUNC:RD->upvalue[0]
1379 | settp CFUNC:RD, LJ_TFUNC
1380 | mov PC, [BASE-8]
1381 | mov [BASE-16], CFUNC:RD
1382 | mov [BASE-8], TMPR
1383 | mov aword [BASE], LJ_TNIL
1384 | mov RDd, 1+3
1385 | jmp ->fff_res
1386 |
1387 |.ffunc_2 ipairs_aux
1388 | mov TAB:RB, [BASE]
1389 | checktab TAB:RB, ->fff_fallback
1390 |.if DUALNUM
1391 | mov RA, [BASE+8]
1392 | checkint RA, ->fff_fallback
1393 |.else
1394 | checknumtp [BASE+8], ->fff_fallback
1395 | movsd xmm0, qword [BASE+8]
1396 |.endif
1397 | mov PC, [BASE-8]
1398 |.if DUALNUM
1399 | add RAd, 1
1400 | setint ITYPE, RA
1401 | mov [BASE-16], ITYPE
1402 |.else
1403 | sseconst_1 xmm1, TMPR
1404 | addsd xmm0, xmm1
1405 | cvttsd2si RAd, xmm0
1406 | movsd qword [BASE-16], xmm0
1407 |.endif
1408 | cmp RAd, TAB:RB->asize; jae >2 // Not in array part?
1409 | mov RD, TAB:RB->array
1410 | lea RD, [RD+RA*8]
1411 |1:
1412 | cmp aword [RD], LJ_TNIL; je ->fff_res0
1413 | // Copy array slot.
1414 | mov RB, [RD]
1415 | mov [BASE-8], RB
1416 |->fff_res2:
1417 | mov RDd, 1+2
1418 | jmp ->fff_res
1419 |2: // Check for empty hash part first. Otherwise call C function.
1420 | cmp dword TAB:RB->hmask, 0; je ->fff_res0
1421 |.if X64WIN
1422 | mov TMPR, BASE
1423 | mov CARG2d, RAd
1424 | mov CARG1, TAB:RB
1425 | mov RB, TMPR
1426 |.else
1427 | mov CARG1, TAB:RB
1428 | mov RB, BASE // Save BASE.
1429 | mov CARG2d, RAd // Caveat: CARG2 == BASE
1430 |.endif
1431 | call extern lj_tab_getinth // (GCtab *t, int32_t key)
1432 | // cTValue * or NULL returned in eax (RD).
1433 | mov BASE, RB
1434 | test RD, RD
1435 | jnz <1
1436 |->fff_res0:
1437 | mov RDd, 1+0
1438 | jmp ->fff_res
1439 |
1440 |.ffunc_1 ipairs
1441 | mov TAB:RB, [BASE]
1442 | mov TMPR, TAB:RB
1443 | checktab TAB:RB, ->fff_fallback
1444#if LJ_52
1445 | cmp aword TAB:RB->metatable, 0; jne ->fff_fallback
1446#endif
1447 | mov CFUNC:RD, [BASE-16]
1448 | cleartp CFUNC:RD
1449 | mov CFUNC:RD, CFUNC:RD->upvalue[0]
1450 | settp CFUNC:RD, LJ_TFUNC
1451 | mov PC, [BASE-8]
1452 | mov [BASE-16], CFUNC:RD
1453 | mov [BASE-8], TMPR
1454 |.if DUALNUM
1455 | mov64 RD, ((uint64_t)LJ_TISNUM<<47)
1456 | mov [BASE], RD
1457 |.else
1458 | mov qword [BASE], 0
1459 |.endif
1460 | mov RDd, 1+3
1461 | jmp ->fff_res
1462 |
1463 |//-- Base library: catch errors ----------------------------------------
1464 |
1465 |.ffunc_1 pcall
1466 | lea RA, [BASE+16]
1467 | sub NARGS:RDd, 1
1468 | mov PCd, 16+FRAME_PCALL
1469 |1:
1470 | movzx RBd, byte [DISPATCH+DISPATCH_GL(hookmask)]
1471 | shr RB, HOOK_ACTIVE_SHIFT
1472 | and RB, 1
1473 | add PC, RB // Remember active hook before pcall.
1474 | // Note: this does a (harmless) copy of the function to the PC slot, too.
1475 | mov KBASE, RD
1476 |2:
1477 | mov RB, [RA+KBASE*8-24]
1478 | mov [RA+KBASE*8-16], RB
1479 | sub KBASE, 1
1480 | ja <2
1481 | jmp ->vm_call_dispatch
1482 |
1483 |.ffunc_2 xpcall
1484 | mov LFUNC:RA, [BASE+8]
1485 | checktp_nc LFUNC:RA, LJ_TFUNC, ->fff_fallback
1486 | mov LFUNC:RB, [BASE] // Swap function and traceback.
1487 | mov [BASE], LFUNC:RA
1488 | mov [BASE+8], LFUNC:RB
1489 | lea RA, [BASE+24]
1490 | sub NARGS:RDd, 2
1491 | mov PCd, 24+FRAME_PCALL
1492 | jmp <1
1493 |
1494 |//-- Coroutine library --------------------------------------------------
1495 |
1496 |.macro coroutine_resume_wrap, resume
1497 |.if resume
1498 |.ffunc_1 coroutine_resume
1499 | mov L:RB, [BASE]
1500 | cleartp L:RB
1501 |.else
1502 |.ffunc coroutine_wrap_aux
1503 | mov CFUNC:RB, [BASE-16]
1504 | cleartp CFUNC:RB
1505 | mov L:RB, CFUNC:RB->upvalue[0].gcr
1506 | cleartp L:RB
1507 |.endif
1508 | mov PC, [BASE-8]
1509 | mov SAVE_PC, PC
1510 | mov TMP1, L:RB
1511 |.if resume
1512 | checktptp [BASE], LJ_TTHREAD, ->fff_fallback
1513 |.endif
1514 | cmp aword L:RB->cframe, 0; jne ->fff_fallback
1515 | cmp byte L:RB->status, LUA_YIELD; ja ->fff_fallback
1516 | mov RA, L:RB->top
1517 | je >1 // Status != LUA_YIELD (i.e. 0)?
1518 | cmp RA, L:RB->base // Check for presence of initial func.
1519 | je ->fff_fallback
1520 | mov PC, [RA-8] // Move initial function up.
1521 | mov [RA], PC
1522 | add RA, 8
1523 |1:
1524 |.if resume
1525 | lea PC, [RA+NARGS:RD*8-16] // Check stack space (-1-thread).
1526 |.else
1527 | lea PC, [RA+NARGS:RD*8-8] // Check stack space (-1).
1528 |.endif
1529 | cmp PC, L:RB->maxstack; ja ->fff_fallback
1530 | mov L:RB->top, PC
1531 |
1532 | mov L:RB, SAVE_L
1533 | mov L:RB->base, BASE
1534 |.if resume
1535 | add BASE, 8 // Keep resumed thread in stack for GC.
1536 |.endif
1537 | mov L:RB->top, BASE
1538 |.if resume
1539 | lea RB, [BASE+NARGS:RD*8-24] // RB = end of source for stack move.
1540 |.else
1541 | lea RB, [BASE+NARGS:RD*8-16] // RB = end of source for stack move.
1542 |.endif
1543 | sub RB, PC // Relative to PC.
1544 |
1545 | cmp PC, RA
1546 | je >3
1547 |2: // Move args to coroutine.
1548 | mov RC, [PC+RB]
1549 | mov [PC-8], RC
1550 | sub PC, 8
1551 | cmp PC, RA
1552 | jne <2
1553 |3:
1554 | mov CARG2, RA
1555 | mov CARG1, TMP1
1556 | call ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1557 |
1558 | mov L:RB, SAVE_L
1559 | mov L:PC, TMP1
1560 | mov BASE, L:RB->base
1561 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
1562 | set_vmstate INTERP
1563 |
1564 | cmp eax, LUA_YIELD
1565 | ja >8
1566 |4:
1567 | mov RA, L:PC->base
1568 | mov KBASE, L:PC->top
1569 | mov L:PC->top, RA // Clear coroutine stack.
1570 | mov PC, KBASE
1571 | sub PC, RA
1572 | je >6 // No results?
1573 | lea RD, [BASE+PC]
1574 | shr PCd, 3
1575 | cmp RD, L:RB->maxstack
1576 | ja >9 // Need to grow stack?
1577 |
1578 | mov RB, BASE
1579 | sub RB, RA
1580 |5: // Move results from coroutine.
1581 | mov RD, [RA]
1582 | mov [RA+RB], RD
1583 | add RA, 8
1584 | cmp RA, KBASE
1585 | jne <5
1586 |6:
1587 |.if resume
1588 | lea RDd, [PCd+2] // nresults+1 = 1 + true + results.
1589 | mov_true ITYPE // Prepend true to results.
1590 | mov [BASE-8], ITYPE
1591 |.else
1592 | lea RDd, [PCd+1] // nresults+1 = 1 + results.
1593 |.endif
1594 |7:
1595 | mov PC, SAVE_PC
1596 | mov MULTRES, RDd
1597 |.if resume
1598 | mov RA, -8
1599 |.else
1600 | xor RAd, RAd
1601 |.endif
1602 | test PCd, FRAME_TYPE
1603 | jz ->BC_RET_Z
1604 | jmp ->vm_return
1605 |
1606 |8: // Coroutine returned with error (at co->top-1).
1607 |.if resume
1608 | mov_false ITYPE // Prepend false to results.
1609 | mov [BASE-8], ITYPE
1610 | mov RA, L:PC->top
1611 | sub RA, 8
1612 | mov L:PC->top, RA // Clear error from coroutine stack.
1613 | // Copy error message.
1614 | mov RD, [RA]
1615 | mov [BASE], RD
1616 | mov RDd, 1+2 // nresults+1 = 1 + false + error.
1617 | jmp <7
1618 |.else
1619 | mov CARG2, L:PC
1620 | mov CARG1, L:RB
1621 | call extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1622 | // Error function does not return.
1623 |.endif
1624 |
1625 |9: // Handle stack expansion on return from yield.
1626 | mov L:RA, TMP1
1627 | mov L:RA->top, KBASE // Undo coroutine stack clearing.
1628 | mov CARG2, PC
1629 | mov CARG1, L:RB
1630 | call extern lj_state_growstack // (lua_State *L, int n)
1631 | mov L:PC, TMP1
1632 | mov BASE, L:RB->base
1633 | jmp <4 // Retry the stack move.
1634 |.endmacro
1635 |
1636 | coroutine_resume_wrap 1 // coroutine.resume
1637 | coroutine_resume_wrap 0 // coroutine.wrap
1638 |
1639 |.ffunc coroutine_yield
1640 | mov L:RB, SAVE_L
1641 | test aword L:RB->cframe, CFRAME_RESUME
1642 | jz ->fff_fallback
1643 | mov L:RB->base, BASE
1644 | lea RD, [BASE+NARGS:RD*8-8]
1645 | mov L:RB->top, RD
1646 | xor RDd, RDd
1647 | mov aword L:RB->cframe, RD
1648 | mov al, LUA_YIELD
1649 | mov byte L:RB->status, al
1650 | jmp ->vm_leave_unw
1651 |
1652 |//-- Math library -------------------------------------------------------
1653 |
1654 | .ffunc_1 math_abs
1655 | mov RB, [BASE]
1656 |.if DUALNUM
1657 | checkint RB, >3
1658 | cmp RBd, 0; jns ->fff_resi
1659 | neg RBd; js >2
1660 |->fff_resbit:
1661 |->fff_resi:
1662 | setint RB
1663 |->fff_resRB:
1664 | mov PC, [BASE-8]
1665 | mov [BASE-16], RB
1666 | jmp ->fff_res1
1667 |2:
1668 | mov64 RB, U64x(41e00000,00000000) // 2^31.
1669 | jmp ->fff_resRB
1670 |3:
1671 | ja ->fff_fallback
1672 |.else
1673 | checknum RB, ->fff_fallback
1674 |.endif
1675 | shl RB, 1
1676 | shr RB, 1
1677 | mov PC, [BASE-8]
1678 | mov [BASE-16], RB
1679 | jmp ->fff_res1
1680 |
1681 |.ffunc_n math_sqrt, sqrtsd
1682 |->fff_resxmm0:
1683 | mov PC, [BASE-8]
1684 | movsd qword [BASE-16], xmm0
1685 | // fallthrough
1686 |
1687 |->fff_res1:
1688 | mov RDd, 1+1
1689 |->fff_res:
1690 | mov MULTRES, RDd
1691 |->fff_res_:
1692 | test PCd, FRAME_TYPE
1693 | jnz >7
1694 |5:
1695 | cmp PC_RB, RDL // More results expected?
1696 | ja >6
1697 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1698 | movzx RAd, PC_RA
1699 | neg RA
1700 | lea BASE, [BASE+RA*8-16] // base = base - (RA+2)*8
1701 | ins_next
1702 |
1703 |6: // Fill up results with nil.
1704 | mov aword [BASE+RD*8-24], LJ_TNIL
1705 | add RD, 1
1706 | jmp <5
1707 |
1708 |7: // Non-standard return case.
1709 | mov RA, -16 // Results start at BASE+RA = BASE-16.
1710 | jmp ->vm_return
1711 |
1712 |.macro math_round, func
1713 | .ffunc math_ .. func
1714 |.if DUALNUM
1715 | mov RB, [BASE]
1716 | checknumx RB, ->fff_resRB, je
1717 | ja ->fff_fallback
1718 |.else
1719 | checknumtp [BASE], ->fff_fallback
1720 |.endif
1721 | movsd xmm0, qword [BASE]
1722 | call ->vm_ .. func .. _sse
1723 |.if DUALNUM
1724 | cvttsd2si RBd, xmm0
1725 | cmp RBd, 0x80000000
1726 | jne ->fff_resi
1727 | cvtsi2sd xmm1, RBd
1728 | ucomisd xmm0, xmm1
1729 | jp ->fff_resxmm0
1730 | je ->fff_resi
1731 |.endif
1732 | jmp ->fff_resxmm0
1733 |.endmacro
1734 |
1735 | math_round floor
1736 | math_round ceil
1737 |
1738 |.ffunc math_log
1739 | cmp NARGS:RDd, 1+1; jne ->fff_fallback // Exactly one argument.
1740 | checknumtp [BASE], ->fff_fallback
1741 | movsd xmm0, qword [BASE]
1742 | mov RB, BASE
1743 | call extern log
1744 | mov BASE, RB
1745 | jmp ->fff_resxmm0
1746 |
1747 |.macro math_extern, func
1748 | .ffunc_n math_ .. func
1749 | mov RB, BASE
1750 | call extern func
1751 | mov BASE, RB
1752 | jmp ->fff_resxmm0
1753 |.endmacro
1754 |
1755 |.macro math_extern2, func
1756 | .ffunc_nn math_ .. func
1757 | mov RB, BASE
1758 | call extern func
1759 | mov BASE, RB
1760 | jmp ->fff_resxmm0
1761 |.endmacro
1762 |
1763 | math_extern log10
1764 | math_extern exp
1765 | math_extern sin
1766 | math_extern cos
1767 | math_extern tan
1768 | math_extern asin
1769 | math_extern acos
1770 | math_extern atan
1771 | math_extern sinh
1772 | math_extern cosh
1773 | math_extern tanh
1774 | math_extern2 pow
1775 | math_extern2 atan2
1776 | math_extern2 fmod
1777 |
1778 |.ffunc_2 math_ldexp
1779 | checknumtp [BASE], ->fff_fallback
1780 | checknumtp [BASE+8], ->fff_fallback
1781 | fld qword [BASE+8]
1782 | fld qword [BASE]
1783 | fscale
1784 | fpop1
1785 | mov PC, [BASE-8]
1786 | fstp qword [BASE-16]
1787 | jmp ->fff_res1
1788 |
1789 |.ffunc_n math_frexp
1790 | mov RB, BASE
1791 |.if X64WIN
1792 | lea CARG2, TMP1 // Caveat: CARG2 == BASE
1793 |.else
1794 | lea CARG1, TMP1
1795 |.endif
1796 | call extern frexp
1797 | mov BASE, RB
1798 | mov RBd, TMP1d
1799 | mov PC, [BASE-8]
1800 | movsd qword [BASE-16], xmm0
1801 |.if DUALNUM
1802 | setint RB
1803 | mov [BASE-8], RB
1804 |.else
1805 | cvtsi2sd xmm1, RBd
1806 | movsd qword [BASE-8], xmm1
1807 |.endif
1808 | mov RDd, 1+2
1809 | jmp ->fff_res
1810 |
1811 |.ffunc_n math_modf
1812 | mov RB, BASE
1813 |.if X64WIN
1814 | lea CARG2, [BASE-16] // Caveat: CARG2 == BASE
1815 |.else
1816 | lea CARG1, [BASE-16]
1817 |.endif
1818 | call extern modf
1819 | mov BASE, RB
1820 | mov PC, [BASE-8]
1821 | movsd qword [BASE-8], xmm0
1822 | mov RDd, 1+2
1823 | jmp ->fff_res
1824 |
1825 |.macro math_minmax, name, cmovop, sseop
1826 | .ffunc_1 name
1827 | mov RAd, 2
1828 |.if DUALNUM
1829 | mov RB, [BASE]
1830 | checkint RB, >4
1831 |1: // Handle integers.
1832 | cmp RAd, RDd; jae ->fff_resRB
1833 | mov TMPR, [BASE+RA*8-8]
1834 | checkint TMPR, >3
1835 | cmp RBd, TMPRd
1836 | cmovop RB, TMPR
1837 | add RAd, 1
1838 | jmp <1
1839 |3:
1840 | ja ->fff_fallback
1841 | // Convert intermediate result to number and continue below.
1842 | cvtsi2sd xmm0, RBd
1843 | jmp >6
1844 |4:
1845 | ja ->fff_fallback
1846 |.else
1847 | checknumtp [BASE], ->fff_fallback
1848 |.endif
1849 |
1850 | movsd xmm0, qword [BASE]
1851 |5: // Handle numbers or integers.
1852 | cmp RAd, RDd; jae ->fff_resxmm0
1853 |.if DUALNUM
1854 | mov RB, [BASE+RA*8-8]
1855 | checknumx RB, >6, jb
1856 | ja ->fff_fallback
1857 | cvtsi2sd xmm1, RBd
1858 | jmp >7
1859 |.else
1860 | checknumtp [BASE+RA*8-8], ->fff_fallback
1861 |.endif
1862 |6:
1863 | movsd xmm1, qword [BASE+RA*8-8]
1864 |7:
1865 | sseop xmm0, xmm1
1866 | add RAd, 1
1867 | jmp <5
1868 |.endmacro
1869 |
1870 | math_minmax math_min, cmovg, minsd
1871 | math_minmax math_max, cmovl, maxsd
1872 |
1873 |//-- String library -----------------------------------------------------
1874 |
1875 |.ffunc string_byte // Only handle the 1-arg case here.
1876 | cmp NARGS:RDd, 1+1; jne ->fff_fallback
1877 | mov STR:RB, [BASE]
1878 | checkstr STR:RB, ->fff_fallback
1879 | mov PC, [BASE-8]
1880 | cmp dword STR:RB->len, 1
1881 | jb ->fff_res0 // Return no results for empty string.
1882 | movzx RBd, byte STR:RB[1]
1883 |.if DUALNUM
1884 | jmp ->fff_resi
1885 |.else
1886 | cvtsi2sd xmm0, RBd; jmp ->fff_resxmm0
1887 |.endif
1888 |
1889 |.ffunc string_char // Only handle the 1-arg case here.
1890 | ffgccheck
1891 | cmp NARGS:RDd, 1+1; jne ->fff_fallback // *Exactly* 1 arg.
1892 |.if DUALNUM
1893 | mov RB, [BASE]
1894 | checkint RB, ->fff_fallback
1895 |.else
1896 | checknumtp [BASE], ->fff_fallback
1897 | cvttsd2si RBd, qword [BASE]
1898 |.endif
1899 | cmp RBd, 255; ja ->fff_fallback
1900 | mov TMP1d, RBd
1901 | mov TMPRd, 1
1902 | lea RD, TMP1 // Points to stack. Little-endian.
1903 |->fff_newstr:
1904 | mov L:RB, SAVE_L
1905 | mov L:RB->base, BASE
1906 | mov CARG3d, TMPRd // Zero-extended to size_t.
1907 | mov CARG2, RD
1908 | mov CARG1, L:RB
1909 | mov SAVE_PC, PC
1910 | call extern lj_str_new // (lua_State *L, char *str, size_t l)
1911 |->fff_resstr:
1912 | // GCstr * returned in eax (RD).
1913 | mov BASE, L:RB->base
1914 | mov PC, [BASE-8]
1915 | settp STR:RD, LJ_TSTR
1916 | mov [BASE-16], STR:RD
1917 | jmp ->fff_res1
1918 |
1919 |.ffunc string_sub
1920 | ffgccheck
1921 | mov TMPRd, -1
1922 | cmp NARGS:RDd, 1+2; jb ->fff_fallback
1923 | jna >1
1924 |.if DUALNUM
1925 | mov TMPR, [BASE+16]
1926 | checkint TMPR, ->fff_fallback
1927 |.else
1928 | checknumtp [BASE+16], ->fff_fallback
1929 | cvttsd2si TMPRd, qword [BASE+16]
1930 |.endif
1931 |1:
1932 | mov STR:RB, [BASE]
1933 | checkstr STR:RB, ->fff_fallback
1934 |.if DUALNUM
1935 | mov ITYPE, [BASE+8]
1936 | mov RAd, ITYPEd // Must clear hiword for lea below.
1937 | sar ITYPE, 47
1938 | cmp ITYPEd, LJ_TISNUM
1939 | jne ->fff_fallback
1940 |.else
1941 | checknumtp [BASE+8], ->fff_fallback
1942 | cvttsd2si RAd, qword [BASE+8]
1943 |.endif
1944 | mov RCd, STR:RB->len
1945 | cmp RCd, TMPRd // len < end? (unsigned compare)
1946 | jb >5
1947 |2:
1948 | test RAd, RAd // start <= 0?
1949 | jle >7
1950 |3:
1951 | sub TMPRd, RAd // start > end?
1952 | jl ->fff_emptystr
1953 | lea RD, [STR:RB+RAd+#STR-1]
1954 | add TMPRd, 1
1955 |4:
1956 | jmp ->fff_newstr
1957 |
1958 |5: // Negative end or overflow.
1959 | jl >6
1960 | lea TMPRd, [TMPRd+RCd+1] // end = end+(len+1)
1961 | jmp <2
1962 |6: // Overflow.
1963 | mov TMPRd, RCd // end = len
1964 | jmp <2
1965 |
1966 |7: // Negative start or underflow.
1967 | je >8
1968 | add RAd, RCd // start = start+(len+1)
1969 | add RAd, 1
1970 | jg <3 // start > 0?
1971 |8: // Underflow.
1972 | mov RAd, 1 // start = 1
1973 | jmp <3
1974 |
1975 |->fff_emptystr: // Range underflow.
1976 | xor TMPRd, TMPRd // Zero length. Any ptr in RD is ok.
1977 | jmp <4
1978 |
1979 |.macro ffstring_op, name
1980 | .ffunc_1 string_ .. name
1981 | ffgccheck
1982 |.if X64WIN
1983 | mov STR:TMPR, [BASE]
1984 | checkstr STR:TMPR, ->fff_fallback
1985 |.else
1986 | mov STR:CARG2, [BASE]
1987 | checkstr STR:CARG2, ->fff_fallback
1988 |.endif
1989 | mov L:RB, SAVE_L
1990 | lea SBUF:CARG1, [DISPATCH+DISPATCH_GL(tmpbuf)]
1991 | mov L:RB->base, BASE
1992 |.if X64WIN
1993 | mov STR:CARG2, STR:TMPR // Caveat: CARG2 == BASE
1994 |.endif
1995 | mov RC, SBUF:CARG1->b
1996 | mov SBUF:CARG1->L, L:RB
1997 | mov SBUF:CARG1->w, RC
1998 | mov SAVE_PC, PC
1999 | call extern lj_buf_putstr_ .. name
2000 | mov CARG1, rax
2001 | call extern lj_buf_tostr
2002 | jmp ->fff_resstr
2003 |.endmacro
2004 |
2005 |ffstring_op reverse
2006 |ffstring_op lower
2007 |ffstring_op upper
2008 |
2009 |//-- Bit library --------------------------------------------------------
2010 |
2011 |.macro .ffunc_bit, name, kind, fdef
2012 | fdef name
2013 |.if kind == 2
2014 | sseconst_tobit xmm1, RB
2015 |.endif
2016 |.if DUALNUM
2017 | mov RB, [BASE]
2018 | checkint RB, >1
2019 |.if kind > 0
2020 | jmp >2
2021 |.else
2022 | jmp ->fff_resbit
2023 |.endif
2024 |1:
2025 | ja ->fff_fallback
2026 | movd xmm0, RB
2027 |.else
2028 | checknumtp [BASE], ->fff_fallback
2029 | movsd xmm0, qword [BASE]
2030 |.endif
2031 |.if kind < 2
2032 | sseconst_tobit xmm1, RB
2033 |.endif
2034 | addsd xmm0, xmm1
2035 | movd RBd, xmm0
2036 |2:
2037 |.endmacro
2038 |
2039 |.macro .ffunc_bit, name, kind
2040 | .ffunc_bit name, kind, .ffunc_1
2041 |.endmacro
2042 |
2043 |.ffunc_bit bit_tobit, 0
2044 | jmp ->fff_resbit
2045 |
2046 |.macro .ffunc_bit_op, name, ins
2047 | .ffunc_bit name, 2
2048 | mov TMPRd, NARGS:RDd // Save for fallback.
2049 | lea RD, [BASE+NARGS:RD*8-16]
2050 |1:
2051 | cmp RD, BASE
2052 | jbe ->fff_resbit
2053 |.if DUALNUM
2054 | mov RA, [RD]
2055 | checkint RA, >2
2056 | ins RBd, RAd
2057 | sub RD, 8
2058 | jmp <1
2059 |2:
2060 | ja ->fff_fallback_bit_op
2061 | movd xmm0, RA
2062 |.else
2063 | checknumtp [RD], ->fff_fallback_bit_op
2064 | movsd xmm0, qword [RD]
2065 |.endif
2066 | addsd xmm0, xmm1
2067 | movd RAd, xmm0
2068 | ins RBd, RAd
2069 | sub RD, 8
2070 | jmp <1
2071 |.endmacro
2072 |
2073 |.ffunc_bit_op bit_band, and
2074 |.ffunc_bit_op bit_bor, or
2075 |.ffunc_bit_op bit_bxor, xor
2076 |
2077 |.ffunc_bit bit_bswap, 1
2078 | bswap RBd
2079 | jmp ->fff_resbit
2080 |
2081 |.ffunc_bit bit_bnot, 1
2082 | not RBd
2083 |.if DUALNUM
2084 | jmp ->fff_resbit
2085 |.else
2086 |->fff_resbit:
2087 | cvtsi2sd xmm0, RBd
2088 | jmp ->fff_resxmm0
2089 |.endif
2090 |
2091 |->fff_fallback_bit_op:
2092 | mov NARGS:RDd, TMPRd // Restore for fallback
2093 | jmp ->fff_fallback
2094 |
2095 |.macro .ffunc_bit_sh, name, ins
2096 |.if DUALNUM
2097 | .ffunc_bit name, 1, .ffunc_2
2098 | // Note: no inline conversion from number for 2nd argument!
2099 | mov RA, [BASE+8]
2100 | checkint RA, ->fff_fallback
2101 |.else
2102 | .ffunc_nn name
2103 | sseconst_tobit xmm2, RB
2104 | addsd xmm0, xmm2
2105 | addsd xmm1, xmm2
2106 | movd RBd, xmm0
2107 | movd RAd, xmm1
2108 |.endif
2109 | ins RBd, cl // Assumes RA is ecx.
2110 | jmp ->fff_resbit
2111 |.endmacro
2112 |
2113 |.ffunc_bit_sh bit_lshift, shl
2114 |.ffunc_bit_sh bit_rshift, shr
2115 |.ffunc_bit_sh bit_arshift, sar
2116 |.ffunc_bit_sh bit_rol, rol
2117 |.ffunc_bit_sh bit_ror, ror
2118 |
2119 |//-----------------------------------------------------------------------
2120 |
2121 |->fff_fallback_2:
2122 | mov NARGS:RDd, 1+2 // Other args are ignored, anyway.
2123 | jmp ->fff_fallback
2124 |->fff_fallback_1:
2125 | mov NARGS:RDd, 1+1 // Other args are ignored, anyway.
2126 |->fff_fallback: // Call fast function fallback handler.
2127 | // BASE = new base, RD = nargs+1
2128 | mov L:RB, SAVE_L
2129 | mov PC, [BASE-8] // Fallback may overwrite PC.
2130 | mov SAVE_PC, PC // Redundant (but a defined value).
2131 | mov L:RB->base, BASE
2132 | lea RD, [BASE+NARGS:RD*8-8]
2133 | lea RA, [RD+8*LUA_MINSTACK] // Ensure enough space for handler.
2134 | mov L:RB->top, RD
2135 | mov CFUNC:RD, [BASE-16]
2136 | cleartp CFUNC:RD
2137 | cmp RA, L:RB->maxstack
2138 | ja >5 // Need to grow stack.
2139 | mov CARG1, L:RB
2140 | call aword CFUNC:RD->f // (lua_State *L)
2141 | mov BASE, L:RB->base
2142 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
2143 | test RDd, RDd; jg ->fff_res // Returned nresults+1?
2144 |1:
2145 | mov RA, L:RB->top
2146 | sub RA, BASE
2147 | shr RAd, 3
2148 | test RDd, RDd
2149 | lea NARGS:RDd, [RAd+1]
2150 | mov LFUNC:RB, [BASE-16]
2151 | jne ->vm_call_tail // Returned -1?
2152 | cleartp LFUNC:RB
2153 | ins_callt // Returned 0: retry fast path.
2154 |
2155 |// Reconstruct previous base for vmeta_call during tailcall.
2156 |->vm_call_tail:
2157 | mov RA, BASE
2158 | test PCd, FRAME_TYPE
2159 | jnz >3
2160 | movzx RBd, PC_RA
2161 | neg RB
2162 | lea BASE, [BASE+RB*8-16] // base = base - (RB+2)*8
2163 | jmp ->vm_call_dispatch // Resolve again for tailcall.
2164 |3:
2165 | mov RB, PC
2166 | and RB, -8
2167 | sub BASE, RB
2168 | jmp ->vm_call_dispatch // Resolve again for tailcall.
2169 |
2170 |5: // Grow stack for fallback handler.
2171 | mov CARG2d, LUA_MINSTACK
2172 | mov CARG1, L:RB
2173 | call extern lj_state_growstack // (lua_State *L, int n)
2174 | mov BASE, L:RB->base
2175 | xor RDd, RDd // Simulate a return 0.
2176 | jmp <1 // Dumb retry (goes through ff first).
2177 |
2178 |->fff_gcstep: // Call GC step function.
2179 | // BASE = new base, RD = nargs+1
2180 | pop RB // Must keep stack at same level.
2181 | mov TMP1, RB // Save return address
2182 | mov L:RB, SAVE_L
2183 | mov SAVE_PC, PC // Redundant (but a defined value).
2184 | mov L:RB->base, BASE
2185 | lea RD, [BASE+NARGS:RD*8-8]
2186 | mov CARG1, L:RB
2187 | mov L:RB->top, RD
2188 | call extern lj_gc_step // (lua_State *L)
2189 | mov BASE, L:RB->base
2190 | mov RD, L:RB->top
2191 | sub RD, BASE
2192 | shr RDd, 3
2193 | add NARGS:RDd, 1
2194 | mov RB, TMP1
2195 | push RB // Restore return address.
2196 | ret
2197 |
2198 |//-----------------------------------------------------------------------
2199 |//-- Special dispatch targets -------------------------------------------
2200 |//-----------------------------------------------------------------------
2201 |
2202 |->vm_record: // Dispatch target for recording phase.
2203 |.if JIT
2204 | movzx RDd, byte [DISPATCH+DISPATCH_GL(hookmask)]
2205 | test RDL, HOOK_VMEVENT // No recording while in vmevent.
2206 | jnz >5
2207 | // Decrement the hookcount for consistency, but always do the call.
2208 | test RDL, HOOK_ACTIVE
2209 | jnz >1
2210 | test RDL, LUA_MASKLINE|LUA_MASKCOUNT
2211 | jz >1
2212 | dec dword [DISPATCH+DISPATCH_GL(hookcount)]
2213 | jmp >1
2214 |.endif
2215 |
2216 |->vm_rethook: // Dispatch target for return hooks.
2217 | movzx RDd, byte [DISPATCH+DISPATCH_GL(hookmask)]
2218 | test RDL, HOOK_ACTIVE // Hook already active?
2219 | jnz >5
2220 | jmp >1
2221 |
2222 |->vm_inshook: // Dispatch target for instr/line hooks.
2223 | movzx RDd, byte [DISPATCH+DISPATCH_GL(hookmask)]
2224 | test RDL, HOOK_ACTIVE // Hook already active?
2225 | jnz >5
2226 |
2227 | test RDL, LUA_MASKLINE|LUA_MASKCOUNT
2228 | jz >5
2229 | dec dword [DISPATCH+DISPATCH_GL(hookcount)]
2230 | jz >1
2231 | test RDL, LUA_MASKLINE
2232 | jz >5
2233 |1:
2234 | mov L:RB, SAVE_L
2235 | mov L:RB->base, BASE
2236 | mov CARG2, PC // Caveat: CARG2 == BASE
2237 | mov CARG1, L:RB
2238 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
2239 | call extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
2240 |3:
2241 | mov BASE, L:RB->base
2242 |4:
2243 | movzx RAd, PC_RA
2244 |5:
2245 | movzx OP, PC_OP
2246 | movzx RDd, PC_RD
2247 | jmp aword [DISPATCH+OP*8+GG_DISP2STATIC] // Re-dispatch to static ins.
2248 |
2249 |->cont_hook: // Continue from hook yield.
2250 | add PC, 4
2251 | mov RA, [RB-40]
2252 | mov MULTRES, RAd // Restore MULTRES for *M ins.
2253 | jmp <4
2254 |
2255 |->vm_hotloop: // Hot loop counter underflow.
2256 |.if JIT
2257 | mov LFUNC:RB, [BASE-16] // Same as curr_topL(L).
2258 | cleartp LFUNC:RB
2259 | mov RB, LFUNC:RB->pc
2260 | movzx RDd, byte [RB+PC2PROTO(framesize)]
2261 | lea RD, [BASE+RD*8]
2262 | mov L:RB, SAVE_L
2263 | mov L:RB->base, BASE
2264 | mov L:RB->top, RD
2265 | mov CARG2, PC
2266 | lea CARG1, [DISPATCH+GG_DISP2J]
2267 | mov aword [DISPATCH+DISPATCH_J(L)], L:RB
2268 | mov SAVE_PC, PC
2269 | call extern lj_trace_hot // (jit_State *J, const BCIns *pc)
2270 | jmp <3
2271 |.endif
2272 |
2273 |->vm_callhook: // Dispatch target for call hooks.
2274 | mov SAVE_PC, PC
2275 |.if JIT
2276 | jmp >1
2277 |.endif
2278 |
2279 |->vm_hotcall: // Hot call counter underflow.
2280 |.if JIT
2281 | mov SAVE_PC, PC
2282 | or PC, 1 // Marker for hot call.
2283 |1:
2284 |.endif
2285 | lea RD, [BASE+NARGS:RD*8-8]
2286 | mov L:RB, SAVE_L
2287 | mov L:RB->base, BASE
2288 | mov L:RB->top, RD
2289 | mov CARG2, PC
2290 | mov CARG1, L:RB
2291 | call extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
2292 | // ASMFunction returned in eax/rax (RD).
2293 | mov SAVE_PC, 0 // Invalidate for subsequent line hook.
2294 |.if JIT
2295 | and PC, -2
2296 |.endif
2297 | mov BASE, L:RB->base
2298 | mov RA, RD
2299 | mov RD, L:RB->top
2300 | sub RD, BASE
2301 | mov RB, RA
2302 | movzx RAd, PC_RA
2303 | shr RDd, 3
2304 | add NARGS:RDd, 1
2305 | jmp RB
2306 |
2307 |->cont_stitch: // Trace stitching.
2308 |.if JIT
2309 | // BASE = base, RC = result, RB = mbase
2310 | mov TRACE:ITYPE, [RB-40] // Save previous trace.
2311 | cleartp TRACE:ITYPE
2312 | mov TMPRd, MULTRES
2313 | movzx RAd, PC_RA
2314 | lea RA, [BASE+RA*8] // Call base.
2315 | sub TMPRd, 1
2316 | jz >2
2317 |1: // Move results down.
2318 | mov RB, [RC]
2319 | mov [RA], RB
2320 | add RC, 8
2321 | add RA, 8
2322 | sub TMPRd, 1
2323 | jnz <1
2324 |2:
2325 | movzx RCd, PC_RA
2326 | movzx RBd, PC_RB
2327 | add RC, RB
2328 | lea RC, [BASE+RC*8-8]
2329 |3:
2330 | cmp RC, RA
2331 | ja >9 // More results wanted?
2332 |
2333 | test TRACE:ITYPE, TRACE:ITYPE
2334 | jz ->cont_nop
2335 | movzx RBd, word TRACE:ITYPE->traceno
2336 | movzx RDd, word TRACE:ITYPE->link
2337 | cmp RDd, RBd
2338 | je ->cont_nop // Blacklisted.
2339 | test RDd, RDd
2340 | jne =>BC_JLOOP // Jump to stitched trace.
2341 |
2342 | // Stitch a new trace to the previous trace.
2343 | mov [DISPATCH+DISPATCH_J(exitno)], RB
2344 | mov L:RB, SAVE_L
2345 | mov L:RB->base, BASE
2346 | mov CARG2, PC
2347 | lea CARG1, [DISPATCH+GG_DISP2J]
2348 | mov aword [DISPATCH+DISPATCH_J(L)], L:RB
2349 | call extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
2350 | mov BASE, L:RB->base
2351 | jmp ->cont_nop
2352 |
2353 |9: // Fill up results with nil.
2354 | mov aword [RA], LJ_TNIL
2355 | add RA, 8
2356 | jmp <3
2357 |.endif
2358 |
2359 |->vm_profhook: // Dispatch target for profiler hook.
2360#if LJ_HASPROFILE
2361 | mov L:RB, SAVE_L
2362 | mov L:RB->base, BASE
2363 | mov CARG2, PC // Caveat: CARG2 == BASE
2364 | mov CARG1, L:RB
2365 | call extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
2366 | mov BASE, L:RB->base
2367 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2368 | sub PC, 4
2369 | jmp ->cont_nop
2370#endif
2371 |
2372 |//-----------------------------------------------------------------------
2373 |//-- Trace exit handler -------------------------------------------------
2374 |//-----------------------------------------------------------------------
2375 |
2376 |// Called from an exit stub with the exit number on the stack.
2377 |// The 16 bit exit number is stored with two (sign-extended) push imm8.
2378 |->vm_exit_handler:
2379 |.if JIT
2380 | push r13; push r12
2381 | push r11; push r10; push r9; push r8
2382 | push rdi; push rsi; push rbp; lea rbp, [rsp+88]; push rbp
2383 | push rbx; push rdx; push rcx; push rax
2384 | movzx RCd, byte [rbp-8] // Reconstruct exit number.
2385 | mov RCH, byte [rbp-16]
2386 | mov [rbp-8], r15; mov [rbp-16], r14
2387 | // DISPATCH is preserved on-trace in LJ_GC64 mode.
2388 | mov RAd, [DISPATCH+DISPATCH_GL(vmstate)] // Get trace number.
2389 | set_vmstate EXIT
2390 | mov [DISPATCH+DISPATCH_J(exitno)], RCd
2391 | mov [DISPATCH+DISPATCH_J(parent)], RAd
2392 |.if X64WIN
2393 | sub rsp, 16*8+4*8 // Room for SSE regs + save area.
2394 |.else
2395 | sub rsp, 16*8 // Room for SSE regs.
2396 |.endif
2397 | add rbp, -128
2398 | movsd qword [rbp-8], xmm15; movsd qword [rbp-16], xmm14
2399 | movsd qword [rbp-24], xmm13; movsd qword [rbp-32], xmm12
2400 | movsd qword [rbp-40], xmm11; movsd qword [rbp-48], xmm10
2401 | movsd qword [rbp-56], xmm9; movsd qword [rbp-64], xmm8
2402 | movsd qword [rbp-72], xmm7; movsd qword [rbp-80], xmm6
2403 | movsd qword [rbp-88], xmm5; movsd qword [rbp-96], xmm4
2404 | movsd qword [rbp-104], xmm3; movsd qword [rbp-112], xmm2
2405 | movsd qword [rbp-120], xmm1; movsd qword [rbp-128], xmm0
2406 | // Caveat: RB is rbp.
2407 | mov L:RB, [DISPATCH+DISPATCH_GL(cur_L)]
2408 | mov BASE, [DISPATCH+DISPATCH_GL(jit_base)]
2409 | mov aword [DISPATCH+DISPATCH_J(L)], L:RB
2410 | mov L:RB->base, BASE
2411 |.if X64WIN
2412 | lea CARG2, [rsp+4*8]
2413 |.else
2414 | mov CARG2, rsp
2415 |.endif
2416 | lea CARG1, [DISPATCH+GG_DISP2J]
2417 | mov qword [DISPATCH+DISPATCH_GL(jit_base)], 0
2418 | call extern lj_trace_exit // (jit_State *J, ExitState *ex)
2419 | // MULTRES or negated error code returned in eax (RD).
2420 | mov RA, L:RB->cframe
2421 | and RA, CFRAME_RAWMASK
2422 | mov [RA+CFRAME_OFS_L], L:RB // Set SAVE_L (on-trace resume/yield).
2423 | mov BASE, L:RB->base
2424 | mov PC, [RA+CFRAME_OFS_PC] // Get SAVE_PC.
2425 | jmp >1
2426 |.endif
2427 |->vm_exit_interp:
2428 | // RD = MULTRES or negated error code, BASE, PC and DISPATCH set.
2429 |.if JIT
2430 | // Restore additional callee-save registers only used in compiled code.
2431 |.if X64WIN
2432 | lea RA, [rsp+10*16+4*8]
2433 |1:
2434 | movdqa xmm15, [RA-10*16]
2435 | movdqa xmm14, [RA-9*16]
2436 | movdqa xmm13, [RA-8*16]
2437 | movdqa xmm12, [RA-7*16]
2438 | movdqa xmm11, [RA-6*16]
2439 | movdqa xmm10, [RA-5*16]
2440 | movdqa xmm9, [RA-4*16]
2441 | movdqa xmm8, [RA-3*16]
2442 | movdqa xmm7, [RA-2*16]
2443 | mov rsp, RA // Reposition stack to C frame.
2444 | movdqa xmm6, [RA-1*16]
2445 | mov r15, CSAVE_1
2446 | mov r14, CSAVE_2
2447 | mov r13, CSAVE_3
2448 | mov r12, CSAVE_4
2449 |.else
2450 | lea RA, [rsp+16]
2451 |1:
2452 | mov r13, [RA-8]
2453 | mov r12, [RA]
2454 | mov rsp, RA // Reposition stack to C frame.
2455 |.endif
2456 | cmp RDd, -LUA_ERRERR; jae >9 // Check for error from exit.
2457 | mov L:RB, SAVE_L
2458 | mov MULTRES, RDd
2459 | mov LFUNC:KBASE, [BASE-16]
2460 | cleartp LFUNC:KBASE
2461 | mov KBASE, LFUNC:KBASE->pc
2462 | mov KBASE, [KBASE+PC2PROTO(k)]
2463 | mov L:RB->base, BASE
2464 | mov qword [DISPATCH+DISPATCH_GL(jit_base)], 0
2465 | set_vmstate INTERP
2466 | // Modified copy of ins_next which handles function header dispatch, too.
2467 | mov RCd, [PC]
2468 | movzx RAd, RCH
2469 | movzx OP, RCL
2470 | add PC, 4
2471 | shr RCd, 16
2472 | cmp MULTRES, -17 // Static dispatch?
2473 | je >5
2474 | cmp OP, BC_FUNCF // Function header?
2475 | jb >3
2476 | cmp OP, BC_FUNCC+2 // Fast function?
2477 | jae >4
2478 |2:
2479 | mov RCd, MULTRES // RC/RD holds nres+1.
2480 |3:
2481 | jmp aword [DISPATCH+OP*8]
2482 |
2483 |4: // Check frame below fast function.
2484 | mov RC, [BASE-8]
2485 | test RCd, FRAME_TYPE
2486 | jnz <2 // Trace stitching continuation?
2487 | // Otherwise set KBASE for Lua function below fast function.
2488 | movzx RCd, byte [RC-3]
2489 | neg RC
2490 | mov LFUNC:KBASE, [BASE+RC*8-32]
2491 | cleartp LFUNC:KBASE
2492 | mov KBASE, LFUNC:KBASE->pc
2493 | mov KBASE, [KBASE+PC2PROTO(k)]
2494 | jmp <2
2495 |
2496 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2497 | mov RA, [DISPATCH+DISPATCH_J(trace)]
2498 | mov TRACE:RA, [RA+RD*8]
2499 | mov RCd, TRACE:RA->startins
2500 | movzx RAd, RCH
2501 | movzx OP, RCL
2502 | shr RCd, 16
2503 | jmp aword [DISPATCH+OP*8+GG_DISP2STATIC]
2504 |
2505 |9: // Rethrow error from the right C frame.
2506 | mov CARG2d, RDd
2507 | mov CARG1, L:RB
2508 | neg CARG2d
2509 | call extern lj_err_trace // (lua_State *L, int errcode)
2510 |.endif
2511 |
2512 |//-----------------------------------------------------------------------
2513 |//-- Math helper functions ----------------------------------------------
2514 |//-----------------------------------------------------------------------
2515 |
2516 |// FP value rounding. Called by math.floor/math.ceil fast functions
2517 |// and from JIT code. arg/ret is xmm0. xmm0-xmm3 and RD (eax) modified.
2518 |.macro vm_round, name, mode, cond
2519 |->name:
2520 |->name .. _sse:
2521 | sseconst_abs xmm2, RD
2522 | sseconst_2p52 xmm3, RD
2523 | movaps xmm1, xmm0
2524 | andpd xmm1, xmm2 // |x|
2525 | ucomisd xmm3, xmm1 // No truncation if 2^52 <= |x|.
2526 | jbe >1
2527 | andnpd xmm2, xmm0 // Isolate sign bit.
2528 |.if mode == 2 // trunc(x)?
2529 | movaps xmm0, xmm1
2530 | addsd xmm1, xmm3 // (|x| + 2^52) - 2^52
2531 | subsd xmm1, xmm3
2532 | sseconst_1 xmm3, RD
2533 | cmpsd xmm0, xmm1, 1 // |x| < result?
2534 | andpd xmm0, xmm3
2535 | subsd xmm1, xmm0 // If yes, subtract -1.
2536 | orpd xmm1, xmm2 // Merge sign bit back in.
2537 |.else
2538 | addsd xmm1, xmm3 // (|x| + 2^52) - 2^52
2539 | subsd xmm1, xmm3
2540 | orpd xmm1, xmm2 // Merge sign bit back in.
2541 | sseconst_1 xmm3, RD
2542 | .if mode == 1 // ceil(x)?
2543 | cmpsd xmm0, xmm1, 6 // x > result?
2544 | andpd xmm0, xmm3
2545 | addsd xmm1, xmm0 // If yes, add 1.
2546 | orpd xmm1, xmm2 // Merge sign bit back in (again).
2547 | .else // floor(x)?
2548 | cmpsd xmm0, xmm1, 1 // x < result?
2549 | andpd xmm0, xmm3
2550 | subsd xmm1, xmm0 // If yes, subtract 1.
2551 | .endif
2552 |.endif
2553 | movaps xmm0, xmm1
2554 |1:
2555 | ret
2556 |.endmacro
2557 |
2558 | vm_round vm_floor, 0, 1
2559 | vm_round vm_ceil, 1, JIT
2560 | vm_round vm_trunc, 2, JIT
2561 |
2562 |// FP modulo x%y. Called by BC_MOD* and vm_arith.
2563 |->vm_mod:
2564 |// Args in xmm0/xmm1, return value in xmm0.
2565 |// Caveat: xmm0-xmm5 and RC (eax) modified!
2566 | movaps xmm5, xmm0
2567 | divsd xmm0, xmm1
2568 | sseconst_abs xmm2, RD
2569 | sseconst_2p52 xmm3, RD
2570 | movaps xmm4, xmm0
2571 | andpd xmm4, xmm2 // |x/y|
2572 | ucomisd xmm3, xmm4 // No truncation if 2^52 <= |x/y|.
2573 | jbe >1
2574 | andnpd xmm2, xmm0 // Isolate sign bit.
2575 | addsd xmm4, xmm3 // (|x/y| + 2^52) - 2^52
2576 | subsd xmm4, xmm3
2577 | orpd xmm4, xmm2 // Merge sign bit back in.
2578 | sseconst_1 xmm2, RD
2579 | cmpsd xmm0, xmm4, 1 // x/y < result?
2580 | andpd xmm0, xmm2
2581 | subsd xmm4, xmm0 // If yes, subtract 1.0.
2582 | movaps xmm0, xmm5
2583 | mulsd xmm1, xmm4
2584 | subsd xmm0, xmm1
2585 | ret
2586 |1:
2587 | mulsd xmm1, xmm0
2588 | movaps xmm0, xmm5
2589 | subsd xmm0, xmm1
2590 | ret
2591 |
2592 |//-----------------------------------------------------------------------
2593 |//-- Miscellaneous functions --------------------------------------------
2594 |//-----------------------------------------------------------------------
2595 |
2596 |// int lj_vm_cpuid(uint32_t f, uint32_t res[4])
2597 |->vm_cpuid:
2598 | mov eax, CARG1d
2599 | .if X64WIN; push rsi; mov rsi, CARG2; .endif
2600 | push rbx
2601 | xor ecx, ecx
2602 | cpuid
2603 | mov [rsi], eax
2604 | mov [rsi+4], ebx
2605 | mov [rsi+8], ecx
2606 | mov [rsi+12], edx
2607 | pop rbx
2608 | .if X64WIN; pop rsi; .endif
2609 | ret
2610 |
2611 |.define NEXT_TAB, TAB:CARG1
2612 |.define NEXT_IDX, CARG2d
2613 |.define NEXT_IDXa, CARG2
2614 |.define NEXT_PTR, RC
2615 |.define NEXT_PTRd, RCd
2616 |.define NEXT_TMP, CARG3
2617 |.define NEXT_ASIZE, CARG4d
2618 |.macro NEXT_RES_IDXL, op2; lea edx, [NEXT_IDX+op2]; .endmacro
2619 |.if X64WIN
2620 |.define NEXT_RES_PTR, [rsp+aword*5]
2621 |.macro NEXT_RES_IDX, op2; add NEXT_IDX, op2; .endmacro
2622 |.else
2623 |.define NEXT_RES_PTR, [rsp+aword*1]
2624 |.macro NEXT_RES_IDX, op2; lea edx, [NEXT_IDX+op2]; .endmacro
2625 |.endif
2626 |
2627 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2628 |// Next idx returned in edx.
2629 |->vm_next:
2630 |.if JIT
2631 | mov NEXT_ASIZE, NEXT_TAB->asize
2632 |1: // Traverse array part.
2633 | cmp NEXT_IDX, NEXT_ASIZE; jae >5
2634 | mov NEXT_TMP, NEXT_TAB->array
2635 | mov NEXT_TMP, qword [NEXT_TMP+NEXT_IDX*8]
2636 | cmp NEXT_TMP, LJ_TNIL; je >2
2637 | lea NEXT_PTR, NEXT_RES_PTR
2638 | mov qword [NEXT_PTR], NEXT_TMP
2639 |.if DUALNUM
2640 | setint NEXT_TMP, NEXT_IDXa
2641 | mov qword [NEXT_PTR+qword*1], NEXT_TMP
2642 |.else
2643 | cvtsi2sd xmm0, NEXT_IDX
2644 | movsd qword [NEXT_PTR+qword*1], xmm0
2645 |.endif
2646 | NEXT_RES_IDX 1
2647 | ret
2648 |2: // Skip holes in array part.
2649 | add NEXT_IDX, 1
2650 | jmp <1
2651 |
2652 |5: // Traverse hash part.
2653 | sub NEXT_IDX, NEXT_ASIZE
2654 |6:
2655 | cmp NEXT_IDX, NEXT_TAB->hmask; ja >9
2656 | imul NEXT_PTRd, NEXT_IDX, #NODE
2657 | add NODE:NEXT_PTR, NEXT_TAB->node
2658 | cmp qword NODE:NEXT_PTR->val, LJ_TNIL; je >7
2659 | NEXT_RES_IDXL NEXT_ASIZE+1
2660 | ret
2661 |7: // Skip holes in hash part.
2662 | add NEXT_IDX, 1
2663 | jmp <6
2664 |
2665 |9: // End of iteration. Set the key to nil (not the value).
2666 | NEXT_RES_IDX NEXT_ASIZE
2667 | lea NEXT_PTR, NEXT_RES_PTR
2668 | mov qword [NEXT_PTR+qword*1], LJ_TNIL
2669 | ret
2670 |.endif
2671 |
2672 |//-----------------------------------------------------------------------
2673 |//-- Assertions ---------------------------------------------------------
2674 |//-----------------------------------------------------------------------
2675 |
2676 |->assert_bad_for_arg_type:
2677#ifdef LUA_USE_ASSERT
2678 | int3
2679#endif
2680 | int3
2681 |
2682 |//-----------------------------------------------------------------------
2683 |//-- FFI helper functions -----------------------------------------------
2684 |//-----------------------------------------------------------------------
2685 |
2686 |// Handler for callback functions. Callback slot number in ah/al.
2687 |->vm_ffi_callback:
2688 |.if FFI
2689 |.type CTSTATE, CTState, PC
2690 | saveregs_ // ebp/rbp already saved. ebp now holds global_State *.
2691 | lea DISPATCH, [ebp+GG_G2DISP]
2692 | mov CTSTATE, GL:ebp->ctype_state
2693 | movzx eax, ax
2694 | mov CTSTATE->cb.slot, eax
2695 | mov CTSTATE->cb.gpr[0], CARG1
2696 | mov CTSTATE->cb.gpr[1], CARG2
2697 | mov CTSTATE->cb.gpr[2], CARG3
2698 | mov CTSTATE->cb.gpr[3], CARG4
2699 | movsd qword CTSTATE->cb.fpr[0], xmm0
2700 | movsd qword CTSTATE->cb.fpr[1], xmm1
2701 | movsd qword CTSTATE->cb.fpr[2], xmm2
2702 | movsd qword CTSTATE->cb.fpr[3], xmm3
2703 |.if X64WIN
2704 | lea rax, [rsp+CFRAME_SIZE+4*8]
2705 |.else
2706 | lea rax, [rsp+CFRAME_SIZE]
2707 | mov CTSTATE->cb.gpr[4], CARG5
2708 | mov CTSTATE->cb.gpr[5], CARG6
2709 | movsd qword CTSTATE->cb.fpr[4], xmm4
2710 | movsd qword CTSTATE->cb.fpr[5], xmm5
2711 | movsd qword CTSTATE->cb.fpr[6], xmm6
2712 | movsd qword CTSTATE->cb.fpr[7], xmm7
2713 |.endif
2714 | mov CTSTATE->cb.stack, rax
2715 | mov CARG2, rsp
2716 | mov SAVE_PC, CTSTATE // Any value outside of bytecode is ok.
2717 | mov CARG1, CTSTATE
2718 | call extern lj_ccallback_enter // (CTState *cts, void *cf)
2719 | // lua_State * returned in eax (RD).
2720 | set_vmstate INTERP
2721 | mov BASE, L:RD->base
2722 | mov RD, L:RD->top
2723 | sub RD, BASE
2724 | mov LFUNC:RB, [BASE-16]
2725 | cleartp LFUNC:RB
2726 | shr RD, 3
2727 | add RD, 1
2728 | ins_callt
2729 |.endif
2730 |
2731 |->cont_ffi_callback: // Return from FFI callback.
2732 |.if FFI
2733 | mov L:RA, SAVE_L
2734 | mov CTSTATE, [DISPATCH+DISPATCH_GL(ctype_state)]
2735 | mov aword CTSTATE->L, L:RA
2736 | mov L:RA->base, BASE
2737 | mov L:RA->top, RB
2738 | mov CARG1, CTSTATE
2739 | mov CARG2, RC
2740 | call extern lj_ccallback_leave // (CTState *cts, TValue *o)
2741 | mov rax, CTSTATE->cb.gpr[0]
2742 | movsd xmm0, qword CTSTATE->cb.fpr[0]
2743 | jmp ->vm_leave_unw
2744 |.endif
2745 |
2746 |->vm_ffi_call: // Call C function via FFI.
2747 | // Caveat: needs special frame unwinding, see below.
2748 |.if FFI
2749 | .type CCSTATE, CCallState, rbx
2750 | push rbp; mov rbp, rsp; push rbx; mov CCSTATE, CARG1
2751 |
2752 | // Readjust stack.
2753 | mov eax, CCSTATE->spadj
2754 | sub rsp, rax
2755 |
2756 | // Copy stack slots.
2757 | movzx ecx, byte CCSTATE->nsp
2758 | sub ecx, 1
2759 | js >2
2760 |1:
2761 | mov rax, [CCSTATE+rcx*8+offsetof(CCallState, stack)]
2762 | mov [rsp+rcx*8+CCALL_SPS_EXTRA*8], rax
2763 | sub ecx, 1
2764 | jns <1
2765 |2:
2766 |
2767 | movzx eax, byte CCSTATE->nfpr
2768 | mov CARG1, CCSTATE->gpr[0]
2769 | mov CARG2, CCSTATE->gpr[1]
2770 | mov CARG3, CCSTATE->gpr[2]
2771 | mov CARG4, CCSTATE->gpr[3]
2772 |.if not X64WIN
2773 | mov CARG5, CCSTATE->gpr[4]
2774 | mov CARG6, CCSTATE->gpr[5]
2775 |.endif
2776 | test eax, eax; jz >5
2777 | movaps xmm0, CCSTATE->fpr[0]
2778 | movaps xmm1, CCSTATE->fpr[1]
2779 | movaps xmm2, CCSTATE->fpr[2]
2780 | movaps xmm3, CCSTATE->fpr[3]
2781 |.if not X64WIN
2782 | cmp eax, 4; jbe >5
2783 | movaps xmm4, CCSTATE->fpr[4]
2784 | movaps xmm5, CCSTATE->fpr[5]
2785 | movaps xmm6, CCSTATE->fpr[6]
2786 | movaps xmm7, CCSTATE->fpr[7]
2787 |.endif
2788 |5:
2789 |
2790 | call aword CCSTATE->func
2791 |
2792 | mov CCSTATE->gpr[0], rax
2793 | movaps CCSTATE->fpr[0], xmm0
2794 |.if not X64WIN
2795 | mov CCSTATE->gpr[1], rdx
2796 | movaps CCSTATE->fpr[1], xmm1
2797 |.endif
2798 |
2799 | mov rbx, [rbp-8]; leave; ret
2800 |.endif
2801 |// Note: vm_ffi_call must be the last function in this object file!
2802 |
2803 |//-----------------------------------------------------------------------
2804}
2805
2806/* Generate the code for a single instruction. */
2807static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2808{
2809 int vk = 0;
2810 |// Note: aligning all instructions does not pay off.
2811 |=>defop:
2812
2813 switch (op) {
2814
2815 /* -- Comparison ops ---------------------------------------------------- */
2816
2817 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2818
2819 |.macro jmp_comp, lt, ge, le, gt, target
2820 ||switch (op) {
2821 ||case BC_ISLT:
2822 | lt target
2823 ||break;
2824 ||case BC_ISGE:
2825 | ge target
2826 ||break;
2827 ||case BC_ISLE:
2828 | le target
2829 ||break;
2830 ||case BC_ISGT:
2831 | gt target
2832 ||break;
2833 ||default: break; /* Shut up GCC. */
2834 ||}
2835 |.endmacro
2836
2837 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2838 | // RA = src1, RD = src2, JMP with RD = target
2839 | ins_AD
2840 | mov ITYPE, [BASE+RA*8]
2841 | mov RB, [BASE+RD*8]
2842 | mov RA, ITYPE
2843 | mov RD, RB
2844 | sar ITYPE, 47
2845 | sar RB, 47
2846 |.if DUALNUM
2847 | cmp ITYPEd, LJ_TISNUM; jne >7
2848 | cmp RBd, LJ_TISNUM; jne >8
2849 | add PC, 4
2850 | cmp RAd, RDd
2851 | jmp_comp jge, jl, jg, jle, >9
2852 |6:
2853 | movzx RDd, PC_RD
2854 | branchPC RD
2855 |9:
2856 | ins_next
2857 |
2858 |7: // RA is not an integer.
2859 | ja ->vmeta_comp
2860 | // RA is a number.
2861 | cmp RBd, LJ_TISNUM; jb >1; jne ->vmeta_comp
2862 | // RA is a number, RD is an integer.
2863 | cvtsi2sd xmm0, RDd
2864 | jmp >2
2865 |
2866 |8: // RA is an integer, RD is not an integer.
2867 | ja ->vmeta_comp
2868 | // RA is an integer, RD is a number.
2869 | cvtsi2sd xmm1, RAd
2870 | movd xmm0, RD
2871 | jmp >3
2872 |.else
2873 | cmp ITYPEd, LJ_TISNUM; jae ->vmeta_comp
2874 | cmp RBd, LJ_TISNUM; jae ->vmeta_comp
2875 |.endif
2876 |1:
2877 | movd xmm0, RD
2878 |2:
2879 | movd xmm1, RA
2880 |3:
2881 | add PC, 4
2882 | ucomisd xmm0, xmm1
2883 | // Unordered: all of ZF CF PF set, ordered: PF clear.
2884 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2885 |.if DUALNUM
2886 | jmp_comp jbe, ja, jb, jae, <9
2887 | jmp <6
2888 |.else
2889 | jmp_comp jbe, ja, jb, jae, >1
2890 | movzx RDd, PC_RD
2891 | branchPC RD
2892 |1:
2893 | ins_next
2894 |.endif
2895 break;
2896
2897 case BC_ISEQV: case BC_ISNEV:
2898 vk = op == BC_ISEQV;
2899 | ins_AD // RA = src1, RD = src2, JMP with RD = target
2900 | mov RB, [BASE+RD*8]
2901 | mov ITYPE, [BASE+RA*8]
2902 | add PC, 4
2903 | mov RD, RB
2904 | mov RA, ITYPE
2905 | sar RB, 47
2906 | sar ITYPE, 47
2907 |.if DUALNUM
2908 | cmp RBd, LJ_TISNUM; jne >7
2909 | cmp ITYPEd, LJ_TISNUM; jne >8
2910 | cmp RDd, RAd
2911 if (vk) {
2912 | jne >9
2913 } else {
2914 | je >9
2915 }
2916 | movzx RDd, PC_RD
2917 | branchPC RD
2918 |9:
2919 | ins_next
2920 |
2921 |7: // RD is not an integer.
2922 | ja >5
2923 | // RD is a number.
2924 | movd xmm1, RD
2925 | cmp ITYPEd, LJ_TISNUM; jb >1; jne >5
2926 | // RD is a number, RA is an integer.
2927 | cvtsi2sd xmm0, RAd
2928 | jmp >2
2929 |
2930 |8: // RD is an integer, RA is not an integer.
2931 | ja >5
2932 | // RD is an integer, RA is a number.
2933 | cvtsi2sd xmm1, RDd
2934 | jmp >1
2935 |
2936 |.else
2937 | cmp RBd, LJ_TISNUM; jae >5
2938 | cmp ITYPEd, LJ_TISNUM; jae >5
2939 | movd xmm1, RD
2940 |.endif
2941 |1:
2942 | movd xmm0, RA
2943 |2:
2944 | ucomisd xmm0, xmm1
2945 |4:
2946 iseqne_fp:
2947 if (vk) {
2948 | jp >2 // Unordered means not equal.
2949 | jne >2
2950 } else {
2951 | jp >2 // Unordered means not equal.
2952 | je >1
2953 }
2954 iseqne_end:
2955 if (vk) {
2956 |1: // EQ: Branch to the target.
2957 | movzx RDd, PC_RD
2958 | branchPC RD
2959 |2: // NE: Fallthrough to next instruction.
2960 |.if not FFI
2961 |3:
2962 |.endif
2963 } else {
2964 |.if not FFI
2965 |3:
2966 |.endif
2967 |2: // NE: Branch to the target.
2968 | movzx RDd, PC_RD
2969 | branchPC RD
2970 |1: // EQ: Fallthrough to next instruction.
2971 }
2972 if (LJ_DUALNUM && (op == BC_ISEQV || op == BC_ISNEV ||
2973 op == BC_ISEQN || op == BC_ISNEN)) {
2974 | jmp <9
2975 } else {
2976 | ins_next
2977 }
2978 |
2979 if (op == BC_ISEQV || op == BC_ISNEV) {
2980 |5: // Either or both types are not numbers.
2981 |.if FFI
2982 | cmp RBd, LJ_TCDATA; je ->vmeta_equal_cd
2983 | cmp ITYPEd, LJ_TCDATA; je ->vmeta_equal_cd
2984 |.endif
2985 | cmp RA, RD
2986 | je <1 // Same GCobjs or pvalues?
2987 | cmp RBd, ITYPEd
2988 | jne <2 // Not the same type?
2989 | cmp RBd, LJ_TISTABUD
2990 | ja <2 // Different objects and not table/ud?
2991 |
2992 | // Different tables or userdatas. Need to check __eq metamethod.
2993 | // Field metatable must be at same offset for GCtab and GCudata!
2994 | cleartp TAB:RA
2995 | mov TAB:RB, TAB:RA->metatable
2996 | test TAB:RB, TAB:RB
2997 | jz <2 // No metatable?
2998 | test byte TAB:RB->nomm, 1<<MM_eq
2999 | jnz <2 // Or 'no __eq' flag set?
3000 if (vk) {
3001 | xor RBd, RBd // ne = 0
3002 } else {
3003 | mov RBd, 1 // ne = 1
3004 }
3005 | jmp ->vmeta_equal // Handle __eq metamethod.
3006 } else {
3007 |.if FFI
3008 |3:
3009 | cmp ITYPEd, LJ_TCDATA
3010 if (LJ_DUALNUM && vk) {
3011 | jne <9
3012 } else {
3013 | jne <2
3014 }
3015 | jmp ->vmeta_equal_cd
3016 |.endif
3017 }
3018 break;
3019 case BC_ISEQS: case BC_ISNES:
3020 vk = op == BC_ISEQS;
3021 | ins_AND // RA = src, RD = str const, JMP with RD = target
3022 | mov RB, [BASE+RA*8]
3023 | add PC, 4
3024 | checkstr RB, >3
3025 | cmp RB, [KBASE+RD*8]
3026 iseqne_test:
3027 if (vk) {
3028 | jne >2
3029 } else {
3030 | je >1
3031 }
3032 goto iseqne_end;
3033 case BC_ISEQN: case BC_ISNEN:
3034 vk = op == BC_ISEQN;
3035 | ins_AD // RA = src, RD = num const, JMP with RD = target
3036 | mov RB, [BASE+RA*8]
3037 | add PC, 4
3038 |.if DUALNUM
3039 | checkint RB, >7
3040 | mov RD, [KBASE+RD*8]
3041 | checkint RD, >8
3042 | cmp RBd, RDd
3043 if (vk) {
3044 | jne >9
3045 } else {
3046 | je >9
3047 }
3048 | movzx RDd, PC_RD
3049 | branchPC RD
3050 |9:
3051 | ins_next
3052 |
3053 |7: // RA is not an integer.
3054 | ja >3
3055 | // RA is a number.
3056 | mov RD, [KBASE+RD*8]
3057 | checkint RD, >1
3058 | // RA is a number, RD is an integer.
3059 | cvtsi2sd xmm0, RDd
3060 | jmp >2
3061 |
3062 |8: // RA is an integer, RD is a number.
3063 | cvtsi2sd xmm0, RBd
3064 | movd xmm1, RD
3065 | ucomisd xmm0, xmm1
3066 | jmp >4
3067 |1:
3068 | movd xmm0, RD
3069 |.else
3070 | checknum RB, >3
3071 |1:
3072 | movsd xmm0, qword [KBASE+RD*8]
3073 |.endif
3074 |2:
3075 | ucomisd xmm0, qword [BASE+RA*8]
3076 |4:
3077 goto iseqne_fp;
3078 case BC_ISEQP: case BC_ISNEP:
3079 vk = op == BC_ISEQP;
3080 | ins_AND // RA = src, RD = primitive type (~), JMP with RD = target
3081 | mov RB, [BASE+RA*8]
3082 | sar RB, 47
3083 | add PC, 4
3084 | cmp RBd, RDd
3085 if (!LJ_HASFFI) goto iseqne_test;
3086 if (vk) {
3087 | jne >3
3088 | movzx RDd, PC_RD
3089 | branchPC RD
3090 |2:
3091 | ins_next
3092 |3:
3093 | cmp RBd, LJ_TCDATA; jne <2
3094 | jmp ->vmeta_equal_cd
3095 } else {
3096 | je >2
3097 | cmp RBd, LJ_TCDATA; je ->vmeta_equal_cd
3098 | movzx RDd, PC_RD
3099 | branchPC RD
3100 |2:
3101 | ins_next
3102 }
3103 break;
3104
3105 /* -- Unary test and copy ops ------------------------------------------- */
3106
3107 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
3108 | ins_AD // RA = dst or unused, RD = src, JMP with RD = target
3109 | mov ITYPE, [BASE+RD*8]
3110 | add PC, 4
3111 if (op == BC_ISTC || op == BC_ISFC) {
3112 | mov RB, ITYPE
3113 }
3114 | sar ITYPE, 47
3115 | cmp ITYPEd, LJ_TISTRUECOND
3116 if (op == BC_IST || op == BC_ISTC) {
3117 | jae >1
3118 } else {
3119 | jb >1
3120 }
3121 if (op == BC_ISTC || op == BC_ISFC) {
3122 | mov [BASE+RA*8], RB
3123 }
3124 | movzx RDd, PC_RD
3125 | branchPC RD
3126 |1: // Fallthrough to the next instruction.
3127 | ins_next
3128 break;
3129
3130 case BC_ISTYPE:
3131 | ins_AD // RA = src, RD = -type
3132 | mov RB, [BASE+RA*8]
3133 | sar RB, 47
3134 | add RBd, RDd
3135 | jne ->vmeta_istype
3136 | ins_next
3137 break;
3138 case BC_ISNUM:
3139 | ins_AD // RA = src, RD = -(TISNUM-1)
3140 | checknumtp [BASE+RA*8], ->vmeta_istype
3141 | ins_next
3142 break;
3143
3144 /* -- Unary ops --------------------------------------------------------- */
3145
3146 case BC_MOV:
3147 | ins_AD // RA = dst, RD = src
3148 | mov RB, [BASE+RD*8]
3149 | mov [BASE+RA*8], RB
3150 | ins_next_
3151 break;
3152 case BC_NOT:
3153 | ins_AD // RA = dst, RD = src
3154 | mov RB, [BASE+RD*8]
3155 | sar RB, 47
3156 | mov RCd, 2
3157 | cmp RB, LJ_TISTRUECOND
3158 | sbb RCd, 0
3159 | shl RC, 47
3160 | not RC
3161 | mov [BASE+RA*8], RC
3162 | ins_next
3163 break;
3164 case BC_UNM:
3165 | ins_AD // RA = dst, RD = src
3166 | mov RB, [BASE+RD*8]
3167 |.if DUALNUM
3168 | checkint RB, >5
3169 | neg RBd
3170 | jo >4
3171 | setint RB
3172 |9:
3173 | mov [BASE+RA*8], RB
3174 | ins_next
3175 |4:
3176 | mov64 RB, U64x(41e00000,00000000) // 2^31.
3177 | jmp <9
3178 |5:
3179 | ja ->vmeta_unm
3180 |.else
3181 | checknum RB, ->vmeta_unm
3182 |.endif
3183 | mov64 RD, U64x(80000000,00000000)
3184 | xor RB, RD
3185 |.if DUALNUM
3186 | jmp <9
3187 |.else
3188 | mov [BASE+RA*8], RB
3189 | ins_next
3190 |.endif
3191 break;
3192 case BC_LEN:
3193 | ins_AD // RA = dst, RD = src
3194 | mov RD, [BASE+RD*8]
3195 | checkstr RD, >2
3196 |.if DUALNUM
3197 | mov RDd, dword STR:RD->len
3198 |1:
3199 | setint RD
3200 | mov [BASE+RA*8], RD
3201 |.else
3202 | xorps xmm0, xmm0
3203 | cvtsi2sd xmm0, dword STR:RD->len
3204 |1:
3205 | movsd qword [BASE+RA*8], xmm0
3206 |.endif
3207 | ins_next
3208 |2:
3209 | cmp ITYPEd, LJ_TTAB; jne ->vmeta_len
3210 | mov TAB:CARG1, TAB:RD
3211#if LJ_52
3212 | mov TAB:RB, TAB:RD->metatable
3213 | cmp TAB:RB, 0
3214 | jnz >9
3215 |3:
3216#endif
3217 |->BC_LEN_Z:
3218 | mov RB, BASE // Save BASE.
3219 | call extern lj_tab_len // (GCtab *t)
3220 | // Length of table returned in eax (RD).
3221 |.if DUALNUM
3222 | // Nothing to do.
3223 |.else
3224 | cvtsi2sd xmm0, RDd
3225 |.endif
3226 | mov BASE, RB // Restore BASE.
3227 | movzx RAd, PC_RA
3228 | jmp <1
3229#if LJ_52
3230 |9: // Check for __len.
3231 | test byte TAB:RB->nomm, 1<<MM_len
3232 | jnz <3
3233 | jmp ->vmeta_len // 'no __len' flag NOT set: check.
3234#endif
3235 break;
3236
3237 /* -- Binary ops -------------------------------------------------------- */
3238
3239 |.macro ins_arithpre, sseins, ssereg
3240 | ins_ABC
3241 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3242 ||switch (vk) {
3243 ||case 0:
3244 | checknumtp [BASE+RB*8], ->vmeta_arith_vn
3245 | .if DUALNUM
3246 | checknumtp [KBASE+RC*8], ->vmeta_arith_vn
3247 | .endif
3248 | movsd xmm0, qword [BASE+RB*8]
3249 | sseins ssereg, qword [KBASE+RC*8]
3250 || break;
3251 ||case 1:
3252 | checknumtp [BASE+RB*8], ->vmeta_arith_nv
3253 | .if DUALNUM
3254 | checknumtp [KBASE+RC*8], ->vmeta_arith_nv
3255 | .endif
3256 | movsd xmm0, qword [KBASE+RC*8]
3257 | sseins ssereg, qword [BASE+RB*8]
3258 || break;
3259 ||default:
3260 | checknumtp [BASE+RB*8], ->vmeta_arith_vv
3261 | checknumtp [BASE+RC*8], ->vmeta_arith_vv
3262 | movsd xmm0, qword [BASE+RB*8]
3263 | sseins ssereg, qword [BASE+RC*8]
3264 || break;
3265 ||}
3266 |.endmacro
3267 |
3268 |.macro ins_arithdn, intins
3269 | ins_ABC
3270 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3271 ||switch (vk) {
3272 ||case 0:
3273 | mov RB, [BASE+RB*8]
3274 | mov RC, [KBASE+RC*8]
3275 | checkint RB, ->vmeta_arith_vno
3276 | checkint RC, ->vmeta_arith_vno
3277 | intins RBd, RCd; jo ->vmeta_arith_vno
3278 || break;
3279 ||case 1:
3280 | mov RB, [BASE+RB*8]
3281 | mov RC, [KBASE+RC*8]
3282 | checkint RB, ->vmeta_arith_nvo
3283 | checkint RC, ->vmeta_arith_nvo
3284 | intins RCd, RBd; jo ->vmeta_arith_nvo
3285 || break;
3286 ||default:
3287 | mov RB, [BASE+RB*8]
3288 | mov RC, [BASE+RC*8]
3289 | checkint RB, ->vmeta_arith_vvo
3290 | checkint RC, ->vmeta_arith_vvo
3291 | intins RBd, RCd; jo ->vmeta_arith_vvo
3292 || break;
3293 ||}
3294 ||if (vk == 1) {
3295 | setint RC
3296 | mov [BASE+RA*8], RC
3297 ||} else {
3298 | setint RB
3299 | mov [BASE+RA*8], RB
3300 ||}
3301 | ins_next
3302 |.endmacro
3303 |
3304 |.macro ins_arithpost
3305 | movsd qword [BASE+RA*8], xmm0
3306 |.endmacro
3307 |
3308 |.macro ins_arith, sseins
3309 | ins_arithpre sseins, xmm0
3310 | ins_arithpost
3311 | ins_next
3312 |.endmacro
3313 |
3314 |.macro ins_arith, intins, sseins
3315 |.if DUALNUM
3316 | ins_arithdn intins
3317 |.else
3318 | ins_arith, sseins
3319 |.endif
3320 |.endmacro
3321
3322 | // RA = dst, RB = src1 or num const, RC = src2 or num const
3323 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
3324 | ins_arith add, addsd
3325 break;
3326 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
3327 | ins_arith sub, subsd
3328 break;
3329 case BC_MULVN: case BC_MULNV: case BC_MULVV:
3330 | ins_arith imul, mulsd
3331 break;
3332 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
3333 | ins_arith divsd
3334 break;
3335 case BC_MODVN:
3336 | ins_arithpre movsd, xmm1
3337 |->BC_MODVN_Z:
3338 | call ->vm_mod
3339 | ins_arithpost
3340 | ins_next
3341 break;
3342 case BC_MODNV: case BC_MODVV:
3343 | ins_arithpre movsd, xmm1
3344 | jmp ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway.
3345 break;
3346 case BC_POW:
3347 | ins_arithpre movsd, xmm1
3348 | mov RB, BASE
3349 | call extern pow
3350 | movzx RAd, PC_RA
3351 | mov BASE, RB
3352 | ins_arithpost
3353 | ins_next
3354 break;
3355
3356 case BC_CAT:
3357 | ins_ABC // RA = dst, RB = src_start, RC = src_end
3358 | mov L:CARG1, SAVE_L
3359 | mov L:CARG1->base, BASE
3360 | lea CARG2, [BASE+RC*8]
3361 | mov CARG3d, RCd
3362 | sub CARG3d, RBd
3363 |->BC_CAT_Z:
3364 | mov L:RB, L:CARG1
3365 | mov SAVE_PC, PC
3366 | call extern lj_meta_cat // (lua_State *L, TValue *top, int left)
3367 | // NULL (finished) or TValue * (metamethod) returned in eax (RC).
3368 | mov BASE, L:RB->base
3369 | test RC, RC
3370 | jnz ->vmeta_binop
3371 | movzx RBd, PC_RB // Copy result to Stk[RA] from Stk[RB].
3372 | movzx RAd, PC_RA
3373 | mov RC, [BASE+RB*8]
3374 | mov [BASE+RA*8], RC
3375 | ins_next
3376 break;
3377
3378 /* -- Constant ops ------------------------------------------------------ */
3379
3380 case BC_KSTR:
3381 | ins_AND // RA = dst, RD = str const (~)
3382 | mov RD, [KBASE+RD*8]
3383 | settp RD, LJ_TSTR
3384 | mov [BASE+RA*8], RD
3385 | ins_next
3386 break;
3387 case BC_KCDATA:
3388 |.if FFI
3389 | ins_AND // RA = dst, RD = cdata const (~)
3390 | mov RD, [KBASE+RD*8]
3391 | settp RD, LJ_TCDATA
3392 | mov [BASE+RA*8], RD
3393 | ins_next
3394 |.endif
3395 break;
3396 case BC_KSHORT:
3397 | ins_AD // RA = dst, RD = signed int16 literal
3398 |.if DUALNUM
3399 | movsx RDd, RDW
3400 | setint RD
3401 | mov [BASE+RA*8], RD
3402 |.else
3403 | movsx RDd, RDW // Sign-extend literal.
3404 | cvtsi2sd xmm0, RDd
3405 | movsd qword [BASE+RA*8], xmm0
3406 |.endif
3407 | ins_next
3408 break;
3409 case BC_KNUM:
3410 | ins_AD // RA = dst, RD = num const
3411 | movsd xmm0, qword [KBASE+RD*8]
3412 | movsd qword [BASE+RA*8], xmm0
3413 | ins_next
3414 break;
3415 case BC_KPRI:
3416 | ins_AD // RA = dst, RD = primitive type (~)
3417 | shl RD, 47
3418 | not RD
3419 | mov [BASE+RA*8], RD
3420 | ins_next
3421 break;
3422 case BC_KNIL:
3423 | ins_AD // RA = dst_start, RD = dst_end
3424 | lea RA, [BASE+RA*8+8]
3425 | lea RD, [BASE+RD*8]
3426 | mov RB, LJ_TNIL
3427 | mov [RA-8], RB // Sets minimum 2 slots.
3428 |1:
3429 | mov [RA], RB
3430 | add RA, 8
3431 | cmp RA, RD
3432 | jbe <1
3433 | ins_next
3434 break;
3435
3436 /* -- Upvalue and function ops ------------------------------------------ */
3437
3438 case BC_UGET:
3439 | ins_AD // RA = dst, RD = upvalue #
3440 | mov LFUNC:RB, [BASE-16]
3441 | cleartp LFUNC:RB
3442 | mov UPVAL:RB, [LFUNC:RB+RD*8+offsetof(GCfuncL, uvptr)]
3443 | mov RB, UPVAL:RB->v
3444 | mov RD, [RB]
3445 | mov [BASE+RA*8], RD
3446 | ins_next
3447 break;
3448 case BC_USETV:
3449#define TV2MARKOFS \
3450 ((int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv))
3451 | ins_AD // RA = upvalue #, RD = src
3452 | mov LFUNC:RB, [BASE-16]
3453 | cleartp LFUNC:RB
3454 | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
3455 | cmp byte UPVAL:RB->closed, 0
3456 | mov RB, UPVAL:RB->v
3457 | mov RA, [BASE+RD*8]
3458 | mov [RB], RA
3459 | jz >1
3460 | // Check barrier for closed upvalue.
3461 | test byte [RB+TV2MARKOFS], LJ_GC_BLACK // isblack(uv)
3462 | jnz >2
3463 |1:
3464 | ins_next
3465 |
3466 |2: // Upvalue is black. Check if new value is collectable and white.
3467 | mov RD, RA
3468 | sar RD, 47
3469 | sub RDd, LJ_TISGCV
3470 | cmp RDd, LJ_TNUMX - LJ_TISGCV // tvisgcv(v)
3471 | jbe <1
3472 | cleartp GCOBJ:RA
3473 | test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(v)
3474 | jz <1
3475 | // Crossed a write barrier. Move the barrier forward.
3476 |.if not X64WIN
3477 | mov CARG2, RB
3478 | mov RB, BASE // Save BASE.
3479 |.else
3480 | xchg CARG2, RB // Save BASE (CARG2 == BASE).
3481 |.endif
3482 | lea GL:CARG1, [DISPATCH+GG_DISP2G]
3483 | call extern lj_gc_barrieruv // (global_State *g, TValue *tv)
3484 | mov BASE, RB // Restore BASE.
3485 | jmp <1
3486 break;
3487#undef TV2MARKOFS
3488 case BC_USETS:
3489 | ins_AND // RA = upvalue #, RD = str const (~)
3490 | mov LFUNC:RB, [BASE-16]
3491 | cleartp LFUNC:RB
3492 | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
3493 | mov STR:RA, [KBASE+RD*8]
3494 | mov RD, UPVAL:RB->v
3495 | settp STR:ITYPE, STR:RA, LJ_TSTR
3496 | mov [RD], STR:ITYPE
3497 | test byte UPVAL:RB->marked, LJ_GC_BLACK // isblack(uv)
3498 | jnz >2
3499 |1:
3500 | ins_next
3501 |
3502 |2: // Check if string is white and ensure upvalue is closed.
3503 | test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(str)
3504 | jz <1
3505 | cmp byte UPVAL:RB->closed, 0
3506 | jz <1
3507 | // Crossed a write barrier. Move the barrier forward.
3508 | mov RB, BASE // Save BASE (CARG2 == BASE).
3509 | mov CARG2, RD
3510 | lea GL:CARG1, [DISPATCH+GG_DISP2G]
3511 | call extern lj_gc_barrieruv // (global_State *g, TValue *tv)
3512 | mov BASE, RB // Restore BASE.
3513 | jmp <1
3514 break;
3515 case BC_USETN:
3516 | ins_AD // RA = upvalue #, RD = num const
3517 | mov LFUNC:RB, [BASE-16]
3518 | cleartp LFUNC:RB
3519 | movsd xmm0, qword [KBASE+RD*8]
3520 | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
3521 | mov RA, UPVAL:RB->v
3522 | movsd qword [RA], xmm0
3523 | ins_next
3524 break;
3525 case BC_USETP:
3526 | ins_AD // RA = upvalue #, RD = primitive type (~)
3527 | mov LFUNC:RB, [BASE-16]
3528 | cleartp LFUNC:RB
3529 | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
3530 | shl RD, 47
3531 | not RD
3532 | mov RA, UPVAL:RB->v
3533 | mov [RA], RD
3534 | ins_next
3535 break;
3536 case BC_UCLO:
3537 | ins_AD // RA = level, RD = target
3538 | branchPC RD // Do this first to free RD.
3539 | mov L:RB, SAVE_L
3540 | cmp aword L:RB->openupval, 0
3541 | je >1
3542 | mov L:RB->base, BASE
3543 | lea CARG2, [BASE+RA*8] // Caveat: CARG2 == BASE
3544 | mov L:CARG1, L:RB // Caveat: CARG1 == RA
3545 | call extern lj_func_closeuv // (lua_State *L, TValue *level)
3546 | mov BASE, L:RB->base
3547 |1:
3548 | ins_next
3549 break;
3550
3551 case BC_FNEW:
3552 | ins_AND // RA = dst, RD = proto const (~) (holding function prototype)
3553 | mov L:RB, SAVE_L
3554 | mov L:RB->base, BASE // Caveat: CARG2/CARG3 may be BASE.
3555 | mov CARG3, [BASE-16]
3556 | cleartp CARG3
3557 | mov CARG2, [KBASE+RD*8] // Fetch GCproto *.
3558 | mov CARG1, L:RB
3559 | mov SAVE_PC, PC
3560 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
3561 | call extern lj_func_newL_gc
3562 | // GCfuncL * returned in eax (RC).
3563 | mov BASE, L:RB->base
3564 | movzx RAd, PC_RA
3565 | settp LFUNC:RC, LJ_TFUNC
3566 | mov [BASE+RA*8], LFUNC:RC
3567 | ins_next
3568 break;
3569
3570 /* -- Table ops --------------------------------------------------------- */
3571
3572 case BC_TNEW:
3573 | ins_AD // RA = dst, RD = hbits|asize
3574 | mov L:RB, SAVE_L
3575 | mov L:RB->base, BASE
3576 | mov RA, [DISPATCH+DISPATCH_GL(gc.total)]
3577 | cmp RA, [DISPATCH+DISPATCH_GL(gc.threshold)]
3578 | mov SAVE_PC, PC
3579 | jae >5
3580 |1:
3581 | mov CARG3d, RDd
3582 | and RDd, 0x7ff
3583 | shr CARG3d, 11
3584 | cmp RDd, 0x7ff
3585 | je >3
3586 |2:
3587 | mov L:CARG1, L:RB
3588 | mov CARG2d, RDd
3589 | call extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
3590 | // Table * returned in eax (RC).
3591 | mov BASE, L:RB->base
3592 | movzx RAd, PC_RA
3593 | settp TAB:RC, LJ_TTAB
3594 | mov [BASE+RA*8], TAB:RC
3595 | ins_next
3596 |3: // Turn 0x7ff into 0x801.
3597 | mov RDd, 0x801
3598 | jmp <2
3599 |5:
3600 | mov L:CARG1, L:RB
3601 | call extern lj_gc_step_fixtop // (lua_State *L)
3602 | movzx RDd, PC_RD
3603 | jmp <1
3604 break;
3605 case BC_TDUP:
3606 | ins_AND // RA = dst, RD = table const (~) (holding template table)
3607 | mov L:RB, SAVE_L
3608 | mov RA, [DISPATCH+DISPATCH_GL(gc.total)]
3609 | mov SAVE_PC, PC
3610 | cmp RA, [DISPATCH+DISPATCH_GL(gc.threshold)]
3611 | mov L:RB->base, BASE
3612 | jae >3
3613 |2:
3614 | mov TAB:CARG2, [KBASE+RD*8] // Caveat: CARG2 == BASE
3615 | mov L:CARG1, L:RB // Caveat: CARG1 == RA
3616 | call extern lj_tab_dup // (lua_State *L, Table *kt)
3617 | // Table * returned in eax (RC).
3618 | mov BASE, L:RB->base
3619 | movzx RAd, PC_RA
3620 | settp TAB:RC, LJ_TTAB
3621 | mov [BASE+RA*8], TAB:RC
3622 | ins_next
3623 |3:
3624 | mov L:CARG1, L:RB
3625 | call extern lj_gc_step_fixtop // (lua_State *L)
3626 | movzx RDd, PC_RD // Need to reload RD.
3627 | not RD
3628 | jmp <2
3629 break;
3630
3631 case BC_GGET:
3632 | ins_AND // RA = dst, RD = str const (~)
3633 | mov LFUNC:RB, [BASE-16]
3634 | cleartp LFUNC:RB
3635 | mov TAB:RB, LFUNC:RB->env
3636 | mov STR:RC, [KBASE+RD*8]
3637 | jmp ->BC_TGETS_Z
3638 break;
3639 case BC_GSET:
3640 | ins_AND // RA = src, RD = str const (~)
3641 | mov LFUNC:RB, [BASE-16]
3642 | cleartp LFUNC:RB
3643 | mov TAB:RB, LFUNC:RB->env
3644 | mov STR:RC, [KBASE+RD*8]
3645 | jmp ->BC_TSETS_Z
3646 break;
3647
3648 case BC_TGETV:
3649 | ins_ABC // RA = dst, RB = table, RC = key
3650 | mov TAB:RB, [BASE+RB*8]
3651 | mov RC, [BASE+RC*8]
3652 | checktab TAB:RB, ->vmeta_tgetv
3653 |
3654 | // Integer key?
3655 |.if DUALNUM
3656 | checkint RC, >5
3657 |.else
3658 | // Convert number to int and back and compare.
3659 | checknum RC, >5
3660 | movd xmm0, RC
3661 | cvttsd2si RCd, xmm0
3662 | cvtsi2sd xmm1, RCd
3663 | ucomisd xmm0, xmm1
3664 | jne ->vmeta_tgetv // Generic numeric key? Use fallback.
3665 |.endif
3666 | cmp RCd, TAB:RB->asize // Takes care of unordered, too.
3667 | jae ->vmeta_tgetv // Not in array part? Use fallback.
3668 | shl RCd, 3
3669 | add RC, TAB:RB->array
3670 | // Get array slot.
3671 | mov ITYPE, [RC]
3672 | cmp ITYPE, LJ_TNIL // Avoid overwriting RB in fastpath.
3673 | je >2
3674 |1:
3675 | mov [BASE+RA*8], ITYPE
3676 | ins_next
3677 |
3678 |2: // Check for __index if table value is nil.
3679 | mov TAB:TMPR, TAB:RB->metatable
3680 | test TAB:TMPR, TAB:TMPR
3681 | jz <1
3682 | test byte TAB:TMPR->nomm, 1<<MM_index
3683 | jz ->vmeta_tgetv // 'no __index' flag NOT set: check.
3684 | jmp <1
3685 |
3686 |5: // String key?
3687 | cmp ITYPEd, LJ_TSTR; jne ->vmeta_tgetv
3688 | cleartp STR:RC
3689 | jmp ->BC_TGETS_Z
3690 break;
3691 case BC_TGETS:
3692 | ins_ABC // RA = dst, RB = table, RC = str const (~)
3693 | mov TAB:RB, [BASE+RB*8]
3694 | not RC
3695 | mov STR:RC, [KBASE+RC*8]
3696 | checktab TAB:RB, ->vmeta_tgets
3697 |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr *
3698 | mov TMPRd, TAB:RB->hmask
3699 | and TMPRd, STR:RC->sid
3700 | imul TMPRd, #NODE
3701 | add NODE:TMPR, TAB:RB->node
3702 | settp ITYPE, STR:RC, LJ_TSTR
3703 |1:
3704 | cmp NODE:TMPR->key, ITYPE
3705 | jne >4
3706 | // Get node value.
3707 | mov ITYPE, NODE:TMPR->val
3708 | cmp ITYPE, LJ_TNIL
3709 | je >5 // Key found, but nil value?
3710 |2:
3711 | mov [BASE+RA*8], ITYPE
3712 | ins_next
3713 |
3714 |4: // Follow hash chain.
3715 | mov NODE:TMPR, NODE:TMPR->next
3716 | test NODE:TMPR, NODE:TMPR
3717 | jnz <1
3718 | // End of hash chain: key not found, nil result.
3719 | mov ITYPE, LJ_TNIL
3720 |
3721 |5: // Check for __index if table value is nil.
3722 | mov TAB:TMPR, TAB:RB->metatable
3723 | test TAB:TMPR, TAB:TMPR
3724 | jz <2 // No metatable: done.
3725 | test byte TAB:TMPR->nomm, 1<<MM_index
3726 | jnz <2 // 'no __index' flag set: done.
3727 | jmp ->vmeta_tgets // Caveat: preserve STR:RC.
3728 break;
3729 case BC_TGETB:
3730 | ins_ABC // RA = dst, RB = table, RC = byte literal
3731 | mov TAB:RB, [BASE+RB*8]
3732 | checktab TAB:RB, ->vmeta_tgetb
3733 | cmp RCd, TAB:RB->asize
3734 | jae ->vmeta_tgetb
3735 | shl RCd, 3
3736 | add RC, TAB:RB->array
3737 | // Get array slot.
3738 | mov ITYPE, [RC]
3739 | cmp ITYPE, LJ_TNIL
3740 | je >2
3741 |1:
3742 | mov [BASE+RA*8], ITYPE
3743 | ins_next
3744 |
3745 |2: // Check for __index if table value is nil.
3746 | mov TAB:TMPR, TAB:RB->metatable
3747 | test TAB:TMPR, TAB:TMPR
3748 | jz <1
3749 | test byte TAB:TMPR->nomm, 1<<MM_index
3750 | jz ->vmeta_tgetb // 'no __index' flag NOT set: check.
3751 | jmp <1
3752 break;
3753 case BC_TGETR:
3754 | ins_ABC // RA = dst, RB = table, RC = key
3755 | mov TAB:RB, [BASE+RB*8]
3756 | cleartp TAB:RB
3757 |.if DUALNUM
3758 | mov RCd, dword [BASE+RC*8]
3759 |.else
3760 | cvttsd2si RCd, qword [BASE+RC*8]
3761 |.endif
3762 | cmp RCd, TAB:RB->asize
3763 | jae ->vmeta_tgetr // Not in array part? Use fallback.
3764 | shl RCd, 3
3765 | add RC, TAB:RB->array
3766 | // Get array slot.
3767 |->BC_TGETR_Z:
3768 | mov ITYPE, [RC]
3769 |->BC_TGETR2_Z:
3770 | mov [BASE+RA*8], ITYPE
3771 | ins_next
3772 break;
3773
3774 case BC_TSETV:
3775 | ins_ABC // RA = src, RB = table, RC = key
3776 | mov TAB:RB, [BASE+RB*8]
3777 | mov RC, [BASE+RC*8]
3778 | checktab TAB:RB, ->vmeta_tsetv
3779 |
3780 | // Integer key?
3781 |.if DUALNUM
3782 | checkint RC, >5
3783 |.else
3784 | // Convert number to int and back and compare.
3785 | checknum RC, >5
3786 | movd xmm0, RC
3787 | cvttsd2si RCd, xmm0
3788 | cvtsi2sd xmm1, RCd
3789 | ucomisd xmm0, xmm1
3790 | jne ->vmeta_tsetv // Generic numeric key? Use fallback.
3791 |.endif
3792 | cmp RCd, TAB:RB->asize // Takes care of unordered, too.
3793 | jae ->vmeta_tsetv
3794 | shl RCd, 3
3795 | add RC, TAB:RB->array
3796 | cmp aword [RC], LJ_TNIL
3797 | je >3 // Previous value is nil?
3798 |1:
3799 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
3800 | jnz >7
3801 |2: // Set array slot.
3802 | mov RB, [BASE+RA*8]
3803 | mov [RC], RB
3804 | ins_next
3805 |
3806 |3: // Check for __newindex if previous value is nil.
3807 | mov TAB:TMPR, TAB:RB->metatable
3808 | test TAB:TMPR, TAB:TMPR
3809 | jz <1
3810 | test byte TAB:TMPR->nomm, 1<<MM_newindex
3811 | jz ->vmeta_tsetv // 'no __newindex' flag NOT set: check.
3812 | jmp <1
3813 |
3814 |5: // String key?
3815 | cmp ITYPEd, LJ_TSTR; jne ->vmeta_tsetv
3816 | cleartp STR:RC
3817 | jmp ->BC_TSETS_Z
3818 |
3819 |7: // Possible table write barrier for the value. Skip valiswhite check.
3820 | barrierback TAB:RB, TMPR
3821 | jmp <2
3822 break;
3823 case BC_TSETS:
3824 | ins_ABC // RA = src, RB = table, RC = str const (~)
3825 | mov TAB:RB, [BASE+RB*8]
3826 | not RC
3827 | mov STR:RC, [KBASE+RC*8]
3828 | checktab TAB:RB, ->vmeta_tsets
3829 |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr *
3830 | mov TMPRd, TAB:RB->hmask
3831 | and TMPRd, STR:RC->sid
3832 | imul TMPRd, #NODE
3833 | mov byte TAB:RB->nomm, 0 // Clear metamethod cache.
3834 | add NODE:TMPR, TAB:RB->node
3835 | settp ITYPE, STR:RC, LJ_TSTR
3836 |1:
3837 | cmp NODE:TMPR->key, ITYPE
3838 | jne >5
3839 | // Ok, key found. Assumes: offsetof(Node, val) == 0
3840 | cmp aword [TMPR], LJ_TNIL
3841 | je >4 // Previous value is nil?
3842 |2:
3843 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
3844 | jnz >7
3845 |3: // Set node value.
3846 | mov ITYPE, [BASE+RA*8]
3847 | mov [TMPR], ITYPE
3848 | ins_next
3849 |
3850 |4: // Check for __newindex if previous value is nil.
3851 | mov TAB:ITYPE, TAB:RB->metatable
3852 | test TAB:ITYPE, TAB:ITYPE
3853 | jz <2
3854 | test byte TAB:ITYPE->nomm, 1<<MM_newindex
3855 | jz ->vmeta_tsets // 'no __newindex' flag NOT set: check.
3856 | jmp <2
3857 |
3858 |5: // Follow hash chain.
3859 | mov NODE:TMPR, NODE:TMPR->next
3860 | test NODE:TMPR, NODE:TMPR
3861 | jnz <1
3862 | // End of hash chain: key not found, add a new one.
3863 |
3864 | // But check for __newindex first.
3865 | mov TAB:TMPR, TAB:RB->metatable
3866 | test TAB:TMPR, TAB:TMPR
3867 | jz >6 // No metatable: continue.
3868 | test byte TAB:TMPR->nomm, 1<<MM_newindex
3869 | jz ->vmeta_tsets // 'no __newindex' flag NOT set: check.
3870 |6:
3871 | mov TMP1, ITYPE
3872 | mov L:CARG1, SAVE_L
3873 | mov L:CARG1->base, BASE
3874 | lea CARG3, TMP1
3875 | mov CARG2, TAB:RB
3876 | mov SAVE_PC, PC
3877 | call extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
3878 | // Handles write barrier for the new key. TValue * returned in eax (RC).
3879 | mov L:CARG1, SAVE_L
3880 | mov BASE, L:CARG1->base
3881 | mov TMPR, rax
3882 | movzx RAd, PC_RA
3883 | jmp <2 // Must check write barrier for value.
3884 |
3885 |7: // Possible table write barrier for the value. Skip valiswhite check.
3886 | barrierback TAB:RB, ITYPE
3887 | jmp <3
3888 break;
3889 case BC_TSETB:
3890 | ins_ABC // RA = src, RB = table, RC = byte literal
3891 | mov TAB:RB, [BASE+RB*8]
3892 | checktab TAB:RB, ->vmeta_tsetb
3893 | cmp RCd, TAB:RB->asize
3894 | jae ->vmeta_tsetb
3895 | shl RCd, 3
3896 | add RC, TAB:RB->array
3897 | cmp aword [RC], LJ_TNIL
3898 | je >3 // Previous value is nil?
3899 |1:
3900 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
3901 | jnz >7
3902 |2: // Set array slot.
3903 | mov ITYPE, [BASE+RA*8]
3904 | mov [RC], ITYPE
3905 | ins_next
3906 |
3907 |3: // Check for __newindex if previous value is nil.
3908 | mov TAB:TMPR, TAB:RB->metatable
3909 | test TAB:TMPR, TAB:TMPR
3910 | jz <1
3911 | test byte TAB:TMPR->nomm, 1<<MM_newindex
3912 | jz ->vmeta_tsetb // 'no __newindex' flag NOT set: check.
3913 | jmp <1
3914 |
3915 |7: // Possible table write barrier for the value. Skip valiswhite check.
3916 | barrierback TAB:RB, TMPR
3917 | jmp <2
3918 break;
3919 case BC_TSETR:
3920 | ins_ABC // RA = src, RB = table, RC = key
3921 | mov TAB:RB, [BASE+RB*8]
3922 | cleartp TAB:RB
3923 |.if DUALNUM
3924 | mov RC, [BASE+RC*8]
3925 |.else
3926 | cvttsd2si RCd, qword [BASE+RC*8]
3927 |.endif
3928 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
3929 | jnz >7
3930 |2:
3931 | cmp RCd, TAB:RB->asize
3932 | jae ->vmeta_tsetr
3933 | shl RCd, 3
3934 | add RC, TAB:RB->array
3935 | // Set array slot.
3936 |->BC_TSETR_Z:
3937 | mov ITYPE, [BASE+RA*8]
3938 | mov [RC], ITYPE
3939 | ins_next
3940 |
3941 |7: // Possible table write barrier for the value. Skip valiswhite check.
3942 | barrierback TAB:RB, TMPR
3943 | jmp <2
3944 break;
3945
3946 case BC_TSETM:
3947 | ins_AD // RA = base (table at base-1), RD = num const (start index)
3948 |1:
3949 | mov TMPRd, dword [KBASE+RD*8] // Integer constant is in lo-word.
3950 | lea RA, [BASE+RA*8]
3951 | mov TAB:RB, [RA-8] // Guaranteed to be a table.
3952 | cleartp TAB:RB
3953 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
3954 | jnz >7
3955 |2:
3956 | mov RDd, MULTRES
3957 | sub RDd, 1
3958 | jz >4 // Nothing to copy?
3959 | add RDd, TMPRd // Compute needed size.
3960 | cmp RDd, TAB:RB->asize
3961 | ja >5 // Doesn't fit into array part?
3962 | sub RDd, TMPRd
3963 | shl TMPRd, 3
3964 | add TMPR, TAB:RB->array
3965 |3: // Copy result slots to table.
3966 | mov RB, [RA]
3967 | add RA, 8
3968 | mov [TMPR], RB
3969 | add TMPR, 8
3970 | sub RDd, 1
3971 | jnz <3
3972 |4:
3973 | ins_next
3974 |
3975 |5: // Need to resize array part.
3976 | mov L:CARG1, SAVE_L
3977 | mov L:CARG1->base, BASE // Caveat: CARG2/CARG3 may be BASE.
3978 | mov CARG2, TAB:RB
3979 | mov CARG3d, RDd
3980 | mov L:RB, L:CARG1
3981 | mov SAVE_PC, PC
3982 | call extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3983 | mov BASE, L:RB->base
3984 | movzx RAd, PC_RA // Restore RA.
3985 | movzx RDd, PC_RD // Restore RD.
3986 | jmp <1 // Retry.
3987 |
3988 |7: // Possible table write barrier for any value. Skip valiswhite check.
3989 | barrierback TAB:RB, RD
3990 | jmp <2
3991 break;
3992
3993 /* -- Calls and vararg handling ----------------------------------------- */
3994
3995 case BC_CALL: case BC_CALLM:
3996 | ins_A_C // RA = base, (RB = nresults+1,) RC = nargs+1 | extra_nargs
3997 if (op == BC_CALLM) {
3998 | add NARGS:RDd, MULTRES
3999 }
4000 | mov LFUNC:RB, [BASE+RA*8]
4001 | checkfunc LFUNC:RB, ->vmeta_call_ra
4002 | lea BASE, [BASE+RA*8+16]
4003 | ins_call
4004 break;
4005
4006 case BC_CALLMT:
4007 | ins_AD // RA = base, RD = extra_nargs
4008 | add NARGS:RDd, MULTRES
4009 | // Fall through. Assumes BC_CALLT follows and ins_AD is a no-op.
4010 break;
4011 case BC_CALLT:
4012 | ins_AD // RA = base, RD = nargs+1
4013 | lea RA, [BASE+RA*8+16]
4014 | mov KBASE, BASE // Use KBASE for move + vmeta_call hint.
4015 | mov LFUNC:RB, [RA-16]
4016 | checktp_nc LFUNC:RB, LJ_TFUNC, ->vmeta_call
4017 |->BC_CALLT_Z:
4018 | mov PC, [BASE-8]
4019 | test PCd, FRAME_TYPE
4020 | jnz >7
4021 |1:
4022 | mov [BASE-16], LFUNC:RB // Copy func+tag down, reloaded below.
4023 | mov MULTRES, NARGS:RDd
4024 | sub NARGS:RDd, 1
4025 | jz >3
4026 |2: // Move args down.
4027 | mov RB, [RA]
4028 | add RA, 8
4029 | mov [KBASE], RB
4030 | add KBASE, 8
4031 | sub NARGS:RDd, 1
4032 | jnz <2
4033 |
4034 | mov LFUNC:RB, [BASE-16]
4035 |3:
4036 | cleartp LFUNC:RB
4037 | mov NARGS:RDd, MULTRES
4038 | cmp byte LFUNC:RB->ffid, 1 // (> FF_C) Calling a fast function?
4039 | ja >5
4040 |4:
4041 | ins_callt
4042 |
4043 |5: // Tailcall to a fast function.
4044 | test PCd, FRAME_TYPE // Lua frame below?
4045 | jnz <4
4046 | movzx RAd, PC_RA
4047 | neg RA
4048 | mov LFUNC:KBASE, [BASE+RA*8-32] // Need to prepare KBASE.
4049 | cleartp LFUNC:KBASE
4050 | mov KBASE, LFUNC:KBASE->pc
4051 | mov KBASE, [KBASE+PC2PROTO(k)]
4052 | jmp <4
4053 |
4054 |7: // Tailcall from a vararg function.
4055 | sub PC, FRAME_VARG
4056 | test PCd, FRAME_TYPEP
4057 | jnz >8 // Vararg frame below?
4058 | sub BASE, PC // Need to relocate BASE/KBASE down.
4059 | mov KBASE, BASE
4060 | mov PC, [BASE-8]
4061 | jmp <1
4062 |8:
4063 | add PCd, FRAME_VARG
4064 | jmp <1
4065 break;
4066
4067 case BC_ITERC:
4068 | ins_A // RA = base, (RB = nresults+1,) RC = nargs+1 (2+1)
4069 | lea RA, [BASE+RA*8+16] // fb = base+2
4070 | mov RB, [RA-32] // Copy state. fb[0] = fb[-4].
4071 | mov RC, [RA-24] // Copy control var. fb[1] = fb[-3].
4072 | mov [RA], RB
4073 | mov [RA+8], RC
4074 | mov LFUNC:RB, [RA-40] // Copy callable. fb[-2] = fb[-5]
4075 | mov [RA-16], LFUNC:RB
4076 | mov NARGS:RDd, 2+1 // Handle like a regular 2-arg call.
4077 | checkfunc LFUNC:RB, ->vmeta_call
4078 | mov BASE, RA
4079 | ins_call
4080 break;
4081
4082 case BC_ITERN:
4083 |.if JIT
4084 | hotloop RBd
4085 |.endif
4086 |->vm_IITERN:
4087 | ins_A // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
4088 | mov TAB:RB, [BASE+RA*8-16]
4089 | cleartp TAB:RB
4090 | mov RCd, [BASE+RA*8-8] // Get index from control var.
4091 | mov TMPRd, TAB:RB->asize
4092 | add PC, 4
4093 | mov ITYPE, TAB:RB->array
4094 |1: // Traverse array part.
4095 | cmp RCd, TMPRd; jae >5 // Index points after array part?
4096 | cmp aword [ITYPE+RC*8], LJ_TNIL; je >4
4097 |.if not DUALNUM
4098 | cvtsi2sd xmm0, RCd
4099 |.endif
4100 | // Copy array slot to returned value.
4101 | mov RB, [ITYPE+RC*8]
4102 | mov [BASE+RA*8+8], RB
4103 | // Return array index as a numeric key.
4104 |.if DUALNUM
4105 | setint ITYPE, RC
4106 | mov [BASE+RA*8], ITYPE
4107 |.else
4108 | movsd qword [BASE+RA*8], xmm0
4109 |.endif
4110 | add RCd, 1
4111 | mov [BASE+RA*8-8], RCd // Update control var.
4112 |2:
4113 | movzx RDd, PC_RD // Get target from ITERL.
4114 | branchPC RD
4115 |3:
4116 | ins_next
4117 |
4118 |4: // Skip holes in array part.
4119 | add RCd, 1
4120 | jmp <1
4121 |
4122 |5: // Traverse hash part.
4123 | sub RCd, TMPRd
4124 |6:
4125 | cmp RCd, TAB:RB->hmask; ja <3 // End of iteration? Branch to ITERL+1.
4126 | imul ITYPEd, RCd, #NODE
4127 | add NODE:ITYPE, TAB:RB->node
4128 | cmp aword NODE:ITYPE->val, LJ_TNIL; je >7
4129 | lea TMPRd, [RCd+TMPRd+1]
4130 | // Copy key and value from hash slot.
4131 | mov RB, NODE:ITYPE->key
4132 | mov RC, NODE:ITYPE->val
4133 | mov [BASE+RA*8], RB
4134 | mov [BASE+RA*8+8], RC
4135 | mov [BASE+RA*8-8], TMPRd
4136 | jmp <2
4137 |
4138 |7: // Skip holes in hash part.
4139 | add RCd, 1
4140 | jmp <6
4141 break;
4142
4143 case BC_ISNEXT:
4144 | ins_AD // RA = base, RD = target (points to ITERN)
4145 | mov CFUNC:RB, [BASE+RA*8-24]
4146 | checkfunc CFUNC:RB, >5
4147 | checktptp [BASE+RA*8-16], LJ_TTAB, >5
4148 | cmp aword [BASE+RA*8-8], LJ_TNIL; jne >5
4149 | cmp byte CFUNC:RB->ffid, FF_next_N; jne >5
4150 | branchPC RD
4151 | mov64 TMPR, ((uint64_t)LJ_KEYINDEX << 32)
4152 | mov [BASE+RA*8-8], TMPR // Initialize control var.
4153 |1:
4154 | ins_next
4155 |5: // Despecialize bytecode if any of the checks fail.
4156 | mov PC_OP, BC_JMP
4157 | branchPC RD
4158 |.if JIT
4159 | cmp byte [PC], BC_ITERN
4160 | jne >6
4161 |.endif
4162 | mov byte [PC], BC_ITERC
4163 | jmp <1
4164 |.if JIT
4165 |6: // Unpatch JLOOP.
4166 | mov RA, [DISPATCH+DISPATCH_J(trace)]
4167 | movzx RCd, word [PC+2]
4168 | mov TRACE:RA, [RA+RC*8]
4169 | mov eax, TRACE:RA->startins
4170 | mov al, BC_ITERC
4171 | mov dword [PC], eax
4172 | jmp <1
4173 |.endif
4174 break;
4175
4176 case BC_VARG:
4177 | ins_ABC // RA = base, RB = nresults+1, RC = numparams
4178 | lea TMPR, [BASE+RC*8+(16+FRAME_VARG)]
4179 | lea RA, [BASE+RA*8]
4180 | sub TMPR, [BASE-8]
4181 | // Note: TMPR may now be even _above_ BASE if nargs was < numparams.
4182 | test RB, RB
4183 | jz >5 // Copy all varargs?
4184 | lea RB, [RA+RB*8-8]
4185 | cmp TMPR, BASE // No vararg slots?
4186 | jnb >2
4187 |1: // Copy vararg slots to destination slots.
4188 | mov RC, [TMPR-16]
4189 | add TMPR, 8
4190 | mov [RA], RC
4191 | add RA, 8
4192 | cmp RA, RB // All destination slots filled?
4193 | jnb >3
4194 | cmp TMPR, BASE // No more vararg slots?
4195 | jb <1
4196 |2: // Fill up remainder with nil.
4197 | mov aword [RA], LJ_TNIL
4198 | add RA, 8
4199 | cmp RA, RB
4200 | jb <2
4201 |3:
4202 | ins_next
4203 |
4204 |5: // Copy all varargs.
4205 | mov MULTRES, 1 // MULTRES = 0+1
4206 | mov RC, BASE
4207 | sub RC, TMPR
4208 | jbe <3 // No vararg slots?
4209 | mov RBd, RCd
4210 | shr RBd, 3
4211 | add RBd, 1
4212 | mov MULTRES, RBd // MULTRES = #varargs+1
4213 | mov L:RB, SAVE_L
4214 | add RC, RA
4215 | cmp RC, L:RB->maxstack
4216 | ja >7 // Need to grow stack?
4217 |6: // Copy all vararg slots.
4218 | mov RC, [TMPR-16]
4219 | add TMPR, 8
4220 | mov [RA], RC
4221 | add RA, 8
4222 | cmp TMPR, BASE // No more vararg slots?
4223 | jb <6
4224 | jmp <3
4225 |
4226 |7: // Grow stack for varargs.
4227 | mov L:RB->base, BASE
4228 | mov L:RB->top, RA
4229 | mov SAVE_PC, PC
4230 | sub TMPR, BASE // Need delta, because BASE may change.
4231 | mov TMP1hi, TMPRd
4232 | mov CARG2d, MULTRES
4233 | sub CARG2d, 1
4234 | mov CARG1, L:RB
4235 | call extern lj_state_growstack // (lua_State *L, int n)
4236 | mov BASE, L:RB->base
4237 | movsxd TMPR, TMP1hi
4238 | mov RA, L:RB->top
4239 | add TMPR, BASE
4240 | jmp <6
4241 break;
4242
4243 /* -- Returns ----------------------------------------------------------- */
4244
4245 case BC_RETM:
4246 | ins_AD // RA = results, RD = extra_nresults
4247 | add RDd, MULTRES // MULTRES >=1, so RD >=1.
4248 | // Fall through. Assumes BC_RET follows and ins_AD is a no-op.
4249 break;
4250
4251 case BC_RET: case BC_RET0: case BC_RET1:
4252 | ins_AD // RA = results, RD = nresults+1
4253 if (op != BC_RET0) {
4254 | shl RAd, 3
4255 }
4256 |1:
4257 | mov PC, [BASE-8]
4258 | mov MULTRES, RDd // Save nresults+1.
4259 | test PCd, FRAME_TYPE // Check frame type marker.
4260 | jnz >7 // Not returning to a fixarg Lua func?
4261 switch (op) {
4262 case BC_RET:
4263 |->BC_RET_Z:
4264 | mov KBASE, BASE // Use KBASE for result move.
4265 | sub RDd, 1
4266 | jz >3
4267 |2: // Move results down.
4268 | mov RB, [KBASE+RA]
4269 | mov [KBASE-16], RB
4270 | add KBASE, 8
4271 | sub RDd, 1
4272 | jnz <2
4273 |3:
4274 | mov RDd, MULTRES // Note: MULTRES may be >255.
4275 | movzx RBd, PC_RB // So cannot compare with RDL!
4276 |5:
4277 | cmp RBd, RDd // More results expected?
4278 | ja >6
4279 break;
4280 case BC_RET1:
4281 | mov RB, [BASE+RA]
4282 | mov [BASE-16], RB
4283 /* fallthrough */
4284 case BC_RET0:
4285 |5:
4286 | cmp PC_RB, RDL // More results expected?
4287 | ja >6
4288 default:
4289 break;
4290 }
4291 | movzx RAd, PC_RA
4292 | neg RA
4293 | lea BASE, [BASE+RA*8-16] // base = base - (RA+2)*8
4294 | mov LFUNC:KBASE, [BASE-16]
4295 | cleartp LFUNC:KBASE
4296 | mov KBASE, LFUNC:KBASE->pc
4297 | mov KBASE, [KBASE+PC2PROTO(k)]
4298 | ins_next
4299 |
4300 |6: // Fill up results with nil.
4301 if (op == BC_RET) {
4302 | mov aword [KBASE-16], LJ_TNIL // Note: relies on shifted base.
4303 | add KBASE, 8
4304 } else {
4305 | mov aword [BASE+RD*8-24], LJ_TNIL
4306 }
4307 | add RD, 1
4308 | jmp <5
4309 |
4310 |7: // Non-standard return case.
4311 | lea RB, [PC-FRAME_VARG]
4312 | test RBd, FRAME_TYPEP
4313 | jnz ->vm_return
4314 | // Return from vararg function: relocate BASE down and RA up.
4315 | sub BASE, RB
4316 if (op != BC_RET0) {
4317 | add RA, RB
4318 }
4319 | jmp <1
4320 break;
4321
4322 /* -- Loops and branches ------------------------------------------------ */
4323
4324 |.define FOR_IDX, [RA]
4325 |.define FOR_STOP, [RA+8]
4326 |.define FOR_STEP, [RA+16]
4327 |.define FOR_EXT, [RA+24]
4328
4329 case BC_FORL:
4330 |.if JIT
4331 | hotloop RBd
4332 |.endif
4333 | // Fall through. Assumes BC_IFORL follows and ins_AJ is a no-op.
4334 break;
4335
4336 case BC_JFORI:
4337 case BC_JFORL:
4338#if !LJ_HASJIT
4339 break;
4340#endif
4341 case BC_FORI:
4342 case BC_IFORL:
4343 vk = (op == BC_IFORL || op == BC_JFORL);
4344 | ins_AJ // RA = base, RD = target (after end of loop or start of loop)
4345 | lea RA, [BASE+RA*8]
4346 if (LJ_DUALNUM) {
4347 | mov RB, FOR_IDX
4348 | checkint RB, >9
4349 | mov TMPR, FOR_STOP
4350 if (!vk) {
4351 | checkint TMPR, ->vmeta_for
4352 | mov ITYPE, FOR_STEP
4353 | test ITYPEd, ITYPEd; js >5
4354 | sar ITYPE, 47;
4355 | cmp ITYPEd, LJ_TISNUM; jne ->vmeta_for
4356 } else {
4357#ifdef LUA_USE_ASSERT
4358 | checkinttp FOR_STOP, ->assert_bad_for_arg_type
4359 | checkinttp FOR_STEP, ->assert_bad_for_arg_type
4360#endif
4361 | mov ITYPE, FOR_STEP
4362 | test ITYPEd, ITYPEd; js >5
4363 | add RBd, ITYPEd; jo >1
4364 | setint RB
4365 | mov FOR_IDX, RB
4366 }
4367 | cmp RBd, TMPRd
4368 | mov FOR_EXT, RB
4369 if (op == BC_FORI) {
4370 | jle >7
4371 |1:
4372 |6:
4373 | branchPC RD
4374 } else if (op == BC_JFORI) {
4375 | branchPC RD
4376 | movzx RDd, PC_RD
4377 | jle =>BC_JLOOP
4378 |1:
4379 |6:
4380 } else if (op == BC_IFORL) {
4381 | jg >7
4382 |6:
4383 | branchPC RD
4384 |1:
4385 } else {
4386 | jle =>BC_JLOOP
4387 |1:
4388 |6:
4389 }
4390 |7:
4391 | ins_next
4392 |
4393 |5: // Invert check for negative step.
4394 if (!vk) {
4395 | sar ITYPE, 47;
4396 | cmp ITYPEd, LJ_TISNUM; jne ->vmeta_for
4397 } else {
4398 | add RBd, ITYPEd; jo <1
4399 | setint RB
4400 | mov FOR_IDX, RB
4401 }
4402 | cmp RBd, TMPRd
4403 | mov FOR_EXT, RB
4404 if (op == BC_FORI) {
4405 | jge <7
4406 } else if (op == BC_JFORI) {
4407 | branchPC RD
4408 | movzx RDd, PC_RD
4409 | jge =>BC_JLOOP
4410 } else if (op == BC_IFORL) {
4411 | jl <7
4412 } else {
4413 | jge =>BC_JLOOP
4414 }
4415 | jmp <6
4416 |9: // Fallback to FP variant.
4417 if (!vk) {
4418 | jae ->vmeta_for
4419 }
4420 } else if (!vk) {
4421 | checknumtp FOR_IDX, ->vmeta_for
4422 }
4423 if (!vk) {
4424 | checknumtp FOR_STOP, ->vmeta_for
4425 } else {
4426#ifdef LUA_USE_ASSERT
4427 | checknumtp FOR_STOP, ->assert_bad_for_arg_type
4428 | checknumtp FOR_STEP, ->assert_bad_for_arg_type
4429#endif
4430 }
4431 | mov RB, FOR_STEP
4432 if (!vk) {
4433 | checknum RB, ->vmeta_for
4434 }
4435 | movsd xmm0, qword FOR_IDX
4436 | movsd xmm1, qword FOR_STOP
4437 if (vk) {
4438 | addsd xmm0, qword FOR_STEP
4439 | movsd qword FOR_IDX, xmm0
4440 | test RB, RB; js >3
4441 } else {
4442 | jl >3
4443 }
4444 | ucomisd xmm1, xmm0
4445 |1:
4446 | movsd qword FOR_EXT, xmm0
4447 if (op == BC_FORI) {
4448 |.if DUALNUM
4449 | jnb <7
4450 |.else
4451 | jnb >2
4452 | branchPC RD
4453 |.endif
4454 } else if (op == BC_JFORI) {
4455 | branchPC RD
4456 | movzx RDd, PC_RD
4457 | jnb =>BC_JLOOP
4458 } else if (op == BC_IFORL) {
4459 |.if DUALNUM
4460 | jb <7
4461 |.else
4462 | jb >2
4463 | branchPC RD
4464 |.endif
4465 } else {
4466 | jnb =>BC_JLOOP
4467 }
4468 |.if DUALNUM
4469 | jmp <6
4470 |.else
4471 |2:
4472 | ins_next
4473 |.endif
4474 |
4475 |3: // Invert comparison if step is negative.
4476 | ucomisd xmm0, xmm1
4477 | jmp <1
4478 break;
4479
4480 case BC_ITERL:
4481 |.if JIT
4482 | hotloop RBd
4483 |.endif
4484 | // Fall through. Assumes BC_IITERL follows and ins_AJ is a no-op.
4485 break;
4486
4487 case BC_JITERL:
4488#if !LJ_HASJIT
4489 break;
4490#endif
4491 case BC_IITERL:
4492 | ins_AJ // RA = base, RD = target
4493 | lea RA, [BASE+RA*8]
4494 | mov RB, [RA]
4495 | cmp RB, LJ_TNIL; je >1 // Stop if iterator returned nil.
4496 if (op == BC_JITERL) {
4497 | mov [RA-8], RB
4498 | jmp =>BC_JLOOP
4499 } else {
4500 | branchPC RD // Otherwise save control var + branch.
4501 | mov [RA-8], RB
4502 }
4503 |1:
4504 | ins_next
4505 break;
4506
4507 case BC_LOOP:
4508 | ins_A // RA = base, RD = target (loop extent)
4509 | // Note: RA/RD is only used by trace recorder to determine scope/extent
4510 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
4511 |.if JIT
4512 | hotloop RBd
4513 |.endif
4514 | // Fall through. Assumes BC_ILOOP follows and ins_A is a no-op.
4515 break;
4516
4517 case BC_ILOOP:
4518 | ins_A // RA = base, RD = target (loop extent)
4519 | ins_next
4520 break;
4521
4522 case BC_JLOOP:
4523 |.if JIT
4524 | ins_AD // RA = base (ignored), RD = traceno
4525 | mov RA, [DISPATCH+DISPATCH_J(trace)]
4526 | mov TRACE:RD, [RA+RD*8]
4527 | mov RD, TRACE:RD->mcode
4528 | mov L:RB, SAVE_L
4529 | mov [DISPATCH+DISPATCH_GL(jit_base)], BASE
4530 | mov [DISPATCH+DISPATCH_GL(tmpbuf.L)], L:RB
4531 | // Save additional callee-save registers only used in compiled code.
4532 |.if X64WIN
4533 | mov CSAVE_4, r12
4534 | mov CSAVE_3, r13
4535 | mov CSAVE_2, r14
4536 | mov CSAVE_1, r15
4537 | mov RA, rsp
4538 | sub rsp, 10*16+4*8
4539 | movdqa [RA-1*16], xmm6
4540 | movdqa [RA-2*16], xmm7
4541 | movdqa [RA-3*16], xmm8
4542 | movdqa [RA-4*16], xmm9
4543 | movdqa [RA-5*16], xmm10
4544 | movdqa [RA-6*16], xmm11
4545 | movdqa [RA-7*16], xmm12
4546 | movdqa [RA-8*16], xmm13
4547 | movdqa [RA-9*16], xmm14
4548 | movdqa [RA-10*16], xmm15
4549 |.else
4550 | sub rsp, 16
4551 | mov [rsp+16], r12
4552 | mov [rsp+8], r13
4553 |.endif
4554 | jmp RD
4555 |.endif
4556 break;
4557
4558 case BC_JMP:
4559 | ins_AJ // RA = unused, RD = target
4560 | branchPC RD
4561 | ins_next
4562 break;
4563
4564 /* -- Function headers -------------------------------------------------- */
4565
4566 /*
4567 ** Reminder: A function may be called with func/args above L->maxstack,
4568 ** i.e. occupying EXTRA_STACK slots. And vmeta_call may add one extra slot,
4569 ** too. This means all FUNC* ops (including fast functions) must check
4570 ** for stack overflow _before_ adding more slots!
4571 */
4572
4573 case BC_FUNCF:
4574 |.if JIT
4575 | hotcall RBd
4576 |.endif
4577 case BC_FUNCV: /* NYI: compiled vararg functions. */
4578 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow and ins_AD is a no-op.
4579 break;
4580
4581 case BC_JFUNCF:
4582#if !LJ_HASJIT
4583 break;
4584#endif
4585 case BC_IFUNCF:
4586 | ins_AD // BASE = new base, RA = framesize, RD = nargs+1
4587 | mov KBASE, [PC-4+PC2PROTO(k)]
4588 | mov L:RB, SAVE_L
4589 | lea RA, [BASE+RA*8] // Top of frame.
4590 | cmp RA, L:RB->maxstack
4591 | ja ->vm_growstack_f
4592 | movzx RAd, byte [PC-4+PC2PROTO(numparams)]
4593 | cmp NARGS:RDd, RAd // Check for missing parameters.
4594 | jbe >3
4595 |2:
4596 if (op == BC_JFUNCF) {
4597 | movzx RDd, PC_RD
4598 | jmp =>BC_JLOOP
4599 } else {
4600 | ins_next
4601 }
4602 |
4603 |3: // Clear missing parameters.
4604 | mov aword [BASE+NARGS:RD*8-8], LJ_TNIL
4605 | add NARGS:RDd, 1
4606 | cmp NARGS:RDd, RAd
4607 | jbe <3
4608 | jmp <2
4609 break;
4610
4611 case BC_JFUNCV:
4612#if !LJ_HASJIT
4613 break;
4614#endif
4615 | int3 // NYI: compiled vararg functions
4616 break; /* NYI: compiled vararg functions. */
4617
4618 case BC_IFUNCV:
4619 | ins_AD // BASE = new base, RA = framesize, RD = nargs+1
4620 | lea RBd, [NARGS:RD*8+FRAME_VARG+8]
4621 | lea RD, [BASE+NARGS:RD*8+8]
4622 | mov LFUNC:KBASE, [BASE-16]
4623 | mov [RD-8], RB // Store delta + FRAME_VARG.
4624 | mov [RD-16], LFUNC:KBASE // Store copy of LFUNC.
4625 | mov L:RB, SAVE_L
4626 | lea RA, [RD+RA*8]
4627 | cmp RA, L:RB->maxstack
4628 | ja ->vm_growstack_v // Need to grow stack.
4629 | mov RA, BASE
4630 | mov BASE, RD
4631 | movzx RBd, byte [PC-4+PC2PROTO(numparams)]
4632 | test RBd, RBd
4633 | jz >2
4634 | add RA, 8
4635 |1: // Copy fixarg slots up to new frame.
4636 | add RA, 8
4637 | cmp RA, BASE
4638 | jnb >3 // Less args than parameters?
4639 | mov KBASE, [RA-16]
4640 | mov [RD], KBASE
4641 | add RD, 8
4642 | mov aword [RA-16], LJ_TNIL // Clear old fixarg slot (help the GC).
4643 | sub RBd, 1
4644 | jnz <1
4645 |2:
4646 if (op == BC_JFUNCV) {
4647 | movzx RDd, PC_RD
4648 | jmp =>BC_JLOOP
4649 } else {
4650 | mov KBASE, [PC-4+PC2PROTO(k)]
4651 | ins_next
4652 }
4653 |
4654 |3: // Clear missing parameters.
4655 | mov aword [RD], LJ_TNIL
4656 | add RD, 8
4657 | sub RBd, 1
4658 | jnz <3
4659 | jmp <2
4660 break;
4661
4662 case BC_FUNCC:
4663 case BC_FUNCCW:
4664 | ins_AD // BASE = new base, RA = ins RA|RD (unused), RD = nargs+1
4665 | mov CFUNC:RB, [BASE-16]
4666 | cleartp CFUNC:RB
4667 | mov KBASE, CFUNC:RB->f
4668 | mov L:RB, SAVE_L
4669 | lea RD, [BASE+NARGS:RD*8-8]
4670 | mov L:RB->base, BASE
4671 | lea RA, [RD+8*LUA_MINSTACK]
4672 | cmp RA, L:RB->maxstack
4673 | mov L:RB->top, RD
4674 if (op == BC_FUNCC) {
4675 | mov CARG1, L:RB // Caveat: CARG1 may be RA.
4676 } else {
4677 | mov CARG2, KBASE
4678 | mov CARG1, L:RB // Caveat: CARG1 may be RA.
4679 }
4680 | ja ->vm_growstack_c // Need to grow stack.
4681 | set_vmstate C
4682 if (op == BC_FUNCC) {
4683 | call KBASE // (lua_State *L)
4684 } else {
4685 | // (lua_State *L, lua_CFunction f)
4686 | call aword [DISPATCH+DISPATCH_GL(wrapf)]
4687 }
4688 | // nresults returned in eax (RD).
4689 | mov BASE, L:RB->base
4690 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
4691 | set_vmstate INTERP
4692 | lea RA, [BASE+RD*8]
4693 | neg RA
4694 | add RA, L:RB->top // RA = (L->top-(L->base+nresults))*8
4695 | mov PC, [BASE-8] // Fetch PC of caller.
4696 | jmp ->vm_returnc
4697 break;
4698
4699 /* ---------------------------------------------------------------------- */
4700
4701 default:
4702 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
4703 exit(2);
4704 break;
4705 }
4706}
4707
4708static int build_backend(BuildCtx *ctx)
4709{
4710 int op;
4711 dasm_growpc(Dst, BC__MAX);
4712 build_subroutines(ctx);
4713 |.code_op
4714 for (op = 0; op < BC__MAX; op++)
4715 build_ins(ctx, (BCOp)op, op);
4716 return BC__MAX;
4717}
4718
4719/* Emit pseudo frame-info for all assembler functions. */
4720static void emit_asm_debug(BuildCtx *ctx)
4721{
4722 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
4723 switch (ctx->mode) {
4724 case BUILD_elfasm:
4725 fprintf(ctx->fp, "\t.section .debug_frame,\"\",@progbits\n");
4726 fprintf(ctx->fp,
4727 ".Lframe0:\n"
4728 "\t.long .LECIE0-.LSCIE0\n"
4729 ".LSCIE0:\n"
4730 "\t.long 0xffffffff\n"
4731 "\t.byte 0x1\n"
4732 "\t.string \"\"\n"
4733 "\t.uleb128 0x1\n"
4734 "\t.sleb128 -8\n"
4735 "\t.byte 0x10\n"
4736 "\t.byte 0xc\n\t.uleb128 0x7\n\t.uleb128 8\n"
4737 "\t.byte 0x80+0x10\n\t.uleb128 0x1\n"
4738 "\t.align 8\n"
4739 ".LECIE0:\n\n");
4740 fprintf(ctx->fp,
4741 ".LSFDE0:\n"
4742 "\t.long .LEFDE0-.LASFDE0\n"
4743 ".LASFDE0:\n"
4744 "\t.long .Lframe0\n"
4745 "\t.quad .Lbegin\n"
4746 "\t.quad %d\n"
4747 "\t.byte 0xe\n\t.uleb128 %d\n" /* def_cfa_offset */
4748 "\t.byte 0x86\n\t.uleb128 0x2\n" /* offset rbp */
4749 "\t.byte 0x83\n\t.uleb128 0x3\n" /* offset rbx */
4750 "\t.byte 0x8f\n\t.uleb128 0x4\n" /* offset r15 */
4751 "\t.byte 0x8e\n\t.uleb128 0x5\n" /* offset r14 */
4752#if LJ_NO_UNWIND
4753 "\t.byte 0x8d\n\t.uleb128 0x6\n" /* offset r13 */
4754 "\t.byte 0x8c\n\t.uleb128 0x7\n" /* offset r12 */
4755#endif
4756 "\t.align 8\n"
4757 ".LEFDE0:\n\n", fcofs, CFRAME_SIZE);
4758#if LJ_HASFFI
4759 fprintf(ctx->fp,
4760 ".LSFDE1:\n"
4761 "\t.long .LEFDE1-.LASFDE1\n"
4762 ".LASFDE1:\n"
4763 "\t.long .Lframe0\n"
4764 "\t.quad lj_vm_ffi_call\n"
4765 "\t.quad %d\n"
4766 "\t.byte 0xe\n\t.uleb128 16\n" /* def_cfa_offset */
4767 "\t.byte 0x86\n\t.uleb128 0x2\n" /* offset rbp */
4768 "\t.byte 0xd\n\t.uleb128 0x6\n" /* def_cfa_register rbp */
4769 "\t.byte 0x83\n\t.uleb128 0x3\n" /* offset rbx */
4770 "\t.align 8\n"
4771 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4772#endif
4773#if !LJ_NO_UNWIND
4774#if LJ_TARGET_SOLARIS
4775 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@unwind\n");
4776#else
4777 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@progbits\n");
4778#endif
4779 fprintf(ctx->fp,
4780 ".Lframe1:\n"
4781 "\t.long .LECIE1-.LSCIE1\n"
4782 ".LSCIE1:\n"
4783 "\t.long 0\n"
4784 "\t.byte 0x1\n"
4785 "\t.string \"zPR\"\n"
4786 "\t.uleb128 0x1\n"
4787 "\t.sleb128 -8\n"
4788 "\t.byte 0x10\n"
4789 "\t.uleb128 6\n" /* augmentation length */
4790 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4791 "\t.long lj_err_unwind_dwarf-.\n"
4792 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4793 "\t.byte 0xc\n\t.uleb128 0x7\n\t.uleb128 8\n"
4794 "\t.byte 0x80+0x10\n\t.uleb128 0x1\n"
4795 "\t.align 8\n"
4796 ".LECIE1:\n\n");
4797 fprintf(ctx->fp,
4798 ".LSFDE2:\n"
4799 "\t.long .LEFDE2-.LASFDE2\n"
4800 ".LASFDE2:\n"
4801 "\t.long .LASFDE2-.Lframe1\n"
4802 "\t.long .Lbegin-.\n"
4803 "\t.long %d\n"
4804 "\t.uleb128 0\n" /* augmentation length */
4805 "\t.byte 0xe\n\t.uleb128 %d\n" /* def_cfa_offset */
4806 "\t.byte 0x86\n\t.uleb128 0x2\n" /* offset rbp */
4807 "\t.byte 0x83\n\t.uleb128 0x3\n" /* offset rbx */
4808 "\t.byte 0x8f\n\t.uleb128 0x4\n" /* offset r15 */
4809 "\t.byte 0x8e\n\t.uleb128 0x5\n" /* offset r14 */
4810 "\t.align 8\n"
4811 ".LEFDE2:\n\n", fcofs, CFRAME_SIZE);
4812#if LJ_HASFFI
4813 fprintf(ctx->fp,
4814 ".Lframe2:\n"
4815 "\t.long .LECIE2-.LSCIE2\n"
4816 ".LSCIE2:\n"
4817 "\t.long 0\n"
4818 "\t.byte 0x1\n"
4819 "\t.string \"zR\"\n"
4820 "\t.uleb128 0x1\n"
4821 "\t.sleb128 -8\n"
4822 "\t.byte 0x10\n"
4823 "\t.uleb128 1\n" /* augmentation length */
4824 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4825 "\t.byte 0xc\n\t.uleb128 0x7\n\t.uleb128 8\n"
4826 "\t.byte 0x80+0x10\n\t.uleb128 0x1\n"
4827 "\t.align 8\n"
4828 ".LECIE2:\n\n");
4829 fprintf(ctx->fp,
4830 ".LSFDE3:\n"
4831 "\t.long .LEFDE3-.LASFDE3\n"
4832 ".LASFDE3:\n"
4833 "\t.long .LASFDE3-.Lframe2\n"
4834 "\t.long lj_vm_ffi_call-.\n"
4835 "\t.long %d\n"
4836 "\t.uleb128 0\n" /* augmentation length */
4837 "\t.byte 0xe\n\t.uleb128 16\n" /* def_cfa_offset */
4838 "\t.byte 0x86\n\t.uleb128 0x2\n" /* offset rbp */
4839 "\t.byte 0xd\n\t.uleb128 0x6\n" /* def_cfa_register rbp */
4840 "\t.byte 0x83\n\t.uleb128 0x3\n" /* offset rbx */
4841 "\t.align 8\n"
4842 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4843#endif
4844#endif
4845 break;
4846#if !LJ_NO_UNWIND
4847 /* Mental note: never let Apple design an assembler.
4848 ** Or a linker. Or a plastic case. But I digress.
4849 */
4850 case BUILD_machasm: {
4851#if LJ_HASFFI
4852 int fcsize = 0;
4853#endif
4854 int i;
4855 fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4856 fprintf(ctx->fp,
4857 "EH_frame1:\n"
4858 "\t.set L$set$x,LECIEX-LSCIEX\n"
4859 "\t.long L$set$x\n"
4860 "LSCIEX:\n"
4861 "\t.long 0\n"
4862 "\t.byte 0x1\n"
4863 "\t.ascii \"zPR\\0\"\n"
4864 "\t.byte 0x1\n"
4865 "\t.byte 128-8\n"
4866 "\t.byte 0x10\n"
4867 "\t.byte 6\n" /* augmentation length */
4868 "\t.byte 0x9b\n" /* indirect|pcrel|sdata4 */
4869 "\t.long _lj_err_unwind_dwarf+4@GOTPCREL\n"
4870 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4871 "\t.byte 0xc\n\t.byte 0x7\n\t.byte 8\n"
4872 "\t.byte 0x80+0x10\n\t.byte 0x1\n"
4873 "\t.align 3\n"
4874 "LECIEX:\n\n");
4875 for (i = 0; i < ctx->nsym; i++) {
4876 const char *name = ctx->sym[i].name;
4877 int32_t size = ctx->sym[i+1].ofs - ctx->sym[i].ofs;
4878 if (size == 0) continue;
4879#if LJ_HASFFI
4880 if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4881#endif
4882 fprintf(ctx->fp,
4883 "%s.eh:\n"
4884 "LSFDE%d:\n"
4885 "\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4886 "\t.long L$set$%d\n"
4887 "LASFDE%d:\n"
4888 "\t.long LASFDE%d-EH_frame1\n"
4889 "\t.long %s-.\n"
4890 "\t.long %d\n"
4891 "\t.byte 0\n" /* augmentation length */
4892 "\t.byte 0xe\n\t.byte %d\n" /* def_cfa_offset */
4893 "\t.byte 0x86\n\t.byte 0x2\n" /* offset rbp */
4894 "\t.byte 0x83\n\t.byte 0x3\n" /* offset rbx */
4895 "\t.byte 0x8f\n\t.byte 0x4\n" /* offset r15 */
4896 "\t.byte 0x8e\n\t.byte 0x5\n" /* offset r14 */
4897 "\t.align 3\n"
4898 "LEFDE%d:\n\n",
4899 name, i, i, i, i, i, i, i, name, size, CFRAME_SIZE, i);
4900 }
4901#if LJ_HASFFI
4902 if (fcsize) {
4903 fprintf(ctx->fp,
4904 "EH_frame2:\n"
4905 "\t.set L$set$y,LECIEY-LSCIEY\n"
4906 "\t.long L$set$y\n"
4907 "LSCIEY:\n"
4908 "\t.long 0\n"
4909 "\t.byte 0x1\n"
4910 "\t.ascii \"zR\\0\"\n"
4911 "\t.byte 0x1\n"
4912 "\t.byte 128-8\n"
4913 "\t.byte 0x10\n"
4914 "\t.byte 1\n" /* augmentation length */
4915 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4916 "\t.byte 0xc\n\t.byte 0x7\n\t.byte 8\n"
4917 "\t.byte 0x80+0x10\n\t.byte 0x1\n"
4918 "\t.align 3\n"
4919 "LECIEY:\n\n");
4920 fprintf(ctx->fp,
4921 "_lj_vm_ffi_call.eh:\n"
4922 "LSFDEY:\n"
4923 "\t.set L$set$yy,LEFDEY-LASFDEY\n"
4924 "\t.long L$set$yy\n"
4925 "LASFDEY:\n"
4926 "\t.long LASFDEY-EH_frame2\n"
4927 "\t.long _lj_vm_ffi_call-.\n"
4928 "\t.long %d\n"
4929 "\t.byte 0\n" /* augmentation length */
4930 "\t.byte 0xe\n\t.byte 16\n" /* def_cfa_offset */
4931 "\t.byte 0x86\n\t.byte 0x2\n" /* offset rbp */
4932 "\t.byte 0xd\n\t.byte 0x6\n" /* def_cfa_register rbp */
4933 "\t.byte 0x83\n\t.byte 0x3\n" /* offset rbx */
4934 "\t.align 3\n"
4935 "LEFDEY:\n\n", fcsize);
4936 }
4937#endif
4938 fprintf(ctx->fp, ".subsections_via_symbols\n");
4939 }
4940 break;
4941#endif
4942 default: /* Difficult for other modes. */
4943 break;
4944 }
4945}
4946
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index 5b3356dc..bda9d7d7 100644
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -18,7 +18,6 @@
18| 18|
19|.if P64 19|.if P64
20|.define X64, 1 20|.define X64, 1
21|.define SSE, 1
22|.if WIN 21|.if WIN
23|.define X64WIN, 1 22|.define X64WIN, 1
24|.endif 23|.endif
@@ -116,24 +115,74 @@
116|.type NODE, Node 115|.type NODE, Node
117|.type NARGS, int 116|.type NARGS, int
118|.type TRACE, GCtrace 117|.type TRACE, GCtrace
118|.type SBUF, SBuf
119| 119|
120|// Stack layout while in interpreter. Must match with lj_frame.h. 120|// Stack layout while in interpreter. Must match with lj_frame.h.
121|//----------------------------------------------------------------------- 121|//-----------------------------------------------------------------------
122|.if not X64 // x86 stack layout. 122|.if not X64 // x86 stack layout.
123| 123|
124|.define CFRAME_SPACE, aword*7 // Delta for esp (see <--). 124|.if WIN
125|
126|.define CFRAME_SPACE, aword*9 // Delta for esp (see <--).
125|.macro saveregs_ 127|.macro saveregs_
126| push edi; push esi; push ebx 128| push edi; push esi; push ebx
129| push extern lj_err_unwind_win
130| fs; push dword [0]
131| fs; mov [0], esp
127| sub esp, CFRAME_SPACE 132| sub esp, CFRAME_SPACE
128|.endmacro 133|.endmacro
129|.macro saveregs 134|.macro restoreregs
130| push ebp; saveregs_ 135| add esp, CFRAME_SPACE
136| fs; pop dword [0]
137| pop edi // Short for esp += 4.
138| pop ebx; pop esi; pop edi; pop ebp
139|.endmacro
140|
141|.else
142|
143|.define CFRAME_SPACE, aword*7 // Delta for esp (see <--).
144|.macro saveregs_
145| push edi; push esi; push ebx
146| sub esp, CFRAME_SPACE
131|.endmacro 147|.endmacro
132|.macro restoreregs 148|.macro restoreregs
133| add esp, CFRAME_SPACE 149| add esp, CFRAME_SPACE
134| pop ebx; pop esi; pop edi; pop ebp 150| pop ebx; pop esi; pop edi; pop ebp
135|.endmacro 151|.endmacro
136| 152|
153|.endif
154|
155|.macro saveregs
156| push ebp; saveregs_
157|.endmacro
158|
159|.if WIN
160|.define SAVE_ERRF, aword [esp+aword*19] // vm_pcall/vm_cpcall only.
161|.define SAVE_NRES, aword [esp+aword*18]
162|.define SAVE_CFRAME, aword [esp+aword*17]
163|.define SAVE_L, aword [esp+aword*16]
164|//----- 16 byte aligned, ^^^ arguments from C caller
165|.define SAVE_RET, aword [esp+aword*15] //<-- esp entering interpreter.
166|.define SAVE_R4, aword [esp+aword*14]
167|.define SAVE_R3, aword [esp+aword*13]
168|.define SAVE_R2, aword [esp+aword*12]
169|//----- 16 byte aligned
170|.define SAVE_R1, aword [esp+aword*11]
171|.define SEH_FUNC, aword [esp+aword*10]
172|.define SEH_NEXT, aword [esp+aword*9] //<-- esp after register saves.
173|.define UNUSED2, aword [esp+aword*8]
174|//----- 16 byte aligned
175|.define UNUSED1, aword [esp+aword*7]
176|.define SAVE_PC, aword [esp+aword*6]
177|.define TMP2, aword [esp+aword*5]
178|.define TMP1, aword [esp+aword*4]
179|//----- 16 byte aligned
180|.define ARG4, aword [esp+aword*3]
181|.define ARG3, aword [esp+aword*2]
182|.define ARG2, aword [esp+aword*1]
183|.define ARG1, aword [esp] //<-- esp while in interpreter.
184|//----- 16 byte aligned, ^^^ arguments for C callee
185|.else
137|.define SAVE_ERRF, aword [esp+aword*15] // vm_pcall/vm_cpcall only. 186|.define SAVE_ERRF, aword [esp+aword*15] // vm_pcall/vm_cpcall only.
138|.define SAVE_NRES, aword [esp+aword*14] 187|.define SAVE_NRES, aword [esp+aword*14]
139|.define SAVE_CFRAME, aword [esp+aword*13] 188|.define SAVE_CFRAME, aword [esp+aword*13]
@@ -154,6 +203,7 @@
154|.define ARG2, aword [esp+aword*1] 203|.define ARG2, aword [esp+aword*1]
155|.define ARG1, aword [esp] //<-- esp while in interpreter. 204|.define ARG1, aword [esp] //<-- esp while in interpreter.
156|//----- 16 byte aligned, ^^^ arguments for C callee 205|//----- 16 byte aligned, ^^^ arguments for C callee
206|.endif
157| 207|
158|// FPARGx overlaps ARGx and ARG(x+1) on x86. 208|// FPARGx overlaps ARGx and ARG(x+1) on x86.
159|.define FPARG3, qword [esp+qword*1] 209|.define FPARG3, qword [esp+qword*1]
@@ -389,7 +439,6 @@
389| fpop 439| fpop
390|.endmacro 440|.endmacro
391| 441|
392|.macro fdup; fld st0; .endmacro
393|.macro fpop1; fstp st1; .endmacro 442|.macro fpop1; fstp st1; .endmacro
394| 443|
395|// Synthesize SSE FP constants. 444|// Synthesize SSE FP constants.
@@ -552,6 +601,10 @@ static void build_subroutines(BuildCtx *ctx)
552 |.else 601 |.else
553 | mov eax, FCARG2 // Error return status for vm_pcall. 602 | mov eax, FCARG2 // Error return status for vm_pcall.
554 | mov esp, FCARG1 603 | mov esp, FCARG1
604 |.if WIN
605 | lea FCARG1, SEH_NEXT
606 | fs; mov [0], FCARG1
607 |.endif
555 |.endif 608 |.endif
556 |->vm_unwind_c_eh: // Landing pad for external unwinder. 609 |->vm_unwind_c_eh: // Landing pad for external unwinder.
557 | mov L:RB, SAVE_L 610 | mov L:RB, SAVE_L
@@ -575,6 +628,10 @@ static void build_subroutines(BuildCtx *ctx)
575 |.else 628 |.else
576 | and FCARG1, CFRAME_RAWMASK 629 | and FCARG1, CFRAME_RAWMASK
577 | mov esp, FCARG1 630 | mov esp, FCARG1
631 |.if WIN
632 | lea FCARG1, SEH_NEXT
633 | fs; mov [0], FCARG1
634 |.endif
578 |.endif 635 |.endif
579 |->vm_unwind_ff_eh: // Landing pad for external unwinder. 636 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
580 | mov L:RB, SAVE_L 637 | mov L:RB, SAVE_L
@@ -588,6 +645,19 @@ static void build_subroutines(BuildCtx *ctx)
588 | set_vmstate INTERP 645 | set_vmstate INTERP
589 | jmp ->vm_returnc // Increments RD/MULTRES and returns. 646 | jmp ->vm_returnc // Increments RD/MULTRES and returns.
590 | 647 |
648 |.if WIN and not X64
649 |->vm_rtlunwind@16: // Thin layer around RtlUnwind.
650 | // (void *cframe, void *excptrec, void *unwinder, int errcode)
651 | mov [esp], FCARG1 // Return value for RtlUnwind.
652 | push FCARG2 // Exception record for RtlUnwind.
653 | push 0 // Ignored by RtlUnwind.
654 | push dword [FCARG1+CFRAME_OFS_SEH]
655 | call extern RtlUnwind@16 // Violates ABI (clobbers too much).
656 | mov FCARG1, eax
657 | mov FCARG2, [esp+4] // errcode (for vm_unwind_c).
658 | ret // Jump to unwinder.
659 |.endif
660 |
591 |//----------------------------------------------------------------------- 661 |//-----------------------------------------------------------------------
592 |//-- Grow stack for calls ----------------------------------------------- 662 |//-- Grow stack for calls -----------------------------------------------
593 |//----------------------------------------------------------------------- 663 |//-----------------------------------------------------------------------
@@ -643,17 +713,18 @@ static void build_subroutines(BuildCtx *ctx)
643 | lea KBASEa, [esp+CFRAME_RESUME] 713 | lea KBASEa, [esp+CFRAME_RESUME]
644 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table. 714 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
645 | add DISPATCH, GG_G2DISP 715 | add DISPATCH, GG_G2DISP
646 | mov L:RB->cframe, KBASEa
647 | mov SAVE_PC, RD // Any value outside of bytecode is ok. 716 | mov SAVE_PC, RD // Any value outside of bytecode is ok.
648 | mov SAVE_CFRAME, RDa 717 | mov SAVE_CFRAME, RDa
649 |.if X64 718 |.if X64
650 | mov SAVE_NRES, RD 719 | mov SAVE_NRES, RD
651 | mov SAVE_ERRF, RD 720 | mov SAVE_ERRF, RD
652 |.endif 721 |.endif
722 | mov L:RB->cframe, KBASEa
653 | cmp byte L:RB->status, RDL 723 | cmp byte L:RB->status, RDL
654 | je >3 // Initial resume (like a call). 724 | je >2 // Initial resume (like a call).
655 | 725 |
656 | // Resume after yield (like a return). 726 | // Resume after yield (like a return).
727 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
657 | set_vmstate INTERP 728 | set_vmstate INTERP
658 | mov byte L:RB->status, RDL 729 | mov byte L:RB->status, RDL
659 | mov BASE, L:RB->base 730 | mov BASE, L:RB->base
@@ -693,20 +764,19 @@ static void build_subroutines(BuildCtx *ctx)
693 | mov RA, INARG_BASE // Caveat: overlaps SAVE_CFRAME! 764 | mov RA, INARG_BASE // Caveat: overlaps SAVE_CFRAME!
694 |.endif 765 |.endif
695 | 766 |
767 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
696 | mov KBASEa, L:RB->cframe // Add our C frame to cframe chain. 768 | mov KBASEa, L:RB->cframe // Add our C frame to cframe chain.
697 | mov SAVE_CFRAME, KBASEa 769 | mov SAVE_CFRAME, KBASEa
698 | mov SAVE_PC, L:RB // Any value outside of bytecode is ok. 770 | mov SAVE_PC, L:RB // Any value outside of bytecode is ok.
771 | add DISPATCH, GG_G2DISP
699 |.if X64 772 |.if X64
700 | mov L:RB->cframe, rsp 773 | mov L:RB->cframe, rsp
701 |.else 774 |.else
702 | mov L:RB->cframe, esp 775 | mov L:RB->cframe, esp
703 |.endif 776 |.endif
704 | 777 |
705 |2: // Entry point for vm_cpcall below (RA = base, RB = L, PC = ftype). 778 |2: // Entry point for vm_resume/vm_cpcall (RA = base, RB = L, PC = ftype).
706 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table. 779 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
707 | add DISPATCH, GG_G2DISP
708 |
709 |3: // Entry point for vm_resume above (RA = base, RB = L, PC = ftype).
710 | set_vmstate INTERP 780 | set_vmstate INTERP
711 | mov BASE, L:RB->base // BASE = old base (used in vmeta_call). 781 | mov BASE, L:RB->base // BASE = old base (used in vmeta_call).
712 | add PC, RA 782 | add PC, RA
@@ -744,14 +814,17 @@ static void build_subroutines(BuildCtx *ctx)
744 | 814 |
745 | mov KBASE, L:RB->stack // Compute -savestack(L, L->top). 815 | mov KBASE, L:RB->stack // Compute -savestack(L, L->top).
746 | sub KBASE, L:RB->top 816 | sub KBASE, L:RB->top
817 | mov DISPATCH, L:RB->glref // Setup pointer to dispatch table.
747 | mov SAVE_ERRF, 0 // No error function. 818 | mov SAVE_ERRF, 0 // No error function.
748 | mov SAVE_NRES, KBASE // Neg. delta means cframe w/o frame. 819 | mov SAVE_NRES, KBASE // Neg. delta means cframe w/o frame.
820 | add DISPATCH, GG_G2DISP
749 | // Handler may change cframe_nres(L->cframe) or cframe_errfunc(L->cframe). 821 | // Handler may change cframe_nres(L->cframe) or cframe_errfunc(L->cframe).
750 | 822 |
751 |.if X64 823 |.if X64
752 | mov KBASEa, L:RB->cframe // Add our C frame to cframe chain. 824 | mov KBASEa, L:RB->cframe // Add our C frame to cframe chain.
753 | mov SAVE_CFRAME, KBASEa 825 | mov SAVE_CFRAME, KBASEa
754 | mov L:RB->cframe, rsp 826 | mov L:RB->cframe, rsp
827 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
755 | 828 |
756 | call CARG4 // (lua_State *L, lua_CFunction func, void *ud) 829 | call CARG4 // (lua_State *L, lua_CFunction func, void *ud)
757 |.else 830 |.else
@@ -762,6 +835,7 @@ static void build_subroutines(BuildCtx *ctx)
762 | mov KBASE, L:RB->cframe // Add our C frame to cframe chain. 835 | mov KBASE, L:RB->cframe // Add our C frame to cframe chain.
763 | mov SAVE_CFRAME, KBASE 836 | mov SAVE_CFRAME, KBASE
764 | mov L:RB->cframe, esp 837 | mov L:RB->cframe, esp
838 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
765 | 839 |
766 | call BASE // (lua_State *L, lua_CFunction func, void *ud) 840 | call BASE // (lua_State *L, lua_CFunction func, void *ud)
767 |.endif 841 |.endif
@@ -869,13 +943,9 @@ static void build_subroutines(BuildCtx *ctx)
869 |.if DUALNUM 943 |.if DUALNUM
870 | mov TMP2, LJ_TISNUM 944 | mov TMP2, LJ_TISNUM
871 | mov TMP1, RC 945 | mov TMP1, RC
872 |.elif SSE 946 |.else
873 | cvtsi2sd xmm0, RC 947 | cvtsi2sd xmm0, RC
874 | movsd TMPQ, xmm0 948 | movsd TMPQ, xmm0
875 |.else
876 | mov ARG4, RC
877 | fild ARG4
878 | fstp TMPQ
879 |.endif 949 |.endif
880 | lea RCa, TMPQ // Store temp. TValue in TMPQ. 950 | lea RCa, TMPQ // Store temp. TValue in TMPQ.
881 | jmp >1 951 | jmp >1
@@ -929,6 +999,19 @@ static void build_subroutines(BuildCtx *ctx)
929 | mov NARGS:RD, 2+1 // 2 args for func(t, k). 999 | mov NARGS:RD, 2+1 // 2 args for func(t, k).
930 | jmp ->vm_call_dispatch_f 1000 | jmp ->vm_call_dispatch_f
931 | 1001 |
1002 |->vmeta_tgetr:
1003 | mov FCARG1, TAB:RB
1004 | mov RB, BASE // Save BASE.
1005 | mov FCARG2, RC // Caveat: FCARG2 == BASE
1006 | call extern lj_tab_getinth@8 // (GCtab *t, int32_t key)
1007 | // cTValue * or NULL returned in eax (RC).
1008 | movzx RA, PC_RA
1009 | mov BASE, RB // Restore BASE.
1010 | test RC, RC
1011 | jnz ->BC_TGETR_Z
1012 | mov dword [BASE+RA*8+4], LJ_TNIL
1013 | jmp ->BC_TGETR2_Z
1014 |
932 |//----------------------------------------------------------------------- 1015 |//-----------------------------------------------------------------------
933 | 1016 |
934 |->vmeta_tsets: 1017 |->vmeta_tsets:
@@ -948,13 +1031,9 @@ static void build_subroutines(BuildCtx *ctx)
948 |.if DUALNUM 1031 |.if DUALNUM
949 | mov TMP2, LJ_TISNUM 1032 | mov TMP2, LJ_TISNUM
950 | mov TMP1, RC 1033 | mov TMP1, RC
951 |.elif SSE 1034 |.else
952 | cvtsi2sd xmm0, RC 1035 | cvtsi2sd xmm0, RC
953 | movsd TMPQ, xmm0 1036 | movsd TMPQ, xmm0
954 |.else
955 | mov ARG4, RC
956 | fild ARG4
957 | fstp TMPQ
958 |.endif 1037 |.endif
959 | lea RCa, TMPQ // Store temp. TValue in TMPQ. 1038 | lea RCa, TMPQ // Store temp. TValue in TMPQ.
960 | jmp >1 1039 | jmp >1
@@ -1020,6 +1099,33 @@ static void build_subroutines(BuildCtx *ctx)
1020 | mov NARGS:RD, 3+1 // 3 args for func(t, k, v). 1099 | mov NARGS:RD, 3+1 // 3 args for func(t, k, v).
1021 | jmp ->vm_call_dispatch_f 1100 | jmp ->vm_call_dispatch_f
1022 | 1101 |
1102 |->vmeta_tsetr:
1103 |.if X64WIN
1104 | mov L:CARG1d, SAVE_L
1105 | mov CARG3d, RC
1106 | mov L:CARG1d->base, BASE
1107 | xchg CARG2d, TAB:RB // Caveat: CARG2d == BASE.
1108 |.elif X64
1109 | mov L:CARG1d, SAVE_L
1110 | mov CARG2d, TAB:RB
1111 | mov L:CARG1d->base, BASE
1112 | mov RB, BASE // Save BASE.
1113 | mov CARG3d, RC // Caveat: CARG3d == BASE.
1114 |.else
1115 | mov L:RA, SAVE_L
1116 | mov ARG2, TAB:RB
1117 | mov RB, BASE // Save BASE.
1118 | mov ARG3, RC
1119 | mov ARG1, L:RA
1120 | mov L:RA->base, BASE
1121 |.endif
1122 | mov SAVE_PC, PC
1123 | call extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
1124 | // TValue * returned in eax (RC).
1125 | movzx RA, PC_RA
1126 | mov BASE, RB // Restore BASE.
1127 | jmp ->BC_TSETR_Z
1128 |
1023 |//-- Comparison metamethods --------------------------------------------- 1129 |//-- Comparison metamethods ---------------------------------------------
1024 | 1130 |
1025 |->vmeta_comp: 1131 |->vmeta_comp:
@@ -1114,6 +1220,26 @@ static void build_subroutines(BuildCtx *ctx)
1114 | jmp <3 1220 | jmp <3
1115 |.endif 1221 |.endif
1116 | 1222 |
1223 |->vmeta_istype:
1224 |.if X64
1225 | mov L:RB, SAVE_L
1226 | mov L:RB->base, BASE // Caveat: CARG2d/CARG3d may be BASE.
1227 | mov CARG2d, RA
1228 | movzx CARG3d, PC_RD
1229 | mov L:CARG1d, L:RB
1230 |.else
1231 | movzx RD, PC_RD
1232 | mov ARG2, RA
1233 | mov L:RB, SAVE_L
1234 | mov ARG3, RD
1235 | mov ARG1, L:RB
1236 | mov L:RB->base, BASE
1237 |.endif
1238 | mov SAVE_PC, PC
1239 | call extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
1240 | mov BASE, L:RB->base
1241 | jmp <6
1242 |
1117 |//-- Arithmetic metamethods --------------------------------------------- 1243 |//-- Arithmetic metamethods ---------------------------------------------
1118 | 1244 |
1119 |->vmeta_arith_vno: 1245 |->vmeta_arith_vno:
@@ -1290,19 +1416,6 @@ static void build_subroutines(BuildCtx *ctx)
1290 | cmp NARGS:RD, 2+1; jb ->fff_fallback 1416 | cmp NARGS:RD, 2+1; jb ->fff_fallback
1291 |.endmacro 1417 |.endmacro
1292 | 1418 |
1293 |.macro .ffunc_n, name
1294 | .ffunc_1 name
1295 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
1296 | fld qword [BASE]
1297 |.endmacro
1298 |
1299 |.macro .ffunc_n, name, op
1300 | .ffunc_1 name
1301 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
1302 | op
1303 | fld qword [BASE]
1304 |.endmacro
1305 |
1306 |.macro .ffunc_nsse, name, op 1419 |.macro .ffunc_nsse, name, op
1307 | .ffunc_1 name 1420 | .ffunc_1 name
1308 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback 1421 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
@@ -1313,14 +1426,6 @@ static void build_subroutines(BuildCtx *ctx)
1313 | .ffunc_nsse name, movsd 1426 | .ffunc_nsse name, movsd
1314 |.endmacro 1427 |.endmacro
1315 | 1428 |
1316 |.macro .ffunc_nn, name
1317 | .ffunc_2 name
1318 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
1319 | cmp dword [BASE+12], LJ_TISNUM; jae ->fff_fallback
1320 | fld qword [BASE]
1321 | fld qword [BASE+8]
1322 |.endmacro
1323 |
1324 |.macro .ffunc_nnsse, name 1429 |.macro .ffunc_nnsse, name
1325 | .ffunc_2 name 1430 | .ffunc_2 name
1326 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback 1431 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
@@ -1418,7 +1523,7 @@ static void build_subroutines(BuildCtx *ctx)
1418 | mov dword [BASE-4], LJ_TTAB // Store metatable as default result. 1523 | mov dword [BASE-4], LJ_TTAB // Store metatable as default result.
1419 | mov [BASE-8], TAB:RB 1524 | mov [BASE-8], TAB:RB
1420 | mov RA, TAB:RB->hmask 1525 | mov RA, TAB:RB->hmask
1421 | and RA, STR:RC->hash 1526 | and RA, STR:RC->sid
1422 | imul RA, #NODE 1527 | imul RA, #NODE
1423 | add NODE:RA, TAB:RB->node 1528 | add NODE:RA, TAB:RB->node
1424 |3: // Rearranged logic, because we expect _not_ to find the key. 1529 |3: // Rearranged logic, because we expect _not_ to find the key.
@@ -1526,11 +1631,7 @@ static void build_subroutines(BuildCtx *ctx)
1526 |.else 1631 |.else
1527 | jae ->fff_fallback 1632 | jae ->fff_fallback
1528 |.endif 1633 |.endif
1529 |.if SSE
1530 | movsd xmm0, qword [BASE]; jmp ->fff_resxmm0 1634 | movsd xmm0, qword [BASE]; jmp ->fff_resxmm0
1531 |.else
1532 | fld qword [BASE]; jmp ->fff_resn
1533 |.endif
1534 | 1635 |
1535 |.ffunc_1 tostring 1636 |.ffunc_1 tostring
1536 | // Only handles the string or number case inline. 1637 | // Only handles the string or number case inline.
@@ -1555,9 +1656,9 @@ static void build_subroutines(BuildCtx *ctx)
1555 |.endif 1656 |.endif
1556 | mov L:FCARG1, L:RB 1657 | mov L:FCARG1, L:RB
1557 |.if DUALNUM 1658 |.if DUALNUM
1558 | call extern lj_str_fromnumber@8 // (lua_State *L, cTValue *o) 1659 | call extern lj_strfmt_number@8 // (lua_State *L, cTValue *o)
1559 |.else 1660 |.else
1560 | call extern lj_str_fromnum@8 // (lua_State *L, lua_Number *np) 1661 | call extern lj_strfmt_num@8 // (lua_State *L, lua_Number *np)
1561 |.endif 1662 |.endif
1562 | // GCstr returned in eax (RD). 1663 | // GCstr returned in eax (RD).
1563 | mov BASE, L:RB->base 1664 | mov BASE, L:RB->base
@@ -1569,55 +1670,35 @@ static void build_subroutines(BuildCtx *ctx)
1569 | je >2 // Missing 2nd arg? 1670 | je >2 // Missing 2nd arg?
1570 |1: 1671 |1:
1571 | cmp dword [BASE+4], LJ_TTAB; jne ->fff_fallback 1672 | cmp dword [BASE+4], LJ_TTAB; jne ->fff_fallback
1572 | mov L:RB, SAVE_L
1573 | mov L:RB->base, BASE // Add frame since C call can throw.
1574 | mov L:RB->top, BASE // Dummy frame length is ok.
1575 | mov PC, [BASE-4] 1673 | mov PC, [BASE-4]
1674 | mov RB, BASE // Save BASE.
1576 |.if X64WIN 1675 |.if X64WIN
1577 | lea CARG3d, [BASE+8] 1676 | mov CARG1d, [BASE]
1578 | mov CARG2d, [BASE] // Caveat: CARG2d == BASE. 1677 | lea CARG3d, [BASE-8]
1579 | mov CARG1d, L:RB 1678 | lea CARG2d, [BASE+8] // Caveat: CARG2d == BASE.
1580 |.elif X64 1679 |.elif X64
1581 | mov CARG2d, [BASE] 1680 | mov CARG1d, [BASE]
1582 | lea CARG3d, [BASE+8] // Caveat: CARG3d == BASE. 1681 | lea CARG2d, [BASE+8]
1583 | mov CARG1d, L:RB 1682 | lea CARG3d, [BASE-8] // Caveat: CARG3d == BASE.
1584 |.else 1683 |.else
1585 | mov TAB:RD, [BASE] 1684 | mov TAB:RD, [BASE]
1586 | mov ARG2, TAB:RD 1685 | mov ARG1, TAB:RD
1587 | mov ARG1, L:RB
1588 | add BASE, 8 1686 | add BASE, 8
1687 | mov ARG2, BASE
1688 | sub BASE, 8+8
1589 | mov ARG3, BASE 1689 | mov ARG3, BASE
1590 |.endif 1690 |.endif
1591 | mov SAVE_PC, PC // Needed for ITERN fallback. 1691 | call extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1592 | call extern lj_tab_next // (lua_State *L, GCtab *t, TValue *key) 1692 | // 1=found, 0=end, -1=error returned in eax (RD).
1593 | // Flag returned in eax (RD). 1693 | mov BASE, RB // Restore BASE.
1594 | mov BASE, L:RB->base 1694 | test RD, RD; jg ->fff_res2 // Found key/value.
1595 | test RD, RD; jz >3 // End of traversal? 1695 | js ->fff_fallback_2 // Invalid key.
1596 | // Copy key and value to results. 1696 | // End of traversal: return nil.
1597 |.if X64 1697 | mov dword [BASE-4], LJ_TNIL
1598 | mov RBa, [BASE+8] 1698 | jmp ->fff_res1
1599 | mov RDa, [BASE+16]
1600 | mov [BASE-8], RBa
1601 | mov [BASE], RDa
1602 |.else
1603 | mov RB, [BASE+8]
1604 | mov RD, [BASE+12]
1605 | mov [BASE-8], RB
1606 | mov [BASE-4], RD
1607 | mov RB, [BASE+16]
1608 | mov RD, [BASE+20]
1609 | mov [BASE], RB
1610 | mov [BASE+4], RD
1611 |.endif
1612 |->fff_res2:
1613 | mov RD, 1+2
1614 | jmp ->fff_res
1615 |2: // Set missing 2nd arg to nil. 1699 |2: // Set missing 2nd arg to nil.
1616 | mov dword [BASE+12], LJ_TNIL 1700 | mov dword [BASE+12], LJ_TNIL
1617 | jmp <1 1701 | jmp <1
1618 |3: // End of traversal: return nil.
1619 | mov dword [BASE-4], LJ_TNIL
1620 | jmp ->fff_res1
1621 | 1702 |
1622 |.ffunc_1 pairs 1703 |.ffunc_1 pairs
1623 | mov TAB:RB, [BASE] 1704 | mov TAB:RB, [BASE]
@@ -1648,19 +1729,12 @@ static void build_subroutines(BuildCtx *ctx)
1648 | add RD, 1 1729 | add RD, 1
1649 | mov dword [BASE-4], LJ_TISNUM 1730 | mov dword [BASE-4], LJ_TISNUM
1650 | mov dword [BASE-8], RD 1731 | mov dword [BASE-8], RD
1651 |.elif SSE 1732 |.else
1652 | movsd xmm0, qword [BASE+8] 1733 | movsd xmm0, qword [BASE+8]
1653 | sseconst_1 xmm1, RBa 1734 | sseconst_1 xmm1, RBa
1654 | addsd xmm0, xmm1 1735 | addsd xmm0, xmm1
1655 | cvtsd2si RD, xmm0 1736 | cvttsd2si RD, xmm0
1656 | movsd qword [BASE-8], xmm0 1737 | movsd qword [BASE-8], xmm0
1657 |.else
1658 | fld qword [BASE+8]
1659 | fld1
1660 | faddp st1
1661 | fist ARG1
1662 | fstp qword [BASE-8]
1663 | mov RD, ARG1
1664 |.endif 1738 |.endif
1665 | mov TAB:RB, [BASE] 1739 | mov TAB:RB, [BASE]
1666 | cmp RD, TAB:RB->asize; jae >2 // Not in array part? 1740 | cmp RD, TAB:RB->asize; jae >2 // Not in array part?
@@ -1678,7 +1752,9 @@ static void build_subroutines(BuildCtx *ctx)
1678 | mov [BASE], RB 1752 | mov [BASE], RB
1679 | mov [BASE+4], RD 1753 | mov [BASE+4], RD
1680 |.endif 1754 |.endif
1681 | jmp ->fff_res2 1755 |->fff_res2:
1756 | mov RD, 1+2
1757 | jmp ->fff_res
1682 |2: // Check for empty hash part first. Otherwise call C function. 1758 |2: // Check for empty hash part first. Otherwise call C function.
1683 | cmp dword TAB:RB->hmask, 0; je ->fff_res0 1759 | cmp dword TAB:RB->hmask, 0; je ->fff_res0
1684 | mov FCARG1, TAB:RB 1760 | mov FCARG1, TAB:RB
@@ -1707,12 +1783,9 @@ static void build_subroutines(BuildCtx *ctx)
1707 |.if DUALNUM 1783 |.if DUALNUM
1708 | mov dword [BASE+12], LJ_TISNUM 1784 | mov dword [BASE+12], LJ_TISNUM
1709 | mov dword [BASE+8], 0 1785 | mov dword [BASE+8], 0
1710 |.elif SSE 1786 |.else
1711 | xorps xmm0, xmm0 1787 | xorps xmm0, xmm0
1712 | movsd qword [BASE+8], xmm0 1788 | movsd qword [BASE+8], xmm0
1713 |.else
1714 | fldz
1715 | fstp qword [BASE+8]
1716 |.endif 1789 |.endif
1717 | mov RD, 1+3 1790 | mov RD, 1+3
1718 | jmp ->fff_res 1791 | jmp ->fff_res
@@ -1819,7 +1892,6 @@ static void build_subroutines(BuildCtx *ctx)
1819 | mov ARG3, RA 1892 | mov ARG3, RA
1820 |.endif 1893 |.endif
1821 | call ->vm_resume // (lua_State *L, TValue *base, 0, 0) 1894 | call ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1822 | set_vmstate INTERP
1823 | 1895 |
1824 | mov L:RB, SAVE_L 1896 | mov L:RB, SAVE_L
1825 |.if X64 1897 |.if X64
@@ -1828,6 +1900,9 @@ static void build_subroutines(BuildCtx *ctx)
1828 | mov L:PC, ARG1 // The callee doesn't modify SAVE_L. 1900 | mov L:PC, ARG1 // The callee doesn't modify SAVE_L.
1829 |.endif 1901 |.endif
1830 | mov BASE, L:RB->base 1902 | mov BASE, L:RB->base
1903 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
1904 | set_vmstate INTERP
1905 |
1831 | cmp eax, LUA_YIELD 1906 | cmp eax, LUA_YIELD
1832 | ja >8 1907 | ja >8
1833 |4: 1908 |4:
@@ -1942,12 +2017,10 @@ static void build_subroutines(BuildCtx *ctx)
1942 |->fff_resi: // Dummy. 2017 |->fff_resi: // Dummy.
1943 |.endif 2018 |.endif
1944 | 2019 |
1945 |.if SSE
1946 |->fff_resn: 2020 |->fff_resn:
1947 | mov PC, [BASE-4] 2021 | mov PC, [BASE-4]
1948 | fstp qword [BASE-8] 2022 | fstp qword [BASE-8]
1949 | jmp ->fff_res1 2023 | jmp ->fff_res1
1950 |.endif
1951 | 2024 |
1952 | .ffunc_1 math_abs 2025 | .ffunc_1 math_abs
1953 |.if DUALNUM 2026 |.if DUALNUM
@@ -1971,8 +2044,6 @@ static void build_subroutines(BuildCtx *ctx)
1971 |.else 2044 |.else
1972 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback 2045 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
1973 |.endif 2046 |.endif
1974 |
1975 |.if SSE
1976 | movsd xmm0, qword [BASE] 2047 | movsd xmm0, qword [BASE]
1977 | sseconst_abs xmm1, RDa 2048 | sseconst_abs xmm1, RDa
1978 | andps xmm0, xmm1 2049 | andps xmm0, xmm1
@@ -1980,15 +2051,6 @@ static void build_subroutines(BuildCtx *ctx)
1980 | mov PC, [BASE-4] 2051 | mov PC, [BASE-4]
1981 | movsd qword [BASE-8], xmm0 2052 | movsd qword [BASE-8], xmm0
1982 | // fallthrough 2053 | // fallthrough
1983 |.else
1984 | fld qword [BASE]
1985 | fabs
1986 | // fallthrough
1987 |->fff_resxmm0: // Dummy.
1988 |->fff_resn:
1989 | mov PC, [BASE-4]
1990 | fstp qword [BASE-8]
1991 |.endif
1992 | 2054 |
1993 |->fff_res1: 2055 |->fff_res1:
1994 | mov RD, 1+1 2056 | mov RD, 1+1
@@ -2015,6 +2077,12 @@ static void build_subroutines(BuildCtx *ctx)
2015 | mov RAa, -8 // Results start at BASE+RA = BASE-8. 2077 | mov RAa, -8 // Results start at BASE+RA = BASE-8.
2016 | jmp ->vm_return 2078 | jmp ->vm_return
2017 | 2079 |
2080 |.if X64
2081 |.define fff_resfp, fff_resxmm0
2082 |.else
2083 |.define fff_resfp, fff_resn
2084 |.endif
2085 |
2018 |.macro math_round, func 2086 |.macro math_round, func
2019 | .ffunc math_ .. func 2087 | .ffunc math_ .. func
2020 |.if DUALNUM 2088 |.if DUALNUM
@@ -2025,107 +2093,75 @@ static void build_subroutines(BuildCtx *ctx)
2025 |.else 2093 |.else
2026 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback 2094 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
2027 |.endif 2095 |.endif
2028 |.if SSE
2029 | movsd xmm0, qword [BASE] 2096 | movsd xmm0, qword [BASE]
2030 | call ->vm_ .. func 2097 | call ->vm_ .. func .. _sse
2031 | .if DUALNUM 2098 |.if DUALNUM
2032 | cvtsd2si RB, xmm0 2099 | cvttsd2si RB, xmm0
2033 | cmp RB, 0x80000000 2100 | cmp RB, 0x80000000
2034 | jne ->fff_resi 2101 | jne ->fff_resi
2035 | cvtsi2sd xmm1, RB 2102 | cvtsi2sd xmm1, RB
2036 | ucomisd xmm0, xmm1 2103 | ucomisd xmm0, xmm1
2037 | jp ->fff_resxmm0 2104 | jp ->fff_resxmm0
2038 | je ->fff_resi 2105 | je ->fff_resi
2039 | .endif
2040 | jmp ->fff_resxmm0
2041 |.else
2042 | fld qword [BASE]
2043 | call ->vm_ .. func
2044 | .if DUALNUM
2045 | fist ARG1
2046 | mov RB, ARG1
2047 | cmp RB, 0x80000000; jne >2
2048 | fdup
2049 | fild ARG1
2050 | fcomparepp
2051 | jp ->fff_resn
2052 | jne ->fff_resn
2053 |2:
2054 | fpop
2055 | jmp ->fff_resi
2056 | .else
2057 | jmp ->fff_resn
2058 | .endif
2059 |.endif 2106 |.endif
2107 | jmp ->fff_resxmm0
2060 |.endmacro 2108 |.endmacro
2061 | 2109 |
2062 | math_round floor 2110 | math_round floor
2063 | math_round ceil 2111 | math_round ceil
2064 | 2112 |
2065 |.if SSE
2066 |.ffunc_nsse math_sqrt, sqrtsd; jmp ->fff_resxmm0 2113 |.ffunc_nsse math_sqrt, sqrtsd; jmp ->fff_resxmm0
2067 |.else
2068 |.ffunc_n math_sqrt; fsqrt; jmp ->fff_resn
2069 |.endif
2070 | 2114 |
2071 |.ffunc math_log 2115 |.ffunc math_log
2072 | cmp NARGS:RD, 1+1; jne ->fff_fallback // Exactly one argument. 2116 | cmp NARGS:RD, 1+1; jne ->fff_fallback // Exactly one argument.
2073 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback 2117 | cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
2074 | fldln2; fld qword [BASE]; fyl2x; jmp ->fff_resn 2118 | movsd xmm0, qword [BASE]
2075 | 2119 |.if not X64
2076 |.ffunc_n math_log10, fldlg2; fyl2x; jmp ->fff_resn 2120 | movsd FPARG1, xmm0
2077 |.ffunc_n math_exp; call ->vm_exp_x87; jmp ->fff_resn 2121 |.endif
2078 | 2122 | mov RB, BASE
2079 |.ffunc_n math_sin; fsin; jmp ->fff_resn 2123 | call extern log
2080 |.ffunc_n math_cos; fcos; jmp ->fff_resn 2124 | mov BASE, RB
2081 |.ffunc_n math_tan; fptan; fpop; jmp ->fff_resn 2125 | jmp ->fff_resfp
2082 |
2083 |.ffunc_n math_asin
2084 | fdup; fmul st0; fld1; fsubrp st1; fsqrt; fpatan
2085 | jmp ->fff_resn
2086 |.ffunc_n math_acos
2087 | fdup; fmul st0; fld1; fsubrp st1; fsqrt; fxch; fpatan
2088 | jmp ->fff_resn
2089 |.ffunc_n math_atan; fld1; fpatan; jmp ->fff_resn
2090 | 2126 |
2091 |.macro math_extern, func 2127 |.macro math_extern, func
2092 |.if SSE
2093 | .ffunc_nsse math_ .. func 2128 | .ffunc_nsse math_ .. func
2094 | .if not X64 2129 |.if not X64
2095 | movsd FPARG1, xmm0 2130 | movsd FPARG1, xmm0
2096 | .endif
2097 |.else
2098 | .ffunc_n math_ .. func
2099 | fstp FPARG1
2100 |.endif 2131 |.endif
2101 | mov RB, BASE 2132 | mov RB, BASE
2102 | call extern lj_vm_ .. func 2133 | call extern func
2103 | mov BASE, RB 2134 | mov BASE, RB
2104 | .if X64 2135 | jmp ->fff_resfp
2105 | jmp ->fff_resxmm0
2106 | .else
2107 | jmp ->fff_resn
2108 | .endif
2109 |.endmacro 2136 |.endmacro
2110 | 2137 |
2138 |.macro math_extern2, func
2139 | .ffunc_nnsse math_ .. func
2140 |.if not X64
2141 | movsd FPARG1, xmm0
2142 | movsd FPARG3, xmm1
2143 |.endif
2144 | mov RB, BASE
2145 | call extern func
2146 | mov BASE, RB
2147 | jmp ->fff_resfp
2148 |.endmacro
2149 |
2150 | math_extern log10
2151 | math_extern exp
2152 | math_extern sin
2153 | math_extern cos
2154 | math_extern tan
2155 | math_extern asin
2156 | math_extern acos
2157 | math_extern atan
2111 | math_extern sinh 2158 | math_extern sinh
2112 | math_extern cosh 2159 | math_extern cosh
2113 | math_extern tanh 2160 | math_extern tanh
2161 | math_extern2 pow
2162 | math_extern2 atan2
2163 | math_extern2 fmod
2114 | 2164 |
2115 |->ff_math_deg:
2116 |.if SSE
2117 |.ffunc_nsse math_rad
2118 | mov CFUNC:RB, [BASE-8]
2119 | mulsd xmm0, qword CFUNC:RB->upvalue[0]
2120 | jmp ->fff_resxmm0
2121 |.else
2122 |.ffunc_n math_rad
2123 | mov CFUNC:RB, [BASE-8]
2124 | fmul qword CFUNC:RB->upvalue[0]
2125 | jmp ->fff_resn
2126 |.endif
2127 |
2128 |.ffunc_nn math_atan2; fpatan; jmp ->fff_resn
2129 |.ffunc_nnr math_ldexp; fscale; fpop1; jmp ->fff_resn 2165 |.ffunc_nnr math_ldexp; fscale; fpop1; jmp ->fff_resn
2130 | 2166 |
2131 |.ffunc_1 math_frexp 2167 |.ffunc_1 math_frexp
@@ -2140,65 +2176,34 @@ static void build_subroutines(BuildCtx *ctx)
2140 | cmp RB, 0x00200000; jb >4 2176 | cmp RB, 0x00200000; jb >4
2141 |1: 2177 |1:
2142 | shr RB, 21; sub RB, RC // Extract and unbias exponent. 2178 | shr RB, 21; sub RB, RC // Extract and unbias exponent.
2143 |.if SSE
2144 | cvtsi2sd xmm0, RB 2179 | cvtsi2sd xmm0, RB
2145 |.else
2146 | mov TMP1, RB; fild TMP1
2147 |.endif
2148 | mov RB, [BASE-4] 2180 | mov RB, [BASE-4]
2149 | and RB, 0x800fffff // Mask off exponent. 2181 | and RB, 0x800fffff // Mask off exponent.
2150 | or RB, 0x3fe00000 // Put mantissa in range [0.5,1) or 0. 2182 | or RB, 0x3fe00000 // Put mantissa in range [0.5,1) or 0.
2151 | mov [BASE-4], RB 2183 | mov [BASE-4], RB
2152 |2: 2184 |2:
2153 |.if SSE
2154 | movsd qword [BASE], xmm0 2185 | movsd qword [BASE], xmm0
2155 |.else
2156 | fstp qword [BASE]
2157 |.endif
2158 | mov RD, 1+2 2186 | mov RD, 1+2
2159 | jmp ->fff_res 2187 | jmp ->fff_res
2160 |3: // Return +-0, +-Inf, NaN unmodified and an exponent of 0. 2188 |3: // Return +-0, +-Inf, NaN unmodified and an exponent of 0.
2161 |.if SSE
2162 | xorps xmm0, xmm0; jmp <2 2189 | xorps xmm0, xmm0; jmp <2
2163 |.else
2164 | fldz; jmp <2
2165 |.endif
2166 |4: // Handle denormals by multiplying with 2^54 and adjusting the bias. 2190 |4: // Handle denormals by multiplying with 2^54 and adjusting the bias.
2167 |.if SSE
2168 | movsd xmm0, qword [BASE] 2191 | movsd xmm0, qword [BASE]
2169 | sseconst_hi xmm1, RBa, 43500000 // 2^54. 2192 | sseconst_hi xmm1, RBa, 43500000 // 2^54.
2170 | mulsd xmm0, xmm1 2193 | mulsd xmm0, xmm1
2171 | movsd qword [BASE-8], xmm0 2194 | movsd qword [BASE-8], xmm0
2172 |.else
2173 | fld qword [BASE]
2174 | mov TMP1, 0x5a800000; fmul TMP1 // x = x*2^54
2175 | fstp qword [BASE-8]
2176 |.endif
2177 | mov RB, [BASE-4]; mov RC, 1076; shl RB, 1; jmp <1 2195 | mov RB, [BASE-4]; mov RC, 1076; shl RB, 1; jmp <1
2178 | 2196 |
2179 |.if SSE
2180 |.ffunc_nsse math_modf 2197 |.ffunc_nsse math_modf
2181 |.else
2182 |.ffunc_n math_modf
2183 |.endif
2184 | mov RB, [BASE+4] 2198 | mov RB, [BASE+4]
2185 | mov PC, [BASE-4] 2199 | mov PC, [BASE-4]
2186 | shl RB, 1; cmp RB, 0xffe00000; je >4 // +-Inf? 2200 | shl RB, 1; cmp RB, 0xffe00000; je >4 // +-Inf?
2187 |.if SSE
2188 | movaps xmm4, xmm0 2201 | movaps xmm4, xmm0
2189 | call ->vm_trunc 2202 | call ->vm_trunc_sse
2190 | subsd xmm4, xmm0 2203 | subsd xmm4, xmm0
2191 |1: 2204 |1:
2192 | movsd qword [BASE-8], xmm0 2205 | movsd qword [BASE-8], xmm0
2193 | movsd qword [BASE], xmm4 2206 | movsd qword [BASE], xmm4
2194 |.else
2195 | fdup
2196 | call ->vm_trunc
2197 | fsub st1, st0
2198 |1:
2199 | fstp qword [BASE-8]
2200 | fstp qword [BASE]
2201 |.endif
2202 | mov RC, [BASE-4]; mov RB, [BASE+4] 2207 | mov RC, [BASE-4]; mov RB, [BASE+4]
2203 | xor RC, RB; js >3 // Need to adjust sign? 2208 | xor RC, RB; js >3 // Need to adjust sign?
2204 |2: 2209 |2:
@@ -2208,25 +2213,10 @@ static void build_subroutines(BuildCtx *ctx)
2208 | xor RB, 0x80000000; mov [BASE+4], RB // Flip sign of fraction. 2213 | xor RB, 0x80000000; mov [BASE+4], RB // Flip sign of fraction.
2209 | jmp <2 2214 | jmp <2
2210 |4: 2215 |4:
2211 |.if SSE
2212 | xorps xmm4, xmm4; jmp <1 // Return +-Inf and +-0. 2216 | xorps xmm4, xmm4; jmp <1 // Return +-Inf and +-0.
2213 |.else
2214 | fldz; fxch; jmp <1 // Return +-Inf and +-0.
2215 |.endif
2216 |
2217 |.ffunc_nnr math_fmod
2218 |1: ; fprem; fnstsw ax; and ax, 0x400; jnz <1
2219 | fpop1
2220 | jmp ->fff_resn
2221 | 2217 |
2222 |.if SSE 2218 |.macro math_minmax, name, cmovop, sseop
2223 |.ffunc_nnsse math_pow; call ->vm_pow; jmp ->fff_resxmm0 2219 | .ffunc_1 name
2224 |.else
2225 |.ffunc_nn math_pow; call ->vm_pow; jmp ->fff_resn
2226 |.endif
2227 |
2228 |.macro math_minmax, name, cmovop, fcmovop, sseop
2229 | .ffunc name
2230 | mov RA, 2 2220 | mov RA, 2
2231 | cmp dword [BASE+4], LJ_TISNUM 2221 | cmp dword [BASE+4], LJ_TISNUM
2232 |.if DUALNUM 2222 |.if DUALNUM
@@ -2242,12 +2232,7 @@ static void build_subroutines(BuildCtx *ctx)
2242 |3: 2232 |3:
2243 | ja ->fff_fallback 2233 | ja ->fff_fallback
2244 | // Convert intermediate result to number and continue below. 2234 | // Convert intermediate result to number and continue below.
2245 |.if SSE
2246 | cvtsi2sd xmm0, RB 2235 | cvtsi2sd xmm0, RB
2247 |.else
2248 | mov TMP1, RB
2249 | fild TMP1
2250 |.endif
2251 | jmp >6 2236 | jmp >6
2252 |4: 2237 |4:
2253 | ja ->fff_fallback 2238 | ja ->fff_fallback
@@ -2255,7 +2240,6 @@ static void build_subroutines(BuildCtx *ctx)
2255 | jae ->fff_fallback 2240 | jae ->fff_fallback
2256 |.endif 2241 |.endif
2257 | 2242 |
2258 |.if SSE
2259 | movsd xmm0, qword [BASE] 2243 | movsd xmm0, qword [BASE]
2260 |5: // Handle numbers or integers. 2244 |5: // Handle numbers or integers.
2261 | cmp RA, RD; jae ->fff_resxmm0 2245 | cmp RA, RD; jae ->fff_resxmm0
@@ -2274,48 +2258,13 @@ static void build_subroutines(BuildCtx *ctx)
2274 | sseop xmm0, xmm1 2258 | sseop xmm0, xmm1
2275 | add RA, 1 2259 | add RA, 1
2276 | jmp <5 2260 | jmp <5
2277 |.else
2278 | fld qword [BASE]
2279 |5: // Handle numbers or integers.
2280 | cmp RA, RD; jae ->fff_resn
2281 | cmp dword [BASE+RA*8-4], LJ_TISNUM
2282 |.if DUALNUM
2283 | jb >6
2284 | ja >9
2285 | fild dword [BASE+RA*8-8]
2286 | jmp >7
2287 |.else
2288 | jae >9
2289 |.endif
2290 |6:
2291 | fld qword [BASE+RA*8-8]
2292 |7:
2293 | fucomi st1; fcmovop st1; fpop1
2294 | add RA, 1
2295 | jmp <5
2296 |.endif
2297 |.endmacro 2261 |.endmacro
2298 | 2262 |
2299 | math_minmax math_min, cmovg, fcmovnbe, minsd 2263 | math_minmax math_min, cmovg, minsd
2300 | math_minmax math_max, cmovl, fcmovbe, maxsd 2264 | math_minmax math_max, cmovl, maxsd
2301 |.if not SSE
2302 |9:
2303 | fpop; jmp ->fff_fallback
2304 |.endif
2305 | 2265 |
2306 |//-- String library ----------------------------------------------------- 2266 |//-- String library -----------------------------------------------------
2307 | 2267 |
2308 |.ffunc_1 string_len
2309 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
2310 | mov STR:RB, [BASE]
2311 |.if DUALNUM
2312 | mov RB, dword STR:RB->len; jmp ->fff_resi
2313 |.elif SSE
2314 | cvtsi2sd xmm0, dword STR:RB->len; jmp ->fff_resxmm0
2315 |.else
2316 | fild dword STR:RB->len; jmp ->fff_resn
2317 |.endif
2318 |
2319 |.ffunc string_byte // Only handle the 1-arg case here. 2268 |.ffunc string_byte // Only handle the 1-arg case here.
2320 | cmp NARGS:RD, 1+1; jne ->fff_fallback 2269 | cmp NARGS:RD, 1+1; jne ->fff_fallback
2321 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback 2270 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
@@ -2326,10 +2275,8 @@ static void build_subroutines(BuildCtx *ctx)
2326 | movzx RB, byte STR:RB[1] 2275 | movzx RB, byte STR:RB[1]
2327 |.if DUALNUM 2276 |.if DUALNUM
2328 | jmp ->fff_resi 2277 | jmp ->fff_resi
2329 |.elif SSE
2330 | cvtsi2sd xmm0, RB; jmp ->fff_resxmm0
2331 |.else 2278 |.else
2332 | mov TMP1, RB; fild TMP1; jmp ->fff_resn 2279 | cvtsi2sd xmm0, RB; jmp ->fff_resxmm0
2333 |.endif 2280 |.endif
2334 | 2281 |
2335 |.ffunc string_char // Only handle the 1-arg case here. 2282 |.ffunc string_char // Only handle the 1-arg case here.
@@ -2341,16 +2288,11 @@ static void build_subroutines(BuildCtx *ctx)
2341 | mov RB, dword [BASE] 2288 | mov RB, dword [BASE]
2342 | cmp RB, 255; ja ->fff_fallback 2289 | cmp RB, 255; ja ->fff_fallback
2343 | mov TMP2, RB 2290 | mov TMP2, RB
2344 |.elif SSE 2291 |.else
2345 | jae ->fff_fallback 2292 | jae ->fff_fallback
2346 | cvttsd2si RB, qword [BASE] 2293 | cvttsd2si RB, qword [BASE]
2347 | cmp RB, 255; ja ->fff_fallback 2294 | cmp RB, 255; ja ->fff_fallback
2348 | mov TMP2, RB 2295 | mov TMP2, RB
2349 |.else
2350 | jae ->fff_fallback
2351 | fld qword [BASE]
2352 | fistp TMP2
2353 | cmp TMP2, 255; ja ->fff_fallback
2354 |.endif 2296 |.endif
2355 |.if X64 2297 |.if X64
2356 | mov TMP3, 1 2298 | mov TMP3, 1
@@ -2371,6 +2313,7 @@ static void build_subroutines(BuildCtx *ctx)
2371 |.endif 2313 |.endif
2372 | mov SAVE_PC, PC 2314 | mov SAVE_PC, PC
2373 | call extern lj_str_new // (lua_State *L, char *str, size_t l) 2315 | call extern lj_str_new // (lua_State *L, char *str, size_t l)
2316 |->fff_resstr:
2374 | // GCstr * returned in eax (RD). 2317 | // GCstr * returned in eax (RD).
2375 | mov BASE, L:RB->base 2318 | mov BASE, L:RB->base
2376 | mov PC, [BASE-4] 2319 | mov PC, [BASE-4]
@@ -2388,14 +2331,10 @@ static void build_subroutines(BuildCtx *ctx)
2388 | jne ->fff_fallback 2331 | jne ->fff_fallback
2389 | mov RB, dword [BASE+16] 2332 | mov RB, dword [BASE+16]
2390 | mov TMP2, RB 2333 | mov TMP2, RB
2391 |.elif SSE 2334 |.else
2392 | jae ->fff_fallback 2335 | jae ->fff_fallback
2393 | cvttsd2si RB, qword [BASE+16] 2336 | cvttsd2si RB, qword [BASE+16]
2394 | mov TMP2, RB 2337 | mov TMP2, RB
2395 |.else
2396 | jae ->fff_fallback
2397 | fld qword [BASE+16]
2398 | fistp TMP2
2399 |.endif 2338 |.endif
2400 |1: 2339 |1:
2401 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback 2340 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
@@ -2410,12 +2349,8 @@ static void build_subroutines(BuildCtx *ctx)
2410 | mov RB, STR:RB->len 2349 | mov RB, STR:RB->len
2411 |.if DUALNUM 2350 |.if DUALNUM
2412 | mov RA, dword [BASE+8] 2351 | mov RA, dword [BASE+8]
2413 |.elif SSE
2414 | cvttsd2si RA, qword [BASE+8]
2415 |.else 2352 |.else
2416 | fld qword [BASE+8] 2353 | cvttsd2si RA, qword [BASE+8]
2417 | fistp ARG3
2418 | mov RA, ARG3
2419 |.endif 2354 |.endif
2420 | mov RC, TMP2 2355 | mov RC, TMP2
2421 | cmp RB, RC // len < end? (unsigned compare) 2356 | cmp RB, RC // len < end? (unsigned compare)
@@ -2459,136 +2394,34 @@ static void build_subroutines(BuildCtx *ctx)
2459 | xor RC, RC // Zero length. Any ptr in RB is ok. 2394 | xor RC, RC // Zero length. Any ptr in RB is ok.
2460 | jmp <4 2395 | jmp <4
2461 | 2396 |
2462 |.ffunc string_rep // Only handle the 1-char case inline. 2397 |.macro ffstring_op, name
2398 | .ffunc_1 string_ .. name
2463 | ffgccheck 2399 | ffgccheck
2464 | cmp NARGS:RD, 2+1; jne ->fff_fallback // Exactly 2 arguments.
2465 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback 2400 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
2466 | cmp dword [BASE+12], LJ_TISNUM 2401 | mov L:RB, SAVE_L
2467 | mov STR:RB, [BASE] 2402 | lea SBUF:FCARG1, [DISPATCH+DISPATCH_GL(tmpbuf)]
2468 |.if DUALNUM 2403 | mov L:RB->base, BASE
2469 | jne ->fff_fallback 2404 | mov STR:FCARG2, [BASE] // Caveat: FCARG2 == BASE
2470 | mov RC, dword [BASE+8] 2405 | mov RCa, SBUF:FCARG1->b
2471 |.elif SSE 2406 | mov SBUF:FCARG1->L, L:RB
2472 | jae ->fff_fallback 2407 | mov SBUF:FCARG1->w, RCa
2473 | cvttsd2si RC, qword [BASE+8] 2408 | mov SAVE_PC, PC
2474 |.else 2409 | call extern lj_buf_putstr_ .. name .. @8
2475 | jae ->fff_fallback 2410 | mov FCARG1, eax
2476 | fld qword [BASE+8] 2411 | call extern lj_buf_tostr@4
2477 | fistp TMP2 2412 | jmp ->fff_resstr
2478 | mov RC, TMP2
2479 |.endif
2480 | test RC, RC
2481 | jle ->fff_emptystr // Count <= 0? (or non-int)
2482 | cmp dword STR:RB->len, 1
2483 | jb ->fff_emptystr // Zero length string?
2484 | jne ->fff_fallback_2 // Fallback for > 1-char strings.
2485 | cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC; jb ->fff_fallback_2
2486 | movzx RA, byte STR:RB[1]
2487 | mov RB, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
2488 |.if X64
2489 | mov TMP3, RC
2490 |.else
2491 | mov ARG3, RC
2492 |.endif
2493 |1: // Fill buffer with char. Yes, this is suboptimal code (do you care?).
2494 | mov [RB], RAL
2495 | add RB, 1
2496 | sub RC, 1
2497 | jnz <1
2498 | mov RD, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
2499 | jmp ->fff_newstr
2500 |
2501 |.ffunc_1 string_reverse
2502 | ffgccheck
2503 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
2504 | mov STR:RB, [BASE]
2505 | mov RC, STR:RB->len
2506 | test RC, RC
2507 | jz ->fff_emptystr // Zero length string?
2508 | cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC; jb ->fff_fallback_1
2509 | add RB, #STR
2510 | mov TMP2, PC // Need another temp register.
2511 |.if X64
2512 | mov TMP3, RC
2513 |.else
2514 | mov ARG3, RC
2515 |.endif
2516 | mov PC, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
2517 |1:
2518 | movzx RA, byte [RB]
2519 | add RB, 1
2520 | sub RC, 1
2521 | mov [PC+RC], RAL
2522 | jnz <1
2523 | mov RD, PC
2524 | mov PC, TMP2
2525 | jmp ->fff_newstr
2526 |
2527 |.macro ffstring_case, name, lo, hi
2528 | .ffunc_1 name
2529 | ffgccheck
2530 | cmp dword [BASE+4], LJ_TSTR; jne ->fff_fallback
2531 | mov STR:RB, [BASE]
2532 | mov RC, STR:RB->len
2533 | cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC; jb ->fff_fallback_1
2534 | add RB, #STR
2535 | mov TMP2, PC // Need another temp register.
2536 |.if X64
2537 | mov TMP3, RC
2538 |.else
2539 | mov ARG3, RC
2540 |.endif
2541 | mov PC, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
2542 | jmp >3
2543 |1: // ASCII case conversion. Yes, this is suboptimal code (do you care?).
2544 | movzx RA, byte [RB+RC]
2545 | cmp RA, lo
2546 | jb >2
2547 | cmp RA, hi
2548 | ja >2
2549 | xor RA, 0x20
2550 |2:
2551 | mov [PC+RC], RAL
2552 |3:
2553 | sub RC, 1
2554 | jns <1
2555 | mov RD, PC
2556 | mov PC, TMP2
2557 | jmp ->fff_newstr
2558 |.endmacro 2413 |.endmacro
2559 | 2414 |
2560 |ffstring_case string_lower, 0x41, 0x5a 2415 |ffstring_op reverse
2561 |ffstring_case string_upper, 0x61, 0x7a 2416 |ffstring_op lower
2562 | 2417 |ffstring_op upper
2563 |//-- Table library ------------------------------------------------------
2564 |
2565 |.ffunc_1 table_getn
2566 | cmp dword [BASE+4], LJ_TTAB; jne ->fff_fallback
2567 | mov RB, BASE // Save BASE.
2568 | mov TAB:FCARG1, [BASE]
2569 | call extern lj_tab_len@4 // LJ_FASTCALL (GCtab *t)
2570 | // Length of table returned in eax (RD).
2571 | mov BASE, RB // Restore BASE.
2572 |.if DUALNUM
2573 | mov RB, RD; jmp ->fff_resi
2574 |.elif SSE
2575 | cvtsi2sd xmm0, RD; jmp ->fff_resxmm0
2576 |.else
2577 | mov ARG1, RD; fild ARG1; jmp ->fff_resn
2578 |.endif
2579 | 2418 |
2580 |//-- Bit library -------------------------------------------------------- 2419 |//-- Bit library --------------------------------------------------------
2581 | 2420 |
2582 |.define TOBIT_BIAS, 0x59c00000 // 2^52 + 2^51 (float, not double!).
2583 |
2584 |.macro .ffunc_bit, name, kind, fdef 2421 |.macro .ffunc_bit, name, kind, fdef
2585 | fdef name 2422 | fdef name
2586 |.if kind == 2 2423 |.if kind == 2
2587 |.if SSE
2588 | sseconst_tobit xmm1, RBa 2424 | sseconst_tobit xmm1, RBa
2589 |.else
2590 | mov TMP1, TOBIT_BIAS
2591 |.endif
2592 |.endif 2425 |.endif
2593 | cmp dword [BASE+4], LJ_TISNUM 2426 | cmp dword [BASE+4], LJ_TISNUM
2594 |.if DUALNUM 2427 |.if DUALNUM
@@ -2604,24 +2437,12 @@ static void build_subroutines(BuildCtx *ctx)
2604 |.else 2437 |.else
2605 | jae ->fff_fallback 2438 | jae ->fff_fallback
2606 |.endif 2439 |.endif
2607 |.if SSE
2608 | movsd xmm0, qword [BASE] 2440 | movsd xmm0, qword [BASE]
2609 |.if kind < 2 2441 |.if kind < 2
2610 | sseconst_tobit xmm1, RBa 2442 | sseconst_tobit xmm1, RBa
2611 |.endif 2443 |.endif
2612 | addsd xmm0, xmm1 2444 | addsd xmm0, xmm1
2613 | movd RB, xmm0 2445 | movd RB, xmm0
2614 |.else
2615 | fld qword [BASE]
2616 |.if kind < 2
2617 | mov TMP1, TOBIT_BIAS
2618 |.endif
2619 | fadd TMP1
2620 | fstp FPARG1
2621 |.if kind > 0
2622 | mov RB, ARG1
2623 |.endif
2624 |.endif
2625 |2: 2446 |2:
2626 |.endmacro 2447 |.endmacro
2627 | 2448 |
@@ -2630,15 +2451,7 @@ static void build_subroutines(BuildCtx *ctx)
2630 |.endmacro 2451 |.endmacro
2631 | 2452 |
2632 |.ffunc_bit bit_tobit, 0 2453 |.ffunc_bit bit_tobit, 0
2633 |.if DUALNUM or SSE
2634 |.if not SSE
2635 | mov RB, ARG1
2636 |.endif
2637 | jmp ->fff_resbit 2454 | jmp ->fff_resbit
2638 |.else
2639 | fild ARG1
2640 | jmp ->fff_resn
2641 |.endif
2642 | 2455 |
2643 |.macro .ffunc_bit_op, name, ins 2456 |.macro .ffunc_bit_op, name, ins
2644 | .ffunc_bit name, 2 2457 | .ffunc_bit name, 2
@@ -2658,17 +2471,10 @@ static void build_subroutines(BuildCtx *ctx)
2658 |.else 2471 |.else
2659 | jae ->fff_fallback_bit_op 2472 | jae ->fff_fallback_bit_op
2660 |.endif 2473 |.endif
2661 |.if SSE
2662 | movsd xmm0, qword [RD] 2474 | movsd xmm0, qword [RD]
2663 | addsd xmm0, xmm1 2475 | addsd xmm0, xmm1
2664 | movd RA, xmm0 2476 | movd RA, xmm0
2665 | ins RB, RA 2477 | ins RB, RA
2666 |.else
2667 | fld qword [RD]
2668 | fadd TMP1
2669 | fstp FPARG1
2670 | ins RB, ARG1
2671 |.endif
2672 | sub RD, 8 2478 | sub RD, 8
2673 | jmp <1 2479 | jmp <1
2674 |.endmacro 2480 |.endmacro
@@ -2685,15 +2491,10 @@ static void build_subroutines(BuildCtx *ctx)
2685 | not RB 2491 | not RB
2686 |.if DUALNUM 2492 |.if DUALNUM
2687 | jmp ->fff_resbit 2493 | jmp ->fff_resbit
2688 |.elif SSE 2494 |.else
2689 |->fff_resbit: 2495 |->fff_resbit:
2690 | cvtsi2sd xmm0, RB 2496 | cvtsi2sd xmm0, RB
2691 | jmp ->fff_resxmm0 2497 | jmp ->fff_resxmm0
2692 |.else
2693 |->fff_resbit:
2694 | mov ARG1, RB
2695 | fild ARG1
2696 | jmp ->fff_resn
2697 |.endif 2498 |.endif
2698 | 2499 |
2699 |->fff_fallback_bit_op: 2500 |->fff_fallback_bit_op:
@@ -2706,22 +2507,13 @@ static void build_subroutines(BuildCtx *ctx)
2706 | // Note: no inline conversion from number for 2nd argument! 2507 | // Note: no inline conversion from number for 2nd argument!
2707 | cmp dword [BASE+12], LJ_TISNUM; jne ->fff_fallback 2508 | cmp dword [BASE+12], LJ_TISNUM; jne ->fff_fallback
2708 | mov RA, dword [BASE+8] 2509 | mov RA, dword [BASE+8]
2709 |.elif SSE 2510 |.else
2710 | .ffunc_nnsse name 2511 | .ffunc_nnsse name
2711 | sseconst_tobit xmm2, RBa 2512 | sseconst_tobit xmm2, RBa
2712 | addsd xmm0, xmm2 2513 | addsd xmm0, xmm2
2713 | addsd xmm1, xmm2 2514 | addsd xmm1, xmm2
2714 | movd RB, xmm0 2515 | movd RB, xmm0
2715 | movd RA, xmm1 2516 | movd RA, xmm1
2716 |.else
2717 | .ffunc_nn name
2718 | mov TMP1, TOBIT_BIAS
2719 | fadd TMP1
2720 | fstp FPARG3
2721 | fadd TMP1
2722 | fstp FPARG1
2723 | mov RA, ARG3
2724 | mov RB, ARG1
2725 |.endif 2517 |.endif
2726 | ins RB, cl // Assumes RA is ecx. 2518 | ins RB, cl // Assumes RA is ecx.
2727 | jmp ->fff_resbit 2519 | jmp ->fff_resbit
@@ -2855,7 +2647,7 @@ static void build_subroutines(BuildCtx *ctx)
2855 | mov FCARG2, PC // Caveat: FCARG2 == BASE 2647 | mov FCARG2, PC // Caveat: FCARG2 == BASE
2856 | mov FCARG1, L:RB 2648 | mov FCARG1, L:RB
2857 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC. 2649 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
2858 | call extern lj_dispatch_ins@8 // (lua_State *L, BCIns *pc) 2650 | call extern lj_dispatch_ins@8 // (lua_State *L, const BCIns *pc)
2859 |3: 2651 |3:
2860 | mov BASE, L:RB->base 2652 | mov BASE, L:RB->base
2861 |4: 2653 |4:
@@ -2926,6 +2718,79 @@ static void build_subroutines(BuildCtx *ctx)
2926 | add NARGS:RD, 1 2718 | add NARGS:RD, 1
2927 | jmp RBa 2719 | jmp RBa
2928 | 2720 |
2721 |->cont_stitch: // Trace stitching.
2722 |.if JIT
2723 | // BASE = base, RC = result, RB = mbase
2724 | mov TRACE:RA, [RB-24] // Save previous trace.
2725 | mov TMP1, TRACE:RA
2726 | mov TMP3, DISPATCH // Need one more register.
2727 | mov DISPATCH, MULTRES
2728 | movzx RA, PC_RA
2729 | lea RA, [BASE+RA*8] // Call base.
2730 | sub DISPATCH, 1
2731 | jz >2
2732 |1: // Move results down.
2733 |.if X64
2734 | mov RBa, [RC]
2735 | mov [RA], RBa
2736 |.else
2737 | mov RB, [RC]
2738 | mov [RA], RB
2739 | mov RB, [RC+4]
2740 | mov [RA+4], RB
2741 |.endif
2742 | add RC, 8
2743 | add RA, 8
2744 | sub DISPATCH, 1
2745 | jnz <1
2746 |2:
2747 | movzx RC, PC_RA
2748 | movzx RB, PC_RB
2749 | add RC, RB
2750 | lea RC, [BASE+RC*8-8]
2751 |3:
2752 | cmp RC, RA
2753 | ja >9 // More results wanted?
2754 |
2755 | mov DISPATCH, TMP3
2756 | mov TRACE:RD, TMP1 // Get previous trace.
2757 | movzx RB, word TRACE:RD->traceno
2758 | movzx RD, word TRACE:RD->link
2759 | cmp RD, RB
2760 | je ->cont_nop // Blacklisted.
2761 | test RD, RD
2762 | jne =>BC_JLOOP // Jump to stitched trace.
2763 |
2764 | // Stitch a new trace to the previous trace.
2765 | mov [DISPATCH+DISPATCH_J(exitno)], RB
2766 | mov L:RB, SAVE_L
2767 | mov L:RB->base, BASE
2768 | mov FCARG2, PC
2769 | lea FCARG1, [DISPATCH+GG_DISP2J]
2770 | mov aword [DISPATCH+DISPATCH_J(L)], L:RBa
2771 | call extern lj_dispatch_stitch@8 // (jit_State *J, const BCIns *pc)
2772 | mov BASE, L:RB->base
2773 | jmp ->cont_nop
2774 |
2775 |9: // Fill up results with nil.
2776 | mov dword [RA+4], LJ_TNIL
2777 | add RA, 8
2778 | jmp <3
2779 |.endif
2780 |
2781 |->vm_profhook: // Dispatch target for profiler hook.
2782#if LJ_HASPROFILE
2783 | mov L:RB, SAVE_L
2784 | mov L:RB->base, BASE
2785 | mov FCARG2, PC // Caveat: FCARG2 == BASE
2786 | mov FCARG1, L:RB
2787 | call extern lj_dispatch_profile@8 // (lua_State *L, const BCIns *pc)
2788 | mov BASE, L:RB->base
2789 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2790 | sub PC, 4
2791 | jmp ->cont_nop
2792#endif
2793 |
2929 |//----------------------------------------------------------------------- 2794 |//-----------------------------------------------------------------------
2930 |//-- Trace exit handler ------------------------------------------------- 2795 |//-- Trace exit handler -------------------------------------------------
2931 |//----------------------------------------------------------------------- 2796 |//-----------------------------------------------------------------------
@@ -2978,10 +2843,9 @@ static void build_subroutines(BuildCtx *ctx)
2978 | movsd qword [ebp-88], xmm1; movsd qword [ebp-96], xmm0 2843 | movsd qword [ebp-88], xmm1; movsd qword [ebp-96], xmm0
2979 |.endif 2844 |.endif
2980 | // Caveat: RB is ebp. 2845 | // Caveat: RB is ebp.
2981 | mov L:RB, [DISPATCH+DISPATCH_GL(jit_L)] 2846 | mov L:RB, [DISPATCH+DISPATCH_GL(cur_L)]
2982 | mov BASE, [DISPATCH+DISPATCH_GL(jit_base)] 2847 | mov BASE, [DISPATCH+DISPATCH_GL(jit_base)]
2983 | mov aword [DISPATCH+DISPATCH_J(L)], L:RBa 2848 | mov aword [DISPATCH+DISPATCH_J(L)], L:RBa
2984 | mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0
2985 | mov L:RB->base, BASE 2849 | mov L:RB->base, BASE
2986 |.if X64WIN 2850 |.if X64WIN
2987 | lea CARG2, [rsp+4*8] 2851 | lea CARG2, [rsp+4*8]
@@ -2991,6 +2855,7 @@ static void build_subroutines(BuildCtx *ctx)
2991 | lea FCARG2, [esp+16] 2855 | lea FCARG2, [esp+16]
2992 |.endif 2856 |.endif
2993 | lea FCARG1, [DISPATCH+GG_DISP2J] 2857 | lea FCARG1, [DISPATCH+GG_DISP2J]
2858 | mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0
2994 | call extern lj_trace_exit@8 // (jit_State *J, ExitState *ex) 2859 | call extern lj_trace_exit@8 // (jit_State *J, ExitState *ex)
2995 | // MULTRES or negated error code returned in eax (RD). 2860 | // MULTRES or negated error code returned in eax (RD).
2996 | mov RAa, L:RB->cframe 2861 | mov RAa, L:RB->cframe
@@ -3037,12 +2902,14 @@ static void build_subroutines(BuildCtx *ctx)
3037 | mov r13, TMPa 2902 | mov r13, TMPa
3038 | mov r12, TMPQ 2903 | mov r12, TMPQ
3039 |.endif 2904 |.endif
3040 | test RD, RD; js >3 // Check for error from exit. 2905 | cmp RD, -LUA_ERRERR; jae >9 // Check for error from exit.
2906 | mov L:RB, SAVE_L
3041 | mov MULTRES, RD 2907 | mov MULTRES, RD
3042 | mov LFUNC:KBASE, [BASE-8] 2908 | mov LFUNC:KBASE, [BASE-8]
3043 | mov KBASE, LFUNC:KBASE->pc 2909 | mov KBASE, LFUNC:KBASE->pc
3044 | mov KBASE, [KBASE+PC2PROTO(k)] 2910 | mov KBASE, [KBASE+PC2PROTO(k)]
3045 | mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0 2911 | mov L:RB->base, BASE
2912 | mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0
3046 | set_vmstate INTERP 2913 | set_vmstate INTERP
3047 | // Modified copy of ins_next which handles function header dispatch, too. 2914 | // Modified copy of ins_next which handles function header dispatch, too.
3048 | mov RC, [PC] 2915 | mov RC, [PC]
@@ -3050,19 +2917,51 @@ static void build_subroutines(BuildCtx *ctx)
3050 | movzx OP, RCL 2917 | movzx OP, RCL
3051 | add PC, 4 2918 | add PC, 4
3052 | shr RC, 16 2919 | shr RC, 16
2920 | cmp MULTRES, -17 // Static dispatch?
2921 | je >5
3053 | cmp OP, BC_FUNCF // Function header? 2922 | cmp OP, BC_FUNCF // Function header?
3054 | jb >2 2923 | jb >3
3055 | mov RC, MULTRES // RC/RD holds nres+1. 2924 | cmp OP, BC_FUNCC+2 // Fast function?
2925 | jae >4
3056 |2: 2926 |2:
2927 | mov RC, MULTRES // RC/RD holds nres+1.
2928 |3:
3057 |.if X64 2929 |.if X64
3058 | jmp aword [DISPATCH+OP*8] 2930 | jmp aword [DISPATCH+OP*8]
3059 |.else 2931 |.else
3060 | jmp aword [DISPATCH+OP*4] 2932 | jmp aword [DISPATCH+OP*4]
3061 |.endif 2933 |.endif
3062 | 2934 |
3063 |3: // Rethrow error from the right C frame. 2935 |4: // Check frame below fast function.
2936 | mov RC, [BASE-4]
2937 | test RC, FRAME_TYPE
2938 | jnz <2 // Trace stitching continuation?
2939 | // Otherwise set KBASE for Lua function below fast function.
2940 | movzx RC, byte [RC-3]
2941 | not RCa
2942 | mov LFUNC:KBASE, [BASE+RC*8-8]
2943 | mov KBASE, LFUNC:KBASE->pc
2944 | mov KBASE, [KBASE+PC2PROTO(k)]
2945 | jmp <2
2946 |
2947 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2948 | mov RA, [DISPATCH+DISPATCH_J(trace)]
2949 | mov TRACE:RA, [RA+RD*4]
2950 | mov RC, TRACE:RA->startins
2951 | movzx RA, RCH
2952 | movzx OP, RCL
2953 | shr RC, 16
2954 |.if X64
2955 | jmp aword [DISPATCH+OP*8+GG_DISP2STATIC]
2956 |.else
2957 | jmp aword [DISPATCH+OP*4+GG_DISP2STATIC]
2958 |.endif
2959 |
2960 |9: // Rethrow error from the right C frame.
2961 | mov FCARG2, RD
3064 | mov FCARG1, L:RB 2962 | mov FCARG1, L:RB
3065 | call extern lj_err_run@4 // (lua_State *L) 2963 | neg FCARG2
2964 | call extern lj_err_trace@8 // (lua_State *L, int errcode)
3066 |.endif 2965 |.endif
3067 | 2966 |
3068 |//----------------------------------------------------------------------- 2967 |//-----------------------------------------------------------------------
@@ -3070,27 +2969,18 @@ static void build_subroutines(BuildCtx *ctx)
3070 |//----------------------------------------------------------------------- 2969 |//-----------------------------------------------------------------------
3071 | 2970 |
3072 |// FP value rounding. Called by math.floor/math.ceil fast functions 2971 |// FP value rounding. Called by math.floor/math.ceil fast functions
3073 |// and from JIT code. 2972 |// and from JIT code. arg/ret is xmm0. xmm0-xmm3 and RD (eax) modified.
3074 | 2973 |.macro vm_round, name, mode, cond
3075 |// x87 variant: Arg/ret on x87 stack. No int/xmm registers modified. 2974 |->name:
3076 |.macro vm_round_x87, mode1, mode2 2975 |.if not X64 and cond
3077 | fnstcw word [esp+4] // Caveat: overwrites ARG1 and ARG2. 2976 | movsd xmm0, qword [esp+4]
3078 | mov [esp+8], eax 2977 | call ->name .. _sse
3079 | mov ax, mode1 2978 | movsd qword [esp+4], xmm0 // Overwrite callee-owned arg.
3080 | or ax, [esp+4] 2979 | fld qword [esp+4]
3081 |.if mode2 ~= 0xffff
3082 | and ax, mode2
3083 |.endif
3084 | mov [esp+6], ax
3085 | fldcw word [esp+6]
3086 | frndint
3087 | fldcw word [esp+4]
3088 | mov eax, [esp+8]
3089 | ret 2980 | ret
3090 |.endmacro 2981 |.endif
3091 | 2982 |
3092 |// SSE variant: arg/ret is xmm0. xmm0-xmm3 and RD (eax) modified. 2983 |->name .. _sse:
3093 |.macro vm_round_sse, mode
3094 | sseconst_abs xmm2, RDa 2984 | sseconst_abs xmm2, RDa
3095 | sseconst_2p52 xmm3, RDa 2985 | sseconst_2p52 xmm3, RDa
3096 | movaps xmm1, xmm0 2986 | movaps xmm1, xmm0
@@ -3128,22 +3018,12 @@ static void build_subroutines(BuildCtx *ctx)
3128 | ret 3018 | ret
3129 |.endmacro 3019 |.endmacro
3130 | 3020 |
3131 |.macro vm_round, name, ssemode, mode1, mode2 3021 | vm_round vm_floor, 0, 1
3132 |->name: 3022 | vm_round vm_ceil, 1, JIT
3133 |.if not SSE 3023 | vm_round vm_trunc, 2, JIT
3134 | vm_round_x87 mode1, mode2
3135 |.endif
3136 |->name .. _sse:
3137 | vm_round_sse ssemode
3138 |.endmacro
3139 |
3140 | vm_round vm_floor, 0, 0x0400, 0xf7ff
3141 | vm_round vm_ceil, 1, 0x0800, 0xfbff
3142 | vm_round vm_trunc, 2, 0x0c00, 0xffff
3143 | 3024 |
3144 |// FP modulo x%y. Called by BC_MOD* and vm_arith. 3025 |// FP modulo x%y. Called by BC_MOD* and vm_arith.
3145 |->vm_mod: 3026 |->vm_mod:
3146 |.if SSE
3147 |// Args in xmm0/xmm1, return value in xmm0. 3027 |// Args in xmm0/xmm1, return value in xmm0.
3148 |// Caveat: xmm0-xmm5 and RC (eax) modified! 3028 |// Caveat: xmm0-xmm5 and RC (eax) modified!
3149 | movaps xmm5, xmm0 3029 | movaps xmm5, xmm0
@@ -3171,488 +3051,6 @@ static void build_subroutines(BuildCtx *ctx)
3171 | movaps xmm0, xmm5 3051 | movaps xmm0, xmm5
3172 | subsd xmm0, xmm1 3052 | subsd xmm0, xmm1
3173 | ret 3053 | ret
3174 |.else
3175 |// Args/ret on x87 stack (y on top). No xmm registers modified.
3176 |// Caveat: needs 3 slots on x87 stack! RC (eax) modified!
3177 | fld st1
3178 | fdiv st1
3179 | fnstcw word [esp+4]
3180 | mov ax, 0x0400
3181 | or ax, [esp+4]
3182 | and ax, 0xf7ff
3183 | mov [esp+6], ax
3184 | fldcw word [esp+6]
3185 | frndint
3186 | fldcw word [esp+4]
3187 | fmulp st1
3188 | fsubp st1
3189 | ret
3190 |.endif
3191 |
3192 |// FP log2(x). Called by math.log(x, base).
3193 |->vm_log2:
3194 |.if X64WIN
3195 | movsd qword [rsp+8], xmm0 // Use scratch area.
3196 | fld1
3197 | fld qword [rsp+8]
3198 | fyl2x
3199 | fstp qword [rsp+8]
3200 | movsd xmm0, qword [rsp+8]
3201 |.elif X64
3202 | movsd qword [rsp-8], xmm0 // Use red zone.
3203 | fld1
3204 | fld qword [rsp-8]
3205 | fyl2x
3206 | fstp qword [rsp-8]
3207 | movsd xmm0, qword [rsp-8]
3208 |.else
3209 | fld1
3210 | fld qword [esp+4]
3211 | fyl2x
3212 |.endif
3213 | ret
3214 |
3215 |// FP exponentiation e^x and 2^x. Called by math.exp fast function and
3216 |// from JIT code. Arg/ret on x87 stack. No int/xmm regs modified.
3217 |// Caveat: needs 3 slots on x87 stack!
3218 |->vm_exp_x87:
3219 | fldl2e; fmulp st1 // e^x ==> 2^(x*log2(e))
3220 |->vm_exp2_x87:
3221 | .if X64WIN
3222 | .define expscratch, dword [rsp+8] // Use scratch area.
3223 | .elif X64
3224 | .define expscratch, dword [rsp-8] // Use red zone.
3225 | .else
3226 | .define expscratch, dword [esp+4] // Needs 4 byte scratch area.
3227 | .endif
3228 | fst expscratch // Caveat: overwrites ARG1.
3229 | cmp expscratch, 0x7f800000; je >1 // Special case: e^+Inf = +Inf
3230 | cmp expscratch, 0xff800000; je >2 // Special case: e^-Inf = 0
3231 |->vm_exp2raw: // Entry point for vm_pow. Without +-Inf check.
3232 | fdup; frndint; fsub st1, st0; fxch // Split into frac/int part.
3233 | f2xm1; fld1; faddp st1; fscale; fpop1 // ==> (2^frac-1 +1) << int
3234 |1:
3235 | ret
3236 |2:
3237 | fpop; fldz; ret
3238 |
3239 |// Generic power function x^y. Called by BC_POW, math.pow fast function,
3240 |// and vm_arith.
3241 |// Args/ret on x87 stack (y on top). RC (eax) modified.
3242 |// Caveat: needs 3 slots on x87 stack!
3243 |->vm_pow:
3244 |.if not SSE
3245 | fist dword [esp+4] // Store/reload int before comparison.
3246 | fild dword [esp+4] // Integral exponent used in vm_powi.
3247 | fucomip st1
3248 | jnz >8 // Branch for FP exponents.
3249 | jp >9 // Branch for NaN exponent.
3250 | fpop // Pop y and fallthrough to vm_powi.
3251 |
3252 |// FP/int power function x^i. Arg1/ret on x87 stack.
3253 |// Arg2 (int) on C stack. RC (eax) modified.
3254 |// Caveat: needs 2 slots on x87 stack!
3255 | mov eax, [esp+4]
3256 | cmp eax, 1; jle >6 // i<=1?
3257 | // Now 1 < (unsigned)i <= 0x80000000.
3258 |1: // Handle leading zeros.
3259 | test eax, 1; jnz >2
3260 | fmul st0
3261 | shr eax, 1
3262 | jmp <1
3263 |2:
3264 | shr eax, 1; jz >5
3265 | fdup
3266 |3: // Handle trailing bits.
3267 | fmul st0
3268 | shr eax, 1; jz >4
3269 | jnc <3
3270 | fmul st1, st0
3271 | jmp <3
3272 |4:
3273 | fmulp st1
3274 |5:
3275 | ret
3276 |6:
3277 | je <5 // x^1 ==> x
3278 | jb >7
3279 | fld1; fdivrp st1
3280 | neg eax
3281 | cmp eax, 1; je <5 // x^-1 ==> 1/x
3282 | jmp <1 // x^-i ==> (1/x)^i
3283 |7:
3284 | fpop; fld1 // x^0 ==> 1
3285 | ret
3286 |
3287 |8: // FP/FP power function x^y.
3288 | fst dword [esp+4]
3289 | fxch
3290 | fst dword [esp+8]
3291 | mov eax, [esp+4]; shl eax, 1
3292 | cmp eax, 0xff000000; je >2 // x^+-Inf?
3293 | mov eax, [esp+8]; shl eax, 1; je >4 // +-0^y?
3294 | cmp eax, 0xff000000; je >4 // +-Inf^y?
3295 | fyl2x
3296 | jmp ->vm_exp2raw
3297 |
3298 |9: // Handle x^NaN.
3299 | fld1
3300 | fucomip st2
3301 | je >1 // 1^NaN ==> 1
3302 | fxch // x^NaN ==> NaN
3303 |1:
3304 | fpop
3305 | ret
3306 |
3307 |2: // Handle x^+-Inf.
3308 | fabs
3309 | fld1
3310 | fucomip st1
3311 | je >3 // +-1^+-Inf ==> 1
3312 | fpop; fabs; fldz; mov eax, 0; setc al
3313 | ror eax, 1; xor eax, [esp+4]; jns >3 // |x|<>1, x^+-Inf ==> +Inf/0
3314 | fxch
3315 |3:
3316 | fpop1; fabs
3317 | ret
3318 |
3319 |4: // Handle +-0^y or +-Inf^y.
3320 | cmp dword [esp+4], 0; jge <3 // y >= 0, x^y ==> |x|
3321 | fpop; fpop
3322 | test eax, eax; jz >5 // y < 0, +-0^y ==> +Inf
3323 | fldz // y < 0, +-Inf^y ==> 0
3324 | ret
3325 |5:
3326 | mov dword [esp+4], 0x7f800000 // Return +Inf.
3327 | fld dword [esp+4]
3328 | ret
3329 |.endif
3330 |
3331 |// Args in xmm0/xmm1. Ret in xmm0. xmm0-xmm2 and RC (eax) modified.
3332 |// Needs 16 byte scratch area for x86. Also called from JIT code.
3333 |->vm_pow_sse:
3334 | cvtsd2si eax, xmm1
3335 | cvtsi2sd xmm2, eax
3336 | ucomisd xmm1, xmm2
3337 | jnz >8 // Branch for FP exponents.
3338 | jp >9 // Branch for NaN exponent.
3339 | // Fallthrough to vm_powi_sse.
3340 |
3341 |// Args in xmm0/eax. Ret in xmm0. xmm0-xmm1 and eax modified.
3342 |->vm_powi_sse:
3343 | cmp eax, 1; jle >6 // i<=1?
3344 | // Now 1 < (unsigned)i <= 0x80000000.
3345 |1: // Handle leading zeros.
3346 | test eax, 1; jnz >2
3347 | mulsd xmm0, xmm0
3348 | shr eax, 1
3349 | jmp <1
3350 |2:
3351 | shr eax, 1; jz >5
3352 | movaps xmm1, xmm0
3353 |3: // Handle trailing bits.
3354 | mulsd xmm0, xmm0
3355 | shr eax, 1; jz >4
3356 | jnc <3
3357 | mulsd xmm1, xmm0
3358 | jmp <3
3359 |4:
3360 | mulsd xmm0, xmm1
3361 |5:
3362 | ret
3363 |6:
3364 | je <5 // x^1 ==> x
3365 | jb >7 // x^0 ==> 1
3366 | neg eax
3367 | call <1
3368 | sseconst_1 xmm1, RDa
3369 | divsd xmm1, xmm0
3370 | movaps xmm0, xmm1
3371 | ret
3372 |7:
3373 | sseconst_1 xmm0, RDa
3374 | ret
3375 |
3376 |8: // FP/FP power function x^y.
3377 |.if X64
3378 | movd rax, xmm1; shl rax, 1
3379 | rol rax, 12; cmp rax, 0xffe; je >2 // x^+-Inf?
3380 | movd rax, xmm0; shl rax, 1; je >4 // +-0^y?
3381 | rol rax, 12; cmp rax, 0xffe; je >5 // +-Inf^y?
3382 | .if X64WIN
3383 | movsd qword [rsp+16], xmm1 // Use scratch area.
3384 | movsd qword [rsp+8], xmm0
3385 | fld qword [rsp+16]
3386 | fld qword [rsp+8]
3387 | .else
3388 | movsd qword [rsp-16], xmm1 // Use red zone.
3389 | movsd qword [rsp-8], xmm0
3390 | fld qword [rsp-16]
3391 | fld qword [rsp-8]
3392 | .endif
3393 |.else
3394 | movsd qword [esp+12], xmm1 // Needs 16 byte scratch area.
3395 | movsd qword [esp+4], xmm0
3396 | cmp dword [esp+12], 0; jne >1
3397 | mov eax, [esp+16]; shl eax, 1
3398 | cmp eax, 0xffe00000; je >2 // x^+-Inf?
3399 |1:
3400 | cmp dword [esp+4], 0; jne >1
3401 | mov eax, [esp+8]; shl eax, 1; je >4 // +-0^y?
3402 | cmp eax, 0xffe00000; je >5 // +-Inf^y?
3403 |1:
3404 | fld qword [esp+12]
3405 | fld qword [esp+4]
3406 |.endif
3407 | fyl2x // y*log2(x)
3408 | fdup; frndint; fsub st1, st0; fxch // Split into frac/int part.
3409 | f2xm1; fld1; faddp st1; fscale; fpop1 // ==> (2^frac-1 +1) << int
3410 |.if X64WIN
3411 | fstp qword [rsp+8] // Use scratch area.
3412 | movsd xmm0, qword [rsp+8]
3413 |.elif X64
3414 | fstp qword [rsp-8] // Use red zone.
3415 | movsd xmm0, qword [rsp-8]
3416 |.else
3417 | fstp qword [esp+4] // Needs 8 byte scratch area.
3418 | movsd xmm0, qword [esp+4]
3419 |.endif
3420 | ret
3421 |
3422 |9: // Handle x^NaN.
3423 | sseconst_1 xmm2, RDa
3424 | ucomisd xmm0, xmm2; je >1 // 1^NaN ==> 1
3425 | movaps xmm0, xmm1 // x^NaN ==> NaN
3426 |1:
3427 | ret
3428 |
3429 |2: // Handle x^+-Inf.
3430 | sseconst_abs xmm2, RDa
3431 | andpd xmm0, xmm2 // |x|
3432 | sseconst_1 xmm2, RDa
3433 | ucomisd xmm0, xmm2; je <1 // +-1^+-Inf ==> 1
3434 | movmskpd eax, xmm1
3435 | xorps xmm0, xmm0
3436 | mov ah, al; setc al; xor al, ah; jne <1 // |x|<>1, x^+-Inf ==> +Inf/0
3437 |3:
3438 | sseconst_hi xmm0, RDa, 7ff00000 // +Inf
3439 | ret
3440 |
3441 |4: // Handle +-0^y.
3442 | movmskpd eax, xmm1; test eax, eax; jnz <3 // y < 0, +-0^y ==> +Inf
3443 | xorps xmm0, xmm0 // y >= 0, +-0^y ==> 0
3444 | ret
3445 |
3446 |5: // Handle +-Inf^y.
3447 | movmskpd eax, xmm1; test eax, eax; jz <3 // y >= 0, +-Inf^y ==> +Inf
3448 | xorps xmm0, xmm0 // y < 0, +-Inf^y ==> 0
3449 | ret
3450 |
3451 |// Callable from C: double lj_vm_foldfpm(double x, int fpm)
3452 |// Computes fpm(x) for extended math functions. ORDER FPM.
3453 |->vm_foldfpm:
3454 |.if JIT
3455 |.if X64
3456 | .if X64WIN
3457 | .define fpmop, CARG2d
3458 | .else
3459 | .define fpmop, CARG1d
3460 | .endif
3461 | cmp fpmop, 1; jb ->vm_floor; je ->vm_ceil
3462 | cmp fpmop, 3; jb ->vm_trunc; ja >2
3463 | sqrtsd xmm0, xmm0; ret
3464 |2:
3465 | .if X64WIN
3466 | movsd qword [rsp+8], xmm0 // Use scratch area.
3467 | fld qword [rsp+8]
3468 | .else
3469 | movsd qword [rsp-8], xmm0 // Use red zone.
3470 | fld qword [rsp-8]
3471 | .endif
3472 | cmp fpmop, 5; ja >2
3473 | .if X64WIN; pop rax; .endif
3474 | je >1
3475 | call ->vm_exp_x87
3476 | .if X64WIN; push rax; .endif
3477 | jmp >7
3478 |1:
3479 | call ->vm_exp2_x87
3480 | .if X64WIN; push rax; .endif
3481 | jmp >7
3482 |2: ; cmp fpmop, 7; je >1; ja >2
3483 | fldln2; fxch; fyl2x; jmp >7
3484 |1: ; fld1; fxch; fyl2x; jmp >7
3485 |2: ; cmp fpmop, 9; je >1; ja >2
3486 | fldlg2; fxch; fyl2x; jmp >7
3487 |1: ; fsin; jmp >7
3488 |2: ; cmp fpmop, 11; je >1; ja >9
3489 | fcos; jmp >7
3490 |1: ; fptan; fpop
3491 |7:
3492 | .if X64WIN
3493 | fstp qword [rsp+8] // Use scratch area.
3494 | movsd xmm0, qword [rsp+8]
3495 | .else
3496 | fstp qword [rsp-8] // Use red zone.
3497 | movsd xmm0, qword [rsp-8]
3498 | .endif
3499 | ret
3500 |.else // x86 calling convention.
3501 | .define fpmop, eax
3502 |.if SSE
3503 | mov fpmop, [esp+12]
3504 | movsd xmm0, qword [esp+4]
3505 | cmp fpmop, 1; je >1; ja >2
3506 | call ->vm_floor; jmp >7
3507 |1: ; call ->vm_ceil; jmp >7
3508 |2: ; cmp fpmop, 3; je >1; ja >2
3509 | call ->vm_trunc; jmp >7
3510 |1:
3511 | sqrtsd xmm0, xmm0
3512 |7:
3513 | movsd qword [esp+4], xmm0 // Overwrite callee-owned args.
3514 | fld qword [esp+4]
3515 | ret
3516 |2: ; fld qword [esp+4]
3517 | cmp fpmop, 5; jb ->vm_exp_x87; je ->vm_exp2_x87
3518 |2: ; cmp fpmop, 7; je >1; ja >2
3519 | fldln2; fxch; fyl2x; ret
3520 |1: ; fld1; fxch; fyl2x; ret
3521 |2: ; cmp fpmop, 9; je >1; ja >2
3522 | fldlg2; fxch; fyl2x; ret
3523 |1: ; fsin; ret
3524 |2: ; cmp fpmop, 11; je >1; ja >9
3525 | fcos; ret
3526 |1: ; fptan; fpop; ret
3527 |.else
3528 | mov fpmop, [esp+12]
3529 | fld qword [esp+4]
3530 | cmp fpmop, 1; jb ->vm_floor; je ->vm_ceil
3531 | cmp fpmop, 3; jb ->vm_trunc; ja >2
3532 | fsqrt; ret
3533 |2: ; cmp fpmop, 5; jb ->vm_exp_x87; je ->vm_exp2_x87
3534 | cmp fpmop, 7; je >1; ja >2
3535 | fldln2; fxch; fyl2x; ret
3536 |1: ; fld1; fxch; fyl2x; ret
3537 |2: ; cmp fpmop, 9; je >1; ja >2
3538 | fldlg2; fxch; fyl2x; ret
3539 |1: ; fsin; ret
3540 |2: ; cmp fpmop, 11; je >1; ja >9
3541 | fcos; ret
3542 |1: ; fptan; fpop; ret
3543 |.endif
3544 |.endif
3545 |9: ; int3 // Bad fpm.
3546 |.endif
3547 |
3548 |// Callable from C: double lj_vm_foldarith(double x, double y, int op)
3549 |// Compute x op y for basic arithmetic operators (+ - * / % ^ and unary -)
3550 |// and basic math functions. ORDER ARITH
3551 |->vm_foldarith:
3552 |.if X64
3553 |
3554 | .if X64WIN
3555 | .define foldop, CARG3d
3556 | .else
3557 | .define foldop, CARG1d
3558 | .endif
3559 | cmp foldop, 1; je >1; ja >2
3560 | addsd xmm0, xmm1; ret
3561 |1: ; subsd xmm0, xmm1; ret
3562 |2: ; cmp foldop, 3; je >1; ja >2
3563 | mulsd xmm0, xmm1; ret
3564 |1: ; divsd xmm0, xmm1; ret
3565 |2: ; cmp foldop, 5; jb ->vm_mod; je ->vm_pow
3566 | cmp foldop, 7; je >1; ja >2
3567 | sseconst_sign xmm1, RDa; xorps xmm0, xmm1; ret
3568 |1: ; sseconst_abs xmm1, RDa; andps xmm0, xmm1; ret
3569 |2: ; cmp foldop, 9; ja >2
3570 |.if X64WIN
3571 | movsd qword [rsp+8], xmm0 // Use scratch area.
3572 | movsd qword [rsp+16], xmm1
3573 | fld qword [rsp+8]
3574 | fld qword [rsp+16]
3575 |.else
3576 | movsd qword [rsp-8], xmm0 // Use red zone.
3577 | movsd qword [rsp-16], xmm1
3578 | fld qword [rsp-8]
3579 | fld qword [rsp-16]
3580 |.endif
3581 | je >1
3582 | fpatan
3583 |7:
3584 |.if X64WIN
3585 | fstp qword [rsp+8] // Use scratch area.
3586 | movsd xmm0, qword [rsp+8]
3587 |.else
3588 | fstp qword [rsp-8] // Use red zone.
3589 | movsd xmm0, qword [rsp-8]
3590 |.endif
3591 | ret
3592 |1: ; fxch; fscale; fpop1; jmp <7
3593 |2: ; cmp foldop, 11; je >1; ja >9
3594 | minsd xmm0, xmm1; ret
3595 |1: ; maxsd xmm0, xmm1; ret
3596 |9: ; int3 // Bad op.
3597 |
3598 |.elif SSE // x86 calling convention with SSE ops.
3599 |
3600 | .define foldop, eax
3601 | mov foldop, [esp+20]
3602 | movsd xmm0, qword [esp+4]
3603 | movsd xmm1, qword [esp+12]
3604 | cmp foldop, 1; je >1; ja >2
3605 | addsd xmm0, xmm1
3606 |7:
3607 | movsd qword [esp+4], xmm0 // Overwrite callee-owned args.
3608 | fld qword [esp+4]
3609 | ret
3610 |1: ; subsd xmm0, xmm1; jmp <7
3611 |2: ; cmp foldop, 3; je >1; ja >2
3612 | mulsd xmm0, xmm1; jmp <7
3613 |1: ; divsd xmm0, xmm1; jmp <7
3614 |2: ; cmp foldop, 5
3615 | je >1; ja >2
3616 | call ->vm_mod; jmp <7
3617 |1: ; pop edx; call ->vm_pow; push edx; jmp <7 // Writes to scratch area.
3618 |2: ; cmp foldop, 7; je >1; ja >2
3619 | sseconst_sign xmm1, RDa; xorps xmm0, xmm1; jmp <7
3620 |1: ; sseconst_abs xmm1, RDa; andps xmm0, xmm1; jmp <7
3621 |2: ; cmp foldop, 9; ja >2
3622 | fld qword [esp+4] // Reload from stack
3623 | fld qword [esp+12]
3624 | je >1
3625 | fpatan; ret
3626 |1: ; fxch; fscale; fpop1; ret
3627 |2: ; cmp foldop, 11; je >1; ja >9
3628 | minsd xmm0, xmm1; jmp <7
3629 |1: ; maxsd xmm0, xmm1; jmp <7
3630 |9: ; int3 // Bad op.
3631 |
3632 |.else // x86 calling convention with x87 ops.
3633 |
3634 | mov eax, [esp+20]
3635 | fld qword [esp+4]
3636 | fld qword [esp+12]
3637 | cmp eax, 1; je >1; ja >2
3638 | faddp st1; ret
3639 |1: ; fsubp st1; ret
3640 |2: ; cmp eax, 3; je >1; ja >2
3641 | fmulp st1; ret
3642 |1: ; fdivp st1; ret
3643 |2: ; cmp eax, 5; jb ->vm_mod; je ->vm_pow
3644 | cmp eax, 7; je >1; ja >2
3645 | fpop; fchs; ret
3646 |1: ; fpop; fabs; ret
3647 |2: ; cmp eax, 9; je >1; ja >2
3648 | fpatan; ret
3649 |1: ; fxch; fscale; fpop1; ret
3650 |2: ; cmp eax, 11; je >1; ja >9
3651 | fucomi st1; fcmovnbe st1; fpop1; ret
3652 |1: ; fucomi st1; fcmovbe st1; fpop1; ret
3653 |9: ; int3 // Bad op.
3654 |
3655 |.endif
3656 | 3054 |
3657 |//----------------------------------------------------------------------- 3055 |//-----------------------------------------------------------------------
3658 |//-- Miscellaneous functions -------------------------------------------- 3056 |//-- Miscellaneous functions --------------------------------------------
@@ -3664,6 +3062,7 @@ static void build_subroutines(BuildCtx *ctx)
3664 | mov eax, CARG1d 3062 | mov eax, CARG1d
3665 | .if X64WIN; push rsi; mov rsi, CARG2; .endif 3063 | .if X64WIN; push rsi; mov rsi, CARG2; .endif
3666 | push rbx 3064 | push rbx
3065 | xor ecx, ecx
3667 | cpuid 3066 | cpuid
3668 | mov [rsi], eax 3067 | mov [rsi], eax
3669 | mov [rsi+4], ebx 3068 | mov [rsi+4], ebx
@@ -3687,6 +3086,7 @@ static void build_subroutines(BuildCtx *ctx)
3687 | mov eax, [esp+4] // Argument 1 is function number. 3086 | mov eax, [esp+4] // Argument 1 is function number.
3688 | push edi 3087 | push edi
3689 | push ebx 3088 | push ebx
3089 | xor ecx, ecx
3690 | cpuid 3090 | cpuid
3691 | mov edi, [esp+16] // Argument 2 is result area. 3091 | mov edi, [esp+16] // Argument 2 is result area.
3692 | mov [edi], eax 3092 | mov [edi], eax
@@ -3699,6 +3099,86 @@ static void build_subroutines(BuildCtx *ctx)
3699 | ret 3099 | ret
3700 |.endif 3100 |.endif
3701 | 3101 |
3102 |.define NEXT_TAB, TAB:FCARG1
3103 |.define NEXT_IDX, FCARG2
3104 |.define NEXT_PTR, RCa
3105 |.define NEXT_PTRd, RC
3106 |.macro NEXT_RES_IDXL, op2; lea edx, [NEXT_IDX+op2]; .endmacro
3107 |.if X64
3108 |.define NEXT_TMP, CARG3d
3109 |.define NEXT_TMPq, CARG3
3110 |.define NEXT_ASIZE, CARG4d
3111 |.macro NEXT_ENTER; .endmacro
3112 |.macro NEXT_LEAVE; ret; .endmacro
3113 |.if X64WIN
3114 |.define NEXT_RES_PTR, [rsp+aword*5]
3115 |.macro NEXT_RES_IDX, op2; add NEXT_IDX, op2; .endmacro
3116 |.else
3117 |.define NEXT_RES_PTR, [rsp+aword*1]
3118 |.macro NEXT_RES_IDX, op2; lea edx, [NEXT_IDX+op2]; .endmacro
3119 |.endif
3120 |.else
3121 |.define NEXT_ASIZE, esi
3122 |.define NEXT_TMP, edi
3123 |.macro NEXT_ENTER; push esi; push edi; .endmacro
3124 |.macro NEXT_LEAVE; pop edi; pop esi; ret; .endmacro
3125 |.define NEXT_RES_PTR, [esp+dword*3]
3126 |.macro NEXT_RES_IDX, op2; add NEXT_IDX, op2; .endmacro
3127 |.endif
3128 |
3129 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
3130 |// Next idx returned in edx.
3131 |->vm_next:
3132 |.if JIT
3133 | NEXT_ENTER
3134 | mov NEXT_ASIZE, NEXT_TAB->asize
3135 |1: // Traverse array part.
3136 | cmp NEXT_IDX, NEXT_ASIZE; jae >5
3137 | mov NEXT_TMP, NEXT_TAB->array
3138 | cmp dword [NEXT_TMP+NEXT_IDX*8+4], LJ_TNIL; je >2
3139 | lea NEXT_PTR, NEXT_RES_PTR
3140 |.if X64
3141 | mov NEXT_TMPq, qword [NEXT_TMP+NEXT_IDX*8]
3142 | mov qword [NEXT_PTR], NEXT_TMPq
3143 |.else
3144 | mov NEXT_ASIZE, dword [NEXT_TMP+NEXT_IDX*8+4]
3145 | mov NEXT_TMP, dword [NEXT_TMP+NEXT_IDX*8]
3146 | mov dword [NEXT_PTR+4], NEXT_ASIZE
3147 | mov dword [NEXT_PTR], NEXT_TMP
3148 |.endif
3149 |.if DUALNUM
3150 | mov dword [NEXT_PTR+dword*3], LJ_TISNUM
3151 | mov dword [NEXT_PTR+dword*2], NEXT_IDX
3152 |.else
3153 | cvtsi2sd xmm0, NEXT_IDX
3154 | movsd qword [NEXT_PTR+dword*2], xmm0
3155 |.endif
3156 | NEXT_RES_IDX 1
3157 | NEXT_LEAVE
3158 |2: // Skip holes in array part.
3159 | add NEXT_IDX, 1
3160 | jmp <1
3161 |
3162 |5: // Traverse hash part.
3163 | sub NEXT_IDX, NEXT_ASIZE
3164 |6:
3165 | cmp NEXT_IDX, NEXT_TAB->hmask; ja >9
3166 | imul NEXT_PTRd, NEXT_IDX, #NODE
3167 | add NODE:NEXT_PTRd, dword NEXT_TAB->node
3168 | cmp dword NODE:NEXT_PTR->val.it, LJ_TNIL; je >7
3169 | NEXT_RES_IDXL NEXT_ASIZE+1
3170 | NEXT_LEAVE
3171 |7: // Skip holes in hash part.
3172 | add NEXT_IDX, 1
3173 | jmp <6
3174 |
3175 |9: // End of iteration. Set the key to nil (not the value).
3176 | NEXT_RES_IDX NEXT_ASIZE
3177 | lea NEXT_PTR, NEXT_RES_PTR
3178 | mov dword [NEXT_PTR+dword*3], LJ_TNIL
3179 | NEXT_LEAVE
3180 |.endif
3181 |
3702 |//----------------------------------------------------------------------- 3182 |//-----------------------------------------------------------------------
3703 |//-- Assertions --------------------------------------------------------- 3183 |//-- Assertions ---------------------------------------------------------
3704 |//----------------------------------------------------------------------- 3184 |//-----------------------------------------------------------------------
@@ -3964,19 +3444,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3964 | // RA is a number. 3444 | // RA is a number.
3965 | cmp dword [BASE+RD*8+4], LJ_TISNUM; jb >1; jne ->vmeta_comp 3445 | cmp dword [BASE+RD*8+4], LJ_TISNUM; jb >1; jne ->vmeta_comp
3966 | // RA is a number, RD is an integer. 3446 | // RA is a number, RD is an integer.
3967 |.if SSE
3968 | cvtsi2sd xmm0, dword [BASE+RD*8] 3447 | cvtsi2sd xmm0, dword [BASE+RD*8]
3969 | jmp >2 3448 | jmp >2
3970 |.else
3971 | fld qword [BASE+RA*8]
3972 | fild dword [BASE+RD*8]
3973 | jmp >3
3974 |.endif
3975 | 3449 |
3976 |8: // RA is an integer, RD is not an integer. 3450 |8: // RA is an integer, RD is not an integer.
3977 | ja ->vmeta_comp 3451 | ja ->vmeta_comp
3978 | // RA is an integer, RD is a number. 3452 | // RA is an integer, RD is a number.
3979 |.if SSE
3980 | cvtsi2sd xmm1, dword [BASE+RA*8] 3453 | cvtsi2sd xmm1, dword [BASE+RA*8]
3981 | movsd xmm0, qword [BASE+RD*8] 3454 | movsd xmm0, qword [BASE+RD*8]
3982 | add PC, 4 3455 | add PC, 4
@@ -3984,29 +3457,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3984 | jmp_comp jbe, ja, jb, jae, <9 3457 | jmp_comp jbe, ja, jb, jae, <9
3985 | jmp <6 3458 | jmp <6
3986 |.else 3459 |.else
3987 | fild dword [BASE+RA*8]
3988 | jmp >2
3989 |.endif
3990 |.else
3991 | checknum RA, ->vmeta_comp 3460 | checknum RA, ->vmeta_comp
3992 | checknum RD, ->vmeta_comp 3461 | checknum RD, ->vmeta_comp
3993 |.endif 3462 |.endif
3994 |.if SSE
3995 |1: 3463 |1:
3996 | movsd xmm0, qword [BASE+RD*8] 3464 | movsd xmm0, qword [BASE+RD*8]
3997 |2: 3465 |2:
3998 | add PC, 4 3466 | add PC, 4
3999 | ucomisd xmm0, qword [BASE+RA*8] 3467 | ucomisd xmm0, qword [BASE+RA*8]
4000 |3: 3468 |3:
4001 |.else
4002 |1:
4003 | fld qword [BASE+RA*8] // Reverse order, i.e like cmp D, A.
4004 |2:
4005 | fld qword [BASE+RD*8]
4006 |3:
4007 | add PC, 4
4008 | fcomparepp
4009 |.endif
4010 | // Unordered: all of ZF CF PF set, ordered: PF clear. 3469 | // Unordered: all of ZF CF PF set, ordered: PF clear.
4011 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't. 3470 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
4012 |.if DUALNUM 3471 |.if DUALNUM
@@ -4046,43 +3505,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4046 | // RD is a number. 3505 | // RD is a number.
4047 | cmp dword [BASE+RA*8+4], LJ_TISNUM; jb >1; jne >5 3506 | cmp dword [BASE+RA*8+4], LJ_TISNUM; jb >1; jne >5
4048 | // RD is a number, RA is an integer. 3507 | // RD is a number, RA is an integer.
4049 |.if SSE
4050 | cvtsi2sd xmm0, dword [BASE+RA*8] 3508 | cvtsi2sd xmm0, dword [BASE+RA*8]
4051 |.else
4052 | fild dword [BASE+RA*8]
4053 |.endif
4054 | jmp >2 3509 | jmp >2
4055 | 3510 |
4056 |8: // RD is an integer, RA is not an integer. 3511 |8: // RD is an integer, RA is not an integer.
4057 | ja >5 3512 | ja >5
4058 | // RD is an integer, RA is a number. 3513 | // RD is an integer, RA is a number.
4059 |.if SSE
4060 | cvtsi2sd xmm0, dword [BASE+RD*8] 3514 | cvtsi2sd xmm0, dword [BASE+RD*8]
4061 | ucomisd xmm0, qword [BASE+RA*8] 3515 | ucomisd xmm0, qword [BASE+RA*8]
4062 |.else
4063 | fild dword [BASE+RD*8]
4064 | fld qword [BASE+RA*8]
4065 |.endif
4066 | jmp >4 3516 | jmp >4
4067 | 3517 |
4068 |.else 3518 |.else
4069 | cmp RB, LJ_TISNUM; jae >5 3519 | cmp RB, LJ_TISNUM; jae >5
4070 | checknum RA, >5 3520 | checknum RA, >5
4071 |.endif 3521 |.endif
4072 |.if SSE
4073 |1: 3522 |1:
4074 | movsd xmm0, qword [BASE+RA*8] 3523 | movsd xmm0, qword [BASE+RA*8]
4075 |2: 3524 |2:
4076 | ucomisd xmm0, qword [BASE+RD*8] 3525 | ucomisd xmm0, qword [BASE+RD*8]
4077 |4: 3526 |4:
4078 |.else
4079 |1:
4080 | fld qword [BASE+RA*8]
4081 |2:
4082 | fld qword [BASE+RD*8]
4083 |4:
4084 | fcomparepp
4085 |.endif
4086 iseqne_fp: 3527 iseqne_fp:
4087 if (vk) { 3528 if (vk) {
4088 | jp >2 // Unordered means not equal. 3529 | jp >2 // Unordered means not equal.
@@ -4205,39 +3646,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4205 | // RA is a number. 3646 | // RA is a number.
4206 | cmp dword [KBASE+RD*8+4], LJ_TISNUM; jb >1 3647 | cmp dword [KBASE+RD*8+4], LJ_TISNUM; jb >1
4207 | // RA is a number, RD is an integer. 3648 | // RA is a number, RD is an integer.
4208 |.if SSE
4209 | cvtsi2sd xmm0, dword [KBASE+RD*8] 3649 | cvtsi2sd xmm0, dword [KBASE+RD*8]
4210 |.else
4211 | fild dword [KBASE+RD*8]
4212 |.endif
4213 | jmp >2 3650 | jmp >2
4214 | 3651 |
4215 |8: // RA is an integer, RD is a number. 3652 |8: // RA is an integer, RD is a number.
4216 |.if SSE
4217 | cvtsi2sd xmm0, dword [BASE+RA*8] 3653 | cvtsi2sd xmm0, dword [BASE+RA*8]
4218 | ucomisd xmm0, qword [KBASE+RD*8] 3654 | ucomisd xmm0, qword [KBASE+RD*8]
4219 |.else
4220 | fild dword [BASE+RA*8]
4221 | fld qword [KBASE+RD*8]
4222 |.endif
4223 | jmp >4 3655 | jmp >4
4224 |.else 3656 |.else
4225 | cmp RB, LJ_TISNUM; jae >3 3657 | cmp RB, LJ_TISNUM; jae >3
4226 |.endif 3658 |.endif
4227 |.if SSE
4228 |1: 3659 |1:
4229 | movsd xmm0, qword [KBASE+RD*8] 3660 | movsd xmm0, qword [KBASE+RD*8]
4230 |2: 3661 |2:
4231 | ucomisd xmm0, qword [BASE+RA*8] 3662 | ucomisd xmm0, qword [BASE+RA*8]
4232 |4: 3663 |4:
4233 |.else
4234 |1:
4235 | fld qword [KBASE+RD*8]
4236 |2:
4237 | fld qword [BASE+RA*8]
4238 |4:
4239 | fcomparepp
4240 |.endif
4241 goto iseqne_fp; 3664 goto iseqne_fp;
4242 case BC_ISEQP: case BC_ISNEP: 3665 case BC_ISEQP: case BC_ISNEP:
4243 vk = op == BC_ISEQP; 3666 vk = op == BC_ISEQP;
@@ -4288,6 +3711,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4288 | ins_next 3711 | ins_next
4289 break; 3712 break;
4290 3713
3714 case BC_ISTYPE:
3715 | ins_AD // RA = src, RD = -type
3716 | add RD, [BASE+RA*8+4]
3717 | jne ->vmeta_istype
3718 | ins_next
3719 break;
3720 case BC_ISNUM:
3721 | ins_AD // RA = src, RD = -(TISNUM-1)
3722 | checknum RA, ->vmeta_istype
3723 | ins_next
3724 break;
3725
4291 /* -- Unary ops --------------------------------------------------------- */ 3726 /* -- Unary ops --------------------------------------------------------- */
4292 3727
4293 case BC_MOV: 3728 case BC_MOV:
@@ -4331,16 +3766,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4331 |.else 3766 |.else
4332 | checknum RD, ->vmeta_unm 3767 | checknum RD, ->vmeta_unm
4333 |.endif 3768 |.endif
4334 |.if SSE
4335 | movsd xmm0, qword [BASE+RD*8] 3769 | movsd xmm0, qword [BASE+RD*8]
4336 | sseconst_sign xmm1, RDa 3770 | sseconst_sign xmm1, RDa
4337 | xorps xmm0, xmm1 3771 | xorps xmm0, xmm1
4338 | movsd qword [BASE+RA*8], xmm0 3772 | movsd qword [BASE+RA*8], xmm0
4339 |.else
4340 | fld qword [BASE+RD*8]
4341 | fchs
4342 | fstp qword [BASE+RA*8]
4343 |.endif
4344 |.if DUALNUM 3773 |.if DUALNUM
4345 | jmp <9 3774 | jmp <9
4346 |.else 3775 |.else
@@ -4356,15 +3785,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4356 |1: 3785 |1:
4357 | mov dword [BASE+RA*8+4], LJ_TISNUM 3786 | mov dword [BASE+RA*8+4], LJ_TISNUM
4358 | mov dword [BASE+RA*8], RD 3787 | mov dword [BASE+RA*8], RD
4359 |.elif SSE 3788 |.else
4360 | xorps xmm0, xmm0 3789 | xorps xmm0, xmm0
4361 | cvtsi2sd xmm0, dword STR:RD->len 3790 | cvtsi2sd xmm0, dword STR:RD->len
4362 |1: 3791 |1:
4363 | movsd qword [BASE+RA*8], xmm0 3792 | movsd qword [BASE+RA*8], xmm0
4364 |.else
4365 | fild dword STR:RD->len
4366 |1:
4367 | fstp qword [BASE+RA*8]
4368 |.endif 3793 |.endif
4369 | ins_next 3794 | ins_next
4370 |2: 3795 |2:
@@ -4382,11 +3807,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4382 | // Length of table returned in eax (RD). 3807 | // Length of table returned in eax (RD).
4383 |.if DUALNUM 3808 |.if DUALNUM
4384 | // Nothing to do. 3809 | // Nothing to do.
4385 |.elif SSE
4386 | cvtsi2sd xmm0, RD
4387 |.else 3810 |.else
4388 | mov ARG1, RD 3811 | cvtsi2sd xmm0, RD
4389 | fild ARG1
4390 |.endif 3812 |.endif
4391 | mov BASE, RB // Restore BASE. 3813 | mov BASE, RB // Restore BASE.
4392 | movzx RA, PC_RA 3814 | movzx RA, PC_RA
@@ -4401,7 +3823,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4401 3823
4402 /* -- Binary ops -------------------------------------------------------- */ 3824 /* -- Binary ops -------------------------------------------------------- */
4403 3825
4404 |.macro ins_arithpre, x87ins, sseins, ssereg 3826 |.macro ins_arithpre, sseins, ssereg
4405 | ins_ABC 3827 | ins_ABC
4406 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN); 3828 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
4407 ||switch (vk) { 3829 ||switch (vk) {
@@ -4410,37 +3832,22 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4410 | .if DUALNUM 3832 | .if DUALNUM
4411 | cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_vn 3833 | cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_vn
4412 | .endif 3834 | .endif
4413 | .if SSE 3835 | movsd xmm0, qword [BASE+RB*8]
4414 | movsd xmm0, qword [BASE+RB*8] 3836 | sseins ssereg, qword [KBASE+RC*8]
4415 | sseins ssereg, qword [KBASE+RC*8]
4416 | .else
4417 | fld qword [BASE+RB*8]
4418 | x87ins qword [KBASE+RC*8]
4419 | .endif
4420 || break; 3837 || break;
4421 ||case 1: 3838 ||case 1:
4422 | checknum RB, ->vmeta_arith_nv 3839 | checknum RB, ->vmeta_arith_nv
4423 | .if DUALNUM 3840 | .if DUALNUM
4424 | cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_nv 3841 | cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_nv
4425 | .endif 3842 | .endif
4426 | .if SSE 3843 | movsd xmm0, qword [KBASE+RC*8]
4427 | movsd xmm0, qword [KBASE+RC*8] 3844 | sseins ssereg, qword [BASE+RB*8]
4428 | sseins ssereg, qword [BASE+RB*8]
4429 | .else
4430 | fld qword [KBASE+RC*8]
4431 | x87ins qword [BASE+RB*8]
4432 | .endif
4433 || break; 3845 || break;
4434 ||default: 3846 ||default:
4435 | checknum RB, ->vmeta_arith_vv 3847 | checknum RB, ->vmeta_arith_vv
4436 | checknum RC, ->vmeta_arith_vv 3848 | checknum RC, ->vmeta_arith_vv
4437 | .if SSE 3849 | movsd xmm0, qword [BASE+RB*8]
4438 | movsd xmm0, qword [BASE+RB*8] 3850 | sseins ssereg, qword [BASE+RC*8]
4439 | sseins ssereg, qword [BASE+RC*8]
4440 | .else
4441 | fld qword [BASE+RB*8]
4442 | x87ins qword [BASE+RC*8]
4443 | .endif
4444 || break; 3851 || break;
4445 ||} 3852 ||}
4446 |.endmacro 3853 |.endmacro
@@ -4478,55 +3885,62 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4478 |.endmacro 3885 |.endmacro
4479 | 3886 |
4480 |.macro ins_arithpost 3887 |.macro ins_arithpost
4481 |.if SSE
4482 | movsd qword [BASE+RA*8], xmm0 3888 | movsd qword [BASE+RA*8], xmm0
4483 |.else
4484 | fstp qword [BASE+RA*8]
4485 |.endif
4486 |.endmacro 3889 |.endmacro
4487 | 3890 |
4488 |.macro ins_arith, x87ins, sseins 3891 |.macro ins_arith, sseins
4489 | ins_arithpre x87ins, sseins, xmm0 3892 | ins_arithpre sseins, xmm0
4490 | ins_arithpost 3893 | ins_arithpost
4491 | ins_next 3894 | ins_next
4492 |.endmacro 3895 |.endmacro
4493 | 3896 |
4494 |.macro ins_arith, intins, x87ins, sseins 3897 |.macro ins_arith, intins, sseins
4495 |.if DUALNUM 3898 |.if DUALNUM
4496 | ins_arithdn intins 3899 | ins_arithdn intins
4497 |.else 3900 |.else
4498 | ins_arith, x87ins, sseins 3901 | ins_arith, sseins
4499 |.endif 3902 |.endif
4500 |.endmacro 3903 |.endmacro
4501 3904
4502 | // RA = dst, RB = src1 or num const, RC = src2 or num const 3905 | // RA = dst, RB = src1 or num const, RC = src2 or num const
4503 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV: 3906 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
4504 | ins_arith add, fadd, addsd 3907 | ins_arith add, addsd
4505 break; 3908 break;
4506 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV: 3909 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
4507 | ins_arith sub, fsub, subsd 3910 | ins_arith sub, subsd
4508 break; 3911 break;
4509 case BC_MULVN: case BC_MULNV: case BC_MULVV: 3912 case BC_MULVN: case BC_MULNV: case BC_MULVV:
4510 | ins_arith imul, fmul, mulsd 3913 | ins_arith imul, mulsd
4511 break; 3914 break;
4512 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV: 3915 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
4513 | ins_arith fdiv, divsd 3916 | ins_arith divsd
4514 break; 3917 break;
4515 case BC_MODVN: 3918 case BC_MODVN:
4516 | ins_arithpre fld, movsd, xmm1 3919 | ins_arithpre movsd, xmm1
4517 |->BC_MODVN_Z: 3920 |->BC_MODVN_Z:
4518 | call ->vm_mod 3921 | call ->vm_mod
4519 | ins_arithpost 3922 | ins_arithpost
4520 | ins_next 3923 | ins_next
4521 break; 3924 break;
4522 case BC_MODNV: case BC_MODVV: 3925 case BC_MODNV: case BC_MODVV:
4523 | ins_arithpre fld, movsd, xmm1 3926 | ins_arithpre movsd, xmm1
4524 | jmp ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway. 3927 | jmp ->BC_MODVN_Z // Avoid 3 copies. It's slow anyway.
4525 break; 3928 break;
4526 case BC_POW: 3929 case BC_POW:
4527 | ins_arithpre fld, movsd, xmm1 3930 | ins_arithpre movsd, xmm1
4528 | call ->vm_pow 3931 | mov RB, BASE
3932 |.if not X64
3933 | movsd FPARG1, xmm0
3934 | movsd FPARG3, xmm1
3935 |.endif
3936 | call extern pow
3937 | movzx RA, PC_RA
3938 | mov BASE, RB
3939 |.if X64
4529 | ins_arithpost 3940 | ins_arithpost
3941 |.else
3942 | fstp qword [BASE+RA*8]
3943 |.endif
4530 | ins_next 3944 | ins_next
4531 break; 3945 break;
4532 3946
@@ -4594,25 +4008,17 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4594 | movsx RD, RDW 4008 | movsx RD, RDW
4595 | mov dword [BASE+RA*8+4], LJ_TISNUM 4009 | mov dword [BASE+RA*8+4], LJ_TISNUM
4596 | mov dword [BASE+RA*8], RD 4010 | mov dword [BASE+RA*8], RD
4597 |.elif SSE 4011 |.else
4598 | movsx RD, RDW // Sign-extend literal. 4012 | movsx RD, RDW // Sign-extend literal.
4599 | cvtsi2sd xmm0, RD 4013 | cvtsi2sd xmm0, RD
4600 | movsd qword [BASE+RA*8], xmm0 4014 | movsd qword [BASE+RA*8], xmm0
4601 |.else
4602 | fild PC_RD // Refetch signed RD from instruction.
4603 | fstp qword [BASE+RA*8]
4604 |.endif 4015 |.endif
4605 | ins_next 4016 | ins_next
4606 break; 4017 break;
4607 case BC_KNUM: 4018 case BC_KNUM:
4608 | ins_AD // RA = dst, RD = num const 4019 | ins_AD // RA = dst, RD = num const
4609 |.if SSE
4610 | movsd xmm0, qword [KBASE+RD*8] 4020 | movsd xmm0, qword [KBASE+RD*8]
4611 | movsd qword [BASE+RA*8], xmm0 4021 | movsd qword [BASE+RA*8], xmm0
4612 |.else
4613 | fld qword [KBASE+RD*8]
4614 | fstp qword [BASE+RA*8]
4615 |.endif
4616 | ins_next 4022 | ins_next
4617 break; 4023 break;
4618 case BC_KPRI: 4024 case BC_KPRI:
@@ -4719,18 +4125,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4719 case BC_USETN: 4125 case BC_USETN:
4720 | ins_AD // RA = upvalue #, RD = num const 4126 | ins_AD // RA = upvalue #, RD = num const
4721 | mov LFUNC:RB, [BASE-8] 4127 | mov LFUNC:RB, [BASE-8]
4722 |.if SSE
4723 | movsd xmm0, qword [KBASE+RD*8] 4128 | movsd xmm0, qword [KBASE+RD*8]
4724 |.else
4725 | fld qword [KBASE+RD*8]
4726 |.endif
4727 | mov UPVAL:RB, [LFUNC:RB+RA*4+offsetof(GCfuncL, uvptr)] 4129 | mov UPVAL:RB, [LFUNC:RB+RA*4+offsetof(GCfuncL, uvptr)]
4728 | mov RA, UPVAL:RB->v 4130 | mov RA, UPVAL:RB->v
4729 |.if SSE
4730 | movsd qword [RA], xmm0 4131 | movsd qword [RA], xmm0
4731 |.else
4732 | fstp qword [RA]
4733 |.endif
4734 | ins_next 4132 | ins_next
4735 break; 4133 break;
4736 case BC_USETP: 4134 case BC_USETP:
@@ -4884,18 +4282,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4884 |.else 4282 |.else
4885 | // Convert number to int and back and compare. 4283 | // Convert number to int and back and compare.
4886 | checknum RC, >5 4284 | checknum RC, >5
4887 |.if SSE
4888 | movsd xmm0, qword [BASE+RC*8] 4285 | movsd xmm0, qword [BASE+RC*8]
4889 | cvtsd2si RC, xmm0 4286 | cvttsd2si RC, xmm0
4890 | cvtsi2sd xmm1, RC 4287 | cvtsi2sd xmm1, RC
4891 | ucomisd xmm0, xmm1 4288 | ucomisd xmm0, xmm1
4892 |.else
4893 | fld qword [BASE+RC*8]
4894 | fist ARG1
4895 | fild ARG1
4896 | fcomparepp
4897 | mov RC, ARG1
4898 |.endif
4899 | jne ->vmeta_tgetv // Generic numeric key? Use fallback. 4289 | jne ->vmeta_tgetv // Generic numeric key? Use fallback.
4900 |.endif 4290 |.endif
4901 | cmp RC, TAB:RB->asize // Takes care of unordered, too. 4291 | cmp RC, TAB:RB->asize // Takes care of unordered, too.
@@ -4941,7 +4331,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
4941 | mov TAB:RB, [BASE+RB*8] 4331 | mov TAB:RB, [BASE+RB*8]
4942 |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. 4332 |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA.
4943 | mov RA, TAB:RB->hmask 4333 | mov RA, TAB:RB->hmask
4944 | and RA, STR:RC->hash 4334 | and RA, STR:RC->sid
4945 | imul RA, #NODE 4335 | imul RA, #NODE
4946 | add NODE:RA, TAB:RB->node 4336 | add NODE:RA, TAB:RB->node
4947 |1: 4337 |1:
@@ -5019,6 +4409,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5019 | mov dword [BASE+RA*8+4], LJ_TNIL 4409 | mov dword [BASE+RA*8+4], LJ_TNIL
5020 | jmp <1 4410 | jmp <1
5021 break; 4411 break;
4412 case BC_TGETR:
4413 | ins_ABC // RA = dst, RB = table, RC = key
4414 | mov TAB:RB, [BASE+RB*8]
4415 |.if DUALNUM
4416 | mov RC, dword [BASE+RC*8]
4417 |.else
4418 | cvttsd2si RC, qword [BASE+RC*8]
4419 |.endif
4420 | cmp RC, TAB:RB->asize
4421 | jae ->vmeta_tgetr // Not in array part? Use fallback.
4422 | shl RC, 3
4423 | add RC, TAB:RB->array
4424 | // Get array slot.
4425 |->BC_TGETR_Z:
4426 |.if X64
4427 | mov RBa, [RC]
4428 | mov [BASE+RA*8], RBa
4429 |.else
4430 | mov RB, [RC]
4431 | mov RC, [RC+4]
4432 | mov [BASE+RA*8], RB
4433 | mov [BASE+RA*8+4], RC
4434 |.endif
4435 |->BC_TGETR2_Z:
4436 | ins_next
4437 break;
5022 4438
5023 case BC_TSETV: 4439 case BC_TSETV:
5024 | ins_ABC // RA = src, RB = table, RC = key 4440 | ins_ABC // RA = src, RB = table, RC = key
@@ -5032,18 +4448,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5032 |.else 4448 |.else
5033 | // Convert number to int and back and compare. 4449 | // Convert number to int and back and compare.
5034 | checknum RC, >5 4450 | checknum RC, >5
5035 |.if SSE
5036 | movsd xmm0, qword [BASE+RC*8] 4451 | movsd xmm0, qword [BASE+RC*8]
5037 | cvtsd2si RC, xmm0 4452 | cvttsd2si RC, xmm0
5038 | cvtsi2sd xmm1, RC 4453 | cvtsi2sd xmm1, RC
5039 | ucomisd xmm0, xmm1 4454 | ucomisd xmm0, xmm1
5040 |.else
5041 | fld qword [BASE+RC*8]
5042 | fist ARG1
5043 | fild ARG1
5044 | fcomparepp
5045 | mov RC, ARG1
5046 |.endif
5047 | jne ->vmeta_tsetv // Generic numeric key? Use fallback. 4455 | jne ->vmeta_tsetv // Generic numeric key? Use fallback.
5048 |.endif 4456 |.endif
5049 | cmp RC, TAB:RB->asize // Takes care of unordered, too. 4457 | cmp RC, TAB:RB->asize // Takes care of unordered, too.
@@ -5094,7 +4502,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5094 | mov TAB:RB, [BASE+RB*8] 4502 | mov TAB:RB, [BASE+RB*8]
5095 |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. 4503 |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA.
5096 | mov RA, TAB:RB->hmask 4504 | mov RA, TAB:RB->hmask
5097 | and RA, STR:RC->hash 4505 | and RA, STR:RC->sid
5098 | imul RA, #NODE 4506 | imul RA, #NODE
5099 | mov byte TAB:RB->nomm, 0 // Clear metamethod cache. 4507 | mov byte TAB:RB->nomm, 0 // Clear metamethod cache.
5100 | add NODE:RA, TAB:RB->node 4508 | add NODE:RA, TAB:RB->node
@@ -5213,6 +4621,39 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5213 | movzx RA, PC_RA // Restore RA. 4621 | movzx RA, PC_RA // Restore RA.
5214 | jmp <2 4622 | jmp <2
5215 break; 4623 break;
4624 case BC_TSETR:
4625 | ins_ABC // RA = src, RB = table, RC = key
4626 | mov TAB:RB, [BASE+RB*8]
4627 |.if DUALNUM
4628 | mov RC, dword [BASE+RC*8]
4629 |.else
4630 | cvttsd2si RC, qword [BASE+RC*8]
4631 |.endif
4632 | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
4633 | jnz >7
4634 |2:
4635 | cmp RC, TAB:RB->asize
4636 | jae ->vmeta_tsetr
4637 | shl RC, 3
4638 | add RC, TAB:RB->array
4639 | // Set array slot.
4640 |->BC_TSETR_Z:
4641 |.if X64
4642 | mov RBa, [BASE+RA*8]
4643 | mov [RC], RBa
4644 |.else
4645 | mov RB, [BASE+RA*8+4]
4646 | mov RA, [BASE+RA*8]
4647 | mov [RC+4], RB
4648 | mov [RC], RA
4649 |.endif
4650 | ins_next
4651 |
4652 |7: // Possible table write barrier for the value. Skip valiswhite check.
4653 | barrierback TAB:RB, RA
4654 | movzx RA, PC_RA // Restore RA.
4655 | jmp <2
4656 break;
5216 4657
5217 case BC_TSETM: 4658 case BC_TSETM:
5218 | ins_AD // RA = base (table at base-1), RD = num const (start index) 4659 | ins_AD // RA = base (table at base-1), RD = num const (start index)
@@ -5389,10 +4830,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5389 break; 4830 break;
5390 4831
5391 case BC_ITERN: 4832 case BC_ITERN:
5392 | ins_A // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
5393 |.if JIT 4833 |.if JIT
5394 | // NYI: add hotloop, record BC_ITERN. 4834 | hotloop RB
5395 |.endif 4835 |.endif
4836 |->vm_IITERN:
4837 | ins_A // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
5396 | mov TMP1, KBASE // Need two more free registers. 4838 | mov TMP1, KBASE // Need two more free registers.
5397 | mov TMP2, DISPATCH 4839 | mov TMP2, DISPATCH
5398 | mov TAB:RB, [BASE+RA*8-16] 4840 | mov TAB:RB, [BASE+RA*8-16]
@@ -5406,10 +4848,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5406 |.if DUALNUM 4848 |.if DUALNUM
5407 | mov dword [BASE+RA*8+4], LJ_TISNUM 4849 | mov dword [BASE+RA*8+4], LJ_TISNUM
5408 | mov dword [BASE+RA*8], RC 4850 | mov dword [BASE+RA*8], RC
5409 |.elif SSE
5410 | cvtsi2sd xmm0, RC
5411 |.else 4851 |.else
5412 | fild dword [BASE+RA*8-8] 4852 | cvtsi2sd xmm0, RC
5413 |.endif 4853 |.endif
5414 | // Copy array slot to returned value. 4854 | // Copy array slot to returned value.
5415 |.if X64 4855 |.if X64
@@ -5425,10 +4865,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5425 | // Return array index as a numeric key. 4865 | // Return array index as a numeric key.
5426 |.if DUALNUM 4866 |.if DUALNUM
5427 | // See above. 4867 | // See above.
5428 |.elif SSE
5429 | movsd qword [BASE+RA*8], xmm0
5430 |.else 4868 |.else
5431 | fstp qword [BASE+RA*8] 4869 | movsd qword [BASE+RA*8], xmm0
5432 |.endif 4870 |.endif
5433 | mov [BASE+RA*8-8], RC // Update control var. 4871 | mov [BASE+RA*8-8], RC // Update control var.
5434 |2: 4872 |2:
@@ -5441,9 +4879,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5441 | 4879 |
5442 |4: // Skip holes in array part. 4880 |4: // Skip holes in array part.
5443 | add RC, 1 4881 | add RC, 1
5444 |.if not (DUALNUM or SSE)
5445 | mov [BASE+RA*8-8], RC
5446 |.endif
5447 | jmp <1 4882 | jmp <1
5448 | 4883 |
5449 |5: // Traverse hash part. 4884 |5: // Traverse hash part.
@@ -5487,14 +4922,28 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5487 | cmp byte CFUNC:RB->ffid, FF_next_N; jne >5 4922 | cmp byte CFUNC:RB->ffid, FF_next_N; jne >5
5488 | branchPC RD 4923 | branchPC RD
5489 | mov dword [BASE+RA*8-8], 0 // Initialize control var. 4924 | mov dword [BASE+RA*8-8], 0 // Initialize control var.
5490 | mov dword [BASE+RA*8-4], 0xfffe7fff 4925 | mov dword [BASE+RA*8-4], LJ_KEYINDEX
5491 |1: 4926 |1:
5492 | ins_next 4927 | ins_next
5493 |5: // Despecialize bytecode if any of the checks fail. 4928 |5: // Despecialize bytecode if any of the checks fail.
5494 | mov PC_OP, BC_JMP 4929 | mov PC_OP, BC_JMP
5495 | branchPC RD 4930 | branchPC RD
4931 |.if JIT
4932 | cmp byte [PC], BC_ITERN
4933 | jne >6
4934 |.endif
5496 | mov byte [PC], BC_ITERC 4935 | mov byte [PC], BC_ITERC
5497 | jmp <1 4936 | jmp <1
4937 |.if JIT
4938 |6: // Unpatch JLOOP.
4939 | mov RA, [DISPATCH+DISPATCH_J(trace)]
4940 | movzx RC, word [PC+2]
4941 | mov TRACE:RA, [RA+RC*4]
4942 | mov eax, TRACE:RA->startins
4943 | mov al, BC_ITERC
4944 | mov dword [PC], eax
4945 | jmp <1
4946 |.endif
5498 break; 4947 break;
5499 4948
5500 case BC_VARG: 4949 case BC_VARG:
@@ -5777,7 +5226,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5777 if (!vk) { 5226 if (!vk) {
5778 | cmp RB, LJ_TISNUM; jae ->vmeta_for 5227 | cmp RB, LJ_TISNUM; jae ->vmeta_for
5779 } 5228 }
5780 |.if SSE
5781 | movsd xmm0, qword FOR_IDX 5229 | movsd xmm0, qword FOR_IDX
5782 | movsd xmm1, qword FOR_STOP 5230 | movsd xmm1, qword FOR_STOP
5783 if (vk) { 5231 if (vk) {
@@ -5790,22 +5238,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5790 | ucomisd xmm1, xmm0 5238 | ucomisd xmm1, xmm0
5791 |1: 5239 |1:
5792 | movsd qword FOR_EXT, xmm0 5240 | movsd qword FOR_EXT, xmm0
5793 |.else
5794 | fld qword FOR_STOP
5795 | fld qword FOR_IDX
5796 if (vk) {
5797 | fadd qword FOR_STEP // nidx = idx + step
5798 | fst qword FOR_IDX
5799 | fst qword FOR_EXT
5800 | test RB, RB; js >1
5801 } else {
5802 | fst qword FOR_EXT
5803 | jl >1
5804 }
5805 | fxch // Swap lim/(n)idx if step non-negative.
5806 |1:
5807 | fcomparepp
5808 |.endif
5809 if (op == BC_FORI) { 5241 if (op == BC_FORI) {
5810 |.if DUALNUM 5242 |.if DUALNUM
5811 | jnb <7 5243 | jnb <7
@@ -5833,11 +5265,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5833 |2: 5265 |2:
5834 | ins_next 5266 | ins_next
5835 |.endif 5267 |.endif
5836 |.if SSE 5268 |
5837 |3: // Invert comparison if step is negative. 5269 |3: // Invert comparison if step is negative.
5838 | ucomisd xmm0, xmm1 5270 | ucomisd xmm0, xmm1
5839 | jmp <1 5271 | jmp <1
5840 |.endif
5841 break; 5272 break;
5842 5273
5843 case BC_ITERL: 5274 case BC_ITERL:
@@ -5875,7 +5306,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5875 | ins_A // RA = base, RD = target (loop extent) 5306 | ins_A // RA = base, RD = target (loop extent)
5876 | // Note: RA/RD is only used by trace recorder to determine scope/extent 5307 | // Note: RA/RD is only used by trace recorder to determine scope/extent
5877 | // This opcode does NOT jump, it's only purpose is to detect a hot loop. 5308 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
5878 |.if JIT 5309 |.if JIT
5879 | hotloop RB 5310 | hotloop RB
5880 |.endif 5311 |.endif
5881 | // Fall through. Assumes BC_ILOOP follows and ins_A is a no-op. 5312 | // Fall through. Assumes BC_ILOOP follows and ins_A is a no-op.
@@ -5894,7 +5325,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
5894 | mov RDa, TRACE:RD->mcode 5325 | mov RDa, TRACE:RD->mcode
5895 | mov L:RB, SAVE_L 5326 | mov L:RB, SAVE_L
5896 | mov [DISPATCH+DISPATCH_GL(jit_base)], BASE 5327 | mov [DISPATCH+DISPATCH_GL(jit_base)], BASE
5897 | mov [DISPATCH+DISPATCH_GL(jit_L)], L:RB 5328 | mov [DISPATCH+DISPATCH_GL(tmpbuf.L)], L:RB
5898 | // Save additional callee-save registers only used in compiled code. 5329 | // Save additional callee-save registers only used in compiled code.
5899 |.if X64WIN 5330 |.if X64WIN
5900 | mov TMPQ, r12 5331 | mov TMPQ, r12
@@ -6061,9 +5492,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
6061 | // (lua_State *L, lua_CFunction f) 5492 | // (lua_State *L, lua_CFunction f)
6062 | call aword [DISPATCH+DISPATCH_GL(wrapf)] 5493 | call aword [DISPATCH+DISPATCH_GL(wrapf)]
6063 } 5494 }
6064 | set_vmstate INTERP
6065 | // nresults returned in eax (RD). 5495 | // nresults returned in eax (RD).
6066 | mov BASE, L:RB->base 5496 | mov BASE, L:RB->base
5497 | mov [DISPATCH+DISPATCH_GL(cur_L)], L:RB
5498 | set_vmstate INTERP
6067 | lea RA, [BASE+RD*8] 5499 | lea RA, [BASE+RD*8]
6068 | neg RA 5500 | neg RA
6069 | add RA, L:RB->top // RA = (L->top-(L->base+nresults))*8 5501 | add RA, L:RB->top // RA = (L->top-(L->base+nresults))*8
@@ -6176,7 +5608,7 @@ static void emit_asm_debug(BuildCtx *ctx)
6176 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs); 5608 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
6177#endif 5609#endif
6178#if !LJ_NO_UNWIND 5610#if !LJ_NO_UNWIND
6179#if (defined(__sun__) && defined(__svr4__)) 5611#if LJ_TARGET_SOLARIS
6180#if LJ_64 5612#if LJ_64
6181 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@unwind\n"); 5613 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@unwind\n");
6182#else 5614#else
@@ -6383,15 +5815,21 @@ static void emit_asm_debug(BuildCtx *ctx)
6383 "LEFDEY:\n\n", fcsize); 5815 "LEFDEY:\n\n", fcsize);
6384 } 5816 }
6385#endif 5817#endif
6386#if LJ_64 5818#if !LJ_64
6387 fprintf(ctx->fp, "\t.subsections_via_symbols\n");
6388#else
6389 fprintf(ctx->fp, 5819 fprintf(ctx->fp,
6390 "\t.non_lazy_symbol_pointer\n" 5820 "\t.non_lazy_symbol_pointer\n"
6391 "L_lj_err_unwind_dwarf$non_lazy_ptr:\n" 5821 "L_lj_err_unwind_dwarf$non_lazy_ptr:\n"
6392 ".indirect_symbol _lj_err_unwind_dwarf\n" 5822 ".indirect_symbol _lj_err_unwind_dwarf\n"
6393 ".long 0\n"); 5823 ".long 0\n\n");
5824 fprintf(ctx->fp, "\t.section __IMPORT,__jump_table,symbol_stubs,pure_instructions+self_modifying_code,5\n");
5825 {
5826 const char *const *xn;
5827 for (xn = ctx->extnames; *xn; xn++)
5828 if (strncmp(*xn, LABEL_PREFIX, sizeof(LABEL_PREFIX)-1))
5829 fprintf(ctx->fp, "L_%s$stub:\n\t.indirect_symbol _%s\n\t.ascii \"\\364\\364\\364\\364\\364\"\n", *xn, *xn);
5830 }
6394#endif 5831#endif
5832 fprintf(ctx->fp, ".subsections_via_symbols\n");
6395 } 5833 }
6396 break; 5834 break;
6397#endif 5835#endif
diff --git a/src/xb1build.bat b/src/xb1build.bat
new file mode 100644
index 00000000..2eb68171
--- /dev/null
+++ b/src/xb1build.bat
@@ -0,0 +1,101 @@
1@rem Script to build LuaJIT with the Xbox One SDK.
2@rem Donated to the public domain.
3@rem
4@rem Open a "Visual Studio .NET Command Prompt" (64 bit host compiler)
5@rem Then cd to this directory and run this script.
6
7@if not defined INCLUDE goto :FAIL
8@if not defined DurangoXDK goto :FAIL
9
10@setlocal
11@echo ---- Host compiler ----
12@set LJCOMPILE=cl /nologo /c /MD /O2 /W3 /D_CRT_SECURE_NO_DEPRECATE
13@set LJLINK=link /nologo
14@set LJMT=mt /nologo
15@set DASMDIR=..\dynasm
16@set DASM=%DASMDIR%\dynasm.lua
17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
18
19%LJCOMPILE% host\minilua.c
20@if errorlevel 1 goto :BAD
21%LJLINK% /out:minilua.exe minilua.obj
22@if errorlevel 1 goto :BAD
23if exist minilua.exe.manifest^
24 %LJMT% -manifest minilua.exe.manifest -outputresource:minilua.exe
25
26@rem Error out for 64 bit host compiler
27@minilua
28@if not errorlevel 8 goto :FAIL
29
30@set DASMFLAGS=-D WIN -D FFI -D P64
31minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h vm_x64.dasc
32@if errorlevel 1 goto :BAD
33
34%LJCOMPILE% /I "." /I %DASMDIR% /D_DURANGO host\buildvm*.c
35@if errorlevel 1 goto :BAD
36%LJLINK% /out:buildvm.exe buildvm*.obj
37@if errorlevel 1 goto :BAD
38if exist buildvm.exe.manifest^
39 %LJMT% -manifest buildvm.exe.manifest -outputresource:buildvm.exe
40
41buildvm -m peobj -o lj_vm.obj
42@if errorlevel 1 goto :BAD
43buildvm -m bcdef -o lj_bcdef.h %ALL_LIB%
44@if errorlevel 1 goto :BAD
45buildvm -m ffdef -o lj_ffdef.h %ALL_LIB%
46@if errorlevel 1 goto :BAD
47buildvm -m libdef -o lj_libdef.h %ALL_LIB%
48@if errorlevel 1 goto :BAD
49buildvm -m recdef -o lj_recdef.h %ALL_LIB%
50@if errorlevel 1 goto :BAD
51buildvm -m vmdef -o jit\vmdef.lua %ALL_LIB%
52@if errorlevel 1 goto :BAD
53buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
54@if errorlevel 1 goto :BAD
55
56@echo ---- Cross compiler ----
57
58@set CWD=%cd%
59@call "%DurangoXDK%\xdk\DurangoVars.cmd" XDK
60@cd /D "%CWD%"
61@shift
62
63@set LJCOMPILE="cl" /nologo /c /W3 /GF /Gm- /GR- /GS- /Gy /openmp- /D_CRT_SECURE_NO_DEPRECATE /D_LIB /D_UNICODE /D_DURANGO
64@set LJLIB="lib" /nologo
65
66@if "%1"=="debug" (
67 @shift
68 @set LJCOMPILE=%LJCOMPILE% /Zi /MDd /Od
69 @set LJLINK=%LJLINK% /debug
70) else (
71 @set LJCOMPILE=%LJCOMPILE% /MD /O2 /DNDEBUG
72)
73
74@if "%1"=="amalg" goto :AMALG
75%LJCOMPILE% /DLUA_BUILD_AS_DLL lj_*.c lib_*.c
76@if errorlevel 1 goto :BAD
77%LJLIB% /OUT:luajit.lib lj_*.obj lib_*.obj
78@if errorlevel 1 goto :BAD
79@goto :NOAMALG
80:AMALG
81%LJCOMPILE% /DLUA_BUILD_AS_DLL ljamalg.c
82@if errorlevel 1 goto :BAD
83%LJLIB% /OUT:luajit.lib ljamalg.obj lj_vm.obj
84@if errorlevel 1 goto :BAD
85:NOAMALG
86
87@del *.obj *.manifest minilua.exe buildvm.exe
88@echo.
89@echo === Successfully built LuaJIT for Xbox One ===
90
91@goto :END
92:BAD
93@echo.
94@echo *******************************************************
95@echo *** Build FAILED -- Please check the error messages ***
96@echo *******************************************************
97@goto :END
98:FAIL
99@echo To run this script you must open a "Visual Studio .NET Command Prompt"
100@echo (64 bit host compiler). The Xbox One SDK must be installed, too.
101:END
diff --git a/src/xedkbuild.bat b/src/xedkbuild.bat
index 240ec878..37322d03 100644
--- a/src/xedkbuild.bat
+++ b/src/xedkbuild.bat
@@ -14,7 +14,7 @@
14@set LJMT=mt /nologo 14@set LJMT=mt /nologo
15@set DASMDIR=..\dynasm 15@set DASMDIR=..\dynasm
16@set DASM=%DASMDIR%\dynasm.lua 16@set DASM=%DASMDIR%\dynasm.lua
17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c 17@set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c lib_buffer.c
18 18
19%LJCOMPILE% host\minilua.c 19%LJCOMPILE% host\minilua.c
20@if errorlevel 1 goto :BAD 20@if errorlevel 1 goto :BAD