Ruby 3.2.2p53 (2023-03-30 revision e51014f9c05aa65cbf203442d37fef7c12390015)
yjit.c
1// This part of YJIT helps interfacing with the rest of CRuby and with the OS.
2// Sometimes our FFI binding generation tool gives undesirable outputs when it
3// sees C features that Rust doesn't support well. We mitigate that by binding
4// functions which have simple parameter types. The boilerplate C functions for
5// that purpose are in this file.
6// Similarly, we wrap OS facilities we need in simple functions to help with
7// FFI and to avoid the need to use external crates.io Rust libraries.
8
9#include "internal.h"
10#include "internal/sanitizers.h"
11#include "internal/string.h"
12#include "internal/hash.h"
13#include "internal/variable.h"
14#include "internal/compile.h"
15#include "internal/class.h"
16#include "internal/fixnum.h"
17#include "gc.h"
18#include "vm_core.h"
19#include "vm_callinfo.h"
20#include "builtin.h"
21#include "insns.inc"
22#include "insns_info.inc"
23#include "vm_sync.h"
24#include "yjit.h"
25#include "vm_insnhelper.h"
26#include "probes.h"
27#include "probes_helper.h"
28#include "iseq.h"
29#include "ruby/debug.h"
30#include "internal/cont.h"
31
32// For mmapp(), sysconf()
33#ifndef _WIN32
34#include <unistd.h>
35#include <sys/mman.h>
36#endif
37
38#include <errno.h>
39
40// We need size_t to have a known size to simplify code generation and FFI.
41// TODO(alan): check this in configure.ac to fail fast on 32 bit platforms.
42STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX);
43// I don't know any C implementation that has uint64_t and puts padding bits
44// into size_t but the standard seems to allow it.
45STATIC_ASSERT(size_t_no_padding_bits, sizeof(size_t) == sizeof(uint64_t));
46
47// This build config impacts the pointer tagging scheme and we only want to
48// support one scheme for simplicity.
49STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM);
50
51// NOTE: We can trust that uint8_t has no "padding bits" since the C spec
52// guarantees it. Wording about padding bits is more explicit in C11 compared
53// to C99. See C11 7.20.1.1p2. All this is to say we have _some_ standards backing to
54// use a Rust `*mut u8` to represent a C `uint8_t *`.
55//
56// If we don't want to trust that we can interpreter the C standard correctly, we
57// could outsource that work to the Rust standard library by sticking to fundamental
58// types in C such as int, long, etc. and use `std::os::raw::c_long` and friends on
59// the Rust side.
60//
61// What's up with the long prefix? Even though we build with `-fvisibility=hidden`
62// we are sometimes a static library where the option doesn't prevent name collision.
63// The "_yjit_" part is for trying to be informative. We might want different
64// suffixes for symbols meant for Rust and symbols meant for broader CRuby.
65
66bool
67rb_yjit_mark_writable(void *mem_block, uint32_t mem_size)
68{
69 return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
70}
71
72void
73rb_yjit_mark_executable(void *mem_block, uint32_t mem_size)
74{
75 // Do not call mprotect when mem_size is zero. Some platforms may return
76 // an error for it. https://github.com/Shopify/ruby/issues/450
77 if (mem_size == 0) {
78 return;
79 }
80 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
81 rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
82 mem_block, (unsigned long)mem_size, strerror(errno));
83 }
84}
85
86// Free the specified memory block.
87bool
88rb_yjit_mark_unused(void *mem_block, uint32_t mem_size)
89{
90 // On Linux, you need to use madvise MADV_DONTNEED to free memory.
91 // We might not need to call this on macOS, but it's not really documented.
92 // We generally prefer to do the same thing on both to ease testing too.
93 madvise(mem_block, mem_size, MADV_DONTNEED);
94
95 // On macOS, mprotect PROT_NONE seems to reduce RSS.
96 // We also call this on Linux to avoid executing unused pages.
97 return mprotect(mem_block, mem_size, PROT_NONE) == 0;
98}
99
100// `start` is inclusive and `end` is exclusive.
101void
102rb_yjit_icache_invalidate(void *start, void *end)
103{
104 // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
105 // but required on ARM before running freshly written code.
106 // On Darwin it's the same as calling sys_icache_invalidate().
107#ifdef __GNUC__
108 __builtin___clear_cache(start, end);
109#elif defined(__aarch64__)
110#error No instruction cache clear available with this compiler on Aarch64!
111#endif
112}
113
114# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
115
116// For a given raw_sample (frame), set the hash with the caller's
117// name, file, and line number. Return the hash with collected frame_info.
118static void
119rb_yjit_add_frame(VALUE hash, VALUE frame)
120{
121 VALUE frame_id = PTR2NUM(frame);
122
123 if (RTEST(rb_hash_aref(hash, frame_id))) {
124 return;
125 }
126 else {
127 VALUE frame_info = rb_hash_new();
128 // Full label for the frame
130 // Absolute path of the frame from rb_iseq_realpath
132 // Line number of the frame
134
135 // If absolute path isn't available use the rb_iseq_path
136 if (NIL_P(file)) {
137 file = rb_profile_frame_path(frame);
138 }
139
140 rb_hash_aset(frame_info, ID2SYM(rb_intern("name")), name);
141 rb_hash_aset(frame_info, ID2SYM(rb_intern("file")), file);
142 rb_hash_aset(frame_info, ID2SYM(rb_intern("samples")), INT2NUM(0));
143 rb_hash_aset(frame_info, ID2SYM(rb_intern("total_samples")), INT2NUM(0));
144 rb_hash_aset(frame_info, ID2SYM(rb_intern("edges")), rb_hash_new());
145 rb_hash_aset(frame_info, ID2SYM(rb_intern("lines")), rb_hash_new());
146
147 if (line != INT2FIX(0)) {
148 rb_hash_aset(frame_info, ID2SYM(rb_intern("line")), line);
149 }
150
151 rb_hash_aset(hash, frame_id, frame_info);
152 }
153}
154
155// Parses the YjitExitLocations raw_samples and line_samples collected by
156// rb_yjit_record_exit_stack and turns them into 3 hashes (raw, lines, and frames) to
157// be used by RubyVM::YJIT.exit_locations. yjit_raw_samples represents the raw frames information
158// (without name, file, and line), and yjit_line_samples represents the line information
159// of the iseq caller.
160VALUE
161rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int samples_len)
162{
163 VALUE result = rb_hash_new();
164 VALUE raw_samples = rb_ary_new_capa(samples_len);
165 VALUE line_samples = rb_ary_new_capa(samples_len);
166 VALUE frames = rb_hash_new();
167 int idx = 0;
168
169 // While the index is less than samples_len, parse yjit_raw_samples and
170 // yjit_line_samples, then add casted values to raw_samples and line_samples array.
171 while (idx < samples_len) {
172 int num = (int)yjit_raw_samples[idx];
173 int line_num = (int)yjit_line_samples[idx];
174 idx++;
175
176 rb_ary_push(raw_samples, SIZET2NUM(num));
177 rb_ary_push(line_samples, INT2NUM(line_num));
178
179 // Loop through the length of samples_len and add data to the
180 // frames hash. Also push the current value onto the raw_samples
181 // and line_samples array respectively.
182 for (int o = 0; o < num; o++) {
183 rb_yjit_add_frame(frames, yjit_raw_samples[idx]);
184 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
185 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
186 idx++;
187 }
188
189 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
190 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
191 idx++;
192
193 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
194 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
195 idx++;
196 }
197
198 // Set add the raw_samples, line_samples, and frames to the results
199 // hash.
200 rb_hash_aset(result, ID2SYM(rb_intern("raw")), raw_samples);
201 rb_hash_aset(result, ID2SYM(rb_intern("lines")), line_samples);
202 rb_hash_aset(result, ID2SYM(rb_intern("frames")), frames);
203
204 return result;
205}
206
207uint32_t
208rb_yjit_get_page_size(void)
209{
210#if defined(_SC_PAGESIZE)
211 long page_size = sysconf(_SC_PAGESIZE);
212 if (page_size <= 0) rb_bug("yjit: failed to get page size");
213
214 // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
215 // Though our design sort of assume we have fine grained control over memory protection
216 // which require small page sizes.
217 if (page_size > 0x40000000l) rb_bug("yjit page size too large");
218
219 return (uint32_t)page_size;
220#else
221#error "YJIT supports POSIX only for now"
222#endif
223}
224
225#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
226// Align the current write position to a multiple of bytes
227static uint8_t *
228align_ptr(uint8_t *ptr, uint32_t multiple)
229{
230 // Compute the pointer modulo the given alignment boundary
231 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
232
233 // If the pointer is already aligned, stop
234 if (rem == 0)
235 return ptr;
236
237 // Pad the pointer by the necessary amount to align it
238 uint32_t pad = multiple - rem;
239
240 return ptr + pad;
241}
242#endif
243
244// Address space reservation. Memory pages are mapped on an as needed basis.
245// See the Rust mm module for details.
246uint8_t *
247rb_yjit_reserve_addr_space(uint32_t mem_size)
248{
249#ifndef _WIN32
250 uint8_t *mem_block;
251
252 // On Linux
253 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
254 uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
255 uint8_t *const cfunc_sample_addr = (void *)&rb_yjit_reserve_addr_space;
256 uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
257 // Align the requested address to page size
258 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
259
260 // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
261 // to improve odds of being in range for 32-bit relative call instructions.
262 do {
263 mem_block = mmap(
264 req_addr,
265 mem_size,
266 PROT_NONE,
267 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
268 -1,
269 0
270 );
271
272 // If we succeeded, stop
273 if (mem_block != MAP_FAILED) {
274 break;
275 }
276
277 // +4MB
278 req_addr += 4 * 1024 * 1024;
279 } while (req_addr < probe_region_end);
280
281 // On MacOS and other platforms
282 #else
283 // Try to map a chunk of memory as executable
284 mem_block = mmap(
285 (void *)rb_yjit_reserve_addr_space,
286 mem_size,
287 PROT_NONE,
288 MAP_PRIVATE | MAP_ANONYMOUS,
289 -1,
290 0
291 );
292 #endif
293
294 // Fallback
295 if (mem_block == MAP_FAILED) {
296 // Try again without the address hint (e.g., valgrind)
297 mem_block = mmap(
298 NULL,
299 mem_size,
300 PROT_NONE,
301 MAP_PRIVATE | MAP_ANONYMOUS,
302 -1,
303 0
304 );
305 }
306
307 // Check that the memory mapping was successful
308 if (mem_block == MAP_FAILED) {
309 perror("ruby: yjit: mmap:");
310 if(errno == ENOMEM) {
311 // No crash report if it's only insufficient memory
312 exit(EXIT_FAILURE);
313 }
314 rb_bug("mmap failed");
315 }
316
317 return mem_block;
318#else
319 // Windows not supported for now
320 return NULL;
321#endif
322}
323
324// Is anyone listening for :c_call and :c_return event currently?
325bool
326rb_c_method_tracing_currently_enabled(rb_execution_context_t *ec)
327{
328 rb_event_flag_t tracing_events;
329 if (rb_multi_ractor_p()) {
330 tracing_events = ruby_vm_event_enabled_global_flags;
331 }
332 else {
333 // At the time of writing, events are never removed from
334 // ruby_vm_event_enabled_global_flags so always checking using it would
335 // mean we don't compile even after tracing is disabled.
336 tracing_events = rb_ec_ractor_hooks(ec)->events;
337 }
338
339 return tracing_events & (RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN);
340}
341
342// The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event
343// like the interpreter. When tracing for c_return is enabled, we patch the code after
344// the C method return to call into this to fire the event.
345void
346rb_full_cfunc_return(rb_execution_context_t *ec, VALUE return_value)
347{
348 rb_control_frame_t *cfp = ec->cfp;
349 RUBY_ASSERT_ALWAYS(cfp == GET_EC()->cfp);
350 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
351
352 RUBY_ASSERT_ALWAYS(RUBYVM_CFUNC_FRAME_P(cfp));
353 RUBY_ASSERT_ALWAYS(me->def->type == VM_METHOD_TYPE_CFUNC);
354
355 // CHECK_CFP_CONSISTENCY("full_cfunc_return"); TODO revive this
356
357 // Pop the C func's frame and fire the c_return TracePoint event
358 // Note that this is the same order as vm_call_cfunc_with_frame().
359 rb_vm_pop_frame(ec);
360 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
361 // Note, this deviates from the interpreter in that users need to enable
362 // a c_return TracePoint for this DTrace hook to work. A reasonable change
363 // since the Ruby return event works this way as well.
364 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
365
366 // Push return value into the caller's stack. We know that it's a frame that
367 // uses cfp->sp because we are patching a call done with gen_send_cfunc().
368 ec->cfp->sp[0] = return_value;
369 ec->cfp->sp++;
370}
371
372unsigned int
373rb_iseq_encoded_size(const rb_iseq_t *iseq)
374{
375 return iseq->body->iseq_size;
376}
377
378// TODO(alan): consider using an opaque pointer for the payload rather than a void pointer
379void *
380rb_iseq_get_yjit_payload(const rb_iseq_t *iseq)
381{
382 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
383 if (iseq->body) {
384 return iseq->body->yjit_payload;
385 }
386 else {
387 // Body is NULL when constructing the iseq.
388 return NULL;
389 }
390}
391
392void
393rb_iseq_set_yjit_payload(const rb_iseq_t *iseq, void *payload)
394{
395 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
396 RUBY_ASSERT_ALWAYS(iseq->body);
397 RUBY_ASSERT_ALWAYS(NULL == iseq->body->yjit_payload);
398 iseq->body->yjit_payload = payload;
399}
400
401void
402rb_iseq_reset_jit_func(const rb_iseq_t *iseq)
403{
404 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
405 iseq->body->jit_func = NULL;
406 // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint,
407 // we'd like to re-compile ISEQs that haven't been converted to trace_* insns.
408 iseq->body->total_calls = 0;
409}
410
411// Get the PC for a given index in an iseq
412VALUE *
413rb_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx)
414{
415 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
416 RUBY_ASSERT_ALWAYS(insn_idx < iseq->body->iseq_size);
417 VALUE *encoded = iseq->body->iseq_encoded;
418 VALUE *pc = &encoded[insn_idx];
419 return pc;
420}
421
422// Get the opcode given a program counter. Can return trace opcode variants.
423int
424rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc)
425{
426 // YJIT should only use iseqs after AST to bytecode compilation
427 RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED));
428
429 const VALUE at_pc = *pc;
430 return rb_vm_insn_addr2opcode((const void *)at_pc);
431}
432
433// used by jit_rb_str_bytesize in codegen.rs
434VALUE
435rb_str_bytesize(VALUE str)
436{
437 return LONG2NUM(RSTRING_LEN(str));
438}
439
440unsigned long
441rb_RSTRING_LEN(VALUE str)
442{
443 return RSTRING_LEN(str);
444}
445
446char *
447rb_RSTRING_PTR(VALUE str)
448{
449 return RSTRING_PTR(str);
450}
451
452rb_proc_t *
453rb_yjit_get_proc_ptr(VALUE procv)
454{
455 rb_proc_t *proc;
456 GetProcPtr(procv, proc);
457 return proc;
458}
459
460// This is defined only as a named struct inside rb_iseq_constant_body.
461// By giving it a separate typedef, we make it nameable by rust-bindgen.
462// Bindgen's temp/anon name isn't guaranteed stable.
463typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct;
464
465const char *
466rb_insn_name(VALUE insn)
467{
468 return insn_name(insn);
469}
470
471// Query the instruction length in bytes for YARV opcode insn
472int
473rb_insn_len(VALUE insn)
474{
475 return insn_len(insn);
476}
477
478unsigned int
479rb_vm_ci_argc(const struct rb_callinfo *ci)
480{
481 return vm_ci_argc(ci);
482}
483
484ID
485rb_vm_ci_mid(const struct rb_callinfo *ci)
486{
487 return vm_ci_mid(ci);
488}
489
490unsigned int
491rb_vm_ci_flag(const struct rb_callinfo *ci)
492{
493 return vm_ci_flag(ci);
494}
495
496const struct rb_callinfo_kwarg *
497rb_vm_ci_kwarg(const struct rb_callinfo *ci)
498{
499 return vm_ci_kwarg(ci);
500}
501
502int
503rb_get_cikw_keyword_len(const struct rb_callinfo_kwarg *cikw)
504{
505 return cikw->keyword_len;
506}
507
508VALUE
509rb_get_cikw_keywords_idx(const struct rb_callinfo_kwarg *cikw, int idx)
510{
511 return cikw->keywords[idx];
512}
513
514rb_method_visibility_t
515rb_METHOD_ENTRY_VISI(const rb_callable_method_entry_t *me)
516{
517 return METHOD_ENTRY_VISI(me);
518}
519
520rb_method_type_t
521rb_get_cme_def_type(const rb_callable_method_entry_t *cme)
522{
523 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
524 return VM_METHOD_TYPE_UNDEF;
525 } else {
526 return cme->def->type;
527 }
528}
529
530ID
531rb_get_cme_def_body_attr_id(const rb_callable_method_entry_t *cme)
532{
533 return cme->def->body.attr.id;
534}
535
536ID rb_get_symbol_id(VALUE namep);
537
538enum method_optimized_type
539rb_get_cme_def_body_optimized_type(const rb_callable_method_entry_t *cme)
540{
541 return cme->def->body.optimized.type;
542}
543
544unsigned int
545rb_get_cme_def_body_optimized_index(const rb_callable_method_entry_t *cme)
546{
547 return cme->def->body.optimized.index;
548}
549
551rb_get_cme_def_body_cfunc(const rb_callable_method_entry_t *cme)
552{
553 return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
554}
555
556uintptr_t
557rb_get_def_method_serial(const rb_method_definition_t *def)
558{
559 return def->method_serial;
560}
561
562ID
563rb_get_def_original_id(const rb_method_definition_t *def)
564{
565 return def->original_id;
566}
567
568int
569rb_get_mct_argc(const rb_method_cfunc_t *mct)
570{
571 return mct->argc;
572}
573
574void *
575rb_get_mct_func(const rb_method_cfunc_t *mct)
576{
577 return (void*)mct->func; // this field is defined as type VALUE (*func)(ANYARGS)
578}
579
580const rb_iseq_t *
581rb_get_def_iseq_ptr(rb_method_definition_t *def)
582{
583 return def_iseq_ptr(def);
584}
585
586VALUE
587rb_get_def_bmethod_proc(rb_method_definition_t *def)
588{
589 RUBY_ASSERT(def->type == VM_METHOD_TYPE_BMETHOD);
590 return def->body.bmethod.proc;
591}
592
593const rb_iseq_t *
594rb_get_iseq_body_local_iseq(const rb_iseq_t *iseq)
595{
596 return iseq->body->local_iseq;
597}
598
599const rb_iseq_t *
600rb_get_iseq_body_parent_iseq(const rb_iseq_t *iseq)
601{
602 return iseq->body->parent_iseq;
603}
604
605unsigned int
606rb_get_iseq_body_local_table_size(const rb_iseq_t *iseq)
607{
608 return iseq->body->local_table_size;
609}
610
611VALUE *
612rb_get_iseq_body_iseq_encoded(const rb_iseq_t *iseq)
613{
614 return iseq->body->iseq_encoded;
615}
616
617bool
618rb_get_iseq_body_builtin_inline_p(const rb_iseq_t *iseq)
619{
620 return iseq->body->builtin_inline_p;
621}
622
623unsigned
624rb_get_iseq_body_stack_max(const rb_iseq_t *iseq)
625{
626 return iseq->body->stack_max;
627}
628
629bool
630rb_get_iseq_flags_has_lead(const rb_iseq_t *iseq)
631{
632 return iseq->body->param.flags.has_lead;
633}
634
635bool
636rb_get_iseq_flags_has_opt(const rb_iseq_t *iseq)
637{
638 return iseq->body->param.flags.has_opt;
639}
640
641bool
642rb_get_iseq_flags_has_kw(const rb_iseq_t *iseq)
643{
644 return iseq->body->param.flags.has_kw;
645}
646
647bool
648rb_get_iseq_flags_has_post(const rb_iseq_t *iseq)
649{
650 return iseq->body->param.flags.has_post;
651}
652
653bool
654rb_get_iseq_flags_has_kwrest(const rb_iseq_t *iseq)
655{
656 return iseq->body->param.flags.has_kwrest;
657}
658
659bool
660rb_get_iseq_flags_has_rest(const rb_iseq_t *iseq)
661{
662 return iseq->body->param.flags.has_rest;
663}
664
665bool
666rb_get_iseq_flags_ruby2_keywords(const rb_iseq_t *iseq)
667{
668 return iseq->body->param.flags.ruby2_keywords;
669}
670
671bool
672rb_get_iseq_flags_has_block(const rb_iseq_t *iseq)
673{
674 return iseq->body->param.flags.has_block;
675}
676
677bool
678rb_get_iseq_flags_ambiguous_param0(const rb_iseq_t *iseq)
679{
680 return iseq->body->param.flags.ambiguous_param0;
681}
682
683bool
684rb_get_iseq_flags_accepts_no_kwarg(const rb_iseq_t *iseq)
685{
686 return iseq->body->param.flags.accepts_no_kwarg;
687}
688
689const rb_seq_param_keyword_struct *
690rb_get_iseq_body_param_keyword(const rb_iseq_t *iseq)
691{
692 return iseq->body->param.keyword;
693}
694
695unsigned
696rb_get_iseq_body_param_size(const rb_iseq_t *iseq)
697{
698 return iseq->body->param.size;
699}
700
701int
702rb_get_iseq_body_param_lead_num(const rb_iseq_t *iseq)
703{
704 return iseq->body->param.lead_num;
705}
706
707int
708rb_get_iseq_body_param_opt_num(const rb_iseq_t *iseq)
709{
710 return iseq->body->param.opt_num;
711}
712
713const VALUE *
714rb_get_iseq_body_param_opt_table(const rb_iseq_t *iseq)
715{
716 return iseq->body->param.opt_table;
717}
718
719VALUE
720rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv, int kw_splat, VALUE block_handler)
721{
722 rb_proc_t *proc;
723 GetProcPtr(recv, proc);
724 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
725}
726
727
728// If true, the iseq is leaf and it can be replaced by a single C call.
729bool
730rb_leaf_invokebuiltin_iseq_p(const rb_iseq_t *iseq)
731{
732 unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
733 unsigned int leave_len = insn_len(BIN(leave));
734
735 return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
736 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
737 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
738 iseq->body->builtin_inline_p
739 );
740}
741
742// Return an rb_builtin_function if the iseq contains only that leaf builtin function.
743const struct rb_builtin_function *
744rb_leaf_builtin_function(const rb_iseq_t *iseq)
745{
746 if (!rb_leaf_invokebuiltin_iseq_p(iseq))
747 return NULL;
748 return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
749}
750
751VALUE
752rb_yjit_str_simple_append(VALUE str1, VALUE str2)
753{
754 return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2));
755}
756
758rb_get_ec_cfp(const rb_execution_context_t *ec)
759{
760 return ec->cfp;
761}
762
763VALUE *
764rb_get_cfp_pc(struct rb_control_frame_struct *cfp)
765{
766 return (VALUE*)cfp->pc;
767}
768
769VALUE *
770rb_get_cfp_sp(struct rb_control_frame_struct *cfp)
771{
772 return cfp->sp;
773}
774
775void
776rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc)
777{
778 cfp->pc = pc;
779}
780
781void
782rb_set_cfp_sp(struct rb_control_frame_struct *cfp, VALUE *sp)
783{
784 cfp->sp = sp;
785}
786
787rb_iseq_t *
788rb_cfp_get_iseq(struct rb_control_frame_struct *cfp)
789{
790 // TODO(alan) could assert frame type here to make sure that it's a ruby frame with an iseq.
791 return (rb_iseq_t*)cfp->iseq;
792}
793
794VALUE
795rb_get_cfp_self(struct rb_control_frame_struct *cfp)
796{
797 return cfp->self;
798}
799
800VALUE *
801rb_get_cfp_ep(struct rb_control_frame_struct *cfp)
802{
803 return (VALUE*)cfp->ep;
804}
805
806const VALUE *
807rb_get_cfp_ep_level(struct rb_control_frame_struct *cfp, uint32_t lv)
808{
809 uint32_t i;
810 const VALUE *ep = (VALUE*)cfp->ep;
811 for (i = 0; i < lv; i++) {
812 ep = VM_ENV_PREV_EP(ep);
813 }
814 return ep;
815}
816
817VALUE
818rb_yarv_class_of(VALUE obj)
819{
820 return rb_class_of(obj);
821}
822
823// YJIT needs this function to never allocate and never raise
824VALUE
825rb_yarv_str_eql_internal(VALUE str1, VALUE str2)
826{
827 // We wrap this since it's static inline
828 return rb_str_eql_internal(str1, str2);
829}
830
831// YJIT needs this function to never allocate and never raise
832VALUE
833rb_yarv_ary_entry_internal(VALUE ary, long offset)
834{
835 return rb_ary_entry_internal(ary, offset);
836}
837
838VALUE
839rb_yarv_fix_mod_fix(VALUE recv, VALUE obj)
840{
841 return rb_fix_mod_fix(recv, obj);
842}
843
844// Print the Ruby source location of some ISEQ for debugging purposes
845void
846rb_yjit_dump_iseq_loc(const rb_iseq_t *iseq, uint32_t insn_idx)
847{
848 char *ptr;
849 long len;
850 VALUE path = rb_iseq_path(iseq);
851 RSTRING_GETMEM(path, ptr, len);
852 fprintf(stderr, "%s %.*s:%u\n", __func__, (int)len, ptr, rb_iseq_line_no(iseq, insn_idx));
853}
854
855// The FL_TEST() macro
856VALUE
857rb_FL_TEST(VALUE obj, VALUE flags)
858{
859 return RB_FL_TEST(obj, flags);
860}
861
862// The FL_TEST_RAW() macro, normally an internal implementation detail
863VALUE
864rb_FL_TEST_RAW(VALUE obj, VALUE flags)
865{
866 return FL_TEST_RAW(obj, flags);
867}
868
869// The RB_TYPE_P macro
870bool
871rb_RB_TYPE_P(VALUE obj, enum ruby_value_type t)
872{
873 return RB_TYPE_P(obj, t);
874}
875
876long
877rb_RSTRUCT_LEN(VALUE st)
878{
879 return RSTRUCT_LEN(st);
880}
881
882// There are RSTRUCT_SETs in ruby/internal/core/rstruct.h and internal/struct.h
883// with different types (int vs long) for k. Here we use the one from ruby/internal/core/rstruct.h,
884// which takes an int.
885void
886rb_RSTRUCT_SET(VALUE st, int k, VALUE v)
887{
888 RSTRUCT_SET(st, k, v);
889}
890
891const struct rb_callinfo *
892rb_get_call_data_ci(const struct rb_call_data *cd)
893{
894 return cd->ci;
895}
896
897bool
898rb_BASIC_OP_UNREDEFINED_P(enum ruby_basic_operators bop, uint32_t klass)
899{
900 return BASIC_OP_UNREDEFINED_P(bop, klass);
901}
902
903VALUE
904rb_RCLASS_ORIGIN(VALUE c)
905{
906 return RCLASS_ORIGIN(c);
907}
908
909// Return the string encoding index
910int
911rb_ENCODING_GET(VALUE obj)
912{
913 return RB_ENCODING_GET(obj);
914}
915
916bool
917rb_yjit_multi_ractor_p(void)
918{
919 return rb_multi_ractor_p();
920}
921
922// For debug builds
923void
924rb_assert_iseq_handle(VALUE handle)
925{
926 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
927 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_iseq));
928}
929
930int
931rb_IMEMO_TYPE_P(VALUE imemo, enum imemo_type imemo_type)
932{
933 return IMEMO_TYPE_P(imemo, imemo_type);
934}
935
936void
937rb_assert_cme_handle(VALUE handle)
938{
939 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
940 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_ment));
941}
942
943// Used for passing a callback and other data over rb_objspace_each_objects
945 rb_iseq_callback callback;
946 void *data;
947};
948
949// Heap-walking callback for rb_yjit_for_each_iseq().
950static int
951for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data)
952{
953 const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data;
954 VALUE v = (VALUE)vstart;
955 for (; v != (VALUE)vend; v += stride) {
956 void *ptr = asan_poisoned_object_p(v);
957 asan_unpoison_object(v, false);
958
959 if (rb_obj_is_iseq(v)) {
960 rb_iseq_t *iseq = (rb_iseq_t *)v;
961 callback_data->callback(iseq, callback_data->data);
962 }
963
964 asan_poison_object_if(ptr, v);
965 }
966 return 0;
967}
968
969// Iterate through the whole GC heap and invoke a callback for each iseq.
970// Used for global code invalidation.
971void
972rb_yjit_for_each_iseq(rb_iseq_callback callback, void *data)
973{
974 struct iseq_callback_data callback_data = { .callback = callback, .data = data };
975 rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data);
976}
977
978// For running write barriers from Rust. Required when we add a new edge in the
979// object graph from `old` to `young`.
980void
981rb_yjit_obj_written(VALUE old, VALUE young, const char *file, int line)
982{
983 rb_obj_written(old, Qundef, young, file, line);
984}
985
986// Acquire the VM lock and then signal all other Ruby threads (ractors) to
987// contend for the VM lock, putting them to sleep. YJIT uses this to evict
988// threads running inside generated code so among other things, it can
989// safely change memory protection of regions housing generated code.
990void
991rb_yjit_vm_lock_then_barrier(unsigned int *recursive_lock_level, const char *file, int line)
992{
993 rb_vm_lock_enter(recursive_lock_level, file, line);
994 rb_vm_barrier();
995}
996
997// Release the VM lock. The lock level must point to the same integer used to
998// acquire the lock.
999void
1000rb_yjit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line)
1001{
1002 rb_vm_lock_leave(recursive_lock_level, file, line);
1003}
1004
1005// Pointer to a YJIT entry point (machine code generated by YJIT)
1006typedef VALUE (*yjit_func_t)(rb_execution_context_t *, rb_control_frame_t *);
1007
1008bool
1009rb_yjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec)
1010{
1011 bool success = true;
1012 RB_VM_LOCK_ENTER();
1013 rb_vm_barrier();
1014
1015 // Compile a block version starting at the first instruction
1016 uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec); // defined in Rust
1017 uint8_t *code_ptr = rb_yjit_iseq_gen_entry_point(iseq, ec);
1018
1019 if (code_ptr) {
1020 iseq->body->jit_func = (yjit_func_t)code_ptr;
1021 }
1022 else {
1023 iseq->body->jit_func = 0;
1024 success = false;
1025 }
1026
1027 RB_VM_LOCK_LEAVE();
1028 return success;
1029}
1030
1031// GC root for interacting with the GC
1033 bool unused; // empty structs are not legal in C99
1034};
1035
1036static void
1037yjit_root_free(void *ptr)
1038{
1039 // Do nothing. The root lives as long as the process.
1040}
1041
1042static size_t
1043yjit_root_memsize(const void *ptr)
1044{
1045 // Count off-gc-heap allocation size of the dependency table
1046 return 0; // TODO: more accurate accounting
1047}
1048
1049// GC callback during compaction
1050static void
1051yjit_root_update_references(void *ptr)
1052{
1053 // Do nothing since we use rb_gc_mark(), which pins.
1054}
1055
1056void rb_yjit_root_mark(void *ptr); // in Rust
1057
1058// Custom type for interacting with the GC
1059// TODO: make this write barrier protected
1060static const rb_data_type_t yjit_root_type = {
1061 "yjit_root",
1062 {rb_yjit_root_mark, yjit_root_free, yjit_root_memsize, yjit_root_update_references},
1063 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1064};
1065
1066// For dealing with refinements
1067void
1068rb_yjit_invalidate_all_method_lookup_assumptions(void)
1069{
1070 // It looks like Module#using actually doesn't need to invalidate all the
1071 // method caches, so we do nothing here for now.
1072}
1073
1074// Number of object shapes, which might be useful for investigating YJIT exit reasons.
1075static VALUE
1076object_shape_count(rb_execution_context_t *ec, VALUE self)
1077{
1078 // next_shape_id starts from 0, so it's the same as the count
1079 return ULONG2NUM((unsigned long)GET_VM()->next_shape_id);
1080}
1081
1082// Primitives used by yjit.rb
1083VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self);
1084VALUE rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self);
1085VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self);
1086VALUE rb_yjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self);
1087VALUE rb_yjit_disasm_iseq(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1088VALUE rb_yjit_insns_compiled(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1089VALUE rb_yjit_code_gc(rb_execution_context_t *ec, VALUE self);
1090VALUE rb_yjit_simulate_oom_bang(rb_execution_context_t *ec, VALUE self);
1091VALUE rb_yjit_get_exit_locations(rb_execution_context_t *ec, VALUE self);
1092
1093// Preprocessed yjit.rb generated during build
1094#include "yjit.rbinc"
1095
1096// Can raise RuntimeError
1097void
1098rb_yjit_init(void)
1099{
1100 // Call the Rust initialization code
1101 void rb_yjit_init_rust(void);
1102 rb_yjit_init_rust();
1103
1104 // Initialize the GC hooks. Do this second as some code depend on Rust initialization.
1105 struct yjit_root_struct *root;
1106 VALUE yjit_root = TypedData_Make_Struct(0, struct yjit_root_struct, &yjit_root_type, root);
1107 rb_gc_register_mark_object(yjit_root);
1108}
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:177
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition assert.h:167
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:39
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:40
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:103
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define ULONG2NUM
Old name of RB_ULONG2NUM.
Definition long.h:60
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:140
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define NIL_P
Old name of RB_NIL_P.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition error.c:794
static VALUE rb_class_of(VALUE obj)
Object to class mapping function.
Definition globals.h:172
Defines RBIMPL_HAS_BUILTIN.
VALUE rb_str_cat(VALUE dst, const char *src, long srclen)
Destructively appends the passed contents to the string.
Definition string.c:3150
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:574
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition rtypeddata.h:489
#define RTEST
This is an old name of RB_TEST.
#define USE_FLONUM
Definition method.h:62
struct rb_iseq_constant_body::@132 param
parameter information
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
ruby_value_type
C-level type of an object.
Definition value_type.h:112