68#include "ruby/internal/config.h"
75#include "internal/class.h"
76#include "internal/cmdlineopt.h"
77#include "internal/cont.h"
78#include "internal/file.h"
79#include "internal/hash.h"
80#include "internal/process.h"
81#include "internal/warnings.h"
83#include "ractor_core.h"
86#define __EXTENSIONS__ 1
90#include "vm_callinfo.h"
94#include "ruby_assert.h"
100#include "insns_info.inc"
101#include "internal/compile.h"
110#ifdef HAVE_SYS_PARAM_H
111# include <sys/param.h>
119# define MAXPATHLEN 1024
123#define MJIT_ATOMIC_SET(var, val) (void)ATOMIC_PTR_EXCHANGE(var, val)
125#define MJIT_TMP_PREFIX "_ruby_mjit_"
133extern void mjit_add_waiting_pid(
rb_vm_t *vm, rb_pid_t pid);
138struct mjit_options mjit_opts;
141bool mjit_enabled =
false;
144bool mjit_call_p =
false;
146bool mjit_cancel_p =
false;
149static bool mjit_compile_p =
false;
151static int active_units_length = 0;
153static int compact_units_length = 0;
157static struct rb_mjit_unit_list unit_queue = { CCAN_LIST_HEAD_INIT(unit_queue.head) };
159static struct rb_mjit_unit_list active_units = { CCAN_LIST_HEAD_INIT(active_units.head) };
161static struct rb_mjit_unit_list compact_units = { CCAN_LIST_HEAD_INIT(compact_units.head) };
163static struct rb_mjit_unit_list stale_units = { CCAN_LIST_HEAD_INIT(stale_units.head) };
165static int current_unit_num;
167static rb_nativethread_lock_t mjit_engine_mutex;
169static bool stop_worker_p;
171static bool worker_stopped =
true;
177static const char *cc_path;
179static const char **cc_common_args;
181static char **cc_added_args;
183static char *pch_file;
185static rb_pid_t pch_owner_pid;
188static enum {PCH_NOT_READY, PCH_FAILED, PCH_SUCCESS} pch_status;
191static double current_cc_ms = 0.0;
195static pid_t current_cc_pid = 0;
198static char *header_file;
200#include "mjit_config.h"
202#if defined(__GNUC__) && \
203 (!defined(__clang__) || \
204 (defined(__clang__) && (defined(__FreeBSD__) || defined(__GLIBC__))))
205# define GCC_PIC_FLAGS "-Wfatal-errors", "-fPIC", "-shared", "-w", "-pipe",
206# define MJIT_CFLAGS_PIPE 1
208# define GCC_PIC_FLAGS
209# define MJIT_CFLAGS_PIPE 0
214#if defined __GNUC__ && !defined __clang__ && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__)
215# define GCC_NOSTDLIB_FLAGS "-nodefaultlibs", "-nostdlib",
217# define GCC_NOSTDLIB_FLAGS
220static const char *
const CC_COMMON_ARGS[] = {
221 MJIT_CC_COMMON MJIT_CFLAGS GCC_PIC_FLAGS
225static const char *
const CC_DEBUG_ARGS[] = {MJIT_DEBUGFLAGS NULL};
226static const char *
const CC_OPTIMIZE_ARGS[] = {MJIT_OPTFLAGS NULL};
228static const char *
const CC_LDSHARED_ARGS[] = {MJIT_LDSHARED MJIT_CFLAGS GCC_PIC_FLAGS NULL};
229static const char *
const CC_DLDFLAGS_ARGS[] = {MJIT_DLDFLAGS NULL};
231static const char *
const CC_LINKER_ARGS[] = {
232#if defined __GNUC__ && !defined __clang__ && !defined(__OpenBSD__)
235 GCC_NOSTDLIB_FLAGS NULL
238static const char *
const CC_LIBS[] = {
239#if defined(__CYGWIN__)
242#if defined __GNUC__ && !defined __clang__
245#if defined __ANDROID__
251#define CC_CODEFLAG_ARGS (mjit_opts.debug ? CC_DEBUG_ARGS : CC_OPTIMIZE_ARGS)
255PRINTF_ARGS(
static void, 2, 3)
256verbose(
int level, const
char *format, ...)
258 if (mjit_opts.verbose >= level) {
260 size_t len = strlen(format);
261 char *full_format = alloca(
sizeof(
char) * (len + 2));
264 memcpy(full_format, format, len);
265 full_format[len] =
'\n';
266 full_format[len+1] =
'\0';
268 va_start(args, format);
269 vfprintf(stderr, full_format, args);
274PRINTF_ARGS(
static void, 1, 2)
275mjit_warning(const
char *format, ...)
277 if (mjit_opts.warnings || mjit_opts.verbose) {
280 fprintf(stderr,
"MJIT warning: ");
281 va_start(args, format);
282 vfprintf(stderr, format, args);
284 fprintf(stderr,
"\n");
293 ccan_list_add_tail(&list->head, &unit->unode);
300 ccan_list_del(&unit->unode);
305remove_file(
const char *filename)
307 if (remove(filename)) {
308 mjit_warning(
"failed to remove \"%s\": %s", filename, strerror(errno));
324 ISEQ_BODY(unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED;
325 ISEQ_BODY(unit->iseq)->mjit_unit = NULL;
327 if (unit->cc_entries) {
328 void *entries = (
void *)unit->cc_entries;
331 if (unit->handle && dlclose(unit->handle)) {
332 mjit_warning(
"failed to close handle for u%d: %s", unit->id, dlerror());
339CRITICAL_SECTION_START(
int level,
const char *msg)
341 verbose(level,
"Locking %s", msg);
343 verbose(level,
"Locked %s", msg);
349CRITICAL_SECTION_FINISH(
int level,
const char *msg)
351 verbose(level,
"Unlocked %s", msg);
355static pid_t mjit_pid = 0;
358sprint_uniq_filename(
char *str,
size_t size,
unsigned long id,
const char *prefix,
const char *suffix)
360 return snprintf(str, size,
"%s/%sp%"PRI_PIDT_PREFIX"uu%lu%s", tmp_dir, prefix, mjit_pid,
id, suffix);
365double ruby_real_ms_time(
void);
366# define real_ms_time() ruby_real_ms_time()
371# ifdef HAVE_CLOCK_GETTIME
373# ifdef CLOCK_MONOTONIC
374 const clockid_t c = CLOCK_MONOTONIC;
376 const clockid_t c = CLOCK_REALTIME;
379 clock_gettime(c, &tv);
380 return tv.tv_nsec / 1000000.0 + tv.tv_sec * 1000.0;
384 gettimeofday(&tv, NULL);
385 return tv.tv_usec / 1000.0 + tv.tv_sec * 1000.0;
398 ccan_list_for_each_safe(&list->head, unit, next, unode) {
399 if (unit->iseq == NULL) {
400 remove_from_list(unit, list);
405 if (best == NULL || ISEQ_BODY(best->iseq)->total_calls < ISEQ_BODY(unit->iseq)->total_calls) {
411 remove_from_list(best, list);
418args_len(
char *
const *args)
422 for (i = 0; (args[i]) != NULL;i++)
430form_args(
int num, ...)
435 char **args, **res, **tmp;
439 for (i = len = 0; i < num; i++) {
440 args = va_arg(argp,
char **);
442 if ((tmp = (
char **)realloc(res,
sizeof(
char *) * (len + n + 1))) == NULL) {
448 MEMCPY(res + len, args,
char *, n + 1);
456#if __has_warning("-Wdeprecated-declarations") || RBIMPL_COMPILER_IS(GCC)
457COMPILER_WARNING_IGNORED(-Wdeprecated-declarations)
462start_process(
const char *abspath,
char *
const *argv)
468 verbose(1,
"MJIT: Failed to open a null device: %s", strerror(errno));
471 if (mjit_opts.verbose >= 2) {
473 fprintf(stderr,
"Starting process: %s", abspath);
474 for (
int i = 0; (arg = argv[i]) != NULL; i++)
475 fprintf(stderr,
" %s", arg);
476 fprintf(stderr,
"\n");
480 if ((pid = vfork()) == 0) {
482 if (mjit_opts.verbose == 0) {
486 dup2(dev_null, STDERR_FILENO);
487 dup2(dev_null, STDOUT_FILENO);
489 (void)close(dev_null);
490 pid = execv(abspath, argv);
494 verbose(1,
"MJIT: Error in execv: %s", abspath);
497 (void)close(dev_null);
506exec_process(
const char *path,
char *
const argv[])
508 int stat, exit_code = -2;
509 pid_t pid = start_process(path, argv);
511 pid_t r = waitpid(pid, &stat, 0);
513 if (errno == EINTR)
continue;
514 fprintf(stderr,
"[%"PRI_PIDT_PREFIX"d] waitpid(%lu): %s (SIGCHLD=%d,%u)\n",
515 getpid(), (
unsigned long)pid, strerror(errno),
516 RUBY_SIGCHLD, SIGCHLD_LOSSY);
520 if (WIFEXITED(stat)) {
521 exit_code = WEXITSTATUS(stat);
524 else if (WIFSIGNALED(stat)) {
534remove_so_file(
const char *so_file,
struct rb_mjit_unit *unit)
536 remove_file(so_file);
541sprint_funcname(
char *funcname,
size_t funcname_size,
const struct rb_mjit_unit *unit)
544 if (iseq == NULL || (!mjit_opts.debug && !mjit_opts.debug_flags)) {
545 snprintf(funcname, funcname_size,
"_mjit%d", unit->id);
550 const char *path =
RSTRING_PTR(rb_iseq_path(iseq));
551 const char *lib =
"/lib/";
553 while (strstr(path, lib))
554 path = strstr(path, lib) + strlen(lib);
555 while (strstr(path, version))
556 path = strstr(path, version) + strlen(version);
559 const char *method =
RSTRING_PTR(ISEQ_BODY(iseq)->location.label);
560 if (!strcmp(method,
"[]")) method =
"AREF";
561 if (!strcmp(method,
"[]=")) method =
"ASET";
564 snprintf(funcname, funcname_size,
"_mjit%d_%s_%s", unit->id, path, method);
565 for (
size_t i = 0; i < strlen(funcname); i++) {
566 char c = funcname[i];
567 if (!((
'a' <= c && c <=
'z') || (
'A' <= c && c <=
'Z') || (
'0' <= c && c <=
'9') || c ==
'_')) {
573static const int c_file_access_mode =
577 O_WRONLY|O_EXCL|O_CREAT;
579#define append_str2(p, str, len) ((char *)memcpy((p), str, (len))+(len))
580#define append_str(p, str) append_str2(p, str, sizeof(str)-1)
581#define append_lit(p, str) append_str2(p, str, rb_strlen_lit(str))
587 const char *rest_args[] = {
595 "-o", pch_file, header_file,
599 verbose(2,
"Creating precompiled header");
600 char **args = form_args(4, cc_common_args, CC_CODEFLAG_ARGS, cc_added_args, rest_args);
602 mjit_warning(
"making precompiled header failed on forming args");
603 pch_status = PCH_FAILED;
607 int exit_code = exec_process(cc_path, args);
610 if (exit_code == 0) {
611 pch_status = PCH_SUCCESS;
614 mjit_warning(
"Making precompiled header failed on compilation. Stopping MJIT worker...");
615 pch_status = PCH_FAILED;
620c_compile(
const char *c_file,
const char *so_file)
622 const char *so_args[] = {
625 "-include-pch", pch_file,
630# if defined(__MACH__)
631 extern VALUE rb_libruby_selfpath;
632 const char *loader_args[] = {
"-bundle_loader",
StringValuePtr(rb_libruby_selfpath), NULL};
634 const char *loader_args[] = {NULL};
637 char **args = form_args(8, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS, cc_added_args,
638 so_args, loader_args, CC_LIBS, CC_DLDFLAGS_ARGS, CC_LINKER_ARGS);
639 if (args == NULL)
return 1;
641 int exit_code = exec_process(cc_path, args);
642 if (!mjit_opts.save_temps)
652 static const char c_ext[] =
".c";
653 static const char so_ext[] = DLEXT;
654 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
656 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
657 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
659 return c_compile(c_file, so_file);
662static void compile_prelude(
FILE *f);
667 VM_ASSERT(unit->type == MJIT_UNIT_BATCH);
668 static const char c_ext[] =
".c";
669 static const char so_ext[] = DLEXT;
670 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
672 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
673 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
677 if (fd < 0 || (f = fdopen(fd,
"w")) == NULL) {
679 if (fd >= 0) (void)close(fd);
680 verbose(1,
"Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
688 ccan_list_for_each(&unit->units.head, child_unit, unode) {
689 if (!success)
continue;
690 if (child_unit->iseq == NULL)
continue;
692 char funcname[MAXPATHLEN];
693 sprint_funcname(funcname,
sizeof(funcname), child_unit);
695 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
696 const char *sep =
"@";
697 const char *iseq_label =
RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
698 const char *iseq_path =
RSTRING_PTR(rb_iseq_path(child_unit->iseq));
699 if (!iseq_label) iseq_label = sep =
"";
700 fprintf(f,
"\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
701 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
713 VM_ASSERT(unit->type == MJIT_UNIT_COMPACT);
714 static const char c_ext[] =
".c";
715 static const char so_ext[] = DLEXT;
716 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
718 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
719 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
723 if (fd < 0 || (f = fdopen(fd,
"w")) == NULL) {
725 if (fd >= 0) (void)close(fd);
726 verbose(1,
"Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
733 compact_units_length = 0;
735 ccan_list_for_each(&active_units.head, batch_unit, unode) {
736 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
737 if (!success)
continue;
738 if (child_unit->iseq == NULL)
continue;
740 char funcname[MAXPATHLEN];
741 sprint_funcname(funcname,
sizeof(funcname), child_unit);
743 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
744 const char *sep =
"@";
745 const char *iseq_label =
RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
746 const char *iseq_path =
RSTRING_PTR(rb_iseq_path(child_unit->iseq));
747 if (!iseq_label) iseq_label = sep =
"";
748 fprintf(f,
"\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
749 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
750 compact_units_length++;
759load_batch_funcs_from_so(
struct rb_mjit_unit *unit,
char *c_file,
char *so_file)
761 double end_time = real_ms_time();
763 void *handle = dlopen(so_file, RTLD_NOW);
764 if (handle == NULL) {
765 mjit_warning(
"failure in loading code from batched '%s': %s", so_file, dlerror());
769 unit->handle = handle;
772 add_to_list(unit, &active_units);
773 active_units_length += unit->units.length;
775 if (!mjit_opts.save_temps)
776 remove_so_file(so_file, unit);
779 ccan_list_for_each(&unit->units.head, child_unit, unode) {
780 char funcname[MAXPATHLEN];
781 sprint_funcname(funcname,
sizeof(funcname), child_unit);
784 if ((func = dlsym(handle, funcname)) == NULL) {
785 mjit_warning(
"skipping to load '%s' from '%s': %s", funcname, so_file, dlerror());
789 if (child_unit->iseq) {
791 const rb_iseq_t *iseq = child_unit->iseq;
792 MJIT_ATOMIC_SET(ISEQ_BODY(iseq)->jit_func, (jit_func_t)func);
794 verbose(1,
"JIT success: %s@%s:%d",
796 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
799 verbose(1,
"JIT skip: A compiled method has been GCed");
802 verbose(1,
"JIT batch (%.1fms): Batched %d methods %s -> %s", end_time - current_cc_ms, unit->units.length, c_file, so_file);
806load_compact_funcs_from_so(
struct rb_mjit_unit *unit,
char *c_file,
char *so_file)
808 double end_time = real_ms_time();
810 void *handle = dlopen(so_file, RTLD_NOW);
811 if (handle == NULL) {
812 mjit_warning(
"failure in loading code from compacted '%s': %s", so_file, dlerror());
816 unit->handle = handle;
819 add_to_list(unit, &compact_units);
821 if (!mjit_opts.save_temps)
822 remove_so_file(so_file, unit);
825 ccan_list_for_each(&active_units.head, batch_unit, unode) {
826 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
827 if (child_unit->iseq == NULL)
continue;
829 char funcname[MAXPATHLEN];
830 sprint_funcname(funcname,
sizeof(funcname), child_unit);
833 if ((func = dlsym(handle, funcname)) == NULL) {
834 mjit_warning(
"skipping to reload '%s' from '%s': %s", funcname, so_file, dlerror());
838 if (child_unit->iseq) {
840 MJIT_ATOMIC_SET(ISEQ_BODY(child_unit->iseq)->jit_func, (jit_func_t)func);
844 verbose(1,
"JIT compaction (%.1fms): Compacted %d methods %s -> %s", end_time - current_cc_ms, active_units_length, c_file, so_file);
849header_name_end(
const char *s)
851 const char *e = s + strlen(s);
853 static const char suffix[] =
".gch";
856 if (e > s+
sizeof(suffix)-1 && strcmp(e-
sizeof(suffix)+1, suffix) == 0) {
857 e -=
sizeof(suffix)-1;
866compile_prelude(
FILE *f)
869 const char *s = pch_file;
870 const char *e = header_name_end(s);
872 fprintf(f,
"#include \"");
888 extern pid_t rb_mjit_fork();
889 pid_t pid = rb_mjit_fork();
891 int exit_code = c_compile_unit(unit);
909 VM_ASSERT(compiled_iseq != NULL);
910 VM_ASSERT(compiled_iseq->mjit_unit != NULL);
911 VM_ASSERT(captured_iseq != NULL);
914 unsigned int new_entries_size = unit->cc_entries_size + captured_iseq->ci_size;
915 VM_ASSERT(captured_iseq->ci_size > 0);
919 int cc_entries_index = unit->cc_entries_size;
920 if (unit->cc_entries_size == 0) {
921 VM_ASSERT(unit->cc_entries == NULL);
922 unit->cc_entries = cc_entries = malloc(
sizeof(
struct rb_callcache *) * new_entries_size);
923 if (cc_entries == NULL)
return -1;
926 void *cc_ptr = (
void *)unit->cc_entries;
927 cc_entries = realloc(cc_ptr,
sizeof(
struct rb_callcache *) * new_entries_size);
928 if (cc_entries == NULL)
return -1;
929 unit->cc_entries = cc_entries;
930 cc_entries += cc_entries_index;
932 unit->cc_entries_size = new_entries_size;
935 for (
unsigned int i = 0; i < captured_iseq->ci_size; i++) {
936 cc_entries[i] = captured_iseq->call_data[i].cc;
939 return cc_entries_index;
942static void mjit_add_iseq_to_process(
const rb_iseq_t *iseq,
const struct rb_mjit_compile_info *compile_info);
948get_uniq_filename(
unsigned long id,
const char *prefix,
const char *suffix)
950 char buff[70], *str = buff;
951 int size = sprint_uniq_filename(buff,
sizeof(buff),
id, prefix, suffix);
955 if (size <= (
int)
sizeof(buff)) {
956 memcpy(str, buff, size);
959 sprint_uniq_filename(str, size,
id, prefix, suffix);
966mjit_cancel_all(
const char *reason)
972 mjit_cancel_p =
true;
973 if (mjit_opts.warnings || mjit_opts.verbose) {
974 fprintf(stderr,
"JIT cancel: Disabled JIT-ed code because %s\n", reason);
980mjit_update_references(
const rb_iseq_t *iseq)
985 CRITICAL_SECTION_START(4,
"mjit_update_references");
986 if (ISEQ_BODY(iseq)->mjit_unit) {
987 ISEQ_BODY(iseq)->mjit_unit->iseq = (
rb_iseq_t *)rb_gc_location((
VALUE)ISEQ_BODY(iseq)->mjit_unit->iseq);
991 mjit_cancel_all(
"GC.compact is used");
998 ccan_list_for_each(&stale_units.head, unit, unode) {
999 if (unit->iseq == iseq) {
1003 CRITICAL_SECTION_FINISH(4,
"mjit_update_references");
1014 if (ISEQ_BODY(iseq)->mjit_unit) {
1017 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL;
1023 ccan_list_for_each(&stale_units.head, unit, unode) {
1024 if (unit->iseq == iseq) {
1038 ccan_list_for_each_safe(&list->head, unit, next, unode) {
1039 ccan_list_del(&unit->unode);
1040 if (!close_handle_p) unit->handle = NULL;
1042 if (list == &stale_units) {
1047 if (unit->handle && dlclose(unit->handle)) {
1048 mjit_warning(
"failed to close handle for u%d: %s", unit->id, dlerror());
1060create_unit(
enum rb_mjit_unit_type type)
1063 unit->id = current_unit_num++;
1065 if (type == MJIT_UNIT_BATCH) {
1066 ccan_list_head_init(&unit->units.head);
1074 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_ISEQ);
1076 ISEQ_BODY(iseq)->mjit_unit = unit;
1084check_unit_queue(
void)
1086 if (mjit_opts.custom)
return;
1087 if (worker_stopped)
return;
1088 if (current_cc_pid != 0)
return;
1091 if (active_units_length >= mjit_opts.max_cache_size)
return;
1094 if (!mjit_compile_p)
return;
1095 mjit_compile_p =
false;
1098 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_BATCH);
1100 VM_ASSERT(unit_queue.length > 0);
1101 while ((child_unit = get_from_list(&unit_queue)) != NULL && (active_units_length + unit->units.length) < mjit_opts.max_cache_size) {
1102 add_to_list(child_unit, &unit->units);
1103 ISEQ_BODY(child_unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_COMPILING;
1107 current_cc_ms = real_ms_time();
1108 current_cc_unit = unit;
1109 bool success = mjit_batch(unit);
1111 mjit_notify_waitpid(1);
1116 if (mjit_opts.wait) {
1117 int exit_code = c_compile_unit(unit);
1118 mjit_notify_waitpid(exit_code);
1121 current_cc_pid = start_c_compile_unit(unit);
1122 if (current_cc_pid == -1) {
1123 mjit_notify_waitpid(1);
1130check_compaction(
void)
1134 int max_compact_size = mjit_opts.max_cache_size / 100;
1135 if (max_compact_size < 10) max_compact_size = 10;
1138 int throttle_threshold = active_units_length / 10;
1140 if (compact_units.length < max_compact_size
1141 && active_units_length - compact_units_length > throttle_threshold
1142 && ((!mjit_opts.wait && unit_queue.length == 0 && active_units.length > 1)
1143 || (active_units_length == mjit_opts.max_cache_size))) {
1144 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_COMPACT);
1147 current_cc_ms = real_ms_time();
1148 current_cc_unit = unit;
1149 bool success = mjit_compact(unit);
1151 mjit_notify_waitpid(1);
1156 if (mjit_opts.wait) {
1157 int exit_code = c_compile_unit(unit);
1158 mjit_notify_waitpid(exit_code);
1161 current_cc_pid = start_c_compile_unit(unit);
1162 if (current_cc_pid == -1) {
1163 mjit_notify_waitpid(1);
1171mjit_notify_waitpid(
int exit_code)
1173 VM_ASSERT(mjit_opts.wait || current_cc_pid != 0);
1177 char c_file[MAXPATHLEN];
1178 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), current_cc_unit->id, MJIT_TMP_PREFIX,
".c");
1181 if (exit_code != 0) {
1182 verbose(2,
"Failed to generate so");
1185 free_unit(current_cc_unit);
1186 current_cc_unit = NULL;
1191 char so_file[MAXPATHLEN];
1192 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), current_cc_unit->id, MJIT_TMP_PREFIX, DLEXT);
1193 switch (current_cc_unit->type) {
1194 case MJIT_UNIT_ISEQ:
1195 rb_bug(
"unreachable: current_cc_unit->type must not be MJIT_UNIT_ISEQ");
1196 case MJIT_UNIT_BATCH:
1197 load_batch_funcs_from_so(current_cc_unit, c_file, so_file);
1198 current_cc_unit = NULL;
1201 if (!stop_worker_p) {
1205 case MJIT_UNIT_COMPACT:
1206 load_compact_funcs_from_so(current_cc_unit, c_file, so_file);
1207 current_cc_unit = NULL;
1212 if (!stop_worker_p) {
1220mjit_target_iseq_p(
const rb_iseq_t *iseq)
1223 return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK)
1224 && !body->builtin_inline_p
1225 && strcmp(
"<internal:mjit>",
RSTRING_PTR(rb_iseq_path(iseq))) != 0;
1229static VALUE rb_mMJIT = 0;
1231static VALUE rb_mMJITC = 0;
1233static VALUE rb_cMJITCompiler = 0;
1235static VALUE rb_cMJITIseqPtr = 0;
1237static VALUE rb_cMJITICPtr = 0;
1239static VALUE rb_mMJITHooks = 0;
1241#define WITH_MJIT_DISABLED(stmt) do { \
1242 bool original_call_p = mjit_call_p; \
1243 mjit_call_p = false; \
1245 mjit_call_p = original_call_p; \
1246 if (mjit_cancel_p) mjit_call_p = false; \
1250MJIT_FUNC_EXPORTED
void
1251rb_mjit_bop_redefined(
int redefined_flag,
enum ruby_basic_operators bop)
1253 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1254 WITH_MJIT_DISABLED({
1260MJIT_FUNC_EXPORTED
void
1263 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1264 WITH_MJIT_DISABLED({
1273rb_mjit_before_ractor_spawn(
void)
1275 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1276 WITH_MJIT_DISABLED({
1282mjit_constant_state_changed(
void *data)
1284 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1286 WITH_MJIT_DISABLED({
1292MJIT_FUNC_EXPORTED
void
1293rb_mjit_constant_state_changed(
ID id)
1295 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1298 rb_workqueue_register(0, mjit_constant_state_changed, (
void *)
id);
1302MJIT_FUNC_EXPORTED
void
1303rb_mjit_constant_ic_update(
const rb_iseq_t *
const iseq,
IC ic,
unsigned insn_idx)
1305 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1306 WITH_MJIT_DISABLED({
1314MJIT_FUNC_EXPORTED
void
1317 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks)
return;
1318 WITH_MJIT_DISABLED({
1325mjit_hook_custom_compile(
const rb_iseq_t *iseq)
1327 WITH_MJIT_DISABLED({
1331 ISEQ_BODY(iseq)->jit_func = (jit_func_t)
NUM2ULONG(jit_func);
1336mjit_add_iseq_to_process(
const rb_iseq_t *iseq,
const struct rb_mjit_compile_info *compile_info)
1338 if (!mjit_enabled)
return;
1339 if (mjit_opts.custom) {
1340 mjit_hook_custom_compile(iseq);
1343 if (pch_status != PCH_SUCCESS || !rb_ractor_main_p())
1345 if (!mjit_target_iseq_p(iseq)) {
1346 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED;
1352 bool recompile_p = !MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func);
1353 if (!ISEQ_BODY(iseq)->mjit_unit || recompile_p) {
1356 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL;
1357 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_NOT_COMPILED;
1358 active_units_length--;
1364 VM_ASSERT(compile_info != NULL);
1365 unit->compile_info = *compile_info;
1367 add_to_list(unit, &unit_queue);
1368 ISEQ_BODY(iseq)->total_calls = 0;
1371 VM_ASSERT(compile_info == NULL);
1372 mjit_compile_p =
true;
1379rb_mjit_add_iseq_to_process(
const rb_iseq_t *iseq)
1381 mjit_add_iseq_to_process(iseq, NULL);
1386#define MJIT_WAIT_TIMEOUT_SECONDS 5
1391 pid_t initial_pid = current_cc_pid;
1392 if (initial_pid == 0) {
1393 mjit_warning(
"initial_pid was 0 on mjit_wait");
1396 if (pch_status == PCH_FAILED)
return;
1399 struct timeval tv = { .tv_sec = 0, .tv_usec = 1000 };
1400 while (current_cc_pid == initial_pid) {
1402 if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS) {
1403 if (unit->type == MJIT_UNIT_ISEQ) {
1404 unit->iseq->body->jit_func = (jit_func_t)MJIT_FUNC_FAILED;
1406 mjit_warning(
"timed out to wait for JIT finish");
1414struct rb_mjit_compile_info*
1417 VM_ASSERT(body->mjit_unit != NULL);
1418 return &body->mjit_unit->compile_info;
1424 if (MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func))
1427 verbose(1,
"JIT recompile: %s@%s:%d",
RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
1428 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
1429 VM_ASSERT(ISEQ_BODY(iseq)->mjit_unit != NULL);
1431 mjit_add_iseq_to_process(iseq, &ISEQ_BODY(iseq)->mjit_unit->compile_info);
1437rb_mjit_recompile_send(
const rb_iseq_t *iseq)
1439 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_send_cache =
true;
1440 mjit_recompile(iseq);
1445rb_mjit_recompile_ivar(
const rb_iseq_t *iseq)
1447 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_ivar_cache =
true;
1448 mjit_recompile(iseq);
1453rb_mjit_recompile_exivar(
const rb_iseq_t *iseq)
1455 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_exivar_cache =
true;
1456 mjit_recompile(iseq);
1461rb_mjit_recompile_inlining(
const rb_iseq_t *iseq)
1463 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_inlining =
true;
1464 mjit_recompile(iseq);
1469rb_mjit_recompile_const(
const rb_iseq_t *iseq)
1471 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_const_cache =
true;
1472 mjit_recompile(iseq);
1475extern VALUE ruby_archlibdir_path, ruby_prefix_path;
1479init_header_filename(
void)
1486 const char *basedir =
"";
1491 basedir_val = ruby_prefix_path;
1495 if (getenv(
"MJIT_SEARCH_BUILD_DIR")) {
1501 const char *hdr = dlsym(RTLD_DEFAULT,
"MJIT_HEADER");
1503 verbose(1,
"No MJIT_HEADER");
1505 else if (hdr[0] !=
'/') {
1506 verbose(1,
"Non-absolute header file path: %s", hdr);
1508 else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) {
1509 verbose(1,
"Non-file header file path: %s", hdr);
1511 else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
1512 !rb_path_check(hdr)) {
1513 verbose(1,
"Unsafe header file: uid=%ld mode=%#o %s",
1514 (
long)st.st_uid, (
unsigned)st.st_mode, hdr);
1520 verbose(3,
"PRELOADENV("PRELOADENV
")=%s", getenv(PRELOADENV));
1522 unsetenv(PRELOADENV);
1523 verbose(3,
"MJIT_HEADER: %s", hdr);
1525 if (!header_file)
return false;
1532 static const char header_name[] = MJIT_HEADER_INSTALL_DIR
"/" MJIT_MIN_HEADER_NAME;
1533 const size_t header_name_len =
sizeof(header_name) - 1;
1535 header_file =
xmalloc(baselen + header_name_len + 1);
1536 p = append_str2(header_file, basedir, baselen);
1537 p = append_str2(p, header_name, header_name_len + 1);
1540 verbose(1,
"Cannot access header file: %s", header_file);
1548 pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX
"h",
".h.gch");
1554system_default_tmpdir(
void)
1557#if defined _CS_DARWIN_USER_TEMP_DIR
1558 char path[MAXPATHLEN];
1559 size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path,
sizeof(path));
1562 if (len >
sizeof(path)) {
1563 confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
1566 memcpy(tmpdir, path, len);
1575check_tmpdir(
const char *dir)
1579 if (!dir)
return FALSE;
1580 if (stat(dir, &st))
return FALSE;
1582# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
1584 if (!S_ISDIR(st.st_mode))
return FALSE;
1588 if (st.st_mode & S_IWOTH) {
1590 if (!(st.st_mode & S_ISVTX))
return FALSE;
1595 if (access(dir, W_OK))
return FALSE;
1603# define RETURN_ENV(name) \
1604 if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
1605 RETURN_ENV(
"TMPDIR");
1607 tmpdir = system_default_tmpdir();
1608 if (check_tmpdir(tmpdir))
return tmpdir;
1614#define MIN_CACHE_SIZE 10
1616#define DEFAULT_MAX_CACHE_SIZE 100
1618#define DEFAULT_CALL_THRESHOLD 10000
1624 stop_worker_p =
false;
1625 worker_stopped =
false;
1631ruby_strndup(
const char *str,
size_t n)
1634 memcpy(ret, str, n);
1642split_flags(
const char *flags)
1644 char *buf[MAXPATHLEN];
1647 for (; flags != NULL; flags = next) {
1648 next = strchr(flags,
' ');
1650 if (strlen(flags) > 0)
1651 buf[i++] =
strdup(flags);
1655 buf[i++] = ruby_strndup(flags, next - flags);
1660 char **ret =
xmalloc(
sizeof(
char *) * (i + 1));
1661 memcpy(ret, buf,
sizeof(
char *) * i);
1666#define opt_match_noarg(s, l, name) \
1667 opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --mjit-" name " is ignored"), 1) : 1)
1668#define opt_match_arg(s, l, name) \
1669 opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--mjit-" name " needs an argument"), 0))
1672mjit_setup_options(
const char *s,
struct mjit_options *mjit_opt)
1674 const size_t l = strlen(s);
1678 else if (opt_match_noarg(s, l,
"warnings")) {
1679 mjit_opt->warnings =
true;
1681 else if (opt_match(s, l,
"debug")) {
1683 mjit_opt->debug_flags =
strdup(s + 1);
1685 mjit_opt->debug =
true;
1687 else if (opt_match_noarg(s, l,
"wait")) {
1688 mjit_opt->wait =
true;
1690 else if (opt_match_noarg(s, l,
"save-temps")) {
1691 mjit_opt->save_temps =
true;
1693 else if (opt_match(s, l,
"verbose")) {
1694 mjit_opt->verbose = *s ? atoi(s + 1) : 1;
1696 else if (opt_match_arg(s, l,
"max-cache")) {
1697 mjit_opt->max_cache_size = atoi(s + 1);
1699 else if (opt_match_arg(s, l,
"call-threshold")) {
1700 mjit_opt->call_threshold = atoi(s + 1);
1703 else if (opt_match_noarg(s, l,
"pause")) {
1704 mjit_opt->pause =
true;
1708 "invalid MJIT option `%s' (--help will show valid MJIT options)", s);
1712#define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc)
1714 M(
"--mjit-warnings",
"",
"Enable printing JIT warnings"),
1715 M(
"--mjit-debug",
"",
"Enable JIT debugging (very slow), or add cflags if specified"),
1716 M(
"--mjit-wait",
"",
"Wait until JIT compilation finishes every time (for testing)"),
1717 M(
"--mjit-save-temps",
"",
"Save JIT temporary files in $TMP or /tmp (for testing)"),
1718 M(
"--mjit-verbose=num",
"",
"Print JIT logs of level num or less to stderr (default: 0)"),
1719 M(
"--mjit-max-cache=num",
"",
"Max number of methods to be JIT-ed in a cache (default: "
1720 STRINGIZE(DEFAULT_MAX_CACHE_SIZE)
")"),
1721 M(
"--mjit-call-threshold=num",
"",
"Number of calls to trigger JIT (for testing, default: "
1722 STRINGIZE(DEFAULT_CALL_THRESHOLD)
")"),
1731mjit_init(
const struct mjit_options *opts)
1733 VM_ASSERT(mjit_enabled);
1739 verbose(1,
"Disabling MJIT because RubyVM::MJIT::Compiler is not defined");
1740 mjit_enabled =
false;
1751 mjit_pid = getpid();
1754 if (mjit_opts.call_threshold == 0)
1755 mjit_opts.call_threshold = DEFAULT_CALL_THRESHOLD;
1756 if (mjit_opts.call_threshold % 2 == 1) {
1757 mjit_opts.call_threshold += 1;
1758 mjit_warning(
"--mjit-call-threshold must be an even number. Using %d instead.", mjit_opts.call_threshold);
1760 mjit_opts.call_threshold /= 2;
1761 if (mjit_opts.max_cache_size <= 0)
1762 mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
1763 if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
1764 mjit_opts.max_cache_size = MIN_CACHE_SIZE;
1767 pch_status = PCH_NOT_READY;
1768 cc_path = CC_COMMON_ARGS[0];
1769 verbose(2,
"MJIT: CC defaults to %s", cc_path);
1770 cc_common_args =
xmalloc(
sizeof(CC_COMMON_ARGS));
1771 memcpy((
void *)cc_common_args, CC_COMMON_ARGS,
sizeof(CC_COMMON_ARGS));
1772 cc_added_args = split_flags(opts->debug_flags);
1773 xfree(opts->debug_flags);
1776 for (
size_t i = 0, j = 0; i <
sizeof(CC_COMMON_ARGS) /
sizeof(
char *); i++) {
1777 if (CC_COMMON_ARGS[i] && strncmp(
"-save-temps", CC_COMMON_ARGS[i], strlen(
"-save-temps")) == 0)
1779 cc_common_args[j] = CC_COMMON_ARGS[i];
1784 tmp_dir = system_tmpdir();
1785 verbose(2,
"MJIT: tmp_dir is %s", tmp_dir);
1787 if (!init_header_filename()) {
1788 mjit_enabled =
false;
1789 verbose(1,
"Failure in MJIT header file name initialization\n");
1792 pch_owner_pid = getpid();
1799 if (!mjit_opts.pause) {
1811 stop_worker_p =
true;
1812 if (current_cc_unit != NULL) {
1813 mjit_wait(current_cc_unit);
1815 worker_stopped =
true;
1820mjit_pause(
bool wait_p)
1822 if (!mjit_enabled) {
1825 if (worker_stopped) {
1831 while (current_cc_unit != NULL) {
1832 mjit_wait(current_cc_unit);
1844 if (!mjit_enabled) {
1847 if (!worker_stopped) {
1852 if (pch_status == PCH_NOT_READY) {
1855 mjit_opts.custom =
true;
1856 pch_status = PCH_SUCCESS;
1864 if (!start_worker()) {
1886mjit_child_after_fork(
void)
1902mjit_finish(
bool close_handle_p)
1908 verbose(2,
"Stopping worker thread");
1913 if (!mjit_opts.save_temps && getpid() == pch_owner_pid && pch_status == PCH_SUCCESS && !mjit_opts.custom)
1914 remove_file(pch_file);
1916 xfree(header_file); header_file = NULL;
1917 xfree((
void *)cc_common_args); cc_common_args = NULL;
1918 for (
char **flag = cc_added_args; *flag != NULL; flag++)
1920 xfree((
void *)cc_added_args); cc_added_args = NULL;
1921 xfree(tmp_dir); tmp_dir = NULL;
1922 xfree(pch_file); pch_file = NULL;
1924 mjit_call_p =
false;
1925 free_list(&unit_queue, close_handle_p);
1926 free_list(&active_units, close_handle_p);
1927 free_list(&compact_units, close_handle_p);
1928 free_list(&stale_units, close_handle_p);
1930 mjit_enabled =
false;
1931 verbose(1,
"Successful MJIT finish");
1943 RUBY_MARK_ENTER(
"mjit");
1946 rb_gc_mark(rb_cMJITCompiler);
1947 rb_gc_mark(rb_cMJITIseqPtr);
1948 rb_gc_mark(rb_cMJITICPtr);
1949 rb_gc_mark(rb_mMJITHooks);
1953 ccan_list_for_each(&active_units.head, unit, unode) {
1954 rb_gc_mark((
VALUE)unit->iseq);
1957 RUBY_MARK_LEAVE(
"mjit");
1965 if (body->mjit_unit && (cc_entries = body->mjit_unit->cc_entries) != NULL) {
1967 for (
unsigned int i = 0; i < body->mjit_unit->cc_entries_size; i++) {
1969 if (cc != NULL && vm_cc_markable(cc)) {
1971 rb_gc_mark((
VALUE)cc);
1972 rb_gc_mark((
VALUE)vm_cc_cme(cc));
1980mjit_compile(
FILE *f,
const rb_iseq_t *iseq,
const char *funcname,
int id)
1982 bool original_call_p = mjit_call_p;
1983 mjit_call_p =
false;
1992 mjit_call_p = original_call_p;
1996#include "mjit.rbinc"
void(* rb_postponed_job_func_t)(void *arg)
Type of postponed jobs.
uint32_t rb_event_flag_t
Represents event(s).
#define NUM2ULONG
Old name of RB_NUM2ULONG.
#define xfree
Old name of ruby_xfree.
#define ID2SYM
Old name of RB_ID2SYM.
#define ULONG2NUM
Old name of RB_ULONG2NUM.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define xmalloc
Old name of ruby_xmalloc.
#define ZALLOC_N
Old name of RB_ZALLOC_N.
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define UINT2NUM
Old name of RB_UINT2NUM.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
void rb_thread_wait_for(struct timeval time)
Identical to rb_thread_sleep(), except it takes struct timeval instead.
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
char * ruby_strdup(const char *str)
This is our own version of strdup(3) that uses ruby_xmalloc() instead of system malloc (benefits our ...
#define strdup(s)
Just another name of ruby_strdup.
#define RUBY_API_VERSION_TEENY
Teeny version.
#define RUBY_API_VERSION_MAJOR
Major version.
#define RUBY_API_VERSION_MINOR
Minor version.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
VALUE type(ANYARGS)
ANYARGS-ed function type.
#define PRI_PIDT_PREFIX
A rb_sprintf() format prefix to be used for a pid_t parameter.
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.