Back to index

plt-scheme  4.2.1
jit.c
Go to the documentation of this file.
00001 /*
00002   MzScheme
00003   Copyright (c) 2006-2009 PLT Scheme Inc.
00004 
00005     This library is free software; you can redistribute it and/or
00006     modify it under the terms of the GNU Library General Public
00007     License as published by the Free Software Foundation; either
00008     version 2 of the License, or (at your option) any later version.
00009 
00010     This library is distributed in the hope that it will be useful,
00011     but WITHOUT ANY WARRANTY; without even the implied warranty of
00012     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013     Library General Public License for more details.
00014 
00015     You should have received a copy of the GNU Library General Public
00016     License along with this library; if not, write to the Free
00017     Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
00018     Boston, MA 02110-1301 USA.
00019 */
00020 
00021 /*
00022   JIT limtations:
00023 
00024   1) See "About short-jump mode" below.
00025 
00026   2) Use jit_patchable_movi_p() when a constant needs to be
00027      visible to the GC.
00028 
00029   3) Immediate operands must be 32-bit values on x86_64, except with
00030      jit_movi, jit_sti, jit_ld, jit_bXi, jit_calli, and jit_finishi.
00031 
00032   4) Function calls are limited to 3 arguments (i.e., jit_prepare()
00033      must never be called with a number greater than 3). This limit
00034      is related to the way the x86_64 port shuffles arguments into
00035      temporary registers.
00036 
00037   5) On x86_64, arguments are delivered in JIT_V2, JIT_V3, and JIT_R2,
00038      in that order. So don't set JIT_R2 before getting the third
00039      argument, etc.
00040 */
00041 
00042 #include "schpriv.h"
00043 #include "schmach.h"
00044 #ifdef MZ_USE_DWARF_LIBUNWIND
00045 # include "unwind/libunwind.h"
00046 #endif
00047 
00048 #ifdef MZ_USE_JIT
00049 
00050 #ifdef __APPLE__
00051 # define _CALL_DARWIN
00052 #endif
00053 
00054 /* Separate JIT_PRECISE_GC lets us test some 3m support in non-3m mode: */
00055 #ifdef MZ_PRECISE_GC
00056 # define JIT_PRECISE_GC
00057 #endif
00058 
00059 /* IMPORTANT! 3m arithmetic checking disabled for the whole file! */
00060 #ifdef MZ_PRECISE_GC
00061 END_XFORM_ARITH;
00062 #endif
00063 
00064 #define JIT_USE_FP_OPS
00065 
00066 #ifdef MZ_USE_JIT_X86_64
00067 # define MZ_USE_JIT_I386
00068 # define JIT_X86_64
00069 #endif
00070 
00071 #include "lightning/lightning.h"
00072 
00073 #ifdef MZ_USE_JIT_X86_64
00074 # define JIT_LOG_WORD_SIZE 3
00075 #else
00076 # define JIT_LOG_WORD_SIZE 2
00077 #endif
00078 #define JIT_WORD_SIZE (1 << JIT_LOG_WORD_SIZE)
00079 #define WORDS_TO_BYTES(x) ((x) << JIT_LOG_WORD_SIZE)
00080 #define MAX_TRY_SHIFT 30
00081 
00082 /* a mzchar is an int: */
00083 #define LOG_MZCHAR_SIZE 2
00084 
00085 #if defined(MZ_USE_JIT_PPC) || defined(MZ_USE_JIT_X86_64)
00086 # define NEED_LONG_JUMPS
00087 #endif
00088 #if defined(MZ_USE_JIT_I386) && !defined(MZ_USE_JIT_X86_64)
00089 # define USE_TINY_JUMPS
00090 #endif
00091 
00092 #define JIT_NOT_RET JIT_R1
00093 #if JIT_NOT_RET == JIT_RET
00094 Fix me! See use.
00095 #endif
00096 
00097 #if 0
00098 static void assert_failure(int where) { printf("JIT assert failed %d\n", where); }
00099 #define JIT_ASSERT(v) if (!(v)) assert_failure(__LINE__);
00100 #else
00101 #define JIT_ASSERT(v) /* */
00102 #endif
00103 
00104 /* Used by vector-set-performance-stats!: */
00105 int scheme_jit_malloced;
00106 
00107 static int skip_checks = 0;
00108 
00109 #define MAX_SHARED_CALL_RANDS 25
00110 static void *shared_tail_code[4][MAX_SHARED_CALL_RANDS];
00111 static void *shared_non_tail_code[4][MAX_SHARED_CALL_RANDS][2];
00112 static void *shared_non_tail_retry_code[2];
00113 static void *shared_non_tail_argc_code[2];
00114 static void *shared_tail_argc_code;
00115 
00116 #define MAX_SHARED_ARITY_CHECK 25
00117 static void *shared_arity_check[MAX_SHARED_ARITY_CHECK][2][2];
00118 
00119 static void *bad_result_arity_code;
00120 static void *unbound_global_code;
00121 static void *quote_syntax_code;
00122 static void *call_original_unary_arith_code;
00123 static void *call_original_binary_arith_code;
00124 static void *call_original_binary_rev_arith_code;
00125 static void *call_original_unary_arith_for_branch_code;
00126 static void *call_original_binary_arith_for_branch_code;
00127 static void *call_original_binary_rev_arith_for_branch_code;
00128 static void *bad_car_code, *bad_cdr_code;
00129 static void *bad_caar_code, *bad_cdar_code, *bad_cadr_code, *bad_cddr_code;
00130 static void *bad_mcar_code, *bad_mcdr_code;
00131 static void *bad_set_mcar_code, *bad_set_mcdr_code;
00132 static void *bad_unbox_code;
00133 static void *bad_vector_length_code;
00134 static void *vector_ref_code, *vector_ref_check_index_code, *vector_set_code, *vector_set_check_index_code;
00135 static void *string_ref_code, *string_ref_check_index_code, *string_set_code, *string_set_check_index_code;
00136 static void *bytes_ref_code, *bytes_ref_check_index_code, *bytes_set_code, *bytes_set_check_index_code;
00137 static void *syntax_e_code;
00138 static void *on_demand_jit_code;
00139 static void *on_demand_jit_arity_code;
00140 static void *get_stack_pointer_code;
00141 static void *stack_cache_pop_code;
00142 static void *struct_pred_code, *struct_pred_multi_code;
00143 static void *struct_pred_branch_code;
00144 static void *struct_get_code, *struct_get_multi_code;
00145 static void *struct_set_code, *struct_set_multi_code;
00146 static void *struct_proc_extract_code;
00147 static void *bad_app_vals_target;
00148 static void *app_values_slow_code, *app_values_multi_slow_code, *app_values_tail_slow_code;
00149 static void *finish_tail_call_code, *finish_tail_call_fixup_code;
00150 
00151 typedef struct {
00152   MZTAG_IF_REQUIRED
00153   GC_CAN_IGNORE jit_state js;
00154   char *limit;
00155   int extra_pushed, max_extra_pushed;
00156   int depth; /* the position of the closure's first value on the stack */
00157   int max_depth;
00158   int *mappings; /* For each element,
00159                   case 0x1 bit:
00160                   . 0 -> case 0x2 bit:
00161                     .        0 -> case rest bits:
00162                     .               0 -> save point
00163                     .               1 -> shift >>2 to get orig pushed count
00164                     .        1 -> shift >>4 to get arity for single orig pushed
00165                     .             shift >>2 to get flags
00166                   . 1 -> shift >>1 to get new (native) pushed */
00167   int num_mappings, mappings_size;
00168   int retained;
00169   int need_set_rs;
00170   void **retain_start;
00171   int local1_busy;
00172   int log_depth;
00173   int self_pos, self_closure_size, self_toplevel_pos;
00174   int self_to_closure_delta, closure_to_args_delta;
00175   int closure_self_on_runstack;
00176   int example_argc;
00177   Scheme_Object **example_argv;
00178   void *self_restart_code;
00179   void *self_nontail_code;
00180   Scheme_Native_Closure *nc; /* for extract_globals and extract_closure_local, only */
00181   Scheme_Closure_Data *self_data;
00182   void *status_at_ptr;
00183   int reg_status;
00184   void *patch_depth;
00185   int rs_virtual_offset;
00186 } mz_jit_state;
00187 
00188 #define mz_RECORD_STATUS(s) (jitter->status_at_ptr = _jit.x.pc, jitter->reg_status = (s))
00189 #define mz_CURRENT_STATUS() ((jitter->status_at_ptr == _jit.x.pc) ? jitter->reg_status : 0)
00190 
00191 #define mz_RS_R0_HAS_RUNSTACK0 0x1
00192 
00193 typedef int (*Native_Check_Arity_Proc)(Scheme_Object *o, int argc, int dummy);
00194 typedef Scheme_Object *(*Native_Get_Arity_Proc)(Scheme_Object *o, int dumm1, int dummy2);
00195 static Native_Check_Arity_Proc check_arity_code;
00196 static Native_Get_Arity_Proc get_arity_code;
00197 
00198 static int generate_non_tail(Scheme_Object *obj, mz_jit_state *jitter, int multi_ok, int need_ends);
00199 static int generate(Scheme_Object *obj, mz_jit_state *jitter, int tail_ok, int multi_ok, int target);
00200 static void *generate_lambda_simple_arity_check(int num_params, int has_rest, int is_method, int permanent);
00201 static void generate_case_lambda(Scheme_Case_Lambda *c, Scheme_Native_Closure_Data *ndata, 
00202                              int is_method);
00203 static void on_demand();
00204 static int generate_non_tail_mark_pos_prefix(mz_jit_state *jitter);
00205 static void generate_non_tail_mark_pos_suffix(mz_jit_state *jitter);
00206 static void *generate_shared_call(int num_rands, mz_jit_state *old_jitter, int multi_ok, int is_tail, 
00207                               int direct_prim, int direct_native, int nontail_self);
00208 
00209 static int generate_two_args(Scheme_Object *rand1, Scheme_Object *rand2, mz_jit_state *jitter, 
00210                              int order_matters, int skipped);
00211 
00212 static int is_simple(Scheme_Object *obj, int depth, int just_markless, mz_jit_state *jitter, int stack_start);
00213 static int lambda_has_been_jitted(Scheme_Native_Closure_Data *ndata);
00214 
00215 #ifdef MZ_PRECISE_GC
00216 static void register_traversers(void);
00217 static void release_native_code(void *fnlized, void *p);
00218 #endif
00219 
00220 int scheme_direct_call_count, scheme_indirect_call_count;
00221 
00222 #ifdef MZ_USE_SINGLE_FLOATS
00223 # define SCHEME_FLOAT_TYPE scheme_float_type
00224 #else
00225 # define SCHEME_FLOAT_TYPE scheme_double_type
00226 #endif
00227 
00228 #define NATIVE_PRESERVES_MARKS 0x1
00229 #define NATIVE_IS_SINGLE_RESULT 0x2
00230 
00231 /* Tracking statistics: */
00232 #if 0
00233 # define NUM_CATEGORIES 23
00234 int jit_sizes[NUM_CATEGORIES];
00235 int jit_counts[NUM_CATEGORIES];
00236 int jit_code_size;
00237 # define START_JIT_DATA() void *__pos = jit_get_ip().ptr; unsigned long __total = 0
00238 # define END_JIT_DATA(where) if (jitter->retain_start) { \
00239                               jit_sizes[where] += __total + ((unsigned long)jit_get_ip().ptr - (unsigned long)__pos); \
00240                               jit_counts[where]++; }
00241 # define PAUSE_JIT_DATA() __total += ((unsigned long)jit_get_ip().ptr - (unsigned long)__pos)
00242 # define RESUME_JIT_DATA() __pos = jit_get_ip().ptr
00243 # define RECORD_CODE_SIZE(s) jit_code_size += s
00244 #else
00245 # define START_JIT_DATA() /* empty */
00246 # define END_JIT_DATA(where) /* empty */
00247 # define PAUSE_JIT_DATA() /* empty */
00248 # define RESUME_JIT_DATA() /* empty */
00249 # define RECORD_CODE_SIZE(s) /* empty */
00250 #endif
00251 
00252 typedef struct {
00253   Scheme_Native_Closure_Data nc;
00254   Scheme_Native_Closure_Data *case_lam;
00255 } Scheme_Native_Closure_Data_Plus_Case;
00256 
00257 /* This structure must be 4 words: */
00258 typedef struct {
00259   void *orig_return_address;
00260   void *stack_frame;
00261   Scheme_Object *cache;
00262   void *filler;
00263 } Stack_Cache_Elem;
00264 
00265 #define STACK_CACHE_SIZE 32
00266 static Stack_Cache_Elem stack_cache_stack[STACK_CACHE_SIZE];
00267 long stack_cache_stack_pos = 0;
00268 
00269 #define IS_NAMED_PRIM(p, nm) (!strcmp(((Scheme_Primitive_Proc *)p)->name, nm))
00270 
00271 #include "codetab.inc"
00272 
00273 static Scheme_Object **fixup_runstack_base;
00274 static int fixup_already_in_place;
00275 
00276 static Scheme_Object *_scheme_tail_apply_from_native_fixup_args(Scheme_Object *rator,
00277                                                                 int argc,
00278                                                                 Scheme_Object **argv)
00279 {
00280   int already = fixup_already_in_place, i;
00281   Scheme_Object **base;
00282 
00283   base = fixup_runstack_base XFORM_OK_MINUS argc XFORM_OK_MINUS already;
00284 
00285   /* Need to shift argc to end of base: */
00286   for (i = 0; i < argc; i++) {
00287     base[already + i] = argv[i];
00288   }
00289 
00290   return _scheme_tail_apply_from_native(rator, argc + already, base);
00291 }
00292 static Scheme_Object *make_global_ref(Scheme_Object *var)
00293 {
00294   GC_CAN_IGNORE Scheme_Object *o;
00295 
00296   o = scheme_alloc_small_object();
00297   o->type = scheme_global_ref_type;
00298   SCHEME_PTR_VAL(o) = var;
00299 
00300   return o;
00301 }
00302 
00303 /*========================================================================*/
00304 /*                              JIT buffer                                */
00305 /*========================================================================*/
00306 
00307 #ifdef SIXTY_FOUR_BIT_INTEGERS
00308 # define JIT_BUFFER_PAD_SIZE 200
00309 #else
00310 # define JIT_BUFFER_PAD_SIZE 100
00311 #endif
00312 
00313 #define _jit (jitter->js)
00314 #define PAST_LIMIT() ((unsigned long)jit_get_ip().ptr > (unsigned long)jitter->limit)
00315 #define CHECK_LIMIT() if (PAST_LIMIT()) return past_limit(jitter);
00316 #if 1
00317 # define past_limit(j) 0
00318 #else
00319 static int past_limit(mz_jit_state *jitter)
00320 {
00321   if (((unsigned long)jit_get_ip().ptr > (unsigned long)jitter->limit + JIT_BUFFER_PAD_SIZE)
00322       || (jitter->retain_start)) {
00323     printf("way past\n");
00324   }
00325   return 0;
00326 }
00327 #endif
00328 
00329 #define JIT_CACHE_SIZE_LIMIT 65536
00330 #define JIT_BUFFER_INIT_SIZE 256
00331 
00332 #define JIT_INIT_MAPPINGS_SIZE 32
00333 
00334 static void *jit_buffer_cache;
00335 static long jit_buffer_cache_size;
00336 static int jit_buffer_cache_registered;
00337 
00338 typedef int (*Generate_Proc)(mz_jit_state *j, void *data);
00339 
00340 static void *get_end_pointer(mz_jit_state *jitter)
00341 {
00342   return jit_get_ip().ptr;
00343 }
00344 
00345 static int mz_retain_it(mz_jit_state *jitter, void *v)
00346 {
00347   if (jitter->retain_start) {
00348     jitter->retain_start[jitter->retained] = v;
00349   }
00350   jitter->retained++;
00351   return jitter->retained;
00352 }
00353 
00354 #ifdef JIT_PRECISE_GC
00355 static void mz_load_retained(mz_jit_state *jitter, int rs, int retptr)
00356 {
00357   void *p;
00358   p = jitter->retain_start + retptr - 1;
00359   (void)jit_patchable_movi_p(rs, p);
00360   jit_ldr_p(rs, rs);
00361 }
00362 #endif
00363 
00364 static void *generate_one(mz_jit_state *old_jitter, 
00365                        Generate_Proc generate,
00366                        void *data,
00367                        int gcable,
00368                        void *save_ptr,
00369                        Scheme_Native_Closure_Data *ndata)
00370 {
00371   mz_jit_state _jitter;
00372   mz_jit_state *jitter = &_jitter;
00373   void *buffer;
00374   int mappings_buffer[JIT_INIT_MAPPINGS_SIZE];
00375   int *mappings = mappings_buffer;
00376   long size = JIT_BUFFER_INIT_SIZE, known_size = 0, size_pre_retained = 0, num_retained = 0, padding;
00377   int mappings_size = JIT_INIT_MAPPINGS_SIZE;
00378   int ok, max_extra_pushed = 0;
00379 #ifdef MZ_PRECISE_GC
00380   Scheme_Object *fnl_obj;
00381 
00382   if (ndata) {
00383     /* When fnl_obj becomes inaccessible, code generated
00384        here can be freed. */
00385     fnl_obj = scheme_box(scheme_false);
00386   } else
00387     fnl_obj = NULL;
00388 #endif
00389 
00390   if (!jit_buffer_cache_registered) {
00391     jit_buffer_cache_registered = 1;
00392     REGISTER_SO(jit_buffer_cache);
00393     REGISTER_SO(stack_cache_stack);
00394 #ifdef MZ_PRECISE_GC
00395     register_traversers();
00396 #endif
00397   }
00398 
00399   while (1) {
00400     memset(jitter, 0, sizeof(_jitter));
00401 #ifdef NEED_LONG_JUMPS
00402     _jitl.long_jumps = 1;
00403 #endif
00404 #ifdef USE_TINY_JUMPS
00405     _jitl.tiny_jumps = 0;
00406 #endif
00407     padding = JIT_BUFFER_PAD_SIZE;
00408     if (known_size) {
00409       size_pre_retained = known_size;
00410       size = size_pre_retained + WORDS_TO_BYTES(num_retained);
00411       padding = 0;
00412       if (gcable) {
00413 #ifdef MZ_PRECISE_GC
00414        buffer = scheme_malloc_code(size);
00415         scheme_jit_malloced += size_pre_retained;
00416 #else
00417        buffer = scheme_malloc_gcable_code(size);
00418 #endif
00419       } else {
00420         buffer = scheme_malloc_code(size);
00421       }
00422       RECORD_CODE_SIZE(size);
00423     } else if (old_jitter) {
00424       /* this is a recursive generate, so use leftover space in
00425         old_jitter's buffer */
00426       buffer = get_end_pointer(old_jitter);
00427       size = ((char *)old_jitter->limit - (char *)buffer);
00428       if (size < JIT_BUFFER_INIT_SIZE) {
00429        old_jitter = NULL;
00430        buffer = NULL;
00431        size = JIT_BUFFER_INIT_SIZE;
00432       } else {
00433        size_pre_retained = size;
00434       }
00435     } else
00436       buffer = NULL;
00437 
00438     if (!buffer) {
00439       if (jit_buffer_cache && (jit_buffer_cache_size >= size)) {
00440        buffer = jit_buffer_cache;
00441        size = jit_buffer_cache_size;
00442        jit_buffer_cache = NULL;
00443       } else {
00444 #ifdef MZ_PRECISE_GC
00445        long minsz;
00446        minsz = GC_malloc_stays_put_threshold();
00447        if (size < minsz)
00448          size = minsz;
00449        buffer = (char *)scheme_malloc_atomic(size);
00450 #else
00451        buffer = scheme_malloc(size);
00452 #endif
00453       }
00454       size_pre_retained = size;
00455     }
00456       
00457     (void)jit_set_ip(buffer).ptr;
00458     jitter->limit = (char *)buffer + size_pre_retained - padding;
00459     if (known_size) {
00460       jitter->retain_start = (void *)jitter->limit;
00461 #ifdef MZ_PRECISE_GC
00462       if (ndata) {
00463        memset(jitter->retain_start, 0, num_retained * sizeof(void*));
00464        ndata->retained = (num_retained ? jitter->retain_start : NULL);
00465        SCHEME_BOX_VAL(fnl_obj) = scheme_make_integer(size_pre_retained);
00466        GC_set_finalizer(fnl_obj, 1, 3,
00467                       release_native_code, buffer,
00468                       NULL, NULL);
00469       }
00470 #endif
00471     } else
00472       jitter->retain_start = NULL;
00473 
00474     jitter->mappings = mappings;
00475     jitter->num_mappings = 0;
00476     jitter->mappings_size = mappings_size;
00477     mappings[0] = 0;
00478     jitter->max_extra_pushed = max_extra_pushed;
00479     jitter->self_pos = 1; /* beyond end of stack */
00480     jitter->self_toplevel_pos = -1;
00481     jitter->status_at_ptr = NULL;
00482 
00483     /* Leave room for retained size on first pass, 
00484        install it if needed) on second pass:*/
00485     if (!known_size || num_retained)
00486       mz_retain_it(jitter, (void *)scheme_make_integer(num_retained));
00487 
00488     ok = generate(jitter, data);
00489 
00490     if (save_ptr) {
00491       mz_retain_it(jitter, save_ptr);
00492     }
00493 #ifdef MZ_PRECISE_GC
00494     if (fnl_obj) {
00495       mz_retain_it(jitter, fnl_obj);
00496     }
00497 #endif
00498 
00499     jitter->limit = (char *)jitter->limit + padding;
00500     if (PAST_LIMIT() || (jitter->retain_start
00501                       && (jitter->retained > num_retained))) {
00502       scheme_console_printf("JIT buffer overflow: %p [%p,%p] (%d)!!\n", 
00503                          jit_get_ip().ptr, 
00504                          buffer, jitter->limit,
00505                          !!jitter->retain_start);
00506       abort();
00507     }
00508 
00509     mappings_size = jitter->mappings_size;
00510     mappings = jitter->mappings;
00511     max_extra_pushed = jitter->max_extra_pushed;
00512 
00513     if (ok) {
00514       /* That was big enough: */
00515       if (known_size) {
00516        /* That was in the permanent area, so return: */
00517        jit_flush_code(buffer, jit_get_ip().ptr);
00518        return buffer;
00519       } else {
00520        /* Allocate permanent area and jit again: */
00521        known_size = ((unsigned long)jit_get_ip().ptr) - (unsigned long)buffer;
00522        if (known_size & (JIT_WORD_SIZE - 1)) {
00523          known_size += (JIT_WORD_SIZE - (known_size & (JIT_WORD_SIZE - 1)));
00524        }
00525        num_retained = jitter->retained;
00526         if (num_retained == 1)  num_retained = 0;
00527        /* Keep this buffer? Don't if it's too big, or if it's
00528           a part of old_jitter, or if there's already a bigger
00529           cache. */
00530        if ((jit_buffer_cache_size < JIT_CACHE_SIZE_LIMIT)
00531            && !old_jitter
00532            && (!jit_buffer_cache
00533               || (jit_buffer_cache_size > size))) {
00534          jit_buffer_cache = buffer;
00535          jit_buffer_cache_size = size;
00536        }
00537       }
00538       /* looping to try again... */
00539     } else {
00540       /* Need more room to try again: */
00541       size = size * 2;
00542       old_jitter = NULL;
00543     }
00544   }
00545 }
00546 
00547 #if 0
00548 # define FOR_LOG(x) x
00549 # define LOG_IT(args) if (jitter->retain_start) { emit_indentation(jitter); printf args; }
00550 static void emit_indentation(mz_jit_state *jitter)
00551 {
00552   int i = jitter->log_depth;
00553   while (i--) {
00554     printf("  ");
00555   }
00556 }
00557 #else
00558 # define FOR_LOG(x) /* empty */
00559 # define LOG_IT(args) /* empty */
00560 #endif
00561 
00562 /*========================================================================*/
00563 /*                               run time                                 */
00564 /*========================================================================*/
00565 
00566 static MZ_INLINE Scheme_Object *do_make_native_closure(Scheme_Native_Closure_Data *code, int size)
00567 {
00568   Scheme_Native_Closure *o;
00569 
00570   o = (Scheme_Native_Closure *)scheme_malloc_tagged(sizeof(Scheme_Native_Closure)
00571                                               + ((size - 1) * sizeof(Scheme_Object *)));
00572   o->so.type = scheme_native_closure_type;
00573   o->code = code;
00574 
00575   return (Scheme_Object *)o;
00576 }
00577 
00578 Scheme_Object *scheme_make_native_closure(Scheme_Native_Closure_Data *code)
00579 {
00580   return do_make_native_closure(code, code->closure_size);
00581 }
00582 
00583 Scheme_Object *scheme_make_native_case_closure(Scheme_Native_Closure_Data *code)
00584 {
00585   return do_make_native_closure(code, -(code->closure_size + 1));
00586 }
00587 
00588 static void call_set_global_bucket(Scheme_Bucket *b, Scheme_Object *val, int set_undef)
00589 {
00590   scheme_set_global_bucket("set!", b, val, set_undef);
00591 }
00592 
00593 static void lexical_binding_wrong_return_arity(int expected, int got, Scheme_Object **argv)
00594 {
00595   scheme_wrong_return_arity(NULL, expected, got, argv, "lexical binding");
00596 }
00597 
00598 static void call_wrong_return_arity(int expected, int got, Scheme_Object **argv)
00599   
00600 {
00601   scheme_wrong_return_arity(NULL, expected, got, argv, NULL);
00602 }
00603 
00604 static void wrong_argument_count(Scheme_Object *proc, int argc, Scheme_Object **argv)
00605 {
00606   scheme_wrong_count((char *)proc, -1, -1, argc, argv);
00607 }
00608 
00609 static void raise_bad_call_with_values(Scheme_Object *f)
00610 {
00611   Scheme_Object *a[1];
00612   a[0] = f;
00613   scheme_wrong_type("call-with-values", "procedure", -1, 1, a);    
00614 }
00615 
00616 static Scheme_Object *call_with_values_from_multiple_result(Scheme_Object *f)
00617 {
00618     Scheme_Thread *p = scheme_current_thread;
00619   if (SAME_OBJ(p->ku.multiple.array, p->values_buffer))
00620     p->values_buffer = NULL;
00621   return _scheme_apply(f, p->ku.multiple.count, p->ku.multiple.array);
00622 }
00623 
00624 static Scheme_Object *call_with_values_from_multiple_result_multi(Scheme_Object *f)
00625 {
00626     Scheme_Thread *p = scheme_current_thread;
00627   if (SAME_OBJ(p->ku.multiple.array, p->values_buffer))
00628     p->values_buffer = NULL;
00629   return _scheme_apply_multi(f, p->ku.multiple.count, p->ku.multiple.array);
00630 }
00631 
00632 static Scheme_Object *tail_call_with_values_from_multiple_result(Scheme_Object *f)
00633 {
00634   Scheme_Thread *p = scheme_current_thread;
00635   int num_rands = p->ku.multiple.count;
00636   
00637   if (num_rands > p->tail_buffer_size) {
00638     /* scheme_tail_apply will allocate */
00639     if (SAME_OBJ(p->ku.multiple.array, p->values_buffer))
00640       p->values_buffer = NULL;
00641   }
00642   return scheme_tail_apply(f, num_rands, p->ku.multiple.array);
00643 }
00644 
00645 static Scheme_Object *clear_runstack(long amt, Scheme_Object *sv)
00646 {
00647   int i;
00648   for (i = 0; i < amt; i++) {
00649     MZ_RUNSTACK[i] = NULL;
00650   }
00651   return sv;
00652 }
00653 
00654 static Scheme_Object *apply_checked_fail(Scheme_Object **args)
00655 {
00656   Scheme_Object *a[3];
00657 
00658   a[0] = args[1];
00659   a[1] = args[3];
00660   a[2] = args[4];
00661 
00662   return _scheme_apply(args[2], 3, a);
00663 }
00664 
00665 /*========================================================================*/
00666 /*                           code-gen utils                               */
00667 /*========================================================================*/
00668 
00669 #define JIT_RUNSTACK JIT_V0
00670 #define JIT_RUNSTACK_BASE JIT_V2
00671 
00672 #ifdef MZ_USE_JIT_PPC
00673 # define JIT_STACK 1
00674 # define JIT_STACK_FRAME 1
00675 #else
00676 # define JIT_STACK JIT_SP
00677 # define JIT_STACK_FRAME JIT_FP
00678 #endif
00679 
00680 #define JIT_UPDATE_THREAD_RSPTR() jit_sti_p(&MZ_RUNSTACK, JIT_RUNSTACK)
00681 #define JIT_UPDATE_THREAD_RSPTR_IF_NEEDED() \
00682     if (jitter->need_set_rs) {   \
00683       JIT_UPDATE_THREAD_RSPTR(); \
00684       jitter->need_set_rs = 0;   \
00685     }
00686 #define JIT_UPDATE_THREAD_RSPTR_FOR_BRANCH_IF_NEEDED() \
00687     if (jitter->need_set_rs) {   \
00688       JIT_UPDATE_THREAD_RSPTR(); \
00689     }
00690 
00691 #if 0
00692 /* Debugging: checking for runstack overflow. A CHECK_RUNSTACK_OVERFLOW() should
00693    be included after each decrement of JIT_RUNTACK. Failure is "reported" by
00694    going into an immediate loop. */
00695 static void *top;
00696 static void *cr_tmp;
00697 # define CHECK_RUNSTACK_OVERFLOW_NOCL() \
00698      jit_sti_l(&cr_tmp, JIT_R0); jit_ldi_l(JIT_R0, &scheme_current_runstack_start); \
00699   top = (_jit.x.pc); (void)jit_bltr_ul(top, JIT_RUNSTACK, JIT_R0); jit_ldi_l(JIT_R0, &cr_tmp)
00700 # define CHECK_RUNSTACK_OVERFLOW() \
00701      CHECK_LIMIT(); CHECK_RUNSTACK_OVERFLOW_NOCL()
00702 #else
00703 # define CHECK_RUNSTACK_OVERFLOW() /* empty */
00704 # define CHECK_RUNSTACK_OVERFLOW_NOCL() /* empty */
00705 #endif
00706 
00707 #if 0
00708 /* Debugging: ... */
00709 static void *top4;
00710 # define VALIDATE_RESULT(reg) top4 = (_jit.x.pc); (void)jit_beqi_ul(top4, reg, 0)
00711 #else
00712 # define VALIDATE_RESULT(reg) /* empty */
00713 #endif
00714 
00715 /* The mz_rs_... family of operations operate on a virtual
00716    JIT_RUNSTACK register to perform a kind of peephole optimization.
00717    The virtual register can be de-sync'd from the actual register, so
00718    that multiple adjustments to the register can be collapsed; this
00719    mostly improves code size, rather than speed. Functions that cause
00720    the register to be de-sync'd are marked as such. Functions that can
00721    accomodate a de-sync'd register on entry are marked as such. All
00722    other fuctions can assume a sync'd regsiter and ensure a sync'd
00723    register. Note that branches and calls normally require a sync'd
00724    register. */
00725 
00726 #if 1
00727 # define mz_rs_dec(n) (jitter->rs_virtual_offset -= (n))
00728 # define mz_rs_inc(n) (jitter->rs_virtual_offset += (n))
00729 # define mz_rs_ldxi(reg, n) jit_ldxi_p(reg, JIT_RUNSTACK, WORDS_TO_BYTES(((n) + jitter->rs_virtual_offset)))
00730 # define mz_rs_ldr(reg) mz_rs_ldxi(reg, 0)
00731 # define mz_rs_stxi(n, reg) jit_stxi_p(WORDS_TO_BYTES(((n) + jitter->rs_virtual_offset)), JIT_RUNSTACK, reg)
00732 # define mz_rs_str(reg) mz_rs_stxi(0, reg)
00733 # define mz_rs_sync() (jitter->rs_virtual_offset \
00734                        ? (jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(jitter->rs_virtual_offset)), \
00735                           jitter->rs_virtual_offset = 0) \
00736                        : 0)
00737 # define mz_rs_sync_0() (jitter->rs_virtual_offset = 0)
00738 #else
00739 # define mz_rs_dec(n) jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(n))
00740 # define mz_rs_inc(n) jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(n))
00741 # define mz_rs_ldr(reg) jit_ldr_p(reg, JIT_RUNSTACK)
00742 # define mz_rs_ldxi(reg, n) jit_ldxi_p(reg, JIT_RUNSTACK, WORDS_TO_BYTES(n))
00743 # define mz_rs_str(reg) jit_str_p(JIT_RUNSTACK, reg)
00744 # define mz_rs_stxi(n, reg) jit_stxi_p(WORDS_TO_BYTES(n), JIT_RUNSTACK, reg)
00745 # define mz_rs_sync() /* empty */
00746 # define mz_rs_sync_0() /* empty */
00747 #endif
00748 
00749 /* No need to sync if a branch just goes to an exception. */
00750 # define mz_rs_sync_fail_branch() /* empty */
00751 
00752 static void new_mapping(mz_jit_state *jitter)
00753 {
00754   jitter->num_mappings++;
00755   if (jitter->num_mappings >= jitter->mappings_size) {
00756     int *a;
00757     a = (int *)scheme_malloc_atomic(jitter->mappings_size * 2 * sizeof(int));
00758     memcpy(a, jitter->mappings, jitter->mappings_size * sizeof(int));
00759     jitter->mappings = a;
00760     jitter->mappings_size *= 2;
00761   }
00762   jitter->mappings[jitter->num_mappings] = 0;
00763 }
00764 
00765 static void mz_pushr_p_it(mz_jit_state *jitter, int reg) 
00766 /* de-sync's rs */
00767 {
00768   int v;
00769 
00770   jitter->extra_pushed++;
00771   if (jitter->extra_pushed > jitter->max_extra_pushed)
00772     jitter->max_extra_pushed = jitter->extra_pushed;
00773 
00774   if (!(jitter->mappings[jitter->num_mappings] & 0x1)
00775       || (jitter->mappings[jitter->num_mappings] < 0)) {
00776     new_mapping(jitter);
00777   }
00778   v = (jitter->mappings[jitter->num_mappings]) >> 1;
00779   v++;
00780   jitter->mappings[jitter->num_mappings] = ((v << 1) | 0x1);
00781   
00782   mz_rs_dec(1);
00783   CHECK_RUNSTACK_OVERFLOW_NOCL();
00784   mz_rs_str(reg);
00785 
00786   jitter->need_set_rs = 1;
00787 }
00788 
00789 static void mz_popr_p_it(mz_jit_state *jitter, int reg) 
00790 /* de-sync's rs */
00791 {
00792   int v;
00793 
00794   jitter->extra_pushed--;
00795 
00796   JIT_ASSERT(jitter->mappings[jitter->num_mappings] & 0x1);
00797   v = jitter->mappings[jitter->num_mappings] >> 1;
00798   v--;
00799   if (!v)
00800     --jitter->num_mappings;
00801   else
00802     jitter->mappings[jitter->num_mappings] = ((v << 1) | 0x1);
00803 
00804   mz_rs_ldr(reg);
00805   mz_rs_inc(1);
00806 
00807   jitter->need_set_rs = 1;
00808 }
00809 
00810 static void mz_runstack_skipped(mz_jit_state *jitter, int n) 
00811 {
00812   int v;
00813 
00814   if (!(jitter->mappings[jitter->num_mappings] & 0x1)
00815       || (jitter->mappings[jitter->num_mappings] > 0)) {
00816     new_mapping(jitter);
00817   }
00818   v = (jitter->mappings[jitter->num_mappings]) >> 1;
00819   JIT_ASSERT(v <= 0);
00820   v -= n;
00821   jitter->mappings[jitter->num_mappings] = ((v << 1) | 0x1);
00822   jitter->self_pos += n;
00823 }
00824 
00825 static void mz_runstack_unskipped(mz_jit_state *jitter, int n) 
00826 {
00827   int v;
00828 
00829   JIT_ASSERT(jitter->mappings[jitter->num_mappings] & 0x1);
00830   v = (jitter->mappings[jitter->num_mappings]) >> 1;
00831   JIT_ASSERT(v + n <= 0);
00832   v += n;
00833   if (!v)
00834     --jitter->num_mappings;
00835   else
00836     jitter->mappings[jitter->num_mappings] = ((v << 1) | 0x1);
00837   jitter->self_pos -= n;
00838 }
00839 
00840 static void mz_runstack_pushed(mz_jit_state *jitter, int n)
00841 {
00842   jitter->depth += n;
00843   if (jitter->depth > jitter->max_depth)
00844     jitter->max_depth = jitter->depth;
00845   jitter->self_pos += n;
00846   if (!jitter->mappings[jitter->num_mappings]
00847       || (jitter->mappings[jitter->num_mappings] & 0x3)) {
00848     new_mapping(jitter);
00849   }
00850   jitter->mappings[jitter->num_mappings] += (n << 2);
00851   jitter->need_set_rs = 1;
00852 }
00853 
00854 static void mz_runstack_closure_pushed(mz_jit_state *jitter, int a, int flags)
00855 {
00856   jitter->depth += 1;
00857   if (jitter->depth > jitter->max_depth)
00858     jitter->max_depth = jitter->depth;
00859   jitter->self_pos += 1;
00860   new_mapping(jitter);
00861   jitter->mappings[jitter->num_mappings] = (a << 4) | (flags << 2) | 0x2;
00862   jitter->need_set_rs = 1;
00863   /* closures are never popped; they go away due to returns or tail calls */
00864 }
00865 
00866 static void mz_runstack_popped(mz_jit_state *jitter, int n)
00867 {
00868   int v;
00869   jitter->depth -= n;
00870   jitter->self_pos -= n;
00871 
00872   v = jitter->mappings[jitter->num_mappings];
00873   JIT_ASSERT(!(v & 0x1));
00874   /* non-procedure slot */
00875   v = v >> 2;
00876   JIT_ASSERT(v >= n);
00877   v -= n;
00878   if (!v)
00879     --jitter->num_mappings;
00880   else
00881     jitter->mappings[jitter->num_mappings] = (v << 2);
00882   jitter->need_set_rs = 1;
00883 }
00884 
00885 static int mz_try_runstack_pop(mz_jit_state *jitter, int n)
00886 {
00887   if (jitter->mappings[jitter->num_mappings] & 0x3)
00888     return 0;
00889   if ((jitter->mappings[jitter->num_mappings] >> 2) < n)
00890     return 0;
00891   mz_runstack_popped(jitter, n);
00892   return 1;
00893 }
00894 
00895 static void mz_runstack_saved(mz_jit_state *jitter)
00896 {
00897   new_mapping(jitter);
00898   /* 0 slot means "saved here" */
00899 }
00900 
00901 static int mz_runstack_restored(mz_jit_state *jitter)
00902 {
00903   /* pop down to 0 slot */
00904   int amt = 0, c;
00905   while ((c = jitter->mappings[jitter->num_mappings])) {
00906     if (c & 0x1) {
00907       /* native push */
00908       c >>= 1;
00909       if (c > 0)
00910        amt += c;
00911     } else if (c & 0x2) {
00912       /* single procedure */
00913       amt++;
00914       jitter->self_pos--;
00915     } else {
00916       /* pushed N */
00917       c = (c >> 2);
00918       amt += c;
00919       jitter->self_pos -= c;
00920     }
00921     --jitter->num_mappings;
00922   }
00923   --jitter->num_mappings;
00924   if (amt)
00925     jitter->need_set_rs = 1;
00926   jitter->depth -= amt;
00927   return amt;
00928 }
00929 
00930 static int mz_remap_it(mz_jit_state *jitter, int i)
00931 {
00932   int j = i, p = jitter->num_mappings, c;
00933   while (p && (j >= 0)) {
00934     c = jitter->mappings[p];
00935     if (c & 0x1) {
00936       /* native push */
00937       c >>= 1;
00938       i += c;
00939       if (c < 0)
00940        j += c;
00941     } else if (c & 0x2) {
00942       /* single procedure */
00943       j--;
00944     } else {
00945       /* pushed N */
00946       j -= (c >> 2);
00947     }
00948     --p;
00949   }
00950   return i;
00951 }
00952 
00953 static int mz_is_closure(mz_jit_state *jitter, int i, int arity, int *_flags)
00954 {
00955   int j = i, p = jitter->num_mappings, c;
00956   while (p && (j >= 0)) {
00957     c = jitter->mappings[p];
00958     if (c & 0x1) {
00959       /* native push */
00960       c >>= 1;
00961       if (c < 0)
00962        j += c;
00963     } else if (c & 0x2) {
00964       /* procedure */
00965       if (!j) {
00966         /* the one we're looking for */
00967         if ((arity == (c >> 4)) || (arity == -1)) {
00968           *_flags = (c >> 2) & 0x3;
00969           return 1;
00970         }
00971       }
00972       j--;
00973     } else {
00974       /* pushed N */
00975       j -= (c >> 2);
00976     }
00977     --p;
00978   }
00979   return 0;
00980 }
00981 
00982 static int stack_safety(mz_jit_state *jitter, int cnt, int offset)
00983 /* de-sync'd rs ok */
00984 {
00985   /* To preserve space safety, we must initialize any stack room
00986      that we make, so that whatever happens to be there isn't
00987      traversed in case of a GC. the value of JIT_RUNSTACK is
00988      handy to use as a "clear" value. */
00989   int i;
00990   for (i = 0; i < cnt; i++) {
00991     mz_rs_stxi(i+offset, JIT_RUNSTACK);
00992     CHECK_LIMIT();
00993   }
00994   return 1;
00995 }
00996 
00997 /* de-sync's rs: */
00998 #define mz_pushr_p(x) mz_pushr_p_it(jitter, x)
00999 #define mz_popr_p(x) mz_popr_p_it(jitter, x)
01000 
01001 #if 0
01002 /* Debugging: at each _finish(), double-check that the runstack register has been
01003    copied into scheme_current_runstack. This code assumes that mz_finishr() is not
01004    used with JIT_R0.  Failure is "reported" by going into an immediate loop, but
01005    check_location is set to the source line number to help indicate where the
01006    problem originated. */
01007 static void *top;
01008 int check_location;
01009 # define CONFIRM_RUNSTACK() (jit_movi_l(JIT_R0, __LINE__), jit_sti_l(&check_location, JIT_R0), \
01010                              jit_ldi_p(JIT_R0, &MZ_RUNSTACK), top = (_jit.x.pc), jit_bner_p(top, JIT_RUNSTACK, JIT_R0))
01011 #else
01012 # define CONFIRM_RUNSTACK() 0
01013 #endif
01014 
01015 #define mz_prepare(x) jit_prepare(x)
01016 #define mz_finish(x) ((void)CONFIRM_RUNSTACK(), jit_finish(x))
01017 #define mz_finishr(x) ((void)CONFIRM_RUNSTACK(), jit_finishr(x))
01018 
01019 #define mz_nonrs_finish(x) jit_finish(x)
01020 
01021 #define mz_retain(x) mz_retain_it(jitter, x)
01022 #define mz_remap(x) mz_remap_it(jitter, x)
01023 
01024 /* 
01025    mz_prolog() and mz_epilog() bracket an internal "function" using a
01026    lighter-weight ABI that keeps all Rx and Vx registers as-is on
01027    entry and exit. Some of those functions are registered in a special
01028    way with add_symbol() so that the backtrace function can follow the
01029    lightweight ABI to get back to the calling code. The lightweight
01030    ABI does not support nested calls (at least not on all platforms;
01031    see LOCAL2 below).
01032 
01033    LOCAL2 and LOCAL3 are available for temporary storage on the C
01034    stack using mz_get_local() and mz_set_local() under certain
01035    circumstances:
01036 
01037    * They can only be used within a function (normally corresponding
01038      to a Scheme lambda) where mz_push_locals() has been called after
01039      jit_prolog(), and where mz_pop_locals() is called before
01040      jit_ret().
01041 
01042    * On some platforms, LOCAL2 and LOCAL3 are the same.
01043 
01044    * On some platforms, a lightweight function created with
01045      mz_prolog() and mz_epilog() uses LOCAL2 to save the return
01046      address. On those platforms, though, LOCAL3 is dufferent from
01047      LOCAL2. So, LOCAL3 can always be used for temporary storage in
01048      such functions (assuming that they're called from a function that
01049      pushes locals, and that nothing else is using LOCAL2).
01050 */
01051 
01052 #ifdef MZ_USE_JIT_PPC
01053 /* JIT_LOCAL1, JIT_LOCAL2, and JIT_LOCAL3 are offsets in the stack frame. */
01054 # define JIT_LOCAL1 56
01055 # define JIT_LOCAL2 60
01056 # define JIT_LOCAL3 64
01057 # define mz_set_local_p(x, l) jit_stxi_p(l, 1, x)
01058 # define mz_get_local_p(x, l) jit_ldxi_p(x, 1, l)
01059 # define mz_patch_branch_at(a, v) (_jitl.long_jumps ? (void)jit_patch_movei(a-4, a-3, v) : (void)jit_patch_branch(a-1, v))
01060 # define mz_patch_ucbranch_at(a, v) (_jitl.long_jumps ? (void)jit_patch_movei(a-4, a-3, v) : (void)jit_patch_ucbranch(a-1, v))
01061 # define mz_prolog(x) (MFLRr(x), mz_set_local_p(x, JIT_LOCAL2))
01062 # define mz_epilog(x) (mz_get_local_p(x, JIT_LOCAL2), jit_jmpr(x))
01063 # define mz_epilog_without_jmp() /* empty */
01064 # define jit_shuffle_saved_regs() /* empty */
01065 # define jit_unshuffle_saved_regs() /* empty */
01066 # define mz_push_locals() /* empty */
01067 # define mz_pop_locals() /* empty */
01068 static void _jit_prolog_again(mz_jit_state *jitter, int n, int ret_addr_reg)
01069 {
01070   /* This must be consistent with _jit_prolog in many ways: */
01071   int frame_size;
01072   int ofs;
01073   int first_saved_reg = JIT_AUX - n;
01074   int num_saved_regs = 32 - first_saved_reg;
01075 
01076   frame_size = 24 + 32 + 12 + num_saved_regs * 4;       /* r27..r31 + args             */
01077   frame_size += 15;                /* the stack must be quad-word     */
01078   frame_size &= ~15;               /* aligned                     */
01079 
01080   STWUrm(1, -frame_size, 1);              /* stwu  r1, -x(r1)            */
01081 
01082   /* We actually only need to save V0-V2, which are at
01083      the end of the saved area: */
01084   first_saved_reg = 29;
01085   num_saved_regs = 3;
01086 
01087   ofs = frame_size - num_saved_regs * 4;
01088   STMWrm(first_saved_reg, ofs, 1);        /* stmw  rI, ofs(r1)           */
01089 #ifdef _CALL_DARWIN
01090   STWrm(ret_addr_reg, frame_size + 8, 1); /* stw   r0, x+8(r1)           */
01091 #else
01092   STWrm(ret_addr_reg, frame_size + 4, 1); /* stw   r0, x+4(r1)           */
01093 #endif
01094 }
01095 #else
01096 # define JIT_LOCAL1 -(JIT_WORD_SIZE * 4)
01097 # define JIT_LOCAL2 -(JIT_WORD_SIZE * 5)
01098 # define mz_set_local_p(x, l) jit_stxi_p((l), JIT_FP, (x))
01099 # define mz_get_local_p(x, l) jit_ldxi_p((x), JIT_FP, (l))
01100 # define mz_patch_branch_at(a, v) jit_patch_branch_at(a, v)
01101 # define mz_patch_ucbranch_at(a, v) jit_patch_ucbranch_at(a, v)
01102 # ifdef _CALL_DARWIN
01103 #  define X86_ALIGN_STACK
01104 #  define STACK_ALIGN_WORDS 3
01105 # endif
01106 # ifdef JIT_X86_64
01107 #  define X86_ALIGN_STACK
01108 #  define STACK_ALIGN_WORDS 1
01109 # endif
01110 # ifdef X86_ALIGN_STACK
01111    /* Maintain 4-byte stack alignment. */
01112 #  define mz_prolog(x) (ADDQiBr(-(STACK_ALIGN_WORDS * JIT_WORD_SIZE), JIT_SP))
01113 #  define mz_epilog_without_jmp() ADDQiBr((STACK_ALIGN_WORDS + 1) * JIT_WORD_SIZE, JIT_SP)
01114 #  define mz_epilog(x) (ADDQiBr(STACK_ALIGN_WORDS * JIT_WORD_SIZE, JIT_SP), RET_())
01115 #  define LOCAL_FRAME_SIZE 3
01116 #  define JIT_LOCAL3 -(JIT_WORD_SIZE * 6)
01117 # else
01118 #  define mz_prolog(x) /* empty */
01119 #  define mz_epilog(x) RET_()
01120 #  define mz_epilog_without_jmp() ADDQir(JIT_WORD_SIZE, JIT_SP)
01121 #  define LOCAL_FRAME_SIZE 2
01122 #  define JIT_LOCAL3 JIT_LOCAL2
01123 # endif
01124 # define mz_push_locals() SUBQir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP)
01125 # define mz_pop_locals() ADDQir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP)
01126 #define _jit_prolog_again(jitter, n, ret_addr_reg) (PUSHQr(ret_addr_reg), jit_base_prolog())
01127 # ifdef MZ_USE_JIT_X86_64
01128 #  define jit_shuffle_saved_regs() (MOVQrr(_ESI, _R12), MOVQrr(_EDI, _R13))
01129 #  define jit_unshuffle_saved_regs() (MOVQrr(_R12, _ESI), MOVQrr(_R13, _EDI))
01130 # else
01131 #  define jit_shuffle_saved_regs() /* empty */
01132 #  define jit_unshuffle_saved_regs() /* empty */
01133 # endif
01134 #endif
01135 
01136 #define mz_patch_branch(a) mz_patch_branch_at(a, (_jit.x.pc))
01137 #define mz_patch_ucbranch(a) mz_patch_ucbranch_at(a, (_jit.x.pc))
01138 
01139 #ifdef NEED_LONG_JUMPS
01140 # define __START_SHORT_JUMPS__(cond) if (cond) { _jitl.long_jumps = 0; }
01141 # define __END_SHORT_JUMPS__(cond) if (cond) { _jitl.long_jumps = 1; }
01142 #else
01143 # define __START_SHORT_JUMPS__(cond) /* empty */
01144 # define __END_SHORT_JUMPS__(cond) /* empty */
01145 #endif
01146 
01147 #ifdef USE_TINY_JUMPS
01148 /* A tiny jump has to be between -128 and 127 bytes. */
01149 # define __START_TINY_JUMPS__(cond) if (cond) { _jitl.tiny_jumps = 1; }
01150 # define __END_TINY_JUMPS__(cond) if (cond) { _jitl.tiny_jumps = 0; }
01151 # define __START_INNER_TINY__(cond) __END_SHORT_JUMPS__(cond); __START_TINY_JUMPS__(1);
01152 # define __END_INNER_TINY__(cond) __END_TINY_JUMPS__(1); __START_SHORT_JUMPS__(cond); 
01153 #else
01154 # define __START_TINY_JUMPS__(cond) __START_SHORT_JUMPS__(cond)
01155 # define __END_TINY_JUMPS__(cond) __END_SHORT_JUMPS__(cond)
01156 # define __START_INNER_TINY__(cond) /* empty */
01157 # define __END_INNER_TINY__(cond) /* empty */
01158 #endif
01159 
01160 #define __START_TINY_OR_SHORT_JUMPS__(tcond, cond) if (tcond) { __START_TINY_JUMPS__(1); } else { __START_SHORT_JUMPS__(cond); }
01161 #define __END_TINY_OR_SHORT_JUMPS__(tcond, cond) if (tcond) { __END_TINY_JUMPS__(1); } else { __END_SHORT_JUMPS__(cond); }
01162 
01163 /* mz_b..i_p supports 64-bit constants on x86_64: */
01164 #ifdef MZ_USE_JIT_X86_64
01165 # define mz_beqi_p(a, v, i) ((void)jit_patchable_movi_p(JIT_REXTMP, i), jit_beqr_p(a, v, JIT_REXTMP))
01166 # define mz_bnei_p(a, v, i) ((void)jit_patchable_movi_p(JIT_REXTMP, i), jit_bner_p(a, v, JIT_REXTMP))
01167 #else
01168 # define mz_beqi_p(a, v, i) jit_beqi_p(a, v, i)
01169 # define mz_bnei_p(a, v, i) jit_bnei_p(a, v, i)
01170 #endif
01171 
01172 /* 
01173  About short-jump mode:
01174    
01175    In
01176       jit_jmpi(code);
01177    or
01178       jit_blti_i(code, v);
01179    the generated instructions can depend on the relative location
01180    between the instruction address and the actual value. Do not enable
01181    short jumps if the relative offset can change between the initial
01182    sizing pass and the final pass. Of course, also don't enable short
01183    jumps if the jump is potentially long (i.e. more than +/- 2^15
01184    on PowerPC, or more than +/- 2^31 on x86_64). Otherwise, enable
01185    short-jump mode as much as possible.
01186 
01187    Tiny-jump mode is like short-jump mode, but the offset must be
01188    within +/- 2^7. Favor tiny jumps over short jumps when possible.
01189 
01190    All mz_finish() and jit_calli() are implicitly long jumps.
01191 */
01192 
01193 /*========================================================================*/
01194 /*                         inlined allocation                             */
01195 /*========================================================================*/
01196 
01197 #if defined(MZ_PRECISE_GC) && !defined(USE_COMPACT_3M_GC)
01198 # define CAN_INLINE_ALLOC
01199 #endif
01200 
01201 #ifdef CAN_INLINE_ALLOC
01202 extern THREAD_LOCAL unsigned long GC_gen0_alloc_page_ptr;
01203 long GC_initial_word(int sizeb);
01204 void GC_initial_words(char *buffer, int sizeb);
01205 long GC_compute_alloc_size(long sizeb);
01206 long GC_alloc_alignment(void);
01207 
01208 static void *retry_alloc_code;
01209 static void *retry_alloc_code_keep_r0_r1;
01210 static void *retry_alloc_code_keep_fpr1;
01211 
01212 static void *retry_alloc_r1; /* set by prepare_retry_alloc() */
01213 
01214 static int generate_alloc_retry(mz_jit_state *jitter, int i);
01215 
01216 #ifdef JIT_USE_FP_OPS
01217 static double save_fp;
01218 #endif
01219 
01220 static void *prepare_retry_alloc(void *p, void *p2)
01221 {
01222   /* Alocate enough to trigger a new page */
01223   long avail, algn;
01224 
01225   algn = GC_alloc_alignment();
01226   avail = algn - (GC_gen0_alloc_page_ptr & (algn - 1));
01227   
01228   if (!avail)
01229     avail = 1;
01230   else if (avail == algn)
01231     avail = 1;
01232 
01233   if (avail > sizeof(long))
01234     avail -= sizeof(long);
01235 
01236   /* We assume that atomic memory and tagged go to the same nursery: */
01237   scheme_malloc_atomic(avail);
01238 
01239   retry_alloc_r1 = p2;
01240 
01241   return p;
01242 }
01243 
01244 static long read_first_word(void *sp)
01245 {
01246   long foo;
01247   memcpy(&foo, sp, sizeof(long));
01248   return foo;
01249 }
01250 
01251 static long initial_tag_word(Scheme_Type tag, int immut)
01252 {
01253   GC_CAN_IGNORE Scheme_Small_Object sp;
01254   memset(&sp, 0, sizeof(Scheme_Small_Object));
01255   sp.iso.so.type = tag;
01256   if (immut) SCHEME_SET_IMMUTABLE(&sp);
01257   return read_first_word((void *)&sp);
01258 }
01259 
01260 static int inline_alloc(mz_jit_state *jitter, int amt, Scheme_Type ty, int immut,
01261                      int keep_r0_r1, int keep_fpr1, int inline_retry)
01262 /* Puts allocated result at JIT_V1; first word is GC tag.
01263    Uses JIT_R2 as temporary. The allocated memory is "dirty" (i.e., not 0ed).
01264    Save FP0 when FP ops are enabled. */
01265 {
01266   GC_CAN_IGNORE jit_insn *ref, *reffail;
01267   long a_word, sz, algn;
01268 #if defined(MZ_USE_PLACES)
01269   long a_words[2];
01270 #endif
01271 
01272   sz = GC_compute_alloc_size(amt);
01273   algn = GC_alloc_alignment();
01274 
01275   __START_TINY_JUMPS__(1);
01276   reffail = _jit.x.pc;
01277   jit_ldi_p(JIT_V1, &GC_gen0_alloc_page_ptr);
01278   jit_subi_l(JIT_R2, JIT_V1, 1);
01279   jit_andi_l(JIT_R2, JIT_R2, (algn - 1));
01280   ref = jit_blti_l(jit_forward(), JIT_R2, (algn - sz));
01281   CHECK_LIMIT();
01282   __END_TINY_JUMPS__(1);
01283 
01284   /* Failure handling */
01285   if (keep_r0_r1) {
01286     if (inline_retry) {
01287       generate_alloc_retry(jitter, 1);
01288       CHECK_LIMIT();
01289     } else {
01290       (void)jit_calli(retry_alloc_code_keep_r0_r1);
01291     }
01292   } else if (keep_fpr1) {
01293     (void)jit_calli(retry_alloc_code_keep_fpr1);
01294   } else {
01295     (void)jit_calli(retry_alloc_code);
01296   }
01297   __START_TINY_JUMPS__(1);
01298   (void)jit_jmpi(reffail);
01299   __END_SHORT_JUMPS__(1);
01300   
01301   __START_TINY_JUMPS__(1);
01302   mz_patch_branch(ref);
01303   jit_addi_ul(JIT_R2, JIT_V1, sz);
01304   (void)jit_sti_l(&GC_gen0_alloc_page_ptr, JIT_R2);
01305 #if !defined(MZ_USE_PLACES)
01306   a_word = GC_initial_word(amt);
01307   jit_movi_l(JIT_R2, a_word);
01308   jit_str_l(JIT_V1, JIT_R2);
01309 
01310   /*SchemeObject header*/
01311   a_word = initial_tag_word(ty, immut);
01312   jit_movi_l(JIT_R2, a_word);
01313   jit_stxi_l(sizeof(long), JIT_V1, JIT_R2);
01314 #else
01315   GC_initial_words(a_words, amt);
01316   jit_movi_l(JIT_R2, a_words[0]);
01317   jit_str_l(JIT_V1, JIT_R2);
01318 
01319   jit_movi_l(JIT_R2, a_words[1]);
01320   jit_stxi_l(sizeof(long), JIT_V1, JIT_R2);
01321 
01322   /*SchemeObject header*/
01323   a_word = initial_tag_word(ty, immut);
01324   jit_movi_l(JIT_R2, a_word);
01325   jit_stxi_l(sizeof(long)*2, JIT_V1, JIT_R2);
01326 #endif
01327 
01328   CHECK_LIMIT();
01329   __END_TINY_JUMPS__(1);
01330 
01331   return 1;
01332 }
01333 #endif
01334 
01335 #ifdef JIT_USE_FP_OPS
01336 # define INLINE_FP_COMP
01337 # ifdef CAN_INLINE_ALLOC
01338 #  define INLINE_FP_OPS
01339 # endif
01340 #endif
01341 
01342 #if defined(INLINE_FP_OPS) && !defined(CAN_INLINE_ALLOC)
01343 static double double_result;
01344 static void *malloc_double(void)
01345 {
01346   return scheme_make_double(double_result);
01347 }
01348 #endif
01349 
01350 #ifdef MZ_PRECISE_GC
01351 # define cons GC_malloc_pair
01352 #else
01353 # define cons scheme_make_pair
01354 #endif
01355 
01356 #ifdef CAN_INLINE_ALLOC
01357 static void *make_list_code, *make_list_star_code;
01358 #else
01359 static Scheme_Object *make_list(long n)
01360 {
01361   GC_CAN_IGNORE Scheme_Object *l = scheme_null;
01362   GC_CAN_IGNORE Scheme_Object **rs = MZ_RUNSTACK;
01363   
01364   while (n--) {
01365     l = cons(rs[n], l);
01366   }
01367 
01368   return l;
01369 }
01370 static Scheme_Object *make_list_star(long n)
01371 {
01372   GC_CAN_IGNORE Scheme_Object **rs = MZ_RUNSTACK;
01373   GC_CAN_IGNORE Scheme_Object *l = rs[--n];
01374   
01375   while (n--) {
01376     l = cons(rs[n], l);
01377   }
01378 
01379   return l;
01380 }
01381 #endif
01382 
01383 #if !defined(CAN_INLINE_ALLOC)
01384 static Scheme_Object *make_vector(long n)
01385 {
01386   Scheme_Object *vec;
01387   vec = scheme_make_vector(n, NULL);
01388   return vec;
01389 }
01390 static Scheme_Object *make_ivector(long n)
01391 {
01392   Scheme_Object *vec;
01393   vec = make_vector(n);
01394   SCHEME_SET_IMMUTABLE(vec);
01395   return vec;
01396 }
01397 static Scheme_Object *make_one_element_vector(Scheme_Object *a)
01398 {
01399   Scheme_Object *vec;
01400   vec = scheme_make_vector(1, a);
01401   return vec;
01402 }
01403 static Scheme_Object *make_one_element_ivector(Scheme_Object *a)
01404 {
01405   Scheme_Object *vec;
01406   vec = make_one_element_vector(a);
01407   SCHEME_SET_IMMUTABLE(vec);
01408   return vec;
01409 }
01410 static Scheme_Object *make_two_element_vector(Scheme_Object *a, Scheme_Object *b)
01411 {
01412   Scheme_Object *vec;
01413   vec = scheme_make_vector(2, a);
01414   SCHEME_VEC_ELS(vec)[1] = b;
01415   return vec;
01416 }
01417 static Scheme_Object *make_two_element_ivector(Scheme_Object *a, Scheme_Object *b)
01418 {
01419   Scheme_Object *vec;
01420   vec = make_two_element_vector(a, b);
01421   SCHEME_SET_IMMUTABLE(vec);
01422   return vec;
01423 }
01424 #endif
01425 
01426 /*========================================================================*/
01427 /*                         bytecode properties                            */
01428 /*========================================================================*/
01429 
01430 #ifdef NEED_LONG_JUMPS
01431 static int is_short(Scheme_Object *obj, int fuel)
01432 {
01433   Scheme_Type t;
01434 
01435   if (fuel <= 0)
01436     return fuel;
01437 
01438   t = SCHEME_TYPE(obj);
01439 
01440   switch (t) {
01441   case scheme_syntax_type:
01442     {
01443       int t;
01444       t = SCHEME_PINT_VAL(obj); 
01445       if (t == CASE_LAMBDA_EXPD)
01446        return fuel - 1;
01447       else
01448        return 0;
01449     }
01450     break;
01451   case scheme_application_type:
01452     {
01453       Scheme_App_Rec *app = (Scheme_App_Rec *)obj;
01454       int i;
01455 
01456       fuel -= app->num_args;
01457       for (i = app->num_args + 1; i--; ) {
01458        fuel = is_short(app->args[i], fuel);
01459       }
01460       return fuel;
01461     }
01462   case scheme_application2_type:
01463     {
01464       Scheme_App2_Rec *app = (Scheme_App2_Rec *)obj;
01465       fuel -= 2;
01466       fuel = is_short(app->rator, fuel);
01467       return is_short(app->rand, fuel);
01468     }
01469   case scheme_application3_type:
01470     {
01471       Scheme_App3_Rec *app = (Scheme_App3_Rec *)obj;
01472       fuel -= 3;
01473       fuel = is_short(app->rator, fuel);
01474       fuel = is_short(app->rand1, fuel);
01475       return is_short(app->rand2, fuel);
01476     }
01477   case scheme_sequence_type:
01478     {
01479       Scheme_Sequence *seq = (Scheme_Sequence *)obj;
01480       int i;
01481 
01482       fuel -= seq->count;
01483       for (i = seq->count; i--; ) {
01484        fuel = is_short(seq->array[i], fuel);
01485       }
01486       return fuel;
01487     }
01488     break;
01489   case scheme_branch_type:
01490     {
01491       Scheme_Branch_Rec *branch = (Scheme_Branch_Rec *)obj;
01492       fuel -= 3;
01493       fuel = is_short(branch->test, fuel);
01494       fuel = is_short(branch->tbranch, fuel);
01495       return is_short(branch->fbranch, fuel);
01496     }
01497   case scheme_toplevel_type:
01498   case scheme_quote_syntax_type:
01499   case scheme_local_type:
01500   case scheme_local_unbox_type:
01501   case scheme_unclosed_procedure_type:
01502     return fuel - 1;
01503   default:
01504     if (t > _scheme_values_types_)
01505       return fuel - 1;
01506     else
01507       return 0;
01508   }
01509 }
01510 #endif
01511 
01512 static int no_sync_change(Scheme_Object *obj, int fuel)
01513 {
01514   Scheme_Type t;
01515 
01516   if (fuel <= 0)
01517     return fuel;
01518 
01519   t = SCHEME_TYPE(obj);
01520 
01521   switch (t) {
01522   case scheme_application2_type:
01523     {
01524       Scheme_App2_Rec *app = (Scheme_App2_Rec *)obj;
01525       if (SCHEME_PRIMP(app->rator)
01526           && (SCHEME_PRIM_PROC_FLAGS(app->rator) & SCHEME_PRIM_IS_UNARY_INLINED)
01527           && (IS_NAMED_PRIM(app->rator, "car")
01528               || IS_NAMED_PRIM(app->rator, "cdr")
01529               || IS_NAMED_PRIM(app->rator, "cadr")
01530               || IS_NAMED_PRIM(app->rator, "cdar")
01531               || IS_NAMED_PRIM(app->rator, "caar")
01532               || IS_NAMED_PRIM(app->rator, "cddr"))) {
01533         return no_sync_change(app->rand, fuel - 1);
01534       }
01535       return 0;
01536     }
01537     break;
01538   case scheme_sequence_type:
01539     {
01540       Scheme_Sequence *seq = (Scheme_Sequence *)obj;
01541       int i;
01542 
01543       fuel -= seq->count;
01544       for (i = seq->count; i--; ) {
01545        fuel = no_sync_change(seq->array[i], fuel);
01546       }
01547       return fuel;
01548     }
01549     break;
01550   case scheme_branch_type:
01551     {
01552       Scheme_Branch_Rec *branch = (Scheme_Branch_Rec *)obj;
01553       fuel -= 3;
01554       fuel = no_sync_change(branch->test, fuel);
01555       fuel = no_sync_change(branch->tbranch, fuel);
01556       return no_sync_change(branch->fbranch, fuel);
01557     }
01558   case scheme_toplevel_type:
01559   case scheme_local_type:
01560   case scheme_local_unbox_type:
01561     return fuel - 1;
01562   default:
01563     if (t > _scheme_values_types_)
01564       return fuel - 1;
01565     else
01566       return 0;
01567   }
01568 }
01569 
01570 Scheme_Object *extract_global(Scheme_Object *o, Scheme_Native_Closure *nc)
01571 {
01572   /* GLOBAL ASSUMPTION: we assume that globals are the last thing
01573      in the closure; grep for "GLOBAL ASSUMPTION" in fun.c. */
01574   Scheme_Object **globs;
01575 
01576   globs = (Scheme_Object **)nc->vals[nc->code->u2.orig_code->closure_size - 1];
01577   return globs[SCHEME_TOPLEVEL_POS(o)];
01578 }
01579 
01580 Scheme_Object *extract_closure_local(Scheme_Object *obj, mz_jit_state *jitter, int extra_push)
01581 {
01582   int pos;
01583 
01584   pos = SCHEME_LOCAL_POS(obj);
01585   pos -= extra_push;
01586   if (pos >= jitter->self_pos - jitter->self_to_closure_delta) {
01587     pos -= (jitter->self_pos - jitter->self_to_closure_delta);
01588     if (pos < jitter->nc->code->u2.orig_code->closure_size) {
01589       /* in the closure */
01590       return jitter->nc->vals[pos];
01591     } else {
01592       /* maybe an example argument... which is useful when
01593          the enclosing function has been lifted, converting
01594          a closure element into an argument */
01595       pos -= jitter->closure_to_args_delta;
01596       if (pos < jitter->example_argc)
01597         return jitter->example_argv[pos];
01598     }
01599   }
01600 
01601   return NULL;
01602 }
01603 
01604 static int check_val_struct_prim(Scheme_Object *p, int arity)
01605 {
01606   if (p && SCHEME_PRIMP(p)) {
01607     if (arity == 1) {
01608       if (((Scheme_Primitive_Proc *)p)->pp.flags & SCHEME_PRIM_IS_STRUCT_PRED)
01609         return 1;
01610       else if (((Scheme_Primitive_Proc *)p)->pp.flags & SCHEME_PRIM_IS_STRUCT_INDEXED_GETTER)
01611         return 2;
01612     } else if (arity == 2) {
01613       if ((((Scheme_Primitive_Proc *)p)->pp.flags & SCHEME_PRIM_IS_STRUCT_OTHER)
01614           && ((((Scheme_Primitive_Proc *)p)->pp.flags & SCHEME_PRIM_STRUCT_OTHER_TYPE_MASK)
01615               == SCHEME_PRIM_STRUCT_TYPE_INDEXED_SETTER))
01616         return 3;
01617     }
01618   }
01619   return 0;
01620 }
01621 
01622 static int inlineable_struct_prim(Scheme_Object *o, mz_jit_state *jitter, int extra_push, int arity)
01623 {
01624   if (jitter->nc) {
01625     if (SAME_TYPE(SCHEME_TYPE(o), scheme_toplevel_type)) {
01626       Scheme_Object *p;
01627       p = extract_global(o, jitter->nc);
01628       p = ((Scheme_Bucket *)p)->val;
01629       return check_val_struct_prim(p, arity);
01630     } else if (SAME_TYPE(SCHEME_TYPE(o), scheme_local_type)) {
01631       Scheme_Object *p;
01632       p = extract_closure_local(o, jitter, extra_push);
01633       return check_val_struct_prim(p, arity);
01634     }
01635   }
01636   return 0;
01637 }
01638 
01639 static int inlined_unary_prim(Scheme_Object *o, Scheme_Object *_app, mz_jit_state *jitter)
01640 {
01641   if (SCHEME_PRIMP(o)
01642       && (SCHEME_PRIM_PROC_FLAGS(o) & SCHEME_PRIM_IS_UNARY_INLINED))
01643     return 1;
01644 
01645   if (inlineable_struct_prim(o, jitter, 1, 1))
01646     return 1;
01647 
01648   return 0;
01649 }
01650 
01651 static int inlined_binary_prim(Scheme_Object *o, Scheme_Object *_app, mz_jit_state *jitter)
01652 {
01653   return ((SCHEME_PRIMP(o)
01654            && (SCHEME_PRIM_PROC_FLAGS(o) & SCHEME_PRIM_IS_BINARY_INLINED))
01655           || inlineable_struct_prim(o, jitter, 2, 2));
01656 }
01657 
01658 static int inlined_nary_prim(Scheme_Object *o, Scheme_Object *_app)
01659 {
01660   return (SCHEME_PRIMP(o)
01661           && (SCHEME_PRIM_PROC_FLAGS(o) & SCHEME_PRIM_IS_NARY_INLINED)
01662           && (((Scheme_App_Rec *)_app)->num_args >= ((Scheme_Primitive_Proc *)o)->mina)
01663           && (((Scheme_App_Rec *)_app)->num_args <= ((Scheme_Primitive_Proc *)o)->mu.maxa));
01664 }
01665 
01666 static int is_noncm(Scheme_Object *a, mz_jit_state *jitter, int depth, int stack_start)
01667 {
01668   if (SCHEME_PRIMP(a)) {
01669     int opts;
01670     opts = ((Scheme_Prim_Proc_Header *)a)->flags & SCHEME_PRIM_OPT_MASK;
01671     if (opts >= SCHEME_PRIM_OPT_NONCM)
01672       /* Structure-type predicates are handled specially, so don't claim NONCM: */
01673       if (!(((Scheme_Prim_Proc_Header *)a)->flags & SCHEME_PRIM_IS_STRUCT_PRED))
01674         return 1;
01675   }
01676 
01677   if (depth 
01678       && jitter->nc 
01679       && SAME_TYPE(SCHEME_TYPE(a), scheme_toplevel_type)
01680       && (SCHEME_TOPLEVEL_FLAGS(a) & SCHEME_TOPLEVEL_CONST)) {
01681     Scheme_Object *p;
01682     p = extract_global(a, jitter->nc);
01683     p = ((Scheme_Bucket *)p)->val;
01684     if (p && SAME_TYPE(SCHEME_TYPE(p), scheme_native_closure_type)) {
01685       Scheme_Native_Closure_Data *ndata = ((Scheme_Native_Closure *)p)->code;
01686       if (ndata->closure_size >= 0) { /* not case-lambda */
01687         if (lambda_has_been_jitted(ndata)) {
01688           if (SCHEME_NATIVE_CLOSURE_DATA_FLAGS(ndata) & NATIVE_PRESERVES_MARKS)
01689             return 1;
01690         } else {
01691           if (SCHEME_CLOSURE_DATA_FLAGS(ndata->u2.orig_code) & CLOS_PRESERVES_MARKS)
01692             return 1;
01693         }
01694       }
01695     }
01696   }
01697 
01698   if (SAME_TYPE(SCHEME_TYPE(a), scheme_local_type)) {
01699     int pos = SCHEME_LOCAL_POS(a) - stack_start;
01700     if (pos >= 0) {
01701       int flags;
01702       if (mz_is_closure(jitter, pos, -1, &flags)) {
01703         return (flags & NATIVE_PRESERVES_MARKS);
01704       }
01705     }
01706   }
01707 
01708   if (depth && SAME_TYPE(SCHEME_TYPE(a), scheme_closure_type)) {
01709     Scheme_Closure_Data *data;
01710 
01711     data = ((Scheme_Closure *)a)->code;
01712     if (SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_PRESERVES_MARKS)
01713       return 1;
01714   }
01715 
01716   return 0;
01717 }
01718 
01719 #define INIT_SIMPLE_DEPTH 10
01720 
01721 static int is_simple(Scheme_Object *obj, int depth, int just_markless, mz_jit_state *jitter, int stack_start)
01722 {
01723   /* Return 1 if evaluating `obj' doesn't change the runstack or cont-mark stack ---
01724      or, if just_markless is 1, doesn't use the cont-mark stack.
01725      If a form doesn't itself change/use the stack, then check all
01726      expressions in tail position, up to some depth. The conservative
01727      answer is always 0. */
01728   Scheme_Type type;
01729 
01730   type = SCHEME_TYPE(obj);
01731 
01732   switch (type) {
01733   case scheme_syntax_type:
01734     {
01735       int t;
01736       t = SCHEME_PINT_VAL(obj); 
01737       return (t == CASE_LAMBDA_EXPD);
01738     }
01739     break;
01740 
01741   case scheme_branch_type:
01742     if (depth) {
01743       Scheme_Branch_Rec *b = (Scheme_Branch_Rec *)obj;
01744       return (is_simple(b->tbranch, depth - 1, just_markless, jitter, stack_start)
01745              && is_simple(b->fbranch, depth - 1, just_markless, jitter, stack_start));
01746     }
01747     break;
01748     
01749   case scheme_let_value_type:
01750     if (depth) {
01751       return is_simple(((Scheme_Let_Value *)obj)->body, depth - 1, just_markless, jitter, stack_start);
01752     }
01753     break;
01754   case scheme_let_one_type:
01755     if (just_markless && depth) {
01756       return is_simple(((Scheme_Let_One *)obj)->body, depth - 1, just_markless, jitter, stack_start + 1);
01757     }
01758     break;
01759   case scheme_let_void_type:
01760     if (just_markless && depth) {
01761       return is_simple(((Scheme_Let_Void *)obj)->body, depth - 1, just_markless, jitter,
01762                        stack_start + ((Scheme_Let_Void *)obj)->count);
01763     }
01764     break;
01765   case scheme_letrec_type:
01766     if (just_markless && depth) {
01767       return is_simple(((Scheme_Letrec *)obj)->body, depth - 1, just_markless, jitter,
01768                        stack_start + ((Scheme_Letrec *)obj)->count);
01769     }
01770     break;
01771 
01772   case scheme_application_type:
01773     if (inlined_nary_prim(((Scheme_App_Rec *)obj)->args[0], obj))
01774       return 1;
01775     if (just_markless) {
01776       return is_noncm(((Scheme_App_Rec *)obj)->args[0], jitter, depth, 
01777                       stack_start + ((Scheme_App_Rec *)obj)->num_args);
01778     }
01779     break;
01780   case scheme_application2_type:
01781     if (inlined_unary_prim(((Scheme_App2_Rec *)obj)->rator, obj, jitter))
01782       return 1;
01783     else if (just_markless) {
01784       return is_noncm(((Scheme_App2_Rec *)obj)->rator, jitter, depth, stack_start + 1);
01785     }
01786     break;
01787   case scheme_application3_type:
01788     if (inlined_binary_prim(((Scheme_App2_Rec *)obj)->rator, obj, jitter))
01789       return 1;
01790     else if (just_markless) {
01791       return is_noncm(((Scheme_App3_Rec *)obj)->rator, jitter, depth, stack_start + 2);
01792     }
01793     break;
01794     
01795   case scheme_toplevel_type:
01796   case scheme_quote_syntax_type:
01797   case scheme_local_type:
01798   case scheme_local_unbox_type:
01799   case scheme_unclosed_procedure_type:
01800     return 1;
01801     break;
01802   }
01803 
01804   return (type > _scheme_values_types_);
01805 }
01806 
01807 static int is_non_gc(Scheme_Object *obj, int depth)
01808 {
01809   /* Return 1 if evaluating `obj' can't trigger a GC. */
01810   Scheme_Type type;
01811 
01812   type = SCHEME_TYPE(obj);
01813 
01814   switch (type) {
01815   case scheme_syntax_type:
01816     break;
01817 
01818   case scheme_branch_type:
01819     if (depth) {
01820       Scheme_Branch_Rec *b = (Scheme_Branch_Rec *)obj;
01821       return (is_non_gc(b->test, depth - 1)
01822              && is_non_gc(b->tbranch, depth - 1)
01823              && is_non_gc(b->fbranch, depth - 1));
01824     }
01825     break;
01826     
01827   case scheme_let_value_type:
01828     if (depth) {
01829       Scheme_Let_Value *lv = (Scheme_Let_Value *)obj;
01830       if (SCHEME_LET_AUTOBOX(lv))
01831         return 0;
01832       return is_non_gc(lv->body, depth - 1);
01833     }
01834     break;
01835   case scheme_let_one_type:
01836     if (depth) {
01837       return (is_non_gc(((Scheme_Let_One *)obj)->value, depth - 1)
01838               && is_non_gc(((Scheme_Let_One *)obj)->body, depth - 1));
01839     }
01840     break;
01841   case scheme_let_void_type:
01842     if (depth) {
01843       Scheme_Let_Void *lv = (Scheme_Let_Void *)obj;
01844       if (SCHEME_LET_AUTOBOX(lv))
01845         return 0;
01846       return is_non_gc(lv->body, depth - 1);
01847     }
01848     break;
01849   case scheme_letrec_type:
01850     break;
01851 
01852   case scheme_application_type:
01853     break;
01854   case scheme_application2_type:
01855     break;
01856   case scheme_application3_type:
01857     break;
01858 
01859   case scheme_toplevel_type:
01860     break;
01861   case scheme_unclosed_procedure_type:
01862     break;
01863 
01864   case scheme_quote_syntax_type:
01865   case scheme_local_type:
01866   case scheme_local_unbox_type:
01867     return 1;
01868     break;
01869   }
01870 
01871   return (type > _scheme_values_types_);
01872 }
01873 
01874 static int ok_to_move_local(Scheme_Object *obj)
01875 {
01876   if (SAME_TYPE(SCHEME_TYPE(obj), scheme_local_type)
01877       && !(SCHEME_LOCAL_FLAGS(obj) & SCHEME_LOCAL_CLEARING_MASK)) {
01878     return 1;
01879   } else
01880     return 0;
01881 }
01882 
01883 static int is_constant_and_avoids_r1(Scheme_Object *obj)
01884 {
01885   Scheme_Type t = SCHEME_TYPE(obj);
01886 
01887   if (SAME_TYPE(t, scheme_toplevel_type)) {
01888     return ((SCHEME_TOPLEVEL_FLAGS(obj) & SCHEME_TOPLEVEL_CONST)
01889             ? 1
01890             : 0);
01891   } else if (SAME_TYPE(t, scheme_local_type) && ok_to_move_local(obj)) {
01892     return 1;
01893   } else
01894     return (t >= _scheme_compiled_values_types_);
01895 }
01896 
01897 /*========================================================================*/
01898 /*                         application codegen                            */
01899 /*========================================================================*/
01900 
01901 static jit_insn *generate_proc_struct_retry(mz_jit_state *jitter, int num_rands, GC_CAN_IGNORE jit_insn *refagain)
01902 {
01903   GC_CAN_IGNORE jit_insn *ref2, *refz1, *refz2, *refz3, *refz4, *refz5;
01904 
01905   ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_proc_struct_type);
01906   jit_ldxi_p(JIT_R1, JIT_V1, &((Scheme_Structure *)0x0)->stype);
01907   jit_ldi_p(JIT_R2, &scheme_reduced_procedure_struct);
01908   refz3 = jit_beqr_p(jit_forward(), JIT_R1, JIT_R2);
01909   jit_ldxi_p(JIT_R1, JIT_R1, &((Scheme_Struct_Type *)0x0)->proc_attr);
01910   refz1 = jit_bmci_i(jit_forward(), JIT_R1, 0x1);
01911   CHECK_LIMIT();
01912 
01913   /* Proc is a field in the record */
01914   jit_rshi_ul(JIT_R1, JIT_R1, 1);
01915   jit_lshi_ul(JIT_R1, JIT_R1, JIT_LOG_WORD_SIZE);
01916   jit_addi_p(JIT_R1, JIT_R1, &((Scheme_Structure *)0x0)->slots);
01917   jit_ldxr_p(JIT_R1, JIT_V1, JIT_R1);
01918 
01919   /* JIT_R1 now has the wrapped procedure */
01920   refz4 = jit_bmsi_i(jit_forward(), JIT_R1, 0x1);
01921   jit_ldr_s(JIT_R2, JIT_R1);
01922   refz2 = jit_bnei_i(jit_forward(), JIT_R2, scheme_native_closure_type);
01923   CHECK_LIMIT();
01924 
01925   /* It's a native closure, but we can't just jump to it, in case
01926      the arity is wrong. */
01927   mz_prepare(2);
01928   jit_movi_i(JIT_R0, num_rands);
01929   jit_pusharg_i(JIT_R0); /* argc */
01930   jit_pusharg_p(JIT_R1); /* closure */
01931   (void)mz_finish(scheme_native_arity_check);
01932   CHECK_LIMIT();
01933   jit_retval(JIT_R0);
01934   refz5 = jit_beqi_i(jit_forward(), JIT_R0, 0);
01935   CHECK_LIMIT();
01936 
01937   /* Extract proc again, then loop */
01938   jit_ldxi_p(JIT_R1, JIT_V1, &((Scheme_Structure *)0x0)->stype);
01939   jit_ldxi_p(JIT_R1, JIT_R1, &((Scheme_Struct_Type *)0x0)->proc_attr);
01940   jit_rshi_ul(JIT_R1, JIT_R1, 1);
01941   jit_lshi_ul(JIT_R1, JIT_R1, JIT_LOG_WORD_SIZE);
01942   jit_addi_p(JIT_R1, JIT_R1, &((Scheme_Structure *)0x0)->slots);
01943   jit_ldxr_p(JIT_V1, JIT_V1, JIT_R1);
01944   (void)jit_jmpi(refagain);
01945   CHECK_LIMIT();
01946 
01947   mz_patch_branch(refz1);
01948   mz_patch_branch(refz2);
01949   mz_patch_branch(refz3);
01950   mz_patch_branch(refz4);
01951   mz_patch_branch(refz5);
01952 
01953   return ref2;
01954 }
01955 
01956 static int generate_direct_prim_tail_call(mz_jit_state *jitter, int num_rands)
01957 {
01958   /* JIT_V1 must have the target function pointer.
01959      Also, scheme_current_runstack must be up-to-date...
01960      unless num-rands == 1, in which case JIT_R0 must
01961      have the argument. */
01962   if (num_rands == 1) {
01963     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
01964     CHECK_RUNSTACK_OVERFLOW();
01965     jit_str_p(JIT_RUNSTACK, JIT_R0);
01966     JIT_UPDATE_THREAD_RSPTR();
01967   }
01968   jit_movi_i(JIT_R1, num_rands);
01969   mz_prepare(2); /* a prim takes 3 args, but a NONCM prim ignores the 3rd */
01970   CHECK_LIMIT();
01971   jit_pusharg_p(JIT_RUNSTACK);
01972   jit_pusharg_i(JIT_R1);
01973   mz_finishr(JIT_V1);
01974   CHECK_LIMIT();
01975   /*  Return: */
01976   mz_pop_locals();
01977   jit_ret();
01978 
01979   return 1;
01980 }
01981 
01982 static int generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, int is_inline)
01983 /* Proc is in V1, args are at RUNSTACK.
01984    If num_rands < 0, then argc is in LOCAL2 and arguments are already below RUNSTACK_BASE.
01985    If direct_native == 2, then some arguments are already in place (shallower in the runstack
01986    than the arguments to move). */
01987 {
01988   int i;
01989   GC_CAN_IGNORE jit_insn *refagain, *ref, *ref2, *ref4, *ref5;
01990 
01991   __START_SHORT_JUMPS__(num_rands < 100);
01992 
01993   /* First, try fast direct jump to native code: */
01994   if (!direct_native) {
01995     ref = jit_bmsi_ul(jit_forward(), JIT_V1, 0x1);
01996     jit_ldr_s(JIT_R1, JIT_V1);
01997     ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_native_closure_type);
01998     CHECK_LIMIT();
01999   } else {
02000     ref = ref2 = NULL;
02001   }
02002 
02003   refagain = _jit.x.pc;
02004 
02005   /* Right kind of function. Extract data and check stack depth: */
02006   jit_ldxi_p(JIT_R0, JIT_V1, &((Scheme_Native_Closure *)0x0)->code);
02007   jit_ldxi_i(JIT_R2, JIT_R0, &((Scheme_Native_Closure_Data *)0x0)->max_let_depth);
02008   jit_ldi_p(JIT_R1, &MZ_RUNSTACK_START);
02009   jit_subr_ul(JIT_R1, JIT_RUNSTACK, JIT_R1);
02010   ref4 = jit_bltr_ul(jit_forward(), JIT_R1, JIT_R2);
02011   CHECK_LIMIT();
02012 
02013   /* Fast jump ok (proc will check argc).
02014      At this point, V1 = closure and R0 = code. */
02015 
02016   /* Check for thread swap: */
02017   (void)jit_movi_p(JIT_R1, &scheme_fuel_counter);
02018   jit_ldr_i(JIT_R2, JIT_R1);
02019   ref5 = jit_blei_i(jit_forward(), JIT_R2, 0);
02020 #ifndef FUEL_AUTODECEREMENTS
02021   jit_subi_p(JIT_R2, JIT_R2, 0x1);
02022   jit_str_i(JIT_R1, JIT_R2);
02023 #endif
02024   CHECK_LIMIT();
02025 
02026   /* Copy args to runstack base: */
02027   if (num_rands >= 0) {
02028     /* Fixed argc: */
02029     if (num_rands) {
02030       jit_subi_p(JIT_R2, JIT_RUNSTACK_BASE, WORDS_TO_BYTES(num_rands)); 
02031       CHECK_RUNSTACK_OVERFLOW();
02032       for (i = num_rands; i--; ) {
02033         jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(i));
02034         jit_stxi_p(WORDS_TO_BYTES(i), JIT_R2, JIT_R1);
02035         CHECK_LIMIT();
02036       }
02037       jit_movr_p(JIT_RUNSTACK, JIT_R2);
02038     } else {
02039       jit_movr_p(JIT_RUNSTACK, JIT_RUNSTACK_BASE);
02040     }
02041     if (direct_native > 1) { /* => some_args_already_in_place */
02042       mz_get_local_p(JIT_R1, JIT_LOCAL2);
02043       jit_lshi_l(JIT_R1, JIT_R1, JIT_LOG_WORD_SIZE);
02044       jit_subr_p(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R1);
02045       CHECK_RUNSTACK_OVERFLOW();
02046     }
02047   } else {
02048     /* Variable argc (in LOCAL2):
02049        arguments are already in place. */
02050   }
02051   /* RUNSTACK, RUNSTACK_BASE, V1, and R0 are ready */
02052   
02053   /* Extract function and data: */
02054   jit_movr_p(JIT_R2, JIT_V1);
02055   if (direct_native) {
02056     jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code);
02057   } else {
02058     jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure_Data *)0x0)->arity_code);
02059   }
02060   /* Set up arguments; JIT_RUNSTACK and JIT_RUNSTACK_BASE must also be ready */
02061   jit_movr_p(JIT_R0, JIT_R2);
02062   if (num_rands >= 0) {
02063     jit_movi_i(JIT_R1, num_rands);
02064     if (direct_native > 1) { /* => some_args_already_in_place */
02065       mz_get_local_p(JIT_R2, JIT_LOCAL2);
02066       jit_addr_i(JIT_R1, JIT_R1, JIT_R2);
02067     }
02068   } else {
02069     mz_get_local_p(JIT_R1, JIT_LOCAL2);    
02070   }
02071   jit_movr_p(JIT_R2, JIT_RUNSTACK);
02072   if (need_set_rs) {
02073     /* In case arity check fails, need to update runstack now: */
02074     JIT_UPDATE_THREAD_RSPTR();
02075   }
02076   /* Now jump: */
02077   jit_jmpr(JIT_V1);
02078   CHECK_LIMIT();
02079 
02080   if (!direct_native && !is_inline && (num_rands >= 0)) {
02081     /* Handle simple applicable struct: */
02082     mz_patch_branch(ref2);
02083     ref2 = generate_proc_struct_retry(jitter, num_rands, refagain);
02084     CHECK_LIMIT();
02085   }
02086 
02087   /* The slow way: */
02088   /*  V1 and RUNSTACK must be intact! */
02089   if (!direct_native) {
02090     mz_patch_branch(ref);
02091     mz_patch_branch(ref2);
02092   }
02093   mz_patch_branch(ref4);
02094   mz_patch_branch(ref5);
02095   CHECK_LIMIT();
02096   if (need_set_rs) {
02097     JIT_UPDATE_THREAD_RSPTR();
02098   }
02099   if (direct_native > 1) { /* => some_args_already_in_place */
02100     /* Need to shuffle argument lists. Since we can pass only
02101        three arguments, use static variables for the others. */
02102     jit_sti_p(&fixup_runstack_base, JIT_RUNSTACK_BASE);
02103     mz_get_local_p(JIT_R1, JIT_LOCAL2);
02104     jit_sti_l(&fixup_already_in_place, JIT_R1);
02105   }
02106   if (num_rands >= 0) {
02107     jit_movi_i(JIT_R0, num_rands);
02108   } else {
02109     mz_get_local_p(JIT_R0, JIT_LOCAL2);    
02110   }
02111   /* Since we've overwritten JIT_RUNSTACK, if this is not shared
02112      code, and if this is 3m, then the runstack no longer
02113      has a pointer to the closure for this code. To ensure that
02114      an appropriate return point exists, jump to static code
02115      for the rest. (This is the slow path, anyway.) */
02116   __END_SHORT_JUMPS__(num_rands < 100);
02117   if (direct_native > 1) {
02118     (void)jit_jmpi(finish_tail_call_fixup_code);
02119   } else {
02120     (void)jit_jmpi(finish_tail_call_code);
02121   }
02122   
02123   return 1;
02124 }
02125 
02126 static int generate_finish_tail_call(mz_jit_state *jitter, int direct_native)
02127 {
02128   mz_prepare(3);
02129   CHECK_LIMIT();
02130   jit_pusharg_p(JIT_RUNSTACK);
02131   jit_pusharg_i(JIT_R0);
02132   jit_pusharg_p(JIT_V1);
02133   if (direct_native > 1) { /* => some_args_already_in_place */
02134     (void)mz_finish(_scheme_tail_apply_from_native_fixup_args);
02135   } else {
02136     (void)mz_finish(_scheme_tail_apply_from_native);
02137   }
02138   CHECK_LIMIT();
02139   /* Return: */
02140   mz_pop_locals();
02141   jit_ret();
02142 
02143   return 1;
02144 }
02145 
02146 static int generate_direct_prim_non_tail_call(mz_jit_state *jitter, int num_rands, int multi_ok, int pop_and_jump)
02147 {
02148   /* See generate_prim_non_tail_call for assumptions. */
02149 
02150   if (pop_and_jump) {
02151     mz_prolog(JIT_R1);
02152   }
02153 
02154   if (num_rands == 1) {
02155     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
02156     CHECK_RUNSTACK_OVERFLOW();
02157     jit_str_p(JIT_RUNSTACK, JIT_R0);
02158     JIT_UPDATE_THREAD_RSPTR();
02159   }
02160 
02161   jit_movi_i(JIT_R1, num_rands);
02162   mz_prepare(2); /* a prim takes 3 args, but a NONCM prim ignores the 3rd */
02163   CHECK_LIMIT();
02164   jit_pusharg_p(JIT_RUNSTACK);
02165   jit_pusharg_i(JIT_R1);
02166   mz_finishr(JIT_V1);
02167   CHECK_LIMIT();
02168   jit_retval(JIT_R0);
02169   VALIDATE_RESULT(JIT_R0);
02170   /* No need to check for multi values or tail-call, because
02171      we only use this for noncm primitives. */
02172 
02173   if (num_rands == 1) {
02174     jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
02175     jitter->need_set_rs = 1;
02176   }
02177 
02178   if (pop_and_jump) {
02179     mz_epilog(JIT_V1);
02180   }
02181 
02182   return 1;
02183 }
02184 
02185 static int generate_retry_call(mz_jit_state *jitter, int num_rands, int multi_ok, GC_CAN_IGNORE jit_insn *reftop)
02186   /* If num_rands < 0, original argc is in V1, and we should
02187      pop argc arguments off runstack before pushing more.
02188      This function is called with short jumps enabled. */
02189 {
02190   GC_CAN_IGNORE jit_insn *ref, *ref2, *refloop;
02191 
02192   if (!reftop) {
02193     reftop = shared_non_tail_retry_code[multi_ok ? 1 : 0];
02194   }
02195 
02196   /* Get new argc: */
02197   (void)jit_ldi_p(JIT_R1, &scheme_current_thread);
02198   jit_ldxi_l(JIT_R2, JIT_R1, &((Scheme_Thread *)0x0)->ku.apply.tail_num_rands);
02199   if (num_rands >= 0) {
02200     jit_movi_l(JIT_V1, 0);
02201   }
02202   /* Thread is in R1. New argc is in R2. Old argc to cancel is in V1. */
02203 
02204   /* Enough room on runstack? */
02205   jit_ldi_p(JIT_R0, &MZ_RUNSTACK_START);
02206   jit_subr_ul(JIT_R0, JIT_RUNSTACK, JIT_R0); /* R0 is space left (in bytes) */
02207   jit_subr_l(JIT_R2, JIT_R2, JIT_V1);
02208   jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE);
02209   ref = jit_bltr_ul(jit_forward(), JIT_R0, JIT_R2);
02210   CHECK_LIMIT();
02211 
02212   /* Yes, there's enough room. Adjust the runstack. */
02213   jit_subr_l(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R2);
02214   CHECK_RUNSTACK_OVERFLOW();
02215 
02216   /* Copy arguments to runstack, then jump to reftop. */
02217   jit_ldxi_l(JIT_R2, JIT_R1, &((Scheme_Thread *)0x0)->ku.apply.tail_num_rands);
02218   jit_ldxi_l(JIT_V1, JIT_R1, &((Scheme_Thread *)0x0)->ku.apply.tail_rands);
02219   jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE);
02220   CHECK_LIMIT();
02221   refloop = _jit.x.pc;
02222   ref2 = jit_blei_l(jit_forward(), JIT_R2, 0);
02223   jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE);
02224   jit_ldxr_p(JIT_R0, JIT_V1, JIT_R2);
02225   jit_stxr_p(JIT_R2, JIT_RUNSTACK, JIT_R0);
02226   (void)jit_jmpi(refloop);
02227   CHECK_LIMIT();
02228 
02229   /* R1 is still the thread. 
02230      Put procedure and argc in place, then jump to apply: */
02231   mz_patch_branch(ref2);
02232   jit_ldxi_l(JIT_V1, JIT_R1, &((Scheme_Thread *)0x0)->ku.apply.tail_rator);
02233   jit_ldxi_l(JIT_R0, JIT_R1, &((Scheme_Thread *)0x0)->ku.apply.tail_num_rands);
02234   __END_SHORT_JUMPS__(1);
02235   (void)jit_jmpi(reftop);
02236   __START_SHORT_JUMPS__(1);
02237   
02238   /* Slow path; restore R0 to SCHEME_TAIL_CALL_WAITING */
02239   mz_patch_branch(ref);
02240   jit_movi_l(JIT_R0, SCHEME_TAIL_CALL_WAITING);
02241 
02242   return 1;
02243 }
02244 
02245 static int generate_clear_previous_args(mz_jit_state *jitter, int num_rands)
02246 {
02247   if (num_rands >= 0) {
02248     int i;
02249     for (i = 0; i < num_rands; i++) {
02250       jit_stxi_p(WORDS_TO_BYTES(i), JIT_RUNSTACK, JIT_RUNSTACK);
02251       CHECK_LIMIT();
02252     }
02253   } else {
02254     /* covered by generate_clear_slow_previous_args */
02255   }
02256   return 1;
02257 }
02258 
02259 static int generate_clear_slow_previous_args(mz_jit_state *jitter)
02260 {
02261   CHECK_LIMIT();
02262   mz_prepare(2);
02263   jit_pusharg_p(JIT_R0);
02264   jit_pusharg_l(JIT_V1);
02265   mz_finish(clear_runstack);
02266   jit_retval(JIT_R0);
02267   return 1;
02268 }
02269 
02270 static int generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, 
02271                               int multi_ok, int nontail_self, int pop_and_jump, int is_inlined)
02272 {
02273   /* Non-tail call.
02274      Proc is in V1, args are at RUNSTACK.
02275      If nontail_self, then R0 has proc pointer, and R2 has max_let_depth.
02276      If num_rands < 0, then argc is in R0, and need to pop runstack before returning.
02277      If num_rands == -1, skip prolog. */
02278   GC_CAN_IGNORE jit_insn *ref, *ref2, *ref4, *ref5, *ref6, *ref7, *ref8, *ref9;
02279   GC_CAN_IGNORE jit_insn *ref10, *reftop = NULL, *refagain;
02280 #ifndef FUEL_AUTODECEREMENTS
02281   GC_CAN_IGNORE jit_insn *ref11;
02282 #endif
02283 
02284   __START_SHORT_JUMPS__(1);
02285 
02286   if (pop_and_jump) {
02287     if (num_rands != -1) {
02288       mz_prolog(JIT_R1);
02289     } else {
02290       reftop = _jit.x.pc;
02291     }
02292   }
02293 
02294   /* Check for inlined native type */
02295   if (!direct_native) {
02296     ref = jit_bmsi_ul(jit_forward(), JIT_V1, 0x1);
02297     jit_ldr_s(JIT_R1, JIT_V1);
02298     ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_native_closure_type);
02299     CHECK_LIMIT();
02300   } else {
02301     ref = ref2 = NULL;
02302   }
02303 
02304   refagain = _jit.x.pc;
02305       
02306   /* Before inlined native, check max let depth */
02307   if (!nontail_self) {
02308     jit_ldxi_p(JIT_R2, JIT_V1, &((Scheme_Native_Closure *)0x0)->code);
02309     jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Native_Closure_Data *)0x0)->max_let_depth);
02310   }
02311   jit_ldi_p(JIT_R1, &MZ_RUNSTACK_START);
02312   jit_subr_ul(JIT_R1, JIT_RUNSTACK, JIT_R1);
02313   ref4 = jit_bltr_ul(jit_forward(), JIT_R1, JIT_R2);
02314   CHECK_LIMIT();
02315 
02316   /* Before inlined native, check stack depth: */
02317   (void)jit_movi_p(JIT_R1, &scheme_jit_stack_boundary); /* assumes USE_STACK_BOUNDARY_VAR */
02318   jit_ldr_l(JIT_R1, JIT_R1);
02319   ref9 = jit_bltr_ul(jit_forward(), JIT_STACK, JIT_R1); /* assumes down-growing stack */
02320   CHECK_LIMIT();
02321 
02322 #ifndef FUEL_AUTODECEREMENTS
02323   /* Finally, check for thread swap: */
02324   (void)jit_movi_p(JIT_R1, &scheme_fuel_counter);
02325   jit_ldr_i(JIT_R2, JIT_R1);
02326   ref11 = jit_blei_i(jit_forward(), JIT_R2, 0);
02327   jit_subi_p(JIT_R2, JIT_R2, 0x1);
02328   jit_str_i(JIT_R1, JIT_R2);
02329 #endif
02330 
02331   /* Fast inlined-native jump ok (proc will check argc, if necessary) */
02332   {
02333     jit_insn *refr;
02334     if (num_rands < 0) {
02335       /* We need to save argc to manually pop the
02336          runstack. So move V1 to R2 and move R0 to V1: */
02337       jit_movr_p(JIT_R2, JIT_V1);
02338       jit_movr_p(JIT_V1, JIT_R0);
02339     }
02340     refr = jit_patchable_movi_p(JIT_R1, jit_forward());
02341     jit_shuffle_saved_regs(); /* maybe copies V regsiters to be restored */
02342     _jit_prolog_again(jitter, 3, JIT_R1); /* saves V registers (or copied V registers) */
02343     if (num_rands >= 0) {
02344       if (nontail_self) { jit_movr_p(JIT_R1, JIT_R0); }
02345       jit_movr_p(JIT_R0, JIT_V1); /* closure */
02346       if (!nontail_self) {
02347         /* nontail_self is only enabled when there are no rest args: */
02348         jit_movi_i(JIT_R1, num_rands); /* argc */
02349         jit_movr_p(JIT_R2, JIT_RUNSTACK); /* argv */
02350       }
02351       jit_addi_p(JIT_RUNSTACK_BASE, JIT_RUNSTACK, WORDS_TO_BYTES(num_rands));
02352     } else {
02353       /* R2 is closure, V1 is argc */
02354       jit_lshi_l(JIT_R1, JIT_V1, JIT_LOG_WORD_SIZE);
02355       jit_addr_p(JIT_RUNSTACK_BASE, JIT_RUNSTACK, JIT_R1);
02356       jit_movr_p(JIT_R0, JIT_R2); /* closure */
02357       jit_movr_i(JIT_R1, JIT_V1); /* argc */
02358       jit_movr_p(JIT_R2, JIT_RUNSTACK); /* argv */
02359     }
02360     CHECK_LIMIT();
02361     mz_push_locals();
02362     if (!nontail_self) {
02363       jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
02364       if (direct_native) {
02365         jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code);
02366       } else {
02367         jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code);
02368         if (need_set_rs) {
02369           /* In case arity check fails, need to update runstack now: */
02370           JIT_UPDATE_THREAD_RSPTR();
02371         }
02372       }
02373       jit_jmpr(JIT_V1); /* callee restores (copied) V registers, etc. */
02374     } else {
02375       /* self-call function pointer is in R1 */
02376       jit_jmpr(JIT_R1);
02377     }
02378     jit_patch_movi(refr, (_jit.x.pc));
02379     jit_unshuffle_saved_regs(); /* maybe uncopies V registers */
02380     /* If num_rands < 0, then V1 has argc */
02381   }
02382   CHECK_LIMIT();
02383   jit_retval(JIT_R0);
02384   VALIDATE_RESULT(JIT_R0);
02385 
02386   /* Fast common-case return */
02387   if (pop_and_jump) {
02388     jit_insn *refc;
02389     __START_INNER_TINY__(1);
02390     refc = jit_blei_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
02391     __END_INNER_TINY__(1);
02392     if (num_rands < 0) { 
02393       /* At this point, argc must be in V1 */
02394       jit_lshi_l(JIT_R1, JIT_V1, JIT_LOG_WORD_SIZE);
02395       jit_addr_p(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R1);
02396     }
02397     if (pop_and_jump) {
02398       mz_epilog(JIT_V1);
02399     }
02400     __START_INNER_TINY__(1);
02401     mz_patch_branch(refc);
02402     __END_INNER_TINY__(1);
02403     CHECK_LIMIT();
02404   }
02405 
02406   if (!multi_ok) {
02407     jit_insn *refm;
02408     __END_SHORT_JUMPS__(1);
02409     refm = jit_beqi_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
02410     mz_patch_branch_at(refm, bad_result_arity_code);
02411     __START_SHORT_JUMPS__(1);
02412   }
02413   ref6 = jit_bnei_p(jit_forward(), JIT_R0, SCHEME_TAIL_CALL_WAITING);
02414   generate_clear_previous_args(jitter, num_rands);
02415   CHECK_LIMIT();
02416   if (pop_and_jump) {
02417     /* Expects argc in V1 if num_rands < 0: */
02418     generate_retry_call(jitter, num_rands, multi_ok, reftop);
02419   }
02420   CHECK_LIMIT();
02421   if (need_set_rs) {
02422     JIT_UPDATE_THREAD_RSPTR();
02423   }
02424   if (num_rands < 0) {
02425     generate_clear_slow_previous_args(jitter);
02426     CHECK_LIMIT();
02427   }
02428   mz_prepare(1);
02429   jit_pusharg_p(JIT_R0);
02430   if (multi_ok) {
02431     (void)mz_finish(scheme_force_value_same_mark);
02432   } else {
02433     (void)mz_finish(scheme_force_one_value_same_mark);
02434   }
02435   ref5 = jit_jmpi(jit_forward());
02436   CHECK_LIMIT();
02437 
02438   /* Maybe it's a prim? */
02439   if (!direct_native) {
02440     mz_patch_branch(ref2);
02441     ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_prim_type);
02442     /* It's a prim. Arity check... fast path when exactly equal to min, only: */
02443     jit_ldxi_i(JIT_R2, JIT_V1, &((Scheme_Primitive_Proc *)0x0)->mina);
02444     if (num_rands >= 0) {
02445       ref7 = jit_bnei_i(jit_forward(), JIT_R2, num_rands);
02446     } else {
02447       ref7 = jit_bner_i(jit_forward(), JIT_R2, JIT_R0);
02448     }
02449     /* Fast prim application */
02450     jit_ldxi_p(JIT_R1, JIT_V1, &((Scheme_Primitive_Proc *)0x0)->prim_val);
02451     if (need_set_rs) {
02452       JIT_UPDATE_THREAD_RSPTR();
02453     }
02454     mz_prepare(3);
02455     jit_pusharg_p(JIT_V1);
02456     if (num_rands < 0) { jit_movr_p(JIT_V1, JIT_R0); } /* save argc to manually pop runstack */
02457     jit_pusharg_p(JIT_RUNSTACK);
02458     jit_pusharg_i(JIT_R2);
02459     (void)mz_finishr(JIT_R1);
02460     CHECK_LIMIT();
02461     jit_retval(JIT_R0);
02462     VALIDATE_RESULT(JIT_R0);
02463     if (!multi_ok) {
02464       jit_insn *refm;
02465       __END_SHORT_JUMPS__(1);
02466       refm = jit_beqi_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
02467       mz_patch_branch_at(refm, bad_result_arity_code);
02468       __START_SHORT_JUMPS__(1);
02469     }
02470     ref10 = jit_bnei_p(jit_forward(), JIT_R0, SCHEME_TAIL_CALL_WAITING);
02471     generate_clear_previous_args(jitter, num_rands);
02472     CHECK_LIMIT();
02473     if (pop_and_jump) {
02474       /* Expects argc in V1 if num_rands < 0: */
02475       generate_retry_call(jitter, num_rands, multi_ok, reftop);
02476     }
02477     CHECK_LIMIT();
02478     if (num_rands < 0) {
02479       generate_clear_slow_previous_args(jitter);
02480       CHECK_LIMIT();
02481     }
02482     mz_prepare(1);
02483     jit_pusharg_p(JIT_R0);
02484     if (multi_ok) {
02485       (void)mz_finish(scheme_force_value_same_mark);
02486     } else {
02487       (void)mz_finish(scheme_force_one_value_same_mark);
02488     }
02489     CHECK_LIMIT();
02490     ref8 = jit_jmpi(jit_forward());
02491 
02492     /* Check for simple applicable struct wrapper */
02493     if (!is_inlined && (num_rands >= 0)) {
02494       mz_patch_branch(ref2);
02495       ref2 = generate_proc_struct_retry(jitter, num_rands, refagain);
02496       CHECK_LIMIT();
02497     }
02498   } else {
02499     ref2 = ref7 = ref8 = ref10 = NULL;
02500   }
02501 
02502   /* The slow way: */
02503   if (!direct_native) {
02504     mz_patch_branch(ref);
02505     mz_patch_branch(ref2);
02506     mz_patch_branch(ref7);
02507   }
02508   mz_patch_branch(ref4);
02509   mz_patch_branch(ref9);
02510 #ifndef FUEL_AUTODECEREMENTS
02511   mz_patch_branch(ref11);
02512 #endif
02513   if (need_set_rs) {
02514     JIT_UPDATE_THREAD_RSPTR();
02515   }
02516   if (num_rands >= 0) {
02517     jit_movi_i(JIT_R0, num_rands);
02518   }
02519   mz_prepare(3);
02520   CHECK_LIMIT();
02521   jit_pusharg_p(JIT_RUNSTACK);
02522   jit_pusharg_i(JIT_R0);
02523   jit_pusharg_p(JIT_V1);
02524   if (num_rands < 0) { jit_movr_p(JIT_V1, JIT_R0); } /* save argc to manually pop runstack */
02525   if (multi_ok) {
02526     (void)mz_finish(_scheme_apply_multi_from_native);
02527   } else {
02528     (void)mz_finish(_scheme_apply_from_native);
02529   }
02530   CHECK_LIMIT();
02531   mz_patch_ucbranch(ref5);
02532   if (!direct_native) {
02533     mz_patch_ucbranch(ref8);
02534   }
02535   jit_retval(JIT_R0);
02536   VALIDATE_RESULT(JIT_R0);
02537   mz_patch_branch(ref6);
02538   if (!direct_native) {
02539     mz_patch_branch(ref10);
02540   }
02541   /* Note: same return code is above for faster common-case return */
02542   if (num_rands < 0) { 
02543     /* At this point, argc must be in V1 */
02544     jit_lshi_l(JIT_R1, JIT_V1, JIT_LOG_WORD_SIZE);
02545     jit_addr_p(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R1);
02546   }
02547   if (pop_and_jump) {
02548     mz_epilog(JIT_V1);
02549   }
02550   CHECK_LIMIT();
02551 
02552   __END_SHORT_JUMPS__(1);
02553 
02554   return 1;
02555 }
02556 
02557 static int generate_self_tail_call(Scheme_Object *rator, mz_jit_state *jitter, int num_rands, jit_insn *slow_code,
02558                                    int args_already_in_place)
02559 {
02560   jit_insn *refslow;
02561   int i, jmp_tiny, jmp_short;
02562   int closure_size = jitter->self_closure_size;
02563 
02564   /* Last argument is in R0 */
02565 
02566 #ifdef JIT_PRECISE_GC
02567   closure_size += 1; /* Skip procedure pointer, too */
02568 #endif
02569 
02570   jmp_tiny = num_rands < 5;
02571   jmp_short = num_rands < 100;
02572 
02573   __START_TINY_OR_SHORT_JUMPS__(jmp_tiny, jmp_short);
02574 
02575   /* Check for thread swap: */
02576   (void)jit_movi_p(JIT_R1, &scheme_fuel_counter);
02577   jit_ldr_i(JIT_R2, JIT_R1);
02578   refslow = jit_blei_i(jit_forward(), JIT_R2, 0);
02579 #ifndef FUEL_AUTODECEREMENTS
02580   jit_subi_p(JIT_R2, JIT_R2, 0x1);
02581   jit_str_i(JIT_R1, JIT_R2);
02582 #endif
02583 
02584   __END_TINY_OR_SHORT_JUMPS__(jmp_tiny, jmp_short);
02585 
02586   /* Copy args to runstack after closure data: */
02587   jit_subi_p(JIT_R2, JIT_RUNSTACK_BASE, WORDS_TO_BYTES(num_rands + closure_size + args_already_in_place)); 
02588   if (num_rands) {
02589     jit_stxi_p(WORDS_TO_BYTES(num_rands - 1 + closure_size + args_already_in_place), JIT_R2, JIT_R0);
02590     for (i = num_rands - 1; i--; ) {
02591       jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(i));
02592       jit_stxi_p(WORDS_TO_BYTES(i + closure_size + args_already_in_place), JIT_R2, JIT_R1);
02593       CHECK_LIMIT();
02594     }
02595   }
02596   jit_movr_p(JIT_RUNSTACK, JIT_R2);
02597 
02598   /* Now jump: */
02599   (void)jit_jmpi(jitter->self_restart_code);
02600   CHECK_LIMIT();
02601 
02602   /* Slow path: */
02603   __START_TINY_OR_SHORT_JUMPS__(jmp_tiny, jmp_short);
02604   mz_patch_branch(refslow);
02605   __END_TINY_OR_SHORT_JUMPS__(jmp_tiny, jmp_short);
02606 
02607   if (args_already_in_place) {
02608     jit_movi_l(JIT_R2, args_already_in_place);
02609     mz_set_local_p(JIT_R2, JIT_LOCAL2);
02610   }
02611 
02612   jit_stxi_p(WORDS_TO_BYTES(num_rands - 1), JIT_RUNSTACK, JIT_R0);
02613   generate(rator, jitter, 0, 0, JIT_V1);
02614   CHECK_LIMIT();
02615   mz_rs_sync();
02616 
02617   (void)jit_jmpi(slow_code);
02618 
02619   return 1;
02620 }
02621 
02622 typedef struct {
02623   int num_rands;
02624   mz_jit_state *old_jitter;
02625   int multi_ok;
02626   int is_tail;
02627   int direct_prim, direct_native, nontail_self;
02628 } Generate_Call_Data;
02629 
02630 static void register_sub_func(mz_jit_state *jitter, void *code, Scheme_Object *protocol)
02631 {
02632   void *code_end;
02633 
02634   code_end = jit_get_ip().ptr;
02635   if (jitter->retain_start)
02636     add_symbol((unsigned long)code, (unsigned long)code_end - 1, protocol, 0);
02637 }
02638 
02639 static void register_helper_func(mz_jit_state *jitter, void *code)
02640 {
02641 #ifdef MZ_USE_DWARF_LIBUNWIND
02642   /* Null indicates that there's no function name to report, but the
02643      stack should be unwound manually using the JJIT-generated convention. */
02644   register_sub_func(jitter, code, scheme_null);
02645 #endif  
02646 }
02647 
02648 int do_generate_shared_call(mz_jit_state *jitter, void *_data)
02649 {
02650   Generate_Call_Data *data = (Generate_Call_Data *)_data;
02651   
02652 #ifdef MZ_USE_JIT_PPC
02653   jitter->js.jitl.nbArgs = data->old_jitter->js.jitl.nbArgs;
02654 #endif
02655 
02656   if (data->is_tail) {
02657     int ok;
02658     void *code;
02659 
02660     code = jit_get_ip().ptr;
02661 
02662     if (data->direct_prim)
02663       ok = generate_direct_prim_tail_call(jitter, data->num_rands);
02664     else
02665       ok = generate_tail_call(jitter, data->num_rands, data->direct_native, 1, 0);
02666 
02667     register_helper_func(jitter, code);
02668 
02669     return ok;
02670   } else {
02671     int ok;
02672     void *code;
02673 
02674     code = jit_get_ip().ptr;
02675 
02676     if (data->direct_prim)
02677       ok = generate_direct_prim_non_tail_call(jitter, data->num_rands, data->multi_ok, 1);
02678     else
02679       ok = generate_non_tail_call(jitter, data->num_rands, data->direct_native, 1, data->multi_ok, data->nontail_self, 1, 0);
02680 
02681     register_sub_func(jitter, code, scheme_false);
02682 
02683     return ok;
02684   }
02685 }
02686 
02687 static void *generate_shared_call(int num_rands, mz_jit_state *old_jitter, int multi_ok, int is_tail, 
02688                               int direct_prim, int direct_native, int nontail_self)
02689 {
02690   Generate_Call_Data data;
02691 
02692   data.num_rands = num_rands;
02693   data.old_jitter = old_jitter;
02694   data.multi_ok = multi_ok;
02695   data.is_tail = is_tail;
02696   data.direct_prim = direct_prim;
02697   data.direct_native = direct_native;
02698   data.nontail_self = nontail_self;
02699 
02700   return generate_one(old_jitter, do_generate_shared_call, &data, 0, NULL, NULL);
02701 }
02702 
02703 static void ensure_retry_available(mz_jit_state *jitter, int multi_ok)
02704 {
02705   int mo = multi_ok ? 1 : 0;
02706   if (!shared_non_tail_retry_code[mo]) {
02707     void *code;
02708     code = generate_shared_call(-1, jitter, multi_ok, 0, 0, 0, 0);
02709     shared_non_tail_retry_code[mo] = code;
02710   }
02711 }
02712 
02713 static int is_a_procedure(Scheme_Object *v, mz_jit_state *jitter)
02714 {
02715   Scheme_Type t;
02716 
02717   if (SCHEME_PROCP(v))
02718     return 1;
02719 
02720   t = SCHEME_TYPE(v);
02721   if (SAME_TYPE(t, scheme_closure_type)
02722       || SAME_TYPE(t, scheme_unclosed_procedure_type))
02723     return 1;
02724   else if (SAME_TYPE(t, scheme_syntax_type)) {
02725     return (SCHEME_PINT_VAL(v) == CASE_LAMBDA_EXPD);
02726   } else if (SAME_TYPE(t, scheme_local_type)) {
02727     int flags;
02728     return mz_is_closure(jitter, SCHEME_LOCAL_POS(v), -1, &flags);
02729   } else if (t == scheme_toplevel_type) {
02730     if (SCHEME_TOPLEVEL_FLAGS(v) & SCHEME_TOPLEVEL_CONST) {
02731       if (jitter->nc) {
02732        Scheme_Object *p;
02733         
02734        p = extract_global(v, jitter->nc);
02735        p = ((Scheme_Bucket *)p)->val;
02736        return SAME_TYPE(SCHEME_TYPE(p), scheme_native_closure_type);
02737       }
02738     }
02739   }
02740 
02741   return 0;
02742 }
02743 
02744 static int generate_nontail_self_setup(mz_jit_state *jitter)
02745 {
02746   void *pp, **pd;
02747   pp = jit_patchable_movi_p(JIT_R2, jit_forward());
02748   pd = (void **)scheme_malloc(2 * sizeof(void *));
02749   pd[0] = pp;
02750   pd[1] = jitter->patch_depth;
02751   jitter->patch_depth = pd;
02752   (void)jit_patchable_movi_p(JIT_R0, jitter->self_nontail_code);
02753 #ifdef JIT_PRECISE_GC
02754   if (jitter->closure_self_on_runstack) {
02755     /* Get this closure's pointer from the run stack */
02756     int depth = jitter->depth + jitter->extra_pushed - 1;
02757     jit_ldxi_p(JIT_V1, JIT_RUNSTACK, WORDS_TO_BYTES(depth));
02758   }
02759 #endif
02760   return 0;
02761 }
02762 
02763 static int can_direct_native(Scheme_Object *p, int num_rands, long *extract_case)
02764 {
02765   if (SAME_TYPE(SCHEME_TYPE(p), scheme_native_closure_type)) {
02766     if (((Scheme_Native_Closure *)p)->code->closure_size < 0) {
02767       /* case-lambda */
02768       int cnt, i;
02769       mzshort *arities;
02770 
02771       cnt = ((Scheme_Native_Closure *)p)->code->closure_size;
02772       cnt = -(cnt + 1);
02773       arities = ((Scheme_Native_Closure *)p)->code->u.arities;
02774       for (i = 0; i < cnt; i++) {
02775         if (arities[i] == num_rands) {
02776           *extract_case = (long)&((Scheme_Native_Closure *)0x0)->vals[i];
02777           return 1;
02778         }
02779       }
02780     } else {
02781       /* not a case-lambda... */
02782       if (scheme_native_arity_check(p, num_rands)
02783           /* If it also accepts num_rands + 1, then it has a vararg,
02784              so don't try direct_native. */
02785           && !scheme_native_arity_check(p, num_rands + 1)) {
02786         return 1;
02787       }
02788     }
02789   }
02790 
02791   return 0;
02792 }
02793 
02794 static int generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, 
02795                      mz_jit_state *jitter, int is_tail, int multi_ok, int no_call)
02796 /* de-sync'd ok 
02797    If no_call is 2, then rator is not necssarily evaluated. 
02798    If no_call is 1, then rator is left in V1 and arguments are on runstack. */
02799 {
02800   int i, offset, need_safety = 0;
02801   int direct_prim = 0, need_non_tail = 0, direct_native = 0, direct_self = 0, nontail_self = 0;
02802   int proc_already_in_place = 0;
02803   Scheme_Object *rator, *v, *arg;
02804   int reorder_ok = 0;
02805   int args_already_in_place = 0;
02806   long extract_case = 0; /* when direct_native, non-0 => offset to extract case-lambda case */
02807   START_JIT_DATA();
02808 
02809   rator = (alt_rands ? alt_rands[0] : app->args[0]);
02810 
02811   if (SCHEME_PRIMP(rator)) {
02812     if ((num_rands >= ((Scheme_Primitive_Proc *)rator)->mina)
02813        && ((num_rands <= ((Scheme_Primitive_Proc *)rator)->mu.maxa)
02814            || (((Scheme_Primitive_Proc *)rator)->mina < 0))
02815        && (is_noncm(rator, jitter, 0, 0)
02816             /* It's also ok to directly call `values' if multiple values are ok: */
02817             || (multi_ok && SAME_OBJ(rator, scheme_values_func))))
02818       direct_prim = 1;
02819   } else {
02820     Scheme_Type t;
02821     t = SCHEME_TYPE(rator);
02822     if ((t == scheme_local_type) && ok_to_move_local(rator)) {
02823       /* We can re-order evaluation of the rator. */
02824       reorder_ok = 1;
02825 
02826       /* Call to known native, or even known self? */
02827       {
02828        int pos, flags;
02829        pos = SCHEME_LOCAL_POS(rator) - num_rands;
02830        if (mz_is_closure(jitter, pos, num_rands, &flags)) {
02831          direct_native = 1;
02832          if ((pos == jitter->self_pos)
02833              && (num_rands < MAX_SHARED_CALL_RANDS)) {
02834             if (is_tail)
02835               direct_self = 1;
02836             else if (jitter->self_nontail_code)
02837               nontail_self = 1;
02838          }
02839        }
02840       }
02841     } else if (t == scheme_toplevel_type) {
02842       if (SCHEME_TOPLEVEL_FLAGS(rator) & SCHEME_TOPLEVEL_CONST) {
02843         /* We can re-order evaluation of the rator. */
02844         reorder_ok = 1;
02845 
02846         if (jitter->nc) {
02847           Scheme_Object *p;
02848 
02849           p = extract_global(rator, jitter->nc);
02850           p = ((Scheme_Bucket *)p)->val;
02851           if (can_direct_native(p, num_rands, &extract_case)) {
02852             direct_native = 1;
02853             
02854             if ((SCHEME_TOPLEVEL_POS(rator) == jitter->self_toplevel_pos)
02855                 && (num_rands < MAX_SHARED_CALL_RANDS)) {
02856               if (is_tail)
02857                 direct_self = 1;
02858               else if (jitter->self_nontail_code)
02859                 nontail_self = 1;
02860             }
02861           }
02862         }
02863       } else if (jitter->nc) {
02864         Scheme_Object *p;
02865 
02866         p = extract_global(rator, jitter->nc);
02867         if (((Scheme_Bucket_With_Flags *)p)->flags & GLOB_IS_CONSISTENT) {
02868           if (can_direct_native(((Scheme_Bucket *)p)->val, num_rands, &extract_case))
02869             direct_native = 1;
02870         }
02871       }
02872     } else if (SAME_TYPE(t, scheme_closure_type)) {
02873       Scheme_Closure_Data *data;
02874       data = ((Scheme_Closure *)rator)->code;
02875       if ((data->num_params == num_rands)
02876           && !(SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_HAS_REST)) {
02877         direct_native = 1;
02878 
02879         if (SAME_OBJ(data->u.jit_clone, jitter->self_data)
02880             && (num_rands < MAX_SHARED_CALL_RANDS)) {
02881           if (is_tail)
02882             direct_self = 1;
02883           else if (jitter->self_nontail_code)
02884             nontail_self = 1;
02885         }
02886       }
02887       reorder_ok = 1;
02888     } else if (t > _scheme_values_types_) {
02889       /* We can re-order evaluation of the rator. */
02890       reorder_ok = 1;
02891     }
02892 
02893 #ifdef JIT_PRECISE_GC
02894     if (jitter->closure_self_on_runstack) {
02895       /* We can get this closure's pointer back from the Scheme stack. */
02896       if (nontail_self)
02897         direct_self = 1;
02898     }
02899 #endif
02900 
02901     if (direct_self)
02902       reorder_ok = 0; /* superceded by direct_self */
02903   }
02904 
02905   /* Direct native tail with same number of args as just received? */
02906   if (direct_native && is_tail && num_rands
02907       && (num_rands == jitter->self_data->num_params)
02908       && !(SCHEME_CLOSURE_DATA_FLAGS(jitter->self_data) & CLOS_HAS_REST)) {
02909     /* Check whether the actual arguments refer to Scheme-stack 
02910        locations that will be filled with argument values; that
02911        is, check how many arguments are already in place for
02912        the call. */
02913     mz_runstack_skipped(jitter, num_rands);
02914     for (i = 0; i < num_rands; i++) {
02915       v = (alt_rands ? alt_rands[i+1] : app->args[i+1]);
02916       if (SAME_TYPE(SCHEME_TYPE(v), scheme_local_type)
02917           && !(SCHEME_LOCAL_FLAGS(v) & SCHEME_LOCAL_OTHER_CLEARS)) {
02918         int pos;
02919         pos = mz_remap(SCHEME_LOCAL_POS(v));
02920         if (pos == (jitter->depth + args_already_in_place))
02921           args_already_in_place++;
02922         else
02923           break;
02924       } else
02925         break;
02926     }
02927     mz_runstack_unskipped(jitter, num_rands);
02928     if (args_already_in_place) {
02929       direct_native = 2;
02930       mz_runstack_skipped(jitter, args_already_in_place);
02931       num_rands -= args_already_in_place;
02932     }
02933   }
02934 
02935   if (num_rands) {
02936     if (!direct_prim || (num_rands > 1)) {
02937       mz_rs_dec(num_rands);
02938       need_safety = num_rands;
02939       CHECK_RUNSTACK_OVERFLOW();
02940       mz_runstack_pushed(jitter, num_rands);
02941     } else {
02942       mz_runstack_skipped(jitter, 1);
02943     }
02944   }
02945 
02946   for (i = num_rands + args_already_in_place + 1; i--; ) {
02947     v = (alt_rands ? alt_rands[i] : app->args[i]);
02948     if (!is_simple(v, INIT_SIMPLE_DEPTH, 1, jitter, 0)) {
02949       need_non_tail = 1;
02950       break;
02951     }
02952   }
02953 
02954   if (need_non_tail) {
02955     offset = generate_non_tail_mark_pos_prefix(jitter);
02956     CHECK_LIMIT();
02957   } else
02958     offset = 0;
02959 
02960   if (!direct_prim && !reorder_ok && !direct_self) {
02961     if (need_safety && !is_non_gc(rator, INIT_SIMPLE_DEPTH)) {
02962       stack_safety(jitter, need_safety, offset);
02963       CHECK_LIMIT();
02964       need_safety = 0;
02965     }
02966 
02967     generate_non_tail(rator, jitter, 0, !need_non_tail); /* sync'd after args below */
02968     CHECK_LIMIT();
02969 
02970     if (num_rands) {
02971       /* Save rator where GC can see it */
02972       Scheme_Type t;
02973       t = SCHEME_TYPE((alt_rands 
02974                        ? alt_rands[1+args_already_in_place] 
02975                        : app->args[1+args_already_in_place]));
02976       if ((num_rands == 1) && (SAME_TYPE(scheme_local_type, t)
02977                             || (t >= _scheme_values_types_))) {
02978        /* App of something complex to a local variable. We
02979           can move the proc directly to V1. */
02980        jit_movr_p(JIT_V1, JIT_R0);
02981        proc_already_in_place = 1;
02982       } else {
02983        mz_rs_stxi(num_rands - 1 + offset, JIT_R0);
02984         if (need_safety)
02985           need_safety--;
02986       }
02987     } else {
02988       jit_movr_p(JIT_V1, JIT_R0);
02989     }
02990   }
02991   /* not sync'd...*/
02992 
02993   for (i = 0; i < num_rands; i++) {
02994     PAUSE_JIT_DATA();
02995     arg = (alt_rands 
02996            ? alt_rands[i+1+args_already_in_place] 
02997            : app->args[i+1+args_already_in_place]);
02998     if (need_safety && !is_non_gc(arg, INIT_SIMPLE_DEPTH)) {
02999       stack_safety(jitter, need_safety - i, offset + i);
03000       CHECK_LIMIT();
03001       need_safety = 0;
03002     }
03003     generate_non_tail(arg, jitter, 0, !need_non_tail); /* sync'd below */
03004     RESUME_JIT_DATA();
03005     CHECK_LIMIT();
03006     if ((i == num_rands - 1) && !direct_prim && !reorder_ok && !direct_self && !proc_already_in_place) {
03007       /* Move rator back to register: */
03008       mz_rs_ldxi(JIT_V1, i + offset);
03009     }
03010     if ((!direct_prim || (num_rands > 1))
03011        && (!direct_self || !is_tail || no_call || (i + 1 < num_rands))) {
03012       mz_rs_stxi(i + offset, JIT_R0);
03013     }
03014   }
03015   /* not sync'd... */
03016 
03017   if (need_non_tail) {
03018     /* Uses JIT_R2: */
03019     generate_non_tail_mark_pos_suffix(jitter);
03020     CHECK_LIMIT();
03021   }
03022 
03023   if (direct_prim) {
03024     if (!no_call) {
03025       (void)jit_movi_p(JIT_V1, ((Scheme_Primitive_Proc *)rator)->prim_val);
03026       if (num_rands == 1) {
03027         mz_runstack_unskipped(jitter, 1);
03028       } else {
03029         mz_rs_sync();
03030         JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
03031       }
03032       LOG_IT(("direct: %s\n", ((Scheme_Primitive_Proc *)rator)->name));
03033     }
03034   }
03035 
03036   if (reorder_ok) {
03037     if (no_call < 2) {
03038       generate(rator, jitter, 0, 0, JIT_V1); /* sync'd below, or not */
03039     }
03040     CHECK_LIMIT();
03041   }
03042 
03043   if (!no_call)
03044     mz_rs_sync();
03045 
03046   END_JIT_DATA(20);
03047 
03048   if (direct_prim || direct_native || direct_self || nontail_self)
03049     scheme_direct_call_count++;
03050   else
03051     scheme_indirect_call_count++;
03052 
03053   if (direct_native && extract_case) {
03054     /* extract case from case-lambda */
03055     jit_ldxi_p(JIT_V1, JIT_V1, extract_case);
03056   }
03057 
03058   if (no_call) {
03059     /* leave actual call to inlining code */
03060   } else if (!(direct_self && is_tail)
03061              && (num_rands >= MAX_SHARED_CALL_RANDS)) {
03062     LOG_IT(("<-many args\n"));
03063     if (is_tail) {
03064       if (direct_prim) {
03065         generate_direct_prim_tail_call(jitter, num_rands);
03066       } else {
03067         if (args_already_in_place) {
03068           jit_movi_l(JIT_R2, args_already_in_place);
03069           mz_set_local_p(JIT_R2, JIT_LOCAL2);
03070         }
03071        generate_tail_call(jitter, num_rands, direct_native, jitter->need_set_rs, 1);
03072       }
03073     } else {
03074       if (direct_prim)
03075        generate_direct_prim_non_tail_call(jitter, num_rands, multi_ok, 0);
03076       else {
03077         if (nontail_self) {
03078           generate_nontail_self_setup(jitter);
03079         }
03080        generate_non_tail_call(jitter, num_rands, direct_native, jitter->need_set_rs, multi_ok, nontail_self, 0, 1);
03081       }
03082     }
03083   } else {
03084     /* Jump to code to implement a tail call for num_rands arguments */
03085     void *code;
03086     int dp = (direct_prim ? 1 : (direct_native ? (1 + direct_native + (nontail_self ? 1 : 0)) : 0));
03087     if (is_tail) {
03088       if (!shared_tail_code[dp][num_rands]) {
03089        code = generate_shared_call(num_rands, jitter, multi_ok, is_tail, direct_prim, direct_native, 0);
03090        shared_tail_code[dp][num_rands] = code;
03091       }
03092       code = shared_tail_code[dp][num_rands];
03093       if (direct_self) {
03094         LOG_IT(("<-self\n"));
03095        generate_self_tail_call(rator, jitter, num_rands, code, args_already_in_place);
03096        CHECK_LIMIT();
03097       } else {
03098         LOG_IT(("<-tail\n"));
03099         if (args_already_in_place) {
03100           jit_movi_l(JIT_R2, args_already_in_place);
03101           mz_set_local_p(JIT_R2, JIT_LOCAL2);
03102         }
03103        (void)jit_jmpi(code);
03104       }
03105     } else {
03106       int mo = (multi_ok ? 1 : 0);
03107 
03108       if (!shared_non_tail_code[dp][num_rands][mo]) {
03109         ensure_retry_available(jitter, multi_ok);
03110        code = generate_shared_call(num_rands, jitter, multi_ok, is_tail, direct_prim, direct_native, nontail_self);
03111        shared_non_tail_code[dp][num_rands][mo] = code;
03112       }
03113       LOG_IT(("<-non-tail %d %d %d\n", dp, num_rands, mo));
03114       code = shared_non_tail_code[dp][num_rands][mo];
03115 
03116       if (nontail_self) {
03117         generate_nontail_self_setup(jitter);
03118       }
03119 
03120       (void)jit_calli(code);
03121 
03122       if (direct_prim) {
03123         if (num_rands == 1) {
03124           /* Popped single argument after return of prim: */
03125           jitter->need_set_rs = 1;
03126         } else {
03127           /* Runstack is up-to-date: */
03128           jitter->need_set_rs = 0;
03129         }
03130       } else {
03131         /* Otherwise, we may have called native code, which may have left
03132            the runstack register out of sync with scheme_current_runstack. */
03133         jitter->need_set_rs = 1;
03134       }
03135     }
03136   }
03137 
03138   END_JIT_DATA(need_non_tail ? 22 : 4);
03139     
03140   return is_tail ? 2 : 1;
03141 }
03142 
03143 static jit_insn *generate_arith_slow_path(mz_jit_state *jitter, Scheme_Object *rator, 
03144                                      jit_insn **_ref, jit_insn **_ref4,
03145                                      jit_insn **for_branch, 
03146                                      int orig_args, int reversed, int arith, int use_v, int v)
03147 /* *_ref is place to set for where to jump (for true case, if for_branch) after completing;
03148    *_ref4 is place to set for where to jump for false if for_branch;
03149    result is place to jump to start slow path if fixnum attempt fails */
03150 {
03151   jit_insn *ref, *ref4, *refslow;
03152 
03153   refslow = _jit.x.pc;
03154 
03155   (void)jit_movi_p(JIT_R2, ((Scheme_Primitive_Proc *)rator)->prim_val);
03156   if (for_branch) {
03157     ref4 = jit_patchable_movi_p(JIT_V1, jit_forward());
03158     mz_set_local_p(JIT_V1, JIT_LOCAL2);
03159   } else
03160     ref4 = NULL;
03161   ref = jit_patchable_movi_p(JIT_V1, jit_forward());
03162 
03163   if (orig_args == 1) {
03164     if (for_branch) {
03165       (void)jit_jmpi(call_original_unary_arith_for_branch_code);
03166     } else {
03167       (void)jit_jmpi(call_original_unary_arith_code);
03168     }
03169   } else {
03170     if (use_v) {
03171       (void)jit_movi_p(JIT_R1, scheme_make_integer(v));
03172       reversed = !reversed;
03173     }
03174 
03175     if (for_branch) {
03176       if (reversed) {
03177        (void)jit_jmpi(call_original_binary_rev_arith_for_branch_code);
03178       } else {
03179        (void)jit_jmpi(call_original_binary_arith_for_branch_code);
03180       }
03181     } else {
03182       if (reversed) {
03183        (void)jit_jmpi(call_original_binary_rev_arith_code);
03184       } else {
03185        (void)jit_jmpi(call_original_binary_arith_code);
03186       }
03187     }
03188   }
03189 
03190   *_ref = ref;
03191   *_ref4 = ref4;
03192 
03193   if (arith == 6) {
03194     /* Add tag back to first arg, just in case. See arithmetic-shift branch to refslow. */
03195     ref = _jit.x.pc;
03196 
03197     if (reversed || use_v) {
03198       jit_ori_l(JIT_R0, JIT_R0, 0x1);
03199     } else {
03200       jit_ori_l(JIT_R1, JIT_R1, 0x1);
03201     }
03202 
03203     __START_TINY_JUMPS__(1);
03204     (void)jit_jmpi(refslow);
03205     __END_TINY_JUMPS__(1);
03206 
03207     return ref;
03208   } else {
03209     return refslow;
03210   }
03211 }
03212 
03213 #ifdef SIXTY_FOUR_BIT_INTEGERS
03214 # define SCHEME_INT_SMALL_ENOUGH(rand2) ((((long)rand2 & 0x7FFFFFFF) == (long)rand2) || (((long)rand2 & 0xFFFFFFFFF8000000) == 0xFFFFFFFFF8000000))
03215 #else
03216 # define SCHEME_INT_SMALL_ENOUGH(rand2) 1
03217 #endif
03218 
03219 static int can_fast_double(int arith, int cmp, int two_args)
03220 {
03221 #ifdef INLINE_FP_OPS
03222   if ((arith == 1)
03223       || (arith == -1)
03224       || (arith == 2)
03225       || (arith == -2)
03226       || (arith == 11))
03227     return 1;
03228 #endif
03229 #ifdef INLINE_FP_COMP
03230   if (!arith
03231       || ((arith == 9) /* min */ && two_args)
03232       || ((arith == 10) /* max */ && two_args))
03233     return 1;
03234 #endif
03235 
03236   return 0;
03237 }
03238 
03239 /* The following FP-generation code is written to work both with a FP
03240    stack (i387) and normal FP regsiters (everything else), though the
03241    double-agent operations that end in _fppop() and _fppush(). In
03242    FP-stack mode, the register names don't actually matter, but the
03243    pushes and pops much balance. The popping branch operations pop
03244    both arguments before branching. */
03245 
03246 #if !defined(MZ_USE_JIT_I386)
03247 /* Not FP stack, so use normal variants. */
03248 #define jit_movi_d_fppush(rd,immd)    jit_movi_d(rd,immd)
03249 #define jit_ldi_d_fppush(rd, is)      jit_ldi_d(rd, is)
03250 #define jit_ldr_d_fppush(rd, rs)      jit_ldr_d(rd, rs)
03251 #define jit_ldxi_d_fppush(rd, rs, is) jit_ldxi_d(rd, rs, is)
03252 #define jit_addr_d_fppop(rd,s1,s2)    jit_addr_d(rd,s1,s2)
03253 #define jit_subr_d_fppop(rd,s1,s2)    jit_subr_d(rd,s1,s2)
03254 #define jit_subrr_d_fppop(rd,s1,s2)   jit_subrr_d(rd,s1,s2)
03255 #define jit_mulr_d_fppop(rd,s1,s2)    jit_mulr_d(rd,s1,s2)
03256 #define jit_divr_d_fppop(rd,s1,s2)    jit_divr_d(rd,s1,s2)
03257 #define jit_divrr_d_fppop(rd,s1,s2)   jit_divrr_d(rd,s1,s2)
03258 #define jit_negr_d_fppop(rd,rs)       jit_negr_d(rd,rs)
03259 #define jit_abs_d_fppop(rd,rs)        jit_abs_d(rd,rs)
03260 #define jit_sti_d_fppop(id, rs)       jit_sti_d(id, rs)
03261 #define jit_stxi_d_fppop(id, rd, rs)  jit_stxi_d(id, rd, rs)
03262 #define jit_bger_d_fppop(d, s1, s2)   jit_bger_d(d, s1, s2)
03263 #define jit_bantiger_d_fppop(d, s1, s2) jit_bantiger_d(d, s1, s2)
03264 #define jit_bler_d_fppop(d, s1, s2)   jit_bler_d(d, s1, s2)
03265 #define jit_bantiler_d_fppop(d, s1, s2) jit_bantiler_d(d, s1, s2)
03266 #define jit_bgtr_d_fppop(d, s1, s2)   jit_bgtr_d(d, s1, s2)
03267 #define jit_bantigtr_d_fppop(d, s1, s2) jit_bantigtr_d(d, s1, s2)
03268 #define jit_bltr_d_fppop(d, s1, s2)   jit_bltr_d(d, s1, s2)
03269 #define jit_bantiltr_d_fppop(d, s1, s2) jit_bantiltr_d(d, s1, s2)
03270 #define jit_beqr_d_fppop(d, s1, s2)   jit_beqr_d(d, s1, s2)
03271 #define jit_bantieqr_d_fppop(d, s1, s2) jit_bantieqr_d(d, s1, s2)
03272 #endif
03273 
03274 static int generate_double_arith(mz_jit_state *jitter, int arith, int cmp, int reversed, int two_args, int second_const,
03275                                  jit_insn **_refd, jit_insn **_refdt,
03276                                  int branch_short)
03277 {
03278 #if defined(INLINE_FP_OPS) || defined(INLINE_FP_COMP)
03279   GC_CAN_IGNORE jit_insn *ref8, *ref9, *ref10, *refd, *refdt;
03280   int no_alloc = 0;
03281 
03282   /* Maybe they're doubles */
03283   __START_TINY_JUMPS__(1);
03284   if (two_args) {
03285     jit_orr_ul(JIT_R2, JIT_R0, JIT_R1);
03286     ref8 = jit_bmsi_ul(jit_forward(), JIT_R2, 0x1);
03287   } else
03288     ref8 = NULL;
03289   jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
03290   ref9 = jit_bnei_p(jit_forward(), JIT_R2, scheme_double_type);
03291   if (two_args) {
03292     jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type);
03293     ref10 = jit_bnei_p(jit_forward(), JIT_R2, scheme_double_type);
03294   } else
03295     ref10 = NULL;
03296   CHECK_LIMIT();
03297   __END_TINY_JUMPS__(1);
03298 
03299   if (!two_args && !second_const && ((arith == 2) || ((arith == -2) && reversed))) {
03300     /* Special case: multiplication by exact 0 */
03301     (void)jit_movi_p(JIT_R0, scheme_make_integer(0));
03302   } else {
03303     /* Yes, they're doubles. */
03304     jit_ldxi_d_fppush(JIT_FPR1, JIT_R0, &((Scheme_Double *)0x0)->double_val);
03305     if (two_args) {
03306       jit_ldxi_d_fppush(JIT_FPR0, JIT_R1, &((Scheme_Double *)0x0)->double_val);
03307     } else if ((arith == -1) && !second_const && reversed) {
03308       reversed = 0;
03309     } else if (arith == 11) {
03310       /* abs needs no extra number */
03311     } else {
03312       double d = second_const;      
03313       jit_movi_d_fppush(JIT_FPR0, d);
03314       reversed = !reversed;
03315       cmp = -cmp;
03316     }
03317     if (arith) {
03318       switch (arith) {
03319       case 1:
03320         jit_addr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03321         break;
03322       case 2:
03323         jit_mulr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03324         break;
03325       case -2:
03326         if (reversed)
03327           jit_divrr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03328         else
03329           jit_divr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03330         break;
03331       case -1:
03332         {
03333           if (!two_args && !second_const && !reversed) {
03334             /* Need a special case to make sure that (- 0.0) => -0.0 */
03335             jit_negr_d_fppop(JIT_FPR1, JIT_FPR1);
03336           } else if (reversed)
03337             jit_subrr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03338           else
03339             jit_subr_d_fppop(JIT_FPR1, JIT_FPR0, JIT_FPR1);
03340         }
03341         break;
03342       case 9: /* min */
03343       case 10: /* max */
03344         {
03345           GC_CAN_IGNORE jit_insn *refc, *refn;
03346           __START_TINY_JUMPS__(1);
03347          /* If R0 is nan, then copy to R1, ensuring nan result */
03348          refn = jit_beqr_d(jit_forward(), JIT_FPR1, JIT_FPR1);
03349           jit_movr_p(JIT_R1, JIT_R0);
03350          mz_patch_branch(refn);
03351           if (arith == 9) {
03352             refc = jit_bger_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03353           } else {
03354            refc = jit_bltr_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03355           }
03356           jit_movr_p(JIT_R0, JIT_R1);
03357           mz_patch_branch(refc);
03358           __END_TINY_JUMPS__(1);
03359           no_alloc = 1;
03360         }
03361         break;
03362       case 11: /* abs */
03363         jit_abs_d_fppop(JIT_FPR1, JIT_FPR1);
03364         break;
03365       default:
03366         break;
03367       }
03368       CHECK_LIMIT();
03369 
03370       if (!no_alloc) {
03371 #ifdef INLINE_FP_OPS
03372 # ifdef CAN_INLINE_ALLOC
03373         inline_alloc(jitter, sizeof(Scheme_Double), scheme_double_type, 0, 0, 1, 0);
03374         CHECK_LIMIT();
03375         jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
03376         (void)jit_stxi_d_fppop(&((Scheme_Double *)0x0)->double_val, JIT_R0, JIT_FPR1);
03377 # else
03378         (void)jit_sti_d_fppop(&double_result, JIT_FPR1);    
03379         JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
03380         mz_prepare(0);
03381         (void)mz_finish(malloc_double);
03382         jit_retval(JIT_R0);
03383 # endif
03384 #endif
03385         CHECK_LIMIT();
03386       }
03387 
03388     } else {
03389       /* The "anti" variants below invert the branch. Unlike the "un" 
03390          variants, the "anti" variants invert the comparison result
03391          after the layer where +nan.0 always generates false. */
03392       __START_SHORT_JUMPS__(branch_short);
03393       switch (cmp) {
03394       case -2:
03395         refd = jit_bantiltr_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03396         break;
03397       case -1:
03398         refd = jit_bantiler_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03399         break;
03400       case 0:
03401         refd = jit_bantieqr_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03402         break;
03403       case 1:
03404         refd = jit_bantiger_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03405         break;
03406       case 2:
03407         refd = jit_bantigtr_d_fppop(jit_forward(), JIT_FPR0, JIT_FPR1);
03408         break;
03409       default:
03410         refd = NULL;
03411         break;
03412       }
03413       __END_SHORT_JUMPS__(branch_short);
03414       *_refd = refd;
03415     }
03416   }
03417 
03418   /* Jump to return result or true branch: */
03419   __START_SHORT_JUMPS__(branch_short);
03420   refdt = jit_jmpi(jit_forward());
03421   *_refdt = refdt;
03422   __END_SHORT_JUMPS__(branch_short);
03423 
03424   /* No, they're not both doubles. */
03425   __START_TINY_JUMPS__(1);
03426   if (two_args) {
03427     mz_patch_branch(ref8);
03428     mz_patch_branch(ref10);
03429   }
03430   mz_patch_branch(ref9);
03431   __END_TINY_JUMPS__(1);
03432 #endif
03433   
03434   return 1;
03435 }
03436 
03437 static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Object *rand, Scheme_Object *rand2, 
03438                        int orig_args, int arith, int cmp, int v, jit_insn **for_branch, int branch_short)
03439 /* needs de-sync */
03440 /* Either arith is non-zero or it's a cmp; the value of each determines the operation:
03441         arith = 1 -> + or add1 (if !rand2)
03442         arith = -1 -> - or sub1
03443         arith = 2 -> *
03444         arith = 3 -> bitwise-and
03445         arith = 4 -> bitwise-ior
03446         arith = 5 -> bitwise-xor
03447         arith = 6 -> arithmetic-shift
03448         arith = 7 -> bitwise-not
03449         arith = 9 -> min
03450         arith = 10 -> max
03451         arith = 11 -> abs
03452         cmp = 0 -> = or zero?
03453         cmp = +/-1 -> >=/<=
03454         cmp = +/-2 -> >/< or positive/negative?
03455         cmp = 3 -> bitwise-bit-test?
03456  */
03457 {
03458   GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref4, *refd = NULL, *refdt = NULL, *refslow;
03459   int skipped, simple_rand, simple_rand2, reversed = 0, has_fixnum_fast = 1;
03460 
03461   LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)rator)->name));
03462 
03463   if (rand2) {
03464     if (SCHEME_INTP(rand2)
03465        && SCHEME_INT_SMALL_ENOUGH(rand2)
03466        && ((arith != 6)
03467            || ((SCHEME_INT_VAL(rand2) <= MAX_TRY_SHIFT)
03468               && (SCHEME_INT_VAL(rand2) >= -MAX_TRY_SHIFT)))
03469         && ((cmp != 3)
03470            || ((SCHEME_INT_VAL(rand2) <= MAX_TRY_SHIFT)
03471               && (SCHEME_INT_VAL(rand2) >= 0)))) {
03472       /* Second is constant, so use constant mode.
03473         For arithmetic shift, only do this if the constant
03474         is in range. */
03475       v = SCHEME_INT_VAL(rand2);
03476       rand2 = NULL;
03477     } else if (SCHEME_INTP(rand)
03478               && SCHEME_INT_SMALL_ENOUGH(rand)
03479               && (arith != 6)
03480                && (cmp != 3)) {
03481       /* First is constant; swap argument order and use constant mode. */
03482       v = SCHEME_INT_VAL(rand);
03483       cmp = -cmp;
03484       rand = rand2;
03485       rand2 = NULL;
03486       reversed = 1;
03487     } else if ((ok_to_move_local(rand2)
03488               || SCHEME_INTP(rand2))
03489               && !(ok_to_move_local(rand)
03490                   || SCHEME_INTP(rand))) {
03491       /* Second expression is side-effect-free, unlike the first; 
03492         swap order and use the fast path for when the first arg is
03493         side-effect free. */
03494       Scheme_Object *t = rand2;
03495       rand2 = rand;
03496       rand = t;
03497       cmp = -cmp;
03498       reversed = 1;
03499     }
03500   }
03501 
03502   if ((arith == -1) && (orig_args == 1) && !v) {
03503     /* Unary subtract */
03504     reversed = 1;
03505   }
03506 
03507   if (rand2) {
03508     simple_rand = (ok_to_move_local(rand)
03509                  || SCHEME_INTP(rand));
03510     if (!simple_rand)
03511       simple_rand2 = SAME_TYPE(SCHEME_TYPE(rand2), scheme_local_type);
03512     else
03513       simple_rand2 = 0;
03514   } else {
03515     simple_rand = 0;
03516     simple_rand2 = 0;
03517   }
03518 
03519   if (rand2 && !simple_rand && !simple_rand2)
03520     skipped = orig_args - 1;    
03521   else
03522     skipped = orig_args;
03523 
03524   mz_runstack_skipped(jitter, skipped);
03525 
03526   if (rand2 && !simple_rand && !simple_rand2) {
03527     mz_runstack_skipped(jitter, 1);
03528     generate_non_tail(rand, jitter, 0, 1); /* sync'd later */
03529     CHECK_LIMIT();
03530     mz_runstack_unskipped(jitter, 1);
03531     mz_rs_dec(1);
03532     CHECK_RUNSTACK_OVERFLOW();
03533     mz_runstack_pushed(jitter, 1);
03534     mz_rs_str(JIT_R0);
03535   }
03536   /* not sync'd... */
03537 
03538   if (simple_rand2) {
03539     if (SAME_TYPE(SCHEME_TYPE(rand), scheme_local_type))
03540       generate(rand, jitter, 0, 0, JIT_R1); /* sync'd below */
03541     else {
03542       generate_non_tail(rand, jitter, 0, 1); /* sync'd below */
03543       CHECK_LIMIT();
03544       jit_movr_p(JIT_R1, JIT_R0);
03545     }
03546     CHECK_LIMIT();
03547     generate(rand2, jitter, 0, 0, JIT_R0); /* sync'd below */
03548   } else {
03549     generate_non_tail(rand2 ? rand2 : rand, jitter, 0, 1); /* sync'd below */
03550   }
03551   CHECK_LIMIT();
03552   /* sync'd in three branches below */
03553 
03554   if (arith == -2) {
03555     if (rand2 || (v != 1) || reversed)
03556       has_fixnum_fast = 0;
03557   }
03558 
03559   /* rand2 in R0, and rand in R1 unless it's simple */
03560 
03561   if (simple_rand || simple_rand2) {
03562     int pos, va;
03563 
03564     if (simple_rand && SCHEME_INTP(rand)) {
03565       (void)jit_movi_p(JIT_R1, rand);
03566       va = JIT_R0;
03567     } else {
03568       if (simple_rand) {
03569         pos = mz_remap(SCHEME_LOCAL_POS(rand));
03570         mz_rs_ldxi(JIT_R1, pos);
03571       }
03572       /* check both fixnum bits at once by ANDing into R2: */
03573       jit_andr_ul(JIT_R2, JIT_R0, JIT_R1);
03574       va = JIT_R2;
03575     }
03576 
03577     mz_rs_sync();
03578 
03579     __START_TINY_JUMPS__(1);
03580     ref2 = jit_bmsi_ul(jit_forward(), va, 0x1);
03581     __END_TINY_JUMPS__(1);
03582 
03583     if (!SCHEME_INTP(rand) && can_fast_double(arith, cmp, 1)) {
03584       /* Maybe they're both doubles... */
03585       generate_double_arith(jitter, arith, cmp, reversed, 1, 0, &refd, &refdt, branch_short);
03586       CHECK_LIMIT();
03587     }
03588 
03589     if (!has_fixnum_fast) {
03590       __START_TINY_JUMPS__(1);
03591       mz_patch_branch(ref2);
03592       __END_TINY_JUMPS__(1);
03593     }
03594 
03595     /* Slow path */
03596     refslow = generate_arith_slow_path(jitter, rator, &ref, &ref4, for_branch, orig_args, reversed, arith, 0, 0);
03597 
03598     if (has_fixnum_fast) {
03599       __START_TINY_JUMPS__(1);
03600       mz_patch_branch(ref2);
03601       __END_TINY_JUMPS__(1);
03602     }
03603     CHECK_LIMIT();
03604   } else if (rand2) {
03605     /* Move rand result back into R1 */
03606     mz_rs_ldr(JIT_R1);
03607     mz_rs_inc(1);
03608     mz_runstack_popped(jitter, 1);
03609 
03610     mz_rs_sync();
03611 
03612     /* check both fixnum bits at once by ANDing into R2: */
03613     jit_andr_ul(JIT_R2, JIT_R0, JIT_R1);
03614     __START_TINY_JUMPS__(1);
03615     ref2 = jit_bmsi_ul(jit_forward(), JIT_R2, 0x1);
03616     __END_TINY_JUMPS__(1);
03617     CHECK_LIMIT();
03618 
03619     if (can_fast_double(arith, cmp, 1)) {
03620       /* Maybe they're both doubles... */
03621       generate_double_arith(jitter, arith, cmp, reversed, 1, 0, &refd, &refdt, branch_short);
03622       CHECK_LIMIT();
03623     }
03624 
03625     if (!has_fixnum_fast) {
03626       __START_TINY_JUMPS__(1);
03627       mz_patch_branch(ref2);
03628       __END_TINY_JUMPS__(1);
03629     }
03630 
03631     /* Slow path */
03632     refslow = generate_arith_slow_path(jitter, rator, &ref, &ref4, for_branch, orig_args, reversed, arith, 0, 0);
03633 
03634     if (has_fixnum_fast) {
03635       /* Fixnum branch: */
03636       __START_TINY_JUMPS__(1);
03637       mz_patch_branch(ref2);
03638       __END_TINY_JUMPS__(1);
03639     }
03640     CHECK_LIMIT();
03641   } else {
03642     mz_rs_sync();
03643     /* Only one argument: */
03644     __START_TINY_JUMPS__(1);
03645     ref2 = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1);
03646     __END_TINY_JUMPS__(1);
03647 
03648     if ((orig_args != 2) /* <- heuristic: we could generate code when an exact argument is
03649                             given, but the extra FP code is probably not worthwhile. */
03650         && can_fast_double(arith, cmp, 0)
03651         /* watch out: divide by 0 is special: */
03652         && ((arith != -2) || v || reversed)) {
03653       /* Maybe it's a double... */
03654       generate_double_arith(jitter, arith, cmp, reversed, 0, v, &refd, &refdt, branch_short);
03655       CHECK_LIMIT();
03656     }
03657 
03658     if (!has_fixnum_fast) {
03659       __START_TINY_JUMPS__(1);
03660       mz_patch_branch(ref2);
03661       __END_TINY_JUMPS__(1);
03662     }
03663 
03664     /* Slow path */
03665     refslow = generate_arith_slow_path(jitter, rator, &ref, &ref4, for_branch, orig_args, reversed, arith, 1, v);
03666 
03667     if (has_fixnum_fast) {
03668       __START_TINY_JUMPS__(1);
03669       mz_patch_branch(ref2);
03670       __END_TINY_JUMPS__(1);
03671     }
03672   }
03673 
03674   CHECK_LIMIT();
03675 
03676   mz_runstack_unskipped(jitter, skipped);
03677 
03678   __START_SHORT_JUMPS__(branch_short);
03679 
03680   if (arith) {
03681     if (rand2) {
03682       /* First arg is in JIT_R1, second is in JIT_R0 */
03683       if (arith == 1) {
03684        jit_andi_ul(JIT_R2, JIT_R1, (~0x1));
03685        (void)jit_boaddr_l(refslow, JIT_R2, JIT_R0);
03686        jit_movr_p(JIT_R0, JIT_R2);
03687       } else if (arith == -1) {
03688        if (reversed) {
03689          jit_movr_p(JIT_R2, JIT_R0);
03690          (void)jit_bosubr_l(refslow, JIT_R2, JIT_R1);
03691        } else {
03692          jit_movr_p(JIT_R2, JIT_R1);
03693          (void)jit_bosubr_l(refslow, JIT_R2, JIT_R0);
03694        }
03695        jit_ori_ul(JIT_R0, JIT_R2, 0x1);
03696       } else if (arith == 2) {
03697         jit_andi_ul(JIT_R2, JIT_R1, (~0x1));
03698         jit_rshi_l(JIT_V1, JIT_R0, 0x1);
03699         (void)jit_bomulr_l(refslow, JIT_V1, JIT_R2);
03700         jit_ori_ul(JIT_R0, JIT_V1, 0x1);
03701       } else if (arith == -2) {
03702         if (has_fixnum_fast) {
03703           /* No fast path for fixnum division, yet */
03704           (void)jit_jmpi(refslow);
03705         }
03706       } else if (arith == 3) {
03707        /* and */
03708        jit_andr_ul(JIT_R0, JIT_R1, JIT_R0);
03709       } else if (arith == 4) {
03710        /* ior */
03711        jit_orr_ul(JIT_R0, JIT_R1, JIT_R0);
03712       } else if (arith == 5) {
03713        /* xor */
03714        jit_andi_ul(JIT_R0, JIT_R0, (~0x1));
03715        jit_xorr_ul(JIT_R0, JIT_R1, JIT_R0);
03716       } else if (arith == 6) {
03717        /* arithmetic-shift 
03718           This is a lot of code, but if you're using
03719           arihtmetic-shift, then you probably want it. */
03720        int v1 = (reversed ? JIT_R0 : JIT_R1);
03721        int v2 = (reversed ? JIT_R1 : JIT_R0);
03722        jit_insn *refi, *refc;
03723 
03724        refi = jit_bgei_l(refslow, v2, (long)scheme_make_integer(0));
03725 
03726        /* Right shift (always works for a small enough shift) */
03727        (void)jit_blti_l(refslow, v2, scheme_make_integer(-MAX_TRY_SHIFT));
03728        jit_notr_l(JIT_V1, v2);
03729        jit_rshi_l(JIT_V1, JIT_V1, 0x1);
03730        jit_addi_l(JIT_V1, JIT_V1, 0x1);
03731         CHECK_LIMIT();
03732 #ifdef MZ_USE_JIT_I386
03733        /* Can't shift from _ECX */
03734        jit_movr_l(JIT_R2, v1);
03735        jit_rshr_l(JIT_R2, JIT_R2, JIT_V1);
03736 #else
03737        jit_rshr_l(JIT_R2, v1, JIT_V1);
03738 #endif
03739        jit_ori_l(JIT_R0, JIT_R2, 0x1);
03740        refc = jit_jmpi(jit_forward());
03741         CHECK_LIMIT();
03742 
03743        /* Left shift */
03744        mz_patch_branch(refi);
03745        (void)jit_bgti_l(refslow, v2, (long)scheme_make_integer(MAX_TRY_SHIFT));
03746        jit_rshi_l(JIT_V1, v2, 0x1);
03747        jit_andi_l(v1, v1, (~0x1));
03748 #ifdef MZ_USE_JIT_I386
03749        /* Can't shift from _ECX */
03750        jit_movr_l(JIT_R2, v1);
03751        jit_lshr_l(JIT_R2, JIT_R2, JIT_V1);
03752 #else
03753        jit_lshr_l(JIT_R2, v1, JIT_V1);
03754 #endif
03755         CHECK_LIMIT();
03756        /* If shifting back right produces a different result, that's overflow... */
03757        jit_rshr_l(JIT_V1, JIT_R2, JIT_V1);
03758        /* !! In case we go refslow, it nseed to add back tag to v1 !! */
03759        (void)jit_bner_p(refslow, JIT_V1, v1);
03760        /* No overflow. */
03761        jit_ori_l(JIT_R0, JIT_R2, 0x1);
03762 
03763        mz_patch_ucbranch(refc);
03764       } else if (arith == 9) {
03765         /* min */
03766         jit_insn *refc;
03767         __START_INNER_TINY__(branch_short);
03768         refc = jit_bltr_l(jit_forward(), JIT_R0, JIT_R1);
03769         jit_movr_l(JIT_R0, JIT_R1);
03770         mz_patch_branch(refc);
03771         __END_INNER_TINY__(branch_short);
03772       } else if (arith == 10) {
03773         /* max */
03774         jit_insn *refc;
03775         __START_INNER_TINY__(branch_short);
03776         refc = jit_bgtr_l(jit_forward(), JIT_R0, JIT_R1);
03777         jit_movr_l(JIT_R0, JIT_R1);
03778         mz_patch_branch(refc);
03779         __END_INNER_TINY__(branch_short);
03780       }
03781     } else {
03782       /* Non-constant arg is in JIT_R0 */
03783       if (arith == 1) {
03784        jit_movr_p(JIT_R2, JIT_R0);
03785        (void)jit_boaddi_l(refslow, JIT_R2, v << 1);
03786        jit_movr_p(JIT_R0, JIT_R2);
03787       } else if (arith == -1) {
03788        if (reversed) {
03789          (void)jit_movi_p(JIT_R2, scheme_make_integer(v));
03790          (void)jit_bosubr_l(refslow, JIT_R2, JIT_R0);
03791          jit_addi_ul(JIT_R0, JIT_R2, 0x1);
03792        } else {
03793          jit_movr_p(JIT_R2, JIT_R0);
03794          (void)jit_bosubi_l(refslow, JIT_R2, v << 1);
03795          jit_movr_p(JIT_R0, JIT_R2);
03796        }
03797       } else if (arith == 2) {
03798         if (v == 1) {
03799           /* R0 already is the answer */
03800         } else if (v == 0) {
03801           (void)jit_movi_p(JIT_R0, scheme_make_integer(0));
03802         } else {
03803           (void)jit_movi_p(JIT_R1, scheme_make_integer(v));
03804           jit_andi_ul(JIT_R2, JIT_R1, (~0x1));
03805           jit_rshi_l(JIT_V1, JIT_R0, 0x1);
03806           (void)jit_bomulr_l(refslow, JIT_V1, JIT_R2);
03807           jit_ori_ul(JIT_R0, JIT_V1, 0x1);
03808         }
03809       } else if (arith == -2) {
03810         if ((v == 1) && !reversed) {
03811           /* R0 already is the answer */
03812         } else {
03813           if (has_fixnum_fast) {
03814             /* No general fast path for fixnum division, yet */
03815             (void)jit_movi_p(JIT_R1, scheme_make_integer(v));
03816             (void)jit_jmpi(refslow);
03817           }
03818         }
03819       } else {
03820        if (arith == 3) {
03821          /* and */
03822          long l = (long)scheme_make_integer(v);
03823          jit_andi_ul(JIT_R0, JIT_R0, l);
03824        } else if (arith == 4) {
03825          /* ior */
03826          long l = (long)scheme_make_integer(v);
03827          jit_ori_ul(JIT_R0, JIT_R0, l);
03828        } else if (arith == 5) {
03829          /* xor */
03830          jit_xori_ul(JIT_R0, JIT_R0, v << 1);
03831        } else if (arith == 6) {
03832          /* arithmetic-shift */
03833          /* We only get here when v is between -MAX_TRY_SHIFT and MAX_TRY_SHIFT, inclusive */
03834          if (v <= 0) {
03835            jit_rshi_l(JIT_R0, JIT_R0, -v);
03836            jit_ori_l(JIT_R0, JIT_R0, 0x1);
03837          } else {
03838            jit_andi_l(JIT_R0, JIT_R0, (~0x1));
03839            jit_lshi_l(JIT_R2, JIT_R0, v);
03840            /* If shifting back right produces a different result, that's overflow... */
03841            jit_rshi_l(JIT_V1, JIT_R2, v);
03842            /* !! In case we go refslow, it nseed to add back tag to JIT_R0 !! */
03843            (void)jit_bner_p(refslow, JIT_V1, JIT_R0);
03844            /* No overflow. */
03845            jit_ori_l(JIT_R0, JIT_R2, 0x1);
03846          }
03847        } else if (arith == 7) {
03848          jit_notr_ul(JIT_R0, JIT_R0);
03849          jit_ori_ul(JIT_R0, JIT_R0, 0x1);
03850        } else if (arith == 9) {
03851           /* min */
03852           jit_insn *refc;
03853           __START_INNER_TINY__(branch_short);
03854           refc = jit_blti_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03855           jit_movi_l(JIT_R0, (long)scheme_make_integer(v));
03856           mz_patch_branch(refc);
03857           __END_INNER_TINY__(branch_short);
03858         } else if (arith == 10) {
03859           /* max */
03860           jit_insn *refc;
03861           __START_INNER_TINY__(branch_short);
03862           refc = jit_bgti_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03863           jit_movi_l(JIT_R0, (long)scheme_make_integer(v));
03864           mz_patch_branch(refc);
03865           __END_INNER_TINY__(branch_short);
03866         } else if (arith == 11) {
03867           /* abs */
03868           jit_insn *refc;
03869           __START_INNER_TINY__(branch_short);
03870           refc = jit_bgei_l(jit_forward(), JIT_R0, (long)scheme_make_integer(0));
03871           __END_INNER_TINY__(branch_short);
03872           /* watch out for most negative fixnum! */
03873           (void)jit_beqi_p(refslow, JIT_R0, (void *)(((long)1 << ((8 * JIT_WORD_SIZE) - 1)) | 0x1));
03874           jit_rshi_l(JIT_R0, JIT_R0, 1);
03875           jit_movi_l(JIT_R1, 0);
03876           jit_subr_l(JIT_R0, JIT_R1, JIT_R0);
03877           jit_lshi_l(JIT_R0, JIT_R0, 1);
03878           jit_ori_l(JIT_R0, JIT_R0, 0x1);
03879           __START_INNER_TINY__(branch_short);
03880           mz_patch_branch(refc);
03881           __END_INNER_TINY__(branch_short);
03882           CHECK_LIMIT();
03883         }
03884       }
03885     }
03886     if (refdt)
03887       mz_patch_ucbranch(refdt);
03888     jit_patch_movi(ref, (_jit.x.pc));
03889   } else {
03890     /* If second is constant, first arg is in JIT_R0. */
03891     /* Otherwise, first arg is in JIT_R1, second is in JIT_R0 */
03892     /* Jump to ref3 to produce false */
03893     switch (cmp) {
03894     case -3:
03895       if (rand2) {
03896         (void)jit_blti_l(refslow, JIT_R1, 0);
03897         (void)jit_bgti_l(refslow, JIT_R1, (long)scheme_make_integer(MAX_TRY_SHIFT));
03898         jit_rshi_l(JIT_R1, JIT_R1, 1);
03899         jit_addi_l(JIT_V1, JIT_R1, 1);
03900         jit_movi_l(JIT_R2, 1);
03901         jit_lshr_l(JIT_R2, JIT_R2, JIT_V1);
03902         ref3 = jit_bmcr_l(jit_forward(), JIT_R0, JIT_R2);
03903       } else {
03904         /* shouldn't get here */
03905         scheme_signal_error("bitwise-bit-test? constant in wrong position");
03906         ref3 = NULL;
03907       }
03908       break;
03909     case -2:
03910       if (rand2) {
03911        ref3 = jit_bger_l(jit_forward(), JIT_R1, JIT_R0);
03912       } else {
03913        ref3 = jit_bgei_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03914       }
03915       break;
03916     case -1:
03917       if (rand2) {
03918        ref3 = jit_bgtr_l(jit_forward(), JIT_R1, JIT_R0);
03919       } else {
03920        ref3 = jit_bgti_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03921       }
03922       break;
03923     case 0:
03924       if (rand2) {
03925        ref3 = jit_bner_l(jit_forward(), JIT_R1, JIT_R0);
03926       } else {
03927        ref3 = jit_bnei_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03928       }
03929       break;
03930     case 1:
03931       if (rand2) {
03932        ref3 = jit_bltr_l(jit_forward(), JIT_R1, JIT_R0);
03933       } else {
03934        ref3 = jit_blti_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03935       }
03936       break;
03937     case 2:
03938       if (rand2) {
03939        ref3 = jit_bler_l(jit_forward(), JIT_R1, JIT_R0);
03940       } else {
03941        ref3 = jit_blei_l(jit_forward(), JIT_R0, (long)scheme_make_integer(v));
03942       }
03943       break;
03944     default:
03945     case 3:
03946       if (rand2) {
03947         (void)jit_blti_l(refslow, JIT_R0, 0);
03948         (void)jit_bgti_l(refslow, JIT_R0, (long)scheme_make_integer(MAX_TRY_SHIFT));
03949         jit_rshi_l(JIT_R0, JIT_R0, 1);
03950         jit_addi_l(JIT_R0, JIT_R0, 1);
03951         jit_movi_l(JIT_V1, 1);
03952         jit_lshr_l(JIT_R0, JIT_V1, JIT_R0);
03953         ref3 = jit_bmcr_l(jit_forward(), JIT_R1, JIT_R0);
03954       } else {
03955         ref3 = jit_bmci_l(jit_forward(), JIT_R0, 1 << (v+1));
03956       }
03957       break;
03958     }
03959 
03960     if (refdt)
03961       mz_patch_ucbranch(refdt);
03962 
03963     if (for_branch) {
03964       for_branch[0] = ref3;
03965       for_branch[1] = refd;
03966       for_branch[2] = ref;
03967       jit_patch_movi(ref4, (_jit.x.pc));
03968     } else {
03969       (void)jit_movi_p(JIT_R0, scheme_true);
03970       ref2 = jit_jmpi(jit_forward());
03971       mz_patch_branch(ref3);
03972       if (refd)
03973         mz_patch_branch(refd);
03974       (void)jit_movi_p(JIT_R0, scheme_false);
03975       mz_patch_ucbranch(ref2);
03976       jit_patch_movi(ref, (_jit.x.pc));
03977     }
03978   }
03979 
03980   __END_SHORT_JUMPS__(branch_short);
03981 
03982   return 1;  
03983 }
03984 
03985 static int generate_inlined_constant_test(mz_jit_state *jitter, Scheme_App2_Rec *app,
03986                                      Scheme_Object *cnst, Scheme_Object *cnst2, 
03987                                      jit_insn **for_branch, int branch_short, int need_sync)
03988 /* de-sync'd ok */
03989 {
03990   GC_CAN_IGNORE jit_insn *ref, *ref2;
03991 
03992   LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)app->rator)->name));
03993 
03994   mz_runstack_skipped(jitter, 1);
03995 
03996   generate_non_tail(app->rand, jitter, 0, 1);
03997   CHECK_LIMIT();
03998 
03999   mz_runstack_unskipped(jitter, 1);
04000 
04001   if (need_sync) mz_rs_sync();
04002 
04003   __START_SHORT_JUMPS__(branch_short);
04004 
04005   if (cnst2) {
04006     ref2 = mz_beqi_p(jit_forward(), JIT_R0, cnst);
04007     ref = mz_bnei_p(jit_forward(), JIT_R0, cnst2);
04008     mz_patch_branch(ref2);
04009   } else {
04010     ref = mz_bnei_p(jit_forward(), JIT_R0, cnst);
04011   }
04012 
04013   if (for_branch) {
04014     for_branch[0] = ref;
04015   } else {
04016     (void)jit_movi_p(JIT_R0, scheme_true);
04017     ref2 = jit_jmpi(jit_forward());
04018     mz_patch_branch(ref);
04019     (void)jit_movi_p(JIT_R0, scheme_false);
04020     mz_patch_ucbranch(ref2);
04021   }
04022 
04023   __END_SHORT_JUMPS__(branch_short);
04024 
04025   return 1;
04026 }
04027 
04028 static int generate_inlined_type_test(mz_jit_state *jitter, Scheme_App2_Rec *app,
04029                                   Scheme_Type lo_ty, Scheme_Type hi_ty, 
04030                                   jit_insn **for_branch, int branch_short, int need_sync)
04031 {
04032   GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref4;
04033   int int_ok;
04034 
04035   int_ok = ((lo_ty <= scheme_integer_type) && (scheme_integer_type <= hi_ty));
04036 
04037   LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)app->rator)->name));
04038 
04039   mz_runstack_skipped(jitter, 1);
04040 
04041   generate_non_tail(app->rand, jitter, 0, 1);
04042   CHECK_LIMIT();
04043 
04044   mz_runstack_unskipped(jitter, 1);
04045 
04046   if (need_sync) mz_rs_sync();
04047 
04048   __START_SHORT_JUMPS__(branch_short);
04049 
04050   ref = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1);
04051   jit_ldxi_s(JIT_R0, JIT_R0, &((Scheme_Object *)0x0)->type);
04052   if (lo_ty == hi_ty) {
04053     ref3 = jit_bnei_p(jit_forward(), JIT_R0, lo_ty);
04054     ref4 = NULL;
04055   } else {
04056     ref3 = jit_blti_p(jit_forward(), JIT_R0, lo_ty);
04057     ref4 = jit_bgti_p(jit_forward(), JIT_R0, hi_ty);
04058   }
04059   if (int_ok) {
04060     mz_patch_branch(ref);
04061   }
04062   if (for_branch) {
04063     if  (!int_ok) {
04064       for_branch[0] = ref;
04065     }
04066     for_branch[1] = ref3;
04067     for_branch[3] = ref4;
04068   } else {
04069     if ((lo_ty <= scheme_integer_type) && (scheme_integer_type <= hi_ty)) {
04070       mz_patch_branch(ref);
04071     }
04072     (void)jit_movi_p(JIT_R0, scheme_true);
04073     ref2 = jit_jmpi(jit_forward());
04074     if  (!int_ok) {
04075       mz_patch_branch(ref);
04076     }
04077     mz_patch_branch(ref3);
04078     if (ref4) {
04079       mz_patch_branch(ref4);
04080     }
04081     (void)jit_movi_p(JIT_R0, scheme_false);
04082     mz_patch_ucbranch(ref2);
04083   }
04084 
04085   __END_SHORT_JUMPS__(branch_short);
04086 
04087   return 1;
04088 }
04089 
04090 static int generate_inlined_struct_op(int kind, mz_jit_state *jitter, 
04091                                   Scheme_Object *rator, Scheme_Object *rand, Scheme_Object *rand2,
04092                                   jit_insn **for_branch, int branch_short, 
04093                                       int multi_ok)
04094 /* de-sync'd ok; for branch, sync'd before */
04095 {
04096   LOG_IT(("inlined struct op\n"));
04097 
04098   if (!rand2) {
04099     generate_two_args(rator, rand, jitter, 1, 1); /* sync'd below */
04100     CHECK_LIMIT();
04101   } else {
04102     Scheme_Object *args[3];
04103     args[0] = rator;
04104     args[1] = rand;
04105     args[2] = rand2;
04106     generate_app(NULL, args, 2, jitter, 0, 0, 1); /* sync'd below */
04107     CHECK_LIMIT();
04108     jit_movr_p(JIT_R0, JIT_V1);
04109     mz_rs_ldr(JIT_R1);
04110     mz_rs_ldxi(JIT_V1, 1);
04111     mz_rs_inc(2); /* sync'd below */
04112     mz_runstack_popped(jitter, 2);
04113   }
04114   mz_rs_sync();
04115 
04116   /* R0 is [potential] predicate/getter/setting, R1 is struct. 
04117      V1 is value for setting. */
04118 
04119   if (for_branch) {
04120     for_branch[2] = jit_patchable_movi_p(JIT_V1, jit_forward());
04121     (void)jit_calli(struct_pred_branch_code);
04122   } else if (kind == 1) {
04123     if (multi_ok) {
04124       (void)jit_calli(struct_pred_multi_code);
04125     } else {
04126       (void)jit_calli(struct_pred_code);
04127     }
04128   } else if (kind == 2) {
04129     if (multi_ok) {
04130       (void)jit_calli(struct_get_multi_code);
04131     } else {
04132       (void)jit_calli(struct_get_code);
04133     }
04134   } else {
04135     if (multi_ok) {
04136       (void)jit_calli(struct_set_multi_code);
04137     } else {
04138       (void)jit_calli(struct_set_code);
04139     }
04140   }
04141 
04142   return 1;
04143 }
04144 
04145 static int generate_cons_alloc(mz_jit_state *jitter, int rev, int inline_retry);
04146 static int generate_vector_alloc(mz_jit_state *jitter, Scheme_Object *rator,
04147                                  Scheme_App_Rec *app, Scheme_App2_Rec *app2, Scheme_App3_Rec *app3);
04148 
04149 static int generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, int is_tail, int multi_ok, 
04150                               jit_insn **for_branch, int branch_short, int need_sync)
04151 /* de-sync's, unless branch */
04152 {
04153   Scheme_Object *rator = app->rator;
04154 
04155   {
04156     int k;
04157     k = inlineable_struct_prim(rator, jitter, 1, 1);
04158     if (k == 1) {
04159       generate_inlined_struct_op(1, jitter, rator, app->rand, NULL, for_branch, branch_short, multi_ok);
04160       scheme_direct_call_count++;
04161       return 1;
04162     } else if ((k == 2) && !for_branch) {
04163       generate_inlined_struct_op(2, jitter, rator, app->rand, NULL, for_branch, branch_short, multi_ok);
04164       scheme_direct_call_count++;
04165       return 1;
04166     }
04167   }
04168 
04169   if (!SCHEME_PRIMP(rator))
04170     return 0;
04171 
04172   if (!(SCHEME_PRIM_PROC_FLAGS(rator) & SCHEME_PRIM_IS_UNARY_INLINED))
04173     return 0;
04174 
04175   scheme_direct_call_count++;
04176 
04177   if (IS_NAMED_PRIM(rator, "not")) {
04178     generate_inlined_constant_test(jitter, app, scheme_false, NULL, for_branch, branch_short, need_sync);
04179     return 1;
04180   } else if (IS_NAMED_PRIM(rator, "null?")) {
04181     generate_inlined_constant_test(jitter, app, scheme_null, NULL, for_branch, branch_short, need_sync);
04182     return 1;
04183   } else if (IS_NAMED_PRIM(rator, "pair?")) {
04184     generate_inlined_type_test(jitter, app, scheme_pair_type, scheme_pair_type, for_branch, branch_short, need_sync);
04185     return 1;
04186   } else if (IS_NAMED_PRIM(rator, "mpair?")) {
04187     generate_inlined_type_test(jitter, app, scheme_mutable_pair_type, scheme_mutable_pair_type, for_branch, branch_short, need_sync);
04188     return 1;
04189   } else if (IS_NAMED_PRIM(rator, "symbol?")) {
04190     generate_inlined_type_test(jitter, app, scheme_symbol_type, scheme_symbol_type, for_branch, branch_short, need_sync);
04191     return 1;
04192   } else if (IS_NAMED_PRIM(rator, "syntax?")) {
04193     generate_inlined_type_test(jitter, app, scheme_stx_type, scheme_stx_type, for_branch, branch_short, need_sync);
04194     return 1;
04195   } else if (IS_NAMED_PRIM(rator, "char?")) {
04196     generate_inlined_type_test(jitter, app, scheme_char_type, scheme_char_type, for_branch, branch_short, need_sync);
04197     return 1;
04198   } else if (IS_NAMED_PRIM(rator, "boolean?")) {
04199     generate_inlined_constant_test(jitter, app, scheme_false, scheme_true, for_branch, branch_short, need_sync);
04200     return 1;
04201   } else if (IS_NAMED_PRIM(rator, "number?")) {
04202     generate_inlined_type_test(jitter, app, scheme_integer_type, scheme_complex_type, for_branch, branch_short, need_sync);
04203     return 1;
04204   } else if (IS_NAMED_PRIM(rator, "real?")) {
04205     generate_inlined_type_test(jitter, app, scheme_integer_type, scheme_double_type, for_branch, branch_short, need_sync);
04206     return 1;
04207   } else if (IS_NAMED_PRIM(rator, "exact-integer?")) {
04208     generate_inlined_type_test(jitter, app, scheme_integer_type, scheme_bignum_type, for_branch, branch_short, need_sync);
04209     return 1;
04210   } else if (IS_NAMED_PRIM(rator, "fixnum?")) {
04211     generate_inlined_type_test(jitter, app, scheme_integer_type, scheme_integer_type, for_branch, branch_short, need_sync);
04212     return 1;
04213   } else if (IS_NAMED_PRIM(rator, "inexact-real?")) {
04214     generate_inlined_type_test(jitter, app, SCHEME_FLOAT_TYPE, scheme_double_type, for_branch, branch_short, need_sync);
04215     return 1;
04216   } else if (IS_NAMED_PRIM(rator, "procedure?")) {
04217     generate_inlined_type_test(jitter, app, scheme_prim_type, scheme_native_closure_type, for_branch, branch_short, need_sync);
04218     return 1;
04219   } else if (IS_NAMED_PRIM(rator, "vector?")) {
04220     generate_inlined_type_test(jitter, app, scheme_vector_type, scheme_vector_type, for_branch, branch_short, need_sync);
04221     return 1;
04222   } else if (IS_NAMED_PRIM(rator, "box?")) {
04223     generate_inlined_type_test(jitter, app, scheme_box_type, scheme_box_type, for_branch, branch_short, need_sync);
04224     return 1;
04225   } else if (IS_NAMED_PRIM(rator, "string?")) {
04226     generate_inlined_type_test(jitter, app, scheme_char_string_type, scheme_char_string_type, for_branch, branch_short, need_sync);
04227     return 1;
04228   } else if (IS_NAMED_PRIM(rator, "bytes?")) {
04229     generate_inlined_type_test(jitter, app, scheme_byte_string_type, scheme_byte_string_type, for_branch, branch_short, need_sync);
04230     return 1;
04231   } else if (IS_NAMED_PRIM(rator, "eof-object?")) {
04232     generate_inlined_constant_test(jitter, app, scheme_eof, NULL, for_branch, branch_short, need_sync);
04233     return 1;
04234   } else if (IS_NAMED_PRIM(rator, "zero?")) {
04235     generate_arith(jitter, rator, app->rand, NULL, 1, 0, 0, 0, for_branch, branch_short);
04236     return 1;
04237   } else if (IS_NAMED_PRIM(rator, "negative?")) {
04238     generate_arith(jitter, rator, app->rand, NULL, 1, 0, -2, 0, for_branch, branch_short);
04239     return 1;
04240   } else if (IS_NAMED_PRIM(rator, "positive?")) {
04241     generate_arith(jitter, rator, app->rand, NULL, 1, 0, 2, 0, for_branch, branch_short);
04242     return 1;
04243   } else if (IS_NAMED_PRIM(rator, "exact-nonnegative-integer?")
04244              || IS_NAMED_PRIM(rator, "exact-positive-integer?")) {
04245     GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref4;
04246     
04247     LOG_IT(("inlined exact-nonnegative-integer?\n"));
04248 
04249     mz_runstack_skipped(jitter, 1);
04250     
04251     generate_non_tail(app->rand, jitter, 0, 1);
04252     CHECK_LIMIT();
04253 
04254     mz_runstack_unskipped(jitter, 1);
04255 
04256     if (need_sync) mz_rs_sync();
04257 
04258     /* Jump ahead if it's a fixnum: */
04259     __START_TINY_JUMPS__(1);
04260     ref = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1);
04261     __END_TINY_JUMPS__(1);
04262 
04263     /* Check for positive bignum: */
04264     __START_SHORT_JUMPS__(branch_short);
04265     jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
04266     ref2 = jit_bnei_p(jit_forward(), JIT_R2, scheme_bignum_type);
04267     jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Stx *)0x0)->iso));
04268     ref3 = jit_bmci_ul(jit_forward(), JIT_R2, 0x1);
04269     __END_SHORT_JUMPS__(branch_short);
04270     /* Ok bignum. Instead of jumping, install the fixnum 1: */
04271     (void)jit_movi_p(JIT_R0, scheme_make_integer(1));
04272 
04273     __START_TINY_JUMPS__(1);
04274     mz_patch_branch(ref);
04275     __END_TINY_JUMPS__(1);
04276     
04277     /* Check whether the fixnum is in range: */
04278     __START_SHORT_JUMPS__(branch_short);
04279     jit_rshi_l(JIT_R0, JIT_R0, 0x1);
04280     if (IS_NAMED_PRIM(rator, "exact-nonnegative-integer?")) {
04281       ref4 = jit_blti_l(jit_forward(), JIT_R0, 0);
04282     } else {
04283       ref4 = jit_blei_l(jit_forward(), JIT_R0, 0);
04284     }
04285 
04286     /* Ok --- it's in range */
04287     
04288     if (for_branch) {
04289       for_branch[0] = ref2;
04290       for_branch[1] = ref3;
04291       for_branch[3] = ref4;
04292     } else {
04293       (void)jit_movi_p(JIT_R0, scheme_true);
04294       ref = jit_jmpi(jit_forward());
04295       mz_patch_branch(ref2);
04296       mz_patch_branch(ref3);
04297       mz_patch_branch(ref4);
04298       (void)jit_movi_p(JIT_R0, scheme_false);
04299       mz_patch_ucbranch(ref);
04300     }
04301     
04302     __END_SHORT_JUMPS__(branch_short);
04303 
04304     return 1;
04305   } else if (!for_branch) {
04306     if (IS_NAMED_PRIM(rator, "car")
04307        || IS_NAMED_PRIM(rator, "cdr")
04308        || IS_NAMED_PRIM(rator, "cadr")
04309        || IS_NAMED_PRIM(rator, "cdar")
04310        || IS_NAMED_PRIM(rator, "caar")
04311        || IS_NAMED_PRIM(rator, "cddr")) {
04312 #     define MAX_LEVELS 2
04313       GC_CAN_IGNORE jit_insn *reffail = NULL, *ref;
04314       int steps, i;
04315       const char *name = ((Scheme_Primitive_Proc *)rator)->name;
04316 
04317       LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)rator)->name));
04318 
04319       for (steps = 0; name[steps+1] != 'r'; steps++) {
04320       }
04321 
04322       mz_runstack_skipped(jitter, 1);
04323 
04324       generate_non_tail(app->rand, jitter, 0, 1);
04325       CHECK_LIMIT();
04326 
04327       mz_runstack_unskipped(jitter, 1);
04328 
04329       mz_rs_sync_fail_branch();
04330 
04331       __START_TINY_JUMPS__(1);
04332 
04333       if (steps > 1) {
04334         jit_movr_p(JIT_R2, JIT_R0); /* save original argument */
04335       }
04336       for (i = 0; i < steps; i++) {
04337         if (!skip_checks) {
04338           if (!i) {
04339             ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04340             reffail = _jit.x.pc;
04341             __END_TINY_JUMPS__(1);
04342             if (steps == 1) {
04343               if (name[1] == 'a') {
04344                 (void)jit_calli(bad_car_code);
04345               } else {
04346                 (void)jit_calli(bad_cdr_code);
04347               }
04348             } else {
04349               if (name[1] == 'a') {
04350                 if (name[2] == 'a') {
04351                   (void)jit_calli(bad_caar_code);
04352                 } else {
04353                   (void)jit_calli(bad_cadr_code);
04354                 }
04355               } else {
04356                 if (name[2] == 'a') {
04357                   (void)jit_calli(bad_cdar_code);
04358                 } else {
04359                   (void)jit_calli(bad_cddr_code);
04360                 }
04361               }
04362             }
04363             __START_TINY_JUMPS__(1);
04364             mz_patch_branch(ref);
04365           } else {
04366             (void)jit_bmsi_ul(reffail, JIT_R0, 0x1);
04367           }
04368           jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type);
04369           (void)jit_bnei_i(reffail, JIT_R1, scheme_pair_type);
04370         } else {
04371           reffail = NULL;
04372         }
04373         if (name[steps - i] == 'a') {
04374           (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Simple_Object *)0x0)->u.pair_val.car);
04375         } else {
04376           (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Simple_Object *)0x0)->u.pair_val.cdr);
04377         }
04378         VALIDATE_RESULT(JIT_R0);
04379         CHECK_LIMIT();
04380       }
04381       __END_TINY_JUMPS__(1);
04382 
04383       return 1;
04384     } else if (IS_NAMED_PRIM(rator, "mcar")
04385                || IS_NAMED_PRIM(rator, "mcdr")) {
04386       GC_CAN_IGNORE jit_insn *reffail = NULL, *ref;
04387       const char *name = ((Scheme_Primitive_Proc *)rator)->name;
04388 
04389       LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)rator)->name));
04390 
04391       mz_runstack_skipped(jitter, 1);
04392 
04393       generate_non_tail(app->rand, jitter, 0, 1);
04394       CHECK_LIMIT();
04395 
04396       mz_runstack_unskipped(jitter, 1);
04397 
04398       mz_rs_sync_fail_branch();
04399 
04400       __START_TINY_JUMPS__(1);
04401 
04402       ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04403       reffail = _jit.x.pc;
04404       __END_TINY_JUMPS__(1);
04405       if (name[2] == 'a') {
04406         (void)jit_calli(bad_mcar_code);
04407       } else {
04408         (void)jit_calli(bad_mcdr_code);
04409       }
04410       __START_TINY_JUMPS__(1);
04411       mz_patch_branch(ref);
04412       jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type);
04413       (void)jit_bnei_i(reffail, JIT_R1, scheme_mutable_pair_type);
04414       if (name[2] == 'a') {
04415         (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Simple_Object *)0x0)->u.pair_val.car);
04416       } else {
04417         (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Simple_Object *)0x0)->u.pair_val.cdr);
04418       }
04419       VALIDATE_RESULT(JIT_R0);
04420       CHECK_LIMIT();
04421       __END_TINY_JUMPS__(1);
04422 
04423       return 1;
04424     } else if (IS_NAMED_PRIM(rator, "vector-length")) {
04425       GC_CAN_IGNORE jit_insn *reffail, *ref;
04426 
04427       LOG_IT(("inlined vector-length\n"));
04428 
04429       mz_runstack_skipped(jitter, 1);
04430 
04431       generate_non_tail(app->rand, jitter, 0, 1);
04432       CHECK_LIMIT();
04433 
04434       mz_runstack_unskipped(jitter, 1);
04435 
04436       mz_rs_sync_fail_branch();
04437 
04438       __START_TINY_JUMPS__(1);
04439       ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04440       __END_TINY_JUMPS__(1);
04441 
04442       reffail = _jit.x.pc;
04443       (void)jit_calli(bad_vector_length_code);
04444 
04445       __START_TINY_JUMPS__(1);
04446       mz_patch_branch(ref);
04447       jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type);
04448       (void)jit_bnei_i(reffail, JIT_R1, scheme_vector_type);
04449       __END_TINY_JUMPS__(1);
04450 
04451       (void)jit_ldxi_i(JIT_R0, JIT_R0, &SCHEME_VEC_SIZE(0x0));
04452       jit_lshi_l(JIT_R0, JIT_R0, 1);
04453       jit_ori_l(JIT_R0, JIT_R0, 0x1);
04454             
04455       return 1;
04456     } else if (IS_NAMED_PRIM(rator, "unbox")) {
04457       GC_CAN_IGNORE jit_insn *reffail, *ref;
04458 
04459       LOG_IT(("inlined unbox\n"));
04460 
04461       mz_runstack_skipped(jitter, 1);
04462 
04463       generate_non_tail(app->rand, jitter, 0, 1);
04464       CHECK_LIMIT();
04465 
04466       mz_runstack_unskipped(jitter, 1);
04467 
04468       mz_rs_sync_fail_branch();
04469 
04470       __START_TINY_JUMPS__(1);
04471       ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04472       __END_TINY_JUMPS__(1);
04473 
04474       reffail = _jit.x.pc;
04475       (void)jit_calli(bad_unbox_code);
04476 
04477       __START_TINY_JUMPS__(1);
04478       mz_patch_branch(ref);
04479       jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type);
04480       (void)jit_bnei_i(reffail, JIT_R1, scheme_box_type);
04481       __END_TINY_JUMPS__(1);
04482 
04483       (void)jit_ldxi_p(JIT_R0, JIT_R0, &SCHEME_BOX_VAL(0x0));
04484       
04485       return 1;
04486     } else if (IS_NAMED_PRIM(rator, "syntax-e")) {
04487       LOG_IT(("inlined syntax-e\n"));
04488 
04489       mz_runstack_skipped(jitter, 1);
04490 
04491       generate_non_tail(app->rand, jitter, 0, 1);
04492       CHECK_LIMIT();
04493 
04494       mz_runstack_unskipped(jitter, 1);
04495 
04496       mz_rs_sync();
04497 
04498       (void)jit_calli(syntax_e_code);
04499       
04500       return 1;
04501     } else if (IS_NAMED_PRIM(rator, "add1")) {
04502       generate_arith(jitter, rator, app->rand, NULL, 1, 1, 0, 1, NULL, 1);
04503       return 1;
04504     } else if (IS_NAMED_PRIM(rator, "sub1")) {
04505       generate_arith(jitter, rator, app->rand, NULL, 1, -1, 0, 1, NULL, 1);
04506       return 1;
04507     } else if (IS_NAMED_PRIM(rator, "-")) {
04508       generate_arith(jitter, rator, app->rand, NULL, 1, -1, 0, 0, NULL, 1);
04509       return 1;
04510     } else if (IS_NAMED_PRIM(rator, "abs")) {
04511       generate_arith(jitter, rator, app->rand, NULL, 1, 11, 0, 0, NULL, 1);
04512       return 1;
04513     } else if (IS_NAMED_PRIM(rator, "bitwise-not")) {
04514       generate_arith(jitter, rator, app->rand, NULL, 1, 7, 0, 9, NULL, 1);
04515       return 1;
04516     } else if (IS_NAMED_PRIM(rator, "vector-immutable")
04517                || IS_NAMED_PRIM(rator, "vector")) {
04518       return generate_vector_alloc(jitter, rator, NULL, app, NULL);
04519     } else if (IS_NAMED_PRIM(rator, "list*")) {
04520       /* on a single argument, `list*' is identity */
04521       mz_runstack_skipped(jitter, 1);
04522       generate_non_tail(app->rand, jitter, 0, 1);
04523       CHECK_LIMIT();
04524       mz_runstack_unskipped(jitter, 1);
04525       return 1;
04526     } else if (IS_NAMED_PRIM(rator, "list")) {
04527       mz_runstack_skipped(jitter, 1);
04528       generate_non_tail(app->rand, jitter, 0, 1);
04529       CHECK_LIMIT();
04530       mz_rs_sync();
04531       mz_runstack_unskipped(jitter, 1);
04532       (void)jit_movi_p(JIT_R1, &scheme_null);
04533       return generate_cons_alloc(jitter, 0, 0);
04534     } else if (IS_NAMED_PRIM(rator, "box")) {
04535       mz_runstack_skipped(jitter, 1);
04536       generate_non_tail(app->rand, jitter, 0, 1);
04537       CHECK_LIMIT();
04538       mz_runstack_unskipped(jitter, 1);
04539       mz_rs_sync();
04540       
04541 #ifdef CAN_INLINE_ALLOC
04542       /* Inlined alloc */
04543       (void)jit_movi_p(JIT_R1, NULL); /* needed because R1 is marked during a GC */
04544       inline_alloc(jitter, sizeof(Scheme_Small_Object), scheme_box_type, 0, 1, 0, 0);
04545       CHECK_LIMIT();
04546       
04547       jit_stxi_p((long)&SCHEME_BOX_VAL(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R0);
04548       jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
04549 #else
04550       /* Non-inlined */
04551       JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
04552       mz_prepare(1);
04553       jit_pusharg_p(JIT_R0);
04554       (void)mz_finish(scheme_box);
04555       jit_retval(JIT_R0);
04556 #endif
04557 
04558       return 1;
04559     }
04560   }
04561 
04562   if (!for_branch) {
04563     scheme_console_printf("Inlining expected.\n");
04564     abort();
04565   }
04566 
04567   --scheme_direct_call_count;
04568 
04569   return 0;
04570 }
04571 
04572 static int generate_two_args(Scheme_Object *rand1, Scheme_Object *rand2, mz_jit_state *jitter, 
04573                              int order_matters, int skipped)
04574 /* de-sync's rs.
04575    Results go into R0 and R1. If !order_matters, and if only the
04576    second is simple, then the arguments will be in reverse order. */
04577 {
04578   int simple1, simple2, direction = 1;
04579   
04580   simple1 = is_constant_and_avoids_r1(rand1);
04581   simple2 = is_constant_and_avoids_r1(rand2);
04582 
04583   if (!simple1) {
04584     if (simple2) {
04585       mz_runstack_skipped(jitter, skipped);
04586 
04587       generate_non_tail(rand1, jitter, 0, 1); /* no sync... */
04588       CHECK_LIMIT();
04589       jit_movr_p(JIT_R1, JIT_R0);
04590 
04591       generate(rand2, jitter, 0, 0, JIT_R0); /* no sync... */
04592       CHECK_LIMIT();
04593 
04594       if (order_matters) {
04595         /* Swap arguments: */
04596         jit_movr_p(JIT_R2, JIT_R0);
04597         jit_movr_p(JIT_R0, JIT_R1);
04598         jit_movr_p(JIT_R1, JIT_R2);
04599       } else
04600         direction = -1;
04601 
04602       mz_runstack_unskipped(jitter, skipped);
04603     } else {
04604       mz_runstack_skipped(jitter, skipped);
04605       generate_non_tail(rand1, jitter, 0, 1); /* no sync... */
04606       CHECK_LIMIT();
04607       mz_runstack_unskipped(jitter, skipped);
04608 
04609       mz_rs_dec(1);
04610       CHECK_RUNSTACK_OVERFLOW();
04611       mz_runstack_pushed(jitter, 1);
04612       mz_rs_str(JIT_R0);
04613       mz_runstack_skipped(jitter, skipped-1);
04614 
04615       generate_non_tail(rand2, jitter, 0, 1); /* no sync... */
04616       CHECK_LIMIT();
04617 
04618       jit_movr_p(JIT_R1, JIT_R0);
04619       mz_rs_ldr(JIT_R0);
04620 
04621       mz_runstack_unskipped(jitter, skipped-1);
04622       mz_rs_inc(1);
04623       mz_runstack_popped(jitter, 1);
04624     }
04625   } else {
04626     mz_runstack_skipped(jitter, skipped);
04627 
04628     if (simple2) {
04629       generate(rand2, jitter, 0, 0, JIT_R1); /* no sync... */
04630       CHECK_LIMIT();
04631     } else {
04632       generate_non_tail(rand2, jitter, 0, 1); /* no sync... */
04633       CHECK_LIMIT();
04634       jit_movr_p(JIT_R1, JIT_R0);
04635     }
04636 
04637     generate(rand1, jitter, 0, 0, JIT_R0); /* no sync... */
04638     CHECK_LIMIT();
04639 
04640     mz_runstack_unskipped(jitter, skipped);
04641   }
04642 
04643   return direction;
04644 }
04645 
04646 static int generate_binary_char(mz_jit_state *jitter, Scheme_App3_Rec *app,
04647                                 jit_insn **for_branch, int branch_short)
04648 /* de-sync'd ok */
04649 {
04650   Scheme_Object *r1, *r2, *rator = app->rator;
04651   GC_CAN_IGNORE jit_insn *reffail = NULL, *ref;
04652   int direct = 0, direction;
04653 
04654   LOG_IT(("inlined %s\n", ((Scheme_Primitive_Proc *)rator)->name));
04655 
04656   r1 = app->rand1;
04657   r2 = app->rand2;
04658   direction = generate_two_args(r1, r2, jitter, 1, 2);
04659   CHECK_LIMIT();
04660 
04661   mz_rs_sync();
04662 
04663   __START_SHORT_JUMPS__(branch_short);
04664   
04665   if (!SCHEME_CHARP(r1)) {
04666     GC_CAN_IGNORE jit_insn *pref;
04667     pref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04668     reffail = _jit.x.pc;
04669     (void)jit_movi_p(JIT_R2, ((Scheme_Primitive_Proc *)rator)->prim_val);
04670     __END_SHORT_JUMPS__(branch_short);
04671     if (direction > 0) {
04672       (void)jit_jmpi(call_original_binary_rev_arith_code);
04673     } else {
04674       (void)jit_jmpi(call_original_binary_arith_code);
04675     }
04676     __START_SHORT_JUMPS__(branch_short);
04677     mz_patch_branch(pref);
04678     jit_ldxi_s(JIT_R2, JIT_R0, (int)&((Scheme_Object *)0x0)->type);
04679     (void)jit_bnei_i(reffail, JIT_R2, scheme_char_type);
04680     CHECK_LIMIT();
04681   } else {
04682     if (!direct)
04683       direct = (SCHEME_CHAR_VAL(r1) < 256);
04684   }
04685   if (!SCHEME_CHARP(r2)) {
04686     if (!reffail) {
04687       GC_CAN_IGNORE jit_insn *pref;
04688       pref = jit_bmci_ul(jit_forward(), JIT_R1, 0x1);
04689       reffail = _jit.x.pc;
04690       (void)jit_movi_p(JIT_R2, ((Scheme_Primitive_Proc *)rator)->prim_val);
04691       __END_SHORT_JUMPS__(branch_short);
04692       if (direction > 0) {
04693         (void)jit_jmpi(call_original_binary_rev_arith_code);
04694       } else {
04695         (void)jit_jmpi(call_original_binary_arith_code);
04696       }
04697       __START_SHORT_JUMPS__(branch_short);
04698       mz_patch_branch(pref);
04699     } else {
04700       (void)jit_bmsi_ul(reffail, JIT_R1, 0x1);
04701     }
04702     jit_ldxi_s(JIT_R2, JIT_R1, (int)&((Scheme_Object *)0x0)->type);
04703     (void)jit_bnei_i(reffail, JIT_R2, scheme_char_type);
04704     CHECK_LIMIT();
04705   } else {
04706     if (!direct)
04707       direct = (SCHEME_CHAR_VAL(r2) < 256);
04708   }
04709 
04710   if (!direct) {
04711     /* Extract character value */
04712     jit_ldxi_i(JIT_R0, JIT_R0, (int)&SCHEME_CHAR_VAL((Scheme_Object *)0x0));
04713     jit_ldxi_i(JIT_R1, JIT_R1, (int)&SCHEME_CHAR_VAL((Scheme_Object *)0x0));
04714     ref = jit_bner_i(jit_forward(), JIT_R0, JIT_R1);
04715   } else {
04716     ref = jit_bner_p(jit_forward(), JIT_R0, JIT_R1);
04717   }
04718   CHECK_LIMIT();
04719   if (for_branch) {
04720     for_branch[0] = ref;
04721   } else {
04722     GC_CAN_IGNORE jit_insn *ref2;
04723     (void)jit_movi_p(JIT_R0, scheme_true);
04724     ref2 = jit_jmpi(jit_forward());
04725     mz_patch_branch(ref);
04726     (void)jit_movi_p(JIT_R0, scheme_false);
04727     mz_patch_ucbranch(ref2);
04728   }
04729     
04730   __END_SHORT_JUMPS__(branch_short);
04731 
04732   return 1;
04733 }
04734 
04735 static int generate_vector_op(mz_jit_state *jitter, int set, int int_ready)
04736 /* if int_ready, JIT_R1 has num index and JIT_V1 has pre-computed offset,
04737    otherwise JIT_R1 has fixnum index */
04738 {
04739   GC_CAN_IGNORE jit_insn *ref, *reffail;
04740 
04741   if (!skip_checks) {
04742     __START_TINY_JUMPS__(1);
04743     ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
04744     __END_TINY_JUMPS__(1);
04745 
04746     reffail = _jit.x.pc;
04747     if (int_ready) {
04748       jit_lshi_ul(JIT_R1, JIT_R1, 1);
04749       jit_ori_l(JIT_R1, JIT_R1, 0x1);
04750     }
04751     if (set) {
04752       (void)jit_calli(vector_set_check_index_code);
04753     } else {
04754       (void)jit_calli(vector_ref_check_index_code);
04755     }
04756     /* doesn't return */
04757     CHECK_LIMIT();
04758 
04759     __START_TINY_JUMPS__(1);
04760     mz_patch_branch(ref);
04761     if (!int_ready)
04762       (void)jit_bmci_ul(reffail, JIT_R1, 0x1);
04763     jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
04764     (void)jit_bnei_i(reffail, JIT_R2, scheme_vector_type);
04765     jit_ldxi_i(JIT_R2, JIT_R0, (int)&SCHEME_VEC_SIZE(0x0));
04766     if (!int_ready) {
04767       jit_rshi_ul(JIT_V1, JIT_R1, 1);
04768       (void)jit_bler_ul(reffail, JIT_R2, JIT_V1);
04769     } else {
04770       (void)jit_bler_ul(reffail, JIT_R2, JIT_R1);
04771     }
04772     CHECK_LIMIT();
04773     __END_TINY_JUMPS__(1);
04774   } else {
04775     if (!int_ready)
04776       jit_rshi_ul(JIT_V1, JIT_R1, 1);
04777   }
04778 
04779   if (!int_ready) {
04780     jit_lshi_ul(JIT_V1, JIT_V1, JIT_LOG_WORD_SIZE);
04781     jit_addi_p(JIT_V1, JIT_V1, (int)&SCHEME_VEC_ELS(0x0));
04782   }
04783   if (set) {
04784     jit_ldr_p(JIT_R2, JIT_RUNSTACK);
04785     jit_stxr_p(JIT_V1, JIT_R0, JIT_R2);
04786     (void)jit_movi_p(JIT_R0, scheme_void);
04787   } else {
04788     jit_ldxr_p(JIT_R0, JIT_R0, JIT_V1);
04789   }
04790 
04791   return 1;
04792 }
04793 
04794 static int generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, int is_tail, int multi_ok, 
04795                                jit_insn **for_branch, int branch_short, int need_sync)
04796 /* de-sync's; for branch, sync'd before  */
04797 {
04798   Scheme_Object *rator = app->rator;
04799 
04800   if (!for_branch
04801       && inlineable_struct_prim(rator, jitter, 2, 2)) {
04802     generate_inlined_struct_op(3, jitter, rator, app->rand1, app->rand2, for_branch, branch_short, multi_ok);
04803     scheme_direct_call_count++;
04804     return 1;
04805   }
04806 
04807 
04808   if (!SCHEME_PRIMP(rator))
04809     return 0;
04810 
04811   if (!(SCHEME_PRIM_PROC_FLAGS(rator) & SCHEME_PRIM_IS_BINARY_INLINED))
04812     return 0;
04813 
04814   scheme_direct_call_count++;
04815 
04816   if (IS_NAMED_PRIM(rator, "eq?")) {
04817     Scheme_Object *a1, *a2;
04818     GC_CAN_IGNORE jit_insn *ref, *ref2;
04819 
04820     LOG_IT(("inlined eq?\n"));
04821     
04822     a1 = app->rand1;
04823     if (SCHEME_TYPE(a1) > _scheme_values_types_) {
04824       a2 = app->rand2;
04825     } else {
04826       a1 = app->rand2;
04827       a2 = app->rand1;
04828     }
04829 
04830     if (SCHEME_TYPE(a1) > _scheme_values_types_) {
04831       /* Compare to constant: */
04832       int retptr;
04833 
04834       mz_runstack_skipped(jitter, 2);
04835 
04836       generate_non_tail(a2, jitter, 0, 1);
04837       CHECK_LIMIT();
04838       if (need_sync) mz_rs_sync();
04839       
04840       mz_runstack_unskipped(jitter, 2);
04841 
04842       if (!SCHEME_INTP(a1)
04843          && !SCHEME_FALSEP(a1)
04844          && !SCHEME_VOIDP(a1)
04845          && !SAME_OBJ(a1, scheme_true))
04846        retptr = mz_retain(a1);
04847       else
04848        retptr = 0;
04849       
04850       __START_SHORT_JUMPS__(branch_short);
04851       
04852 #ifdef JIT_PRECISE_GC
04853       if (retptr) {
04854        mz_load_retained(jitter, JIT_R1, retptr);
04855        ref = jit_bner_p(jit_forward(), JIT_R0, JIT_R1);
04856       } else
04857 #endif
04858        ref = mz_bnei_p(jit_forward(), JIT_R0, a1);
04859 
04860       if (for_branch) {
04861        for_branch[0] = ref;
04862       } else {
04863        (void)jit_movi_p(JIT_R0, scheme_true);
04864        ref2 = jit_jmpi(jit_forward());
04865        mz_patch_branch(ref);
04866        (void)jit_movi_p(JIT_R0, scheme_false);
04867        mz_patch_ucbranch(ref2);
04868       }
04869       
04870       __END_SHORT_JUMPS__(branch_short);
04871     } else {
04872       /* Two complex expressions: */
04873       generate_two_args(a2, a1, jitter, 0, 2);
04874       CHECK_LIMIT();
04875 
04876       if (need_sync) mz_rs_sync();
04877 
04878       __START_SHORT_JUMPS__(branch_short);
04879 
04880       ref = jit_bner_p(jit_forward(), JIT_R0, JIT_R1);
04881       if (for_branch) {
04882        for_branch[0] = ref;
04883       } else {
04884        (void)jit_movi_p(JIT_R0, scheme_true);
04885        ref2 = jit_jmpi(jit_forward());
04886        mz_patch_branch(ref);
04887        (void)jit_movi_p(JIT_R0, scheme_false);
04888        mz_patch_ucbranch(ref2);
04889       }
04890       
04891       __END_SHORT_JUMPS__(branch_short);      
04892     }
04893     
04894     return 1;
04895   } else if (IS_NAMED_PRIM(rator, "=")) {
04896     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, 0, 0, for_branch, branch_short);
04897     return 1;
04898   } else if (IS_NAMED_PRIM(rator, "<=")) {
04899     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, -1, 0, for_branch, branch_short);
04900     return 1;
04901   } else if (IS_NAMED_PRIM(rator, "<")) {
04902     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, -2, 0, for_branch, branch_short);
04903     return 1;
04904   } else if (IS_NAMED_PRIM(rator, ">=")) {
04905     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, 1, 0, for_branch, branch_short);
04906     return 1;
04907   } else if (IS_NAMED_PRIM(rator, ">")) {
04908     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, 2, 0, for_branch, branch_short);
04909     return 1;
04910   } else if (IS_NAMED_PRIM(rator, "bitwise-bit-set?")) {
04911     generate_arith(jitter, rator, app->rand1, app->rand2, 2, 0, 3, 0, for_branch, branch_short);
04912     return 1;
04913   } else if (IS_NAMED_PRIM(rator, "char=?")) {
04914     generate_binary_char(jitter, app, for_branch, branch_short);
04915     return 1;
04916   } else if (!for_branch) {
04917     if (IS_NAMED_PRIM(rator, "+")) {
04918       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 1, 0, 0, NULL, 1);
04919       return 1;
04920     } else if (IS_NAMED_PRIM(rator, "-")) {
04921       generate_arith(jitter, rator, app->rand1, app->rand2, 2, -1, 0, 0, NULL, 1);
04922       return 1;
04923     } else if (IS_NAMED_PRIM(rator, "*")) {
04924       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 2, 0, 0, NULL, 1);
04925       return 1;
04926     } else if (IS_NAMED_PRIM(rator, "/")) {
04927       generate_arith(jitter, rator, app->rand1, app->rand2, 2, -2, 0, 0, NULL, 1);
04928       return 1;
04929     } else if (IS_NAMED_PRIM(rator, "min")) {
04930       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 9, 0, 0, NULL, 1);
04931       return 1;
04932     } else if (IS_NAMED_PRIM(rator, "max")) {
04933       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 10, 0, 0, NULL, 1);
04934       return 1;
04935     } else if (IS_NAMED_PRIM(rator, "bitwise-and")) {
04936       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 3, 0, 0, NULL, 1);
04937       return 1;
04938     } else if (IS_NAMED_PRIM(rator, "bitwise-ior")) {
04939       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 4, 0, 0, NULL, 1);
04940       return 1;
04941     } else if (IS_NAMED_PRIM(rator, "bitwise-xor")) {
04942       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 5, 0, 0, NULL, 1);
04943       return 1;
04944     } else if (IS_NAMED_PRIM(rator, "arithmetic-shift")) {
04945       generate_arith(jitter, rator, app->rand1, app->rand2, 2, 6, 0, 0, NULL, 1);
04946       return 1;
04947     } else if (IS_NAMED_PRIM(rator, "vector-ref")
04948               || IS_NAMED_PRIM(rator, "string-ref")
04949               || IS_NAMED_PRIM(rator, "bytes-ref")) {
04950       int simple;
04951       int which;
04952 
04953       if (IS_NAMED_PRIM(rator, "vector-ref"))
04954        which = 0;
04955       else if (IS_NAMED_PRIM(rator, "string-ref"))
04956        which = 1;
04957       else
04958        which = 2;
04959 
04960       LOG_IT(("inlined vector-/string-/bytes-ref\n"));
04961 
04962       simple = (SCHEME_INTP(app->rand2)
04963               && (SCHEME_INT_VAL(app->rand2) >= 0));
04964 
04965       if (!simple) {
04966         generate_two_args(app->rand1, app->rand2, jitter, 1, 2);
04967         CHECK_LIMIT();
04968 
04969         mz_rs_sync();
04970 
04971         if (!which) {
04972           /* vector-ref is relatively simple and worth inlining */
04973           generate_vector_op(jitter, 0, 0);
04974           CHECK_LIMIT();
04975        } else if (which == 1) {
04976          (void)jit_calli(string_ref_check_index_code);
04977        } else {
04978          (void)jit_calli(bytes_ref_check_index_code);
04979        }
04980       } else {
04981        long offset;
04982 
04983         mz_runstack_skipped(jitter, 2);
04984       
04985         generate_non_tail(app->rand1, jitter, 0, 1);
04986         CHECK_LIMIT();
04987 
04988         mz_rs_sync();
04989       
04990        offset = SCHEME_INT_VAL(app->rand2);
04991        (void)jit_movi_p(JIT_R1, offset);
04992        if (!which)
04993          offset = ((int)&SCHEME_VEC_ELS(0x0)) + WORDS_TO_BYTES(offset);
04994        else if (which == 1)
04995          offset = offset << LOG_MZCHAR_SIZE;
04996        jit_movi_l(JIT_V1, offset);
04997        if (!which) {
04998           /* vector-ref is relatively simple and worth inlining */
04999           generate_vector_op(jitter, 0, 1);
05000           CHECK_LIMIT();
05001        } else if (which == 1) {
05002          (void)jit_calli(string_ref_code);
05003        } else {
05004          (void)jit_calli(bytes_ref_code);
05005        }
05006 
05007         mz_runstack_unskipped(jitter, 2);
05008       }
05009 
05010       return 1;
05011     } else if (IS_NAMED_PRIM(rator, "set-mcar!")
05012                || IS_NAMED_PRIM(rator, "set-mcdr!")) {
05013       GC_CAN_IGNORE jit_insn *reffail, *ref;
05014       int set_mcar;
05015 
05016       set_mcar = IS_NAMED_PRIM(rator, "set-mcar!");
05017 
05018       LOG_IT(("inlined set-mcar!\n"));
05019 
05020       generate_two_args(app->rand1, app->rand2, jitter, 1, 2);
05021       CHECK_LIMIT();
05022       mz_rs_sync_fail_branch();
05023 
05024       __START_TINY_JUMPS__(1);
05025       ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
05026       reffail = _jit.x.pc;
05027       __END_TINY_JUMPS__(1);
05028       if (set_mcar)
05029         (void)jit_calli(bad_set_mcar_code);
05030       else
05031         (void)jit_calli(bad_set_mcdr_code);
05032       __START_TINY_JUMPS__(1);
05033       mz_patch_branch(ref);
05034       jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
05035       (void)jit_bnei_i(reffail, JIT_R2, scheme_mutable_pair_type);
05036       __END_TINY_JUMPS__(1);
05037       CHECK_LIMIT();
05038 
05039       if (set_mcar)
05040         (void)jit_stxi_p(&((Scheme_Simple_Object *)0x0)->u.pair_val.car, JIT_R0, JIT_R1);
05041       else
05042         (void)jit_stxi_p(&((Scheme_Simple_Object *)0x0)->u.pair_val.cdr, JIT_R0, JIT_R1);
05043       
05044       (void)jit_movi_p(JIT_R0, scheme_void);
05045 
05046       return 1;
05047     } else if (IS_NAMED_PRIM(rator, "cons")
05048                || IS_NAMED_PRIM(rator, "list*")) {
05049       LOG_IT(("inlined cons\n"));
05050 
05051       generate_two_args(app->rand1, app->rand2, jitter, 1, 2);
05052       CHECK_LIMIT();
05053       mz_rs_sync();
05054 
05055       return generate_cons_alloc(jitter, 0, 0);
05056     } else if (IS_NAMED_PRIM(rator, "mcons")) {
05057       LOG_IT(("inlined mcons\n"));
05058 
05059       generate_two_args(app->rand1, app->rand2, jitter, 1, 2);
05060       CHECK_LIMIT();
05061       mz_rs_sync();
05062 
05063 #ifdef CAN_INLINE_ALLOC
05064       /* Inlined alloc */
05065       inline_alloc(jitter, sizeof(Scheme_Simple_Object), scheme_mutable_pair_type, 0, 1, 0, 0);
05066       CHECK_LIMIT();
05067 
05068       jit_stxi_p((long)&SCHEME_MCAR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R0);
05069       jit_stxi_p((long)&SCHEME_MCDR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R1);
05070       jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
05071 #else
05072       /* Non-inlined alloc */
05073       JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05074       mz_prepare(2);
05075       jit_pusharg_p(JIT_R1);
05076       jit_pusharg_p(JIT_R0);
05077       (void)mz_finish(scheme_make_mutable_pair);
05078       jit_retval(JIT_R0);
05079 #endif
05080 
05081       return 1;
05082     } else if (IS_NAMED_PRIM(rator, "list")) {
05083       LOG_IT(("inlined list\n"));
05084 
05085       generate_two_args(app->rand1, app->rand2, jitter, 1, 2);
05086       CHECK_LIMIT();
05087 
05088       mz_rs_dec(1);
05089       CHECK_RUNSTACK_OVERFLOW();
05090       mz_runstack_pushed(jitter, 1);
05091       mz_rs_str(JIT_R0);
05092       (void)jit_movi_p(JIT_R0, &scheme_null);
05093       CHECK_LIMIT();
05094       mz_rs_sync();
05095 
05096       generate_cons_alloc(jitter, 1, 0);
05097       CHECK_LIMIT();
05098 
05099       jit_ldr_p(JIT_R1, JIT_RUNSTACK);
05100       jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
05101       mz_runstack_popped(jitter, 1);
05102       CHECK_LIMIT();
05103       
05104       return generate_cons_alloc(jitter, 1, 0);
05105     } else if (IS_NAMED_PRIM(rator, "vector-immutable")
05106                || IS_NAMED_PRIM(rator, "vector")) {
05107       return generate_vector_alloc(jitter, rator, NULL, NULL, app);
05108     }
05109   }
05110 
05111   if (!for_branch) {
05112     scheme_console_printf("Inlining expected.\n");
05113     abort();
05114   }
05115 
05116   --scheme_direct_call_count;
05117 
05118   return 0;
05119 }
05120 
05121 static int generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int is_tail, int multi_ok, 
05122                              jit_insn **for_branch, int branch_short)
05123 /* de-sync's; for branch, sync'd before */
05124 {
05125   Scheme_Object *rator = app->args[0];
05126 
05127   if (!SCHEME_PRIMP(rator))
05128     return 0;
05129 
05130   if (!(SCHEME_PRIM_PROC_FLAGS(rator) & SCHEME_PRIM_IS_NARY_INLINED))
05131     return 0;
05132 
05133   if (app->num_args < ((Scheme_Primitive_Proc *)rator)->mina)
05134     return 0;
05135   if (app->num_args > ((Scheme_Primitive_Proc *)rator)->mu.maxa)
05136     return 0;
05137 
05138   scheme_direct_call_count++;
05139 
05140   if (!for_branch) {
05141     if (IS_NAMED_PRIM(rator, "vector-set!")
05142        || IS_NAMED_PRIM(rator, "string-set!")
05143        || IS_NAMED_PRIM(rator, "bytes-set!")) {
05144       int simple, constval;
05145       int which;
05146       int pushed;
05147 
05148       if (IS_NAMED_PRIM(rator, "vector-set!"))
05149        which = 0;
05150       else if (IS_NAMED_PRIM(rator, "string-set!"))
05151        which = 1;
05152       else
05153        which = 2;
05154 
05155       LOG_IT(("inlined vector-set!\n"));
05156 
05157       simple = (SCHEME_INTP(app->args[2])
05158               && (SCHEME_INT_VAL(app->args[2]) >= 0));
05159 
05160       constval = (SCHEME_TYPE(app->args[3]) > _scheme_values_types_);
05161       
05162       if (constval && simple)
05163        pushed = 1;
05164       else
05165        pushed = 2;
05166 
05167       mz_runstack_skipped(jitter, 3 - pushed);
05168 
05169       if (pushed) {
05170         mz_rs_dec(pushed);
05171         CHECK_RUNSTACK_OVERFLOW();
05172        mz_runstack_pushed(jitter, pushed);
05173         stack_safety(jitter, pushed, 0);
05174         CHECK_LIMIT();
05175       }
05176 
05177       generate_non_tail(app->args[1], jitter, 0, 1); /* sync'd below */
05178       CHECK_LIMIT();
05179       if (!constval || !simple) {
05180        mz_rs_str(JIT_R0);
05181       } else {
05182        jit_movr_p(JIT_V1, JIT_R0);
05183       }
05184 
05185       if (!simple) {
05186        generate_non_tail(app->args[2], jitter, 0, 1); /* sync'd below */
05187        CHECK_LIMIT();
05188        if (!constval) {
05189          mz_rs_stxi(1, JIT_R0);
05190        } else {
05191          jit_movr_p(JIT_R1, JIT_R0);
05192        }
05193       }
05194 
05195       generate_non_tail(app->args[3], jitter, 0, 1); /* sync'd below */
05196       CHECK_LIMIT();
05197       mz_rs_sync();
05198  
05199       if (!constval || !simple) {
05200        jit_movr_p(JIT_R2, JIT_R0);
05201        jit_ldr_p(JIT_R0, JIT_RUNSTACK);
05202        jit_str_p(JIT_RUNSTACK, JIT_R2);
05203        if (!simple && !constval) {
05204          jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(1));
05205        }
05206       } else {
05207        jit_str_p(JIT_RUNSTACK, JIT_R0);
05208        jit_movr_p(JIT_R0, JIT_V1);
05209       }
05210 
05211       if (!simple) {
05212        if (!which) {
05213           /* vector-set! is relatively simple and worth inlining */
05214           generate_vector_op(jitter, 1, 0);
05215           CHECK_LIMIT();
05216        } else if (which == 1) {
05217          (void)jit_calli(string_set_check_index_code);
05218        } else {
05219          (void)jit_calli(bytes_set_check_index_code);
05220        }
05221       } else {
05222        long offset;
05223        offset = SCHEME_INT_VAL(app->args[2]);
05224        (void)jit_movi_p(JIT_R1, offset);
05225        if (!which)
05226          offset = ((int)&SCHEME_VEC_ELS(0x0)) + WORDS_TO_BYTES(offset);
05227        else if (which == 1)
05228          offset = offset << LOG_MZCHAR_SIZE;
05229        jit_movi_l(JIT_V1, offset);
05230        if (!which) {
05231           /* vector-set! is relatively simple and worth inlining */
05232           generate_vector_op(jitter, 1, 1);
05233           CHECK_LIMIT();
05234        } else if (which == 1) {
05235          (void)jit_calli(string_set_code);
05236        } else {
05237          (void)jit_calli(bytes_set_code);
05238        }
05239       }
05240       
05241       mz_rs_inc(pushed); /* no sync */
05242       mz_runstack_popped(jitter, pushed);
05243 
05244       mz_runstack_unskipped(jitter, 3 - pushed);
05245 
05246       return 1;
05247     } else if (IS_NAMED_PRIM(rator, "vector-immutable")
05248                || IS_NAMED_PRIM(rator, "vector")) {
05249       return generate_vector_alloc(jitter, rator, app, NULL, NULL);
05250     } else if (IS_NAMED_PRIM(rator, "list")
05251                || IS_NAMED_PRIM(rator, "list*")) {
05252       int c = app->num_args;
05253       int star;
05254 
05255       star = IS_NAMED_PRIM(rator, "list*");
05256 
05257       if (c)
05258         generate_app(app, NULL, c, jitter, 0, 0, 2);
05259       CHECK_LIMIT();
05260       mz_rs_sync();
05261 
05262 #ifdef CAN_INLINE_ALLOC
05263       jit_movi_l(JIT_R2, c);
05264       if (star)
05265         (void)jit_calli(make_list_star_code);
05266       else
05267         (void)jit_calli(make_list_code);
05268 #else
05269       JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05270       jit_movi_l(JIT_R0, c);
05271       mz_prepare(1);
05272       jit_pusharg_l(JIT_R0);
05273       if (star)
05274         (void)mz_finish(make_list_star);
05275       else
05276         (void)mz_finish(make_list);
05277       jit_retval(JIT_R0);
05278 #endif
05279 
05280       if (c) {
05281         mz_rs_inc(c); /* no sync */
05282         mz_runstack_popped(jitter, c);
05283       }
05284 
05285       return 1;
05286     } else if (IS_NAMED_PRIM(rator, "checked-procedure-check-and-extract")) {
05287       generate_app(app, NULL, 5, jitter, 0, 0, 2);  /* sync'd below */
05288       CHECK_LIMIT();
05289       mz_rs_sync();
05290 
05291       (void)jit_calli(struct_proc_extract_code);
05292       CHECK_LIMIT();
05293 
05294       mz_rs_inc(5);
05295       mz_runstack_popped(jitter, 5);
05296 
05297       return 1;
05298     }
05299   }
05300 
05301   if (!for_branch) {
05302     scheme_console_printf("Inlining expected.\n");
05303     abort();
05304   }
05305 
05306   --scheme_direct_call_count;
05307 
05308   return 0;
05309 }
05310 
05311 static int generate_cons_alloc(mz_jit_state *jitter, int rev, int inline_retry)
05312 {
05313   /* Args should be in R0 (car) and R1 (cdr) */
05314 
05315 #ifdef CAN_INLINE_ALLOC
05316   /* Inlined alloc */
05317   inline_alloc(jitter, sizeof(Scheme_Simple_Object), scheme_pair_type, 0, 1, 0, inline_retry);
05318   CHECK_LIMIT();
05319   
05320   if (rev) {
05321     jit_stxi_p((long)&SCHEME_CAR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R1);
05322     jit_stxi_p((long)&SCHEME_CDR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R0);
05323   } else {
05324     jit_stxi_p((long)&SCHEME_CAR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R0);
05325     jit_stxi_p((long)&SCHEME_CDR(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R1);
05326   }
05327   jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
05328 #else
05329   /* Non-inlined */
05330   JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05331   mz_prepare(2);
05332   if (rev) {
05333     jit_pusharg_p(JIT_R0);
05334     jit_pusharg_p(JIT_R1);
05335   } else {
05336     jit_pusharg_p(JIT_R1);
05337     jit_pusharg_p(JIT_R0);
05338   }
05339   (void)mz_finish(scheme_make_pair);
05340   jit_retval(JIT_R0);
05341 #endif
05342 
05343   return 1;
05344 }
05345 
05346 static int generate_vector_alloc(mz_jit_state *jitter, Scheme_Object *rator,
05347                                  Scheme_App_Rec *app, Scheme_App2_Rec *app2, Scheme_App3_Rec *app3)
05348 /* de-sync'd ok */
05349 {
05350   int imm, i, c;
05351 
05352   imm = IS_NAMED_PRIM(rator, "vector-immutable");
05353 
05354   if (app2) {
05355     mz_runstack_skipped(jitter, 1);
05356     generate_non_tail(app2->rand, jitter, 0, 1); /* sync'd below */
05357     CHECK_LIMIT();
05358     mz_runstack_unskipped(jitter, 1);
05359     c = 1;
05360   } else if (app3) {
05361     generate_two_args(app3->rand1, app3->rand2, jitter, 1, 2);  /* sync'd below */
05362     c = 2;
05363   } else {
05364     c = app->num_args;
05365     if (c)
05366       generate_app(app, NULL, c, jitter, 0, 0, 2);  /* sync'd below */
05367   }
05368   CHECK_LIMIT();
05369 
05370   mz_rs_sync();
05371 
05372 #ifdef CAN_INLINE_ALLOC
05373   /* Inlined alloc */
05374   if (app2)
05375     (void)jit_movi_p(JIT_R1, NULL); /* needed because R1 is marked during a GC */
05376   inline_alloc(jitter, sizeof(Scheme_Vector) + ((c - 1) * sizeof(Scheme_Object*)), scheme_vector_type, 
05377                imm, app2 || app3, 0, 0);
05378   CHECK_LIMIT();
05379 
05380   if ((c == 2) || (c == 1)) {
05381     jit_stxi_p((long)&SCHEME_VEC_ELS(0x0)[0] + GC_OBJHEAD_SIZE, JIT_V1, JIT_R0);
05382   }
05383   if (c == 2) {
05384     jit_stxi_p((long)&SCHEME_VEC_ELS(0x0)[1] + GC_OBJHEAD_SIZE, JIT_V1, JIT_R1);
05385   }
05386   jit_movi_l(JIT_R1, c);
05387   jit_stxi_i((long)&SCHEME_VEC_SIZE(0x0) + GC_OBJHEAD_SIZE, JIT_V1, JIT_R1);
05388   jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
05389 #else
05390   /* Non-inlined */
05391   JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05392   if (c == 1) {
05393     mz_prepare(1);
05394     jit_pusharg_p(JIT_R0);
05395     if (imm)
05396       (void)mz_finish(make_one_element_ivector);
05397     else
05398       (void)mz_finish(make_one_element_vector);
05399   } else if (c == 2) {
05400     mz_prepare(2);
05401     jit_pusharg_p(JIT_R1);
05402     jit_pusharg_p(JIT_R0);
05403     if (imm)
05404       (void)mz_finish(make_two_element_ivector);
05405     else
05406       (void)mz_finish(make_two_element_vector);
05407   } else {
05408     jit_movi_l(JIT_R1, c);
05409     mz_prepare(1);
05410     jit_pusharg_l(JIT_R1);
05411     if (imm)
05412       (void)mz_finish(make_ivector);
05413     else
05414       (void)mz_finish(make_vector);
05415   }
05416   jit_retval(JIT_R0);
05417 #endif
05418 
05419   CHECK_LIMIT();
05420 
05421   if (app) {
05422     for (i = 0; i < c; i++) {
05423       jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(i));
05424       jit_stxi_p((long)&SCHEME_VEC_ELS(0x0)[i], JIT_R0, JIT_R1);
05425       CHECK_LIMIT();
05426     }
05427     
05428     if (c) {
05429       /* could use mz_rs */
05430       jit_addi_l(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(c));
05431       mz_runstack_popped(jitter, c);
05432     }
05433   }
05434 
05435   return 1;
05436 }
05437 
05438 int generate_inlined_test(mz_jit_state *jitter, Scheme_Object *obj, int branch_short, jit_insn **refs, int need_sync)
05439 /* de-sync'd ok; syncs before jump */
05440 {
05441   switch (SCHEME_TYPE(obj)) {
05442   case scheme_application2_type:
05443     return generate_inlined_unary(jitter, (Scheme_App2_Rec *)obj, 0, 0, refs, branch_short, need_sync);
05444   case scheme_application3_type:
05445     return generate_inlined_binary(jitter, (Scheme_App3_Rec *)obj, 0, 0, refs, branch_short, need_sync);
05446   }
05447 
05448   return 0;
05449 }
05450 
05451 
05452 /*========================================================================*/
05453 /*                           lambda codegen                               */
05454 /*========================================================================*/
05455 
05456 #ifdef JIT_PRECISE_GC
05457 static Scheme_Object example_so = { scheme_native_closure_type, 0 };
05458 #endif
05459 
05460 static void ensure_closure_native(Scheme_Closure_Data *data, 
05461                               Scheme_Native_Closure_Data *case_lam)
05462 {
05463   if (!data->u.native_code || SCHEME_FALSEP((Scheme_Object *)data->u.native_code)) {
05464     Scheme_Native_Closure_Data *code;
05465     code = scheme_generate_lambda(data, 0, case_lam);
05466     data->u.native_code = code;
05467   }
05468 }
05469 
05470 static int generate_closure(Scheme_Closure_Data *data, 
05471                          mz_jit_state *jitter,
05472                             int immediately_filled)
05473 {
05474   Scheme_Native_Closure_Data *code;
05475   int retptr;
05476   
05477   ensure_closure_native(data, NULL);
05478   code = data->u.native_code;
05479 
05480 #ifdef JIT_PRECISE_GC
05481   if (data->closure_size < 100) {
05482     int sz;
05483     long init_word;
05484     sz = (sizeof(Scheme_Native_Closure)
05485           + ((data->closure_size - 1) * sizeof(Scheme_Object *)));
05486 # ifdef CAN_INLINE_ALLOC
05487     if (immediately_filled) {
05488       /* Inlined alloc */
05489       inline_alloc(jitter, sz, scheme_native_closure_type, 0, 0, 0, 0);
05490       CHECK_LIMIT();
05491       jit_addi_p(JIT_R0, JIT_V1, GC_OBJHEAD_SIZE);
05492     } else
05493 # endif
05494       {
05495         /* Non-inlined alloc */
05496         JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05497   
05498         jit_movi_l(JIT_R0, sz);
05499         mz_prepare(1);
05500         jit_pusharg_l(JIT_R0);
05501         if (immediately_filled) {
05502           (void)mz_finish(GC_malloc_one_small_dirty_tagged);
05503         } else {
05504           (void)mz_finish(GC_malloc_one_small_tagged);
05505         }
05506         jit_retval(JIT_R0);
05507         memcpy(&init_word, &example_so, sizeof(long));
05508         jit_movi_l(JIT_R1, init_word);
05509         jit_str_l(JIT_R0, JIT_R1); 
05510       }
05511     retptr = mz_retain(code);
05512     mz_load_retained(jitter, JIT_R1, retptr);
05513     jit_stxi_p((long)&((Scheme_Native_Closure *)0x0)->code, JIT_R0, JIT_R1);
05514 
05515     return 1;
05516   }
05517 #endif
05518 
05519   JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05520 
05521   mz_prepare(1);
05522   retptr = mz_retain(code);
05523 #ifdef JIT_PRECISE_GC
05524   mz_load_retained(jitter, JIT_R0, retptr);
05525 #else
05526   (void)jit_patchable_movi_p(JIT_R0, code); /* !! */
05527 #endif
05528   jit_pusharg_p(JIT_R0);
05529   (void)mz_finish(scheme_make_native_closure);
05530   jit_retval(JIT_R0);
05531 
05532   return 1;
05533 }
05534 
05535 static int generate_closure_fill(Scheme_Closure_Data *data, 
05536                              mz_jit_state *jitter)
05537 {
05538   /* Fill in closure */
05539   int j, size, pos;
05540   mzshort *map;
05541   size = data->closure_size;
05542   map = data->closure_map;
05543   jit_addi_p(JIT_R2, JIT_R0, &((Scheme_Native_Closure *)0x0)->vals);
05544   for (j = 0; j < size; j++) {
05545     CHECK_LIMIT();
05546     pos = mz_remap(map[j]);
05547     jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(pos));
05548     jit_stxi_p(WORDS_TO_BYTES(j), JIT_R2, JIT_R1);
05549   }
05550   return 1;
05551 }
05552 
05553 Scheme_Native_Closure_Data *scheme_generate_case_lambda(Scheme_Case_Lambda *c)
05554 {
05555   Scheme_Closure_Data *data;
05556   Scheme_Native_Closure_Data *ndata;
05557   Scheme_Object *name, *o;
05558   int max_let_depth = 0, i, count, is_method = 0;
05559 
05560   ndata = MALLOC_ONE_RT(Scheme_Native_Closure_Data);
05561 #ifdef MZTAG_REQUIRED
05562   ndata->iso.so.type = scheme_rt_native_code;
05563 #endif
05564   name = c->name;
05565   if (name && SCHEME_BOXP(name)) {
05566     name = SCHEME_BOX_VAL(name);
05567     is_method = 1;
05568   }
05569   ndata->u2.name = name;
05570   count = c->count;
05571   for (i = 0; i < count; i++) {
05572     o = c->array[i];
05573     if (SCHEME_PROCP(o))
05574       o = (Scheme_Object *)((Scheme_Closure *)o)->code;
05575     data = (Scheme_Closure_Data *)o;
05576     ensure_closure_native(data, ndata);
05577     if (data->u.native_code->max_let_depth > max_let_depth)
05578       max_let_depth = data->u.native_code->max_let_depth;
05579   }
05580   ndata->max_let_depth = max_let_depth;
05581   ndata->closure_size = -(count + 1); /* Indicates case-lambda */
05582 
05583   if (count) {
05584     o = c->array[0];
05585     if (SCHEME_PROCP(o))
05586       o = (Scheme_Object *)((Scheme_Closure *)o)->code;
05587     data = (Scheme_Closure_Data *)o;
05588     is_method = ((SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_IS_METHOD) ? 1 : 0);
05589   }
05590 
05591   generate_case_lambda(c, ndata, is_method);
05592 
05593   return ndata;
05594 }
05595 
05596 static void ensure_case_closure_native(Scheme_Case_Lambda *c)
05597 {
05598   if (!c->native_code || SCHEME_FALSEP((Scheme_Object *)c->native_code)) {
05599     Scheme_Native_Closure_Data *ndata;
05600     ndata = scheme_generate_case_lambda(c);
05601     c->native_code = ndata;
05602   }
05603 }
05604 
05605 static int generate_case_closure(Scheme_Object *obj, mz_jit_state *jitter, int target)
05606 /* de-sync's */
05607 {
05608   Scheme_Case_Lambda *c = (Scheme_Case_Lambda *)obj;
05609   Scheme_Native_Closure_Data *ndata;
05610   Scheme_Closure_Data *data;
05611   Scheme_Object *o;
05612   int i, offset, count, retptr;
05613 
05614   ensure_case_closure_native(c);
05615   ndata = c->native_code;
05616 
05617   mz_rs_sync();
05618 
05619   JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
05620   mz_prepare(1);
05621   retptr = mz_retain(ndata);
05622 #ifdef JIT_PRECISE_GC
05623   mz_load_retained(jitter, JIT_R0, retptr);
05624 #else
05625   (void)jit_patchable_movi_p(JIT_R0, ndata); /* !! */
05626 #endif
05627   jit_pusharg_p(JIT_R0);
05628   (void)mz_finish(scheme_make_native_case_closure);
05629   jit_retval(JIT_R1);
05630   CHECK_LIMIT();
05631 
05632   count = c->count;
05633   
05634   for (i = 0; i < count; i++) {
05635     o = c->array[i];
05636     if (SCHEME_PROCP(o))
05637       o = (Scheme_Object *)((Scheme_Closure *)o)->code;
05638     data = (Scheme_Closure_Data *)o;
05639     mz_pushr_p(JIT_R1);
05640     mz_rs_sync();
05641     generate_closure(data, jitter, 1);
05642     CHECK_LIMIT();
05643     generate_closure_fill(data, jitter);
05644     CHECK_LIMIT();
05645     mz_popr_p(JIT_R1);
05646     offset = WORDS_TO_BYTES(i) + (unsigned long)&((Scheme_Native_Closure *)0x0)->vals;
05647     jit_stxi_p(offset, JIT_R1, JIT_R0);
05648     CHECK_LIMIT();
05649   }
05650   jit_movr_p(target, JIT_R1);
05651   
05652   return 1;
05653 }
05654 
05655 /*========================================================================*/
05656 /*                          non-tail codegen                              */
05657 /*========================================================================*/
05658 
05659 static int generate_non_tail_mark_pos_prefix(mz_jit_state *jitter)
05660 {
05661   /* dsync'd ok.
05662      This part of a non-tail setup can be done once for a sequence
05663      of non-tail calls. In that case, pass 0 for the `mark_pos_ends'
05664      argument to generate_non_tail(), so that it can skip this prefix
05665      and suffix. In case this prefix needs to adjust the runstack,
05666      the result indicates the number of pushed values. */
05667   jit_ldi_l(JIT_R2, &scheme_current_cont_mark_pos);
05668   jit_addi_l(JIT_R2, JIT_R2, 2);
05669   jit_sti_l(&scheme_current_cont_mark_pos, JIT_R2);
05670   return 0 /* = number of pushed items */;
05671 }
05672 
05673 static void generate_non_tail_mark_pos_suffix(mz_jit_state *jitter)
05674 /* dsync'd ok */
05675 {
05676   jit_ldi_l(JIT_R2, &scheme_current_cont_mark_pos);
05677   jit_subi_l(JIT_R2, JIT_R2, 2);
05678   jit_sti_l(&scheme_current_cont_mark_pos, JIT_R2);
05679 }
05680 
05681 static int generate_non_tail(Scheme_Object *obj, mz_jit_state *jitter, int multi_ok, int mark_pos_ends)
05682 /* de-sync's rs */
05683 {
05684   if (is_simple(obj, INIT_SIMPLE_DEPTH, 0, jitter, 0)) {
05685     /* Simple; doesn't change the stack or set marks: */
05686     int v;
05687     FOR_LOG(jitter->log_depth++);
05688     v = generate(obj, jitter, 0, multi_ok, JIT_R0);
05689     FOR_LOG(--jitter->log_depth);
05690     return v;
05691   }
05692 
05693   {
05694     int amt, need_ends = 1, using_local1 = 0;
05695     START_JIT_DATA();
05696     
05697     /* Might change the stack or marks: */
05698     if (is_simple(obj, INIT_SIMPLE_DEPTH, 1, jitter, 0)) {
05699       need_ends = 0;
05700     } else {
05701       LOG_IT(("non-tail\n"));
05702       if (mark_pos_ends)
05703        generate_non_tail_mark_pos_prefix(jitter);
05704       jit_ldi_p(JIT_R2, &scheme_current_cont_mark_stack);
05705       if (!jitter->local1_busy) {
05706         using_local1 = 1;
05707         jitter->local1_busy = 1;
05708         mz_set_local_p(JIT_R2, JIT_LOCAL1);
05709       } else {
05710         /* mark stack is an integer... turn it into a pointer */
05711         jit_lshi_l(JIT_R2, JIT_R2, 0x1);
05712         jit_ori_l(JIT_R2, JIT_R2, 0x1);
05713         mz_pushr_p(JIT_R2); /* no sync */
05714       }
05715       CHECK_LIMIT();
05716     }
05717     mz_runstack_saved(jitter);
05718     CHECK_LIMIT();
05719     
05720     PAUSE_JIT_DATA();
05721     FOR_LOG(jitter->log_depth++);
05722 
05723     generate(obj, jitter, 0, multi_ok, JIT_R0); /* no sync */
05724 
05725     FOR_LOG(--jitter->log_depth);
05726     RESUME_JIT_DATA();
05727     CHECK_LIMIT();
05728 
05729     amt = mz_runstack_restored(jitter);
05730     if (amt) {
05731       mz_rs_inc(amt);
05732     }
05733     if (need_ends) {
05734       if (using_local1) {
05735         mz_get_local_p(JIT_R2, JIT_LOCAL1);
05736         jitter->local1_busy = 0;
05737       } else {
05738         mz_popr_p(JIT_R2); /* no sync */
05739         jit_rshi_l(JIT_R2, JIT_R2, 0x1); /* pointer back to integer */
05740       }
05741       jit_sti_p(&scheme_current_cont_mark_stack, JIT_R2);
05742       if (mark_pos_ends)
05743        generate_non_tail_mark_pos_suffix(jitter);
05744       CHECK_LIMIT();
05745     }
05746 
05747     END_JIT_DATA(21);
05748   }
05749     
05750   return 1;
05751 }
05752 
05753 /*========================================================================*/
05754 /*                          expression codegen                            */
05755 /*========================================================================*/
05756 
05757 static int generate_ignored_non_tail(Scheme_Object *obj, mz_jit_state *jitter, int multi_ok, int need_ends)
05758 /* de-sync's */
05759 {
05760   Scheme_Type t = SCHEME_TYPE(obj);
05761 
05762   if (SAME_TYPE(t, scheme_local_type)
05763       || SAME_TYPE(t, scheme_local_unbox_type)) {
05764     /* Must be here to clear */
05765     if (SCHEME_LOCAL_FLAGS(obj) & SCHEME_LOCAL_CLEAR_ON_READ) {
05766       int pos;
05767       START_JIT_DATA();
05768       pos = mz_remap(SCHEME_LOCAL_POS(obj));
05769       LOG_IT(("clear %d\n", pos));
05770       mz_rs_stxi(pos, JIT_RUNSTACK);
05771       END_JIT_DATA(2);      
05772     }
05773     return 1;
05774   }
05775 
05776   return generate_non_tail(obj, jitter, multi_ok, need_ends);
05777 }
05778 
05779 static Scheme_Object *generate_k(void)
05780 {
05781   Scheme_Thread *p = scheme_current_thread;
05782   Scheme_Object *obj = (Scheme_Object *)p->ku.k.p1;
05783   mz_jit_state *jitter = (mz_jit_state *)p->ku.k.p2;
05784   int v;
05785 
05786   p->ku.k.p1 = NULL;
05787   p->ku.k.p2 = NULL;
05788 
05789   v = generate(obj, jitter, p->ku.k.i1, p->ku.k.i2, p->ku.k.i3);
05790 
05791   return scheme_make_integer(v);
05792 }
05793 
05794 static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int multi_ok, int target)
05795 /* de-sync's; result goes to target */
05796 {
05797   Scheme_Type type;
05798 
05799 #ifdef DO_STACK_CHECK
05800 # include "mzstkchk.h"
05801   {
05802     Scheme_Object *ok;
05803     Scheme_Thread *p = scheme_current_thread;
05804     mz_jit_state *jitter_copy;
05805 
05806     jitter_copy = MALLOC_ONE_RT(mz_jit_state);
05807     memcpy(jitter_copy, jitter, sizeof(mz_jit_state));
05808 #ifdef MZTAG_REQUIRED
05809     jitter_copy->type = scheme_rt_jitter_data;
05810 #endif
05811 
05812     p->ku.k.p1 = (void *)obj;
05813     p->ku.k.p2 = (void *)jitter_copy;
05814     p->ku.k.i1 = is_tail;
05815     p->ku.k.i2 = multi_ok;
05816     p->ku.k.i3 = target;
05817 
05818     ok = scheme_handle_stack_overflow(generate_k);
05819 
05820     memcpy(jitter, jitter_copy, sizeof(mz_jit_state));
05821 
05822     return SCHEME_INT_VAL(ok);
05823   }
05824 #endif
05825   
05826   type = SCHEME_TYPE(obj);
05827   switch (type) {
05828   case scheme_toplevel_type:
05829     {
05830       int pos;
05831       /* Other parts of the JIT rely on this code not modifying R1 */
05832       START_JIT_DATA();
05833       LOG_IT(("top-level\n"));
05834       mz_rs_sync_fail_branch();
05835       /* Load global array: */
05836       pos = mz_remap(SCHEME_TOPLEVEL_DEPTH(obj));
05837       mz_rs_ldxi(JIT_R2, pos);
05838       /* Load bucket: */
05839       pos = SCHEME_TOPLEVEL_POS(obj);
05840       jit_ldxi_p(JIT_R2, JIT_R2, WORDS_TO_BYTES(pos));
05841       /* Extract bucket value */
05842       jit_ldxi_p(target, JIT_R2, &(SCHEME_VAR_BUCKET(0x0)->val));
05843       CHECK_LIMIT();
05844       if (!(SCHEME_TOPLEVEL_FLAGS(obj) 
05845            & (SCHEME_TOPLEVEL_CONST | SCHEME_TOPLEVEL_READY))) {
05846        /* Is it NULL? */
05847        (void)jit_beqi_p(unbound_global_code, target, 0);
05848       }
05849       END_JIT_DATA(0);
05850       return 1;
05851     }
05852   case scheme_local_type:
05853     {
05854       /* Other parts of the JIT rely on this code modifying the target register, only */
05855       int pos;
05856       START_JIT_DATA();
05857       pos = mz_remap(SCHEME_LOCAL_POS(obj));
05858       LOG_IT(("local %d [%d]\n", pos, SCHEME_LOCAL_FLAGS(obj)));
05859       if (pos || (mz_CURRENT_STATUS() != mz_RS_R0_HAS_RUNSTACK0)) {
05860         mz_rs_ldxi(target, pos);
05861         VALIDATE_RESULT(target);
05862       } else if (target != JIT_R0) {
05863         jit_movr_p(target, JIT_R0);
05864       }
05865       if (SCHEME_LOCAL_FLAGS(obj) & SCHEME_LOCAL_CLEAR_ON_READ) {
05866         mz_rs_stxi(pos, JIT_RUNSTACK);
05867       }
05868       END_JIT_DATA(2);
05869       return 1;
05870     }
05871   case scheme_local_unbox_type:
05872     {
05873       int pos;
05874       START_JIT_DATA();
05875       LOG_IT(("unbox local\n"));
05876 
05877       pos = mz_remap(SCHEME_LOCAL_POS(obj));
05878       mz_rs_ldxi(JIT_R0, pos);
05879       jit_ldr_p(target, JIT_R0);
05880       if (SCHEME_LOCAL_FLAGS(obj) & SCHEME_LOCAL_CLEAR_ON_READ) {
05881         LOG_IT(("clear-on-read\n"));
05882         mz_rs_stxi(pos, JIT_RUNSTACK);
05883       }
05884       VALIDATE_RESULT(target);
05885 
05886       END_JIT_DATA(3);
05887       return 1;
05888     }
05889   case scheme_syntax_type:
05890     {
05891       int pos;
05892       pos = SCHEME_PINT_VAL(obj);
05893       switch (pos) {
05894       case CASE_LAMBDA_EXPD:
05895        {
05896          START_JIT_DATA();
05897          LOG_IT(("case-lambda\n"));
05898          /* case-lambda */
05899          generate_case_closure(SCHEME_IPTR_VAL(obj), jitter, target);
05900          END_JIT_DATA(5);
05901        } 
05902        break;
05903       case BEGIN0_EXPD:
05904        {
05905          Scheme_Sequence *seq;
05906          jit_insn *ref, *ref2;
05907          int i;
05908          START_JIT_DATA();
05909 
05910          LOG_IT(("begin0\n"));
05911 
05912          seq = (Scheme_Sequence *)SCHEME_IPTR_VAL(obj);
05913        
05914          /* Evaluate first expression, and for consistency with bytecode
05915             evaluation, allow multiple values. */
05916          generate_non_tail(seq->array[0], jitter, 1, 1);
05917          CHECK_LIMIT();
05918 
05919          /* Save value(s) */
05920          jit_movr_p(JIT_V1, JIT_R0);
05921          mz_pushr_p(JIT_V1);
05922          mz_pushr_p(JIT_V1);
05923          mz_pushr_p(JIT_V1);
05924           mz_rs_sync();
05925          __START_SHORT_JUMPS__(1);
05926          ref = jit_bnei_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
05927          CHECK_LIMIT();
05928          /* Save away multiple values */
05929          mz_popr_p(JIT_V1); /* sync'd below... */
05930          mz_popr_p(JIT_V1);
05931          mz_popr_p(JIT_V1);
05932          jit_ldi_p(JIT_R0, &scheme_current_thread);
05933          CHECK_LIMIT();
05934          jit_ldxi_l(JIT_V1, JIT_R0, &((Scheme_Thread *)0x0)->ku.multiple.count);
05935          jit_lshi_l(JIT_V1, JIT_V1, 0x1);
05936          jit_ori_l(JIT_V1, JIT_V1, 0x1);
05937          mz_pushr_p(JIT_V1); /* sync'd below */
05938          jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Thread *)0x0)->ku.multiple.array);
05939          mz_pushr_p(JIT_V1); /* sync'd below */
05940          CHECK_LIMIT();
05941          (void)jit_movi_p(JIT_R1, 0x0);
05942          mz_pushr_p(JIT_R1); /* pushing 0 indicates that multi-array follows */
05943          /* If multi-value array is values buffer, zero out values buffer */
05944          jit_ldxi_p(JIT_R2, JIT_R0, &((Scheme_Thread *)0x0)->values_buffer);
05945           mz_rs_sync();
05946          ref2 = jit_bner_p(jit_forward(), JIT_V1, JIT_R2);
05947          jit_stxi_p(&((Scheme_Thread *)0x0)->values_buffer, JIT_R0, JIT_R1);
05948          CHECK_LIMIT();
05949 
05950          /* evaluate remaining expressions */
05951          mz_patch_branch(ref);
05952          mz_patch_branch(ref2);
05953          __END_SHORT_JUMPS__(1);
05954          for (i = 1; i < seq->count; i++) {
05955            generate_ignored_non_tail(seq->array[i], jitter, 1, 1); /* sync's below */
05956             CHECK_LIMIT();
05957          }
05958 
05959          /* Restore values, if necessary */
05960          mz_popr_p(JIT_R0);
05961          mz_popr_p(JIT_R1);
05962          mz_popr_p(JIT_R2);
05963           mz_rs_sync();
05964          CHECK_LIMIT();
05965          __START_TINY_JUMPS__(1);
05966          ref = jit_bnei_p(jit_forward(), JIT_R0, 0x0);
05967          CHECK_LIMIT();
05968          jit_ldi_p(JIT_R0, &scheme_current_thread);
05969          jit_stxi_p(&((Scheme_Thread *)0x0)->ku.multiple.array, JIT_R0, JIT_R1);
05970          jit_rshi_ul(JIT_R2, JIT_R2, 0x1);
05971          jit_stxi_l(&((Scheme_Thread *)0x0)->ku.multiple.count, JIT_R0, JIT_R2);
05972          (void)jit_movi_p(JIT_R0, SCHEME_MULTIPLE_VALUES);
05973 
05974          mz_patch_branch(ref);
05975           if (target != JIT_R0)
05976             jit_movr_p(target, JIT_R0);
05977          __END_TINY_JUMPS__(1);
05978 
05979          END_JIT_DATA(6);
05980        }
05981        break;
05982       case SET_EXPD:
05983        {
05984          Scheme_Object *p, *v;
05985          int pos, set_undef;
05986          START_JIT_DATA();
05987        
05988          LOG_IT(("set!\n"));
05989 
05990          p = SCHEME_IPTR_VAL(obj);
05991          v = SCHEME_CAR(p);
05992          set_undef = SCHEME_TRUEP(v);
05993          p = SCHEME_CDR(p);
05994          v = SCHEME_CAR(p);
05995          p = SCHEME_CDR(p);
05996 
05997          generate_non_tail(p, jitter, 0, 1);
05998          CHECK_LIMIT();
05999           mz_rs_sync();
06000           
06001          /* Load global+stx array: */
06002          pos = mz_remap(SCHEME_TOPLEVEL_DEPTH(v));
06003          jit_ldxi_p(JIT_R2, JIT_RUNSTACK, WORDS_TO_BYTES(pos));
06004          /* Try already-renamed stx: */
06005          pos = SCHEME_TOPLEVEL_POS(v);
06006          jit_ldxi_p(JIT_R2, JIT_R2, WORDS_TO_BYTES(pos));
06007          CHECK_LIMIT();
06008        
06009          /* R0 has values, R2 has pos */
06010          JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06011          mz_prepare(3);
06012          (void)jit_movi_i(JIT_R1, set_undef);
06013          jit_pusharg_p(JIT_R1);
06014          jit_pusharg_p(JIT_R0);
06015          jit_pusharg_p(JIT_R2);
06016          CHECK_LIMIT();
06017          (void)mz_finish(call_set_global_bucket);
06018          CHECK_LIMIT();
06019          (void)jit_movi_p(target, scheme_void);
06020          END_JIT_DATA(7);
06021        }
06022        break;
06023       case APPVALS_EXPD:
06024         {
06025           Scheme_Object *p, *v;
06026           GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref5, *refloop;
06027           START_JIT_DATA();
06028 
06029          LOG_IT(("appvals\n"));
06030 
06031           p = SCHEME_IPTR_VAL(obj);
06032          v = SCHEME_CAR(p);
06033           p = SCHEME_CDR(p);
06034 
06035           generate_non_tail(v, jitter, 0, 1);
06036          CHECK_LIMIT();
06037 
06038           /* If v is not known to produce a procedure, then check result: */
06039           if (!is_a_procedure(v, jitter)) {
06040             mz_rs_sync();
06041             (void)jit_bmsi_l(bad_app_vals_target, JIT_R0, 0x1);
06042             jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type);
06043             (void)jit_blti_i(bad_app_vals_target, JIT_R1, scheme_prim_type);
06044             (void)jit_bgti_i(bad_app_vals_target, JIT_R1, scheme_native_closure_type);
06045             CHECK_LIMIT();
06046           }
06047 
06048           mz_pushr_p(JIT_R0);
06049           generate_non_tail(p, jitter, 1, 1);
06050           CHECK_LIMIT();
06051 
06052           mz_popr_p(JIT_V1);
06053           /* Function is in V1, argument(s) in R0 */
06054 
06055           mz_rs_sync();
06056 
06057           __START_SHORT_JUMPS__(1);
06058           ref = jit_beqi_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
06059           /* Single-value case: --------------- */
06060           /* We definitely have stack space for one argument, because we
06061              just used it for the rator. */
06062           if (is_tail) {
06063             jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK_BASE, WORDS_TO_BYTES(1));
06064           } else {
06065             jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
06066           }
06067           CHECK_RUNSTACK_OVERFLOW();
06068           jit_str_p(JIT_RUNSTACK, JIT_R0);
06069           jit_movi_l(JIT_R0, 1);
06070           ref2 = jit_jmpi(jit_forward());
06071           CHECK_LIMIT();
06072 
06073           /* Multiple-values case: ------------ */
06074           mz_patch_branch(ref);
06075           /* Get new argc: */
06076           (void)jit_ldi_p(JIT_R1, &scheme_current_thread);
06077           jit_ldxi_l(JIT_R2, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.count);
06078           /* Enough room on runstack? */
06079           jit_ldi_p(JIT_R0, &MZ_RUNSTACK_START);
06080           if (is_tail) {
06081             jit_subr_ul(JIT_R0, JIT_RUNSTACK_BASE, JIT_R0);
06082           } else {
06083             jit_subr_ul(JIT_R0, JIT_RUNSTACK, JIT_R0); 
06084           }
06085           CHECK_LIMIT();
06086           /* R0 is space left (in bytes), R2 is argc */
06087           jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE);
06088           if (is_tail) {
06089             __END_SHORT_JUMPS__(1);
06090             (void)jit_bltr_ul(app_values_tail_slow_code, JIT_R0, JIT_R2);
06091             __START_SHORT_JUMPS__(1);
06092             ref5 = 0;
06093           } else {
06094             GC_CAN_IGNORE jit_insn *refok;
06095             refok = jit_bger_ul(jit_forward(), JIT_R0, JIT_R2);
06096             __END_SHORT_JUMPS__(1);
06097             if (multi_ok) {
06098               (void)jit_calli(app_values_multi_slow_code);
06099             } else {
06100               (void)jit_calli(app_values_slow_code);
06101             }
06102             __START_SHORT_JUMPS__(1);
06103             ref5 = jit_jmpi(jit_forward());
06104             mz_patch_branch(refok);
06105           }
06106           CHECK_LIMIT();
06107           if (is_tail) {
06108             jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK_BASE, JIT_R2);
06109           } else {
06110             jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R2);
06111           }
06112           CHECK_RUNSTACK_OVERFLOW();
06113           /* Copy args: */
06114           jit_ldxi_l(JIT_R1, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.array);
06115           refloop = _jit.x.pc;
06116           ref3 = jit_blei_l(jit_forward(), JIT_R2, 0);
06117           jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE);
06118           jit_ldxr_p(JIT_R0, JIT_R1, JIT_R2);
06119           jit_stxr_p(JIT_R2, JIT_RUNSTACK, JIT_R0);
06120           (void)jit_jmpi(refloop);
06121           CHECK_LIMIT();
06122           mz_patch_branch(ref3);
06123           (void)jit_ldi_p(JIT_R0, &scheme_current_thread);
06124           jit_ldxi_l(JIT_R0, JIT_R0, &((Scheme_Thread *)0x0)->ku.multiple.count);
06125           
06126           /* Perform call --------------------- */
06127           /* Function is in V1, argc in R0, args on RUNSTACK */
06128           mz_patch_ucbranch(ref2);
06129           __END_SHORT_JUMPS__(1);
06130 
06131           if (is_tail) {
06132             if (!shared_tail_argc_code) {
06133               shared_tail_argc_code = generate_shared_call(-1, jitter, 1, 1, 0, 0, 0);
06134             }
06135             mz_set_local_p(JIT_R0, JIT_LOCAL2);
06136             (void)jit_jmpi(shared_tail_argc_code);
06137           } else {
06138             int mo = multi_ok ? 1 : 0;
06139             void *code;
06140             if (!shared_non_tail_argc_code[mo]) {
06141               ensure_retry_available(jitter, multi_ok);
06142               code = generate_shared_call(-2, jitter, multi_ok, 0, 0, 0, 0);
06143               shared_non_tail_argc_code[mo] = code;
06144             }
06145             code = shared_non_tail_argc_code[mo];
06146             (void)jit_calli(code);
06147             /* non-tail code pops args off runstack for us */
06148             jitter->need_set_rs = 1;
06149             mz_patch_ucbranch(ref5);
06150             if (target != JIT_R0)
06151               jit_movr_p(target, JIT_R0);
06152           }
06153           CHECK_LIMIT();
06154 
06155           END_JIT_DATA(81);
06156 
06157           if (is_tail)
06158             return 2;
06159         }
06160         break;
06161       case BOXENV_EXPD:
06162        {
06163          Scheme_Object *p, *v;
06164          int pos;
06165          START_JIT_DATA();
06166 
06167          LOG_IT(("boxenv\n"));
06168 
06169           mz_rs_sync();
06170          JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06171 
06172          p = (Scheme_Object *)SCHEME_IPTR_VAL(obj);
06173          v = SCHEME_CAR(p);
06174          pos = mz_remap(SCHEME_INT_VAL(v));
06175          p = SCHEME_CDR(p);
06176 
06177          jit_ldxi_p(JIT_R2, JIT_RUNSTACK, WORDS_TO_BYTES(pos));
06178          mz_prepare(1);
06179          jit_pusharg_p(JIT_R2);
06180          (void)mz_finish(scheme_make_envunbox);
06181          jit_retval(JIT_R0);
06182          jit_stxi_p(WORDS_TO_BYTES(pos), JIT_RUNSTACK, JIT_R0);
06183          CHECK_LIMIT();
06184 
06185          generate(p, jitter, is_tail, multi_ok, target);
06186 
06187          END_JIT_DATA(8);
06188        }
06189        break;
06190       case REF_EXPD:
06191         {
06192           mz_rs_sync();
06193 
06194           obj = SCHEME_IPTR_VAL(obj);
06195       
06196           /* Load global array: */
06197           pos = mz_remap(SCHEME_TOPLEVEL_DEPTH(obj));
06198           jit_ldxi_p(JIT_R2, JIT_RUNSTACK, WORDS_TO_BYTES(pos));
06199           /* Load bucket: */
06200           pos = SCHEME_TOPLEVEL_POS(obj);
06201           jit_ldxi_p(JIT_R2, JIT_R2, WORDS_TO_BYTES(pos));
06202           CHECK_LIMIT();
06203 
06204           JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06205           mz_prepare(1);
06206          jit_pusharg_p(JIT_R2);
06207          (void)mz_finish(make_global_ref);
06208          CHECK_LIMIT();
06209           jit_retval(target);
06210           VALIDATE_RESULT(target);
06211         }
06212         break;
06213       case SPLICE_EXPD:
06214         {
06215           scheme_signal_error("cannot JIT a top-level splice form");
06216         }
06217         break;
06218       default:
06219        {
06220           mz_rs_sync();
06221          JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06222          obj = SCHEME_IPTR_VAL(obj);
06223          (void)jit_patchable_movi_p(JIT_R2, obj); /* !! */
06224          CHECK_LIMIT();
06225          mz_prepare(1);
06226          jit_pusharg_p(JIT_R2);
06227          (void)mz_finish(scheme_syntax_executers[pos]);
06228          CHECK_LIMIT();
06229          jit_retval(target);
06230           VALIDATE_RESULT(target);
06231        }
06232       }
06233       return 1;
06234     }
06235   case scheme_application_type:
06236     {
06237       Scheme_App_Rec *app = (Scheme_App_Rec *)obj;
06238       int r;
06239 
06240       LOG_IT(("app %d\n", app->num_args));
06241 
06242       r = generate_inlined_nary(jitter, app, is_tail, multi_ok, NULL, 1);
06243       CHECK_LIMIT();
06244       if (r) {
06245         if (target != JIT_R0)
06246           jit_movr_p(target, JIT_R0);
06247        return r;
06248       }
06249 
06250       r = generate_app(app, NULL, app->num_args, jitter, is_tail, multi_ok, 0);
06251 
06252       CHECK_LIMIT();
06253       if (target != JIT_R0)
06254         jit_movr_p(target, JIT_R0);
06255 
06256       return r;
06257     }
06258   case scheme_application2_type:
06259     {
06260       Scheme_App2_Rec *app = (Scheme_App2_Rec *)obj;
06261       Scheme_Object *args[2];
06262       int r;
06263 
06264       r = generate_inlined_unary(jitter, app, is_tail, multi_ok, NULL, 1, 0);
06265       CHECK_LIMIT();
06266       if (r) {
06267         if (target != JIT_R0)
06268           jit_movr_p(target, JIT_R0);
06269        return r;
06270       }
06271 
06272       LOG_IT(("app 2\n"));
06273 
06274       args[0] = app->rator;
06275       args[1] = app->rand;
06276       
06277       r = generate_app(NULL, args, 1, jitter, is_tail, multi_ok, 0);
06278 
06279       CHECK_LIMIT();
06280       if (target != JIT_R0)
06281         jit_movr_p(target, JIT_R0);
06282 
06283       return r;
06284     }
06285   case scheme_application3_type:
06286     {
06287       Scheme_App3_Rec *app = (Scheme_App3_Rec *)obj;
06288       Scheme_Object *args[3];
06289       int r;
06290 
06291       r = generate_inlined_binary(jitter, app, is_tail, multi_ok, NULL, 1, 0);
06292       CHECK_LIMIT();
06293       if (r) {
06294         if (target != JIT_R0)
06295           jit_movr_p(target, JIT_R0);
06296        return r;
06297       }
06298 
06299       LOG_IT(("app 3\n"));
06300       
06301       args[0] = app->rator;
06302       args[1] = app->rand1;
06303       args[2] = app->rand2;
06304 
06305       r = generate_app(NULL, args, 2, jitter, is_tail, multi_ok, 0);
06306 
06307       CHECK_LIMIT();
06308       if (target != JIT_R0)
06309         jit_movr_p(target, JIT_R0);
06310 
06311       return r;
06312     }
06313   case scheme_sequence_type:
06314     {
06315       Scheme_Sequence *seq = (Scheme_Sequence *)obj;
06316       int cnt = seq->count, i;
06317       START_JIT_DATA();
06318 
06319       LOG_IT(("begin\n"));
06320 
06321       for (i = 0; i < cnt - 1; i++) {
06322        generate_ignored_non_tail(seq->array[i], jitter, 1, 1);
06323        CHECK_LIMIT();
06324       }
06325 
06326       END_JIT_DATA(11);
06327 
06328       return generate(seq->array[cnt - 1], jitter, is_tail, multi_ok, target);
06329     }
06330   case scheme_branch_type:
06331     {  
06332       Scheme_Branch_Rec *branch = (Scheme_Branch_Rec *)obj;
06333       jit_insn *refs[6], *ref2;
06334       int nsrs, nsrs1, g1, g2, amt, need_sync;
06335 #ifdef NEED_LONG_JUMPS
06336       int then_short_ok, else_short_ok;
06337 #else
06338       int then_short_ok = 1;
06339 #endif
06340       START_JIT_DATA();
06341 
06342 #ifdef NEED_LONG_JUMPS
06343       /* It's possible that the code for a then
06344         or else branch will be so large that we might
06345         need a long jump. Conservatively analyze the
06346         `then' and `else' expressions. */
06347       then_short_ok = (is_short(branch->tbranch, 32) > 0);
06348       else_short_ok = (is_short(branch->fbranch, 32) > 0);
06349 #endif
06350 
06351       LOG_IT(("if...\n"));
06352 
06353       /* Places to patch for the false branch: */
06354       refs[0] = NULL;
06355       refs[1] = NULL;
06356       refs[2] = NULL; /* a movi patch, instead of a branch */
06357       refs[3] = NULL;
06358       refs[4] = NULL;
06359       refs[5] = NULL; /* a jmpi instead of a test branch */
06360 
06361       /* Avoid rs_sync if neither branch changes the sync state. */
06362       if ((no_sync_change(branch->tbranch, 32) > 0)
06363           && (no_sync_change(branch->fbranch, 32) > 0))
06364         need_sync = 0;
06365       else
06366         need_sync = 1;
06367 
06368       if (!generate_inlined_test(jitter, branch->test, then_short_ok, refs, need_sync)) {
06369        CHECK_LIMIT();
06370        generate_non_tail(branch->test, jitter, 0, 1);
06371         if (need_sync) mz_rs_sync();
06372        CHECK_LIMIT();
06373        __START_SHORT_JUMPS__(then_short_ok);
06374        refs[0] = jit_beqi_p(jit_forward(), JIT_R0, scheme_false);
06375        __END_SHORT_JUMPS__(then_short_ok);
06376       }
06377       CHECK_LIMIT();
06378 
06379       /* True branch */
06380       mz_runstack_saved(jitter);
06381       nsrs = jitter->need_set_rs;
06382       PAUSE_JIT_DATA();
06383       LOG_IT(("...then...\n"));
06384       FOR_LOG(++jitter->log_depth);
06385       g1 = generate(branch->tbranch, jitter, is_tail, multi_ok, target);
06386       RESUME_JIT_DATA();
06387       CHECK_LIMIT();
06388       amt = mz_runstack_restored(jitter);
06389       if (g1 != 2) {
06390        if (!is_tail) {
06391           if (amt)
06392             mz_rs_inc(amt);
06393           if (need_sync) mz_rs_sync();
06394         }
06395        __START_SHORT_JUMPS__(else_short_ok);
06396        ref2 = jit_jmpi(jit_forward());
06397        __END_SHORT_JUMPS__(else_short_ok);
06398        nsrs1 = jitter->need_set_rs;
06399       } else {
06400        ref2 = 0;
06401        nsrs1 = 0;
06402       }
06403       jitter->need_set_rs = nsrs;
06404       if (need_sync) mz_rs_sync_0();
06405 
06406       /* False branch */
06407       mz_runstack_saved(jitter);
06408       __START_SHORT_JUMPS__(then_short_ok);
06409       if (refs[0]) {
06410        mz_patch_branch(refs[0]);
06411       }
06412       if (refs[1]) {
06413        mz_patch_branch(refs[1]);
06414       }
06415       if (refs[2]) {
06416        jit_patch_movi(refs[2], (_jit.x.pc));
06417       }
06418       if (refs[3]) {
06419        mz_patch_branch(refs[3]);
06420       }
06421       if (refs[4]) {
06422        mz_patch_branch(refs[4]);
06423       }
06424       if (refs[5]) {
06425        mz_patch_ucbranch(refs[5]);
06426       }
06427       __END_SHORT_JUMPS__(then_short_ok);
06428       PAUSE_JIT_DATA();
06429       FOR_LOG(jitter->log_depth--);
06430       LOG_IT(("...else\n"));
06431       FOR_LOG(++jitter->log_depth);
06432       g2 = generate(branch->fbranch, jitter, is_tail, multi_ok, target);
06433       RESUME_JIT_DATA();
06434       CHECK_LIMIT();
06435       amt = mz_runstack_restored(jitter);
06436       if (g2 != 2) {
06437         if (!is_tail) {
06438           if (amt)
06439             mz_rs_inc(amt);
06440           if (need_sync) mz_rs_sync();
06441         }
06442       } else {
06443        jitter->need_set_rs = 0;
06444       }
06445       if (g1 != 2) {
06446        __START_SHORT_JUMPS__(else_short_ok);
06447        mz_patch_ucbranch(ref2);
06448        __END_SHORT_JUMPS__(else_short_ok);
06449       }
06450       FOR_LOG(jitter->log_depth--);
06451 
06452       END_JIT_DATA(12);
06453 
06454       /* Return result */
06455 
06456       if ((g1 == 2) && (g2 == 2))
06457        return 2;
06458 
06459       if (nsrs1)
06460        jitter->need_set_rs = 1;
06461 
06462       return 1;
06463     }
06464   case scheme_unclosed_procedure_type:
06465     {
06466       Scheme_Closure_Data *data = (Scheme_Closure_Data *)obj;
06467       START_JIT_DATA();
06468 
06469       LOG_IT(("lambda\n"));
06470       
06471       mz_rs_sync();
06472       
06473       /* Allocate closure */
06474       generate_closure(data, jitter, 1);
06475       CHECK_LIMIT();
06476 
06477       generate_closure_fill(data, jitter);
06478 
06479       CHECK_LIMIT();
06480       if (target != JIT_R0)
06481         jit_movr_p(target, JIT_R0);
06482 
06483       END_JIT_DATA(13);
06484       return 0;
06485     }
06486   case scheme_let_value_type:
06487     {
06488       Scheme_Let_Value *lv = (Scheme_Let_Value *)obj;
06489       int ab = SCHEME_LET_AUTOBOX(lv), i, pos;
06490       START_JIT_DATA();
06491 
06492       LOG_IT(("let...\n"));
06493 
06494       if (lv->count == 1) {
06495        /* Expect one result: */
06496        generate_non_tail(lv->value, jitter, 0, 1); /* no sync */
06497        CHECK_LIMIT();
06498        if (ab) {
06499          pos = mz_remap(lv->position);
06500          mz_rs_ldxi(JIT_R2, pos);
06501          jit_str_p(JIT_R2, JIT_R0);
06502        } else {
06503          pos = mz_remap(lv->position);
06504          mz_rs_stxi(pos, JIT_R0);
06505        }
06506        CHECK_LIMIT();
06507       } else {
06508        /* Expect multiple results: */
06509        jit_insn *ref, *ref2, *ref3;
06510 
06511        generate_non_tail(lv->value, jitter, 1, 1);
06512        CHECK_LIMIT();
06513 
06514         mz_rs_sync();
06515     
06516        __START_SHORT_JUMPS__(1);
06517 
06518        /* Did we get multiple results? If not, go to error: */
06519        ref = jit_bnei_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES);
06520        /* Load count and result array: */
06521        jit_ldi_p(JIT_R2, &scheme_current_thread);
06522        jit_ldxi_l(JIT_R1, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.count);
06523        jit_ldxi_p(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array);
06524        CHECK_LIMIT();
06525        /* If we got the expected count, jump to installing values: */
06526        ref2 = jit_beqi_i(jit_forward(), JIT_R1, lv->count);
06527        /* Otherwise, jump to error: */
06528        ref3 = jit_jmpi(jit_forward());
06529        CHECK_LIMIT();
06530 
06531        /* Jump here when we didn't get multiple values. Set count to 1
06532           and "array" to single value: */
06533        mz_patch_branch(ref);
06534        jit_movi_i(JIT_R1, 1);
06535        jit_movr_p(JIT_R2, JIT_R0);
06536        CHECK_LIMIT();
06537          
06538        /* Error starts here: */
06539        mz_patch_ucbranch(ref3);
06540        JIT_UPDATE_THREAD_RSPTR_FOR_BRANCH_IF_NEEDED();
06541        mz_prepare(3);
06542        jit_pusharg_p(JIT_R2);
06543        jit_pusharg_i(JIT_R1);
06544        CHECK_LIMIT();
06545        jit_movi_i(JIT_V1, lv->count);
06546        jit_pusharg_i(JIT_V1);
06547        (void)mz_finish(lexical_binding_wrong_return_arity);
06548        CHECK_LIMIT();
06549 
06550        /* Continue with expected values; R2 has value array: */
06551        mz_patch_branch(ref2);
06552        __END_SHORT_JUMPS__(1);
06553        for (i = 0; i < lv->count; i++) {
06554          jit_ldxi_p(JIT_R1, JIT_R2, WORDS_TO_BYTES(i));
06555          if (ab) {
06556            pos = mz_remap(lv->position + i);
06557            jit_ldxi_p(JIT_R0, JIT_RUNSTACK, WORDS_TO_BYTES(pos));
06558            jit_str_p(JIT_R0, JIT_R1);
06559          } else {
06560            pos = mz_remap(lv->position + i);
06561            jit_stxi_p(WORDS_TO_BYTES(pos), JIT_RUNSTACK, JIT_R1);
06562          }
06563          CHECK_LIMIT();
06564        }
06565       }
06566 
06567       END_JIT_DATA(14);
06568 
06569       LOG_IT(("...in\n"));
06570 
06571       return generate(lv->body, jitter, is_tail, multi_ok, target);
06572     }
06573   case scheme_let_void_type:
06574     {
06575       Scheme_Let_Void *lv = (Scheme_Let_Void *)obj;
06576       int c = lv->count;
06577       START_JIT_DATA();
06578 
06579       LOG_IT(("letv...\n"));
06580 
06581       mz_rs_dec(c);
06582       CHECK_RUNSTACK_OVERFLOW();
06583       stack_safety(jitter, c, 0);
06584       mz_runstack_pushed(jitter, c);
06585 
06586       if (SCHEME_LET_AUTOBOX(lv)) {
06587        int i;
06588         mz_rs_sync();
06589        JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06590        for (i = 0; i < c; i++) {
06591          CHECK_LIMIT();
06592          (void)jit_movi_p(JIT_R0, scheme_undefined);
06593          mz_prepare(1);
06594          jit_pusharg_p(JIT_R0);
06595          (void)mz_finish(scheme_make_envunbox);
06596          jit_retval(JIT_R0);
06597          jit_stxi_p(WORDS_TO_BYTES(i), JIT_RUNSTACK, JIT_R0);
06598        }
06599       }
06600       CHECK_LIMIT();
06601 
06602       END_JIT_DATA(15);
06603 
06604       LOG_IT(("...in\n"));
06605 
06606       return generate(lv->body, jitter, is_tail, multi_ok, target);
06607     }
06608   case scheme_letrec_type:
06609     {
06610       Scheme_Letrec *l = (Scheme_Letrec *)obj;
06611       int i, nsrs;
06612       START_JIT_DATA();
06613 
06614       LOG_IT(("letrec...\n"));
06615 
06616       mz_rs_sync();
06617 
06618       /* Create unfinished closures */
06619       for (i = 0; i < l->count; i++) {
06620        ((Scheme_Closure_Data *)l->procs[i])->context = (Scheme_Object *)l;
06621        generate_closure((Scheme_Closure_Data *)l->procs[i], jitter, i + 1 == l->count);
06622        CHECK_LIMIT();
06623        jit_stxi_p(WORDS_TO_BYTES(i), JIT_RUNSTACK, JIT_R0);
06624       }
06625 
06626       /* Close them: */
06627       for (i = l->count; i--; ) {
06628        if (i != l->count - 1) {
06629          /* Last one we created is still in JIT_R0: */
06630          jit_ldxi_p(JIT_R0, JIT_RUNSTACK, WORDS_TO_BYTES(i));
06631        }
06632        generate_closure_fill((Scheme_Closure_Data *)l->procs[i], jitter);
06633        CHECK_LIMIT();
06634       }
06635 
06636       END_JIT_DATA(16);
06637 
06638       LOG_IT(("...in\n"));
06639 
06640       /* Assuming we can replace the last l->count, push closure info instead: */
06641       nsrs = jitter->need_set_rs;
06642       if (mz_try_runstack_pop(jitter, l->count)) {
06643         int i;
06644         for (i = l->count; i--; ) {
06645           Scheme_Closure_Data *data2 = (Scheme_Closure_Data *)l->procs[i];
06646           mz_runstack_closure_pushed(jitter, 
06647                                      (data2->num_params
06648                                       - ((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_HAS_REST)
06649                                          ? 1
06650                                          : 0)),
06651                                      (((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_PRESERVES_MARKS)
06652                                        ? NATIVE_PRESERVES_MARKS
06653                                        : 0)
06654                                       | ((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_SINGLE_RESULT)
06655                                          ? NATIVE_IS_SINGLE_RESULT
06656                                          : 0)));
06657         }
06658         jitter->need_set_rs = nsrs;
06659       }
06660 
06661       return generate(l->body, jitter, is_tail, multi_ok, target);
06662     }
06663   case scheme_let_one_type:
06664     {
06665       Scheme_Let_One *lv = (Scheme_Let_One *)obj;
06666       START_JIT_DATA();
06667 
06668       LOG_IT(("leto...\n"));
06669 
06670       mz_runstack_skipped(jitter, 1);
06671 
06672       PAUSE_JIT_DATA();
06673       generate_non_tail(lv->value, jitter, 0, 1); /* no sync */
06674       RESUME_JIT_DATA();
06675       CHECK_LIMIT();
06676       
06677       mz_runstack_unskipped(jitter, 1);
06678 
06679       mz_rs_dec(1);
06680       CHECK_RUNSTACK_OVERFLOW();
06681       mz_runstack_pushed(jitter, 1);
06682 
06683       mz_rs_str(JIT_R0);
06684        
06685       END_JIT_DATA(17);
06686 
06687       LOG_IT(("...in\n"));
06688 
06689       mz_RECORD_STATUS(mz_RS_R0_HAS_RUNSTACK0);
06690 
06691       return generate(lv->body, jitter, is_tail, multi_ok, target);
06692     }
06693   case scheme_with_cont_mark_type:
06694     {
06695       Scheme_With_Continuation_Mark *wcm = (Scheme_With_Continuation_Mark *)obj;
06696       START_JIT_DATA();
06697 
06698       LOG_IT(("wcm...\n"));
06699 
06700       /* Key: */
06701       generate_non_tail(wcm->key, jitter, 0, 1); /* sync'd below */
06702       CHECK_LIMIT();
06703       if (SCHEME_TYPE(wcm->val) > _scheme_values_types_) {
06704        /* No need to push mark onto value stack: */
06705        jit_movr_p(JIT_V1, JIT_R0);
06706        generate_non_tail(wcm->val, jitter, 0, 1); /* sync'd below */
06707        CHECK_LIMIT();
06708       } else {
06709        mz_pushr_p(JIT_R0);
06710        generate_non_tail(wcm->val, jitter, 0, 1); /* sync'd below */
06711        CHECK_LIMIT();
06712        mz_popr_p(JIT_V1); /* sync'd below */
06713       }
06714 
06715       mz_rs_sync();
06716       JIT_UPDATE_THREAD_RSPTR_IF_NEEDED();
06717 
06718       mz_prepare(2);
06719       jit_pusharg_p(JIT_R0);
06720       jit_pusharg_p(JIT_V1);
06721       (void)mz_finish(scheme_set_cont_mark);
06722       CHECK_LIMIT();
06723 
06724       END_JIT_DATA(18);
06725 
06726       LOG_IT(("...in\n"));
06727        
06728       return generate(wcm->body, jitter, is_tail, multi_ok, target);
06729     }
06730   case scheme_quote_syntax_type:
06731     {
06732       Scheme_Quote_Syntax *qs = (Scheme_Quote_Syntax *)obj;
06733       int i, c, p;
06734       START_JIT_DATA();
06735       
06736       LOG_IT(("quote-syntax\n"));
06737       
06738       i = qs->position;
06739       c = mz_remap(qs->depth);
06740       p = qs->midpoint;
06741       
06742       mz_rs_sync();
06743 
06744       jit_movi_i(JIT_R0, WORDS_TO_BYTES(c));
06745       jit_movi_i(JIT_R1, WORDS_TO_BYTES(i + p + 1));
06746       jit_movi_i(JIT_R2, WORDS_TO_BYTES(p));
06747       (void)jit_calli(quote_syntax_code);
06748 
06749       CHECK_LIMIT();
06750       if (target != JIT_R0)
06751         jit_movr_p(target, JIT_R0);
06752       
06753       END_JIT_DATA(10);
06754 
06755       return 1;
06756     }
06757   default:
06758     {
06759       int retptr;
06760       Scheme_Type type = SCHEME_TYPE(obj);
06761       START_JIT_DATA();
06762 
06763       /* Other parts of the JIT rely on this code modifying R0, only */
06764 
06765       LOG_IT(("const\n"));
06766 
06767       /* Avoid compiling closures multiple times: */
06768       if (jitter->retain_start) {
06769        if (type == scheme_closure_type) {
06770          /* Empty closure? If so, compile the code and get a native closure: */
06771          Scheme_Closure *c = (Scheme_Closure *)obj;
06772          if (ZERO_SIZED_CLOSUREP(c))
06773            obj = scheme_jit_closure((Scheme_Object *)c->code, NULL);
06774        } else if (type == scheme_case_closure_type) {
06775          /* Empty case closure? Turn in into a JITted empty case closure. */
06776          obj = scheme_unclose_case_lambda(obj, 1);
06777        }
06778       }
06779 
06780       if (!SCHEME_INTP(obj)
06781          && !SAME_OBJ(obj, scheme_true)
06782          && !SAME_OBJ(obj, scheme_false)
06783          && !SAME_OBJ(obj, scheme_void)
06784          && !SAME_OBJ(obj, scheme_null)) {
06785        retptr = mz_retain(obj);
06786       } else
06787        retptr = 0;
06788 
06789 #ifdef JIT_PRECISE_GC
06790       if (retptr)
06791        mz_load_retained(jitter, target, retptr);
06792       else
06793 #endif
06794        (void)jit_patchable_movi_p(target, obj); /* !! */
06795 
06796       END_JIT_DATA(19);
06797       return 1;
06798     }
06799   }
06800 }
06801 
06802 /*========================================================================*/
06803 /*                          procedure codegen                             */
06804 /*========================================================================*/
06805 
06806 static void generate_function_prolog(mz_jit_state *jitter, void *code, int max_let_depth)
06807 {
06808   int in;
06809   START_JIT_DATA();
06810 
06811   jit_prolog(3);
06812     
06813   in = jit_arg_p();
06814   jit_getarg_p(JIT_R0, in); /* closure */
06815   in = jit_arg_i();
06816   jit_getarg_i(JIT_R1, in); /* argc */
06817   in = jit_arg_p();
06818   jit_getarg_p(JIT_R2, in); /* argv */
06819 
06820   jit_ldi_p(JIT_RUNSTACK, &MZ_RUNSTACK);
06821 
06822   END_JIT_DATA(1);
06823 }
06824 
06825 static int generate_function_getarg(mz_jit_state *jitter, int has_rest, int num_params)
06826 {
06827   int i, cnt;
06828   jit_insn *ref;
06829   int set_ref;
06830 
06831   mz_push_locals();
06832 
06833   /* If rands == runstack and there are no rest args, set runstack
06834      base to runstack + rands (and don't copy rands), otherwise set
06835      base to runstack and proceed normally. Implement this by
06836      optimistically assuming rands == runstack, so that there's just
06837      one jump. Skip this optimization when the procedure has
06838      rest args, because we'll have to copy anyway. */
06839   if (!has_rest && num_params) {
06840     jit_lshi_l(JIT_RUNSTACK_BASE, JIT_R1, JIT_LOG_WORD_SIZE);
06841     jit_addr_p(JIT_RUNSTACK_BASE, JIT_R2, JIT_RUNSTACK_BASE);
06842     __START_TINY_OR_SHORT_JUMPS__(num_params < 10, num_params < 100);
06843     ref = jit_beqr_p(jit_forward(), JIT_RUNSTACK, JIT_R2);
06844     __END_TINY_OR_SHORT_JUMPS__(num_params < 10, num_params < 100);
06845     set_ref = 1;
06846   } else {
06847     ref = 0;
06848     set_ref = 0;
06849   }
06850   jit_movr_p(JIT_RUNSTACK_BASE, JIT_RUNSTACK);
06851 
06852   /* Make stack room for arguments: */
06853   cnt = num_params;
06854   if (cnt) {
06855     CHECK_LIMIT();
06856     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(cnt));
06857     CHECK_RUNSTACK_OVERFLOW();
06858     if (has_rest)
06859       --cnt;
06860   }
06861 
06862   /* Extract arguments to runstack: */
06863   for (i = cnt; i--; ) {
06864     jit_ldxi_p(JIT_V1, JIT_R2, WORDS_TO_BYTES(i));
06865     jit_stxi_p(WORDS_TO_BYTES(i), JIT_RUNSTACK, JIT_V1);
06866     CHECK_LIMIT();
06867   }
06868 
06869   if (set_ref) {
06870     __START_TINY_OR_SHORT_JUMPS__(num_params < 10, num_params < 100);
06871     mz_patch_branch(ref);
06872     __END_TINY_OR_SHORT_JUMPS__(num_params < 10, num_params < 100);
06873   }
06874 
06875   return cnt;
06876 }
06877 
06878 static int save_struct_temp(mz_jit_state *jitter)
06879 {
06880 #ifdef MZ_USE_JIT_PPC
06881   jit_movr_p(JIT_V(3), JIT_V1);
06882 #endif
06883 #ifdef MZ_USE_JIT_I386
06884 # ifdef X86_ALIGN_STACK
06885   mz_set_local_p(JIT_V1, JIT_LOCAL3);
06886 # else
06887   jit_pushr_p(JIT_V1);
06888 # endif
06889 #endif
06890   return 1;
06891 }
06892 
06893 static int restore_struct_temp(mz_jit_state *jitter, int reg)
06894 {
06895 #ifdef MZ_USE_JIT_PPC
06896   jit_movr_p(reg, JIT_V(3));
06897 #endif
06898 #ifdef MZ_USE_JIT_I386
06899 # ifdef X86_ALIGN_STACK
06900   mz_get_local_p(reg, JIT_LOCAL3);
06901 # else
06902   jit_popr_p(reg);
06903 # endif
06904 #endif
06905   return 1;
06906 }
06907 
06908 static int do_generate_common(mz_jit_state *jitter, void *_data)
06909 {
06910   int in, i, ii, iii;
06911   GC_CAN_IGNORE jit_insn *ref, *ref2;
06912 
06913   /* *** check_arity_code *** */
06914   /* Called as a function: */
06915   check_arity_code = (Native_Check_Arity_Proc)jit_get_ip().ptr;
06916   jit_prolog(3); /* only need 1 argument, but return path overlaps with proc conventions */
06917   in = jit_arg_p();
06918   jit_getarg_p(JIT_R0, in); /* closure */
06919   in = jit_arg_p();
06920   jit_getarg_i(JIT_R2, in); /* argc */
06921   mz_push_locals();
06922   jit_movi_i(JIT_R1, -1);
06923   jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
06924   jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code);
06925   jit_jmpr(JIT_V1); /* leads to a jit_ret() that assumes 3 arguments */
06926   CHECK_LIMIT();
06927 
06928   /* *** get_arity_code *** */
06929   /* Called as a function: */
06930   get_arity_code = (Native_Get_Arity_Proc)jit_get_ip().ptr;
06931   jit_prolog(3); /* only need 1 argument, but return path overlaps with proc conventions */
06932   in = jit_arg_p();
06933   jit_getarg_p(JIT_R0, in); /* closure */
06934   mz_push_locals();
06935   jit_movi_i(JIT_R1, -1);
06936   (void)jit_movi_p(JIT_R2, 0x0);
06937   jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
06938   jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code);
06939   jit_jmpr(JIT_V1); /* leads to a jit_ret() that assumes 3 arguments */
06940   CHECK_LIMIT();
06941 
06942   /* *** bad_result_arity_code *** */
06943   /* Jumped-to from non-tail contexts  */
06944   bad_result_arity_code = (Native_Get_Arity_Proc)jit_get_ip().ptr;
06945   jit_ldi_p(JIT_R2, &scheme_current_thread);
06946   jit_ldxi_l(JIT_R1, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.count);
06947   jit_ldxi_p(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array);
06948   CHECK_LIMIT();
06949   mz_prepare(3);
06950   jit_pusharg_p(JIT_R2);
06951   jit_pusharg_i(JIT_R1);
06952   CHECK_LIMIT();
06953   jit_movi_i(JIT_V1, 1);
06954   jit_pusharg_i(JIT_V1);
06955   (void)mz_finish(call_wrong_return_arity);
06956   CHECK_LIMIT();
06957 
06958   /* *** unbound_global_code *** */
06959   unbound_global_code = jit_get_ip().ptr;
06960   JIT_UPDATE_THREAD_RSPTR();
06961   mz_prepare(1);
06962   jit_pusharg_p(JIT_R2);
06963   (void)mz_finish(scheme_unbound_global);
06964   CHECK_LIMIT();
06965 
06966   /* *** quote_syntax_code *** */
06967   /* R0 is WORDS_TO_BYTES(c), R1 is WORDS_TO_BYTES(i+p+1), R2 is WORDS_TO_BYTES(p) */
06968   quote_syntax_code = jit_get_ip().ptr;
06969   mz_prolog(JIT_V1);
06970   __START_SHORT_JUMPS__(1);
06971   /* Load global array: */
06972   jit_ldxr_p(JIT_V1, JIT_RUNSTACK, JIT_R0);
06973 #ifdef JIT_PRECISE_GC
06974   /* Save global-array index before we lose it: */
06975   mz_set_local_p(JIT_R0, JIT_LOCAL3);
06976 #endif
06977   /* Load syntax object: */
06978   jit_ldxr_p(JIT_R0, JIT_V1, JIT_R1);
06979   /* Is it null? */
06980   ref = jit_bnei_p(jit_forward(), JIT_R0, 0x0);
06981   CHECK_LIMIT();
06982   /* Syntax object is NULL, so we need to create it. */
06983   jit_ldxr_p(JIT_R0, JIT_V1, JIT_R2); /* put element at p in R0 */
06984 #ifndef JIT_PRECISE_GC
06985   /* Save global array: */
06986   mz_set_local_p(JIT_V1, JIT_LOCAL3);
06987 #endif
06988   /* Move R1 to V1 to save it: */
06989   jit_movr_p(JIT_V1, JIT_R1);
06990   /* Compute i in JIT_R1: */
06991   jit_subr_p(JIT_R1, JIT_R1, JIT_R2);
06992   jit_subi_p(JIT_R1, JIT_R1, WORDS_TO_BYTES(1));
06993   jit_rshi_ul(JIT_R1, JIT_R1, JIT_LOG_WORD_SIZE);
06994   CHECK_LIMIT();
06995   /* Call scheme_delayed_rename: */
06996   JIT_UPDATE_THREAD_RSPTR();
06997   CHECK_LIMIT();
06998   mz_prepare(2);
06999   jit_pusharg_l(JIT_R1);
07000   jit_pusharg_p(JIT_R0);
07001   (void)mz_finish(scheme_delayed_rename);
07002   CHECK_LIMIT();
07003   jit_retval(JIT_R0);
07004   /* Restore global array into JIT_R1, and put computed element at i+p+1: */
07005 #ifdef JIT_PRECISE_GC
07006   mz_get_local_p(JIT_R1, JIT_LOCAL3);
07007   jit_ldxr_p(JIT_R1, JIT_RUNSTACK, JIT_R1);
07008 #else
07009   mz_get_local_p(JIT_R1, JIT_LOCAL3);
07010 #endif
07011   jit_stxr_p(JIT_V1, JIT_R1, JIT_R0);
07012   mz_patch_branch(ref);
07013   __END_SHORT_JUMPS__(1);
07014   mz_epilog(JIT_V1);
07015 
07016   /* *** bad_[m]{car,cdr,...}_code *** */
07017   /* Bad argument is in R0 for car/cdr, R2 otherwise */
07018   for (i = 0; i < 8; i++) {
07019     void *code;
07020     
07021     code = jit_get_ip().ptr;
07022     switch (i) {
07023     case 0:
07024       bad_car_code = code;
07025       break;
07026     case 1:
07027       bad_cdr_code = code;
07028       break;
07029     case 2:
07030       bad_caar_code = code;
07031       break;
07032     case 3:
07033       bad_cadr_code = code;
07034       break;
07035     case 4:
07036       bad_cdar_code = code;
07037       break;
07038     case 5:
07039       bad_cddr_code = code;      
07040       break;
07041     case 6:
07042       bad_mcar_code = code;
07043       break;
07044     case 7:
07045       bad_mcdr_code = code;
07046       break;
07047     }
07048     mz_prolog(JIT_R1);
07049     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
07050     CHECK_RUNSTACK_OVERFLOW();
07051     if ((i < 2) || (i > 5)) {
07052       jit_str_p(JIT_RUNSTACK, JIT_R0);
07053     } else {
07054       jit_str_p(JIT_RUNSTACK, JIT_R2);
07055     }
07056     JIT_UPDATE_THREAD_RSPTR();
07057     CHECK_LIMIT();
07058     jit_movi_i(JIT_R1, 1);
07059     jit_prepare(2);
07060     jit_pusharg_p(JIT_RUNSTACK);
07061     jit_pusharg_i(JIT_R1);
07062     switch (i) {
07063     case 0:
07064       (void)mz_finish(scheme_checked_car);
07065       break;
07066     case 1:
07067       (void)mz_finish(scheme_checked_cdr);
07068       break;
07069     case 2:
07070       (void)mz_finish(scheme_checked_caar);
07071       break;
07072     case 3:
07073       (void)mz_finish(scheme_checked_cadr);
07074       break;
07075     case 4:
07076       (void)mz_finish(scheme_checked_cdar);
07077       break;
07078     case 5:
07079       (void)mz_finish(scheme_checked_cddr);
07080       break;
07081     case 6:
07082       (void)mz_finish(scheme_checked_mcar);
07083       break;
07084     case 7:
07085       (void)mz_finish(scheme_checked_mcdr);
07086       break;
07087     }
07088     CHECK_LIMIT();
07089 
07090     register_sub_func(jitter, code, scheme_false);
07091   }
07092 
07093   /* *** bad_set_{car,cdr}_code *** */
07094   /* Bad argument is in R0, other is in R1 */
07095   for (i = 0; i < 2; i++) {
07096     void *code;
07097     code = jit_get_ip().ptr;
07098     switch (i) {
07099     case 0:
07100       bad_set_mcar_code = code;
07101       break;
07102     case 1:
07103       bad_set_mcdr_code = code;
07104       break;
07105     }
07106     mz_prolog(JIT_R2);
07107     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(2));
07108     CHECK_RUNSTACK_OVERFLOW();
07109     jit_str_p(JIT_RUNSTACK, JIT_R0);
07110     jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_R1);
07111     JIT_UPDATE_THREAD_RSPTR();
07112     CHECK_LIMIT();
07113     jit_movi_i(JIT_R1, 2);
07114     jit_prepare(2);
07115     jit_pusharg_p(JIT_RUNSTACK);
07116     jit_pusharg_i(JIT_R1);
07117     switch (i) {
07118     case 0:
07119       (void)mz_finish(scheme_checked_set_mcar);
07120       break;
07121     case 1:
07122       (void)mz_finish(scheme_checked_set_mcdr);
07123       break;
07124     }
07125     CHECK_LIMIT();
07126     register_sub_func(jitter, code, scheme_false);
07127   }
07128 
07129   /* *** bad_unbox_code *** */
07130   /* R0 is argument */
07131   bad_unbox_code = jit_get_ip().ptr;
07132   mz_prolog(JIT_R1);
07133   jit_prepare(1);
07134   jit_pusharg_i(JIT_R0);
07135   (void)mz_finish(scheme_unbox);
07136   CHECK_LIMIT();
07137   register_sub_func(jitter, bad_unbox_code, scheme_false);
07138 
07139   /* *** bad_vector_length_code *** */
07140   /* R0 is argument */
07141   bad_vector_length_code = jit_get_ip().ptr;
07142   mz_prolog(JIT_R1);
07143   jit_prepare(1);
07144   jit_pusharg_i(JIT_R0);
07145   (void)mz_finish(scheme_vector_length);
07146   CHECK_LIMIT();
07147   register_sub_func(jitter, bad_vector_length_code, scheme_false);
07148 
07149   /* *** call_original_unary_arith_code *** */
07150   /* R0 is arg, R2 is code pointer, V1 is return address */
07151   for (i = 0; i < 3; i++) {
07152     int argc, j;
07153     void *code;
07154     for (j = 0; j < 2; j++) {
07155       code = jit_get_ip().ptr;
07156       if (!i) {
07157        if (!j)
07158          call_original_unary_arith_code = code;
07159        else
07160          call_original_unary_arith_for_branch_code = code;
07161        argc = 1;
07162       } else if (i == 1) {
07163        if (!j)
07164          call_original_binary_arith_code = code;
07165        else
07166          call_original_binary_arith_for_branch_code = code;
07167        argc = 2;
07168       } else {
07169        if (!j)
07170          call_original_binary_rev_arith_code = code;
07171        else
07172          call_original_binary_rev_arith_for_branch_code = code;
07173        argc = 2;
07174       }
07175       jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(argc));
07176       CHECK_RUNSTACK_OVERFLOW();
07177       if (i == 2) {
07178        jit_str_p(JIT_RUNSTACK, JIT_R0);
07179        jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_R1);
07180       } else if (i == 1) {
07181        jit_str_p(JIT_RUNSTACK, JIT_R1);
07182        jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_R0);
07183       } else {
07184        jit_str_p(JIT_RUNSTACK, JIT_R0);
07185       }
07186       jit_movi_i(JIT_R1, argc);
07187       if (!j) {
07188         /* For stack-trace reporting, stuff return address into LOCAL2 */
07189         mz_set_local_p(JIT_V1, JIT_LOCAL2);
07190       }
07191       JIT_UPDATE_THREAD_RSPTR();
07192       mz_prepare(2);
07193       jit_pusharg_p(JIT_RUNSTACK);
07194       jit_pusharg_p(JIT_R1);
07195       (void)mz_finishr(JIT_R2);
07196       CHECK_LIMIT();
07197       jit_retval(JIT_R0);
07198       VALIDATE_RESULT(JIT_R0);
07199       jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(argc));
07200       JIT_UPDATE_THREAD_RSPTR();
07201       if (!j) {
07202        jit_jmpr(JIT_V1);
07203       } else {
07204        /* In for_branch mode, V1 is target for false, LOCAL2 is target for true */
07205        mz_get_local_p(JIT_R1, JIT_LOCAL2);
07206        __START_TINY_JUMPS__(1);
07207        ref = jit_beqi_p(jit_forward(), JIT_R0, scheme_true);
07208        jit_jmpr(JIT_V1);
07209        mz_patch_branch(ref);
07210        jit_jmpr(JIT_R1);
07211        __END_TINY_JUMPS__(1);
07212       }
07213       CHECK_LIMIT();
07214 
07215       register_sub_func(jitter, code, scheme_void);
07216     }
07217   }
07218 
07219   /* *** on_demand_jit_[arity_]code *** */
07220   /* Used as the code stub for a closure whose
07221      code is not yet compiled. See generate_function_prolog
07222      for the state of registers on entry */
07223   on_demand_jit_code = jit_get_ip().ptr;
07224   jit_prolog(3);
07225   in = jit_arg_p();
07226   jit_getarg_p(JIT_R0, in); /* closure */
07227   in = jit_arg_i();
07228   jit_getarg_i(JIT_R1, in); /* argc */
07229   in = jit_arg_p();
07230   jit_getarg_p(JIT_R2, in); /* argv */
07231   CHECK_LIMIT();
07232   jit_ldi_p(JIT_RUNSTACK, &MZ_RUNSTACK);
07233   mz_push_locals();
07234   on_demand_jit_arity_code = jit_get_ip().ptr; /* <<<- arity variant starts here */
07235   jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(3));
07236   CHECK_RUNSTACK_OVERFLOW();
07237   jit_str_p(JIT_RUNSTACK, JIT_R0);
07238   jit_lshi_ul(JIT_R1, JIT_R1, 0x1);
07239   jit_ori_ul(JIT_R1, JIT_R1, 0x1);
07240   CHECK_LIMIT();
07241   jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_R1);
07242   jit_stxi_p(WORDS_TO_BYTES(2), JIT_RUNSTACK, JIT_R2);
07243   JIT_UPDATE_THREAD_RSPTR();
07244   (void)jit_calli(on_demand); /* DARWIN: stack needs to be 16-byte aligned */
07245   CHECK_LIMIT();
07246   /* Restore registers and runstack, and jump to arity checking
07247      of newly-created code when argv == runstack (i.e., a tail call): */
07248   jit_ldr_p(JIT_R0, JIT_RUNSTACK);
07249   jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(1));
07250   jit_rshi_ul(JIT_R1, JIT_R1, 0x1);
07251   jit_ldxi_p(JIT_R2, JIT_RUNSTACK, WORDS_TO_BYTES(2));
07252   jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(3));
07253   CHECK_LIMIT();
07254   ref = jit_bner_p(jit_forward(), JIT_RUNSTACK, JIT_R2);
07255   /* Also, check that the runstack is big enough with the revised
07256      max_let_depth. We can use JIT_V2 here because RUNSTACK_BASE is not
07257      yet ready: */
07258   jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
07259   jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->max_let_depth);
07260   jit_ldi_p(JIT_V2, &MZ_RUNSTACK_START);
07261   jit_subr_ul(JIT_V2, JIT_RUNSTACK, JIT_V2);
07262   ref2 = jit_bltr_ul(jit_forward(), JIT_V2, JIT_V1);
07263   CHECK_LIMIT();
07264   /* This is the tail-call fast path: */
07265   jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
07266   jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code);
07267   /* Set runstack base to end of arguments on runstack: */
07268   jit_movr_p(JIT_RUNSTACK_BASE, JIT_R1);
07269   jit_lshi_ul(JIT_RUNSTACK_BASE, JIT_RUNSTACK_BASE, JIT_LOG_WORD_SIZE);
07270   jit_addr_p(JIT_RUNSTACK_BASE, JIT_RUNSTACK_BASE, JIT_RUNSTACK);
07271   jit_jmpr(JIT_V1);
07272   CHECK_LIMIT();
07273   /* Slower path (non-tail) when argv != runstack. */
07274   mz_patch_branch(ref);
07275   mz_patch_branch(ref2);
07276   CHECK_LIMIT();
07277   JIT_UPDATE_THREAD_RSPTR();
07278   mz_prepare(3);
07279   jit_pusharg_p(JIT_R2);
07280   jit_pusharg_p(JIT_R1);
07281   jit_pusharg_p(JIT_R0);
07282   (void)mz_finish(_scheme_apply_multi_from_native);
07283   CHECK_LIMIT();
07284   mz_pop_locals();
07285   jit_ret();
07286   CHECK_LIMIT();
07287   register_helper_func(jitter, on_demand_jit_code);
07288 
07289   /* *** app_values_tail_slow_code *** */
07290   /* RELIES ON jit_prolog(3) FROM ABOVE */
07291   /* Rator in V1, arguments are in thread's multiple-values cells. */
07292   app_values_tail_slow_code = jit_get_ip().ptr;
07293   JIT_UPDATE_THREAD_RSPTR();
07294   mz_prepare(1);
07295   jit_pusharg_p(JIT_V1);
07296   (void)mz_finish(tail_call_with_values_from_multiple_result);
07297   jit_retval(JIT_R0);
07298   VALIDATE_RESULT(JIT_R0);
07299   /* Return: */
07300   mz_pop_locals();
07301   jit_ret();  
07302   CHECK_LIMIT();
07303 
07304   /* *** finish_tail_call_[fixup_]code *** */
07305   /* RELIES ON jit_prolog(3) FROM ABOVE */
07306   finish_tail_call_code = jit_get_ip().ptr;
07307   generate_finish_tail_call(jitter, 0);
07308   CHECK_LIMIT();
07309   register_helper_func(jitter, finish_tail_call_code);
07310   finish_tail_call_fixup_code = jit_get_ip().ptr;
07311   generate_finish_tail_call(jitter, 2);
07312   CHECK_LIMIT();
07313   register_helper_func(jitter, finish_tail_call_fixup_code);
07314 
07315   /* *** get_stack_pointer_code *** */
07316   get_stack_pointer_code = jit_get_ip().ptr;
07317   jit_leaf(0);
07318   jit_movr_p(JIT_R0, JIT_STACK_FRAME);
07319   /* Get frame pointer of caller... */
07320 #ifdef MZ_USE_JIT_PPC
07321   jit_ldr_p(JIT_R0, JIT_R0);
07322 #endif
07323 #ifdef MZ_USE_JIT_I386
07324   jit_ldr_p(JIT_R0, JIT_R0);
07325 #endif
07326   jit_movr_p(JIT_RET, JIT_R0);
07327   jit_ret();
07328   CHECK_LIMIT();
07329 
07330   /* *** stack_cache_pop_code *** */
07331   /* DANGER: this code must save and restore (or avoid)
07332      any registers that a function call would normally save 
07333      and restore. JIT_AUX, which is used by things like jit_ldi,
07334      is such a register for PPC. */
07335   stack_cache_pop_code = jit_get_ip().ptr;
07336   jit_movr_p(JIT_R0, JIT_RET);
07337 #ifdef MZ_USE_JIT_PPC
07338   jit_movr_p(JIT_R(3), JIT_AUX);
07339 #endif
07340   /* Decrement stack_cache_stack_pos */
07341   jit_ldi_l(JIT_R1, &stack_cache_stack_pos);
07342   jit_subi_i(JIT_R2, JIT_R1, 1);
07343   jit_sti_l(&stack_cache_stack_pos, JIT_R2);
07344   CHECK_LIMIT();
07345   /* Extract old return address and jump to it */
07346   jit_lshi_l(JIT_R1, JIT_R1, (JIT_LOG_WORD_SIZE + 2));
07347   jit_addi_l(JIT_R1, JIT_R1, (int)&((Stack_Cache_Elem *)0x0)->orig_return_address);
07348   (void)jit_movi_p(JIT_R2, &stack_cache_stack);
07349   jit_ldxr_p(JIT_R2, JIT_R2, JIT_R1);
07350   jit_movr_p(JIT_RET, JIT_R0);
07351 #ifdef MZ_USE_JIT_PPC
07352   jit_movr_p(JIT_AUX, JIT_R(3));
07353 #endif
07354   jit_jmpr(JIT_R2);
07355   CHECK_LIMIT();
07356 
07357   /* *** bad_app_vals_target *** */
07358   /* Non-proc is in R0 */
07359   bad_app_vals_target = jit_get_ip().ptr;
07360   JIT_UPDATE_THREAD_RSPTR();
07361   mz_prepare(1);
07362   jit_pusharg_p(JIT_R0);
07363   (void)mz_finish(raise_bad_call_with_values);
07364   /* Doesn't return */
07365   CHECK_LIMIT();
07366 
07367   /* *** app_values[_multi]_slow_code *** */
07368   /* Rator in V1, arguments are in thread's multiple-values cells. */
07369   for (i = 0; i < 2; i++) {
07370     if (i)
07371       app_values_multi_slow_code = jit_get_ip().ptr;
07372     else
07373       app_values_slow_code = jit_get_ip().ptr;
07374     mz_prolog(JIT_R1);
07375     JIT_UPDATE_THREAD_RSPTR();
07376     mz_prepare(1);
07377     jit_pusharg_p(JIT_V1);
07378     if (i) {
07379       (void)mz_finish(call_with_values_from_multiple_result_multi);
07380     } else {
07381       (void)mz_finish(call_with_values_from_multiple_result);
07382     }
07383     jit_retval(JIT_R0);
07384     VALIDATE_RESULT(JIT_R0);
07385     mz_epilog(JIT_R1);
07386     CHECK_LIMIT();
07387   }
07388 
07389   /* *** {vector,string,bytes}_{ref,set}_[check_index_]code *** */
07390   /* R0 is vector/string/bytes, R1 is index (Scheme number in check-index mode), 
07391      V1 is vector/string/bytes offset in non-check-index mode (and for
07392      vector, it includes the offset to the start of the elements array.
07393      In set mode, value is on run stack. */
07394   for (iii = 0; iii < 2; iii++) { /* ref, set */
07395     for (ii = 0; ii < 3; ii++) { /* vector, string, bytes */
07396       for (i = 0; i < 2; i++) { /* check index? */
07397        jit_insn *ref, *reffail;
07398        Scheme_Type ty;
07399        int offset, count_offset, log_elem_size;
07400 
07401        switch (ii) {
07402        case 0:
07403          ty = scheme_vector_type;
07404          offset = (int)&SCHEME_VEC_ELS(0x0);
07405          count_offset = (int)&SCHEME_VEC_SIZE(0x0);
07406          log_elem_size = JIT_LOG_WORD_SIZE;
07407          if (!iii) {
07408            if (!i) {
07409              vector_ref_code = jit_get_ip().ptr;
07410            } else {
07411              vector_ref_check_index_code = jit_get_ip().ptr;
07412            }
07413          } else {
07414            if (!i) {
07415              vector_set_code = jit_get_ip().ptr;
07416            } else {
07417              vector_set_check_index_code = jit_get_ip().ptr;
07418            }
07419          }
07420          break;
07421        case 1:
07422          ty = scheme_char_string_type;
07423          offset = (int)&SCHEME_CHAR_STR_VAL(0x0);
07424          count_offset = (int)&SCHEME_CHAR_STRLEN_VAL(0x0);
07425          log_elem_size = LOG_MZCHAR_SIZE;
07426          if (!iii) {
07427            if (!i) {
07428              string_ref_code = jit_get_ip().ptr;
07429            } else {
07430              string_ref_check_index_code = jit_get_ip().ptr;
07431            }
07432          } else {
07433            if (!i) {
07434              string_set_code = jit_get_ip().ptr;
07435            } else {
07436              string_set_check_index_code = jit_get_ip().ptr;
07437            }
07438          }
07439          break;
07440        default:
07441        case 2:
07442          ty = scheme_byte_string_type;
07443          offset = (int)&SCHEME_BYTE_STR_VAL(0x0);
07444          count_offset = (int)&SCHEME_BYTE_STRLEN_VAL(0x0);
07445          log_elem_size = 0;
07446          if (!iii) {
07447            if (!i) {
07448              bytes_ref_code = jit_get_ip().ptr;
07449            } else {
07450              bytes_ref_check_index_code = jit_get_ip().ptr;
07451            }
07452          } else {
07453            if (!i) {
07454              bytes_set_code = jit_get_ip().ptr;
07455            } else {
07456              bytes_set_check_index_code = jit_get_ip().ptr;
07457            }
07458          }
07459          break;
07460        }
07461 
07462        __START_TINY_JUMPS__(1);
07463 
07464        mz_prolog(JIT_R2);
07465 
07466        ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
07467        CHECK_LIMIT();
07468 
07469        /* Slow path: */
07470        reffail = _jit.x.pc;
07471        if (!i) {
07472          jit_lshi_ul(JIT_R1, JIT_R1, 1);
07473          jit_ori_ul(JIT_R1, JIT_R1, 0x1);
07474        }
07475        jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(2));
07476         CHECK_RUNSTACK_OVERFLOW();
07477        jit_str_p(JIT_RUNSTACK, JIT_R0);
07478        jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_R1);
07479        if (!iii) {
07480          jit_movi_i(JIT_R1, 2);
07481        } else {
07482          /* In set mode, value was already on run stack */
07483          jit_movi_i(JIT_R1, 3);
07484        }
07485        JIT_UPDATE_THREAD_RSPTR();
07486        jit_prepare(2);
07487        jit_pusharg_p(JIT_RUNSTACK);
07488        jit_pusharg_i(JIT_R1);
07489        switch (ii) {
07490        case 0:
07491          if (!iii) {
07492            (void)mz_finish(scheme_checked_vector_ref);
07493          } else {
07494            (void)mz_finish(scheme_checked_vector_set);
07495          }
07496          break;
07497        case 1:
07498          if (!iii) {
07499            (void)mz_finish(scheme_checked_string_ref);
07500            /* might return, if char was outside Latin-1 */
07501            jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(2));
07502            JIT_UPDATE_THREAD_RSPTR();
07503            jit_retval(JIT_R0);
07504            mz_epilog(JIT_R2);
07505          } else {
07506            (void)mz_finish(scheme_checked_string_set);
07507          }
07508          break;
07509        case 2:
07510          if (!iii) {
07511            (void)mz_finish(scheme_checked_byte_string_ref);
07512          } else {
07513            (void)mz_finish(scheme_checked_byte_string_set);
07514          }
07515          break;
07516        }
07517        /* doesn't return */
07518        CHECK_LIMIT();
07519 
07520         /* Continue fast path */
07521     
07522        mz_patch_branch(ref);
07523        if (i) {
07524          (void)jit_bmci_ul(reffail, JIT_R1, 0x1);
07525          (void)jit_blei_l(reffail, JIT_R1, 0x0);
07526        }
07527        jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
07528        (void)jit_bnei_i(reffail, JIT_R2, ty);
07529         if (iii) {
07530           jit_ldxi_s(JIT_R2, JIT_R0, &(MZ_OPT_HASH_KEY((Scheme_Inclhash_Object *)0x0)));
07531           (void)jit_bmsi_ul(reffail, JIT_R2, 0x1);
07532         }
07533        jit_ldxi_i(JIT_R2, JIT_R0, count_offset);
07534         CHECK_LIMIT();
07535        if (i) {
07536          /* index from expression: */
07537          jit_rshi_ul(JIT_V1, JIT_R1, 1);
07538          (void)jit_bler_ul(reffail, JIT_R2, JIT_V1);
07539          if (log_elem_size)
07540            jit_lshi_ul(JIT_V1, JIT_V1, log_elem_size);
07541          if (!ii) /* vector */
07542            jit_addi_p(JIT_V1, JIT_V1, offset);
07543        } else {
07544          /* constant index supplied: */
07545          (void)jit_bler_ul(reffail, JIT_R2, JIT_R1);
07546        }
07547        if (!iii) {
07548          /* ref mode: */
07549          switch (ii) {
07550          case 0: /* vector */
07551            jit_ldxr_p(JIT_R0, JIT_R0, JIT_V1);
07552            break;
07553          case 1: /* string */
07554            jit_ldxi_p(JIT_R2, JIT_R0, offset);
07555            jit_ldxr_i(JIT_R2, JIT_R2, JIT_V1);
07556            /* Non-Latin-1 char: use slow path: */
07557            jit_extr_i_l(JIT_R2, JIT_R2);
07558            (void)jit_bgti_l(reffail, JIT_R2, 255);
07559            /* Latin-1: extract from scheme_char_constants: */
07560            jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE);
07561            (void)jit_movi_p(JIT_R0, scheme_char_constants);
07562            jit_ldxr_p(JIT_R0, JIT_R0, JIT_R2);
07563            break;
07564          case 2: /* bytes */
07565            jit_ldxi_p(JIT_R0, JIT_R0, offset);
07566            jit_ldxr_c(JIT_R0, JIT_R0, JIT_V1);
07567            jit_extr_uc_ul(JIT_R0, JIT_R0);
07568            jit_lshi_l(JIT_R0, JIT_R0, 0x1);
07569            jit_ori_l(JIT_R0, JIT_R0, 0x1);
07570            break;
07571          }
07572        } else {
07573          /* set mode: */
07574          jit_ldr_p(JIT_R2, JIT_RUNSTACK);
07575          switch (ii) {
07576          case 0: /* vector */
07577            jit_stxr_p(JIT_V1, JIT_R0, JIT_R2);
07578            break;
07579          case 1: /* string */
07580             (void)jit_bmsi_l(reffail, JIT_R2, 0x1);
07581            jit_ldxi_s(JIT_R2, JIT_R2, &((Scheme_Object *)0x0)->type);
07582            (void)jit_bnei_i(reffail, JIT_R2, scheme_char_type);
07583            jit_ldr_p(JIT_R2, JIT_RUNSTACK);
07584            jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Small_Object *)0x0)->u.char_val);
07585            jit_ldxi_p(JIT_R0, JIT_R0, offset);
07586            jit_stxr_i(JIT_V1, JIT_R0, JIT_R2);
07587            break;
07588          case 2: /* bytes */
07589            (void)jit_bmci_l(reffail, JIT_R2, 0x1);
07590            jit_rshi_ul(JIT_R2, JIT_R2, 1);
07591            (void)jit_bmsi_l(reffail, JIT_R2, ~0xFF);
07592            jit_ldxi_p(JIT_R0, JIT_R0, offset);
07593            jit_stxr_c(JIT_V1, JIT_R0, JIT_R2);
07594            break;
07595          }
07596          (void)jit_movi_p(JIT_R0, scheme_void);
07597        }
07598        mz_epilog(JIT_R2);
07599        CHECK_LIMIT();
07600 
07601        __END_TINY_JUMPS__(1);
07602       }
07603     }
07604   }
07605 
07606   /* *** syntax_ecode *** */
07607   /* R0 is (potential) syntax object */
07608   {
07609     jit_insn *ref, *reffail;
07610     syntax_e_code = jit_get_ip().ptr;
07611     __START_TINY_JUMPS__(1);
07612     mz_prolog(JIT_R2);
07613 
07614     ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
07615 
07616     reffail = _jit.x.pc;
07617     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
07618     CHECK_RUNSTACK_OVERFLOW();
07619     jit_str_p(JIT_RUNSTACK, JIT_R0);
07620     jit_movi_i(JIT_R1, 1);
07621     JIT_UPDATE_THREAD_RSPTR();
07622     CHECK_LIMIT();
07623     jit_prepare(2);
07624     jit_pusharg_p(JIT_RUNSTACK);
07625     jit_pusharg_i(JIT_R1);
07626     (void)mz_finish(scheme_checked_syntax_e);
07627     jit_retval(JIT_R0);
07628     jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1));
07629     mz_epilog(JIT_R2);
07630     CHECK_LIMIT();
07631     
07632     /* It's not a fixnum... */
07633     mz_patch_branch(ref);
07634     jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
07635     (void)jit_bnei_i(reffail, JIT_R2, scheme_stx_type);
07636     
07637     /* It's a syntax object... needs to propagate? */
07638     jit_ldxi_l(JIT_R2, JIT_R0, &((Scheme_Stx *)0x0)->u.lazy_prefix);
07639     ref = jit_beqi_l(jit_forward(), JIT_R2, 0x0);
07640     CHECK_LIMIT();
07641 
07642     /* Maybe needs to propagate; check STX_SUBSTX_FLAG flag */
07643     jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Stx *)0x0)->iso));
07644     (void)jit_bmsi_ul(reffail, JIT_R2, STX_SUBSTX_FLAG);
07645     
07646     /* No propagations. Extract value. */
07647     mz_patch_branch(ref);
07648     jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Stx *)0x0)->val);
07649 
07650     mz_epilog(JIT_R2);
07651     CHECK_LIMIT();
07652     __END_TINY_JUMPS__(1);
07653   }
07654 
07655   /* *** struct_{pred,get,set}[_branch]_code *** */
07656   /* R0 is (potential) struct proc, R1 is (potential) struct. */
07657   /* In branch mode, V1 is target address for false branch. */
07658   /* In set mode, V1 is value to install.                   */
07659   for (ii = 0; ii < 2; ii++) {
07660     for (i = 0; i < 4; i++) {
07661       void *code, *code_end;
07662       int kind, for_branch;
07663       jit_insn *ref, *ref2, *refslow, *bref1, *bref2, *bref3, *bref4, *bref5, *bref6, *bref8;
07664 
07665       if ((ii == 1) && (i == 1)) continue; /* no multi variant of pred branch */
07666 
07667       code = jit_get_ip().ptr;
07668 
07669       if (!i) {
07670        kind = 1;
07671        for_branch = 0;
07672         if (ii == 1) 
07673           struct_pred_multi_code = jit_get_ip().ptr;
07674         else
07675           struct_pred_code = jit_get_ip().ptr;
07676       } else if (i == 1) {
07677        kind = 1;
07678        for_branch = 1;
07679         struct_pred_branch_code = jit_get_ip().ptr;
07680        /* Save target address for false branch: */
07681         save_struct_temp(jitter);
07682       } else if (i == 2) {
07683        kind = 2;
07684        for_branch = 0;
07685         if (ii == 1) 
07686           struct_get_multi_code = jit_get_ip().ptr;
07687         else
07688           struct_get_code = jit_get_ip().ptr;
07689       } else {
07690        kind = 3;
07691        for_branch = 0;
07692         if (ii == 1) 
07693           struct_set_multi_code = jit_get_ip().ptr;
07694         else
07695           struct_set_code = jit_get_ip().ptr;
07696         /* Save value to install: */
07697         save_struct_temp(jitter);
07698       }
07699 
07700       mz_prolog(JIT_V1);
07701 
07702       __START_SHORT_JUMPS__(1);
07703 
07704       ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
07705       CHECK_LIMIT();
07706 
07707       /* Slow path: non-struct proc, or argument type is
07708         bad for a getter. */
07709       refslow = _jit.x.pc;
07710       jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES((kind == 3) ? 2 : 1));
07711       CHECK_RUNSTACK_OVERFLOW();
07712       JIT_UPDATE_THREAD_RSPTR();
07713       jit_str_p(JIT_RUNSTACK, JIT_R1);
07714       if (kind == 3) {
07715         restore_struct_temp(jitter, JIT_V1);        
07716         jit_stxi_p(WORDS_TO_BYTES(1), JIT_RUNSTACK, JIT_V1);
07717       }
07718       jit_movi_i(JIT_V1, ((kind == 3) ? 2 : 1));
07719       jit_prepare(3);
07720       jit_pusharg_p(JIT_RUNSTACK);
07721       jit_pusharg_p(JIT_V1);
07722       jit_pusharg_p(JIT_R0);
07723       if (ii == 1) {
07724         (void)mz_finish(_scheme_apply_multi_from_native);
07725       } else {
07726         (void)mz_finish(_scheme_apply_from_native);
07727       }
07728       jit_retval(JIT_R0);
07729       VALIDATE_RESULT(JIT_R0);
07730       jit_addi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES((kind == 3) ? 2 : 1));
07731       JIT_UPDATE_THREAD_RSPTR();
07732       if (!for_branch) {
07733        mz_epilog(JIT_V1);
07734        bref5 = NULL;
07735        bref6 = NULL;
07736       } else {
07737        /* Need to check for true or false. */
07738        bref5 = jit_beqi_p(jit_forward(), JIT_R0, scheme_false);
07739        bref6 = jit_jmpi(jit_forward());
07740       }
07741       CHECK_LIMIT();
07742 
07743       /* Continue trying fast path: check proc */
07744       mz_patch_branch(ref);
07745       jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
07746       (void)jit_bnei_i(refslow, JIT_R2, scheme_prim_type);
07747       jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Primitive_Proc *)0x0)->pp.flags);
07748       if (kind == 3) {
07749         jit_andi_i(JIT_R2, JIT_R2, SCHEME_PRIM_STRUCT_OTHER_TYPE_MASK);
07750         (void)jit_bnei_i(refslow, JIT_R2, SCHEME_PRIM_STRUCT_TYPE_INDEXED_SETTER);
07751       } else {
07752         (void)jit_bmci_i(refslow, JIT_R2, ((kind == 1) 
07753                                            ? SCHEME_PRIM_IS_STRUCT_PRED
07754                                            : SCHEME_PRIM_IS_STRUCT_INDEXED_GETTER));
07755       }
07756       CHECK_LIMIT();
07757       /* Check argument: */
07758       if (kind == 1) {
07759        bref1 = jit_bmsi_ul(jit_forward(), JIT_R1, 0x1);
07760        jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type);
07761         __START_INNER_TINY__(1);
07762        ref2 = jit_beqi_i(jit_forward(), JIT_R2, scheme_structure_type);
07763         __END_INNER_TINY__(1);
07764        bref2 = jit_bnei_i(jit_forward(), JIT_R2, scheme_proc_struct_type);
07765       } else {
07766        (void)jit_bmsi_ul(refslow, JIT_R1, 0x1);
07767        jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type);
07768         __START_INNER_TINY__(1);
07769        ref2 = jit_beqi_i(jit_forward(), JIT_R2, scheme_structure_type);
07770         __END_INNER_TINY__(1);
07771        (void)jit_bnei_i(refslow, JIT_R2, scheme_proc_struct_type);
07772        bref1 = bref2 = NULL;
07773       }
07774       __START_INNER_TINY__(1);
07775       mz_patch_branch(ref2);
07776       __END_INNER_TINY__(1);
07777       CHECK_LIMIT();
07778 
07779       /* Put argument struct type in R2, target struct type in V1 */
07780       jit_ldxi_p(JIT_R2, JIT_R1, &((Scheme_Structure *)0x0)->stype);
07781       jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Primitive_Closure *)0x0)->val);
07782       if (kind >= 2) {
07783        jit_ldxi_p(JIT_V1, JIT_V1, &((Struct_Proc_Info *)0x0)->struct_type);
07784       }
07785       CHECK_LIMIT();
07786 
07787       /* common case: types are the same */
07788       if (kind >= 2) {
07789         __START_INNER_TINY__(1);
07790         bref8 = jit_beqr_p(jit_forward(), JIT_R2, JIT_V1);
07791         __END_INNER_TINY__(1);
07792       } else
07793         bref8 = NULL;
07794 
07795       jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Struct_Type *)0x0)->name_pos);
07796       jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Struct_Type *)0x0)->name_pos);
07797       /* Now R2 is argument depth, V1 is target depth */
07798       if (kind == 1) {
07799        bref3 = jit_bltr_i(jit_forward(), JIT_R2, JIT_V1);
07800       } else {
07801        (void)jit_bltr_i(refslow, JIT_R2, JIT_V1);
07802        bref3 = NULL;
07803       }
07804       CHECK_LIMIT();
07805       /* Lookup argument type at target type depth, put it in R2: */
07806       jit_lshi_ul(JIT_R2, JIT_V1, JIT_LOG_WORD_SIZE);
07807       jit_addi_p(JIT_R2, JIT_R2, &((Scheme_Struct_Type *)0x0)->parent_types);
07808       jit_ldxi_p(JIT_V1, JIT_R1, &((Scheme_Structure *)0x0)->stype);
07809       jit_ldxr_p(JIT_R2, JIT_V1, JIT_R2);
07810       CHECK_LIMIT();
07811 
07812       /* Re-load target type into V1: */
07813       jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Primitive_Closure *)0x0)->val);
07814       if (kind >= 2) {
07815        jit_ldxi_p(JIT_V1, JIT_V1, &((Struct_Proc_Info *)0x0)->struct_type);
07816       }
07817 
07818       if (kind == 1) {
07819        bref4 = jit_bner_p(jit_forward(), JIT_R2, JIT_V1);
07820 
07821        /* True branch: */
07822        if (!for_branch) {
07823          (void)jit_movi_p(JIT_R0, scheme_true);
07824        } else {
07825          mz_patch_ucbranch(bref6);
07826 #ifdef MZ_USE_JIT_I386
07827 # ifndef X86_ALIGN_STACK
07828          jit_popr_p(JIT_V1);
07829 # endif
07830 #endif
07831        }
07832        mz_epilog(JIT_V1);
07833 
07834        /* False branch: */
07835        mz_patch_branch(bref1);
07836        mz_patch_branch(bref2);
07837        mz_patch_branch(bref3);
07838        mz_patch_branch(bref4);
07839        if (for_branch) {
07840          mz_patch_branch(bref5);
07841           restore_struct_temp(jitter, JIT_V1);
07842          mz_epilog_without_jmp();
07843          jit_jmpr(JIT_V1);
07844        } else {
07845          (void)jit_movi_p(JIT_R0, scheme_false);
07846          mz_epilog(JIT_V1);
07847        }
07848       } else {
07849        (void)jit_bner_p(refslow, JIT_R2, JIT_V1);
07850        bref4 = NULL;
07851         __START_INNER_TINY__(1);
07852         mz_patch_branch(bref8);
07853         __END_INNER_TINY__(1);
07854        /* Extract field */
07855        jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Primitive_Closure *)0x0)->val);
07856        jit_ldxi_i(JIT_V1, JIT_V1, &((Struct_Proc_Info *)0x0)->field);
07857        jit_lshi_ul(JIT_V1, JIT_V1, JIT_LOG_WORD_SIZE);
07858        jit_addi_p(JIT_V1, JIT_V1, &((Scheme_Structure *)0x0)->slots);
07859         if (kind == 3) {
07860           restore_struct_temp(jitter, JIT_R0);
07861           jit_stxr_p(JIT_V1, JIT_R1, JIT_R0);
07862           (void)jit_movi_p(JIT_R0, scheme_void);
07863         } else {
07864           jit_ldxr_p(JIT_R0, JIT_R1, JIT_V1);
07865         }
07866        mz_epilog(JIT_V1);
07867       }
07868       CHECK_LIMIT();
07869       
07870       __END_SHORT_JUMPS__(1);
07871 
07872       if (jitter->retain_start) {
07873        code_end = jit_get_ip().ptr;
07874        add_symbol((unsigned long)code, (unsigned long)code_end - 1, scheme_false, 0);
07875       }
07876     }
07877   }
07878 
07879 #ifdef CAN_INLINE_ALLOC
07880   /* *** retry_alloc_code[{_keep_r0_r1,_keep_fpr1}] *** */
07881   for (i = 0; i < 3; i++) { 
07882     if (!i)
07883       retry_alloc_code = jit_get_ip().ptr;
07884     else if (i == 1)
07885       retry_alloc_code_keep_r0_r1 = jit_get_ip().ptr;
07886     else
07887       retry_alloc_code_keep_fpr1 = jit_get_ip().ptr;
07888 
07889     mz_prolog(JIT_V1);
07890     generate_alloc_retry(jitter, i);
07891     CHECK_LIMIT();
07892     mz_epilog(JIT_V1);
07893     CHECK_LIMIT();
07894   }
07895 #endif
07896 
07897 #ifdef CAN_INLINE_ALLOC
07898   /* *** make_list_code *** */
07899   /* R2 has length, args are on runstack */
07900   for (i = 0; i < 2; i++) {
07901     jit_insn *ref, *refnext;
07902 
07903     if (i == 0)
07904       make_list_code = jit_get_ip().ptr;  
07905     else
07906       make_list_star_code = jit_get_ip().ptr;  
07907     mz_prolog(JIT_R1);
07908     jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE);
07909     if (i == 0)
07910       (void)jit_movi_p(JIT_R0, &scheme_null);
07911     else {
07912       jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE);
07913       jit_ldxr_p(JIT_R0, JIT_RUNSTACK, JIT_R2);
07914     }
07915 
07916     __START_SHORT_JUMPS__(1);
07917     ref = jit_beqi_l(jit_forward(), JIT_R2, 0);
07918     refnext = _jit.x.pc;
07919     __END_SHORT_JUMPS__(1);
07920     CHECK_LIMIT();
07921 
07922     jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE);
07923     jit_ldxr_p(JIT_R1, JIT_RUNSTACK, JIT_R2);
07924     mz_set_local_p(JIT_R2, JIT_LOCAL3);
07925 
07926     generate_cons_alloc(jitter, 1, 1);
07927     CHECK_LIMIT();
07928 
07929     mz_get_local_p(JIT_R2, JIT_LOCAL3);
07930 
07931     __START_SHORT_JUMPS__(1);
07932     (void)jit_bnei_l(refnext, JIT_R2, 0);
07933     mz_patch_branch(ref);
07934     __END_SHORT_JUMPS__(1);
07935 
07936     mz_epilog(JIT_R1);
07937   }
07938 #endif
07939 
07940   return 1;
07941 }
07942 
07943 static int do_generate_more_common(mz_jit_state *jitter, void *_data)
07944 {
07945   /* *** check_proc_extract_code *** */
07946   /* arguments are on the Scheme stack */
07947   {
07948     void *code_end;
07949     jit_insn *ref, *ref2, *ref3, *refslow;
07950     
07951     struct_proc_extract_code = jit_get_ip().ptr;
07952     mz_prolog(JIT_V1);
07953       
07954     __START_SHORT_JUMPS__(1);
07955 
07956     mz_rs_ldr(JIT_R0);
07957     ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1);
07958     CHECK_LIMIT();
07959 
07960     /* Slow path: call C implementation */
07961     refslow = _jit.x.pc;
07962     JIT_UPDATE_THREAD_RSPTR();
07963     jit_movi_i(JIT_V1, 5);
07964     jit_prepare(2);
07965     jit_pusharg_p(JIT_RUNSTACK);
07966     jit_pusharg_p(JIT_V1);
07967     (void)mz_finish(scheme_extract_checked_procedure);
07968     jit_retval(JIT_R0);
07969     VALIDATE_RESULT(JIT_R0);
07970     mz_epilog(JIT_V1);
07971 
07972     /* Continue trying fast path: check proc */
07973     mz_patch_branch(ref);
07974     jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type);
07975     (void)jit_bnei_i(refslow, JIT_R2, scheme_struct_type_type);
07976     jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Struct_Type *)0x0)->iso));
07977     (void)jit_bmci_ul(refslow, JIT_R2, STRUCT_TYPE_CHECKED_PROC);
07978     CHECK_LIMIT();
07979 
07980     mz_rs_ldxi(JIT_R1, 1);
07981     (void)jit_bmsi_ul(refslow, JIT_R1, 0x1);
07982     jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type);
07983     __START_INNER_TINY__(1);
07984     ref2 = jit_beqi_i(jit_forward(), JIT_R2, scheme_structure_type);
07985     __END_INNER_TINY__(1);
07986     (void)jit_bnei_i(refslow, JIT_R2, scheme_proc_struct_type);
07987     __START_INNER_TINY__(1);
07988     mz_patch_branch(ref2);
07989     __END_INNER_TINY__(1);
07990     CHECK_LIMIT();
07991 
07992     /* Put argument struct type in R2, target struct type is in R0 */
07993     jit_ldxi_p(JIT_R2, JIT_R1, &((Scheme_Structure *)0x0)->stype);
07994     jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Struct_Type *)0x0)->name_pos);
07995     jit_ldxi_i(JIT_V1, JIT_R0, &((Scheme_Struct_Type *)0x0)->name_pos);
07996 
07997     /* Now R2 is argument depth, V1 is target depth */
07998     (void)jit_bltr_i(refslow, JIT_R2, JIT_V1);
07999     CHECK_LIMIT();
08000     /* Lookup argument type at target type depth, put it in R2: */
08001     jit_lshi_ul(JIT_R2, JIT_V1, JIT_LOG_WORD_SIZE);
08002     jit_addi_p(JIT_R2, JIT_R2, &((Scheme_Struct_Type *)0x0)->parent_types);
08003     jit_ldxi_p(JIT_V1, JIT_R1, &((Scheme_Structure *)0x0)->stype);
08004     jit_ldxr_p(JIT_R2, JIT_V1, JIT_R2);
08005     CHECK_LIMIT();
08006     (void)jit_bner_p(refslow, JIT_R2, JIT_R0);
08007 
08008     /* Type matches. Extract checker. */
08009     jit_ldxi_p(JIT_V1, JIT_R1, &(((Scheme_Structure *)0x0)->slots[0]));
08010 
08011     /* Checker is in V1. Set up args on runstack, then apply it. */
08012     mz_rs_dec(2);
08013     mz_rs_ldxi(JIT_R2, 5);
08014     mz_rs_str(JIT_R2);
08015     mz_rs_ldxi(JIT_R2, 6);
08016     mz_rs_stxi(1, JIT_R2);
08017     CHECK_LIMIT();
08018     mz_rs_sync();
08019 
08020     __END_SHORT_JUMPS__(1);
08021     generate_non_tail_call(jitter, 2, 0, 1, 0, 0, 0, 0);
08022     CHECK_LIMIT();
08023     __START_SHORT_JUMPS__(1);
08024 
08025     mz_rs_inc(2);
08026     mz_rs_sync();
08027     ref3 = jit_bnei_p(refslow, JIT_R0, scheme_false);
08028     CHECK_LIMIT();
08029 
08030     /* Check failed. Apply the failure procedure. */
08031     JIT_UPDATE_THREAD_RSPTR();
08032     jit_prepare(1);
08033     jit_pusharg_p(JIT_RUNSTACK);
08034     (void)mz_finish(apply_checked_fail);
08035     CHECK_LIMIT();
08036     jit_retval(JIT_R0);
08037     VALIDATE_RESULT(JIT_R0);
08038     mz_epilog(JIT_V1);
08039     CHECK_LIMIT();
08040 
08041     /* Check passed. Extract the procedure. */
08042     mz_patch_branch(ref3);
08043     mz_rs_ldxi(JIT_R1, 1);
08044     jit_ldxi_p(JIT_R0, JIT_R1, &(((Scheme_Structure *)0x0)->slots[1]));
08045 
08046     mz_epilog(JIT_V1);
08047     CHECK_LIMIT();
08048       
08049     __END_SHORT_JUMPS__(1);
08050 
08051     if (jitter->retain_start) {
08052       code_end = jit_get_ip().ptr;
08053       add_symbol((unsigned long)struct_proc_extract_code, (unsigned long)code_end - 1, scheme_false, 0);
08054     }
08055   }
08056 
08057   return 1;
08058 }
08059 
08060 #ifdef CAN_INLINE_ALLOC
08061 static int generate_alloc_retry(mz_jit_state *jitter, int i)
08062 {
08063 #ifdef JIT_USE_FP_OPS
08064   if (i == 2) {
08065     (void)jit_sti_d_fppop(&save_fp, JIT_FPR1);
08066   }
08067 #endif
08068   JIT_UPDATE_THREAD_RSPTR();
08069   jit_prepare(2);
08070   CHECK_LIMIT();
08071   if (i == 1) {
08072     jit_pusharg_p(JIT_R1);
08073     jit_pusharg_p(JIT_R0);
08074   } else {
08075     (void)jit_movi_p(JIT_R0, NULL);
08076     jit_pusharg_p(JIT_R0);
08077     jit_pusharg_p(JIT_R0);
08078   }
08079   (void)mz_finish(prepare_retry_alloc);
08080   jit_retval(JIT_R0);
08081   if (i == 1) {
08082     jit_ldi_l(JIT_R1, &retry_alloc_r1);
08083   }
08084 #ifdef JIT_USE_FP_OPS
08085   if (i == 2) {
08086     (void)jit_ldi_d_fppush(JIT_FPR1, &save_fp);
08087   }
08088 #endif
08089   return 1;
08090 }
08091 #endif
08092 
08093 typedef struct {
08094   Scheme_Closure_Data *data;
08095   void *arity_code, *code, *tail_code, *code_end, **patch_depth;
08096   int max_extra, max_depth;
08097   Scheme_Native_Closure *nc;
08098   int argc;
08099   Scheme_Object **argv;
08100 } Generate_Closure_Data;
08101 
08102 static int do_generate_closure(mz_jit_state *jitter, void *_data)
08103 {
08104   Generate_Closure_Data *gdata = (Generate_Closure_Data *)_data;
08105   Scheme_Closure_Data *data = gdata->data;
08106   void *code, *tail_code, *code_end, *arity_code;
08107   int i, r, cnt, has_rest, is_method, num_params, to_args, argc;
08108   Scheme_Object **argv;
08109 
08110   code = jit_get_ip().ptr;
08111 
08112   jitter->nc = gdata->nc;
08113 
08114   argc = gdata->argc;
08115   argv = gdata->argv;
08116 
08117   generate_function_prolog(jitter, code, 
08118                         /* max_extra_pushed may be wrong the first time around,
08119                            but it will be right the last time around */
08120                         WORDS_TO_BYTES(data->max_let_depth + jitter->max_extra_pushed));
08121   CHECK_LIMIT();
08122 
08123   cnt = generate_function_getarg(jitter, 
08124                              (SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_HAS_REST),
08125                              data->num_params);
08126   CHECK_LIMIT();
08127 
08128   /* A tail call with arity checking can start here.
08129      (This is a little reundant checking when `code' is the
08130      etry point, but that's the slow path anyway.) */
08131   
08132   has_rest = ((SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_HAS_REST) ? 1 : 0);
08133   is_method = ((SCHEME_CLOSURE_DATA_FLAGS(data) & CLOS_IS_METHOD) ? 1 : 0);
08134   num_params = data->num_params;
08135   if (num_params && has_rest)
08136     --num_params;
08137 
08138   if (num_params < MAX_SHARED_ARITY_CHECK) {
08139     void *shared_arity_code;
08140 
08141     shared_arity_code = shared_arity_check[num_params][has_rest][is_method];
08142     if (!shared_arity_code) {
08143       shared_arity_code = generate_lambda_simple_arity_check(num_params, has_rest, is_method, 1);
08144       shared_arity_check[num_params][has_rest][is_method] = shared_arity_code;
08145     }
08146 
08147     arity_code = jit_get_ip().ptr;
08148   
08149     if (!has_rest)
08150       (void)jit_bnei_i(shared_arity_code, JIT_R1, num_params);
08151     else
08152       (void)jit_blti_i(shared_arity_code, JIT_R1, num_params);
08153   } else
08154     arity_code = generate_lambda_simple_arity_check(num_params, has_rest, is_method, 0);
08155 
08156   /* A tail call starts here. Caller must ensure that the
08157      stack is big enough, right number of arguments, closure
08158      is in R0. If the closure has a rest arg, also ensure
08159      argc in R1 and argv in R2. */
08160   tail_code = jit_get_ip().ptr;
08161 
08162   /* 0 params and has_rest => (lambda args E) where args is not in E,
08163      so accept any number of arguments and ignore them. */
08164 
08165   if (has_rest && data->num_params) {
08166     /* If runstack == argv and argc == cnt, then we didn't
08167        copy args down, and we need to make room for scheme_null. */
08168     jit_insn *ref, *ref2, *ref3;
08169          
08170     CHECK_LIMIT();
08171     
08172     __START_SHORT_JUMPS__(cnt < 100);
08173 
08174     ref = jit_bner_p(jit_forward(), JIT_RUNSTACK, JIT_R2);
08175     ref3 = jit_bgti_p(jit_forward(), JIT_R1, cnt);
08176     jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(cnt+1));
08177     CHECK_RUNSTACK_OVERFLOW();
08178     for (i = cnt; i--; ) {
08179       jit_ldxi_p(JIT_V1, JIT_R2, WORDS_TO_BYTES(i));
08180       jit_stxi_p(WORDS_TO_BYTES(i), JIT_RUNSTACK, JIT_V1);
08181       CHECK_LIMIT();
08182     }
08183     (void)jit_movi_p(JIT_V1, scheme_null);
08184     jit_stxi_p(WORDS_TO_BYTES(cnt), JIT_RUNSTACK, JIT_V1);
08185     ref2 = jit_jmpi(jit_forward());
08186     CHECK_LIMIT();
08187 
08188     /* Build a list for extra arguments: */
08189     mz_patch_branch(ref);
08190     mz_patch_branch(ref3);
08191 #ifndef JIT_PRECISE_GC
08192     if (data->closure_size)
08193 #endif
08194       {
08195         mz_pushr_p(JIT_R0);
08196         mz_rs_sync();
08197       }
08198     JIT_UPDATE_THREAD_RSPTR();
08199     CHECK_LIMIT();
08200     mz_prepare(3);
08201     jit_movi_i(JIT_V1, cnt);
08202     jit_pusharg_i(JIT_V1);
08203     jit_pusharg_p(JIT_R2);
08204     jit_pusharg_i(JIT_R1);
08205     CHECK_LIMIT();
08206     (void)mz_finish(scheme_build_list_offset);
08207     jit_retval(JIT_V1);
08208 #ifndef JIT_PRECISE_GC
08209     if (data->closure_size)
08210 #endif
08211       {
08212         mz_popr_p(JIT_R0);
08213         mz_rs_sync();
08214       }
08215     jit_stxi_p(WORDS_TO_BYTES(cnt), JIT_RUNSTACK, JIT_V1);
08216     mz_patch_ucbranch(ref2); /* jump here if we copied and produced null */
08217     CHECK_LIMIT();
08218 
08219     __END_SHORT_JUMPS__(cnt < 100);
08220 
08221     has_rest = 1;
08222     if (argc < (data->num_params - 1)) {
08223       argv = NULL;
08224       argc = 0;
08225     }
08226   } else {
08227     has_rest = 0;
08228     if (argc != data->num_params) {
08229       argv = NULL;
08230       argc = 0;
08231     }
08232   }
08233 
08234 #ifdef JIT_PRECISE_GC
08235   /* Keeping the native-closure code pointer on the runstack ensures
08236      that the code won't be GCed while we're running it. If the
08237      closure is empty, it's ok, faster, and useful to keep it,
08238      otherwise keep just the code pointer for space safety. */
08239   if (!data->closure_size) {
08240     jitter->closure_self_on_runstack = 1;
08241     mz_pushr_p(JIT_R0);  /* no sync */
08242   } else {
08243     jit_ldxi_p(JIT_R1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code);
08244     mz_pushr_p(JIT_R1);  /* no sync */
08245   }
08246   to_args = 0;
08247 #else
08248   to_args = 0;
08249 #endif
08250 
08251   /* Extract closure to runstack: */
08252   cnt = data->closure_size;
08253   to_args += cnt;
08254   if (cnt) {
08255     mz_rs_dec(cnt);
08256     CHECK_RUNSTACK_OVERFLOW();
08257     
08258     for (i = cnt; i--; ) {
08259       int pos;
08260       pos = WORDS_TO_BYTES(i) + (long)&((Scheme_Native_Closure *)0x0)->vals;
08261       jit_ldxi_p(JIT_R1, JIT_R0, pos);
08262       mz_rs_stxi(i, JIT_R1);
08263       CHECK_LIMIT();
08264     }
08265   }
08266 
08267   mz_rs_sync();
08268 
08269   /* If we have a letrec context, record arities */
08270   if (data->context && SAME_TYPE(SCHEME_TYPE(data->context), scheme_letrec_type)) {
08271     Scheme_Letrec *lr = (Scheme_Letrec *)data->context;
08272     int pos, self_pos = -1;
08273     for (i = data->closure_size; i--; ) {
08274       pos = data->closure_map[i];
08275       if (pos < lr->count) {
08276        Scheme_Closure_Data *data2 = (Scheme_Closure_Data *)lr->procs[pos];
08277        mz_runstack_closure_pushed(jitter, 
08278                                    (data2->num_params
08279                                     - ((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_HAS_REST)
08280                                        ? 1
08281                                        : 0)),
08282                                    (((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_PRESERVES_MARKS)
08283                                      ? NATIVE_PRESERVES_MARKS
08284                                      : 0)
08285                                     | ((SCHEME_CLOSURE_DATA_FLAGS(data2) & CLOS_SINGLE_RESULT)
08286                                      ? NATIVE_IS_SINGLE_RESULT
08287                                        : 0)));
08288        if (SAME_OBJ(lr->procs[pos], (Scheme_Object *)data)) {
08289          self_pos = i;
08290        }
08291       } else
08292        mz_runstack_pushed(jitter, 1);
08293     }
08294     if ((self_pos >= 0) && !has_rest) {
08295       jitter->self_pos = self_pos;
08296       jitter->self_closure_size = data->closure_size;
08297     }
08298   } else {
08299     mz_runstack_pushed(jitter, cnt);
08300 
08301     /* A define-values context? */
08302     if (data->context && SAME_TYPE(SCHEME_TYPE(data->context), scheme_toplevel_type)) {
08303       jitter->self_toplevel_pos = SCHEME_TOPLEVEL_POS(data->context);
08304       jitter->self_closure_size = data->closure_size;
08305     }
08306   }
08307 
08308   LOG_IT(("PROC: %s, %d args, flags: %x\n", 
08309           (data->name ? scheme_format_utf8("~s", 2, 1, &data->name, NULL) : "???"),
08310           data->num_params,
08311           SCHEME_CLOSURE_DATA_FLAGS(data)));
08312   FOR_LOG(jitter->log_depth++);
08313 
08314   jitter->self_data = data;
08315 
08316   jitter->self_restart_code =