diff -urNp disc/arch/ia64/kernel/acpi.c krel/arch/ia64/kernel/acpi.c --- disc/arch/ia64/kernel/acpi.c Fri Aug 9 13:44:24 2002 +++ krel/arch/ia64/kernel/acpi.c Fri Aug 9 13:45:29 2002 @@ -326,6 +326,7 @@ acpi_find_iosapic (int global_vector, u3 } +#ifndef CONFIG_IA64_SGI_SN static int __init acpi_parse_iosapic (acpi_table_entry_header *header) { @@ -350,6 +351,7 @@ acpi_parse_iosapic (acpi_table_entry_hea return 0; } +#endif static int __init acpi_parse_plat_int_src (acpi_table_entry_header *header) @@ -748,11 +750,13 @@ acpi_boot_init (char *cmdline) /* I/O APIC */ +#ifndef CONFIG_IA64_SGI_SN result = acpi_table_parse_madt(ACPI_MADT_IOSAPIC, acpi_parse_iosapic); if (1 > result) { printk(KERN_ERR PREFIX "Error parsing MADT - no IOAPIC entries!\n"); return ((result == 0) ? -ENODEV : result); } +#endif /* System-Level Interrupt Routing */ diff -urNp disc/arch/ia64/kernel/efi.c krel/arch/ia64/kernel/efi.c --- disc/arch/ia64/kernel/efi.c Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/efi.c Fri Aug 9 13:45:29 2002 @@ -328,34 +328,18 @@ efi_map_pal_code (void) md->phys_addr); continue; } - /* - * The only ITLB entry in region 7 that is used is the one installed by - * __start(). That entry covers a 64MB range. - */ - mask = ~((1 << KERNEL_TR_PAGE_SHIFT) - 1); - vaddr = PAGE_OFFSET + md->phys_addr; /* - * We must check that the PAL mapping won't overlap with the kernel - * mapping. - * - * PAL code is guaranteed to be aligned on a power of 2 between 4k and - * 256KB and that only one ITR is needed to map it. This implies that the - * PAL code is always aligned on its size, i.e., the closest matching page - * size supported by the TLB. Therefore PAL code is guaranteed never to - * cross a 64MB unless it is bigger than 64MB (very unlikely!). So for - * now the following test is enough to determine whether or not we need a - * dedicated ITR for the PAL code. + * Set up a dedicated ITR to map PAL. We do so in the identity + * mapped virtual area, so data mappings will be provided + * automatically with DTC entries by the Alt-dtlb-miss handler. */ - if ((vaddr & mask) == (KERNEL_START & mask)) { - printk("%s: no need to install ITR for PAL code\n", __FUNCTION__); - continue; - } + mask = ~(IA64_GRANULE_SIZE - 1); + vaddr = PAGE_OFFSET + md->phys_addr; if (md->num_pages << EFI_PAGE_SHIFT > IA64_GRANULE_SIZE) panic("Woah! PAL code size bigger than a granule!"); - mask = ~((1 << IA64_GRANULE_SHIFT) - 1); printk("CPU %d: mapping PAL code [0x%lx-0x%lx) into [0x%lx-0x%lx)\n", smp_processor_id(), md->phys_addr, md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT), @@ -370,6 +354,11 @@ efi_map_pal_code (void) ia64_set_psr(psr); /* restore psr */ ia64_srlz_i(); } + + if (pal_code_count == 0) { + printk(KERN_ERR "CPU %d: No EFI Pal Code memory range found.\n", + smp_processor_id()); + } } void __init @@ -548,15 +537,15 @@ efi_enter_virtual_mode (void) * Now that EFI is in virtual mode, we arrange for EFI functions to be * called directly: */ - efi.get_time = __va(runtime->get_time); - efi.set_time = __va(runtime->set_time); - efi.get_wakeup_time = __va(runtime->get_wakeup_time); - efi.set_wakeup_time = __va(runtime->set_wakeup_time); - efi.get_variable = __va(runtime->get_variable); - efi.get_next_variable = __va(runtime->get_next_variable); - efi.set_variable = __va(runtime->set_variable); - efi.get_next_high_mono_count = __va(runtime->get_next_high_mono_count); - efi.reset_system = __va(runtime->reset_system); + efi.get_time = (efi_get_time_t *)runtime->get_time; + efi.set_time = (efi_set_time_t *)runtime->set_time; + efi.get_wakeup_time = (efi_get_wakeup_time_t *)runtime->get_wakeup_time; + efi.set_wakeup_time = (efi_set_wakeup_time_t *)runtime->set_wakeup_time; + efi.get_variable = (efi_get_variable_t *)runtime->get_variable; + efi.get_next_variable = (efi_get_next_variable_t *)runtime->get_next_variable; + efi.set_variable = (efi_set_variable_t *)runtime->set_variable; + efi.get_next_high_mono_count = (efi_get_next_high_mono_count_t *)runtime->get_next_high_mono_count; + efi.reset_system = (efi_reset_system_t *)runtime->reset_system; } /* diff -urNp disc/arch/ia64/kernel/efi_stub.S krel/arch/ia64/kernel/efi_stub.S --- disc/arch/ia64/kernel/efi_stub.S Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/efi_stub.S Fri Aug 9 13:45:29 2002 @@ -62,7 +62,7 @@ GLOBAL_ENTRY(efi_call_phys) mov b6=r2 ;; andcm r16=loc3,r16 // get psr with IT, DT, and RT bits cleared - br.call.sptk.many rp=ia64_switch_mode + br.call.sptk.many rp=ia64_switch_mode_phys .ret0: mov out4=in5 mov out0=in1 mov out1=in2 @@ -73,7 +73,7 @@ GLOBAL_ENTRY(efi_call_phys) br.call.sptk.many rp=b6 // call the EFI function .ret1: mov ar.rsc=0 // put RSE in enforced lazy, LE mode mov r16=loc3 - br.call.sptk.many rp=ia64_switch_mode // return to virtual mode + br.call.sptk.many rp=ia64_switch_mode_virt // return to virtual mode .ret2: mov ar.rsc=loc4 // restore RSE configuration mov ar.pfs=loc1 mov rp=loc0 diff -urNp disc/arch/ia64/kernel/entry.S krel/arch/ia64/kernel/entry.S --- disc/arch/ia64/kernel/entry.S Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/entry.S Fri Aug 9 13:45:29 2002 @@ -141,15 +141,12 @@ GLOBAL_ENTRY(ia64_switch_to) ;; st8 [r22]=sp // save kernel stack pointer of old task shr.u r26=r20,IA64_GRANULE_SHIFT - shr.u r17=r20,KERNEL_TR_PAGE_SHIFT - ;; - cmp.ne p6,p7=KERNEL_TR_PAGE_NUM,r17 adds r21=IA64_TASK_THREAD_KSP_OFFSET,in0 ;; /* * If we've already mapped this task's page, we can skip doing it again. */ -(p6) cmp.eq p7,p6=r26,r27 + cmp.eq p7,p6=r26,r27 (p6) br.cond.dpnt .map ;; .done: diff -urNp disc/arch/ia64/kernel/head.S krel/arch/ia64/kernel/head.S --- disc/arch/ia64/kernel/head.S Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/head.S Fri Aug 9 13:45:29 2002 @@ -64,17 +64,20 @@ start_ap: */ rsm psr.i | psr.ic mov r16=((ia64_rid(IA64_REGION_ID_KERNEL, PAGE_OFFSET) << 8) | (IA64_GRANULE_SHIFT << 2)) + mov r3=ip ;; srlz.i mov r18=KERNEL_TR_PAGE_SHIFT<<2 movl r17=KERNEL_START + dep r2=0,r3,0,12 ;; mov rr[r17]=r16 mov cr.itir=r18 mov cr.ifa=r17 mov r16=IA64_TR_KERNEL - movl r18=((1 << KERNEL_TR_PAGE_SHIFT) | PAGE_KERNEL) + movl r18=PAGE_KERNELRX ;; + or r18=r2,r18 srlz.i ;; itr.i itr[r16]=r18 @@ -84,6 +87,22 @@ start_ap: srlz.i /* + * Map kernel global data + */ + movl r17=VGLOBAL_BASE + movl r18=PAGE_KERNEL + ;; + mov cr.ifa=r17 + mov r16=IA64_TR_KERNEL_GLOBAL + or r18=r2,r18 + ;; + srlz.i + ;; + itr.d dtr[r16]=r18 + ;; + srlz.i + + /* * Switch into virtual mode: */ movl r16=(IA64_PSR_IT|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFH|IA64_PSR_BN \ @@ -132,27 +151,55 @@ start_ap: movl r3=cpucount ;; ld4 r3=[r3] // r3 <- smp_processor_id() - movl r2=init_tasks - ;; - shladd r2=r3,3,r2 - ;; - ld8 r2=[r2] #else mov r3=0 - movl r2=init_task_union - ;; #endif + + // Make init_tasks an identity mapped pointer + movl r18=init_tasks + ;; cmp4.ne isAP,isBP=r3,r0 - ;; // RAW on r2 - extr r3=r2,0,61 // r3 == phys addr of task struct - mov r16=KERNEL_TR_PAGE_NUM + shladd r18=r3,3,r18 + ;; + ld8 r2=[r18] + ;; + tpa r3=r2 // r3 == phys addr of task struct + ;; + dep r2=-1,r3,61,3 // convert phys to identity mapped virtual + ;; + st8 [r18]=r2 // and save it back in init_tasks[thiscpu] + + + // load mapping for stack (virtaddr in r2, physaddr in r3) + rsm psr.ic + movl r17=PAGE_KERNEL + ;; + srlz.d + dep r18=0,r3,0,12 + ;; + or r18=r17,r18 + ;; + mov r17=rr[r2] + ;; + dep r17=0,r17,8,24 ;; + mov cr.itir=r17 + mov cr.ifa=r2 + + mov r16=IA64_TR_CURRENT_STACK + ;; + itr.d dtr[r16]=r18 + ;; + ssm psr.ic + srlz.d // load the "current" pointer (r13) and ar.k6 with the current task mov r13=r2 mov IA64_KR(CURRENT)=r3 // Physical address - // initialize k4 to a safe value (64-128MB is mapped by TR_KERNEL) + // k4 indicates which large kernel page is mapped + extr.u r16=r3,IA64_GRANULE_SHIFT,64-IA64_GRANULE_SHIFT + ;; mov IA64_KR(CURRENT_STACK)=r16 /* * Reserve space at the top of the stack for "struct pt_regs". Kernel threads @@ -673,14 +720,14 @@ GLOBAL_ENTRY(__ia64_init_fpu) END(__ia64_init_fpu) /* - * Switch execution mode from virtual to physical or vice versa. + * Switch execution mode from virtual to physical * * Inputs: * r16 = new psr to establish * * Note: RSE must already be in enforced lazy mode */ -GLOBAL_ENTRY(ia64_switch_mode) +GLOBAL_ENTRY(ia64_switch_mode_phys) { alloc r2=ar.pfs,0,0,0,0 rsm psr.i | psr.ic // disable interrupts and interrupt collection @@ -690,35 +737,86 @@ GLOBAL_ENTRY(ia64_switch_mode) { flushrs // must be first insn in group srlz.i - shr.u r19=r15,61 // r19 <- top 3 bits of current IP } ;; mov cr.ipsr=r16 // set new PSR - add r3=1f-ia64_switch_mode,r15 - xor r15=0x7,r19 // flip the region bits + add r3=1f-ia64_switch_mode_phys,r15 mov r17=ar.bsp mov r14=rp // get return address into a general register + ;; - // switch RSE backing store: + // going to physical mode, use tpa to translate virt->phys + tpa r17=r17 + tpa r3=r3 + tpa sp=sp + tpa r14=r14 ;; - dep r17=r15,r17,61,3 // make ar.bsp physical or virtual + mov r18=ar.rnat // save ar.rnat - ;; mov ar.bspstore=r17 // this steps on ar.rnat - dep r3=r15,r3,61,3 // make rfi return address physical or virtual + mov cr.iip=r3 + mov cr.ifs=r0 + ;; + mov ar.rnat=r18 // restore ar.rnat + rfi // must be last insn in group + ;; +1: mov rp=r14 + br.ret.sptk.many rp +END(ia64_switch_mode_phys) + +/* + * Switch execution mode from physical to virtual + * + * Inputs: + * r16 = new psr to establish + * + * Note: RSE must already be in enforced lazy mode + */ +GLOBAL_ENTRY(ia64_switch_mode_virt) + { + alloc r2=ar.pfs,0,0,0,0 + rsm psr.i | psr.ic // disable interrupts and interrupt collection + mov r15=ip + } + ;; + { + flushrs // must be first insn in group + srlz.i + } + ;; + mov cr.ipsr=r16 // set new PSR + add r3=1f-ia64_switch_mode_virt,r15 + + mov r17=ar.bsp + mov r14=rp // get return address into a general register ;; + + // going to virtual + // - for code addresses, set upper bits of addr to VPERNODE_BASE + // - for stack addresses, set upper 3 bits to 0xe.... Dont change any of the + // lower bits since we want it to stay identity mapped + movl r18=VPERNODE_BASE + dep r3=0,r3,KERNEL_TR_PAGE_SHIFT,64-KERNEL_TR_PAGE_SHIFT + dep r14=0,r14,KERNEL_TR_PAGE_SHIFT,64-KERNEL_TR_PAGE_SHIFT + dep r17=-1,r17,61,3 + dep sp=-1,sp,61,3 + ;; + or r3=r3,r18 + or r14=r14,r18 + ;; + + mov r18=ar.rnat // save ar.rnat + mov ar.bspstore=r17 // this steps on ar.rnat mov cr.iip=r3 mov cr.ifs=r0 - dep sp=r15,sp,61,3 // make stack pointer physical or virtual ;; mov ar.rnat=r18 // restore ar.rnat - dep r14=r15,r14,61,3 // make function return address physical or virtual rfi // must be last insn in group ;; 1: mov rp=r14 br.ret.sptk.many rp -END(ia64_switch_mode) +END(ia64_switch_mode_virt) #ifdef CONFIG_IA64_BRL_EMU @@ -811,3 +909,85 @@ GLOBAL_ENTRY(ia64_spinlock_contention) END(ia64_spinlock_contention) #endif + +#ifdef CONFIG_NUMA + +#define PSR_BITS_TO_CLEAR \ + (IA64_PSR_I | IA64_PSR_IT | IA64_PSR_DT | IA64_PSR_RT | \ + IA64_PSR_DD | IA64_PSR_SS | IA64_PSR_RI | IA64_PSR_ED | \ + IA64_PSR_DFL | IA64_PSR_DFH) + +#define PSR_BITS_TO_SET \ + (IA64_PSR_BN) + +/* + * ccNUMA systems bring up all cpus running from the copy of the + * kernel that elilo loaded into memory. Processors that find that + * they are not using the kernel text/rodata that is on their local + * node can use this routine to reset their TLB mappings to point + * at the correct copy. + * + * This is like the magic trick where you pull a table cloth out + * from under a table covered with plates, glasses and silverware, + * except in this version we slide an identical tablecloth in to + * replace the one we pulled out. + * + * Inputs: + * in0 = physical address of local node copy to be mapped + */ + +GLOBAL_ENTRY(ia64_remap_kernel) + .prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(8) + alloc loc1=ar.pfs,8,5,7,0 + mov loc0=rp + .body + ;; + mov loc4=ar.rsc // save RSE configuration + mov ar.rsc=0 // put RSE in enforced lazy, LE mode + ;; + + movl r16=PSR_BITS_TO_CLEAR + + mov loc3=psr // save processor status word + movl r17=PSR_BITS_TO_SET + ;; + or loc3=loc3,r17 + ;; + andcm r16=loc3,r16 // get psr with IT, DT, and RT bits cleared + br.call.sptk.few rp=ia64_switch_mode_phys +.ret3: + rsm psr.i | psr.ic + movl r19=VPERNODE_BASE + movl r20=KERNEL_TR_PAGE_SHIFT<<2 + movl r21=PAGE_KERNELRX + mov r22=IA64_TR_KERNEL + ;; + ptr.i r19,r20 // purge old code mapping + ptr.d r19,r20 // and the data mapping too + ;; + srlz.i + ;; + + mov cr.ifa=r19 + mov cr.itir=r20 + or r21=r21,in0 + ;; + + srlz.i + ;; + itr.i itr[r22]=r21 + ;; + itr.d dtr[r22]=r21 + ;; + srlz.i + ;; + + mov r16=loc3 + br.call.sptk.few rp=ia64_switch_mode_virt // return to virtual mode +.ret4: mov ar.rsc=loc4 // restore RSE configuration + mov ar.pfs=loc1 + mov rp=loc0 + br.ret.sptk.few rp +END(ia64_remap_kernel) + +#endif diff -urNp disc/arch/ia64/kernel/ia64_ksyms.c krel/arch/ia64/kernel/ia64_ksyms.c --- disc/arch/ia64/kernel/ia64_ksyms.c Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/ia64_ksyms.c Fri Aug 9 13:45:29 2002 @@ -156,3 +156,8 @@ EXPORT_SYMBOL(ia64_mv); #endif EXPORT_SYMBOL(machvec_noop); +#ifdef CONFIG_DISCONTIGMEM +EXPORT_SYMBOL(invalid_mem_map); +EXPORT_SYMBOL(zero_page_memmap_ptr); +#endif + diff -urNp disc/arch/ia64/kernel/ivt.S krel/arch/ia64/kernel/ivt.S --- disc/arch/ia64/kernel/ivt.S Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/ivt.S Fri Aug 9 13:45:29 2002 @@ -125,8 +125,13 @@ ENTRY(vhpt_miss) shr.u r18=r16,PGDIR_SHIFT // get bits 33-63 of the faulting address ;; (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place + .global ia64_ivt_patch1 +ia64_ivt_patch1: +{ .mlx // we patch this bundle to include physical address of swapper_pg_dir srlz.d // ensure "rsm psr.dt" has taken effect -(p6) movl r19=__pa(swapper_pg_dir) // region 5 is rooted at swapper_pg_dir +(p6) movl r19=swapper_pg_dir // region 5 is rooted at swapper_pg_dir +} +.pred.rel "mutex", p6, p7 (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3 ;; @@ -414,8 +419,13 @@ ENTRY(nested_dtlb_miss) shr.u r18=r16,PGDIR_SHIFT // get bits 33-63 of faulting address ;; (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place - srlz.d -(p6) movl r19=__pa(swapper_pg_dir) // region 5 is rooted at swapper_pg_dir + .global ia64_ivt_patch2 +ia64_ivt_patch2: +{ .mlx // we patch this bundle to include physical address of swapper_pg_dir + srlz.d +(p6) movl r19=swapper_pg_dir // region 5 is rooted at swapper_pg_dir +} + .pred.rel "mutex", p6, p7 (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3 ;; diff -urNp disc/arch/ia64/kernel/mca.c krel/arch/ia64/kernel/mca.c --- disc/arch/ia64/kernel/mca.c Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/mca.c Fri Aug 9 13:45:29 2002 @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -432,17 +433,17 @@ ia64_mca_init(void) IA64_MCA_DEBUG("ia64_mca_init: registered mca rendezvous spinloop and wakeup mech.\n"); - ia64_mc_info.imi_mca_handler = __pa(mca_hldlr_ptr->fp); + ia64_mc_info.imi_mca_handler = __tpa(mca_hldlr_ptr->fp); /* * XXX - disable SAL checksum by setting size to 0; should be - * __pa(ia64_os_mca_dispatch_end) - __pa(ia64_os_mca_dispatch); + * __tpa(ia64_os_mca_dispatch_end) - __tpa(ia64_os_mca_dispatch); */ ia64_mc_info.imi_mca_handler_size = 0; /* Register the os mca handler with SAL */ if ((rc = ia64_sal_set_vectors(SAL_VECTOR_OS_MCA, ia64_mc_info.imi_mca_handler, - mca_hldlr_ptr->gp, + __tpa(mca_hldlr_ptr->gp), ia64_mc_info.imi_mca_handler_size, 0, 0, 0))) { @@ -452,15 +453,15 @@ ia64_mca_init(void) } IA64_MCA_DEBUG("ia64_mca_init: registered os mca handler with SAL at 0x%lx, gp = 0x%lx\n", - ia64_mc_info.imi_mca_handler, mca_hldlr_ptr->gp); + ia64_mc_info.imi_mca_handler, __tpa(mca_hldlr_ptr->gp)); /* * XXX - disable SAL checksum by setting size to 0, should be * IA64_INIT_HANDLER_SIZE */ - ia64_mc_info.imi_monarch_init_handler = __pa(mon_init_ptr->fp); + ia64_mc_info.imi_monarch_init_handler = __tpa(mon_init_ptr->fp); ia64_mc_info.imi_monarch_init_handler_size = 0; - ia64_mc_info.imi_slave_init_handler = __pa(slave_init_ptr->fp); + ia64_mc_info.imi_slave_init_handler = __tpa(slave_init_ptr->fp); ia64_mc_info.imi_slave_init_handler_size = 0; IA64_MCA_DEBUG("ia64_mca_init: os init handler at %lx\n", @@ -469,10 +470,10 @@ ia64_mca_init(void) /* Register the os init handler with SAL */ if ((rc = ia64_sal_set_vectors(SAL_VECTOR_OS_INIT, ia64_mc_info.imi_monarch_init_handler, - __pa(ia64_get_gp()), + __tpa(ia64_get_gp()), ia64_mc_info.imi_monarch_init_handler_size, ia64_mc_info.imi_slave_init_handler, - __pa(ia64_get_gp()), + __tpa(ia64_get_gp()), ia64_mc_info.imi_slave_init_handler_size))) { printk("ia64_mca_init: Failed to register m/s init handlers with SAL. rc = %ld\n", @@ -844,6 +845,7 @@ ia64_init_handler (struct pt_regs *regs) ia64_sal_clear_state_info(SAL_INFO_TYPE_INIT); init_handler_platform(regs); /* call platform specific routines */ + } /* diff -urNp disc/arch/ia64/kernel/minstate.h krel/arch/ia64/kernel/minstate.h --- disc/arch/ia64/kernel/minstate.h Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/minstate.h Fri Aug 9 13:45:29 2002 @@ -59,7 +59,7 @@ (pUser) addl rKRBS=IA64_RBS_OFFSET,r1; /* compute base of register backing store */ \ ;; \ (pUser) mov rARRNAT=ar.rnat; \ -(pKern) dep r1=0,sp,61,3; /* compute physical addr of sp */ \ +(pKern) tpa r1=sp; /* compute physical addr of sp */ \ (pUser) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1; /* compute base of memory stack */ \ (pUser) mov rARBSPSTORE=ar.bspstore; /* save ar.bspstore */ \ (pUser) dep rKRBS=-1,rKRBS,61,3; /* compute kernel virtual addr of RBS */\ diff -urNp disc/arch/ia64/kernel/pal.S krel/arch/ia64/kernel/pal.S --- disc/arch/ia64/kernel/pal.S Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/pal.S Fri Aug 9 13:45:29 2002 @@ -164,7 +164,7 @@ GLOBAL_ENTRY(ia64_pal_call_phys_static) ;; mov loc4=ar.rsc // save RSE configuration dep.z loc2=loc2,0,61 // convert pal entry point to physical - dep.z r8=r8,0,61 // convert rp to physical + tpa r8=r8 // convert rp to physical ;; mov b7 = loc2 // install target to branch reg mov ar.rsc=0 // put RSE in enforced lazy, LE mode @@ -174,13 +174,13 @@ GLOBAL_ENTRY(ia64_pal_call_phys_static) or loc3=loc3,r17 // add in psr the bits to set ;; andcm r16=loc3,r16 // removes bits to clear from psr - br.call.sptk.many rp=ia64_switch_mode + br.call.sptk.many rp=ia64_switch_mode_phys .ret1: mov rp = r8 // install return address (physical) br.cond.sptk.many b7 1: mov ar.rsc=0 // put RSE in enforced lazy, LE mode mov r16=loc3 // r16= original psr - br.call.sptk.many rp=ia64_switch_mode // return to virtual mode + br.call.sptk.many rp=ia64_switch_mode_virt // return to virtual mode .ret2: mov psr.l = loc3 // restore init PSR @@ -228,13 +228,13 @@ GLOBAL_ENTRY(ia64_pal_call_phys_stacked) mov b7 = loc2 // install target to branch reg ;; andcm r16=loc3,r16 // removes bits to clear from psr - br.call.sptk.many rp=ia64_switch_mode + br.call.sptk.many rp=ia64_switch_mode_phys .ret6: br.call.sptk.many rp=b7 // now make the call .ret7: mov ar.rsc=0 // put RSE in enforced lazy, LE mode mov r16=loc3 // r16= original psr - br.call.sptk.many rp=ia64_switch_mode // return to virtual mode + br.call.sptk.many rp=ia64_switch_mode_virt // return to virtual mode .ret8: mov psr.l = loc3 // restore init PSR mov ar.pfs = loc1 diff -urNp disc/arch/ia64/kernel/setup.c krel/arch/ia64/kernel/setup.c --- disc/arch/ia64/kernel/setup.c Fri Aug 9 13:45:05 2002 +++ krel/arch/ia64/kernel/setup.c Fri Aug 9 13:45:29 2002 @@ -275,8 +275,8 @@ find_memory (void) + strlen(__va(ia64_boot_param->command_line)) + 1); n++; - rsvd_region[n].start = KERNEL_START; - rsvd_region[n].end = KERNEL_END; + rsvd_region[n].start = __imva(KERNEL_START); + rsvd_region[n].end = __imva(KERNEL_END); n++; #ifdef CONFIG_BLK_DEV_INITRD @@ -334,6 +334,46 @@ find_memory (void) #endif } +/* + * There are two places in the performance critical path of + * the exception handling code where we need to know the physical + * address of the swapper_pg_dir structure. This routine + * patches the "movl" instructions to load the value needed. + */ +static void __init +patch_ivt_with_phys_swapper_pg_dir(void) +{ + extern char ia64_ivt_patch1[], ia64_ivt_patch2[]; + unsigned long spd = __tpa(swapper_pg_dir); + unsigned long *p; + + p = (unsigned long *)__imva(ia64_ivt_patch1); + + *p = (*p & 0x3fffffffffffUL) | + ((spd & 0x000000ffffc00000UL)<<24); + p++; + *p = (*p & 0xf000080fff800000UL) | + ((spd & 0x8000000000000000UL) >> 4) | + ((spd & 0x7fffff0000000000UL) >> 40) | + ((spd & 0x00000000001f0000UL) << 29) | + ((spd & 0x0000000000200000UL) << 23) | + ((spd & 0x000000000000ff80UL) << 43) | + ((spd & 0x000000000000007fUL) << 36); + + p = (unsigned long *)__imva(ia64_ivt_patch2); + + *p = (*p & 0x3fffffffffffUL) | + ((spd & 0x000000ffffc00000UL)<<24); + p++; + *p = (*p & 0xf000080fff800000UL) | + ((spd & 0x8000000000000000UL) >> 4) | + ((spd & 0x7fffff0000000000UL) >> 40) | + ((spd & 0x00000000001f0000UL) << 29) | + ((spd & 0x0000000000200000UL) << 23) | + ((spd & 0x000000000000ff80UL) << 43) | + ((spd & 0x000000000000007fUL) << 36); +} + void __init setup_arch (char **cmdline_p) { @@ -342,6 +382,8 @@ setup_arch (char **cmdline_p) unw_init(); + patch_ivt_with_phys_swapper_pg_dir(); + *cmdline_p = __va(ia64_boot_param->command_line); strncpy(saved_command_line, *cmdline_p, sizeof(saved_command_line)); saved_command_line[COMMAND_LINE_SIZE-1] = '\0'; /* for safety */ @@ -432,12 +474,13 @@ setup_arch (char **cmdline_p) # endif #endif + platform_setup(cmdline_p); + #ifdef CONFIG_IA64_MCA /* enable IA-64 Machine Check Abort Handling */ ia64_mca_init(); #endif - platform_setup(cmdline_p); paging_init(); unw_create_gate_table(); @@ -457,6 +500,7 @@ show_cpuinfo (struct seq_file *m, void * # define cpu 0 #endif char family[32], features[128], *cp; + unsigned long phys_cpu_id; struct cpuinfo_ia64 *c = v; unsigned long mask; @@ -479,8 +523,14 @@ show_cpuinfo (struct seq_file *m, void * if (mask) sprintf(cp, " 0x%lx", mask); +#ifdef CONFIG_SMP + phys_cpu_id = cpu_physical_id(cpu); +#else + phys_cpu_id = (ia64_get_lid() >> 16) & 0xffff; +#endif seq_printf(m, "processor : %d\n" + "sapic : (%04lx:%04lx)\n" "vendor : %s\n" "arch : IA-64\n" "family : %s\n" @@ -493,14 +543,16 @@ show_cpuinfo (struct seq_file *m, void * "cpu MHz : %lu.%06lu\n" "itc MHz : %lu.%06lu\n" "BogoMIPS : %lu.%02lu\n\n", - cpu, c->vendor, family, c->model, c->revision, c->archrev, + cpu, + (phys_cpu_id>>8)&0xff, phys_cpu_id&0xff, + c->vendor, family, c->model, c->revision, c->archrev, features, c->ppn, c->number, c->proc_freq / 1000000, c->proc_freq % 1000000, c->itc_freq / 1000000, c->itc_freq % 1000000, lpj*HZ/500000, (lpj*HZ/5000) % 100); return 0; -#undef lpj -#undef cpu +#undef cpu +#undef lpj } static void * @@ -589,6 +641,7 @@ identify_cpu (struct cpuinfo_ia64 *c) } + /* * cpu_init() initializes state that is per-CPU. This function acts * as a 'CPU state barrier', nothing should get across. diff -urNp disc/arch/ia64/kernel/smpboot.c krel/arch/ia64/kernel/smpboot.c --- disc/arch/ia64/kernel/smpboot.c Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/kernel/smpboot.c Fri Aug 9 13:45:29 2002 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -319,6 +320,11 @@ smp_callin (void) { int cpuid, phys_id; extern void ia64_init_itm(void); +#ifdef CONFIG_DISCONTIGMEM + cnodeid_t kloadnode; + extern unsigned long kcopybase[PLAT_MAX_COMPACT_NODES]; + extern void ia64_remap_kernel(long); +#endif #ifdef CONFIG_PERFMON extern void perfmon_init_percpu(void); @@ -332,6 +338,15 @@ smp_callin (void) BUG(); } +#ifdef CONFIG_DISCONTIGMEM + /* + * Check to see if there is a closer copy of the kernel to use. + */ + kloadnode = paddr_to_nid(__tpa(&kcopybase[0])); + if (node_cpuid[cpuid].nid != kloadnode && kcopybase[node_cpuid[cpuid].nid] != ~0UL) + ia64_remap_kernel(__pa(kcopybase[node_cpuid[cpuid].nid])); +#endif + smp_setup_percpu_timer(); /* @@ -340,6 +355,7 @@ smp_callin (void) Dprintk("Going to syncup ITC with BP.\n"); ia64_sync_itc(0); + /* * Get our bogomips. */ @@ -567,7 +583,7 @@ init_smp_config(void) /* Tell SAL where to drop the AP's. */ ap_startup = (struct fptr *) start_ap; sal_ret = ia64_sal_set_vectors(SAL_VECTOR_OS_BOOT_RENDEZ, - __pa(ap_startup->fp), __pa(ap_startup->gp), 0, 0, 0, 0); + __tpa(ap_startup->fp), __tpa(ap_startup->gp), 0, 0, 0, 0); if (sal_ret < 0) { printk("SMP: Can't set SAL AP Boot Rendezvous: %s\n Forcing UP mode\n", ia64_sal_strerror(sal_ret)); diff -urNp disc/arch/ia64/mm/extable.c krel/arch/ia64/mm/extable.c --- disc/arch/ia64/mm/extable.c Mon Jul 29 15:23:19 2002 +++ krel/arch/ia64/mm/extable.c Fri Aug 9 13:45:29 2002 @@ -6,9 +6,8 @@ */ #include - +#include #include -#include extern const struct exception_table_entry __start___ex_table[]; extern const struct exception_table_entry __stop___ex_table[]; @@ -16,28 +15,28 @@ extern const struct exception_table_entr static inline const struct exception_table_entry * search_one_table (const struct exception_table_entry *first, const struct exception_table_entry *last, - unsigned long ip, unsigned long gp) + unsigned long value) { + /* Abort early if the search value is out of range. */ + if (value < first->addr || value > last->addr) + return 0; + while (first <= last) { const struct exception_table_entry *mid; long diff; mid = &first[(last - first)/2]; - diff = (mid->addr + gp) - ip; + diff = mid->addr - value; if (diff == 0) return mid; else if (diff < 0) - first = mid + 1; + first = mid+1; else - last = mid - 1; + last = mid-1; } return 0; } -#ifndef CONFIG_MODULES -register unsigned long main_gp __asm__("gp"); -#endif - struct exception_fixup search_exception_table (unsigned long addr) { @@ -46,9 +45,9 @@ search_exception_table (unsigned long ad #ifndef CONFIG_MODULES /* There is only the kernel to search. */ - entry = search_one_table(__start___ex_table, __stop___ex_table - 1, addr, main_gp); + entry = search_one_table(__start___ex_table, __stop___ex_table - 1, addr); if (entry) - fix.cont = entry->cont + main_gp; + fix.cont = entry->cont; return fix; #else struct archdata *archdata; @@ -58,13 +57,9 @@ search_exception_table (unsigned long ad for (mp = module_list; mp; mp = mp->next) { if (!mp->ex_table_start) continue; - archdata = (struct archdata *) mp->archdata_start; - if (!archdata) - continue; - entry = search_one_table(mp->ex_table_start, mp->ex_table_end - 1, - addr, (unsigned long) archdata->gp); + entry = search_one_table(mp->ex_table_start, mp->ex_table_end - 1, addr); if (entry) { - fix.cont = entry->cont + (unsigned long) archdata->gp; + fix.cont = entry->cont; return fix; } } diff -urNp disc/arch/ia64/mm/init.c krel/arch/ia64/mm/init.c --- disc/arch/ia64/mm/init.c Fri Aug 9 13:45:05 2002 +++ krel/arch/ia64/mm/init.c Fri Aug 9 13:45:29 2002 @@ -32,13 +32,111 @@ mmu_gather_t mmu_gathers[NR_CPUS]; /* References to section boundaries: */ -extern char _stext, _etext, _edata, __init_begin, __init_end; +extern char _stext, _etext, _edata, __init_begin, __init_etext, __init_sdata, __init_end; extern void ia64_tlb_init (void); unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL; static unsigned long totalram_pages; +struct page *zero_page_memmap_ptr; /* map entry for zero page */ + +#ifdef CONFIG_NUMA + +/* + * Set ktreplicate to 0 to disable kernel text replication. + */ +static int ktreplicate=1; + +static int __init replicate_setup(char *str) +{ + get_option(&str, &ktreplicate); + return 1; +} + +__setup("ktreplicate=", replicate_setup); + + +/* + * Addresses of per-node copies of kernel text/readonly-data + */ +unsigned long kcopybase[PLAT_MAX_COMPACT_NODES]; + +/* + * make properly aligned copies of kernel text and read-only + * data on other nodes. + */ +static void +replicate_kernel(void) +{ + cnodeid_t cnodeid; + char *kpage; + unsigned long kstart = (unsigned long)&_stext; + unsigned long kend = (unsigned long)&__init_etext; + unsigned long size; + cnodeid_t kloadnode; + int cpuid; + extern void ia64_remap_kernel(long); + + memset(kcopybase, -1, sizeof(kcopybase)); + + if (!ktreplicate) { + printk("Kernel text replication is disabled\n"); + return; + } + + kloadnode = paddr_to_nid(__tpa(&kcopybase[0])); + kcopybase[kloadnode] = __imva(&kcopybase[0]) & ~(KERNEL_TR_PAGE_SIZE - 1); + + /* + * Copy to all of the other nodes + */ + for (cnodeid = 0; cnodeid < numnodes; cnodeid++) { + if (cnodeid == kloadnode) + continue; + size = kend & (KERNEL_TR_PAGE_SIZE - 1); + size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + + /* allocate big page aligned from target node */ + kpage = __alloc_bootmem_node(BOOT_NODE_DATA(cnodeid), size, + KERNEL_TR_PAGE_SIZE, 0L); + + if (kpage == 0) { + printk("Could not replicate kernel to node %d\n", cnodeid); + continue; + } + + /* + * Kernel (_stext) probably does not begin on a big page + * boundary. Free up any pages before needed portion + */ + size = kstart & (KERNEL_TR_PAGE_SIZE - 1); + if (size) + free_bootmem_node(BOOT_NODE_DATA(cnodeid), + __pa(kpage), size); + + kcopybase[cnodeid] = (unsigned long)kpage; + + /* now make the copy */ + memcpy(kpage + size, &_stext, kend-kstart); + + printk("Replicated kernel to %p on node %d\n", (long*)(kpage + size), cnodeid); + } + + /* + * Search the node_cpuid table for our hard_processor_id. We + * are not necessarily on the first cpu in the list. + */ + for (cpuid=0; cpuidflags); + set_page_count(virt_to_page(addr), 1); + free_page(addr); + ++totalram_pages; + ++pages_freed; + } + } +#else + addr = __imva(&__init_begin); + eaddr = __imva(&__init_end); + for (; addr < eaddr; addr += PAGE_SIZE) { clear_bit(PG_reserved, &virt_to_page(addr)->flags); set_page_count(virt_to_page(addr), 1); free_page(addr); ++totalram_pages; + ++pages_freed; } +#endif + printk ("Freeing unused kernel memory: %ldkB freed\n", - (&__init_end - &__init_begin) >> 10); + (pages_freed << PAGE_SHIFT) >> 10); } void @@ -188,83 +319,84 @@ si_meminfo (struct sysinfo *val) return; } -#ifdef CONFIG_DISCONTIGMEM void show_mem(void) { - int i, nid; - int reserved, cached, slab, free, active; - pg_data_t *pgdat; - zone_t *zone; - - printk("%4s %7s %7s %9s %7s %7s %7s\n", "node","total", - "free","reserved","swap","slab","active"); + printk("Mem-info:\n"); + show_free_areas(); - /* - * Iterate over each node's pg_data_t and look at its pages - */ - for(nid = 0; nid < numnodes; nid++) { - pgdat = NODE_DATA(nid); - reserved = cached = slab = free = active = 0; +#ifdef CONFIG_DISCONTIGMEM + { + int i, nid; + int reserved, cached, slab, free, active; + pg_data_t *pgdat; + zone_t *zone; + + printk("%4s %7s %7s %9s %7s %7s %7s\n", "node","total", + "free","reserved","swap","slab","active"); /* - * Get info about each page in the node + * Iterate over each node's pg_data_t and look at its pages */ - printk("%4d ", pgdat->node_id); - for(i = 0; i < pgdat->node_size; i++) { - if (PageReserved(pgdat->node_mem_map + i)) + for(nid = 0; nid < numnodes; nid++) { + pgdat = NODE_DATA(nid); + reserved = cached = slab = free = active = 0; + + /* + * Get info about each page in the node + */ + printk("%4d ", pgdat->node_id); + for(i = 0; i < pgdat->node_size; i++) { + if (PageReserved(pgdat->node_mem_map + i)) + reserved++; + if (PageSwapCache(pgdat->node_mem_map + i)) + cached++; + if (PageSlab(pgdat->node_mem_map + i)) + slab++; + if (PageActive(pgdat->node_mem_map + i)) + active++; + } + for (zone = pgdat->node_zones; + zone < pgdat->node_zones + MAX_NR_ZONES; zone++) + free += zone->free_pages; + + printk("%7ld ", pgdat->node_size); + printk("%7d ", free); + printk("%9d ", reserved); + printk("%7d ", cached); + printk("%7d ", slab); + printk("%7d\n", active); + } + printk("Total of %ld pages in page table cache\n", pgtable_cache_size); + show_buffers(); + printk("%d free buffer pages\n", nr_free_buffer_pages()); + } +#else /* !CONFIG_DISCONTIGMEM */ + { + int i, total = 0, reserved = 0; + int shared = 0, cached = 0; + printk("Free swap: %6dkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); + i = max_mapnr; + while (i-- > 0) { + if (!VALID_PAGE(mem_map + i)) + continue; + total++; + if (PageReserved(mem_map+i)) reserved++; - if (PageSwapCache(pgdat->node_mem_map + i)) + else if (PageSwapCache(mem_map+i)) cached++; - if (PageSlab(pgdat->node_mem_map + i)) - slab++; - if (PageActive(pgdat->node_mem_map + i)) - active++; + else if (page_count(mem_map + i)) + shared += page_count(mem_map + i) - 1; } - for (zone = pgdat->node_zones; - zone < pgdat->node_zones + MAX_NR_ZONES; zone++) - free += zone->free_pages; - - printk("%7ld ", pgdat->node_size); - printk("%7d ", free); - printk("%9d ", reserved); - printk("%7d ", cached); - printk("%7d ", slab); - printk("%7d\n", active); + printk("%d pages of RAM\n", total); + printk("%d reserved pages\n", reserved); + printk("%d pages shared\n", shared); + printk("%d pages swap cached\n", cached); + printk("%ld pages in page table cache\n", pgtable_cache_size); + show_buffers(); } - show_buffers(); -} -#else /* !CONFIG_DISCONTIGMEM */ -void -show_mem(void) -{ - int i, total = 0, reserved = 0; - int shared = 0, cached = 0; - - printk("Mem-info:\n"); - show_free_areas(); - - printk("Free swap: %6dkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); - i = max_mapnr; - while (i-- > 0) { - if (!VALID_PAGE(mem_map + i)) - continue; - total++; - if (PageReserved(mem_map+i)) - reserved++; - else if (PageSwapCache(mem_map+i)) - cached++; - else if (page_count(mem_map + i)) - shared += page_count(mem_map + i) - 1; - } - printk("%d pages of RAM\n", total); - printk("%d reserved pages\n", reserved); - printk("%d pages shared\n", shared); - printk("%d pages swap cached\n", cached); - printk("%ld pages in page table cache\n", pgtable_cache_size); - show_buffers(); -} #endif /* !CONFIG_DISCONTIGMEM */ +} /* * This is like put_dirty_page() but installs a clean page with PAGE_GATE protection @@ -279,7 +411,7 @@ put_gate_page (struct page *page, unsign if (!PageReserved(page)) printk("put_gate_page: gate page at 0x%p not in reserved memory\n", - page_address(page)); + (long*)page_address(page)); pgd = pgd_offset_k(address); /* note: this is NOT pgd_offset()! */ @@ -330,7 +462,7 @@ ia64_mmu_init (void *my_cpu_data) ia64_srlz_d(); ia64_itr(0x2, IA64_TR_PERCPU_DATA, PERCPU_ADDR, - pte_val(mk_pte_phys(__pa(my_cpu_data), PAGE_KERNEL)), PAGE_SHIFT); + pte_val(mk_pte_phys(__tpa(my_cpu_data), PAGE_KERNEL)), PAGE_SHIFT); ia64_set_psr(psr); ia64_srlz_i(); @@ -543,6 +675,8 @@ paging_init (void) #endif unsigned long zones_size[MAX_NR_ZONES]; + clear_page((void*)__imva(empty_zero_page)); + /* initialize mem_map[] */ memset(zones_size, 0, sizeof(zones_size)); @@ -601,6 +735,8 @@ paging_init (void) free_area_init(zones_size); #endif /* !CONFIG_VIRTUAL_MEM_MAP */ #endif /* CONFIG_DISCONTIGMEM */ + + zero_page_memmap_ptr = virt_to_page(__imva(empty_zero_page)); } static int @@ -639,14 +775,17 @@ mem_init (void) #endif high_memory = __va(max_low_pfn * PAGE_SIZE); +#ifdef CONFIG_NUMA + replicate_kernel(); +#endif totalram_pages += free_all_bootmem(); reserved_pages = 0; efi_memmap_walk(count_reserved_pages, &reserved_pages); - codesize = (unsigned long) &_etext - (unsigned long) &_stext; - datasize = (unsigned long) &_edata - (unsigned long) &_etext; - initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; + codesize = &_etext - &_stext; + datasize = &_edata - &__init_end; + initsize = (&__init_etext - &__init_begin) + (&__init_end - &__init_sdata); printk("Memory: %luk/%luk available (%luk code, %luk reserved, %luk data, %luk init)\n", (unsigned long) nr_free_pages() << (PAGE_SHIFT - 10), @@ -669,9 +808,112 @@ mem_init (void) show_mem() ; /* install the gate page in the global page table: */ - put_gate_page(virt_to_page(__start_gate_section), GATE_ADDR); + put_gate_page(virt_to_page(__imva(__start_gate_section)), GATE_ADDR); #ifdef CONFIG_IA32_SUPPORT ia32_gdt_init(); #endif + } + + + + +/**********************************************************************************************/ +#ifdef CONFIG_IA64_SGI_SN_DEBUG +/* + * Debug versions of __va, __pa, __imva & _tpa. These routines + * catch common misuse of the macros. + * + * See include/asm-ia64/page.h for more details + * + */ + + +#define __xva(x) ({ia64_va _v; _v.l = (long) (x); _v.f.reg = -1; _v.p;}) +#define __xtpa(x) ({ia64_va _v; asm("tpa %0=%1" : "=r"(_v.l) : "r"(x)); _v.l;}) + + +extern char _end; +extern char __gp; + +void +bkpt(long l, int i) +{ + i = i; +} + +unsigned long +__pa_debug(long v) +{ + ia64_va a; + int reg; + + a.l = v; + reg = a.f.reg; + a.f.reg = 0; + if (v != 0 && (reg < 6 || (v != MAX_DMA_ADDRESS && a.l != __xtpa(v)))) { + bkpt(v, 0); + printk("BUG __pa 0x%lx, from 0x%lx\n", v, (unsigned long) __builtin_return_address(0)); + panic ("__pa_debug"); + } + return a.l; +} + +void * +__va_debug(long p) +{ + ia64_va a; + int reg; + + a.l = p; + reg = a.f.reg; + a.f.reg = 7; + if (reg != 0 || __xtpa(a.p) != p) { + bkpt(p, 1); + printk("BUG __va 0x%lx, from 0x%lx\n", p, (unsigned long) __builtin_return_address(0)); + panic ("__va_debug"); + } + return a.p; +} + +unsigned long +__tpa_debug(long v) +{ + ia64_va a; + + a.l = v; + if (a.f.reg != 7) { + bkpt(v, 2); + printk("BUG __tpa 0x%lx, from 0x%lx\n", v, (unsigned long) __builtin_return_address(0)); + panic ("__tpa_debug"); + } + + return __xtpa(v); +} + + +unsigned long +__imva_debug(long v) +{ + ia64_va a; + unsigned long p; + + a.l = v; + if (a.f.reg != 7) { + bkpt(v, 3); + printk("BUG __imva 0x%lx, from 0x%lx\n", v, (unsigned long) __builtin_return_address(0)); + panic ("__imva_debug"); + } + + p = __xtpa(v); + a.l = p; + a.f.reg = 7; + if (__xtpa(a.p) != p) { + bkpt(p, 4); + printk("BUG __imva 0x%lx -> 0x%lx, from 0x%lx\n", v, a.l, (unsigned long) __builtin_return_address(0)); + panic ("__imva_debug"); + } + return a.l; +} +#endif diff -urNp disc/arch/ia64/vmlinux.lds.S krel/arch/ia64/vmlinux.lds.S --- disc/arch/ia64/vmlinux.lds.S Mon Jul 29 15:23:21 2002 +++ krel/arch/ia64/vmlinux.lds.S Fri Aug 9 13:45:29 2002 @@ -3,10 +3,22 @@ #include #include #include +#include OUTPUT_FORMAT("elf64-ia64-little") OUTPUT_ARCH(ia64) ENTRY(phys_start) + +PHDRS +{ + headers PT_PHDR PHDRS; + pernode PT_LOAD; + global PT_LOAD; +} + +#define PERNODEADDR(section) KERNPHYSBASE + ADDR(section) - VPERNODE_BASE +#define GLOBALADDR(section) KERNPHYSBASE + ADDR(section) - VGLOBAL_BASE + SECTIONS { /* Sections to be discarded */ @@ -19,26 +31,83 @@ SECTIONS } v = PAGE_OFFSET; /* this symbol is here to make debugging easier... */ - phys_start = _start - PAGE_OFFSET; + phys_start = KERNPHYSBASE + _start - VPERNODE_BASE; - . = KERNEL_START; + . = VPERNODE_BASE; _text = .; _stext = .; - .text : AT(ADDR(.text) - PAGE_OFFSET) + .text : AT(PERNODEADDR(.text)) { *(.text.ivt) + . = ALIGN(PAGE_SIZE); + __start_gate_section = .; + *(.text.gate) + __stop_gate_section = .; *(.text) - } - .text2 : AT(ADDR(.text2) - PAGE_OFFSET) - { *(.text2) } + } :pernode + .text2 : AT(PERNODEADDR(.text2)) + { *(.text2) } :pernode #ifdef CONFIG_SMP - .text.lock : AT(ADDR(.text.lock) - PAGE_OFFSET) - { *(.text.lock) } + .text.lock : AT(PERNODEADDR(.text.lock)) + { *(.text.lock) } :pernode #endif _etext = .; /* Read-only data */ + .rodata : AT(PERNODEADDR(.rodata)) + { *(.rodata) *(.rodata.*) } :pernode + + /* Unwind info & table: */ + . = ALIGN(8); + .IA_64.unwind_info : AT(PERNODEADDR(.IA_64.unwind_info)) + { *(.IA_64.unwind_info*) } + . = ALIGN(8); + ia64_unw_start = .; + .IA_64.unwind : AT(PERNODEADDR(.IA_64.unwind)) + { *(.IA_64.unwind*) } + ia64_unw_end = .; + . = ALIGN(16); + + /* Initialization code (must be last in :pernode area) */ + + . = ALIGN(PAGE_SIZE); + __init_begin = .; + .text.init : AT(PERNODEADDR(.text.init)) + { *(.text.init) } :pernode + + __init_etext = .; + /* Jump virtual address from pernode to global area */ + . = VGLOBAL_BASE + (. - VPERNODE_BASE); + __init_sdata = .; + + /* Initialization data (must be first in :global area) */ + + .data.init : AT(GLOBALADDR(.data.init)) + { *(.data.init) } :global + . = ALIGN(16); + __setup_start = .; + .setup.init : AT(GLOBALADDR(.setup.init)) + { *(.setup.init) } + __setup_end = .; + __initcall_start = .; + .initcall.init : AT(GLOBALADDR(.initcall.init)) + { *(.initcall.init) } + __initcall_end = .; + . = ALIGN(PAGE_SIZE); + __init_end = .; + + /* The initial task and kernel stack */ + init_task : AT(GLOBALADDR(init_task)) + { *(init_task) } + + .data.page_aligned : AT(GLOBALADDR(.data.page_aligned)) + { + *(.data.idt) + *(__special_page_section) + } + + __gp = ALIGN(16) + 0x200000; /* gp must be 16-byte aligned for exc. table */ /* Global data */ _data = .; @@ -46,106 +115,59 @@ SECTIONS /* Exception table */ . = ALIGN(16); __start___ex_table = .; - __ex_table : AT(ADDR(__ex_table) - PAGE_OFFSET) - { *(__ex_table) } + __ex_table : AT(GLOBALADDR(__ex_table)) + { *(__ex_table) } :global __stop___ex_table = .; #if defined(CONFIG_IA64_GENERIC) /* Machine Vector */ . = ALIGN(16); machvec_start = .; - .machvec : AT(ADDR(.machvec) - PAGE_OFFSET) + .machvec : AT(GLOBALADDR(.machvec)) { *(.machvec) } machvec_end = .; #endif __start___ksymtab = .; /* Kernel symbol table */ - __ksymtab : AT(ADDR(__ksymtab) - PAGE_OFFSET) + __ksymtab : AT(GLOBALADDR(__ksymtab)) { *(__ksymtab) } __stop___ksymtab = .; __start___kallsyms = .; /* All kernel symbols for debugging */ - __kallsyms : AT(ADDR(__kallsyms) - PAGE_OFFSET) + __kallsyms : AT(GLOBALADDR(__kallsyms)) { *(__kallsyms) } __stop___kallsyms = .; - /* Unwind info & table: */ - . = ALIGN(8); - .IA_64.unwind_info : AT(ADDR(.IA_64.unwind_info) - PAGE_OFFSET) - { *(.IA_64.unwind_info*) } - ia64_unw_start = .; - .IA_64.unwind : AT(ADDR(.IA_64.unwind) - PAGE_OFFSET) - { *(.IA_64.unwind*) } - ia64_unw_end = .; - - .rodata : AT(ADDR(.rodata) - PAGE_OFFSET) - { *(.rodata) *(.rodata.*) } - .kstrtab : AT(ADDR(.kstrtab) - PAGE_OFFSET) + .kstrtab : AT(GLOBALADDR(.kstrtab)) { *(.kstrtab) } - .opd : AT(ADDR(.opd) - PAGE_OFFSET) + .opd : AT(GLOBALADDR(.opd)) { *(.opd) } - /* Initialization code and data: */ - - . = ALIGN(PAGE_SIZE); - __init_begin = .; - .text.init : AT(ADDR(.text.init) - PAGE_OFFSET) - { *(.text.init) } - - .data.init : AT(ADDR(.data.init) - PAGE_OFFSET) - { *(.data.init) } - . = ALIGN(16); - __setup_start = .; - .setup.init : AT(ADDR(.setup.init) - PAGE_OFFSET) - { *(.setup.init) } - __setup_end = .; - __initcall_start = .; - .initcall.init : AT(ADDR(.initcall.init) - PAGE_OFFSET) - { *(.initcall.init) } - __initcall_end = .; - . = ALIGN(PAGE_SIZE); - __init_end = .; - - /* The initial task and kernel stack */ - init_task : AT(ADDR(init_task) - PAGE_OFFSET) - { *(init_task) } - - .data.page_aligned : AT(ADDR(.data.page_aligned) - PAGE_OFFSET) - { *(__special_page_section) - __start_gate_section = .; - *(.text.gate) - __stop_gate_section = .; - } - - . = ALIGN(SMP_CACHE_BYTES); - .data.cacheline_aligned : AT(ADDR(.data.cacheline_aligned) - PAGE_OFFSET) + . = ALIGN(L1_CACHE_BYTES); + .data.cacheline_aligned : AT(GLOBALADDR(.data.cacheline_aligned)) { *(.data.cacheline_aligned) } /* Kernel symbol names for modules: */ - .kstrtab : AT(ADDR(.kstrtab) - PAGE_OFFSET) + .kstrtab : AT(GLOBALADDR(.kstrtab)) { *(.kstrtab) } - .data : AT(ADDR(.data) - PAGE_OFFSET) + .data : AT(GLOBALADDR(.data)) { *(.data) *(.gnu.linkonce.d*) CONSTRUCTORS } - - . = ALIGN(16); - __gp = . + 0x200000; /* gp must be 16-byte aligned for exc. table */ - - .got : AT(ADDR(.got) - PAGE_OFFSET) + .got : AT(GLOBALADDR(.got)) { *(.got.plt) *(.got) } /* We want the small data sections together, so single-instruction offsets can access them all, and initialized data all before uninitialized, so we can shorten the on-disk segment size. */ - .sdata : AT(ADDR(.sdata) - PAGE_OFFSET) + .sdata : AT(GLOBALADDR(.sdata)) { *(.sdata) } _edata = .; _bss = .; - .sbss : AT(ADDR(.sbss) - PAGE_OFFSET) + .sbss : AT(GLOBALADDR(.sbss)) { *(.sbss) *(.scommon) } - .bss : AT(ADDR(.bss) - PAGE_OFFSET) + .bss : AT(GLOBALADDR(.bss)) { *(.bss) *(COMMON) } - . = ALIGN(64 / 8); + . = ALIGN(L1_CACHE_BYTES / 8); _end = .; /* Stabs debugging sections. */ diff -urNp disc/fs/proc/kcore.c krel/fs/proc/kcore.c --- disc/fs/proc/kcore.c Fri Sep 14 01:04:43 2001 +++ krel/fs/proc/kcore.c Fri Aug 9 13:45:29 2002 @@ -21,6 +21,9 @@ #include #include +#ifdef CONFIG_NUMA +extern char _stext, __init_etext, __init_sdata, _end; +#endif static int open_kcore(struct inode * inode, struct file * filp) { @@ -112,6 +115,12 @@ struct memelfnote extern char saved_command_line[]; +#ifdef CONFIG_NUMA +#define KCORE_NPHDRS 4 +#else +#define KCORE_NPHDRS 2 +#endif + static size_t get_kcore_size(int *num_vma, size_t *elf_buflen) { size_t try, size; @@ -131,7 +140,7 @@ static size_t get_kcore_size(int *num_vm *num_vma = *num_vma + 1; } *elf_buflen = sizeof(struct elfhdr) + - (*num_vma + 2)*sizeof(struct elf_phdr) + + (*num_vma + KCORE_NPHDRS)*sizeof(struct elf_phdr) + 3 * sizeof(struct memelfnote); *elf_buflen = PAGE_ALIGN(*elf_buflen); return (size - PAGE_OFFSET + *elf_buflen); @@ -212,7 +221,7 @@ static void elf_kcore_store_hdr(char *bu elf->e_flags = 0; elf->e_ehsize = sizeof(struct elfhdr); elf->e_phentsize= sizeof(struct elf_phdr); - elf->e_phnum = 2 + num_vma; + elf->e_phnum = KCORE_NPHDRS + num_vma; elf->e_shentsize= 0; elf->e_shnum = 0; elf->e_shstrndx = 0; @@ -237,12 +246,38 @@ static void elf_kcore_store_hdr(char *bu offset += sizeof(struct elf_phdr); phdr->p_type = PT_LOAD; phdr->p_flags = PF_R|PF_W|PF_X; - phdr->p_offset = dataoff; + phdr->p_offset = PAGE_OFFSET - 0x8000000000000000; phdr->p_vaddr = PAGE_OFFSET; phdr->p_paddr = __pa(PAGE_OFFSET); phdr->p_filesz = phdr->p_memsz = ((unsigned long)high_memory - PAGE_OFFSET); phdr->p_align = PAGE_SIZE; +#ifdef CONFIG_NUMA + /* pernode area */ + phdr = (struct elf_phdr *) bufp; + bufp += sizeof(struct elf_phdr); + offset += sizeof(struct elf_phdr); + phdr->p_type = PT_LOAD; + phdr->p_flags = PF_R|PF_X; + phdr->p_offset = (size_t)&_stext - 0x8000000000000000; + phdr->p_vaddr = (unsigned long)&_stext; + phdr->p_paddr = __tpa(&_stext); + phdr->p_filesz = phdr->p_memsz = &__init_etext - &_stext; + phdr->p_align = PAGE_SIZE; + + /* global area */ + phdr = (struct elf_phdr *) bufp; + bufp += sizeof(struct elf_phdr); + offset += sizeof(struct elf_phdr); + phdr->p_type = PT_LOAD; + phdr->p_flags = PF_R|PF_W|PF_X; + phdr->p_offset = (size_t)&__init_sdata - 0x8000000000000000; + phdr->p_vaddr = (unsigned long)&__init_sdata; + phdr->p_paddr = __tpa(&__init_sdata); + phdr->p_filesz = phdr->p_memsz = &_end - &__init_sdata; + phdr->p_align = PAGE_SIZE; +#endif + /* setup ELF PT_LOAD program header for every vmalloc'd area */ for (m=vmlist; m; m=m->next) { if (m->flags & VM_IOREMAP) /* don't dump ioremap'd stuff! (TA) */ @@ -254,9 +289,13 @@ static void elf_kcore_store_hdr(char *bu phdr->p_type = PT_LOAD; phdr->p_flags = PF_R|PF_W|PF_X; - phdr->p_offset = (size_t)m->addr - PAGE_OFFSET + dataoff; + phdr->p_offset = (size_t)m->addr - 0x8000000000000000; phdr->p_vaddr = (size_t)m->addr; +#ifdef CONFIG_DISCONTIGMEM + phdr->p_paddr = __tpa(m->addr); +#else phdr->p_paddr = __pa(m->addr); +#endif phdr->p_filesz = phdr->p_memsz = m->size; phdr->p_align = PAGE_SIZE; } @@ -354,9 +393,7 @@ static ssize_t read_kcore(struct file *f buffer += tsz; acc += tsz; - /* leave now if filled buffer already */ - if (buflen == 0) - return acc; + return acc; } else read_unlock(&vmlist_lock); @@ -382,7 +419,7 @@ static ssize_t read_kcore(struct file *f } #endif /* fill the remainder of the buffer from kernel VM space */ - start = (unsigned long)__va(*fpos - elf_buflen); + start = *fpos + 0x8000000000000000; if ((tsz = (PAGE_SIZE - (start & ~PAGE_MASK))) > buflen) tsz = buflen; @@ -432,6 +469,16 @@ static ssize_t read_kcore(struct file *f return -EFAULT; } kfree(elf_buf); +#ifdef CONFIG_NUMA + } else if (start >= (unsigned long)&_stext && + start < (unsigned long)&__init_etext) { + if (copy_to_user(buffer, (char *)start, tsz)) + return -EFAULT; + } else if (start >= (unsigned long)&__init_sdata && + start < (unsigned long)&_end) { + if (copy_to_user(buffer, (char *)start, tsz)) + return -EFAULT; +#endif } else if ((start > PAGE_OFFSET) && (start < (unsigned long)high_memory)) { if (kern_addr_valid(start)) { diff -urNp disc/include/asm-ia64/asmmacro.h krel/include/asm-ia64/asmmacro.h --- disc/include/asm-ia64/asmmacro.h Thu Apr 5 21:51:47 2001 +++ krel/include/asm-ia64/asmmacro.h Fri Aug 9 13:45:29 2002 @@ -38,17 +38,17 @@ name: #if __GNUC__ >= 3 # define EX(y,x...) \ - .xdata4 "__ex_table", @gprel(99f), @gprel(y); \ + .xdata8 "__ex_table", 99f, y; \ [99:] x # define EXCLR(y,x...) \ - .xdata4 "__ex_table", @gprel(99f), @gprel(y)+4; \ + .xdata8 "__ex_table", 99f, y+4; \ [99:] x #else # define EX(y,x...) \ - .xdata4 "__ex_table", @gprel(99f), @gprel(y); \ + .xdata8 "__ex_table", 99f, y; \ 99: x # define EXCLR(y,x...) \ - .xdata4 "__ex_table", @gprel(99f), @gprel(y)+4; \ + .xdata8 "__ex_table", 99f, y+4; \ 99: x #endif diff -urNp disc/include/asm-ia64/kregs.h krel/include/asm-ia64/kregs.h --- disc/include/asm-ia64/kregs.h Mon Jul 29 15:23:26 2002 +++ krel/include/asm-ia64/kregs.h Fri Aug 9 13:45:29 2002 @@ -26,10 +26,11 @@ /* * Translation registers: */ -#define IA64_TR_KERNEL 0 /* itr0, dtr0: maps kernel image (code & data) */ +#define IA64_TR_KERNEL 0 /* itr0, dtr0: maps pernode kernel image (code & rodata) */ #define IA64_TR_PALCODE 1 /* itr1: maps PALcode as required by EFI */ #define IA64_TR_PERCPU_DATA 1 /* dtr1: percpu data */ #define IA64_TR_CURRENT_STACK 2 /* dtr2: maps kernel's memory- & register-stacks */ +#define IA64_TR_KERNEL_GLOBAL 3 /* dtr3: maps kernel global data*/ /* Processor status register bits: */ #define IA64_PSR_BE_BIT 1 diff -urNp disc/include/asm-ia64/page.h krel/include/asm-ia64/page.h --- disc/include/asm-ia64/page.h Fri Aug 9 13:45:05 2002 +++ krel/include/asm-ia64/page.h Fri Aug 9 13:45:29 2002 @@ -31,6 +31,12 @@ #define PAGE_ALIGN(addr) (((addr) + PAGE_SIZE - 1) & PAGE_MASK) #ifdef __ASSEMBLY__ +/* + * NOTE: These macros DO NOT work for kernel text/data addresses on all platforms + * since kernel text/static-data is not necessarily identity mapped. + * Use these macros ONLY for identity mapped addresses. + * (See __tpa & __imva) + */ # define __pa(x) ((x) - PAGE_OFFSET) # define __va(x) ((x) + PAGE_OFFSET) #else /* !__ASSEMBLY */ @@ -78,13 +84,47 @@ typedef union ia64_va { } ia64_va; /* - * Note: These macros depend on the fact that PAGE_OFFSET has all + * __pa(vaddr) - convert identity mapped reg 7 virtual address to a physical address. + * __va(paddr) - convert identity mapped reg 7 physical address to a virtual address. + * + * NOTE: These macros DO NOT work for kernel text/data addresses on all platforms + * since kernel text/static-data is not necessarily identity mapped. + * Use these macros ONLY for identify mapped addresses. + * + * NOTE: These macros depend on the fact that PAGE_OFFSET has all * region bits set to 1 and all other bits set to zero. They are * expressed in this way to ensure they result in a single "dep" - * instruction. + * instruction. The __pa & __va macros also work only for addresses that are + * identity mapped. Use the __tpa & __imva addresses for kernel text/static-data addresses. + * + * + * __imva(vaddr) - convert a kernel text/static-data virtual address to an identity mapped + * virtual address. + * __tpa(vaddr) - convert a kernel text/static-data virtual address to a physical address + * This macro works for all region 7 addresses, whether identity mapped or not. + * (performance may be slightly slower - only an issue in highly performance + * critical areas). + * + * Note: these macros work for all region 7 addresses. + * + * Note: turn on DEBUG_PA_VA to get additional debugging on these macros. */ + +#ifdef CONFIG_IA64_SGI_SN_DEBUG +extern unsigned long __pa_debug(long); +extern void * __va_debug(long); +extern unsigned long __tpa_debug(long); +extern unsigned long __imva_debug(long); +#define __pa(x) __pa_debug((long)(x)) +#define __va(x) __va_debug((long)(x)) +#define __tpa(x) __tpa_debug((long)(x)) +#define __imva(x) __imva_debug((long)(x)) +#else #define __pa(x) ({ia64_va _v; _v.l = (long) (x); _v.f.reg = 0; _v.l;}) #define __va(x) ({ia64_va _v; _v.l = (long) (x); _v.f.reg = -1; _v.p;}) +#define __tpa(x) ({ia64_va _v; asm("tpa %0=%1" : "=r"(_v.l) : "r"(x)); _v.l;}) +#define __imva(x) ((long)__va(__tpa(x))) +#endif #define REGION_NUMBER(x) ({ia64_va _v; _v.l = (long) (x); _v.f.reg;}) #define REGION_OFFSET(x) ({ia64_va _v; _v.l = (long) (x); _v.f.off;}) diff -urNp disc/include/asm-ia64/pgtable.h krel/include/asm-ia64/pgtable.h --- disc/include/asm-ia64/pgtable.h Fri Aug 9 13:45:05 2002 +++ krel/include/asm-ia64/pgtable.h Fri Aug 9 13:45:29 2002 @@ -459,7 +459,8 @@ extern void paging_init (void); * for zero-mapped memory areas etc.. */ extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) +extern struct page *zero_page_memmap_ptr; +#define ZERO_PAGE(vaddr) (zero_page_memmap_ptr) /* We provide our own get_unmapped_area to cope with VA holes for userland */ #define HAVE_ARCH_UNMAPPED_AREA @@ -513,6 +514,5 @@ extern void arch_memmap_zone_init (memma */ #define KERNEL_TR_PAGE_SHIFT _PAGE_SIZE_64M #define KERNEL_TR_PAGE_SIZE (1 << KERNEL_TR_PAGE_SHIFT) -#define KERNEL_TR_PAGE_NUM ((KERNEL_START - PAGE_OFFSET) / KERNEL_TR_PAGE_SIZE) #endif /* _ASM_IA64_PGTABLE_H */ diff -urNp disc/include/asm-ia64/system.h krel/include/asm-ia64/system.h --- disc/include/asm-ia64/system.h Mon Jul 29 15:23:28 2002 +++ krel/include/asm-ia64/system.h Fri Aug 9 13:45:29 2002 @@ -18,7 +18,22 @@ #include #include -#define KERNEL_START (PAGE_OFFSET + 68*1024*1024) +/* + * Virtual base address of per-node private space (used for kernel + * code and read-only data) + */ +#define VPERNODE_BASE 0xe002000000000000 + +/* + * Virtual base address of kernel global space (read-write data) + */ +#define VGLOBAL_BASE 0xe002100000000000 + +/* physical address to load kernel */ +#define KERNPHYSBASE (64*1024*1024) + +/* virtual address to load kernel */ +#define KERNEL_START (VPERNODE_BASE) #define GATE_ADDR (0xa000000000000000 + PAGE_SIZE) #define PERCPU_ADDR (0xa000000000000000 + 2*PAGE_SIZE) diff -urNp disc/include/asm-ia64/uaccess.h krel/include/asm-ia64/uaccess.h --- disc/include/asm-ia64/uaccess.h Mon Jul 29 15:23:28 2002 +++ krel/include/asm-ia64/uaccess.h Fri Aug 9 13:45:29 2002 @@ -139,25 +139,25 @@ asm (".section \"__ex_table\", \"a\"\n\t #define __get_user_64(addr) \ asm ("\n"_LL"\tld8 %0=%2%P2\t// %0 and %1 get overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)+4\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f+4\n" \ _LL \ : "=r"(__gu_val), "=r"(__gu_err) : "m"(__m(addr)), "1"(__gu_err)); #define __get_user_32(addr) \ asm ("\n"_LL"\tld4 %0=%2%P2\t// %0 and %1 get overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)+4\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f+4\n" \ _LL \ : "=r"(__gu_val), "=r"(__gu_err) : "m"(__m(addr)), "1"(__gu_err)); #define __get_user_16(addr) \ asm ("\n"_LL"\tld2 %0=%2%P2\t// %0 and %1 get overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)+4\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f+4\n" \ _LL \ : "=r"(__gu_val), "=r"(__gu_err) : "m"(__m(addr)), "1"(__gu_err)); #define __get_user_8(addr) \ asm ("\n"_LL"\tld1 %0=%2%P2\t// %0 and %1 get overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)+4\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f+4\n" \ _LL \ : "=r"(__gu_val), "=r"(__gu_err) : "m"(__m(addr)), "1"(__gu_err)); @@ -201,28 +201,28 @@ extern void __put_user_unknown (void); #define __put_user_64(x,addr) \ asm volatile ( \ "\n"_LL"\tst8 %1=%r2%P1\t// %0 gets overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f\n" \ _LL \ : "=r"(__pu_err) : "m"(__m(addr)), "rO"(x), "0"(__pu_err)) #define __put_user_32(x,addr) \ asm volatile ( \ "\n"_LL"\tst4 %1=%r2%P1\t// %0 gets overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f\n" \ _LL \ : "=r"(__pu_err) : "m"(__m(addr)), "rO"(x), "0"(__pu_err)) #define __put_user_16(x,addr) \ asm volatile ( \ "\n"_LL"\tst2 %1=%r2%P1\t// %0 gets overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f\n" \ _LL \ : "=r"(__pu_err) : "m"(__m(addr)), "rO"(x), "0"(__pu_err)) #define __put_user_8(x,addr) \ asm volatile ( \ "\n"_LL"\tst1 %1=%r2%P1\t// %0 gets overwritten by exception handler\n" \ - "\t.xdata4 \"__ex_table\", @gprel(1b), @gprel(1f)\n" \ + "\t.xdata8 \"__ex_table\", 1b, 1f\n" \ _LL \ : "=r"(__pu_err) : "m"(__m(addr)), "rO"(x), "0"(__pu_err)) @@ -309,8 +309,8 @@ extern unsigned long __strnlen_user (con }) struct exception_table_entry { - int addr; /* gp-relative address of insn this fixup is for */ - int cont; /* gp-relative continuation address; if bit 2 is set, r9 is set to 0 */ + unsigned long addr; /* address of insn this fixup is for */ + unsigned long cont; /* continuation address; if bit 2 is set, r9 is set to 0 */ }; struct exception_fixup { diff -urNp disc/kernel/sched.c krel/kernel/sched.c --- disc/kernel/sched.c Fri Dec 21 18:42:04 2001 +++ krel/kernel/sched.c Fri Aug 9 13:45:29 2002 @@ -120,7 +120,12 @@ extern struct task_struct *child_reaper; #else +#ifdef CONFIG_DISCONTIGMEM +#define idle_task(cpu) (init_tasks[0]) +#else #define idle_task(cpu) (&init_task) +#endif + #define can_schedule(p,cpu) (1) #endif @@ -1304,7 +1309,12 @@ void __init init_idle(void) struct schedule_data * sched_data; sched_data = &aligned_data[smp_processor_id()].schedule_data; +#ifdef CONFIG_DISCONTIGMEM + /* The init_task structure may not be identity mapped on all IA64 platforms */ + if (current != (struct task_struct*)__imva(&init_task) && task_on_runqueue(current)) { +#else if (current != &init_task && task_on_runqueue(current)) { +#endif printk("UGH! (%d:%d) was on the runqueue, removing.\n", smp_processor_id(), current->pid); del_from_runqueue(current);