--- //depot/user/rwatson/netperf/sys/alpha/alpha/machdep.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/alpha/alpha/machdep.c	2005/01/19 20:22:17
@@ -2406,3 +2406,29 @@
 	pcpu->pc_idlepcb.apcb_ptbr = thread0.td_pcb->pcb_hw.apcb_ptbr;
 	pcpu->pc_current_asngen = 1;
 }
+
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_ipl = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_ipl);
+	}
+}
--- //depot/user/rwatson/netperf/sys/alpha/alpha/vm_machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/alpha/alpha/vm_machdep.c	2005/01/19 20:22:17
@@ -202,6 +202,10 @@
 	 */
 	td2->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 /*
@@ -321,6 +325,10 @@
 	 */
 	td->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 void
--- //depot/user/rwatson/netperf/sys/alpha/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/alpha/include/proc.h	2004/11/25 02:37:15
@@ -52,7 +52,8 @@
 	u_int64_t	md_hae;		/* user HAE register value */
 	void            *osf_sigtramp;  /* user-level signal trampoline */
 	u_int		md_kernnest;	/* nesting level in the kernel */
-	register_t	md_savecrit;	/* save PSL for critical section */
+	register_t	md_saved_ipl;	/* save IPL for critical section */
+	u_int		md_spinlock_count;
 };
 
 #define	MDP_UAC_NOPRINT	0x0010		/* Don't print unaligned traps */
--- //depot/user/rwatson/netperf/sys/amd64/amd64/machdep.c	2004/12/04 17:43:45
+++ //depot/user/rwatson/percpu/sys/amd64/amd64/machdep.c	2005/01/19 20:22:17
@@ -1287,6 +1287,32 @@
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_flags = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_flags);
+	}
+}
+
 /*
  * Construct a PCB from a trapframe. This is called from kdb_trap() where
  * we want to start a backtrace from the function that caused us to enter
--- //depot/user/rwatson/netperf/sys/amd64/amd64/vm_machdep.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/amd64/amd64/vm_machdep.c	2005/01/19 20:22:17
@@ -142,7 +142,7 @@
 	pcb2->pcb_rsp = (register_t)td2->td_frame - sizeof(void *);
 	pcb2->pcb_rbx = (register_t)td2;		/* fork_trampoline argument */
 	pcb2->pcb_rip = (register_t)fork_trampoline;
-	pcb2->pcb_rflags = td2->td_frame->tf_rflags & ~PSL_I; /* ints disabled */
+	pcb2->pcb_rflags = PSL_KERNEL; /* ints disabled */
 	/*-
 	 * pcb2->pcb_dr*:	cloned above.
 	 * pcb2->pcb_savefpu:	cloned above.
@@ -151,6 +151,10 @@
 	 * pcb2->pcb_[fg]sbase:	cloned above
 	 */
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = pcb2->pcb_rflags | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_rsp is loaded pointing to the cpu_switch() stack frame
@@ -289,6 +293,10 @@
 	 * pcb2->pcb_onfault:	cloned above (always NULL here?).
 	 * pcb2->pcb_[fg]sbase: cloned above
 	 */
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/amd64/include/proc.h	2004/04/07 03:50:45
+++ //depot/user/rwatson/percpu/sys/amd64/include/proc.h	2004/11/25 02:37:15
@@ -37,7 +37,8 @@
  * Machine-dependent part of the proc structure for AMD64.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/arm/arm/machdep.c	2004/11/06 12:44:50
+++ //depot/user/rwatson/percpu/sys/arm/arm/machdep.c	2004/11/25 02:37:15
@@ -315,6 +315,32 @@
 {
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_cspr = disable_interrupts(I32_bit | F32_bit);
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		restore_interrupts(td->td_md.md_saved_cspr);
+	}
+}
+
 /*
  * Clear registers on exec
  */
--- //depot/user/rwatson/netperf/sys/arm/arm/vm_machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/arm/arm/vm_machdep.c	2005/01/19 20:22:17
@@ -169,6 +169,10 @@
 	tf->tf_r0 = 0;
 	tf->tf_r1 = 0;
 	pcb2->un_32.pcb32_sp = (u_int)sf;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_cspr = 0;
 }
 				
 void
@@ -303,6 +307,10 @@
 	td->td_pcb->un_32.pcb32_sp = (u_int)sf;
 	td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + td->td_kstack_pages
 	    * PAGE_SIZE + USPACE_UNDEF_STACK_TOP;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_cspr = 0;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/arm/include/proc.h	2004/05/23 16:56:02
+++ //depot/user/rwatson/percpu/sys/arm/include/proc.h	2004/11/25 02:37:15
@@ -46,7 +46,8 @@
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_cspr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/conf/files.alpha	2004/11/24 14:41:38
+++ //depot/user/rwatson/percpu/sys/conf/files.alpha	2004/11/25 02:37:15
@@ -43,7 +43,6 @@
 alpha/alpha/clock.c		standard
 alpha/alpha/clock_if.m		standard
 alpha/alpha/cpuconf.c		standard
-alpha/alpha/critical.c		standard
 alpha/alpha/db_disasm.c		optional	ddb
 alpha/alpha/db_interface.c	optional	ddb
 alpha/alpha/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.amd64	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/conf/files.amd64	2005/01/19 20:22:17
@@ -68,7 +68,6 @@
 amd64/amd64/bios.c		standard
 amd64/amd64/busdma_machdep.c	standard
 amd64/amd64/cpu_switch.S	standard
-amd64/amd64/critical.c		standard
 amd64/amd64/db_disasm.c		optional	ddb
 amd64/amd64/db_interface.c	optional	ddb
 amd64/amd64/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.arm	2004/11/24 14:41:38
+++ //depot/user/rwatson/percpu/sys/conf/files.arm	2004/11/25 02:37:15
@@ -12,7 +12,6 @@
 arm/arm/cpufunc_asm_sa1.S 	standard
 arm/arm/cpufunc_asm_armv4.S 	standard
 arm/arm/cpufunc_asm_sa11x0.S 	standard
-arm/arm/critical.c		standard
 arm/arm/db_disasm.c		optional	ddb
 arm/arm/db_interface.c		optional	ddb
 arm/arm/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.i386	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/conf/files.i386	2005/01/19 20:22:17
@@ -220,7 +220,6 @@
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional	ddb
 i386/i386/db_interface.c	optional	ddb
 i386/i386/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.ia64	2004/11/24 14:41:38
+++ //depot/user/rwatson/percpu/sys/conf/files.ia64	2004/11/25 02:37:15
@@ -90,7 +90,6 @@
 ia64/ia64/clock.c		standard
 ia64/ia64/clock_if.m		standard
 ia64/ia64/context.S		standard
-ia64/ia64/critical.c		standard
 ia64/ia64/db_interface.c	optional	ddb
 ia64/ia64/db_trace.c		optional	ddb
 ia64/ia64/dump_machdep.c	standard
--- //depot/user/rwatson/netperf/sys/conf/files.pc98	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/conf/files.pc98	2005/01/19 20:22:17
@@ -142,7 +142,6 @@
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional	ddb
 i386/i386/db_interface.c	optional	ddb
 i386/i386/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.powerpc	2004/08/16 14:53:15
+++ //depot/user/rwatson/percpu/sys/conf/files.powerpc	2004/11/25 02:37:15
@@ -36,7 +36,6 @@
 powerpc/powerpc/copyinout.c	standard
 powerpc/powerpc/copystr.c	standard
 powerpc/powerpc/cpu.c		standard
-powerpc/powerpc/critical.c	standard
 powerpc/powerpc/elf_machdep.c	standard
 powerpc/powerpc/fpu.c		standard
 powerpc/powerpc/fuswintr.c	standard
--- //depot/user/rwatson/netperf/sys/conf/files.sparc64	2004/11/24 14:41:38
+++ //depot/user/rwatson/percpu/sys/conf/files.sparc64	2004/11/25 02:37:15
@@ -74,7 +74,6 @@
 sparc64/sparc64/cheetah.c	standard
 sparc64/sparc64/clock.c		standard
 sparc64/sparc64/counter.c	standard
-sparc64/sparc64/critical.c	standard
 sparc64/sparc64/db_disasm.c	optional	ddb
 sparc64/sparc64/db_interface.c	optional	ddb
 sparc64/sparc64/db_trace.c	optional	ddb
--- //depot/user/rwatson/netperf/sys/i386/i386/machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/i386/i386/machdep.c	2005/01/19 20:22:17
@@ -2200,6 +2200,32 @@
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_flags = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_flags);
+	}
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
--- //depot/user/rwatson/netperf/sys/i386/i386/vm_machdep.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/i386/i386/vm_machdep.c	2005/01/19 20:22:17
@@ -256,6 +256,10 @@
 	}
 	mtx_unlock_spin(&sched_lock);
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_esp is loaded pointing to the cpu_switch() stack frame
@@ -364,7 +368,7 @@
 
 /*
  * Initialize machine state (pcb and trap frame) for a new thread about to
- * upcall. Pu t enough state in the new thread's PCB to get it to go back 
+ * upcall. Put enough state in the new thread's PCB to get it to go back 
  * userret(), where we can intercept it again to set the return (upcall)
  * Address and stack, along with those from upcals that are from other sources
  * such as those generated in thread_userret() itself.
@@ -416,7 +420,7 @@
 	pcb2->pcb_esp = (int)td->td_frame - sizeof(void *); /* trampoline arg */
 	pcb2->pcb_ebx = (int)td;			    /* trampoline arg */
 	pcb2->pcb_eip = (int)fork_trampoline;
-	pcb2->pcb_psl &= ~(PSL_I);	/* interrupts must be disabled */
+	pcb2->pcb_psl = PSL_KERNEL;		/* ints disabled */
 	pcb2->pcb_gs = rgs();
 	/*
 	 * If we didn't copy the pcb, we'd need to do the following registers:
@@ -427,7 +431,11 @@
 	 * pcb2->pcb_gs:	cloned above.  XXXKSE ???
 	 * pcb2->pcb_ext:	cleared below.
 	 */
-	 pcb2->pcb_ext = NULL;
+	pcb2->pcb_ext = NULL;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/i386/include/proc.h	2004/06/28 19:42:50
+++ //depot/user/rwatson/percpu/sys/i386/include/proc.h	2004/11/25 02:37:15
@@ -47,7 +47,8 @@
  * Machine-dependent part of the proc structure for i386.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/ia64/ia64/machdep.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/ia64/ia64/machdep.c	2005/01/19 20:22:17
@@ -381,6 +381,32 @@
 }
 
 void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_intr = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_intr);
+	}
+}
+
+void
 map_pal_code(void)
 {
 	pt_entry_t pte;
--- //depot/user/rwatson/netperf/sys/ia64/ia64/vm_machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/ia64/ia64/vm_machdep.c	2005/01/19 20:22:17
@@ -158,6 +158,10 @@
 	pcb->pcb_special.sp = (uintptr_t)tf - 16;
 	pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td, (void (*)(void*))fork_return, td);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_intr = 1;
 }
 
 void
@@ -271,6 +275,10 @@
 	td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16;
 	td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_intr = 1;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/ia64/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/ia64/include/proc.h	2004/11/25 02:37:15
@@ -30,7 +30,8 @@
 #define	_MACHINE_PROC_H_
 
 struct mdthread {
-	register_t	md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_intr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/kern/kern_fork.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/kern/kern_fork.c	2005/01/19 20:22:17
@@ -72,7 +72,6 @@
 #include <vm/vm_extern.h>
 #include <vm/uma.h>
 
-#include <machine/critical.h>
 
 #ifndef _SYS_SYSPROTO_H_
 struct fork_args {
@@ -764,7 +763,6 @@
 
 	sched_lock.mtx_lock = (uintptr_t)td;
 	mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED);
-	cpu_critical_fork_exit();
 	CTR4(KTR_PROC, "fork_exit: new thread %p (kse %p, pid %d, %s)",
 		td, td->td_sched, p->p_pid, p->p_comm);
 
--- //depot/user/rwatson/netperf/sys/kern/kern_mutex.c	2004/10/19 18:01:38
+++ //depot/user/rwatson/percpu/sys/kern/kern_mutex.c	2004/11/25 02:37:15
@@ -377,7 +377,7 @@
 #if defined(SMP) || LOCK_DEBUG > 0 || 1
 	_get_spin_lock(m, curthread, opts, file, line);
 #else
-	critical_enter();
+	spinlock_enter();
 #endif
 	LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file,
 	    line);
@@ -399,7 +399,7 @@
 #if defined(SMP) || LOCK_DEBUG > 0 || 1
 	_rel_spin_lock(m);
 #else
-	critical_exit();
+	spinlock_exit();
 #endif
 }
 
@@ -593,7 +593,7 @@
 			break;
 
 		/* Give interrupts a chance while we spin. */
-		critical_exit();
+		spinlock_exit();
 		while (m->mtx_lock != MTX_UNOWNED) {
 			if (i++ < 10000000) {
 				cpu_spinwait();
@@ -612,7 +612,7 @@
 			}
 			cpu_spinwait();
 		}
-		critical_enter();
+		spinlock_enter();
 	}
 
 	if (LOCK_LOG_TEST(&m->mtx_object, opts))
--- //depot/user/rwatson/netperf/sys/kern/kern_proc.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/kern/kern_proc.c	2005/01/19 20:22:17
@@ -63,7 +63,6 @@
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/uma.h>
-#include <machine/critical.h>
 
 MALLOC_DEFINE(M_PGRP, "pgrp", "process group header");
 MALLOC_DEFINE(M_SESSION, "session", "session header");
--- //depot/user/rwatson/netperf/sys/kern/kern_switch.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/kern/kern_switch.c	2005/01/19 20:22:17
@@ -105,7 +105,6 @@
 #if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 #include <sys/smp.h>
 #endif
-#include <machine/critical.h>
 #if defined(SMP) && defined(SCHED_4BSD)
 #include <sys/sysctl.h>
 #endif
@@ -568,8 +567,6 @@
 	struct thread *td;
 
 	td = curthread;
-	if (td->td_critnest == 0)
-		cpu_critical_enter(td);
 	td->td_critnest++;
 	CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td,
 	    (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest);
@@ -597,7 +594,6 @@
 		}
 #endif
 		td->td_critnest = 0;
-		cpu_critical_exit(td);
 	} else {
 		td->td_critnest--;
 	}
--- //depot/user/rwatson/netperf/sys/pc98/i386/machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/pc98/i386/machdep.c	2005/01/19 20:22:17
@@ -2257,6 +2257,32 @@
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_flags = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_flags);
+	}
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
--- //depot/user/rwatson/netperf/sys/powerpc/include/cpufunc.h	2004/08/07 02:40:42
+++ //depot/user/rwatson/percpu/sys/powerpc/include/cpufunc.h	2004/11/25 02:37:15
@@ -149,8 +149,11 @@
 static __inline void
 intr_restore(register_t msr)
 {
+	register_t	value;
 
-	mtmsr(msr);
+	value = mfmsr() & ~(PSL_EE | PSL_RI);
+	value |= msr & (PSL_EE | PSL_RI);
+	mtmsr(value);
 }
 
 static __inline void
--- //depot/user/rwatson/netperf/sys/powerpc/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/powerpc/include/proc.h	2004/11/25 02:37:15
@@ -39,7 +39,8 @@
  * Machine-dependent part of the proc structure
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_msr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/powerpc/powerpc/machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/machdep.c	2005/01/19 20:22:17
@@ -875,6 +875,32 @@
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_msr = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_msr);
+	}
+}
+
 /*
  * kcopy(const void *src, void *dst, size_t len);
  *
--- //depot/user/rwatson/netperf/sys/powerpc/powerpc/vm_machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/vm_machdep.c	2005/01/19 20:22:17
@@ -154,6 +154,10 @@
 	pcb->pcb_lr = (register_t)fork_trampoline;
 	pcb->pcb_usr = kernel_pmap->pm_sr[USER_SR];
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_msr = PSL_EE | PSL_RI;
+
 	/*
  	 * Now cpu_switch() can schedule the new process.
 	 */
@@ -329,6 +333,10 @@
 	pcb2->pcb_sp = (register_t)cf;
 	pcb2->pcb_lr = (register_t)fork_trampoline;
 	pcb2->pcb_usr = kernel_pmap->pm_sr[USER_SR];
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_msr = PSL_EE | PSL_RI;
 }
 
 void
--- //depot/user/rwatson/netperf/sys/sparc64/include/proc.h	2004/04/08 03:11:34
+++ //depot/user/rwatson/percpu/sys/sparc64/include/proc.h	2004/11/25 02:37:15
@@ -42,7 +42,8 @@
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_pil;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/sparc64/sparc64/machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/machdep.c	2005/01/19 20:22:17
@@ -230,6 +230,33 @@
 	}
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_pil = rdpr(pil);
+		wrpr(pil, 0, 14);
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		wrpr(pil, td->td_md.md_saved_pil, 0);
+	}
+}
+
 unsigned
 tick_get_timecount(struct timecounter *tc)
 {
--- //depot/user/rwatson/netperf/sys/sparc64/sparc64/vm_machdep.c	2004/11/27 21:43:45
+++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/vm_machdep.c	2005/01/19 20:22:17
@@ -173,6 +173,10 @@
 	fr->fr_local[2] = (u_long)tf;
 	pcb->pcb_pc = (u_long)fork_trampoline - 8;
 	pcb->pcb_sp = (u_long)fr - SPOFF;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_pil = 0;
 }
 
 void
@@ -287,6 +291,10 @@
 	pcb2->pcb_sp = (u_long)fp - SPOFF;
 	pcb2->pcb_pc = (u_long)fork_trampoline - 8;
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_pil = 0;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 */
--- //depot/user/rwatson/netperf/sys/sys/lock.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/sys/lock.h	2004/11/25 02:37:15
@@ -196,6 +196,8 @@
 extern struct lock_class lock_class_mtx_spin;
 extern struct lock_class lock_class_sx;
 
+void	spinlock_enter(void);
+void	spinlock_exit(void);
 void	witness_init(struct lock_object *);
 void	witness_destroy(struct lock_object *);
 int	witness_defineorder(struct lock_object *, struct lock_object *);
--- //depot/user/rwatson/netperf/sys/sys/mutex.h	2004/08/05 21:29:14
+++ //depot/user/rwatson/percpu/sys/sys/mutex.h	2004/11/25 02:37:15
@@ -164,7 +164,7 @@
 #define _get_spin_lock(mp, tid, opts, file, line) do {			\
 	struct thread *_tid = (tid);					\
 									\
-	critical_enter();						\
+	spinlock_enter();						\
 	if (!_obtain_lock((mp), _tid)) {				\
 		if ((mp)->mtx_lock == (uintptr_t)_tid)			\
 			(mp)->mtx_recurse++;				\
@@ -191,8 +191,8 @@
  * Since spin locks are not _too_ common, inlining this code is not too big 
  * a deal.
  *
- * Since we always perform a critical_enter() when attempting to acquire a
- * spin lock, we need to always perform a matching critical_exit() when
+ * Since we always perform a spinlock_enter() when attempting to acquire a
+ * spin lock, we need to always perform a matching spinlock_exit() when
  * releasing a spin lock.  This includes the recursion cases.
  */
 #ifndef _rel_spin_lock
@@ -201,7 +201,7 @@
 		(mp)->mtx_recurse--;					\
 	else								\
 		_release_lock_quick((mp));				\
-	critical_exit();						\
+	spinlock_exit();						\
 } while (0)
 #endif
 
@@ -289,8 +289,8 @@
 #define	mtx_unlock_spin_flags(m, opts)					\
 	_rel_spin_lock((m))
 #else	/* SMP */
-#define	mtx_lock_spin_flags(m, opts)	critical_enter()
-#define	mtx_unlock_spin_flags(m, opts)	critical_exit()
+#define	mtx_lock_spin_flags(m, opts)	spinlock_enter()
+#define	mtx_unlock_spin_flags(m, opts)	spinlock_exit()
 #endif	/* SMP */
 #endif	/* LOCK_DEBUG > 0 || MUTEX_NOINLINE */
 
--- //depot/user/rwatson/netperf/sys/vm/uma_core.c	2005/01/01 20:04:14
+++ //depot/user/rwatson/percpu/sys/vm/uma_core.c	2005/01/19 20:22:17
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2004, Robert N. M. Watson
  * Copyright (c) 2004, 2005,
  *     Bosko Milekic <bmilekic@freebsd.org>
  * Copyright (c) 2002, 2003, 2004, 2005,
@@ -384,48 +385,19 @@
 zone_timeout(uma_zone_t zone)
 {
 	uma_keg_t keg;
-	uma_cache_t cache;
 	u_int64_t alloc;
-	int cpu;
 
 	keg = zone->uz_keg;
 	alloc = 0;
 
 	/*
-	 * Aggregate per cpu cache statistics back to the zone.
-	 *
-	 * XXX This should be done in the sysctl handler.
-	 *
-	 * I may rewrite this to set a flag in the per cpu cache instead of
-	 * locking.  If the flag is not cleared on the next round I will have
-	 * to lock and do it here instead so that the statistics don't get too
-	 * far out of sync.
-	 */
-	if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL)) {
-		for (cpu = 0; cpu <= mp_maxid; cpu++) {
-			if (CPU_ABSENT(cpu))
-				continue;
-			CPU_LOCK(cpu);
-			cache = &zone->uz_cpu[cpu];
-			/* Add them up, and reset */
-			alloc += cache->uc_allocs;
-			cache->uc_allocs = 0;
-			CPU_UNLOCK(cpu);
-		}
-	}
-
-	/* Now push these stats back into the zone.. */
-	ZONE_LOCK(zone);
-	zone->uz_allocs += alloc;
-
-	/*
 	 * Expand the zone hash table.
 	 *
 	 * This is done if the number of slabs is larger than the hash size.
 	 * What I'm trying to do here is completely reduce collisions.  This
 	 * may be a little aggressive.  Should I allow for two collisions max?
 	 */
-
+	ZONE_LOCK(zone);
 	if (keg->uk_flags & UMA_ZONE_HASH &&
 	    keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
 		struct uma_hash newhash;
@@ -613,6 +585,10 @@
 /*
  * Drains the per cpu caches for a zone.
  *
+ * NOTE: This may only be called while the zone is being turn down, and not
+ * during normal operation.  This is necessary in order that we do not have
+ * to migrate CPUs to drain the per-CPU caches.
+ *
  * Arguments:
  *	zone     The zone to drain, must be unlocked.
  *
@@ -626,12 +602,20 @@
 	int cpu;
 
 	/*
-	 * We have to lock each cpu cache before locking the zone
+	 * XXX: It is safe to not lock the per-CPU caches, because we're
+	 * tearing down the zone anyway.  I.e., there will be no further use
+	 * of the caches at this point.
+	 *
+	 * XXX: It would good to be able to assert that the zone is being
+	 * torn down to prevent improper use of cache_drain().
+	 *
+	 * XXX: We lock the zone before passing into bucket_cache_drain() as
+	 * it is used elsewhere.  Should the tear-down path be made special
+	 * there in some form?
 	 */
 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
 		if (CPU_ABSENT(cpu))
 			continue;
-		CPU_LOCK(cpu);
 		cache = &zone->uz_cpu[cpu];
 		bucket_drain(zone, cache->uc_allocbucket);
 		bucket_drain(zone, cache->uc_freebucket);
@@ -644,11 +628,6 @@
 	ZONE_LOCK(zone);
 	bucket_cache_drain(zone);
 	ZONE_UNLOCK(zone);
-	for (cpu = 0; cpu <= mp_maxid; cpu++) {
-		if (CPU_ABSENT(cpu))
-			continue;
-		CPU_UNLOCK(cpu);
-	}
 }
 
 /*
@@ -1786,6 +1765,9 @@
 	uma_cache_t cache;
 	uma_bucket_t bucket;
 	int cpu;
+#ifdef INVARIANTS
+	int count;
+#endif
 	int badness;
 
 	/* This is the fast path allocation */
@@ -1820,12 +1802,33 @@
 		}
 	}
 
+	/*
+	 * If possible, allocate from the per-CPU cache.  There are two
+	 * requirements for safe access to the per-CPU cache: (1) the thread
+	 * accessing the cache must not be preempted or yield during access,
+	 * and (2) the thread must not migrate CPUs without switching which
+	 * cache it accesses.  We rely on a critical section to prevent
+	 * preemption and migration.  We release the critical section in
+	 * order to acquire the zone mutex if we are unable to free to the
+	 * current cache; when we re-acquire the critical section, we must
+	 * detect and handle migration if it has occurred.
+	 */
+#ifdef INVARIANTS
+	count = 0;
+#endif
 zalloc_restart:
+	critical_enter();
 	cpu = PCPU_GET(cpuid);
-	CPU_LOCK(cpu);
 	cache = &zone->uz_cpu[cpu];
 
 zalloc_start:
+#ifdef INVARIANTS
+	count++;
+	KASSERT(count < 10, ("uma_zalloc_arg: count == 10"));
+#endif
+#if 0
+	critical_assert();
+#endif
 	bucket = cache->uc_allocbucket;
 
 	if (bucket) {
@@ -1838,12 +1841,12 @@
 			KASSERT(item != NULL,
 			    ("uma_zalloc: Bucket pointer mangled."));
 			cache->uc_allocs++;
+			critical_exit();
 #ifdef INVARIANTS
 			ZONE_LOCK(zone);
 			uma_dbg_alloc(zone, NULL, item);
 			ZONE_UNLOCK(zone);
 #endif
-			CPU_UNLOCK(cpu);
 			if (zone->uz_ctor != NULL) {
 				if (zone->uz_ctor(item, zone->uz_keg->uk_size,
 				    udata, flags) != 0) {
@@ -1873,7 +1876,33 @@
 			}
 		}
 	}
+	/*
+	 * Attempt to retrieve the item from the per-CPU cache has failed, so
+	 * we must go back to the zone.  This requires the zone lock, so we
+	 * must drop the critical section, then re-acquire it when we go back
+	 * to the cache.  Since the critical section is released, we may be
+	 * preempted or migrate.  As such, make sure not to maintain any
+	 * thread-local state specific to the cache from prior to releasing
+	 * the critical section.
+	 */
+	critical_exit();
 	ZONE_LOCK(zone);
+	critical_enter();
+	cpu = PCPU_GET(cpuid);
+	cache = &zone->uz_cpu[cpu];
+	bucket = cache->uc_allocbucket;
+	if (bucket != NULL) {
+		if (bucket != NULL && bucket->ub_cnt > 0) {
+			ZONE_UNLOCK(zone);
+			goto zalloc_start;
+		}
+		bucket = cache->uc_freebucket;
+		if (bucket != NULL && bucket->ub_cnt > 0) {
+			ZONE_UNLOCK(zone);
+			goto zalloc_start;
+		}
+	}
+
 	/* Since we have locked the zone we may as well send back our stats */
 	zone->uz_allocs += cache->uc_allocs;
 	cache->uc_allocs = 0;
@@ -1897,8 +1926,8 @@
 		ZONE_UNLOCK(zone);
 		goto zalloc_start;
 	}
-	/* We are no longer associated with this cpu!!! */
-	CPU_UNLOCK(cpu);
+	/* We are no longer associated with this CPU. */
+	critical_exit();
 
 	/* Bump up our uz_count so we get here less */
 	if (zone->uz_count < BUCKET_MAX)
@@ -2211,6 +2240,9 @@
 	uma_bucket_t bucket;
 	int bflags;
 	int cpu;
+#ifdef INVARIANTS
+	int count;
+#endif
 	enum zfreeskip skip;
 
 	/* This is the fast path free */
@@ -2236,12 +2268,33 @@
 		skip = SKIP_DTOR;
 	}
 
+#ifdef INVARIANTS
+	count = 0;
+#endif
+	/*
+	 * If possible, free to the per-CPU cache.  There are two
+	 * requirements for safe access to the per-CPU cache: (1) the thread
+	 * accessing the cache must not be preempted or yield during access,
+	 * and (2) the thread must not migrate CPUs without switching which
+	 * cache it accesses.  We rely on a critical section to prevent
+	 * preemption and migration.  We release the critical section in
+	 * order to acquire the zone mutex if we are unable to free to the
+	 * current cache; when we re-acquire the critical section, we must
+	 * detect and handle migration if it has occurred.
+	 */
 zfree_restart:
+	critical_enter();
 	cpu = PCPU_GET(cpuid);
-	CPU_LOCK(cpu);
 	cache = &zone->uz_cpu[cpu];
 
 zfree_start:
+#ifdef INVARIANTS
+	count++;
+	KASSERT(count < 10, ("uma_zfree_arg: count == 10"));
+#endif
+#if 0
+	critical_assert();
+#endif
 	bucket = cache->uc_freebucket;
 
 	if (bucket) {
@@ -2255,6 +2308,7 @@
 			    ("uma_zfree: Freeing to non free bucket index."));
 			bucket->ub_bucket[bucket->ub_cnt] = item;
 			bucket->ub_cnt++;
+			critical_exit();
 #ifdef INVARIANTS
 			ZONE_LOCK(zone);
 			if (keg->uk_flags & UMA_ZONE_MALLOC)
@@ -2263,7 +2317,6 @@
 				uma_dbg_free(zone, NULL, item);
 			ZONE_UNLOCK(zone);
 #endif
-			CPU_UNLOCK(cpu);
 			return;
 		} else if (cache->uc_allocbucket) {
 #ifdef UMA_DEBUG_ALLOC
@@ -2287,9 +2340,32 @@
 	 *
 	 * 1) The buckets are NULL
 	 * 2) The alloc and free buckets are both somewhat full.
+	 *
+	 * We must go back the zone, hich requires acquiring the zone lock,
+	 * which in turn means we must release and re-acquire the critical
+	 * section.  Since the critical section is released, we may be
+	 * preempted or migrate.  As such, make sure not to maintain any
+	 * thread-local state specific to the cache from prior to releasing
+	 * the critical section.
 	 */
-
+	critical_exit();
 	ZONE_LOCK(zone);
+	critical_enter();
+	cpu = PCPU_GET(cpuid);
+	cache = &zone->uz_cpu[cpu];
+	if (cache->uc_freebucket != NULL) {
+		if (cache->uc_freebucket->ub_cnt <
+		    cache->uc_freebucket->ub_entries) {
+			ZONE_UNLOCK(zone);
+			goto zfree_start;
+		}
+		if (cache->uc_allocbucket != NULL &&
+		    (cache->uc_allocbucket->ub_cnt <
+		    cache->uc_freebucket->ub_cnt)) {
+			ZONE_UNLOCK(zone);
+			goto zfree_start;
+		}
+	}
 
 	bucket = cache->uc_freebucket;
 	cache->uc_freebucket = NULL;
@@ -2311,8 +2387,8 @@
 		cache->uc_freebucket = bucket;
 		goto zfree_start;
 	}
-	/* We're done with this CPU now */
-	CPU_UNLOCK(cpu);
+	/* We are nolonger associated with this CPU. */
+	critical_exit();
 
 	/* And the zone.. */
 	ZONE_UNLOCK(zone);
@@ -2726,6 +2802,7 @@
 	int cachefree;
 	uma_bucket_t bucket;
 	uma_cache_t cache;
+	u_int64_t alloc;
 
 	cnt = 0;
 	mtx_lock(&uma_mtx);
@@ -2749,6 +2826,13 @@
 	  LIST_FOREACH(z, &zk->uk_zones, uz_link) {
 		if (cnt == 0)	/* list may have changed size */
 			break;
+		/*
+		 * XXXRW: This is the last remaining use of a per-CPU mutex
+		 * to protect access to a per-CPU zone cache.  We should be
+		 * migrating CPUs here to pick up data from each cache
+		 * instead.  Because this locking no longer protects other
+		 * use of the cache, we can now get inconsistent results.
+		 */
 		if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) {
 			for (cpu = 0; cpu <= mp_maxid; cpu++) {
 				if (CPU_ABSENT(cpu))
@@ -2758,6 +2842,7 @@
 		}
 		ZONE_LOCK(z);
 		cachefree = 0;
+		alloc = 0;
 		if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) {
 			for (cpu = 0; cpu <= mp_maxid; cpu++) {
 				if (CPU_ABSENT(cpu))
@@ -2767,9 +2852,15 @@
 					cachefree += cache->uc_allocbucket->ub_cnt;
 				if (cache->uc_freebucket != NULL)
 					cachefree += cache->uc_freebucket->ub_cnt;
+				alloc += cache->uc_allocs;
+				cache->uc_allocs = 0;
 				CPU_UNLOCK(cpu);
 			}
 		}
+
+		/* Now push alloc stats back into zone.  */
+		z->uz_allocs += alloc;
+
 		LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) {
 			cachefree += bucket->ub_cnt;
 		}
