Index: kern/subr_uio.c =================================================================== --- kern/subr_uio.c (revision 227544) +++ kern/subr_uio.c (working copy) @@ -85,6 +85,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o vm_map_entry_t entry; vm_pindex_t upindex; vm_prot_t prot; + vm_page_bits_t vbits; boolean_t wired; KASSERT((uaddr & PAGE_MASK) == 0, @@ -95,6 +96,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o * unwired in sf_buf_mext(). */ kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr)); + vbits = kern_pg->valid; kern_pg->valid = VM_PAGE_BITS_ALL; KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1, ("vm_pgmoveco: kern_pg is not correctly wired")); @@ -105,6 +107,13 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o return(EFAULT); } VM_OBJECT_LOCK(uobject); + if (vm_page_insert(kern_pg, uobject, upindex) != 0) { + kern_pg->valid = vbits; + VM_OBJECT_UNLOCK(uobject); + vm_map_lookup_done(map, entry); + return(ENOMEM); + } + vm_page_dirty(kern_pg); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) @@ -122,8 +131,6 @@ retry: if (uobject->backing_object != NULL) pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE); } - vm_page_insert(kern_pg, uobject, upindex); - vm_page_dirty(kern_pg); VM_OBJECT_UNLOCK(uobject); vm_map_lookup_done(map, entry); return(KERN_SUCCESS); Index: vm/sg_pager.c =================================================================== --- vm/sg_pager.c (revision 227544) +++ vm/sg_pager.c (working copy) @@ -179,6 +179,10 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m /* Construct a new fake page. */ page = vm_page_getfake(paddr, memattr); VM_OBJECT_LOCK(object); + if (vm_page_insert(page, object, offset) != 0) { + vm_page_putfake(page); + return (VM_PAGER_FAIL); + } TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq); /* Free the original pages and insert this fake page into the object. */ @@ -187,7 +191,6 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m vm_page_free(m[i]); vm_page_unlock(m[i]); } - vm_page_insert(page, object, offset); m[reqpage] = page; page->valid = VM_PAGE_BITS_ALL; Index: vm/vm_object.c =================================================================== --- vm/vm_object.c (revision 227544) +++ vm/vm_object.c (working copy) @@ -1357,9 +1357,7 @@ retry: VM_OBJECT_LOCK(new_object); goto retry; } - vm_page_lock(m); vm_page_rename(m, new_object, idx); - vm_page_unlock(m); /* * page automatically made dirty by rename and * cache handled @@ -1614,9 +1612,7 @@ restart: * If the page was mapped to a process, it can remain * mapped through the rename. */ - vm_page_lock(p); vm_page_rename(p, object, new_pindex); - vm_page_unlock(p); /* page automatically made dirty by rename */ } } Index: vm/vm_fault.c =================================================================== --- vm/vm_fault.c (revision 227544) +++ vm/vm_fault.c (working copy) @@ -769,9 +769,7 @@ vnode_locked: * process'es object. The page is * automatically made dirty. */ - vm_page_lock(fs.m); vm_page_rename(fs.m, fs.first_object, fs.first_pindex); - vm_page_unlock(fs.m); vm_page_busy(fs.m); fs.first_m = fs.m; fs.m = NULL; Index: vm/device_pager.c =================================================================== --- vm/device_pager.c (revision 227544) +++ vm/device_pager.c (working copy) @@ -307,11 +307,14 @@ old_dev_pager_fault(vm_object_t object, vm_ooffset */ page = vm_page_getfake(paddr, memattr); VM_OBJECT_LOCK(object); + if (vm_page_insert(page, object, offset) != 0) { + vm_page_putfake(page); + return (VM_PAGER_FAIL); + } vm_page_lock(*mres); vm_page_free(*mres); vm_page_unlock(*mres); *mres = page; - vm_page_insert(page, object, pidx); } page->valid = VM_PAGE_BITS_ALL; return (VM_PAGER_OK); Index: vm/vm_page.c =================================================================== --- vm/vm_page.c (revision 227544) +++ vm/vm_page.c (working copy) @@ -109,6 +109,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include /* @@ -776,13 +777,15 @@ vm_page_dirty(vm_page_t m) * The object and page must be locked. * This routine may not block. */ -void +int vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) { vm_page_t neighbor; + vm_pindex_t cpindex; VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); if (m->object != NULL) panic("vm_page_insert: page already inserted"); + cpindex = m->pindex; /* * Record the object/offset pair in this page @@ -802,9 +805,14 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm } else TAILQ_INSERT_TAIL(&object->memq, m, listq); } - if (vm_radix_insert(&object->rtree, pindex, m) != 0) - panic("vm_page_insert: unable to insert the new page"); + if (vm_radix_insert(&object->rtree, pindex, m) != 0) { + TAILQ_REMOVE(&object->memq, m, listq); + m->object = NULL; + m->pindex = cpindex; + return (ENOMEM); + } + /* * show that the object has one more resident page. */ @@ -821,6 +829,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm */ if (m->aflags & PGA_WRITEABLE) vm_object_set_writeable_dirty(object); + return (0); } /* @@ -965,9 +974,20 @@ vm_page_prev(vm_page_t m) void vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) { + u_int i; + MPASS(m->object != NULL); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED); + + vm_page_lock(m); vm_page_remove(m); - vm_page_insert(m, new_object, new_pindex); + vm_page_unlock(m); + while (vm_page_insert(m, new_object, new_pindex) != 0) { + pagedaemon_wakeup(); + for (i = 0; i < 10000000; i++) + cpu_spinwait(); + } vm_page_dirty(m); } @@ -1104,7 +1124,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind struct vnode *vp = NULL; vm_object_t m_object; vm_page_t m; - int flags, req_class; + int cachedp, flags, req_class; KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0), ("vm_page_alloc: inconsistent object/req")); @@ -1112,6 +1132,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); req_class = req & VM_ALLOC_CLASS_MASK; + cachedp = 0; /* * The page daemon is allowed to dig deeper into the free page list. @@ -1194,6 +1215,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind ("vm_page_alloc: cached page %p is PG_ZERO", m)); KASSERT(m->valid != 0, ("vm_page_alloc: cached page %p is invalid", m)); + cachedp++; if (m->object == object && m->pindex == pindex) cnt.v_reactivated++; else @@ -1245,7 +1267,22 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind if (object->memattr != VM_MEMATTR_DEFAULT && object->type != OBJT_DEVICE && object->type != OBJT_SG) pmap_page_set_memattr(m, object->memattr); - vm_page_insert(m, object, pindex); + if (vm_page_insert(m, object, pindex) != 0) { + + /* See the comment below about hold count handling. */ + if (vp != NULL) + vdrop(vp); + vm_page_lock(m); + if (req & VM_ALLOC_WIRED) + vm_page_unwire(m, 0); + if (cachedp) + vm_page_cache(m); + else + vm_page_free(m); + vm_page_unlock(m); + pagedaemon_wakeup(); + return (NULL); + } } else m->pindex = pindex; @@ -2453,11 +2490,8 @@ vm_page_cowfault(vm_page_t m) pindex = m->pindex; retry_alloc: - pmap_remove_all(m); - vm_page_remove(m); mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); if (mnew == NULL) { - vm_page_insert(m, object, pindex); vm_page_unlock(m); VM_OBJECT_UNLOCK(object); VM_WAIT; @@ -2483,8 +2517,9 @@ vm_page_cowfault(vm_page_t m) vm_page_lock(mnew); vm_page_free(mnew); vm_page_unlock(mnew); - vm_page_insert(m, object, pindex); } else { /* clear COW & copy page */ + pmap_remove_all(m); + vm_page_remove(m); if (!so_zerocp_fullpage) pmap_copy_page(m, mnew); mnew->valid = VM_PAGE_BITS_ALL; Index: vm/vm_page.h =================================================================== --- vm/vm_page.h (revision 227544) +++ vm/vm_page.h (working copy) @@ -370,7 +370,7 @@ void vm_page_dontneed(vm_page_t); void vm_page_deactivate (vm_page_t); vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t); vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr); -void vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t); +int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t); vm_page_t vm_page_lookup (vm_object_t, vm_pindex_t); vm_page_t vm_page_next(vm_page_t m); int vm_page_pa_tryrelock(pmap_t, vm_paddr_t, vm_paddr_t *); Index: vm/vm_contig.c =================================================================== --- vm/vm_contig.c (revision 227544) +++ vm/vm_contig.c (working copy) @@ -277,9 +277,22 @@ retry: vm_map_unlock(map); return (0); } + if (vm_page_insert(m, object, OFF_TO_IDX(offset + i)) != 0) { + do { + vm_page_free(m); + if (i == 0) + break; + i -= PAGE_SIZE; + m = vm_page_lookup(object, + OFF_TO_IDX(offset + i)); + } while (1); + VM_OBJECT_UNLOCK(object); + vm_map_delete(map, addr, addr + size); + vm_map_unlock(map); + return (0); + } if (memattr != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, memattr); - vm_page_insert(m, object, OFF_TO_IDX(offset + i)); if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL; @@ -305,6 +318,7 @@ contigmapping(vm_map_t map, vm_size_t size, vm_pag { vm_object_t object = kernel_object; vm_offset_t addr, tmp_addr; + vm_page_t tmp_m; vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { @@ -314,13 +328,29 @@ contigmapping(vm_map_t map, vm_size_t size, vm_pag vm_object_reference(object); vm_map_insert(map, object, addr - VM_MIN_KERNEL_ADDRESS, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0); - vm_map_unlock(map); VM_OBJECT_LOCK(object); + + /* Ensure first all the pages can be safely inserted. */ + tmp_m = m; for (tmp_addr = addr; tmp_addr < addr + size; tmp_addr += PAGE_SIZE) { + if (vm_page_insert(tmp_m, object, + OFF_TO_IDX(tmp_addr - VM_MIN_KERNEL_ADDRESS)) != 0) { + MPASS(tmp_m >= m); + while (tmp_m != m) { + tmp_m--; + vm_page_remove(tmp_m); + } + VM_OBJECT_UNLOCK(object); + vm_map_delete(map, addr, addr + size); + vm_map_unlock(map); + return (0); + } + tmp_m++; + } + vm_map_unlock(map); + for (tmp_addr = addr; tmp_addr < addr + size; tmp_addr += PAGE_SIZE) { if (memattr != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, memattr); - vm_page_insert(m, object, - OFF_TO_IDX(tmp_addr - VM_MIN_KERNEL_ADDRESS)); if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL;