Fix compilation error with slab allocator and no debugging.

[srvx.git] / src / alloc-slab.c

diff --git a/src/alloc-slab.c b/src/alloc-slab.c
index 27d38b717582a88ab302f276d7d3b5bee9e1de5b..2b923193ca8ffaa0ad27f0c21ba30e4398003dea 100644 (file)
--- a/src/alloc-slab.c
+++ b/src/alloc-slab.c
@@ -25,18 +25,19 @@
 # error The slab allocator requires that your system have the mmap() system call.
 #endif
 
-#define SLAB_DEBUG 0
+#define SLAB_DEBUG 1
+#define SLAB_RESERVE 1024
 
 #if SLAB_DEBUG
 
-#define ALLOC_MAGIC 0x1acf
-#define FREE_MAGIC  0xfc1d
+#define ALLOC_MAGIC 0x1a
+#define FREE_MAGIC  0xcf
 
 struct alloc_header {
-    unsigned int file_id : 8;
     unsigned int size : 24;
+    unsigned int magic : 8;
+    unsigned int file_id : 8;
     unsigned int line : 16;
-    unsigned int magic : 16;
 };
 
 static const char *file_ids[256];
@@ -96,7 +97,7 @@ struct slabset {
 #define SLAB_MIN     (2 * sizeof(void*))
 #define SLAB_GRAIN   sizeof(void*)
 #define SLAB_ALIGN   SLAB_GRAIN
-#define SMALL_CUTOFF 582
+#define SMALL_CUTOFF 576
 /* Element size < SMALL_CUTOFF -> use small slabs.
  * Larger elements are allocated directly using mmap().  The largest
  * regularly allocated struct in srvx 1.x is smaller than
@@ -105,7 +106,9 @@ struct slabset {
  */
 
 static struct slabset little_slabs[SMALL_CUTOFF / SLAB_GRAIN];
-static struct slab *free_slabs;
+static struct slab *free_slab_head;
+static struct slab *free_slab_tail;
+unsigned long free_slab_count;
 unsigned long big_alloc_count;
 unsigned long big_alloc_size;
 unsigned long slab_count;
@@ -191,13 +194,15 @@ slab_alloc(struct slabset *sset)
         unsigned int ii, step;
 
         /* Allocate new slab. */
-        if (free_slabs) {
-            slab = free_slabs;
-            free_slabs = slab->next;
+        if (free_slab_head) {
+            slab = free_slab_head;
+            if (!(free_slab_head = slab->next))
+                free_slab_tail = NULL;
         } else {
             item = slab_map(slab_pagesize());
             slab = (struct slab*)((char*)item + slab_pagesize() - sizeof(*slab));
             slab->base = item;
+            slab_count++;
         }
 
         /* Populate free list. */
@@ -221,8 +226,6 @@ slab_alloc(struct slabset *sset)
         assert(!slab->prev || slab == slab->prev->next);
         sset->child = slab;
         sset->nslabs++;
-        slab_count++;
-        /* log_module(MAIN_LOG, LOG_DEBUG, "Allocated new %u-slab %p.", sset->size, slab); */
     }
 
     slab = sset->child;
@@ -231,7 +234,6 @@ slab_alloc(struct slabset *sset)
            <= (slab_pagesize() - sizeof(*slab) - sset->size));
     slab->free = *item;
     if (++slab->used == sset->items_per_slab) {
-        /* log_module(MAIN_LOG, LOG_DEBUG, "%u-slab %p is empty.", sset->size, item); */
         if (sset->child != slab) {
             /* Unlink slab and reinsert before sset->child. */
             if (slab->prev)
@@ -257,15 +259,12 @@ slab_alloc(struct slabset *sset)
 static void
 slab_unalloc(void *ptr, size_t size)
 {
-    void **item;
     struct slab *slab, *new_next;
 
-    item = ptr;
     assert(size < SMALL_CUTOFF);
     slab = (struct slab*)((((unsigned long)ptr | (slab_pagesize() - 1)) + 1) - sizeof(*slab));
-    *item = slab->free;
-    memset(item + 1, 0xde, size - sizeof(*item));
-    slab->free = item;
+    *(void**)ptr = slab->free;
+    slab->free = ptr;
     slab->parent->nallocs--;
 
     if (slab->used-- == slab->parent->items_per_slab
@@ -284,12 +283,9 @@ slab_unalloc(void *ptr, size_t size)
         slab->parent->child = slab;
         assert(!slab->next || slab == slab->next->prev);
         assert(!slab->prev || slab == slab->prev->next);
-        /* log_module(MAIN_LOG, LOG_DEBUG, "%u-slab %p became partial.", slab->parent->size, slab); */
     } else if (!slab->used) {
-        /* log_module(MAIN_LOG, LOG_DEBUG, "%u-slab %p became full.", slab->parent->size, slab); */
         /* Unlink slab from its parent. */
         slab->parent->nslabs--;
-        slab_count--;
         if (slab->prev)
             slab->prev->next = slab->next;
         if (slab->next)
@@ -302,11 +298,38 @@ slab_unalloc(void *ptr, size_t size)
             assert(!new_next->prev || new_next == new_next->prev->next);
         }
 
+#if SLAB_RESERVE
+        if (!free_slab_count) {
+            /* Make sure we have enough free slab pages. */
+            while (free_slab_count < SLAB_RESERVE) {
+                struct slab *tslab;
+                void *item;
+
+                item = slab_map(slab_pagesize());
+                tslab = (struct slab*)((char*)item + slab_pagesize() - sizeof(*slab));
+                tslab->base = item;
+                tslab->prev = free_slab_tail;
+                free_slab_tail = tslab;
+                if (!free_slab_head)
+                    free_slab_head = tslab;
+                free_slab_count++;
+                slab_count++;
+            }
+        }
+
+        /* Unmap old slab, so accesses to stale pointers will fault. */
+        munmap(slab->base, slab_pagesize());
+        slab_count--;
+#else
         /* Link to list of free slabs. */
-        slab->prev = NULL;
         slab->parent = NULL;
-        slab->next = free_slabs;
-        free_slabs = slab;
+        slab->prev = free_slab_tail;
+        slab->next = NULL;
+        free_slab_tail = slab;
+        if (!free_slab_head)
+            free_slab_head = slab;
+        free_slab_count++;
+#endif
     }
 }
 
@@ -360,6 +383,7 @@ slab_realloc(const char *file, unsigned int line, void *ptr, size_t size)
         return ptr;
     newblock = slab_malloc(file, line, size);
     memcpy(newblock, ptr, osize);
+    slab_free(file, line, ptr);
     return newblock;
 }
 
@@ -376,33 +400,39 @@ slab_strdup(const char *file, unsigned int line, const char *src)
 }
 
 void
-slab_free(UNUSED_ARG(const char *file), UNUSED_ARG(unsigned int line), void *ptr)
+slab_free(const char *file, unsigned int line, void *ptr)
 {
     alloc_header_t *hdr;
-    size_t real;
+    size_t user, real;
 
     if (!ptr)
         return;
     verify(ptr);
     hdr = (alloc_header_t*)ptr - 1;
 #if SLAB_DEBUG
+    hdr->file_id = get_file_id(file);
+    hdr->line = line;
     hdr->magic = FREE_MAGIC;
-    real = hdr->size + sizeof(*hdr);
+    user = hdr->size;
 #else
-    real = *hdr + sizeof(*hdr);
+    user = *hdr;
+    (void)file; (void)line;
 #endif
-    real = (real + SLAB_GRAIN - 1) & ~(SLAB_GRAIN - 1);
+    real = (user + sizeof(*hdr) + SLAB_GRAIN - 1) & ~(SLAB_GRAIN - 1);
     if (real < SMALL_CUTOFF) {
+        memset(hdr + 1, 0xde, real - sizeof(*hdr));
         slab_unalloc(hdr, real);
         slab_alloc_count--;
-        slab_alloc_size -= real - sizeof(*hdr);
+        slab_alloc_size -= user;
     } else {
         munmap(hdr, slab_round_up(real));
         big_alloc_count--;
-        big_alloc_size -= real - sizeof(*hdr);
+        big_alloc_size -= user;
     }
 }
 
+/* Undefine the verify macro in case we're not debugging. */
+#undef verify
 void
 verify(const void *ptr)
 {