--- skbuff.c.linksys 2008-12-03 21:51:16.000000000 +0100 +++ skbuff.c 2008-04-17 03:54:01.000000000 +0200 @@ -64,8 +64,6 @@ /* NSP Optimizations: This definition controls the static memory allocation optimization. */ #define TI_STATIC_ALLOCATIONS -int sysctl_hot_list_len = 512; - static kmem_cache_t *skbuff_head_cache; #undef DEBUG_SKB @@ -84,10 +82,10 @@ typedef struct DRIVER_BUFFER { /* Pointer to the allocated data buffer. This is the static data buffer - * allocated for the TI-Cache. 60 bytes out of the below buffer are required - * by the SKB shared info. We always reserve at least MAX_RESERVED_HEADROOM bytes - * so that the packets always have sufficient headroom. */ - char ptr_buffer[MAX_SIZE_STATIC_BUFFER + MAX_RESERVED_HEADROOM + 60]; + * allocated for the TI-Cache. 60 bytes out of the below buffer are required + * by the SKB shared info. We always reserve at least MAX_RESERVED_HEADROOM bytes + * so that the packets always have sufficient headroom. */ + char ptr_buffer[MAX_SIZE_STATIC_BUFFER + MAX_RESERVED_HEADROOM + 60]; /* List of the driver buffers. */ struct DRIVER_BUFFER* ptr_next; @@ -98,32 +96,26 @@ /* List of the driver buffers. */ DRIVER_BUFFER* ptr_available_driver_buffers; - /* The number of available buffers. */ - int num_available_buffers; + /* The number of available buffers. */ + int num_available_buffers; + }DRIVER_BUFFER_MCB; DRIVER_BUFFER_MCB driver_mcb; -int hybrid_mode = 0; +int hybrid_mode = 0; #endif /* TI_STATIC_ALLOCATIONS */ -static union { - struct sk_buff_head list; - char pad[SMP_CACHE_BYTES]; -} skb_head_pool[NR_CPUS]; - #ifdef TI_STATIC_ALLOCATIONS int num_of_buffers(char* buf) { - unsigned long flags; - int len = 0; - - local_irq_save(flags); - - len = sprintf(buf,"Numbers of free Buffers %d\n",driver_mcb.num_available_buffers); - /* Criticial Section: Unlock interrupts. */ - local_irq_restore(flags); - return len; + unsigned long flags; + int len = 0; + local_irq_save(flags); + len = sprintf(buf,"Numbers of free Buffers %d\n",driver_mcb.num_available_buffers); + /* Criticial Section: Unlock interrupts. */ + local_irq_restore(flags); + return len; } int read_num_of_free_buff() { @@ -169,76 +161,6 @@ BUG(); } -static __inline__ struct sk_buff *skb_head_from_pool(void) -{ - struct sk_buff_head *list; - struct sk_buff *skb = NULL; - unsigned long flags; - - local_irq_save(flags); - - list = &skb_head_pool[smp_processor_id()].list; - - if (skb_queue_len(list)) { - - skb = __skb_dequeue(list); - } - local_irq_restore(flags); - return skb; -} - -/************************************************************************** - * FUNCTION NAME : skb_head_to_pool - ************************************************************************** - * DESCRIPTION : - * This function was optimized to put the SKB back into the pool, the - * orignal code would keep only 'sysctl_hot_list_len' entries in the list - * the remaining skb's would be released back to the SLAB. -***************************************************************************/ -static __inline__ void skb_head_to_pool(struct sk_buff *skb) -{ -#ifdef TI_STATIC_ALLOCATIONS - struct sk_buff_head* list; - unsigned long flags; - - local_irq_save(flags); - list = &skb_head_pool[smp_processor_id()].list; - if (skb_queue_len(list) < sysctl_hot_list_len) - { - __skb_queue_head(list, skb); - local_irq_restore(flags); - return; - }else - { - local_irq_restore(flags); - kmem_cache_free(skbuff_head_cache, skb); - return; - } -#else - struct sk_buff_head *list; - unsigned long flags; - - local_irq_save(flags); - - list = &skb_head_pool[smp_processor_id()].list; - - if (skb_queue_len(list) < sysctl_hot_list_len) - { - __skb_queue_head(list, skb); - local_irq_restore(flags); - return; - } - local_irq_restore(flags); - kmem_cache_free(skbuff_head_cache, skb); -#endif /* TI_STATIC_ALLOCATIONS */ -} - -/* Allocate a new skbuff. We do this ourselves so we can fill in a few - * 'private' fields and also do memory statistics to find all the - * [BEEP] leaks. - * - */ - /** * alloc_skb - allocate a network buffer * @size: size to allocate @@ -267,15 +189,9 @@ gfp_mask &= ~__GFP_WAIT; } - /* Get the HEAD */ - skb = skb_head_from_pool(); - if (skb == NULL) - { - skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); - if (skb == NULL) - goto nohead; - - } + skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (skb == NULL) + goto nohead; /* Get the DATA. Size must match skb_add_mtu(). */ size = SKB_DATA_ALIGN(size); @@ -301,12 +217,14 @@ atomic_set(&(skb_shinfo(skb)->dataref), 1); skb_shinfo(skb)->nr_frags = 0; skb_shinfo(skb)->frag_list = NULL; - return skb; nodata: - skb_head_to_pool(skb); + return skb; + nohead: - return NULL; + kmem_cache_free(skbuff_head_cache, skb); + skb = NULL; + goto nodata; } /* @@ -399,25 +317,6 @@ /************************************************************************** - * FUNCTION NAME : ti_release_skb - ************************************************************************** - * DESCRIPTION : - * This function is called from the ti_alloc_skb when there were no more - * data buffers available. The allocated SKB had to released back to the - * data pool. The reason why this function was moved from the fast path - * below was because '__skb_queue_head' is an inline function which adds - * a large code chunk on the fast path. - * - * NOTES : - * This function is called with interrupts disabled. - **************************************************************************/ -static void ti_release_skb (struct sk_buff_head* list, struct sk_buff* skb) -{ - __skb_queue_head(list, skb); - return; -} - -/************************************************************************** * FUNCTION NAME : ti_alloc_skb ************************************************************************** * DESCRIPTION : @@ -430,101 +329,99 @@ **************************************************************************/ struct sk_buff *ti_alloc_skb(unsigned int size,int gfp_mask) { - register struct sk_buff* skb; - unsigned long flags; - struct sk_buff_head* list; + register struct sk_buff* skb; + unsigned long flags; DRIVER_BUFFER* ptr_node = NULL; - /* Critical Section Begin: Lock out interrupts. */ - local_irq_save(flags); + /* YES. Now get a data block from the head of statically allocated block. */ + ptr_node = driver_mcb.ptr_available_driver_buffers; - /* Get the SKB Pool list associated with the processor and dequeue the head. */ - list = &skb_head_pool[smp_processor_id()].list; - skb = __skb_dequeue(list); + if (ptr_node != NULL || hybrid_mode == 1) { - /* Align the data size. */ - size = SKB_DATA_ALIGN(size); - - /* Did we get one. */ - if (skb != NULL) - { - /* YES. Now get a data block from the head of statically allocated block. */ - ptr_node = driver_mcb.ptr_available_driver_buffers; - if (ptr_node != NULL) - { - /* YES. Got a data block. Advance the free list pointer to the next available buffer. */ - driver_mcb.ptr_available_driver_buffers = ptr_node->ptr_next; - ptr_node->ptr_next = NULL; - - /* Decrement the number of available data buffers. */ - driver_mcb.num_available_buffers = driver_mcb.num_available_buffers - 1; - } - else - { - /* NO. Was unable to get a data block. So put the SKB back on the free list. - * This is slow path. */ -#ifdef DEBUG_SKB - printk ("DEBUG: No Buffer memory available: Number of free buffer:%d.\n", - driver_mcb.num_available_buffers); -#endif - ti_release_skb (list, skb); - } - } + /* Critical Section Begin: Lock out interrupts. */ + local_irq_save(flags); - /* Critical Section End: Unlock interrupts. */ - local_irq_restore(flags); + skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (skb == NULL) + goto nohead; + + /* Align the data size. */ + size = SKB_DATA_ALIGN(size); + + if (ptr_node != NULL) + { + /* YES. Got a data block. Advance the free list pointer to the next available buffer. */ + driver_mcb.ptr_available_driver_buffers = ptr_node->ptr_next; + ptr_node->ptr_next = NULL; + + /* Decrement the number of available data buffers. */ + driver_mcb.num_available_buffers = driver_mcb.num_available_buffers - 1; + } + + /* Critical Section End: Unlock interrupts. */ + local_irq_restore(flags); + + /* Did we get an SKB and data buffer. Proceed only if we were succesful in getting both else drop */ + if (ptr_node != NULL) + { + /* XXX: does not include slab overhead */ + skb->truesize = size + sizeof(struct sk_buff); + + /* Load the data pointers. */ + skb->head = ptr_node->ptr_buffer; + skb->data = ptr_node->ptr_buffer + MAX_RESERVED_HEADROOM; + skb->tail = ptr_node->ptr_buffer + MAX_RESERVED_HEADROOM; + skb->end = ptr_node->ptr_buffer + size + MAX_RESERVED_HEADROOM; + + /* Set up other state */ + skb->len = 0; + skb->cloned = 0; + skb->data_len = 0; + + /* Mark the SKB indicating that the SKB is from the TI cache. */ + skb->cb[45] = 1; + + atomic_set(&skb->users, 1); + atomic_set(&(skb_shinfo(skb)->dataref), 1); + skb_shinfo(skb)->nr_frags = 0; + skb_shinfo(skb)->frag_list = NULL; + return skb; + } + + nohead: + + kmem_cache_free(skbuff_head_cache, skb); + + /* Control comes here only when there is no statically allocated data buffers + * available. This case is handled using the mode selected + * + * 1. Hybrid Mode. + * In that case lets jump to the old allocation code. This way we + * can allocate a small number of data buffers upfront and the rest will hit + * this portion of the code, which is slow path. Note the number of hits here + * should be kept as low as possible to satisfy performance requirements. + * + * 2. Pure Static Mode. + * Return NULL the user should have tuned the number of static buffers for + * worst case scenario. So return NULL and let the drivers handle the error. */ + if (hybrid_mode == 1) + { + /* Hybrid Mode: Old allocation. */ + if (net_ratelimit()) + printk("DEBUG:Going Hybrid"); + return alloc_skb(size,gfp_mask); + } + else + { + /* Pure Static Mode: No buffers available. */ + return NULL; + } + + } else { - /* Did we get an SKB and data buffer. Proceed only if we were succesful in getting both else drop */ - if (skb != NULL && ptr_node != NULL) - { - /* XXX: does not include slab overhead */ - skb->truesize = size + sizeof(struct sk_buff); - - /* Load the data pointers. */ - skb->head = ptr_node->ptr_buffer; - skb->data = ptr_node->ptr_buffer + MAX_RESERVED_HEADROOM; - skb->tail = ptr_node->ptr_buffer + MAX_RESERVED_HEADROOM; - skb->end = ptr_node->ptr_buffer + size + MAX_RESERVED_HEADROOM; - - /* Set up other state */ - skb->len = 0; - skb->cloned = 0; - skb->data_len = 0; - - /* Mark the SKB indicating that the SKB is from the TI cache. */ - skb->cb[45] = 1; - - atomic_set(&skb->users, 1); - atomic_set(&(skb_shinfo(skb)->dataref), 1); - skb_shinfo(skb)->nr_frags = 0; - skb_shinfo(skb)->frag_list = NULL; - return skb; - } - else - { - /* Control comes here only when there is no statically allocated data buffers - * available. This case is handled using the mode selected - * - * 1. Hybrid Mode. - * In that case lets jump to the old allocation code. This way we - * can allocate a small number of data buffers upfront and the rest will hit - * this portion of the code, which is slow path. Note the number of hits here - * should be kept as low as possible to satisfy performance requirements. - * - * 2. Pure Static Mode. - * Return NULL the user should have tuned the number of static buffers for - * worst case scenario. So return NULL and let the drivers handle the error. */ - if (hybrid_mode == 1) - { - /* Hybrid Mode: Old allocation. */ - return dev_alloc_skb(size); - } - else - { - /* Pure Static Mode: No buffers available. */ - return NULL; - } - } + /* Pure Static Mode: No buffers available. */ + return NULL; + } } /************************************************************************** @@ -564,35 +461,43 @@ **************************************************************************/ static void ti_skb_release_data(struct sk_buff *skb) { - DRIVER_BUFFER* ptr_node; - unsigned long flags; + DRIVER_BUFFER* ptr_node; + unsigned long flags; + char* ptr_num_buffers; + int num_buffers; - /* The SKB data can be cleaned only if the packet has not been cloned and we - * are the only one holding a reference to the data. */ + /* The SKB data can be cleaned only if the packet has not been cloned and we + * are the only one holding a reference to the data. */ if (!skb->cloned || atomic_dec_and_test(&(skb_shinfo(skb)->dataref))) - { - /* Are there any fragments associated with the SKB ?*/ + { + /* Are there any fragments associated with the SKB ?*/ if ((skb_shinfo(skb)->nr_frags != 0) || (skb_shinfo(skb)->frag_list != NULL)) - { - /* Slow Path: Try and clean up the fragments. */ - ti_skb_release_fragment (skb); - } - - /* Cleanup the SKB data memory. This is fast path. */ - ptr_node = (DRIVER_BUFFER *)skb->head; - - /* Critical Section: Lock out interrupts. */ - local_irq_save(flags); - - /* Add the data buffer to the list of available buffers. */ - ptr_node->ptr_next = driver_mcb.ptr_available_driver_buffers; - driver_mcb.ptr_available_driver_buffers = ptr_node; - - /* Increment the number of available data buffers. */ - driver_mcb.num_available_buffers = driver_mcb.num_available_buffers + 1; - - /* Criticial Section: Unlock interrupts. */ - local_irq_restore(flags); + { + /* Slow Path: Try and clean up the fragments. */ + ti_skb_release_fragment(skb); + } + + /* YES. Now get a data block from the head of statically allocated block. */ + ptr_node = driver_mcb.ptr_available_driver_buffers; + if (ptr_node != NULL) + { + /* Cleanup the SKB data memory. This is fast path. */ + ptr_node = (DRIVER_BUFFER *)skb->head; + + /* Critical Section: Lock out interrupts. */ + local_irq_save(flags); + + /* Add the data buffer to the list of available buffers. */ + ptr_node->ptr_next = driver_mcb.ptr_available_driver_buffers; + driver_mcb.ptr_available_driver_buffers = ptr_node; + + /* Increment the number of available data buffers. */ + driver_mcb.num_available_buffers = driver_mcb.num_available_buffers + 1; + + /* Criticial Section: Unlock interrupts. */ + local_irq_restore(flags); + } + } return; } @@ -606,7 +511,7 @@ void kfree_skbmem(struct sk_buff *skb) { skb_release_data(skb); - skb_head_to_pool(skb); + kmem_cache_free(skbuff_head_cache, skb); } /** @@ -627,9 +532,10 @@ } dst_release(skb->dst); + if(skb->destructor) { #if defined(CONFIG_MIPS_AVALANCHE_SOC) - /* nothing */ + /* nothing */ #else if (in_irq()) { printk(KERN_WARNING "Warning: kfree_skb on hard IRQ %p\n", @@ -642,24 +548,25 @@ nf_conntrack_put(skb->nfct); #endif - /* NSP Optimization: Any module will call this function to clean the packet memory. - * Check if the packet belongs to the TI-Cache and if so then put the packet back - * into the static memory pool. If not then release into the SLAB. */ +/* NSP Optimization: Any module will call this function to clean the packet memory. +* Check if the packet belongs to the TI-Cache and if so then put the packet back +* into the static memory pool. If not then release into the SLAB. */ + #ifdef TI_STATIC_ALLOCATIONS - /* Check if the SKB is from the TI cache. */ - if (skb->cb[45] == 1) - { - skb_headerinit(skb, NULL, 0); /* clean state */ - ti_skb_release_data(skb); - skb_head_to_pool(skb); - } - else - { - skb_headerinit(skb, NULL, 0); /* clean state */ - kfree_skbmem(skb); - } + /* Check if the SKB is from the TI cache. */ + if (skb->cb[45] == 1) + { + skb_headerinit(skb, NULL, 0); /* clean state */ + ti_skb_release_data(skb); + kmem_cache_free(skbuff_head_cache, skb); + } + else + { + skb_headerinit(skb, NULL, 0); /* clean state */ + kfree_skbmem(skb); + } #else - skb_headerinit(skb, NULL, 0); /* clean state */ + skb_headerinit(skb, NULL, 0); /* clean state */ kfree_skbmem(skb); #endif /* TI_STATIC_ALLOCATIONS */ } @@ -682,19 +589,16 @@ { struct sk_buff *n; - n = skb_head_from_pool(); - if (!n) - { - /* NSP Optimizations: We do not want the SKB's to ever be allocated from the dynamic - * memory pool. This causes the SLAB manager to be used. If this happens we can run - * into performance problems. Instead everytime print out a message and return ERROR. - * These messages are important as they need to be monitored to determine MAX. number - * of buffers that are required. */ - n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); - if (!n) - { - return NULL; - } + /* NSP Optimizations: We do not want the SKB's to ever be allocated from the dynamic + * memory pool. This causes the SLAB manager to be used. If this happens we can run + * into performance problems. Instead everytime print out a message and return ERROR. + * These messages are important as they need to be monitored to determine MAX. number + * of buffers that are required. */ + + n = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (!n) + { + return NULL; } #define C(x) n->x = skb->x @@ -766,14 +670,14 @@ new->nh.raw=old->nh.raw+offset; new->mac.raw=old->mac.raw+offset; - /* PANKAJ: While copying the skb header don't inherit the cb[45]. */ - { - /* Retain the control buffer value which decides the pool... Just another reason to - * use a seperate field in skb than cb[45]. */ - char retained_value = new->cb[45]; - memcpy(new->cb, old->cb, sizeof(old->cb)); - new->cb[45] = retained_value; - } + /* PANKAJ: While copying the skb header don't inherit the cb[45]. */ + { + /* Retain the control buffer value which decides the pool... Just another reason to + * use a seperate field in skb than cb[45]. */ + char retained_value = new->cb[45]; + memcpy(new->cb, old->cb, sizeof(old->cb)); + new->cb[45] = retained_value; + } atomic_set(&new->users, 1); new->pkt_type=old->pkt_type; @@ -819,7 +723,7 @@ /* * Allocate the copy buffer */ - n=alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask); + n=ti_alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask); if(n==NULL) return NULL; @@ -868,19 +772,19 @@ /* Free old data. */ - /* PANKAJ: Debugging.... */ - if (skb->cb[45] == 1) - { - /* Release the old data back to the pool. */ - ti_skb_release_data(skb); - - /* We have now allocated data from the dynamic memory pool. Use the skb to indicate so. */ - skb->cb[45] = 0; - } - else - { - skb_release_data(skb); - } + /* PANKAJ: Debugging.... */ + if (skb->cb[45] == 1) + { + /* Release the old data back to the pool. */ + ti_skb_release_data(skb); + + /* We have now allocated data from the dynamic memory pool. Use the skb to indicate so. */ + skb->cb[45] = 0; + } + else + { + skb_release_data(skb); + } skb->head = data; skb->end = data + size; @@ -926,7 +830,7 @@ /* * Allocate the copy buffer */ - n=alloc_skb(skb->end - skb->head, gfp_mask); + n=ti_alloc_skb(skb->end - skb->head, gfp_mask); if(n==NULL) return NULL; @@ -1005,19 +909,19 @@ if (skb_shinfo(skb)->frag_list) skb_clone_fraglist(skb); - /* PANKAJ: Debugging.... */ - if (skb->cb[45] == 1) - { - /* Release the old data back to the pool. */ - ti_skb_release_data(skb); - - /* We have now allocated data from the dynamic memory pool. Use the skb to indicate so. */ - skb->cb[45] = 0; - } - else - { - skb_release_data(skb); - } + /* PANKAJ: Debugging.... */ + if (skb->cb[45] == 1) + { + /* Release the old data back to the pool. */ + ti_skb_release_data(skb); + + /* We have now allocated data from the dynamic memory pool. Use the skb to indicate so. */ + skb->cb[45] = 0; + } + else + { + skb_release_data(skb); + } off = (data+nhead) - skb->head; @@ -1089,7 +993,7 @@ * Allocate the copy buffer */ - n=alloc_skb(newheadroom + skb->len + newtailroom, + n=ti_alloc_skb(newheadroom + skb->len + newtailroom, gfp_mask); if(n==NULL) return NULL; @@ -1551,27 +1455,14 @@ } } -#if 0 -/* - * Tune the memory allocator for a new MTU size. - */ -void skb_add_mtu(int mtu) -{ - /* Must match allocation in alloc_skb */ - mtu = SKB_DATA_ALIGN(mtu) + sizeof(struct skb_shared_info); - - kmem_add_cache_size(mtu); -} -#endif - void __init skb_init(void) { int i; - struct sk_buff *skb; + struct sk_buff *skb; - char* ptr_num_buffers; - int num_buffers; - char* ptr_approach; + char* ptr_num_buffers; + int num_buffers; + char* ptr_approach; skbuff_head_cache = kmem_cache_create("skbuff_head_cache", @@ -1582,64 +1473,45 @@ if (!skbuff_head_cache) panic("cannot create skbuff cache"); - for (i=0; iptr_buffer, - MAX_SIZE_STATIC_BUFFER + MAX_RESERVED_HEADROOM + 60); + /* Increment the number of available data buffers. */ + driver_mcb.num_available_buffers = driver_mcb.num_available_buffers + 1; + + /* Mick: Cache Invalidate. */ + skbuff_data_invalidate ((void *)ptr_driver_buffer->ptr_buffer, + MAX_SIZE_STATIC_BUFFER + MAX_RESERVED_HEADROOM + 60); } - printk ("NSP Optimizations: Succesfully allocated TI-Cached Memory Pool.\n"); + printk ("NSP Optimizations: Succesfully allocated TI-Cached Memory Pool.\n"); #endif /* TI_STATIC_ALLOCATIONS */