From aae3e195b2be5cb001d31aee5979837da8ccee3f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 9 Apr 2014 11:58:17 +0200 Subject: [PATCH 227/365] percpu_ida: Use local locks the local_irq_save() + spin_lock() does not work that well on -RT Signed-off-by: Sebastian Andrzej Siewior --- lib/percpu_ida.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/percpu_ida.c b/lib/percpu_ida.c index 6d40944960de..822a2c027e72 100644 --- a/lib/percpu_ida.c +++ b/lib/percpu_ida.c @@ -26,6 +26,9 @@ #include #include #include +#include + +static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock); struct percpu_ida_cpu { /* @@ -148,13 +151,13 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state) unsigned long flags; int tag; - local_irq_save(flags); + local_lock_irqsave(irq_off_lock, flags); tags = this_cpu_ptr(pool->tag_cpu); /* Fastpath */ tag = alloc_local_tag(tags); if (likely(tag >= 0)) { - local_irq_restore(flags); + local_unlock_irqrestore(irq_off_lock, flags); return tag; } @@ -173,6 +176,7 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state) if (!tags->nr_free) alloc_global_tags(pool, tags); + if (!tags->nr_free) steal_tags(pool, tags); @@ -184,7 +188,7 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state) } spin_unlock(&pool->lock); - local_irq_restore(flags); + local_unlock_irqrestore(irq_off_lock, flags); if (tag >= 0 || state == TASK_RUNNING) break; @@ -196,7 +200,7 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state) schedule(); - local_irq_save(flags); + local_lock_irqsave(irq_off_lock, flags); tags = this_cpu_ptr(pool->tag_cpu); } if (state != TASK_RUNNING) @@ -221,7 +225,7 @@ void percpu_ida_free(struct percpu_ida *pool, unsigned tag) BUG_ON(tag >= pool->nr_tags); - local_irq_save(flags); + local_lock_irqsave(irq_off_lock, flags); tags = this_cpu_ptr(pool->tag_cpu); spin_lock(&tags->lock); @@ -253,7 +257,7 @@ void percpu_ida_free(struct percpu_ida *pool, unsigned tag) spin_unlock(&pool->lock); } - local_irq_restore(flags); + local_unlock_irqrestore(irq_off_lock, flags); } EXPORT_SYMBOL_GPL(percpu_ida_free); @@ -345,7 +349,7 @@ int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn, struct percpu_ida_cpu *remote; unsigned cpu, i, err = 0; - local_irq_save(flags); + local_lock_irqsave(irq_off_lock, flags); for_each_possible_cpu(cpu) { remote = per_cpu_ptr(pool->tag_cpu, cpu); spin_lock(&remote->lock); @@ -367,7 +371,7 @@ int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn, } spin_unlock(&pool->lock); out: - local_irq_restore(flags); + local_unlock_irqrestore(irq_off_lock, flags); return err; } EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); -- 2.28.0