summaryrefslogtreecommitdiff
path: root/net/rds/ib.c
diff options
context:
space:
mode:
authorZach Brown <zach.brown@oracle.com>2010-07-15 12:34:33 -0700
committerAndy Grover <andy.grover@oracle.com>2010-09-08 18:16:44 -0700
commitea819867b788728aca60717e4fdacb3df771f670 (patch)
tree68952e283e4b119622c6e8244d96e41c623ae4b0 /net/rds/ib.c
parent1bde04a63d532c2540d6fdee0a661530a62b1686 (diff)
RDS/IB: protect the list of IB devices
The RDS IB device list wasn't protected by any locking. Traversal in both the get_mr and FMR flushing paths could race with additon and removal. List manipulation is done with RCU primatives and is protected by the write side of a rwsem. The list traversal in the get_mr fast path is protected by a rcu read critical section. The FMR list traversal is more problematic because it can block while traversing the list. We protect this with the read side of the rwsem. Signed-off-by: Zach Brown <zach.brown@oracle.com>
Diffstat (limited to 'net/rds/ib.c')
-rw-r--r--net/rds/ib.c27
1 files changed, 20 insertions, 7 deletions
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 3eb5617649c6..b12a3951167d 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -53,6 +53,12 @@ MODULE_PARM_DESC(fmr_message_size, " Max size of a RDMA transfer");
module_param(rds_ib_retry_count, int, 0444);
MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
+/*
+ * we have a clumsy combination of RCU and a rwsem protecting this list
+ * because it is used both in the get_mr fast path and while blocking in
+ * the FMR flushing path.
+ */
+DECLARE_RWSEM(rds_ib_devices_lock);
struct list_head rds_ib_devices;
/* NOTE: if also grabbing ibdev lock, grab this first */
@@ -171,7 +177,10 @@ void rds_ib_add_one(struct ib_device *device)
INIT_LIST_HEAD(&rds_ibdev->ipaddr_list);
INIT_LIST_HEAD(&rds_ibdev->conn_list);
- list_add_tail(&rds_ibdev->list, &rds_ib_devices);
+
+ down_write(&rds_ib_devices_lock);
+ list_add_tail_rcu(&rds_ibdev->list, &rds_ib_devices);
+ up_write(&rds_ib_devices_lock);
atomic_inc(&rds_ibdev->refcount);
ib_set_client_data(device, &rds_ib_client, rds_ibdev);
@@ -230,16 +239,20 @@ void rds_ib_remove_one(struct ib_device *device)
rds_ib_dev_shutdown(rds_ibdev);
+ /* stop connection attempts from getting a reference to this device. */
+ ib_set_client_data(device, &rds_ib_client, NULL);
+
+ down_write(&rds_ib_devices_lock);
+ list_del_rcu(&rds_ibdev->list);
+ up_write(&rds_ib_devices_lock);
+
/*
- * prevent future connection attempts from getting a reference to this
- * device and wait for currently racing connection attempts to finish
- * getting their reference
+ * This synchronize rcu is waiting for readers of both the ib
+ * client data and the devices list to finish before we drop
+ * both of those references.
*/
- ib_set_client_data(device, &rds_ib_client, NULL);
synchronize_rcu();
rds_ib_dev_put(rds_ibdev);
-
- list_del(&rds_ibdev->list);
rds_ib_dev_put(rds_ibdev);
}