summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Documentation/ABI/stable/sysfs-class-infiniband17
-rw-r--r--Documentation/infiniband/core_locking.rst (renamed from Documentation/infiniband/core_locking.txt)64
-rw-r--r--Documentation/infiniband/index.rst23
-rw-r--r--Documentation/infiniband/ipoib.rst (renamed from Documentation/infiniband/ipoib.txt)24
-rw-r--r--Documentation/infiniband/opa_vnic.rst (renamed from Documentation/infiniband/opa_vnic.txt)110
-rw-r--r--Documentation/infiniband/sysfs.rst (renamed from Documentation/infiniband/sysfs.txt)4
-rw-r--r--Documentation/infiniband/tag_matching.rst (renamed from Documentation/infiniband/tag_matching.txt)5
-rw-r--r--Documentation/infiniband/user_mad.rst (renamed from Documentation/infiniband/user_mad.txt)33
-rw-r--r--Documentation/infiniband/user_verbs.rst (renamed from Documentation/infiniband/user_verbs.txt)12
-rw-r--r--MAINTAINERS8
-rw-r--r--drivers/infiniband/Kconfig12
-rw-r--r--drivers/infiniband/core/Makefile3
-rw-r--r--drivers/infiniband/core/core_priv.h9
-rw-r--r--drivers/infiniband/core/cq.c37
-rw-r--r--drivers/infiniband/core/device.c131
-rw-r--r--drivers/infiniband/core/mr_pool.c8
-rw-r--r--drivers/infiniband/core/nldev.c233
-rw-r--r--drivers/infiniband/core/rw.c201
-rw-r--r--drivers/infiniband/core/ucm.c1350
-rw-r--r--drivers/infiniband/core/ucma.c114
-rw-r--r--drivers/infiniband/core/umem.c13
-rw-r--r--drivers/infiniband/core/umem_odp.c106
-rw-r--r--drivers/infiniband/core/user_mad.c53
-rw-r--r--drivers/infiniband/core/uverbs_cmd.c26
-rw-r--r--drivers/infiniband/core/uverbs_main.c40
-rw-r--r--drivers/infiniband/core/uverbs_std_types_cq.c19
-rw-r--r--drivers/infiniband/core/uverbs_std_types_mr.c1
-rw-r--r--drivers/infiniband/core/uverbs_uapi.c4
-rw-r--r--drivers/infiniband/core/verbs.c156
-rw-r--r--drivers/infiniband/hw/Makefile1
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.c66
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.h9
-rw-r--r--drivers/infiniband/hw/bnxt_re/main.c8
-rw-r--r--drivers/infiniband/hw/cxgb3/cxio_hal.c30
-rw-r--r--drivers/infiniband/hw/cxgb3/cxio_hal.h3
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_cm.c2
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_provider.c160
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c21
-rw-r--r--drivers/infiniband/hw/cxgb4/cq.c54
-rw-r--r--drivers/infiniband/hw/cxgb4/device.c9
-rw-r--r--drivers/infiniband/hw/cxgb4/iw_cxgb4.h11
-rw-r--r--drivers/infiniband/hw/cxgb4/mem.c8
-rw-r--r--drivers/infiniband/hw/cxgb4/provider.c9
-rw-r--r--drivers/infiniband/hw/cxgb4/qp.c94
-rw-r--r--drivers/infiniband/hw/cxgb4/resource.c16
-rw-r--r--drivers/infiniband/hw/efa/efa.h9
-rw-r--r--drivers/infiniband/hw/efa/efa_com.c74
-rw-r--r--drivers/infiniband/hw/efa/efa_com.h1
-rw-r--r--drivers/infiniband/hw/efa/efa_com_cmd.c8
-rw-r--r--drivers/infiniband/hw/efa/efa_main.c10
-rw-r--r--drivers/infiniband/hw/efa/efa_verbs.c248
-rw-r--r--drivers/infiniband/hw/hfi1/mad.c9
-rw-r--r--drivers/infiniband/hw/hfi1/pio.c3
-rw-r--r--drivers/infiniband/hw/hfi1/rc.c26
-rw-r--r--drivers/infiniband/hw/hfi1/tid_rdma.c7
-rw-r--r--drivers/infiniband/hw/hfi1/user_pages.c11
-rw-r--r--drivers/infiniband/hw/hfi1/verbs.c6
-rw-r--r--drivers/infiniband/hw/hns/Kconfig15
-rw-r--r--drivers/infiniband/hw/hns/Makefile13
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_alloc.c101
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_cmd.c4
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_cq.c81
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_db.c12
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_device.h108
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hem.c504
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hem.h16
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v1.c74
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.c270
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.h23
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_main.c29
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_mr.c166
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_pd.c2
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_qp.c207
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_srq.c40
-rw-r--r--drivers/infiniband/hw/i40iw/i40iw_verbs.c54
-rw-r--r--drivers/infiniband/hw/mlx4/cq.c43
-rw-r--r--drivers/infiniband/hw/mlx4/main.c21
-rw-r--r--drivers/infiniband/hw/mlx4/mlx4_ib.h9
-rw-r--r--drivers/infiniband/hw/mlx4/mr.c16
-rw-r--r--drivers/infiniband/hw/mlx4/qp.c11
-rw-r--r--drivers/infiniband/hw/mlx4/srq.c9
-rw-r--r--drivers/infiniband/hw/mlx5/cq.c56
-rw-r--r--drivers/infiniband/hw/mlx5/ib_rep.c19
-rw-r--r--drivers/infiniband/hw/mlx5/ib_rep.h2
-rw-r--r--drivers/infiniband/hw/mlx5/main.c59
-rw-r--r--drivers/infiniband/hw/mlx5/mem.c20
-rw-r--r--drivers/infiniband/hw/mlx5/mlx5_ib.h30
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c554
-rw-r--r--drivers/infiniband/hw/mlx5/odp.c44
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c236
-rw-r--r--drivers/infiniband/hw/mthca/mthca_memfree.c6
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.c52
-rw-r--r--drivers/infiniband/hw/nes/Kconfig16
-rw-r--r--drivers/infiniband/hw/nes/Makefile4
-rw-r--r--drivers/infiniband/hw/nes/nes.c1205
-rw-r--r--drivers/infiniband/hw/nes/nes.h574
-rw-r--r--drivers/infiniband/hw/nes/nes_cm.c3992
-rw-r--r--drivers/infiniband/hw/nes/nes_cm.h470
-rw-r--r--drivers/infiniband/hw/nes/nes_context.h193
-rw-r--r--drivers/infiniband/hw/nes/nes_hw.c3887
-rw-r--r--drivers/infiniband/hw/nes/nes_hw.h1380
-rw-r--r--drivers/infiniband/hw/nes/nes_mgt.c1155
-rw-r--r--drivers/infiniband/hw/nes/nes_mgt.h97
-rw-r--r--drivers/infiniband/hw/nes/nes_nic.c1870
-rw-r--r--drivers/infiniband/hw/nes/nes_utils.c916
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c3759
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.h198
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.c8
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.h2
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_main.c8
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c38
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.h7
-rw-r--r--drivers/infiniband/hw/qedr/main.c8
-rw-r--r--drivers/infiniband/hw/qedr/verbs.c82
-rw-r--r--drivers/infiniband/hw/qedr/verbs.h7
-rw-r--r--drivers/infiniband/hw/qib/qib_rc.c26
-rw-r--r--drivers/infiniband/hw/qib/qib_user_pages.c11
-rw-r--r--drivers/infiniband/hw/qib/qib_user_sdma.c11
-rw-r--r--drivers/infiniband/hw/qib/qib_verbs.c6
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib.h4
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_main.c8
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_verbs.c22
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_verbs.h7
-rw-r--r--drivers/infiniband/hw/usnic/usnic_uiom.c7
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma.h2
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c46
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c8
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c3
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c16
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h7
-rw-r--r--drivers/infiniband/sw/rdmavt/cq.c57
-rw-r--r--drivers/infiniband/sw/rdmavt/cq.h7
-rw-r--r--drivers/infiniband/sw/rdmavt/mr.c3
-rw-r--r--drivers/infiniband/sw/rdmavt/qp.c35
-rw-r--r--drivers/infiniband/sw/rdmavt/vt.c7
-rw-r--r--drivers/infiniband/sw/rxe/rxe_mr.c3
-rw-r--r--drivers/infiniband/sw/rxe/rxe_pool.c1
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.c40
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.h2
-rw-r--r--drivers/infiniband/ulp/ipoib/Kconfig2
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_ethtool.c3
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_verbs.c7
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.c12
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.h64
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c12
-rw-r--r--drivers/infiniband/ulp/iser/iser_memory.c121
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c156
-rw-r--r--drivers/infiniband/ulp/isert/ib_isert.c19
-rw-r--r--drivers/infiniband/ulp/srp/ib_srp.c21
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/Makefile2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/cmd.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ecpf.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ecpf.h4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.c15
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.h2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eq.c444
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch.c34
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch.h33
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c181
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/events.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c3
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h9
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/main.c56
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h13
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c334
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/rdma.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/sriov.c22
-rw-r--r--drivers/nvme/host/rdma.c2
-rw-r--r--include/linux/mlx5/device.h2
-rw-r--r--include/linux/mlx5/driver.h11
-rw-r--r--include/linux/mlx5/eq.h23
-rw-r--r--include/linux/mlx5/eswitch.h32
-rw-r--r--include/linux/mlx5/fs.h1
-rw-r--r--include/linux/mlx5/mlx5_ifc.h35
-rw-r--r--include/linux/mlx5/qp.h3
-rw-r--r--include/net/devlink.h6
-rw-r--r--include/rdma/ib_umem.h19
-rw-r--r--include/rdma/ib_umem_odp.h20
-rw-r--r--include/rdma/ib_verbs.h208
-rw-r--r--include/rdma/mr_pool.h2
-rw-r--r--include/rdma/rdma_netlink.h8
-rw-r--r--include/rdma/rdma_vt.h2
-rw-r--r--include/rdma/rdmavt_qp.h108
-rw-r--r--include/rdma/rw.h9
-rw-r--r--include/rdma/signature.h122
-rw-r--r--include/uapi/rdma/ib_user_cm.h326
-rw-r--r--include/uapi/rdma/rdma_netlink.h31
-rw-r--r--net/core/devlink.c6
-rw-r--r--net/rds/ib_cm.c8
189 files changed, 4844 insertions, 24598 deletions
diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband
index 17211ceb9bf4..aed21b8916a2 100644
--- a/Documentation/ABI/stable/sysfs-class-infiniband
+++ b/Documentation/ABI/stable/sysfs-class-infiniband
@@ -423,23 +423,6 @@ Description:
(e.g. driver restart on the VM which owns the VF).
-sysfs interface for NetEffect RNIC Low-Level iWARP driver (nes)
----------------------------------------------------------------
-
-What: /sys/class/infiniband/nesX/hw_rev
-What: /sys/class/infiniband/nesX/hca_type
-What: /sys/class/infiniband/nesX/board_id
-Date: Feb, 2008
-KernelVersion: v2.6.25
-Contact: linux-rdma@vger.kernel.org
-Description:
- hw_rev: (RO) Hardware revision number
-
- hca_type: (RO) Host Channel Adapter type (NEX020)
-
- board_id: (RO) Manufacturing board id
-
-
sysfs interface for Chelsio T4/T5 RDMA driver (cxgb4)
-----------------------------------------------------
diff --git a/Documentation/infiniband/core_locking.txt b/Documentation/infiniband/core_locking.rst
index 4b1f36b6ada0..f34669beb4fe 100644
--- a/Documentation/infiniband/core_locking.txt
+++ b/Documentation/infiniband/core_locking.rst
@@ -1,4 +1,6 @@
-INFINIBAND MIDLAYER LOCKING
+===========================
+InfiniBand Midlayer Locking
+===========================
This guide is an attempt to make explicit the locking assumptions
made by the InfiniBand midlayer. It describes the requirements on
@@ -6,45 +8,47 @@ INFINIBAND MIDLAYER LOCKING
protocols that use the midlayer.
Sleeping and interrupt context
+==============================
With the following exceptions, a low-level driver implementation of
all of the methods in struct ib_device may sleep. The exceptions
are any methods from the list:
- create_ah
- modify_ah
- query_ah
- destroy_ah
- post_send
- post_recv
- poll_cq
- req_notify_cq
- map_phys_fmr
+ - create_ah
+ - modify_ah
+ - query_ah
+ - destroy_ah
+ - post_send
+ - post_recv
+ - poll_cq
+ - req_notify_cq
+ - map_phys_fmr
which may not sleep and must be callable from any context.
The corresponding functions exported to upper level protocol
consumers:
- ib_create_ah
- ib_modify_ah
- ib_query_ah
- ib_destroy_ah
- ib_post_send
- ib_post_recv
- ib_req_notify_cq
- ib_map_phys_fmr
+ - ib_create_ah
+ - ib_modify_ah
+ - ib_query_ah
+ - ib_destroy_ah
+ - ib_post_send
+ - ib_post_recv
+ - ib_req_notify_cq
+ - ib_map_phys_fmr
are therefore safe to call from any context.
In addition, the function
- ib_dispatch_event
+ - ib_dispatch_event
used by low-level drivers to dispatch asynchronous events through
the midlayer is also safe to call from any context.
Reentrancy
+----------
All of the methods in struct ib_device exported by a low-level
driver must be fully reentrant. The low-level driver is required to
@@ -62,6 +66,7 @@ Reentrancy
information between different calls of ib_poll_cq() is not defined.
Callbacks
+---------
A low-level driver must not perform a callback directly from the
same callchain as an ib_device method call. For example, it is not
@@ -74,18 +79,18 @@ Callbacks
completion event handlers for the same CQ are not called
simultaneously. The driver must guarantee that only one CQ event
handler for a given CQ is running at a time. In other words, the
- following situation is not allowed:
+ following situation is not allowed::
- CPU1 CPU2
+ CPU1 CPU2
- low-level driver ->
- consumer CQ event callback:
- /* ... */
- ib_req_notify_cq(cq, ...);
- low-level driver ->
- /* ... */ consumer CQ event callback:
- /* ... */
- return from CQ event handler
+ low-level driver ->
+ consumer CQ event callback:
+ /* ... */
+ ib_req_notify_cq(cq, ...);
+ low-level driver ->
+ /* ... */ consumer CQ event callback:
+ /* ... */
+ return from CQ event handler
The context in which completion event and asynchronous event
callbacks run is not defined. Depending on the low-level driver, it
@@ -93,6 +98,7 @@ Callbacks
Upper level protocol consumers may not sleep in a callback.
Hot-plug
+--------
A low-level driver announces that a device is ready for use by
consumers when it calls ib_register_device(), all initialization
diff --git a/Documentation/infiniband/index.rst b/Documentation/infiniband/index.rst
new file mode 100644
index 000000000000..22eea64de722
--- /dev/null
+++ b/Documentation/infiniband/index.rst
@@ -0,0 +1,23 @@
+:orphan:
+
+==========
+InfiniBand
+==========
+
+.. toctree::
+ :maxdepth: 1
+
+ core_locking
+ ipoib
+ opa_vnic
+ sysfs
+ tag_matching
+ user_mad
+ user_verbs
+
+.. only:: subproject and html
+
+ Indices
+ =======
+
+ * :ref:`genindex`
diff --git a/Documentation/infiniband/ipoib.txt b/Documentation/infiniband/ipoib.rst
index 47c1dd9818f2..0dd36154c0c9 100644
--- a/Documentation/infiniband/ipoib.txt
+++ b/Documentation/infiniband/ipoib.rst
@@ -1,4 +1,6 @@
-IP OVER INFINIBAND
+==================
+IP over InfiniBand
+==================
The ib_ipoib driver is an implementation of the IP over InfiniBand
protocol as specified by RFC 4391 and 4392, issued by the IETF ipoib
@@ -8,16 +10,17 @@ IP OVER INFINIBAND
masqueraded to the kernel as ethernet interfaces).
Partitions and P_Keys
+=====================
When the IPoIB driver is loaded, it creates one interface for each
port using the P_Key at index 0. To create an interface with a
different P_Key, write the desired P_Key into the main interface's
- /sys/class/net/<intf name>/create_child file. For example:
+ /sys/class/net/<intf name>/create_child file. For example::
echo 0x8001 > /sys/class/net/ib0/create_child
This will create an interface named ib0.8001 with P_Key 0x8001. To
- remove a subinterface, use the "delete_child" file:
+ remove a subinterface, use the "delete_child" file::
echo 0x8001 > /sys/class/net/ib0/delete_child
@@ -28,6 +31,7 @@ Partitions and P_Keys
rtnl_link_ops, where children created using either way behave the same.
Datagram vs Connected modes
+===========================
The IPoIB driver supports two modes of operation: datagram and
connected. The mode is set and read through an interface's
@@ -51,6 +55,7 @@ Datagram vs Connected modes
networking stack to use the smaller UD MTU for these neighbours.
Stateless offloads
+==================
If the IB HW supports IPoIB stateless offloads, IPoIB advertises
TCP/IP checksum and/or Large Send (LSO) offloading capability to the
@@ -60,9 +65,10 @@ Stateless offloads
on/off using ethtool calls. Currently LRO is supported only for
checksum offload capable devices.
- Stateless offloads are supported only in datagram mode.
+ Stateless offloads are supported only in datagram mode.
Interrupt moderation
+====================
If the underlying IB device supports CQ event moderation, one can
use ethtool to set interrupt mitigation parameters and thus reduce
@@ -71,6 +77,7 @@ Interrupt moderation
moderation is supported.
Debugging Information
+=====================
By compiling the IPoIB driver with CONFIG_INFINIBAND_IPOIB_DEBUG set
to 'y', tracing messages are compiled into the driver. They are
@@ -79,7 +86,7 @@ Debugging Information
runtime through files in /sys/module/ib_ipoib/.
CONFIG_INFINIBAND_IPOIB_DEBUG also enables files in the debugfs
- virtual filesystem. By mounting this filesystem, for example with
+ virtual filesystem. By mounting this filesystem, for example with::
mount -t debugfs none /sys/kernel/debug
@@ -96,10 +103,13 @@ Debugging Information
performance, because it adds tests to the fast path.
References
+==========
Transmission of IP over InfiniBand (IPoIB) (RFC 4391)
- http://ietf.org/rfc/rfc4391.txt
+ http://ietf.org/rfc/rfc4391.txt
+
IP over InfiniBand (IPoIB) Architecture (RFC 4392)
- http://ietf.org/rfc/rfc4392.txt
+ http://ietf.org/rfc/rfc4392.txt
+
IP over InfiniBand: Connected Mode (RFC 4755)
http://ietf.org/rfc/rfc4755.txt
diff --git a/Documentation/infiniband/opa_vnic.txt b/Documentation/infiniband/opa_vnic.rst
index 282e17be798a..2f888d9ffec0 100644
--- a/Documentation/infiniband/opa_vnic.txt
+++ b/Documentation/infiniband/opa_vnic.rst
@@ -1,3 +1,7 @@
+=================================================================
+Intel Omni-Path (OPA) Virtual Network Interface Controller (VNIC)
+=================================================================
+
Intel Omni-Path (OPA) Virtual Network Interface Controller (VNIC) feature
supports Ethernet functionality over Omni-Path fabric by encapsulating
the Ethernet packets between HFI nodes.
@@ -17,70 +21,72 @@ an independent Ethernet network. The configuration is performed by an
Ethernet Manager (EM) which is part of the trusted Fabric Manager (FM)
application. HFI nodes can have multiple VNICs each connected to a
different virtual Ethernet switch. The below diagram presents a case
-of two virtual Ethernet switches with two HFI nodes.
-
- +-------------------+
- | Subnet/ |
- | Ethernet |
- | Manager |
- +-------------------+
- / /
- / /
- / /
- / /
-+-----------------------------+ +------------------------------+
-| Virtual Ethernet Switch | | Virtual Ethernet Switch |
-| +---------+ +---------+ | | +---------+ +---------+ |
-| | VPORT | | VPORT | | | | VPORT | | VPORT | |
-+--+---------+----+---------+-+ +-+---------+----+---------+---+
- | \ / |
- | \ / |
- | \/ |
- | / \ |
- | / \ |
- +-----------+------------+ +-----------+------------+
- | VNIC | VNIC | | VNIC | VNIC |
- +-----------+------------+ +-----------+------------+
- | HFI | | HFI |
- +------------------------+ +------------------------+
+of two virtual Ethernet switches with two HFI nodes::
+
+ +-------------------+
+ | Subnet/ |
+ | Ethernet |
+ | Manager |
+ +-------------------+
+ / /
+ / /
+ / /
+ / /
+ +-----------------------------+ +------------------------------+
+ | Virtual Ethernet Switch | | Virtual Ethernet Switch |
+ | +---------+ +---------+ | | +---------+ +---------+ |
+ | | VPORT | | VPORT | | | | VPORT | | VPORT | |
+ +--+---------+----+---------+-+ +-+---------+----+---------+---+
+ | \ / |
+ | \ / |
+ | \/ |
+ | / \ |
+ | / \ |
+ +-----------+------------+ +-----------+------------+
+ | VNIC | VNIC | | VNIC | VNIC |
+ +-----------+------------+ +-----------+------------+
+ | HFI | | HFI |
+ +------------------------+ +------------------------+
The Omni-Path encapsulated Ethernet packet format is as described below.
-Bits Field
-------------------------------------
+==================== ================================
+Bits Field
+==================== ================================
Quad Word 0:
-0-19 SLID (lower 20 bits)
-20-30 Length (in Quad Words)
-31 BECN bit
-32-51 DLID (lower 20 bits)
-52-56 SC (Service Class)
-57-59 RC (Routing Control)
-60 FECN bit
-61-62 L2 (=10, 16B format)
-63 LT (=1, Link Transfer Head Flit)
+0-19 SLID (lower 20 bits)
+20-30 Length (in Quad Words)
+31 BECN bit
+32-51 DLID (lower 20 bits)
+52-56 SC (Service Class)
+57-59 RC (Routing Control)
+60 FECN bit
+61-62 L2 (=10, 16B format)
+63 LT (=1, Link Transfer Head Flit)
Quad Word 1:
-0-7 L4 type (=0x78 ETHERNET)
-8-11 SLID[23:20]
-12-15 DLID[23:20]
-16-31 PKEY
-32-47 Entropy
-48-63 Reserved
+0-7 L4 type (=0x78 ETHERNET)
+8-11 SLID[23:20]
+12-15 DLID[23:20]
+16-31 PKEY
+32-47 Entropy
+48-63 Reserved
Quad Word 2:
-0-15 Reserved
-16-31 L4 header
-32-63 Ethernet Packet
+0-15 Reserved
+16-31 L4 header
+32-63 Ethernet Packet
Quad Words 3 to N-1:
-0-63 Ethernet packet (pad extended)
+0-63 Ethernet packet (pad extended)
Quad Word N (last):
-0-23 Ethernet packet (pad extended)
-24-55 ICRC
-56-61 Tail
-62-63 LT (=01, Link Transfer Tail Flit)
+0-23 Ethernet packet (pad extended)
+24-55 ICRC
+56-61 Tail
+62-63 LT (=01, Link Transfer Tail Flit)
+==================== ================================
Ethernet packet is padded on the transmit side to ensure that the VNIC OPA
packet is quad word aligned. The 'Tail' field contains the number of bytes
@@ -123,7 +129,7 @@ operation. It also handles the encapsulation of Ethernet packets with an
Omni-Path header in the transmit path. For each VNIC interface, the
information required for encapsulation is configured by the EM via VEMA MAD
interface. It also passes any control information to the HW dependent driver
-by invoking the RDMA netdev control operations.
+by invoking the RDMA netdev control operations::
+-------------------+ +----------------------+
| | | Linux |
diff --git a/Documentation/infiniband/sysfs.txt b/Documentation/infiniband/sysfs.rst
index 9fab5062f84b..f0abd6fa48f4 100644
--- a/Documentation/infiniband/sysfs.txt
+++ b/Documentation/infiniband/sysfs.rst
@@ -1,4 +1,6 @@
-SYSFS FILES
+===========
+Sysfs files
+===========
The sysfs interface has moved to
Documentation/ABI/stable/sysfs-class-infiniband.
diff --git a/Documentation/infiniband/tag_matching.txt b/Documentation/infiniband/tag_matching.rst
index d2a3bf819226..ef56ea585f92 100644
--- a/Documentation/infiniband/tag_matching.txt
+++ b/Documentation/infiniband/tag_matching.rst
@@ -1,12 +1,16 @@
+==================
Tag matching logic
+==================
The MPI standard defines a set of rules, known as tag-matching, for matching
source send operations to destination receives. The following parameters must
match the following source and destination parameters:
+
* Communicator
* User tag - wild card may be specified by the receiver
* Source rank – wild car may be specified by the receiver
* Destination rank – wild
+
The ordering rules require that when more than one pair of send and receive
message envelopes may match, the pair that includes the earliest posted-send
and the earliest posted-receive is the pair that must be used to satisfy the
@@ -35,6 +39,7 @@ the header to initiate an RDMA READ operation directly to the matching buffer.
A fin message needs to be received in order for the buffer to be reused.
Tag matching implementation
+===========================
There are two types of matching objects used, the posted receive list and the
unexpected message list. The application posts receive buffers through calls
diff --git a/Documentation/infiniband/user_mad.txt b/Documentation/infiniband/user_mad.rst
index 7aca13a54a3a..d88abfc0e370 100644
--- a/Documentation/infiniband/user_mad.txt
+++ b/Documentation/infiniband/user_mad.rst
@@ -1,6 +1,9 @@
-USERSPACE MAD ACCESS
+====================
+Userspace MAD access
+====================
Device files
+============
Each port of each InfiniBand device has a "umad" device and an
"issm" device attached. For example, a two-port HCA will have two
@@ -8,12 +11,13 @@ Device files
device of each type (for switch port 0).
Creating MAD agents
+===================
A MAD agent can be created by filling in a struct ib_user_mad_reg_req
and then calling the IB_USER_MAD_REGISTER_AGENT ioctl on a file
descriptor for the appropriate device file. If the registration
request succeeds, a 32-bit id will be returned in the structure.
- For example:
+ For example::
struct ib_user_mad_reg_req req = { /* ... */ };
ret = ioctl(fd, IB_USER_MAD_REGISTER_AGENT, (char *) &req);
@@ -26,12 +30,14 @@ Creating MAD agents
ioctl. Also, all agents registered through a file descriptor will
be unregistered when the descriptor is closed.
- 2014 -- a new registration ioctl is now provided which allows additional
+ 2014
+ a new registration ioctl is now provided which allows additional
fields to be provided during registration.
Users of this registration call are implicitly setting the use of
pkey_index (see below).
Receiving MADs
+==============
MADs are received using read(). The receive side now supports
RMPP. The buffer passed to read() must be at least one
@@ -41,7 +47,8 @@ Receiving MADs
MAD (RMPP), the errno is set to ENOSPC and the length of the
buffer needed is set in mad.length.
- Example for normal MAD (non RMPP) reads:
+ Example for normal MAD (non RMPP) reads::
+
struct ib_user_mad *mad;
mad = malloc(sizeof *mad + 256);
ret = read(fd, mad, sizeof *mad + 256);
@@ -50,7 +57,8 @@ Receiving MADs
free(mad);
}
- Example for RMPP reads:
+ Example for RMPP reads::
+
struct ib_user_mad *mad;
mad = malloc(sizeof *mad + 256);
ret = read(fd, mad, sizeof *mad + 256);
@@ -76,11 +84,12 @@ Receiving MADs
poll()/select() may be used to wait until a MAD can be read.
Sending MADs
+============
MADs are sent using write(). The agent ID for sending should be
filled into the id field of the MAD, the destination LID should be
filled into the lid field, and so on. The send side does support
- RMPP so arbitrary length MAD can be sent. For example:
+ RMPP so arbitrary length MAD can be sent. For example::
struct ib_user_mad *mad;
@@ -97,6 +106,7 @@ Sending MADs
perror("write");
Transaction IDs
+===============
Users of the umad devices can use the lower 32 bits of the
transaction ID field (that is, the least significant half of the
@@ -105,6 +115,7 @@ Transaction IDs
the kernel and will be overwritten before a MAD is sent.
P_Key Index Handling
+====================
The old ib_umad interface did not allow setting the P_Key index for
MADs that are sent and did not provide a way for obtaining the P_Key
@@ -119,6 +130,7 @@ P_Key Index Handling
default, and the IB_USER_MAD_ENABLE_PKEY ioctl will be removed.
Setting IsSM Capability Bit
+===========================
To set the IsSM capability bit for a port, simply open the
corresponding issm device file. If the IsSM bit is already set,
@@ -129,25 +141,26 @@ Setting IsSM Capability Bit
the issm file.
/dev files
+==========
To create the appropriate character device files automatically with
- udev, a rule like
+ udev, a rule like::
KERNEL=="umad*", NAME="infiniband/%k"
KERNEL=="issm*", NAME="infiniband/%k"
- can be used. This will create device nodes named
+ can be used. This will create device nodes named::
/dev/infiniband/umad0
/dev/infiniband/issm0
for the first port, and so on. The InfiniBand device and port
- associated with these devices can be determined from the files
+ associated with these devices can be determined from the files::
/sys/class/infiniband_mad/umad0/ibdev
/sys/class/infiniband_mad/umad0/port
- and
+ and::
/sys/class/infiniband_mad/issm0/ibdev
/sys/class/infiniband_mad/issm0/port
diff --git a/Documentation/infiniband/user_verbs.txt b/Documentation/infiniband/user_verbs.rst
index 47ebf2f80b2b..8ddc4b1cfef2 100644
--- a/Documentation/infiniband/user_verbs.txt
+++ b/Documentation/infiniband/user_verbs.rst
@@ -1,4 +1,6 @@
-USERSPACE VERBS ACCESS
+======================
+Userspace verbs access
+======================
The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_VERBS,
enables direct userspace access to IB hardware via "verbs," as
@@ -13,6 +15,7 @@ USERSPACE VERBS ACCESS
libmthca userspace driver be installed.
User-kernel communication
+=========================
Userspace communicates with the kernel for slow path, resource
management operations via the /dev/infiniband/uverbsN character
@@ -28,6 +31,7 @@ User-kernel communication
system call.
Resource management
+===================
Since creation and destruction of all IB resources is done by
commands passed through a file descriptor, the kernel can keep track
@@ -41,6 +45,7 @@ Resource management
prevent one process from touching another process's resources.
Memory pinning
+==============
Direct userspace I/O requires that memory regions that are potential
I/O targets be kept resident at the same physical address. The
@@ -54,13 +59,14 @@ Memory pinning
number of pages pinned by a process.
/dev files
+==========
To create the appropriate character device files automatically with
- udev, a rule like
+ udev, a rule like::
KERNEL=="uverbs*", NAME="infiniband/%k"
- can be used. This will create device nodes named
+ can be used. This will create device nodes named::
/dev/infiniband/uverbs0
diff --git a/MAINTAINERS b/MAINTAINERS
index d0ed735994a5..83a62b911692 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -10840,14 +10840,6 @@ F: driver/net/net_failover.c
F: include/net/net_failover.h
F: Documentation/networking/net_failover.rst
-NETEFFECT IWARP RNIC DRIVER (IW_NES)
-M: Faisal Latif <faisal.latif@intel.com>
-L: linux-rdma@vger.kernel.org
-W: http://www.intel.com/Products/Server/Adapters/Server-Cluster/Server-Cluster-overview.htm
-S: Supported
-F: drivers/infiniband/hw/nes/
-F: include/uapi/rdma/nes-abi.h
-
NETEM NETWORK EMULATOR
M: Stephen Hemminger <stephen@networkplumber.org>
L: netem@lists.linux-foundation.org (moderated for non-subscribers)
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index 8ba41cbf1869..42af4cd40ba2 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -36,17 +36,6 @@ config INFINIBAND_USER_ACCESS
libibverbs, libibcm and a hardware driver library from
rdma-core <https://github.com/linux-rdma/rdma-core>.
-config INFINIBAND_USER_ACCESS_UCM
- tristate "Userspace CM (UCM, DEPRECATED)"
- depends on BROKEN || COMPILE_TEST
- depends on INFINIBAND_USER_ACCESS
- help
- The UCM module has known security flaws, which no one is
- interested to fix. The user-space part of this code was
- dropped from the upstream a long time ago.
-
- This option is DEPRECATED and planned to be removed.
-
config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI
bool "Allow experimental legacy verbs in new ioctl uAPI (EXPERIMENTAL)"
depends on INFINIBAND_USER_ACCESS
@@ -98,7 +87,6 @@ source "drivers/infiniband/hw/efa/Kconfig"
source "drivers/infiniband/hw/i40iw/Kconfig"
source "drivers/infiniband/hw/mlx4/Kconfig"
source "drivers/infiniband/hw/mlx5/Kconfig"
-source "drivers/infiniband/hw/nes/Kconfig"
source "drivers/infiniband/hw/ocrdma/Kconfig"
source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
source "drivers/infiniband/hw/usnic/Kconfig"
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index 313f2349b518..42f1b2a4f746 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -6,7 +6,6 @@ obj-$(CONFIG_INFINIBAND) += ib_core.o ib_cm.o iw_cm.o \
$(infiniband-y)
obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o
obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)
-obj-$(CONFIG_INFINIBAND_USER_ACCESS_UCM) += ib_ucm.o $(user_access-y)
ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
device.o fmr_pool.o cache.o netlink.o \
@@ -29,8 +28,6 @@ rdma_ucm-y := ucma.o
ib_umad-y := user_mad.o
-ib_ucm-y := ucm.o
-
ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
rdma_core.o uverbs_std_types.o uverbs_ioctl.o \
uverbs_std_types_cq.o \
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index ff40a450b5d2..a953c2fa2e78 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -88,6 +88,15 @@ typedef int (*nldev_callback)(struct ib_device *device,
int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
struct netlink_callback *cb);
+struct ib_client_nl_info {
+ struct sk_buff *nl_msg;
+ struct device *cdev;
+ unsigned int port;
+ u64 abi;
+};
+int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,
+ struct ib_client_nl_info *res);
+
enum ib_cache_gid_default_mode {
IB_CACHE_GID_DEFAULT_MODE_SET,
IB_CACHE_GID_DEFAULT_MODE_DELETE
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index a24c900fbdf6..00d70f166209 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -113,7 +113,7 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
}
/**
- * __ib_alloc_cq - allocate a completion queue
+ * __ib_alloc_cq_user - allocate a completion queue
* @dev: device to allocate the CQ for
* @private: driver private data, accessible from cq->cq_context
* @nr_cqe: number of CQEs to allocate
@@ -139,23 +139,26 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
struct ib_cq *cq;
int ret = -ENOMEM;
- cq = dev->ops.create_cq(dev, &cq_attr, NULL);
- if (IS_ERR(cq))
- return cq;
+ cq = rdma_zalloc_drv_obj(dev, ib_cq);
+ if (!cq)
+ return ERR_PTR(ret);
cq->device = dev;
- cq->uobject = NULL;
- cq->event_handler = NULL;
cq->cq_context = private;
cq->poll_ctx = poll_ctx;
atomic_set(&cq->usecnt, 0);
cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
if (!cq->wc)
- goto out_destroy_cq;
+ goto out_free_cq;
cq->res.type = RDMA_RESTRACK_CQ;
rdma_restrack_set_task(&cq->res, caller);
+
+ ret = dev->ops.create_cq(cq, &cq_attr, NULL);
+ if (ret)
+ goto out_free_wc;
+
rdma_restrack_kadd(&cq->res);
switch (cq->poll_ctx) {
@@ -178,29 +181,29 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
break;
default:
ret = -EINVAL;
- goto out_free_wc;
+ goto out_destroy_cq;
}
return cq;
-out_free_wc:
- kfree(cq->wc);
- rdma_restrack_del(&cq->res);
out_destroy_cq:
+ rdma_restrack_del(&cq->res);
cq->device->ops.destroy_cq(cq, udata);
+out_free_wc:
+ kfree(cq->wc);
+out_free_cq:
+ kfree(cq);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(__ib_alloc_cq_user);
/**
- * ib_free_cq - free a completion queue
+ * ib_free_cq_user - free a completion queue
* @cq: completion queue to free.
* @udata: User data or NULL for kernel object
*/
void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
{
- int ret;
-
if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
return;
@@ -218,9 +221,9 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
WARN_ON_ONCE(1);
}
- kfree(cq->wc);
rdma_restrack_del(&cq->res);
- ret = cq->device->ops.destroy_cq(cq, udata);
- WARN_ON_ONCE(ret);
+ cq->device->ops.destroy_cq(cq, udata);
+ kfree(cq->wc);
+ kfree(cq);
}
EXPORT_SYMBOL(ib_free_cq_user);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 29f7b15c81d9..8a6ccb936dfe 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -270,7 +270,7 @@ struct ib_port_data_rcu {
struct ib_port_data pdata[];
};
-static int ib_device_check_mandatory(struct ib_device *device)
+static void ib_device_check_mandatory(struct ib_device *device)
{
#define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }
static const struct {
@@ -305,8 +305,6 @@ static int ib_device_check_mandatory(struct ib_device *device)
break;
}
}
-
- return 0;
}
/*
@@ -375,7 +373,7 @@ struct ib_device *ib_device_get_by_name(const char *name,
down_read(&devices_rwsem);
device = __ib_device_get_by_name(name);
if (device && driver_id != RDMA_DRIVER_UNKNOWN &&
- device->driver_id != driver_id)
+ device->ops.driver_id != driver_id)
device = NULL;
if (device) {
@@ -1193,10 +1191,7 @@ static int setup_device(struct ib_device *device)
int ret;
setup_dma_device(device);
-
- ret = ib_device_check_mandatory(device);
- if (ret)
- return ret;
+ ib_device_check_mandatory(device);
ret = setup_port_data(device);
if (ret) {
@@ -1479,7 +1474,7 @@ void ib_unregister_driver(enum rdma_driver_id driver_id)
down_read(&devices_rwsem);
xa_for_each (&devices, index, ib_dev) {
- if (ib_dev->driver_id != driver_id)
+ if (ib_dev->ops.driver_id != driver_id)
continue;
get_device(&ib_dev->dev);
@@ -1749,6 +1744,104 @@ void ib_unregister_client(struct ib_client *client)
}
EXPORT_SYMBOL(ib_unregister_client);
+static int __ib_get_global_client_nl_info(const char *client_name,
+ struct ib_client_nl_info *res)
+{
+ struct ib_client *client;
+ unsigned long index;
+ int ret = -ENOENT;
+
+ down_read(&clients_rwsem);
+ xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
+ if (strcmp(client->name, client_name) != 0)
+ continue;
+ if (!client->get_global_nl_info) {
+ ret = -EOPNOTSUPP;
+ break;
+ }
+ ret = client->get_global_nl_info(res);
+ if (WARN_ON(ret == -ENOENT))
+ ret = -EINVAL;
+ if (!ret && res->cdev)
+ get_device(res->cdev);
+ break;
+ }
+ up_read(&clients_rwsem);
+ return ret;
+}
+
+static int __ib_get_client_nl_info(struct ib_device *ibdev,
+ const char *client_name,
+ struct ib_client_nl_info *res)
+{
+ unsigned long index;
+ void *client_data;
+ int ret = -ENOENT;
+
+ down_read(&ibdev->client_data_rwsem);
+ xan_for_each_marked (&ibdev->client_data, index, client_data,
+ CLIENT_DATA_REGISTERED) {
+ struct ib_client *client = xa_load(&clients, index);
+
+ if (!client || strcmp(client->name, client_name) != 0)
+ continue;
+ if (!client->get_nl_info) {
+ ret = -EOPNOTSUPP;
+ break;
+ }
+ ret = client->get_nl_info(ibdev, client_data, res);
+ if (WARN_ON(ret == -ENOENT))
+ ret = -EINVAL;
+
+ /*
+ * The cdev is guaranteed valid as long as we are inside the
+ * client_data_rwsem as remove_one can't be called. Keep it
+ * valid for the caller.
+ */
+ if (!ret && res->cdev)
+ get_device(res->cdev);
+ break;
+ }
+ up_read(&ibdev->client_data_rwsem);
+
+ return ret;
+}
+
+/**
+ * ib_get_client_nl_info - Fetch the nl_info from a client
+ * @device - IB device
+ * @client_name - Name of the client
+ * @res - Result of the query
+ */
+int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,
+ struct ib_client_nl_info *res)
+{
+ int ret;
+
+ if (ibdev)
+ ret = __ib_get_client_nl_info(ibdev, client_name, res);
+ else
+ ret = __ib_get_global_client_nl_info(client_name, res);
+#ifdef CONFIG_MODULES
+ if (ret == -ENOENT) {
+ request_module("rdma-client-%s", client_name);
+ if (ibdev)
+ ret = __ib_get_client_nl_info(ibdev, client_name, res);
+ else
+ ret = __ib_get_global_client_nl_info(client_name, res);
+ }
+#endif
+ if (ret) {
+ if (ret == -ENOENT)
+ return -EOPNOTSUPP;
+ return ret;
+ }
+
+ if (WARN_ON(!res->cdev))
+ return -EINVAL;
+ return 0;
+}
+
/**
* ib_set_client_data - Set IB client context
* @device:Device to set context for
@@ -2039,7 +2132,7 @@ struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
(uintptr_t)ndev) {
if (rcu_access_pointer(cur->netdev) == ndev &&
(driver_id == RDMA_DRIVER_UNKNOWN ||
- cur->ib_dev->driver_id == driver_id) &&
+ cur->ib_dev->ops.driver_id == driver_id) &&
ib_device_try_get(cur->ib_dev)) {
res = cur->ib_dev;
break;
@@ -2344,12 +2437,28 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
#define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name)
+ if (ops->driver_id != RDMA_DRIVER_UNKNOWN) {
+ WARN_ON(dev_ops->driver_id != RDMA_DRIVER_UNKNOWN &&
+ dev_ops->driver_id != ops->driver_id);
+ dev_ops->driver_id = ops->driver_id;
+ }
+ if (ops->owner) {
+ WARN_ON(dev_ops->owner && dev_ops->owner != ops->owner);
+ dev_ops->owner = ops->owner;
+ }
+ if (ops->uverbs_abi_ver)
+ dev_ops->uverbs_abi_ver = ops->uverbs_abi_ver;
+
+ dev_ops->uverbs_no_driver_id_binding |=
+ ops->uverbs_no_driver_id_binding;
+
SET_DEVICE_OP(dev_ops, add_gid);
SET_DEVICE_OP(dev_ops, advise_mr);
SET_DEVICE_OP(dev_ops, alloc_dm);
SET_DEVICE_OP(dev_ops, alloc_fmr);
SET_DEVICE_OP(dev_ops, alloc_hw_stats);
SET_DEVICE_OP(dev_ops, alloc_mr);
+ SET_DEVICE_OP(dev_ops, alloc_mr_integrity);
SET_DEVICE_OP(dev_ops, alloc_mw);
SET_DEVICE_OP(dev_ops, alloc_pd);
SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);
@@ -2409,6 +2518,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, iw_reject);
SET_DEVICE_OP(dev_ops, iw_rem_ref);
SET_DEVICE_OP(dev_ops, map_mr_sg);
+ SET_DEVICE_OP(dev_ops, map_mr_sg_pi);
SET_DEVICE_OP(dev_ops, map_phys_fmr);
SET_DEVICE_OP(dev_ops, mmap);
SET_DEVICE_OP(dev_ops, modify_ah);
@@ -2445,6 +2555,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, unmap_fmr);
SET_OBJ_SIZE(dev_ops, ib_ah);
+ SET_OBJ_SIZE(dev_ops, ib_cq);
SET_OBJ_SIZE(dev_ops, ib_pd);
SET_OBJ_SIZE(dev_ops, ib_srq);
SET_OBJ_SIZE(dev_ops, ib_ucontext);
diff --git a/drivers/infiniband/core/mr_pool.c b/drivers/infiniband/core/mr_pool.c
index d117f21ce9fd..c0e2df128b34 100644
--- a/drivers/infiniband/core/mr_pool.c
+++ b/drivers/infiniband/core/mr_pool.c
@@ -34,14 +34,18 @@ void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr)
EXPORT_SYMBOL(ib_mr_pool_put);
int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,
- enum ib_mr_type type, u32 max_num_sg)
+ enum ib_mr_type type, u32 max_num_sg, u32 max_num_meta_sg)
{
struct ib_mr *mr;
unsigned long flags;
int ret, i;
for (i = 0; i < nr; i++) {
- mr = ib_alloc_mr(qp->pd, type, max_num_sg);
+ if (type == IB_MR_TYPE_INTEGRITY)
+ mr = ib_alloc_mr_integrity(qp->pd, max_num_sg,
+ max_num_meta_sg);
+ else
+ mr = ib_alloc_mr(qp->pd, type, max_num_sg);
if (IS_ERR(mr)) {
ret = PTR_ERR(mr);
goto out;
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 69188cbbd99b..5499f5629dc2 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -42,84 +42,94 @@
#include "cma_priv.h"
#include "restrack.h"
+/*
+ * Sort array elements by the netlink attribute name
+ */
static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
- [RDMA_NLDEV_ATTR_DEV_INDEX] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING,
- .len = IB_DEVICE_NAME_MAX - 1},
- [RDMA_NLDEV_ATTR_PORT_INDEX] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_FW_VERSION] = { .type = NLA_NUL_STRING,
- .len = IB_FW_VERSION_NAME_MAX - 1},
- [RDMA_NLDEV_ATTR_NODE_GUID] = { .type = NLA_U64 },
- [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID] = { .type = NLA_U64 },
- [RDMA_NLDEV_ATTR_SUBNET_PREFIX] = { .type = NLA_U64 },
- [RDMA_NLDEV_ATTR_LID] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_SM_LID] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_LMC] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_PORT_STATE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_RES_SUMMARY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,
- .len = 16 },
- [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },
- [RDMA_NLDEV_ATTR_RES_QP] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_QP_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_LQPN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_RQPN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_RQ_PSN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_SQ_PSN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_RES_STATE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_RES_PID] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_KERN_NAME] = { .type = NLA_NUL_STRING,
- .len = TASK_COMM_LEN },
+ [RDMA_NLDEV_ATTR_CHARDEV] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_CHARDEV_ABI] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_CHARDEV_NAME] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_CHARDEV_TYPE] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE },
+ [RDMA_NLDEV_ATTR_DEV_INDEX] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING,
+ .len = IB_DEVICE_NAME_MAX },
+ [RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_DEV_PROTOCOL] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_DRIVER] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_DRIVER_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_DRIVER_STRING] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_DRIVER_S32] = { .type = NLA_S32 },
+ [RDMA_NLDEV_ATTR_DRIVER_S64] = { .type = NLA_S64 },
+ [RDMA_NLDEV_ATTR_DRIVER_U32] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_DRIVER_U64] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_FW_VERSION] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_LID] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_LINK_TYPE] = { .type = NLA_NUL_STRING,
+ .len = IFNAMSIZ },
+ [RDMA_NLDEV_ATTR_LMC] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_NDEV_INDEX] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_NDEV_NAME] = { .type = NLA_NUL_STRING,
+ .len = IFNAMSIZ },
+ [RDMA_NLDEV_ATTR_NODE_GUID] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_PORT_INDEX] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_PORT_STATE] = { .type = NLA_U8 },
[RDMA_NLDEV_ATTR_RES_CM_ID] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_CM_IDN] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_PS] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = {
- .len = sizeof(struct __kernel_sockaddr_storage) },
- [RDMA_NLDEV_ATTR_RES_DST_ADDR] = {
- .len = sizeof(struct __kernel_sockaddr_storage) },
[RDMA_NLDEV_ATTR_RES_CQ] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = { .type = NLA_NESTED },
[RDMA_NLDEV_ATTR_RES_CQE] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 },
- [RDMA_NLDEV_ATTR_RES_POLL_CTX] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_RES_MR] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_LKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_CQN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_CTXN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_DST_ADDR] = {
+ .len = sizeof(struct __kernel_sockaddr_storage) },
[RDMA_NLDEV_ATTR_RES_IOVA] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_RES_KERN_NAME] = { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_RES_LKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_LQPN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_MR] = { .type = NLA_NESTED },
[RDMA_NLDEV_ATTR_RES_MRLEN] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_RES_MRN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE] = { .type = NLA_U8 },
[RDMA_NLDEV_ATTR_RES_PD] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_PD_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_NDEV_INDEX] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_NDEV_NAME] = { .type = NLA_NUL_STRING,
- .len = IFNAMSIZ },
- [RDMA_NLDEV_ATTR_DRIVER] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_DRIVER_ENTRY] = { .type = NLA_NESTED },
- [RDMA_NLDEV_ATTR_DRIVER_STRING] = { .type = NLA_NUL_STRING,
- .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
- [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_DRIVER_S32] = { .type = NLA_S32 },
- [RDMA_NLDEV_ATTR_DRIVER_U32] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_DRIVER_S64] = { .type = NLA_S64 },
- [RDMA_NLDEV_ATTR_DRIVER_U64] = { .type = NLA_U64 },
[RDMA_NLDEV_ATTR_RES_PDN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_CQN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_MRN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_CM_IDN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_RES_CTXN] = { .type = NLA_U32 },
- [RDMA_NLDEV_ATTR_LINK_TYPE] = { .type = NLA_NUL_STRING,
- .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
- [RDMA_NLDEV_SYS_ATTR_NETNS_MODE] = { .type = NLA_U8 },
- [RDMA_NLDEV_ATTR_DEV_PROTOCOL] = { .type = NLA_NUL_STRING,
- .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
+ [RDMA_NLDEV_ATTR_RES_PD_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_PID] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_POLL_CTX] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_RES_PS] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_QP] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_QP_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_RQPN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_RQ_PSN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_SQ_PSN] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = {
+ .len = sizeof(struct __kernel_sockaddr_storage) },
+ [RDMA_NLDEV_ATTR_RES_STATE] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_RES_SUMMARY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,
+ .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
+ [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_SM_LID] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_SUBNET_PREFIX] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID] = { .type = NLA_U32 },
[RDMA_NLDEV_NET_NS_FD] = { .type = NLA_U32 },
+ [RDMA_NLDEV_SYS_ATTR_NETNS_MODE] = { .type = NLA_U8 },
};
static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -1347,6 +1357,90 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
return 0;
}
+static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+ char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE];
+ struct ib_client_nl_info data = {};
+ struct ib_device *ibdev = NULL;
+ struct sk_buff *msg;
+ u32 index;
+ int err;
+
+ err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
+ extack);
+ if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE])
+ return -EINVAL;
+
+ nla_strlcpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE],
+ sizeof(client_name));
+
+ if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {
+ index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+ ibdev = ib_device_get_by_index(sock_net(skb->sk), index);
+ if (!ibdev)
+ return -EINVAL;
+
+ if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
+ data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
+ if (!rdma_is_port_valid(ibdev, data.port)) {
+ err = -EINVAL;
+ goto out_put;
+ }
+ } else {
+ data.port = -1;
+ }
+ } else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
+ return -EINVAL;
+ }
+
+ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!msg) {
+ err = -ENOMEM;
+ goto out_put;
+ }
+ nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+ RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+ RDMA_NLDEV_CMD_GET_CHARDEV),
+ 0, 0);
+
+ data.nl_msg = msg;
+ err = ib_get_client_nl_info(ibdev, client_name, &data);
+ if (err)
+ goto out_nlmsg;
+
+ err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV,
+ huge_encode_dev(data.cdev->devt),
+ RDMA_NLDEV_ATTR_PAD);
+ if (err)
+ goto out_data;
+ err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi,
+ RDMA_NLDEV_ATTR_PAD);
+ if (err)
+ goto out_data;
+ if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME,
+ dev_name(data.cdev))) {
+ err = -EMSGSIZE;
+ goto out_data;
+ }
+
+ nlmsg_end(msg, nlh);
+ put_device(data.cdev);
+ if (ibdev)
+ ib_device_put(ibdev);
+ return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+
+out_data:
+ put_device(data.cdev);
+out_nlmsg:
+ nlmsg_free(msg);
+out_put:
+ if (ibdev)
+ ib_device_put(ibdev);
+ return err;
+}
+
static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack)
{
@@ -1404,6 +1498,9 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
.doit = nldev_get_doit,
.dump = nldev_get_dumpit,
},
+ [RDMA_NLDEV_CMD_GET_CHARDEV] = {
+ .doit = nldev_get_chardev,
+ },
[RDMA_NLDEV_CMD_SET] = {
.doit = nldev_set_doit,
.flags = RDMA_NL_ADMIN_PERM,
diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index 32ca8429eaae..dce06108c8c3 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -51,10 +51,34 @@ static inline bool rdma_rw_io_needs_mr(struct ib_device *dev, u8 port_num,
return false;
}
-static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev)
+static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev,
+ bool pi_support)
{
+ u32 max_pages;
+
+ if (pi_support)
+ max_pages = dev->attrs.max_pi_fast_reg_page_list_len;
+ else
+ max_pages = dev->attrs.max_fast_reg_page_list_len;
+
/* arbitrary limit to avoid allocating gigantic resources */
- return min_t(u32, dev->attrs.max_fast_reg_page_list_len, 256);
+ return min_t(u32, max_pages, 256);
+}
+
+static inline int rdma_rw_inv_key(struct rdma_rw_reg_ctx *reg)
+{
+ int count = 0;
+
+ if (reg->mr->need_inval) {
+ reg->inv_wr.opcode = IB_WR_LOCAL_INV;
+ reg->inv_wr.ex.invalidate_rkey = reg->mr->lkey;
+ reg->inv_wr.next = &reg->reg_wr.wr;
+ count++;
+ } else {
+ reg->inv_wr.next = NULL;
+ }
+
+ return count;
}
/* Caller must have zero-initialized *reg. */
@@ -62,7 +86,8 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
struct rdma_rw_reg_ctx *reg, struct scatterlist *sg,
u32 sg_cnt, u32 offset)
{
- u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
+ u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device,
+ qp->integrity_en);
u32 nents = min(sg_cnt, pages_per_mr);
int count = 0, ret;
@@ -70,14 +95,7 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
if (!reg->mr)
return -EAGAIN;
- if (reg->mr->need_inval) {
- reg->inv_wr.opcode = IB_WR_LOCAL_INV;
- reg->inv_wr.ex.invalidate_rkey = reg->mr->lkey;
- reg->inv_wr.next = &reg->reg_wr.wr;
- count++;
- } else {
- reg->inv_wr.next = NULL;
- }
+ count += rdma_rw_inv_key(reg);
ret = ib_map_mr_sg(reg->mr, sg, nents, &offset, PAGE_SIZE);
if (ret < 0 || ret < nents) {
@@ -102,7 +120,8 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct rdma_rw_reg_ctx *prev = NULL;
- u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
+ u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device,
+ qp->integrity_en);
int i, j, ret = 0, count = 0;
ctx->nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr;
@@ -343,13 +362,14 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct ib_device *dev = qp->pd->device;
- u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
+ u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device,
+ qp->integrity_en);
struct ib_rdma_wr *rdma_wr;
- struct ib_send_wr *prev_wr = NULL;
int count = 0, ret;
if (sg_cnt > pages_per_mr || prot_sg_cnt > pages_per_mr) {
- pr_err("SG count too large\n");
+ pr_err("SG count too large: sg_cnt=%d, prot_sg_cnt=%d, pages_per_mr=%d\n",
+ sg_cnt, prot_sg_cnt, pages_per_mr);
return -EINVAL;
}
@@ -358,75 +378,58 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
return -ENOMEM;
sg_cnt = ret;
- ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir);
- if (!ret) {
- ret = -ENOMEM;
- goto out_unmap_sg;
+ if (prot_sg_cnt) {
+ ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir);
+ if (!ret) {
+ ret = -ENOMEM;
+ goto out_unmap_sg;
+ }
+ prot_sg_cnt = ret;
}
- prot_sg_cnt = ret;
ctx->type = RDMA_RW_SIG_MR;
ctx->nr_ops = 1;
- ctx->sig = kcalloc(1, sizeof(*ctx->sig), GFP_KERNEL);
- if (!ctx->sig) {
+ ctx->reg = kcalloc(1, sizeof(*ctx->reg), GFP_KERNEL);
+ if (!ctx->reg) {
ret = -ENOMEM;
goto out_unmap_prot_sg;
}
- ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->data, sg, sg_cnt, 0);
- if (ret < 0)
- goto out_free_ctx;
- count += ret;
- prev_wr = &ctx->sig->data.reg_wr.wr;
-
- ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->prot,
- prot_sg, prot_sg_cnt, 0);
- if (ret < 0)
- goto out_destroy_data_mr;
- count += ret;
-
- if (ctx->sig->prot.inv_wr.next)
- prev_wr->next = &ctx->sig->prot.inv_wr;
- else
- prev_wr->next = &ctx->sig->prot.reg_wr.wr;
- prev_wr = &ctx->sig->prot.reg_wr.wr;
-
- ctx->sig->sig_mr = ib_mr_pool_get(qp, &qp->sig_mrs);
- if (!ctx->sig->sig_mr) {
+ ctx->reg->mr = ib_mr_pool_get(qp, &qp->sig_mrs);
+ if (!ctx->reg->mr) {
ret = -EAGAIN;
- goto out_destroy_prot_mr;
+ goto out_free_ctx;
}
- if (ctx->sig->sig_mr->need_inval) {
- memset(&ctx->sig->sig_inv_wr, 0, sizeof(ctx->sig->sig_inv_wr));
+ count += rdma_rw_inv_key(ctx->reg);
- ctx->sig->sig_inv_wr.opcode = IB_WR_LOCAL_INV;
- ctx->sig->sig_inv_wr.ex.invalidate_rkey = ctx->sig->sig_mr->rkey;
+ memcpy(ctx->reg->mr->sig_attrs, sig_attrs, sizeof(struct ib_sig_attrs));
- prev_wr->next = &ctx->sig->sig_inv_wr;
- prev_wr = &ctx->sig->sig_inv_wr;
+ ret = ib_map_mr_sg_pi(ctx->reg->mr, sg, sg_cnt, NULL, prot_sg,
+ prot_sg_cnt, NULL, SZ_4K);
+ if (unlikely(ret)) {
+ pr_err("failed to map PI sg (%d)\n", sg_cnt + prot_sg_cnt);
+ goto out_destroy_sig_mr;
}
- ctx->sig->sig_wr.wr.opcode = IB_WR_REG_SIG_MR;
- ctx->sig->sig_wr.wr.wr_cqe = NULL;
- ctx->sig->sig_wr.wr.sg_list = &ctx->sig->data.sge;
- ctx->sig->sig_wr.wr.num_sge = 1;
- ctx->sig->sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE;
- ctx->sig->sig_wr.sig_attrs = sig_attrs;
- ctx->sig->sig_wr.sig_mr = ctx->sig->sig_mr;
- if (prot_sg_cnt)
- ctx->sig->sig_wr.prot = &ctx->sig->prot.sge;
- prev_wr->next = &ctx->sig->sig_wr.wr;
- prev_wr = &ctx->sig->sig_wr.wr;
+ ctx->reg->reg_wr.wr.opcode = IB_WR_REG_MR_INTEGRITY;
+ ctx->reg->reg_wr.wr.wr_cqe = NULL;
+ ctx->reg->reg_wr.wr.num_sge = 0;
+ ctx->reg->reg_wr.wr.send_flags = 0;
+ ctx->reg->reg_wr.access = IB_ACCESS_LOCAL_WRITE;
+ if (rdma_protocol_iwarp(qp->device, port_num))
+ ctx->reg->reg_wr.access |= IB_ACCESS_REMOTE_WRITE;
+ ctx->reg->reg_wr.mr = ctx->reg->mr;
+ ctx->reg->reg_wr.key = ctx->reg->mr->lkey;
count++;
- ctx->sig->sig_sge.addr = 0;
- ctx->sig->sig_sge.length = ctx->sig->data.sge.length;
- if (sig_attrs->wire.sig_type != IB_SIG_TYPE_NONE)
- ctx->sig->sig_sge.length += ctx->sig->prot.sge.length;
+ ctx->reg->sge.addr = ctx->reg->mr->iova;
+ ctx->reg->sge.length = ctx->reg->mr->length;
+ if (sig_attrs->wire.sig_type == IB_SIG_TYPE_NONE)
+ ctx->reg->sge.length -= ctx->reg->mr->sig_attrs->meta_length;
- rdma_wr = &ctx->sig->data.wr;
- rdma_wr->wr.sg_list = &ctx->sig->sig_sge;
+ rdma_wr = &ctx->reg->wr;
+ rdma_wr->wr.sg_list = &ctx->reg->sge;
rdma_wr->wr.num_sge = 1;
rdma_wr->remote_addr = remote_addr;
rdma_wr->rkey = rkey;
@@ -434,21 +437,18 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
else
rdma_wr->wr.opcode = IB_WR_RDMA_READ;
- prev_wr->next = &rdma_wr->wr;
- prev_wr = &rdma_wr->wr;
+ ctx->reg->reg_wr.wr.next = &rdma_wr->wr;
count++;
return count;
-out_destroy_prot_mr:
- if (prot_sg_cnt)
- ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr);
-out_destroy_data_mr:
- ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr);
+out_destroy_sig_mr:
+ ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr);
out_free_ctx:
- kfree(ctx->sig);
+ kfree(ctx->reg);
out_unmap_prot_sg:
- ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
+ if (prot_sg_cnt)
+ ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
out_unmap_sg:
ib_dma_unmap_sg(dev, sg, sg_cnt, dir);
return ret;
@@ -491,22 +491,8 @@ struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
switch (ctx->type) {
case RDMA_RW_SIG_MR:
- rdma_rw_update_lkey(&ctx->sig->data, true);
- if (ctx->sig->prot.mr)
- rdma_rw_update_lkey(&ctx->sig->prot, true);
-
- ctx->sig->sig_mr->need_inval = true;
- ib_update_fast_reg_key(ctx->sig->sig_mr,
- ib_inc_rkey(ctx->sig->sig_mr->lkey));
- ctx->sig->sig_sge.lkey = ctx->sig->sig_mr->lkey;
-
- if (ctx->sig->data.inv_wr.next)
- first_wr = &ctx->sig->data.inv_wr;
- else
- first_wr = &ctx->sig->data.reg_wr.wr;
- last_wr = &ctx->sig->data.wr.wr;
- break;
case RDMA_RW_MR:
+ /* fallthrough */
for (i = 0; i < ctx->nr_ops; i++) {
rdma_rw_update_lkey(&ctx->reg[i],
ctx->reg[i].wr.wr.opcode !=
@@ -605,7 +591,7 @@ EXPORT_SYMBOL(rdma_rw_ctx_destroy);
/**
* rdma_rw_ctx_destroy_signature - release all resources allocated by
- * rdma_rw_ctx_init_signature
+ * rdma_rw_ctx_signature_init
* @ctx: context to release
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
@@ -623,16 +609,12 @@ void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
if (WARN_ON_ONCE(ctx->type != RDMA_RW_SIG_MR))
return;
- ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr);
- ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
+ ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr);
+ kfree(ctx->reg);
- if (ctx->sig->prot.mr) {
- ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr);
+ ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
+ if (prot_sg_cnt)
ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir);
- }
-
- ib_mr_pool_put(qp, &qp->sig_mrs, ctx->sig->sig_mr);
- kfree(ctx->sig);
}
EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);
@@ -653,7 +635,7 @@ unsigned int rdma_rw_mr_factor(struct ib_device *device, u8 port_num,
unsigned int mr_pages;
if (rdma_rw_can_use_mr(device, port_num))
- mr_pages = rdma_rw_fr_page_list_len(device);
+ mr_pages = rdma_rw_fr_page_list_len(device, false);
else
mr_pages = device->attrs.max_sge_rd;
return DIV_ROUND_UP(maxpages, mr_pages);
@@ -679,9 +661,8 @@ void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)
* we'll need two additional MRs for the registrations and the
* invalidation.
*/
- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
- factor += 6; /* (inv + reg) * (data + prot + sig) */
- else if (rdma_rw_can_use_mr(dev, attr->port_num))
+ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN ||
+ rdma_rw_can_use_mr(dev, attr->port_num))
factor += 2; /* inv + reg */
attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs;
@@ -697,20 +678,22 @@ void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)
int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)
{
struct ib_device *dev = qp->pd->device;
- u32 nr_mrs = 0, nr_sig_mrs = 0;
+ u32 nr_mrs = 0, nr_sig_mrs = 0, max_num_sg = 0;
int ret = 0;
- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) {
+ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) {
nr_sig_mrs = attr->cap.max_rdma_ctxs;
- nr_mrs = attr->cap.max_rdma_ctxs * 2;
+ nr_mrs = attr->cap.max_rdma_ctxs;
+ max_num_sg = rdma_rw_fr_page_list_len(dev, true);
} else if (rdma_rw_can_use_mr(dev, attr->port_num)) {
nr_mrs = attr->cap.max_rdma_ctxs;
+ max_num_sg = rdma_rw_fr_page_list_len(dev, false);
}
if (nr_mrs) {
ret = ib_mr_pool_init(qp, &qp->rdma_mrs, nr_mrs,
IB_MR_TYPE_MEM_REG,
- rdma_rw_fr_page_list_len(dev));
+ max_num_sg, 0);
if (ret) {
pr_err("%s: failed to allocated %d MRs\n",
__func__, nr_mrs);
@@ -720,10 +703,10 @@ int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)
if (nr_sig_mrs) {
ret = ib_mr_pool_init(qp, &qp->sig_mrs, nr_sig_mrs,
- IB_MR_TYPE_SIGNATURE, 2);
+ IB_MR_TYPE_INTEGRITY, max_num_sg, max_num_sg);
if (ret) {
pr_err("%s: failed to allocated %d SIG MRs\n",
- __func__, nr_mrs);
+ __func__, nr_sig_mrs);
goto out_free_rdma_mrs;
}
}
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
deleted file mode 100644
index 8e7da2d41fd8..000000000000
--- a/drivers/infiniband/core/ucm.c
+++ /dev/null
@@ -1,1350 +0,0 @@
-/*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/completion.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/err.h>
-#include <linux/poll.h>
-#include <linux/sched.h>
-#include <linux/file.h>
-#include <linux/mount.h>
-#include <linux/cdev.h>
-#include <linux/xarray.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-
-#include <linux/nospec.h>
-
-#include <linux/uaccess.h>
-
-#include <rdma/ib.h>
-#include <rdma/ib_cm.h>
-#include <rdma/ib_user_cm.h>
-#include <rdma/ib_marshall.h>
-
-#include "core_priv.h"
-
-MODULE_AUTHOR("Libor Michalek");
-MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");
-MODULE_LICENSE("Dual BSD/GPL");
-
-struct ib_ucm_device {
- int devnum;
- struct cdev cdev;
- struct device dev;
- struct ib_device *ib_dev;
-};
-
-struct ib_ucm_file {
- struct mutex file_mutex;
- struct file *filp;
- struct ib_ucm_device *device;
-
- struct list_head ctxs;
- struct list_head events;
- wait_queue_head_t poll_wait;
-};
-
-struct ib_ucm_context {
- int id;
- struct completion comp;
- atomic_t ref;
- int events_reported;
-
- struct ib_ucm_file *file;
- struct ib_cm_id *cm_id;
- __u64 uid;
-
- struct list_head events; /* list of pending events. */
- struct list_head file_list; /* member in file ctx list */
-};
-
-struct ib_ucm_event {
- struct ib_ucm_context *ctx;
- struct list_head file_list; /* member in file event list */
- struct list_head ctx_list; /* member in ctx event list */
-
- struct ib_cm_id *cm_id;
- struct ib_ucm_event_resp resp;
- void *data;
- void *info;
- int data_len;
- int info_len;
-};
-
-enum {
- IB_UCM_MAJOR = 231,
- IB_UCM_BASE_MINOR = 224,
- IB_UCM_MAX_DEVICES = RDMA_MAX_PORTS,
- IB_UCM_NUM_FIXED_MINOR = 32,
- IB_UCM_NUM_DYNAMIC_MINOR = IB_UCM_MAX_DEVICES - IB_UCM_NUM_FIXED_MINOR,
-};
-
-#define IB_UCM_BASE_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_BASE_MINOR)
-static dev_t dynamic_ucm_dev;
-
-static void ib_ucm_add_one(struct ib_device *device);
-static void ib_ucm_remove_one(struct ib_device *device, void *client_data);
-
-static struct ib_client ucm_client = {
- .name = "ucm",
- .add = ib_ucm_add_one,
- .remove = ib_ucm_remove_one
-};
-
-static DEFINE_XARRAY_ALLOC(ctx_id_table);
-static DECLARE_BITMAP(dev_map, IB_UCM_MAX_DEVICES);
-
-static struct ib_ucm_context *ib_ucm_ctx_get(struct ib_ucm_file *file, int id)
-{
- struct ib_ucm_context *ctx;
-
- xa_lock(&ctx_id_table);
- ctx = xa_load(&ctx_id_table, id);
- if (!ctx)
- ctx = ERR_PTR(-ENOENT);
- else if (ctx->file != file)
- ctx = ERR_PTR(-EINVAL);
- else
- atomic_inc(&ctx->ref);
- xa_unlock(&ctx_id_table);
-
- return ctx;
-}
-
-static void ib_ucm_ctx_put(struct ib_ucm_context *ctx)
-{
- if (atomic_dec_and_test(&ctx->ref))
- complete(&ctx->comp);
-}
-
-static inline int ib_ucm_new_cm_id(int event)
-{
- return event == IB_CM_REQ_RECEIVED || event == IB_CM_SIDR_REQ_RECEIVED;
-}
-
-static void ib_ucm_cleanup_events(struct ib_ucm_context *ctx)
-{
- struct ib_ucm_event *uevent;
-
- mutex_lock(&ctx->file->file_mutex);
- list_del(&ctx->file_list);
- while (!list_empty(&ctx->events)) {
-
- uevent = list_entry(ctx->events.next,
- struct ib_ucm_event, ctx_list);
- list_del(&uevent->file_list);
- list_del(&uevent->ctx_list);
- mutex_unlock(&ctx->file->file_mutex);
-
- /* clear incoming connections. */
- if (ib_ucm_new_cm_id(uevent->resp.event))
- ib_destroy_cm_id(uevent->cm_id);
-
- kfree(uevent);
- mutex_lock(&ctx->file->file_mutex);
- }
- mutex_unlock(&ctx->file->file_mutex);
-}
-
-static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file)
-{
- struct ib_ucm_context *ctx;
-
- ctx = kzalloc(sizeof *ctx, GFP_KERNEL);
- if (!ctx)
- return NULL;
-
- atomic_set(&ctx->ref, 1);
- init_completion(&ctx->comp);
- ctx->file = file;
- INIT_LIST_HEAD(&ctx->events);
-
- if (xa_alloc(&ctx_id_table, &ctx->id, ctx, xa_limit_32b, GFP_KERNEL))
- goto error;
-
- list_add_tail(&ctx->file_list, &file->ctxs);
- return ctx;
-
-error:
- kfree(ctx);
- return NULL;
-}
-
-static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq,
- const struct ib_cm_req_event_param *kreq)
-{
- ureq->remote_ca_guid = kreq->remote_ca_guid;
- ureq->remote_qkey = kreq->remote_qkey;
- ureq->remote_qpn = kreq->remote_qpn;
- ureq->qp_type = kreq->qp_type;
- ureq->starting_psn = kreq->starting_psn;
- ureq->responder_resources = kreq->responder_resources;
- ureq->initiator_depth = kreq->initiator_depth;
- ureq->local_cm_response_timeout = kreq->local_cm_response_timeout;
- ureq->flow_control = kreq->flow_control;
- ureq->remote_cm_response_timeout = kreq->remote_cm_response_timeout;
- ureq->retry_count = kreq->retry_count;
- ureq->rnr_retry_count = kreq->rnr_retry_count;
- ureq->srq = kreq->srq;
- ureq->port = kreq->port;
-
- ib_copy_path_rec_to_user(&ureq->primary_path, kreq->primary_path);
- if (kreq->alternate_path)
- ib_copy_path_rec_to_user(&ureq->alternate_path,
- kreq->alternate_path);
-}
-
-static void ib_ucm_event_rep_get(struct ib_ucm_rep_event_resp *urep,
- const struct ib_cm_rep_event_param *krep)
-{
- urep->remote_ca_guid = krep->remote_ca_guid;
- urep->remote_qkey = krep->remote_qkey;
- urep->remote_qpn = krep->remote_qpn;
- urep->starting_psn = krep->starting_psn;
- urep->responder_resources = krep->responder_resources;
- urep->initiator_depth = krep->initiator_depth;
- urep->target_ack_delay = krep->target_ack_delay;
- urep->failover_accepted = krep->failover_accepted;
- urep->flow_control = krep->flow_control;
- urep->rnr_retry_count = krep->rnr_retry_count;
- urep->srq = krep->srq;
-}
-
-static void ib_ucm_event_sidr_rep_get(struct ib_ucm_sidr_rep_event_resp *urep,
- const struct ib_cm_sidr_rep_event_param *krep)
-{
- urep->status = krep->status;
- urep->qkey = krep->qkey;
- urep->qpn = krep->qpn;
-};
-
-static int ib_ucm_event_process(const struct ib_cm_event *evt,
- struct ib_ucm_event *uvt)
-{
- void *info = NULL;
-
- switch (evt->event) {
- case IB_CM_REQ_RECEIVED:
- ib_ucm_event_req_get(&uvt->resp.u.req_resp,
- &evt->param.req_rcvd);
- uvt->data_len = IB_CM_REQ_PRIVATE_DATA_SIZE;
- uvt->resp.present = IB_UCM_PRES_PRIMARY;
- uvt->resp.present |= (evt->param.req_rcvd.alternate_path ?
- IB_UCM_PRES_ALTERNATE : 0);
- break;
- case IB_CM_REP_RECEIVED:
- ib_ucm_event_rep_get(&uvt->resp.u.rep_resp,
- &evt->param.rep_rcvd);
- uvt->data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
- break;
- case IB_CM_RTU_RECEIVED:
- uvt->data_len = IB_CM_RTU_PRIVATE_DATA_SIZE;
- uvt->resp.u.send_status = evt->param.send_status;
- break;
- case IB_CM_DREQ_RECEIVED:
- uvt->data_len = IB_CM_DREQ_PRIVATE_DATA_SIZE;
- uvt->resp.u.send_status = evt->param.send_status;
- break;
- case IB_CM_DREP_RECEIVED:
- uvt->data_len = IB_CM_DREP_PRIVATE_DATA_SIZE;
- uvt->resp.u.send_status = evt->param.send_status;
- break;
- case IB_CM_MRA_RECEIVED:
- uvt->resp.u.mra_resp.timeout =
- evt->param.mra_rcvd.service_timeout;
- uvt->data_len = IB_CM_MRA_PRIVATE_DATA_SIZE;
- break;
- case IB_CM_REJ_RECEIVED:
- uvt->resp.u.rej_resp.reason = evt->param.rej_rcvd.reason;
- uvt->data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
- uvt->info_len = evt->param.rej_rcvd.ari_length;
- info = evt->param.rej_rcvd.ari;
- break;
- case IB_CM_LAP_RECEIVED:
- ib_copy_path_rec_to_user(&uvt->resp.u.lap_resp.path,
- evt->param.lap_rcvd.alternate_path);
- uvt->data_len = IB_CM_LAP_PRIVATE_DATA_SIZE;
- uvt->resp.present = IB_UCM_PRES_ALTERNATE;
- break;
- case IB_CM_APR_RECEIVED:
- uvt->resp.u.apr_resp.status = evt->param.apr_rcvd.ap_status;
- uvt->data_len = IB_CM_APR_PRIVATE_DATA_SIZE;
- uvt->info_len = evt->param.apr_rcvd.info_len;
- info = evt->param.apr_rcvd.apr_info;
- break;
- case IB_CM_SIDR_REQ_RECEIVED:
- uvt->resp.u.sidr_req_resp.pkey =
- evt->param.sidr_req_rcvd.pkey;
- uvt->resp.u.sidr_req_resp.port =
- evt->param.sidr_req_rcvd.port;
- uvt->data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE;
- break;
- case IB_CM_SIDR_REP_RECEIVED:
- ib_ucm_event_sidr_rep_get(&uvt->resp.u.sidr_rep_resp,
- &evt->param.sidr_rep_rcvd);
- uvt->data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
- uvt->info_len = evt->param.sidr_rep_rcvd.info_len;
- info = evt->param.sidr_rep_rcvd.info;
- break;
- default:
- uvt->resp.u.send_status = evt->param.send_status;
- break;
- }
-
- if (uvt->data_len) {
- uvt->data = kmemdup(evt->private_data, uvt->data_len, GFP_KERNEL);
- if (!uvt->data)
- goto err1;
-
- uvt->resp.present |= IB_UCM_PRES_DATA;
- }
-
- if (uvt->info_len) {
- uvt->info = kmemdup(info, uvt->info_len, GFP_KERNEL);
- if (!uvt->info)
- goto err2;
-
- uvt->resp.present |= IB_UCM_PRES_INFO;
- }
- return 0;
-
-err2:
- kfree(uvt->data);
-err1:
- return -ENOMEM;
-}
-
-static int ib_ucm_event_handler(struct ib_cm_id *cm_id,
- const struct ib_cm_event *event)
-{
- struct ib_ucm_event *uevent;
- struct ib_ucm_context *ctx;
- int result = 0;
-
- ctx = cm_id->context;
-
- uevent = kzalloc(sizeof *uevent, GFP_KERNEL);
- if (!uevent)
- goto err1;
-
- uevent->ctx = ctx;
- uevent->cm_id = cm_id;
- uevent->resp.uid = ctx->uid;
- uevent->resp.id = ctx->id;
- uevent->resp.event = event->event;
-
- result = ib_ucm_event_process(event, uevent);
- if (result)
- goto err2;
-
- mutex_lock(&ctx->file->file_mutex);
- list_add_tail(&uevent->file_list, &ctx->file->events);
- list_add_tail(&uevent->ctx_list, &ctx->events);
- wake_up_interruptible(&ctx->file->poll_wait);
- mutex_unlock(&ctx->file->file_mutex);
- return 0;
-
-err2:
- kfree(uevent);
-err1:
- /* Destroy new cm_id's */
- return ib_ucm_new_cm_id(event->event);
-}
-
-static ssize_t ib_ucm_event(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_context *ctx;
- struct ib_ucm_event_get cmd;
- struct ib_ucm_event *uevent;
- int result = 0;
-
- if (out_len < sizeof(struct ib_ucm_event_resp))
- return -ENOSPC;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- mutex_lock(&file->file_mutex);
- while (list_empty(&file->events)) {
- mutex_unlock(&file->file_mutex);
-
- if (file->filp->f_flags & O_NONBLOCK)
- return -EAGAIN;
-
- if (wait_event_interruptible(file->poll_wait,
- !list_empty(&file->events)))
- return -ERESTARTSYS;
-
- mutex_lock(&file->file_mutex);
- }
-
- uevent = list_entry(file->events.next, struct ib_ucm_event, file_list);
-
- if (ib_ucm_new_cm_id(uevent->resp.event)) {
- ctx = ib_ucm_ctx_alloc(file);
- if (!ctx) {
- result = -ENOMEM;
- goto done;
- }
-
- ctx->cm_id = uevent->cm_id;
- ctx->cm_id->context = ctx;
- uevent->resp.id = ctx->id;
- }
-
- if (copy_to_user(u64_to_user_ptr(cmd.response),
- &uevent->resp, sizeof(uevent->resp))) {
- result = -EFAULT;
- goto done;
- }
-
- if (uevent->data) {
- if (cmd.data_len < uevent->data_len) {
- result = -ENOMEM;
- goto done;
- }
- if (copy_to_user(u64_to_user_ptr(cmd.data),
- uevent->data, uevent->data_len)) {
- result = -EFAULT;
- goto done;
- }
- }
-
- if (uevent->info) {
- if (cmd.info_len < uevent->info_len) {
- result = -ENOMEM;
- goto done;
- }
- if (copy_to_user(u64_to_user_ptr(cmd.info),
- uevent->info, uevent->info_len)) {
- result = -EFAULT;
- goto done;
- }
- }
-
- list_del(&uevent->file_list);
- list_del(&uevent->ctx_list);
- uevent->ctx->events_reported++;
-
- kfree(uevent->data);
- kfree(uevent->info);
- kfree(uevent);
-done:
- mutex_unlock(&file->file_mutex);
- return result;
-}
-
-static ssize_t ib_ucm_create_id(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_create_id cmd;
- struct ib_ucm_create_id_resp resp;
- struct ib_ucm_context *ctx;
- int result;
-
- if (out_len < sizeof(resp))
- return -ENOSPC;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- mutex_lock(&file->file_mutex);
- ctx = ib_ucm_ctx_alloc(file);
- mutex_unlock(&file->file_mutex);
- if (!ctx)
- return -ENOMEM;
-
- ctx->uid = cmd.uid;
- ctx->cm_id = ib_create_cm_id(file->device->ib_dev,
- ib_ucm_event_handler, ctx);
- if (IS_ERR(ctx->cm_id)) {
- result = PTR_ERR(ctx->cm_id);
- goto err1;
- }
-
- resp.id = ctx->id;
- if (copy_to_user(u64_to_user_ptr(cmd.response),
- &resp, sizeof(resp))) {
- result = -EFAULT;
- goto err2;
- }
- return 0;
-
-err2:
- ib_destroy_cm_id(ctx->cm_id);
-err1:
- xa_erase(&ctx_id_table, ctx->id);
- kfree(ctx);
- return result;
-}
-
-static ssize_t ib_ucm_destroy_id(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_destroy_id cmd;
- struct ib_ucm_destroy_id_resp resp;
- struct ib_ucm_context *ctx;
- int result = 0;
-
- if (out_len < sizeof(resp))
- return -ENOSPC;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- xa_lock(&ctx_id_table);
- ctx = xa_load(&ctx_id_table, cmd.id);
- if (!ctx)
- ctx = ERR_PTR(-ENOENT);
- else if (ctx->file != file)
- ctx = ERR_PTR(-EINVAL);
- else
- __xa_erase(&ctx_id_table, ctx->id);
- xa_unlock(&ctx_id_table);
-
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- ib_ucm_ctx_put(ctx);
- wait_for_completion(&ctx->comp);
-
- /* No new events will be generated after destroying the cm_id. */
- ib_destroy_cm_id(ctx->cm_id);
- /* Cleanup events not yet reported to the user. */
- ib_ucm_cleanup_events(ctx);
-
- resp.events_reported = ctx->events_reported;
- if (copy_to_user(u64_to_user_ptr(cmd.response),
- &resp, sizeof(resp)))
- result = -EFAULT;
-
- kfree(ctx);
- return result;
-}
-
-static ssize_t ib_ucm_attr_id(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_attr_id_resp resp;
- struct ib_ucm_attr_id cmd;
- struct ib_ucm_context *ctx;
- int result = 0;
-
- if (out_len < sizeof(resp))
- return -ENOSPC;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- resp.service_id = ctx->cm_id->service_id;
- resp.service_mask = ctx->cm_id->service_mask;
- resp.local_id = ctx->cm_id->local_id;
- resp.remote_id = ctx->cm_id->remote_id;
-
- if (copy_to_user(u64_to_user_ptr(cmd.response),
- &resp, sizeof(resp)))
- result = -EFAULT;
-
- ib_ucm_ctx_put(ctx);
- return result;
-}
-
-static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_uverbs_qp_attr resp;
- struct ib_ucm_init_qp_attr cmd;
- struct ib_ucm_context *ctx;
- struct ib_qp_attr qp_attr;
- int result = 0;
-
- if (out_len < sizeof(resp))
- return -ENOSPC;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- resp.qp_attr_mask = 0;
- memset(&qp_attr, 0, sizeof qp_attr);
- qp_attr.qp_state = cmd.qp_state;
- result = ib_cm_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask);
- if (result)
- goto out;
-
- ib_copy_qp_attr_to_user(ctx->cm_id->device, &resp, &qp_attr);
-
- if (copy_to_user(u64_to_user_ptr(cmd.response),
- &resp, sizeof(resp)))
- result = -EFAULT;
-
-out:
- ib_ucm_ctx_put(ctx);
- return result;
-}
-
-static int ucm_validate_listen(__be64 service_id, __be64 service_mask)
-{
- service_id &= service_mask;
-
- if (((service_id & IB_CMA_SERVICE_ID_MASK) == IB_CMA_SERVICE_ID) ||
- ((service_id & IB_SDP_SERVICE_ID_MASK) == IB_SDP_SERVICE_ID))
- return -EINVAL;
-
- return 0;
-}
-
-static ssize_t ib_ucm_listen(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_listen cmd;
- struct ib_ucm_context *ctx;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- result = ucm_validate_listen(cmd.service_id, cmd.service_mask);
- if (result)
- goto out;
-
- result = ib_cm_listen(ctx->cm_id, cmd.service_id, cmd.service_mask);
-out:
- ib_ucm_ctx_put(ctx);
- return result;
-}
-
-static ssize_t ib_ucm_notify(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_notify cmd;
- struct ib_ucm_context *ctx;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- result = ib_cm_notify(ctx->cm_id, (enum ib_event_type) cmd.event);
- ib_ucm_ctx_put(ctx);
- return result;
-}
-
-static int ib_ucm_alloc_data(const void **dest, u64 src, u32 len)
-{
- void *data;
-
- *dest = NULL;
-
- if (!len)
- return 0;
-
- data = memdup_user(u64_to_user_ptr(src), len);
- if (IS_ERR(data))
- return PTR_ERR(data);
-
- *dest = data;
- return 0;
-}
-
-static int ib_ucm_path_get(struct sa_path_rec **path, u64 src)
-{
- struct ib_user_path_rec upath;
- struct sa_path_rec *sa_path;
-
- *path = NULL;
-
- if (!src)
- return 0;
-
- sa_path = kmalloc(sizeof(*sa_path), GFP_KERNEL);
- if (!sa_path)
- return -ENOMEM;
-
- if (copy_from_user(&upath, u64_to_user_ptr(src),
- sizeof(upath))) {
-
- kfree(sa_path);
- return -EFAULT;
- }
-
- ib_copy_path_rec_from_user(sa_path, &upath);
- *path = sa_path;
- return 0;
-}
-
-static ssize_t ib_ucm_send_req(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_cm_req_param param;
- struct ib_ucm_context *ctx;
- struct ib_ucm_req cmd;
- int result;
-
- param.private_data = NULL;
- param.primary_path = NULL;
- param.alternate_path = NULL;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
- if (result)
- goto done;
-
- result = ib_ucm_path_get(&param.primary_path, cmd.primary_path);
- if (result)
- goto done;
-
- result = ib_ucm_path_get(&param.alternate_path, cmd.alternate_path);
- if (result)
- goto done;
-
- param.private_data_len = cmd.len;
- param.service_id = cmd.sid;
- param.qp_num = cmd.qpn;
- param.qp_type = cmd.qp_type;
- param.starting_psn = cmd.psn;
- param.peer_to_peer = cmd.peer_to_peer;
- param.responder_resources = cmd.responder_resources;
- param.initiator_depth = cmd.initiator_depth;
- param.remote_cm_response_timeout = cmd.remote_cm_response_timeout;
- param.flow_control = cmd.flow_control;
- param.local_cm_response_timeout = cmd.local_cm_response_timeout;
- param.retry_count = cmd.retry_count;
- param.rnr_retry_count = cmd.rnr_retry_count;
- param.max_cm_retries = cmd.max_cm_retries;
- param.srq = cmd.srq;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = ib_send_cm_req(ctx->cm_id, &param);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
-done:
- kfree(param.private_data);
- kfree(param.primary_path);
- kfree(param.alternate_path);
- return result;
-}
-
-static ssize_t ib_ucm_send_rep(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_cm_rep_param param;
- struct ib_ucm_context *ctx;
- struct ib_ucm_rep cmd;
- int result;
-
- param.private_data = NULL;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
- if (result)
- return result;
-
- param.qp_num = cmd.qpn;
- param.starting_psn = cmd.psn;
- param.private_data_len = cmd.len;
- param.responder_resources = cmd.responder_resources;
- param.initiator_depth = cmd.initiator_depth;
- param.failover_accepted = cmd.failover_accepted;
- param.flow_control = cmd.flow_control;
- param.rnr_retry_count = cmd.rnr_retry_count;
- param.srq = cmd.srq;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- ctx->uid = cmd.uid;
- result = ib_send_cm_rep(ctx->cm_id, &param);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
- kfree(param.private_data);
- return result;
-}
-
-static ssize_t ib_ucm_send_private_data(struct ib_ucm_file *file,
- const char __user *inbuf, int in_len,
- int (*func)(struct ib_cm_id *cm_id,
- const void *private_data,
- u8 private_data_len))
-{
- struct ib_ucm_private_data cmd;
- struct ib_ucm_context *ctx;
- const void *private_data = NULL;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&private_data, cmd.data, cmd.len);
- if (result)
- return result;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = func(ctx->cm_id, private_data, cmd.len);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
- kfree(private_data);
- return result;
-}
-
-static ssize_t ib_ucm_send_rtu(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_rtu);
-}
-
-static ssize_t ib_ucm_send_dreq(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_dreq);
-}
-
-static ssize_t ib_ucm_send_drep(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_drep);
-}
-
-static ssize_t ib_ucm_send_info(struct ib_ucm_file *file,
- const char __user *inbuf, int in_len,
- int (*func)(struct ib_cm_id *cm_id,
- int status,
- const void *info,
- u8 info_len,
- const void *data,
- u8 data_len))
-{
- struct ib_ucm_context *ctx;
- struct ib_ucm_info cmd;
- const void *data = NULL;
- const void *info = NULL;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&data, cmd.data, cmd.data_len);
- if (result)
- goto done;
-
- result = ib_ucm_alloc_data(&info, cmd.info, cmd.info_len);
- if (result)
- goto done;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = func(ctx->cm_id, cmd.status, info, cmd.info_len,
- data, cmd.data_len);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
-done:
- kfree(data);
- kfree(info);
- return result;
-}
-
-static ssize_t ib_ucm_send_rej(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_rej);
-}
-
-static ssize_t ib_ucm_send_apr(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_apr);
-}
-
-static ssize_t ib_ucm_send_mra(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_context *ctx;
- struct ib_ucm_mra cmd;
- const void *data = NULL;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&data, cmd.data, cmd.len);
- if (result)
- return result;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = ib_send_cm_mra(ctx->cm_id, cmd.timeout, data, cmd.len);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
- kfree(data);
- return result;
-}
-
-static ssize_t ib_ucm_send_lap(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_ucm_context *ctx;
- struct sa_path_rec *path = NULL;
- struct ib_ucm_lap cmd;
- const void *data = NULL;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&data, cmd.data, cmd.len);
- if (result)
- goto done;
-
- result = ib_ucm_path_get(&path, cmd.path);
- if (result)
- goto done;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = ib_send_cm_lap(ctx->cm_id, path, data, cmd.len);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
-done:
- kfree(data);
- kfree(path);
- return result;
-}
-
-static ssize_t ib_ucm_send_sidr_req(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_cm_sidr_req_param param = {};
- struct ib_ucm_context *ctx;
- struct ib_ucm_sidr_req cmd;
- int result;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
- if (result)
- goto done;
-
- result = ib_ucm_path_get(&param.path, cmd.path);
- if (result)
- goto done;
-
- param.private_data_len = cmd.len;
- param.service_id = cmd.sid;
- param.timeout_ms = cmd.timeout;
- param.max_cm_retries = cmd.max_cm_retries;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = ib_send_cm_sidr_req(ctx->cm_id, &param);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
-done:
- kfree(param.private_data);
- kfree(param.path);
- return result;
-}
-
-static ssize_t ib_ucm_send_sidr_rep(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len)
-{
- struct ib_cm_sidr_rep_param param;
- struct ib_ucm_sidr_rep cmd;
- struct ib_ucm_context *ctx;
- int result;
-
- param.info = NULL;
-
- if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
- return -EFAULT;
-
- result = ib_ucm_alloc_data(&param.private_data,
- cmd.data, cmd.data_len);
- if (result)
- goto done;
-
- result = ib_ucm_alloc_data(&param.info, cmd.info, cmd.info_len);
- if (result)
- goto done;
-
- param.qp_num = cmd.qpn;
- param.qkey = cmd.qkey;
- param.status = cmd.status;
- param.info_length = cmd.info_len;
- param.private_data_len = cmd.data_len;
-
- ctx = ib_ucm_ctx_get(file, cmd.id);
- if (!IS_ERR(ctx)) {
- result = ib_send_cm_sidr_rep(ctx->cm_id, &param);
- ib_ucm_ctx_put(ctx);
- } else
- result = PTR_ERR(ctx);
-
-done:
- kfree(param.private_data);
- kfree(param.info);
- return result;
-}
-
-static ssize_t (*ucm_cmd_table[])(struct ib_ucm_file *file,
- const char __user *inbuf,
- int in_len, int out_len) = {
- [IB_USER_CM_CMD_CREATE_ID] = ib_ucm_create_id,
- [IB_USER_CM_CMD_DESTROY_ID] = ib_ucm_destroy_id,
- [IB_USER_CM_CMD_ATTR_ID] = ib_ucm_attr_id,
- [IB_USER_CM_CMD_LISTEN] = ib_ucm_listen,
- [IB_USER_CM_CMD_NOTIFY] = ib_ucm_notify,
- [IB_USER_CM_CMD_SEND_REQ] = ib_ucm_send_req,
- [IB_USER_CM_CMD_SEND_REP] = ib_ucm_send_rep,
- [IB_USER_CM_CMD_SEND_RTU] = ib_ucm_send_rtu,
- [IB_USER_CM_CMD_SEND_DREQ] = ib_ucm_send_dreq,
- [IB_USER_CM_CMD_SEND_DREP] = ib_ucm_send_drep,
- [IB_USER_CM_CMD_SEND_REJ] = ib_ucm_send_rej,
- [IB_USER_CM_CMD_SEND_MRA] = ib_ucm_send_mra,
- [IB_USER_CM_CMD_SEND_LAP] = ib_ucm_send_lap,
- [IB_USER_CM_CMD_SEND_APR] = ib_ucm_send_apr,
- [IB_USER_CM_CMD_SEND_SIDR_REQ] = ib_ucm_send_sidr_req,
- [IB_USER_CM_CMD_SEND_SIDR_REP] = ib_ucm_send_sidr_rep,
- [IB_USER_CM_CMD_EVENT] = ib_ucm_event,
- [IB_USER_CM_CMD_INIT_QP_ATTR] = ib_ucm_init_qp_attr,
-};
-
-static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,
- size_t len, loff_t *pos)
-{
- struct ib_ucm_file *file = filp->private_data;
- struct ib_ucm_cmd_hdr hdr;
- ssize_t result;
-
- if (!ib_safe_file_access(filp)) {
- pr_err_once("ucm_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
- task_tgid_vnr(current), current->comm);
- return -EACCES;
- }
-
- if (len < sizeof(hdr))
- return -EINVAL;
-
- if (copy_from_user(&hdr, buf, sizeof(hdr)))
- return -EFAULT;
-
- if (hdr.cmd >= ARRAY_SIZE(ucm_cmd_table))
- return -EINVAL;
- hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucm_cmd_table));
-
- if (hdr.in + sizeof(hdr) > len)
- return -EINVAL;
-
- result = ucm_cmd_table[hdr.cmd](file, buf + sizeof(hdr),
- hdr.in, hdr.out);
- if (!result)
- result = len;
-
- return result;
-}
-
-static __poll_t ib_ucm_poll(struct file *filp,
- struct poll_table_struct *wait)
-{
- struct ib_ucm_file *file = filp->private_data;
- __poll_t mask = 0;
-
- poll_wait(filp, &file->poll_wait, wait);
-
- if (!list_empty(&file->events))
- mask = EPOLLIN | EPOLLRDNORM;
-
- return mask;
-}
-
-/*
- * ib_ucm_open() does not need the BKL:
- *
- * - no global state is referred to;
- * - there is no ioctl method to race against;
- * - no further module initialization is required for open to work
- * after the device is registered.
- */
-static int ib_ucm_open(struct inode *inode, struct file *filp)
-{
- struct ib_ucm_file *file;
-
- file = kmalloc(sizeof(*file), GFP_KERNEL);
- if (!file)
- return -ENOMEM;
-
- INIT_LIST_HEAD(&file->events);
- INIT_LIST_HEAD(&file->ctxs);
- init_waitqueue_head(&file->poll_wait);
-
- mutex_init(&file->file_mutex);
-
- filp->private_data = file;
- file->filp = filp;
- file->device = container_of(inode->i_cdev, struct ib_ucm_device, cdev);
-
- return stream_open(inode, filp);
-}
-
-static int ib_ucm_close(struct inode *inode, struct file *filp)
-{
- struct ib_ucm_file *file = filp->private_data;
- struct ib_ucm_context *ctx;
-
- mutex_lock(&file->file_mutex);
- while (!list_empty(&file->ctxs)) {
- ctx = list_entry(file->ctxs.next,
- struct ib_ucm_context, file_list);
- mutex_unlock(&file->file_mutex);
-
- xa_erase(&ctx_id_table, ctx->id);
- ib_destroy_cm_id(ctx->cm_id);
- ib_ucm_cleanup_events(ctx);
- kfree(ctx);
-
- mutex_lock(&file->file_mutex);
- }
- mutex_unlock(&file->file_mutex);
- kfree(file);
- return 0;
-}
-
-static void ib_ucm_release_dev(struct device *dev)
-{
- struct ib_ucm_device *ucm_dev;
-
- ucm_dev = container_of(dev, struct ib_ucm_device, dev);
- kfree(ucm_dev);
-}
-
-static void ib_ucm_free_dev(struct ib_ucm_device *ucm_dev)
-{
- clear_bit(ucm_dev->devnum, dev_map);
-}
-
-static const struct file_operations ucm_fops = {
- .owner = THIS_MODULE,
- .open = ib_ucm_open,
- .release = ib_ucm_close,
- .write = ib_ucm_write,
- .poll = ib_ucm_poll,
- .llseek = no_llseek,
-};
-
-static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- struct ib_ucm_device *ucm_dev;
-
- ucm_dev = container_of(dev, struct ib_ucm_device, dev);
- return sprintf(buf, "%s\n", ucm_dev->ib_dev->name);
-}
-static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
-
-static void ib_ucm_add_one(struct ib_device *device)
-{
- int devnum;
- dev_t base;
- struct ib_ucm_device *ucm_dev;
-
- if (!device->ops.alloc_ucontext || !rdma_cap_ib_cm(device, 1))
- return;
-
- ucm_dev = kzalloc(sizeof *ucm_dev, GFP_KERNEL);
- if (!ucm_dev)
- return;
-
- device_initialize(&ucm_dev->dev);
- ucm_dev->ib_dev = device;
- ucm_dev->dev.release = ib_ucm_release_dev;
-
- devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);
- if (devnum >= IB_UCM_MAX_DEVICES)
- goto err;
- ucm_dev->devnum = devnum;
- set_bit(devnum, dev_map);
- if (devnum >= IB_UCM_NUM_FIXED_MINOR)
- base = dynamic_ucm_dev + devnum - IB_UCM_NUM_FIXED_MINOR;
- else
- base = IB_UCM_BASE_DEV + devnum;
-
- cdev_init(&ucm_dev->cdev, &ucm_fops);
- ucm_dev->cdev.owner = THIS_MODULE;
- kobject_set_name(&ucm_dev->cdev.kobj, "ucm%d", ucm_dev->devnum);
-
- ucm_dev->dev.class = &cm_class;
- ucm_dev->dev.parent = device->dev.parent;
- ucm_dev->dev.devt = base;
-
- dev_set_name(&ucm_dev->dev, "ucm%d", ucm_dev->devnum);
- if (cdev_device_add(&ucm_dev->cdev, &ucm_dev->dev))
- goto err_devnum;
-
- if (device_create_file(&ucm_dev->dev, &dev_attr_ibdev))
- goto err_dev;
-
- ib_set_client_data(device, &ucm_client, ucm_dev);
- return;
-
-err_dev:
- cdev_device_del(&ucm_dev->cdev, &ucm_dev->dev);
-err_devnum:
- ib_ucm_free_dev(ucm_dev);
-err:
- put_device(&ucm_dev->dev);
- return;
-}
-
-static void ib_ucm_remove_one(struct ib_device *device, void *client_data)
-{
- struct ib_ucm_device *ucm_dev = client_data;
-
- if (!ucm_dev)
- return;
-
- cdev_device_del(&ucm_dev->cdev, &ucm_dev->dev);
- ib_ucm_free_dev(ucm_dev);
- put_device(&ucm_dev->dev);
-}
-
-static CLASS_ATTR_STRING(abi_version, S_IRUGO,
- __stringify(IB_USER_CM_ABI_VERSION));
-
-static int __init ib_ucm_init(void)
-{
- int ret;
-
- ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR,
- "infiniband_cm");
- if (ret) {
- pr_err("ucm: couldn't register device number\n");
- goto error1;
- }
-
- ret = alloc_chrdev_region(&dynamic_ucm_dev, 0, IB_UCM_NUM_DYNAMIC_MINOR,
- "infiniband_cm");
- if (ret) {
- pr_err("ucm: couldn't register dynamic device number\n");
- goto err_alloc;
- }
-
- ret = class_create_file(&cm_class, &class_attr_abi_version.attr);
- if (ret) {
- pr_err("ucm: couldn't create abi_version attribute\n");
- goto error2;
- }
-
- ret = ib_register_client(&ucm_client);
- if (ret) {
- pr_err("ucm: couldn't register client\n");
- goto error3;
- }
- return 0;
-
-error3:
- class_remove_file(&cm_class, &class_attr_abi_version.attr);
-error2:
- unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);
-err_alloc:
- unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);
-error1:
- return ret;
-}
-
-static void __exit ib_ucm_cleanup(void)
-{
- ib_unregister_client(&ucm_client);
- class_remove_file(&cm_class, &class_attr_abi_version.attr);
- unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);
- unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);
- WARN_ON(!xa_empty(&ctx_id_table));
-}
-
-module_init(ib_ucm_init);
-module_exit(ib_ucm_cleanup);
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 140a338a135f..0274e9b704be 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -52,6 +52,8 @@
#include <rdma/rdma_cm_ib.h>
#include <rdma/ib_addr.h>
#include <rdma/ib.h>
+#include <rdma/rdma_netlink.h>
+#include "core_priv.h"
MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access");
@@ -81,7 +83,7 @@ struct ucma_file {
};
struct ucma_context {
- int id;
+ u32 id;
struct completion comp;
atomic_t ref;
int events_reported;
@@ -94,7 +96,7 @@ struct ucma_context {
struct list_head list;
struct list_head mc_list;
/* mark that device is in process of destroying the internal HW
- * resources, protected by the global mut
+ * resources, protected by the ctx_table lock
*/
int closing;
/* sync between removal event and id destroy, protected by file mut */
@@ -104,7 +106,7 @@ struct ucma_context {
struct ucma_multicast {
struct ucma_context *ctx;
- int id;
+ u32 id;
int events_reported;
u64 uid;
@@ -122,9 +124,8 @@ struct ucma_event {
struct work_struct close_work;
};
-static DEFINE_MUTEX(mut);
-static DEFINE_IDR(ctx_idr);
-static DEFINE_IDR(multicast_idr);
+static DEFINE_XARRAY_ALLOC(ctx_table);
+static DEFINE_XARRAY_ALLOC(multicast_table);
static const struct file_operations ucma_fops;
@@ -133,7 +134,7 @@ static inline struct ucma_context *_ucma_find_context(int id,
{
struct ucma_context *ctx;
- ctx = idr_find(&ctx_idr, id);
+ ctx = xa_load(&ctx_table, id);
if (!ctx)
ctx = ERR_PTR(-ENOENT);
else if (ctx->file != file || !ctx->cm_id)
@@ -145,7 +146,7 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)
{
struct ucma_context *ctx;
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
ctx = _ucma_find_context(id, file);
if (!IS_ERR(ctx)) {
if (ctx->closing)
@@ -153,7 +154,7 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)
else
atomic_inc(&ctx->ref);
}
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
return ctx;
}
@@ -216,10 +217,7 @@ static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)
INIT_LIST_HEAD(&ctx->mc_list);
ctx->file = file;
- mutex_lock(&mut);
- ctx->id = idr_alloc(&ctx_idr, ctx, 0, 0, GFP_KERNEL);
- mutex_unlock(&mut);
- if (ctx->id < 0)
+ if (xa_alloc(&ctx_table, &ctx->id, ctx, xa_limit_32b, GFP_KERNEL))
goto error;
list_add_tail(&ctx->list, &file->ctx_list);
@@ -238,13 +236,10 @@ static struct ucma_multicast* ucma_alloc_multicast(struct ucma_context *ctx)
if (!mc)
return NULL;
- mutex_lock(&mut);
- mc->id = idr_alloc(&multicast_idr, NULL, 0, 0, GFP_KERNEL);
- mutex_unlock(&mut);
- if (mc->id < 0)
+ mc->ctx = ctx;
+ if (xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b, GFP_KERNEL))
goto error;
- mc->ctx = ctx;
list_add_tail(&mc->list, &ctx->mc_list);
return mc;
@@ -319,9 +314,9 @@ static void ucma_removal_event_handler(struct rdma_cm_id *cm_id)
* handled separately below.
*/
if (ctx->cm_id == cm_id) {
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
ctx->closing = 1;
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
queue_work(ctx->file->close_wq, &ctx->close_work);
return;
}
@@ -523,9 +518,7 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
err2:
rdma_destroy_id(cm_id);
err1:
- mutex_lock(&mut);
- idr_remove(&ctx_idr, ctx->id);
- mutex_unlock(&mut);
+ xa_erase(&ctx_table, ctx->id);
mutex_lock(&file->mut);
list_del(&ctx->list);
mutex_unlock(&file->mut);
@@ -537,13 +530,13 @@ static void ucma_cleanup_multicast(struct ucma_context *ctx)
{
struct ucma_multicast *mc, *tmp;
- mutex_lock(&mut);
+ mutex_lock(&ctx->file->mut);
list_for_each_entry_safe(mc, tmp, &ctx->mc_list, list) {
list_del(&mc->list);
- idr_remove(&multicast_idr, mc->id);
+ xa_erase(&multicast_table, mc->id);
kfree(mc);
}
- mutex_unlock(&mut);
+ mutex_unlock(&ctx->file->mut);
}
static void ucma_cleanup_mc_events(struct ucma_multicast *mc)
@@ -614,11 +607,11 @@ static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf,
if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
return -EFAULT;
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
ctx = _ucma_find_context(cmd.id, file);
if (!IS_ERR(ctx))
- idr_remove(&ctx_idr, ctx->id);
- mutex_unlock(&mut);
+ __xa_erase(&ctx_table, ctx->id);
+ xa_unlock(&ctx_table);
if (IS_ERR(ctx))
return PTR_ERR(ctx);
@@ -630,14 +623,14 @@ static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf,
flush_workqueue(ctx->file->close_wq);
/* At this point it's guaranteed that there is no inflight
* closing task */
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
if (!ctx->closing) {
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
ucma_put_ctx(ctx);
wait_for_completion(&ctx->comp);
rdma_destroy_id(ctx->cm_id);
} else {
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
}
resp.events_reported = ucma_free_ctx(ctx);
@@ -951,8 +944,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx,
}
}
- if (copy_to_user(response, resp,
- sizeof(*resp) + (i * sizeof(struct ib_path_rec_data))))
+ if (copy_to_user(response, resp, struct_size(resp, path_data, i)))
ret = -EFAULT;
kfree(resp);
@@ -1432,9 +1424,7 @@ static ssize_t ucma_process_join(struct ucma_file *file,
goto err3;
}
- mutex_lock(&mut);
- idr_replace(&multicast_idr, mc, mc->id);
- mutex_unlock(&mut);
+ xa_store(&multicast_table, mc->id, mc, 0);
mutex_unlock(&file->mut);
ucma_put_ctx(ctx);
@@ -1444,9 +1434,7 @@ err3:
rdma_leave_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr);
ucma_cleanup_mc_events(mc);
err2:
- mutex_lock(&mut);
- idr_remove(&multicast_idr, mc->id);
- mutex_unlock(&mut);
+ xa_erase(&multicast_table, mc->id);
list_del(&mc->list);
kfree(mc);
err1:
@@ -1508,8 +1496,8 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,
if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
return -EFAULT;
- mutex_lock(&mut);
- mc = idr_find(&multicast_idr, cmd.id);
+ xa_lock(&multicast_table);
+ mc = xa_load(&multicast_table, cmd.id);
if (!mc)
mc = ERR_PTR(-ENOENT);
else if (mc->ctx->file != file)
@@ -1517,8 +1505,8 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,
else if (!atomic_inc_not_zero(&mc->ctx->ref))
mc = ERR_PTR(-ENXIO);
else
- idr_remove(&multicast_idr, mc->id);
- mutex_unlock(&mut);
+ __xa_erase(&multicast_table, mc->id);
+ xa_unlock(&multicast_table);
if (IS_ERR(mc)) {
ret = PTR_ERR(mc);
@@ -1615,14 +1603,14 @@ static ssize_t ucma_migrate_id(struct ucma_file *new_file,
* events being added before existing events.
*/
ucma_lock_files(cur_file, new_file);
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
list_move_tail(&ctx->list, &new_file->ctx_list);
ucma_move_events(ctx, new_file);
ctx->file = new_file;
resp.events_reported = ctx->events_reported;
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
ucma_unlock_files(cur_file, new_file);
response:
@@ -1757,18 +1745,15 @@ static int ucma_close(struct inode *inode, struct file *filp)
ctx->destroying = 1;
mutex_unlock(&file->mut);
- mutex_lock(&mut);
- idr_remove(&ctx_idr, ctx->id);
- mutex_unlock(&mut);
-
+ xa_erase(&ctx_table, ctx->id);
flush_workqueue(file->close_wq);
/* At that step once ctx was marked as destroying and workqueue
* was flushed we are safe from any inflights handlers that
* might put other closing task.
*/
- mutex_lock(&mut);
+ xa_lock(&ctx_table);
if (!ctx->closing) {
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
ucma_put_ctx(ctx);
wait_for_completion(&ctx->comp);
/* rdma_destroy_id ensures that no event handlers are
@@ -1776,7 +1761,7 @@ static int ucma_close(struct inode *inode, struct file *filp)
*/
rdma_destroy_id(ctx->cm_id);
} else {
- mutex_unlock(&mut);
+ xa_unlock(&ctx_table);
}
ucma_free_ctx(ctx);
@@ -1805,6 +1790,19 @@ static struct miscdevice ucma_misc = {
.fops = &ucma_fops,
};
+static int ucma_get_global_nl_info(struct ib_client_nl_info *res)
+{
+ res->abi = RDMA_USER_CM_ABI_VERSION;
+ res->cdev = ucma_misc.this_device;
+ return 0;
+}
+
+static struct ib_client rdma_cma_client = {
+ .name = "rdma_cm",
+ .get_global_nl_info = ucma_get_global_nl_info,
+};
+MODULE_ALIAS_RDMA_CLIENT("rdma_cm");
+
static ssize_t show_abi_version(struct device *dev,
struct device_attribute *attr,
char *buf)
@@ -1833,7 +1831,14 @@ static int __init ucma_init(void)
ret = -ENOMEM;
goto err2;
}
+
+ ret = ib_register_client(&rdma_cma_client);
+ if (ret)
+ goto err3;
+
return 0;
+err3:
+ unregister_net_sysctl_table(ucma_ctl_table_hdr);
err2:
device_remove_file(ucma_misc.this_device, &dev_attr_abi_version);
err1:
@@ -1843,11 +1848,10 @@ err1:
static void __exit ucma_cleanup(void)
{
+ ib_unregister_client(&rdma_cma_client);
unregister_net_sysctl_table(ucma_ctl_table_hdr);
device_remove_file(ucma_misc.this_device, &dev_attr_abi_version);
misc_deregister(&ucma_misc);
- idr_destroy(&ctx_idr);
- idr_destroy(&multicast_idr);
}
module_init(ucma_init);
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index e7ea819fcb11..08da840ed7ee 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -54,9 +54,10 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->sg_nents, 0) {
page = sg_page_iter_page(&sg_iter);
- if (!PageDirty(page) && umem->writable && dirty)
- set_page_dirty_lock(page);
- put_page(page);
+ if (umem->writable && dirty)
+ put_user_pages_dirty_lock(&page, 1);
+ else
+ put_user_page(page);
}
sg_free_table(&umem->sg_head);
@@ -244,7 +245,6 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,
umem->context = context;
umem->length = size;
umem->address = addr;
- umem->page_shift = PAGE_SHIFT;
umem->writable = ib_access_writable(access);
umem->owning_mm = mm = current->mm;
mmgrab(mm);
@@ -361,6 +361,9 @@ static void __ib_umem_release_tail(struct ib_umem *umem)
*/
void ib_umem_release(struct ib_umem *umem)
{
+ if (!umem)
+ return;
+
if (umem->is_odp) {
ib_umem_odp_release(to_ib_umem_odp(umem));
__ib_umem_release_tail(umem);
@@ -385,7 +388,7 @@ int ib_umem_page_count(struct ib_umem *umem)
n = 0;
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
- n += sg_dma_len(sg) >> umem->page_shift;
+ n += sg_dma_len(sg) >> PAGE_SHIFT;
return n;
}
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index f962b5bbfa40..2a75c6f8d827 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -59,7 +59,7 @@ static u64 node_start(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
- return ib_umem_start(&umem_odp->umem);
+ return ib_umem_start(umem_odp);
}
/* Note that the representation of the intervals in the interval tree
@@ -72,7 +72,7 @@ static u64 node_last(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
- return ib_umem_end(&umem_odp->umem) - 1;
+ return ib_umem_end(umem_odp) - 1;
}
INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
@@ -107,8 +107,6 @@ static void ib_umem_notifier_end_account(struct ib_umem_odp *umem_odp)
static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,
u64 start, u64 end, void *cookie)
{
- struct ib_umem *umem = &umem_odp->umem;
-
/*
* Increase the number of notifiers running, to
* prevent any further fault handling on this MR.
@@ -119,8 +117,8 @@ static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,
* all pending page faults. */
smp_wmb();
complete_all(&umem_odp->notifier_completion);
- umem->context->invalidate_range(umem_odp, ib_umem_start(umem),
- ib_umem_end(umem));
+ umem_odp->umem.context->invalidate_range(
+ umem_odp, ib_umem_start(umem_odp), ib_umem_end(umem_odp));
return 0;
}
@@ -151,6 +149,7 @@ static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
{
struct ib_ucontext_per_mm *per_mm =
container_of(mn, struct ib_ucontext_per_mm, mn);
+ int rc;
if (mmu_notifier_range_blockable(range))
down_read(&per_mm->umem_rwsem);
@@ -167,11 +166,14 @@ static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
return 0;
}
- return rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, range->start,
- range->end,
- invalidate_range_start_trampoline,
- mmu_notifier_range_blockable(range),
- NULL);
+ rc = rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, range->start,
+ range->end,
+ invalidate_range_start_trampoline,
+ mmu_notifier_range_blockable(range),
+ NULL);
+ if (rc)
+ up_read(&per_mm->umem_rwsem);
+ return rc;
}
static int invalidate_range_end_trampoline(struct ib_umem_odp *item, u64 start,
@@ -205,10 +207,9 @@ static const struct mmu_notifier_ops ib_umem_notifiers = {
static void add_umem_to_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
- struct ib_umem *umem = &umem_odp->umem;
down_write(&per_mm->umem_rwsem);
- if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
+ if (likely(ib_umem_start(umem_odp) != ib_umem_end(umem_odp)))
rbt_ib_umem_insert(&umem_odp->interval_tree,
&per_mm->umem_tree);
up_write(&per_mm->umem_rwsem);
@@ -217,10 +218,9 @@ static void add_umem_to_per_mm(struct ib_umem_odp *umem_odp)
static void remove_umem_from_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
- struct ib_umem *umem = &umem_odp->umem;
down_write(&per_mm->umem_rwsem);
- if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
+ if (likely(ib_umem_start(umem_odp) != ib_umem_end(umem_odp)))
rbt_ib_umem_remove(&umem_odp->interval_tree,
&per_mm->umem_tree);
complete_all(&umem_odp->notifier_completion);
@@ -351,7 +351,7 @@ struct ib_umem_odp *ib_alloc_odp_umem(struct ib_umem_odp *root,
umem->context = ctx;
umem->length = size;
umem->address = addr;
- umem->page_shift = PAGE_SHIFT;
+ odp_data->page_shift = PAGE_SHIFT;
umem->writable = root->umem.writable;
umem->is_odp = 1;
odp_data->per_mm = per_mm;
@@ -405,18 +405,19 @@ int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
struct mm_struct *mm = umem->owning_mm;
int ret_val;
+ umem_odp->page_shift = PAGE_SHIFT;
if (access & IB_ACCESS_HUGETLB) {
struct vm_area_struct *vma;
struct hstate *h;
down_read(&mm->mmap_sem);
- vma = find_vma(mm, ib_umem_start(umem));
+ vma = find_vma(mm, ib_umem_start(umem_odp));
if (!vma || !is_vm_hugetlb_page(vma)) {
up_read(&mm->mmap_sem);
return -EINVAL;
}
h = hstate_vma(vma);
- umem->page_shift = huge_page_shift(h);
+ umem_odp->page_shift = huge_page_shift(h);
up_read(&mm->mmap_sem);
}
@@ -424,16 +425,16 @@ int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
init_completion(&umem_odp->notifier_completion);
- if (ib_umem_num_pages(umem)) {
+ if (ib_umem_odp_num_pages(umem_odp)) {
umem_odp->page_list =
vzalloc(array_size(sizeof(*umem_odp->page_list),
- ib_umem_num_pages(umem)));
+ ib_umem_odp_num_pages(umem_odp)));
if (!umem_odp->page_list)
return -ENOMEM;
umem_odp->dma_list =
vzalloc(array_size(sizeof(*umem_odp->dma_list),
- ib_umem_num_pages(umem)));
+ ib_umem_odp_num_pages(umem_odp)));
if (!umem_odp->dma_list) {
ret_val = -ENOMEM;
goto out_page_list;
@@ -456,16 +457,14 @@ out_page_list:
void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
{
- struct ib_umem *umem = &umem_odp->umem;
-
/*
* Ensure that no more pages are mapped in the umem.
*
* It is the driver's responsibility to ensure, before calling us,
* that the hardware will not attempt to access the MR any more.
*/
- ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem),
- ib_umem_end(umem));
+ ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
+ ib_umem_end(umem_odp));
remove_umem_from_per_mm(umem_odp);
put_per_mm(umem_odp);
@@ -487,7 +486,7 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
* The function returns -EFAULT if the DMA mapping operation fails. It returns
* -EAGAIN if a concurrent invalidation prevents us from updating the page.
*
- * The page is released via put_page even if the operation failed. For
+ * The page is released via put_user_page even if the operation failed. For
* on-demand pinning, the page is released whenever it isn't stored in the
* umem.
*/
@@ -498,8 +497,8 @@ static int ib_umem_odp_map_dma_single_page(
u64 access_mask,
unsigned long current_seq)
{
- struct ib_umem *umem = &umem_odp->umem;
- struct ib_device *dev = umem->context->device;
+ struct ib_ucontext *context = umem_odp->umem.context;
+ struct ib_device *dev = context->device;
dma_addr_t dma_addr;
int remove_existing_mapping = 0;
int ret = 0;
@@ -514,10 +513,9 @@ static int ib_umem_odp_map_dma_single_page(
goto out;
}
if (!(umem_odp->dma_list[page_index])) {
- dma_addr = ib_dma_map_page(dev,
- page,
- 0, BIT(umem->page_shift),
- DMA_BIDIRECTIONAL);
+ dma_addr =
+ ib_dma_map_page(dev, page, 0, BIT(umem_odp->page_shift),
+ DMA_BIDIRECTIONAL);
if (ib_dma_mapping_error(dev, dma_addr)) {
ret = -EFAULT;
goto out;
@@ -536,15 +534,16 @@ static int ib_umem_odp_map_dma_single_page(
}
out:
- put_page(page);
+ put_user_page(page);
if (remove_existing_mapping) {
ib_umem_notifier_start_account(umem_odp);
- umem->context->invalidate_range(
+ context->invalidate_range(
umem_odp,
- ib_umem_start(umem) + (page_index << umem->page_shift),
- ib_umem_start(umem) +
- ((page_index + 1) << umem->page_shift));
+ ib_umem_start(umem_odp) +
+ (page_index << umem_odp->page_shift),
+ ib_umem_start(umem_odp) +
+ ((page_index + 1) << umem_odp->page_shift));
ib_umem_notifier_end_account(umem_odp);
ret = -EAGAIN;
}
@@ -581,27 +580,26 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
u64 bcnt, u64 access_mask,
unsigned long current_seq)
{
- struct ib_umem *umem = &umem_odp->umem;
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = umem_odp->umem.owning_mm;
struct page **local_page_list = NULL;
u64 page_mask, off;
- int j, k, ret = 0, start_idx, npages = 0, page_shift;
- unsigned int flags = 0;
+ int j, k, ret = 0, start_idx, npages = 0;
+ unsigned int flags = 0, page_shift;
phys_addr_t p = 0;
if (access_mask == 0)
return -EINVAL;
- if (user_virt < ib_umem_start(umem) ||
- user_virt + bcnt > ib_umem_end(umem))
+ if (user_virt < ib_umem_start(umem_odp) ||
+ user_virt + bcnt > ib_umem_end(umem_odp))
return -EFAULT;
local_page_list = (struct page **)__get_free_page(GFP_KERNEL);
if (!local_page_list)
return -ENOMEM;
- page_shift = umem->page_shift;
+ page_shift = umem_odp->page_shift;
page_mask = ~(BIT(page_shift) - 1);
off = user_virt & (~page_mask);
user_virt = user_virt & page_mask;
@@ -621,7 +619,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
if (access_mask & ODP_WRITE_ALLOWED_BIT)
flags |= FOLL_WRITE;
- start_idx = (user_virt - ib_umem_start(umem)) >> page_shift;
+ start_idx = (user_virt - ib_umem_start(umem_odp)) >> page_shift;
k = start_idx;
while (bcnt > 0) {
@@ -659,7 +657,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
ret = -EFAULT;
break;
}
- put_page(local_page_list[j]);
+ put_user_page(local_page_list[j]);
continue;
}
@@ -686,8 +684,8 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
* ib_umem_odp_map_dma_single_page().
*/
if (npages - (j + 1) > 0)
- release_pages(&local_page_list[j+1],
- npages - (j + 1));
+ put_user_pages(&local_page_list[j+1],
+ npages - (j + 1));
break;
}
}
@@ -711,21 +709,20 @@ EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
u64 bound)
{
- struct ib_umem *umem = &umem_odp->umem;
int idx;
u64 addr;
- struct ib_device *dev = umem->context->device;
+ struct ib_device *dev = umem_odp->umem.context->device;
- virt = max_t(u64, virt, ib_umem_start(umem));
- bound = min_t(u64, bound, ib_umem_end(umem));
+ virt = max_t(u64, virt, ib_umem_start(umem_odp));
+ bound = min_t(u64, bound, ib_umem_end(umem_odp));
/* Note that during the run of this function, the
* notifiers_count of the MR is > 0, preventing any racing
* faults from completion. We might be racing with other
* invalidations, so we must make sure we free each page only
* once. */
mutex_lock(&umem_odp->umem_mutex);
- for (addr = virt; addr < bound; addr += BIT(umem->page_shift)) {
- idx = (addr - ib_umem_start(umem)) >> umem->page_shift;
+ for (addr = virt; addr < bound; addr += BIT(umem_odp->page_shift)) {
+ idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
if (umem_odp->page_list[idx]) {
struct page *page = umem_odp->page_list[idx];
dma_addr_t dma = umem_odp->dma_list[idx];
@@ -733,7 +730,8 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
WARN_ON(!dma_addr);
- ib_dma_unmap_page(dev, dma_addr, PAGE_SIZE,
+ ib_dma_unmap_page(dev, dma_addr,
+ BIT(umem_odp->page_shift),
DMA_BIDIRECTIONAL);
if (dma & ODP_WRITE_ALLOWED_BIT) {
struct page *head_page = compound_head(page);
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 671f07ba1fad..9f8a48016b41 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -54,6 +54,7 @@
#include <rdma/ib_mad.h>
#include <rdma/ib_user_mad.h>
+#include <rdma/rdma_netlink.h>
#include "core_priv.h"
@@ -744,7 +745,7 @@ found:
"process %s did not enable P_Key index support.\n",
current->comm);
dev_warn(&file->port->dev,
- " Documentation/infiniband/user_mad.txt has info on the new ABI.\n");
+ " Documentation/infiniband/user_mad.rst has info on the new ABI.\n");
}
}
@@ -1124,11 +1125,48 @@ static const struct file_operations umad_sm_fops = {
.llseek = no_llseek,
};
+static int ib_umad_get_nl_info(struct ib_device *ibdev, void *client_data,
+ struct ib_client_nl_info *res)
+{
+ struct ib_umad_device *umad_dev = client_data;
+
+ if (!rdma_is_port_valid(ibdev, res->port))
+ return -EINVAL;
+
+ res->abi = IB_USER_MAD_ABI_VERSION;
+ res->cdev = &umad_dev->ports[res->port - rdma_start_port(ibdev)].dev;
+
+ return 0;
+}
+
static struct ib_client umad_client = {
.name = "umad",
.add = ib_umad_add_one,
- .remove = ib_umad_remove_one
+ .remove = ib_umad_remove_one,
+ .get_nl_info = ib_umad_get_nl_info,
};
+MODULE_ALIAS_RDMA_CLIENT("umad");
+
+static int ib_issm_get_nl_info(struct ib_device *ibdev, void *client_data,
+ struct ib_client_nl_info *res)
+{
+ struct ib_umad_device *umad_dev =
+ ib_get_client_data(ibdev, &umad_client);
+
+ if (!rdma_is_port_valid(ibdev, res->port))
+ return -EINVAL;
+
+ res->abi = IB_USER_MAD_ABI_VERSION;
+ res->cdev = &umad_dev->ports[res->port - rdma_start_port(ibdev)].sm_dev;
+
+ return 0;
+}
+
+static struct ib_client issm_client = {
+ .name = "issm",
+ .get_nl_info = ib_issm_get_nl_info,
+};
+MODULE_ALIAS_RDMA_CLIENT("issm");
static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr,
char *buf)
@@ -1387,13 +1425,17 @@ static int __init ib_umad_init(void)
}
ret = ib_register_client(&umad_client);
- if (ret) {
- pr_err("couldn't register ib_umad client\n");
+ if (ret)
goto out_class;
- }
+
+ ret = ib_register_client(&issm_client);
+ if (ret)
+ goto out_client;
return 0;
+out_client:
+ ib_unregister_client(&umad_client);
out_class:
class_unregister(&umad_class);
@@ -1411,6 +1453,7 @@ out:
static void __exit ib_umad_cleanup(void)
{
+ ib_unregister_client(&issm_client);
ib_unregister_client(&umad_client);
class_unregister(&umad_class);
unregister_chrdev_region(base_umad_dev,
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 63fe14c7c68f..750c4d484329 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -756,7 +756,9 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
mr->device = pd->device;
mr->pd = pd;
+ mr->type = IB_MR_TYPE_USER;
mr->dm = NULL;
+ mr->sig_attrs = NULL;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
mr->res.type = RDMA_RESTRACK_MR;
@@ -1021,12 +1023,11 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
attr.comp_vector = cmd->comp_vector;
attr.flags = cmd->flags;
- cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
- if (IS_ERR(cq)) {
- ret = PTR_ERR(cq);
+ cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
+ if (!cq) {
+ ret = -ENOMEM;
goto err_file;
}
-
cq->device = ib_dev;
cq->uobject = &obj->uobject;
cq->comp_handler = ib_uverbs_comp_handler;
@@ -1034,6 +1035,10 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
atomic_set(&cq->usecnt, 0);
+ ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
+ if (ret)
+ goto err_free;
+
obj->uobject.object = cq;
memset(&resp, 0, sizeof resp);
resp.base.cq_handle = obj->uobject.id;
@@ -1054,7 +1059,9 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
err_cb:
ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
-
+ cq = NULL;
+err_free:
+ kfree(cq);
err_file:
if (ev_file)
ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
@@ -3715,9 +3722,6 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
* trailing driver_data flex array. In this case the size of the base struct
* cannot be changed.
*/
-#define offsetof_after(_struct, _member) \
- (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
-
#define UAPI_DEF_WRITE_IO(req, resp) \
.write.has_resp = 1 + \
BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
@@ -3748,11 +3752,11 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
*/
#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
.write.has_resp = 1, \
- .write.req_size = offsetof_after(req, req_last_member), \
- .write.resp_size = offsetof_after(resp, resp_last_member)
+ .write.req_size = offsetofend(req, req_last_member), \
+ .write.resp_size = offsetofend(resp, resp_last_member)
#define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
- .write.req_size = offsetof_after(req, req_last_member)
+ .write.req_size = offsetofend(req, req_last_member)
const struct uapi_definition uverbs_def_write_intf[] = {
DECLARE_UVERBS_OBJECT(
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 84a5e9a6d483..11c13c1381cf 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -51,6 +51,7 @@
#include <rdma/ib.h>
#include <rdma/uverbs_std_types.h>
+#include <rdma/rdma_netlink.h>
#include "uverbs.h"
#include "core_priv.h"
@@ -198,7 +199,7 @@ void ib_uverbs_release_file(struct kref *ref)
ib_dev = srcu_dereference(file->device->ib_dev,
&file->device->disassociate_srcu);
if (ib_dev && !ib_dev->ops.disassociate_ucontext)
- module_put(ib_dev->owner);
+ module_put(ib_dev->ops.owner);
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
if (atomic_dec_and_test(&file->device->refcount))
@@ -1065,7 +1066,7 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
module_dependent = !(ib_dev->ops.disassociate_ucontext);
if (module_dependent) {
- if (!try_module_get(ib_dev->owner)) {
+ if (!try_module_get(ib_dev->ops.owner)) {
ret = -ENODEV;
goto err;
}
@@ -1100,7 +1101,7 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
return stream_open(inode, filp);
err_module:
- module_put(ib_dev->owner);
+ module_put(ib_dev->ops.owner);
err:
mutex_unlock(&dev->lists_mutex);
@@ -1148,12 +1149,41 @@ static const struct file_operations uverbs_mmap_fops = {
.compat_ioctl = ib_uverbs_ioctl,
};
+static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data,
+ struct ib_client_nl_info *res)
+{
+ struct ib_uverbs_device *uverbs_dev = client_data;
+ int ret;
+
+ if (res->port != -1)
+ return -EINVAL;
+
+ res->abi = ibdev->ops.uverbs_abi_ver;
+ res->cdev = &uverbs_dev->dev;
+
+ /*
+ * To support DRIVER_ID binding in userspace some of the driver need
+ * upgrading to expose their PCI dependent revision information
+ * through get_context instead of relying on modalias matching. When
+ * the drivers are fixed they can drop this flag.
+ */
+ if (!ibdev->ops.uverbs_no_driver_id_binding) {
+ ret = nla_put_u32(res->nl_msg, RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID,
+ ibdev->ops.driver_id);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
static struct ib_client uverbs_client = {
.name = "uverbs",
.no_kverbs_req = true,
.add = ib_uverbs_add_one,
- .remove = ib_uverbs_remove_one
+ .remove = ib_uverbs_remove_one,
+ .get_nl_info = ib_uverbs_get_nl_info,
};
+MODULE_ALIAS_RDMA_CLIENT("uverbs");
static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,
char *buf)
@@ -1186,7 +1216,7 @@ static ssize_t abi_version_show(struct device *device,
srcu_key = srcu_read_lock(&dev->disassociate_srcu);
ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
if (ib_dev)
- ret = sprintf(buf, "%d\n", ib_dev->uverbs_abi_ver);
+ ret = sprintf(buf, "%u\n", ib_dev->ops.uverbs_abi_ver);
srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
return ret;
diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index 07ea4e3c4566..e39fe6a8aac4 100644
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -111,9 +111,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
INIT_LIST_HEAD(&obj->comp_list);
INIT_LIST_HEAD(&obj->async_list);
- cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
- if (IS_ERR(cq)) {
- ret = PTR_ERR(cq);
+ cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
+ if (!cq) {
+ ret = -ENOMEM;
goto err_event_file;
}
@@ -122,10 +122,15 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
cq->comp_handler = ib_uverbs_comp_handler;
cq->event_handler = ib_uverbs_cq_event_handler;
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
- obj->uobject.object = cq;
- obj->uobject.user_handle = user_handle;
atomic_set(&cq->usecnt, 0);
cq->res.type = RDMA_RESTRACK_CQ;
+
+ ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
+ if (ret)
+ goto err_free;
+
+ obj->uobject.object = cq;
+ obj->uobject.user_handle = user_handle;
rdma_restrack_uadd(&cq->res);
ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe,
@@ -136,7 +141,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
return 0;
err_cq:
ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
-
+ cq = NULL;
+err_free:
+ kfree(cq);
err_event_file:
if (ev_file)
uverbs_uobject_put(ev_file_uobj);
diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c
index 997f7a3a558a..c1286a52dc84 100644
--- a/drivers/infiniband/core/uverbs_std_types_mr.c
+++ b/drivers/infiniband/core/uverbs_std_types_mr.c
@@ -128,6 +128,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(
mr->device = pd->device;
mr->pd = pd;
+ mr->type = IB_MR_TYPE_DM;
mr->dm = dm;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
index 7a987acf0c0b..00c547887132 100644
--- a/drivers/infiniband/core/uverbs_uapi.c
+++ b/drivers/infiniband/core/uverbs_uapi.c
@@ -22,6 +22,8 @@ static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
return ERR_PTR(-EOVERFLOW);
elm = kzalloc(alloc_size, GFP_KERNEL);
+ if (!elm)
+ return ERR_PTR(-ENOMEM);
rc = radix_tree_insert(&uapi->radix, key, elm);
if (rc) {
kfree(elm);
@@ -645,7 +647,7 @@ struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
return ERR_PTR(-ENOMEM);
INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
- uapi->driver_id = ibdev->driver_id;
+ uapi->driver_id = ibdev->ops.driver_id;
rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
if (rc)
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index e666a1f7608d..4a04e94a72db 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -209,7 +209,7 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
EXPORT_SYMBOL(ib_rate_to_mbps);
__attribute_const__ enum rdma_transport_type
-rdma_node_get_transport(enum rdma_node_type node_type)
+rdma_node_get_transport(unsigned int node_type)
{
if (node_type == RDMA_NODE_USNIC)
@@ -299,6 +299,7 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
mr->device = pd->device;
mr->pd = pd;
+ mr->type = IB_MR_TYPE_DMA;
mr->uobject = NULL;
mr->need_inval = false;
@@ -316,7 +317,7 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
EXPORT_SYMBOL(__ib_alloc_pd);
/**
- * ib_dealloc_pd - Deallocates a protection domain.
+ * ib_dealloc_pd_user - Deallocates a protection domain.
* @pd: The protection domain to deallocate.
* @udata: Valid user data or NULL for kernel object
*
@@ -1157,6 +1158,10 @@ struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
qp_init_attr->cap.max_recv_sge))
return ERR_PTR(-EINVAL);
+ if ((qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) &&
+ !(device->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER))
+ return ERR_PTR(-EINVAL);
+
/*
* If the callers is using the RDMA API calculate the resources
* needed for the RDMA READ/WRITE operations.
@@ -1232,6 +1237,8 @@ struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
qp->max_write_sge = qp_init_attr->cap.max_send_sge;
qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
device->attrs.max_sge_rd);
+ if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
+ qp->integrity_en = true;
return qp;
@@ -1916,21 +1923,28 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
const char *caller)
{
struct ib_cq *cq;
+ int ret;
- cq = device->ops.create_cq(device, cq_attr, NULL);
-
- if (!IS_ERR(cq)) {
- cq->device = device;
- cq->uobject = NULL;
- cq->comp_handler = comp_handler;
- cq->event_handler = event_handler;
- cq->cq_context = cq_context;
- atomic_set(&cq->usecnt, 0);
- cq->res.type = RDMA_RESTRACK_CQ;
- rdma_restrack_set_task(&cq->res, caller);
- rdma_restrack_kadd(&cq->res);
+ cq = rdma_zalloc_drv_obj(device, ib_cq);
+ if (!cq)
+ return ERR_PTR(-ENOMEM);
+
+ cq->device = device;
+ cq->uobject = NULL;
+ cq->comp_handler = comp_handler;
+ cq->event_handler = event_handler;
+ cq->cq_context = cq_context;
+ atomic_set(&cq->usecnt, 0);
+ cq->res.type = RDMA_RESTRACK_CQ;
+ rdma_restrack_set_task(&cq->res, caller);
+
+ ret = device->ops.create_cq(cq, cq_attr, NULL);
+ if (ret) {
+ kfree(cq);
+ return ERR_PTR(ret);
}
+ rdma_restrack_kadd(&cq->res);
return cq;
}
EXPORT_SYMBOL(__ib_create_cq);
@@ -1949,7 +1963,9 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
return -EBUSY;
rdma_restrack_del(&cq->res);
- return cq->device->ops.destroy_cq(cq, udata);
+ cq->device->ops.destroy_cq(cq, udata);
+ kfree(cq);
+ return 0;
}
EXPORT_SYMBOL(ib_destroy_cq_user);
@@ -1966,6 +1982,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
{
struct ib_pd *pd = mr->pd;
struct ib_dm *dm = mr->dm;
+ struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
int ret;
rdma_restrack_del(&mr->res);
@@ -1974,6 +1991,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
atomic_dec(&pd->usecnt);
if (dm)
atomic_dec(&dm->usecnt);
+ kfree(sig_attrs);
}
return ret;
@@ -1981,7 +1999,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
EXPORT_SYMBOL(ib_dereg_mr_user);
/**
- * ib_alloc_mr() - Allocates a memory region
+ * ib_alloc_mr_user() - Allocates a memory region
* @pd: protection domain associated with the region
* @mr_type: memory region type
* @max_num_sg: maximum sg entries available for registration.
@@ -2001,6 +2019,9 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
if (!pd->device->ops.alloc_mr)
return ERR_PTR(-EOPNOTSUPP);
+ if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY))
+ return ERR_PTR(-EINVAL);
+
mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
if (!IS_ERR(mr)) {
mr->device = pd->device;
@@ -2011,12 +2032,66 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
mr->need_inval = false;
mr->res.type = RDMA_RESTRACK_MR;
rdma_restrack_kadd(&mr->res);
+ mr->type = mr_type;
+ mr->sig_attrs = NULL;
}
return mr;
}
EXPORT_SYMBOL(ib_alloc_mr_user);
+/**
+ * ib_alloc_mr_integrity() - Allocates an integrity memory region
+ * @pd: protection domain associated with the region
+ * @max_num_data_sg: maximum data sg entries available for registration
+ * @max_num_meta_sg: maximum metadata sg entries available for
+ * registration
+ *
+ * Notes:
+ * Memory registration page/sg lists must not exceed max_num_sg,
+ * also the integrity page/sg lists must not exceed max_num_meta_sg.
+ *
+ */
+struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_data_sg,
+ u32 max_num_meta_sg)
+{
+ struct ib_mr *mr;
+ struct ib_sig_attrs *sig_attrs;
+
+ if (!pd->device->ops.alloc_mr_integrity ||
+ !pd->device->ops.map_mr_sg_pi)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (!max_num_meta_sg)
+ return ERR_PTR(-EINVAL);
+
+ sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
+ if (!sig_attrs)
+ return ERR_PTR(-ENOMEM);
+
+ mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
+ max_num_meta_sg);
+ if (IS_ERR(mr)) {
+ kfree(sig_attrs);
+ return mr;
+ }
+
+ mr->device = pd->device;
+ mr->pd = pd;
+ mr->dm = NULL;
+ mr->uobject = NULL;
+ atomic_inc(&pd->usecnt);
+ mr->need_inval = false;
+ mr->res.type = RDMA_RESTRACK_MR;
+ rdma_restrack_kadd(&mr->res);
+ mr->type = IB_MR_TYPE_INTEGRITY;
+ mr->sig_attrs = sig_attrs;
+
+ return mr;
+}
+EXPORT_SYMBOL(ib_alloc_mr_integrity);
+
/* "Fast" memory regions */
struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
@@ -2226,19 +2301,17 @@ EXPORT_SYMBOL(ib_create_wq);
*/
int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
{
- int err;
struct ib_cq *cq = wq->cq;
struct ib_pd *pd = wq->pd;
if (atomic_read(&wq->usecnt))
return -EBUSY;
- err = wq->device->ops.destroy_wq(wq, udata);
- if (!err) {
- atomic_dec(&pd->usecnt);
- atomic_dec(&cq->usecnt);
- }
- return err;
+ wq->device->ops.destroy_wq(wq, udata);
+ atomic_dec(&pd->usecnt);
+ atomic_dec(&cq->usecnt);
+
+ return 0;
}
EXPORT_SYMBOL(ib_destroy_wq);
@@ -2376,6 +2449,43 @@ int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,
EXPORT_SYMBOL(ib_set_vf_guid);
/**
+ * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
+ * information) and set an appropriate memory region for registration.
+ * @mr: memory region
+ * @data_sg: dma mapped scatterlist for data
+ * @data_sg_nents: number of entries in data_sg
+ * @data_sg_offset: offset in bytes into data_sg
+ * @meta_sg: dma mapped scatterlist for metadata
+ * @meta_sg_nents: number of entries in meta_sg
+ * @meta_sg_offset: offset in bytes into meta_sg
+ * @page_size: page vector desired page size
+ *
+ * Constraints:
+ * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
+ *
+ * Return: 0 on success.
+ *
+ * After this completes successfully, the memory region
+ * is ready for registration.
+ */
+int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset, unsigned int page_size)
+{
+ if (unlikely(!mr->device->ops.map_mr_sg_pi ||
+ WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
+ return -EOPNOTSUPP;
+
+ mr->page_size = page_size;
+
+ return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
+ data_sg_offset, meta_sg,
+ meta_sg_nents, meta_sg_offset);
+}
+EXPORT_SYMBOL(ib_map_mr_sg_pi);
+
+/**
* ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
* and set it the memory region.
* @mr: memory region
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index 77094be1b262..433fca59febd 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -7,7 +7,6 @@ obj-$(CONFIG_INFINIBAND_EFA) += efa/
obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
-obj-$(CONFIG_INFINIBAND_NES) += nes/
obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/
obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA) += vmw_pvrdma/
obj-$(CONFIG_INFINIBAND_USNIC) += usnic/
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 2c3685faa57a..a91653aabf38 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -805,10 +805,8 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
rdev->sqp_ah = NULL;
}
- if (!IS_ERR_OR_NULL(qp->rumem))
- ib_umem_release(qp->rumem);
- if (!IS_ERR_OR_NULL(qp->sumem))
- ib_umem_release(qp->sumem);
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
mutex_lock(&rdev->qp_lock);
list_del(&qp->list);
@@ -1201,12 +1199,8 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
qp_destroy:
bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
free_umem:
- if (udata) {
- if (qp->rumem)
- ib_umem_release(qp->rumem);
- if (qp->sumem)
- ib_umem_release(qp->sumem);
- }
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
fail:
kfree(qp);
return ERR_PTR(rc);
@@ -1302,8 +1296,7 @@ void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
if (qplib_srq->cq)
nq = qplib_srq->cq->nq;
bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
- if (srq->umem)
- ib_umem_release(srq->umem);
+ ib_umem_release(srq->umem);
atomic_dec(&rdev->srq_count);
if (nq)
nq->budget--;
@@ -1412,8 +1405,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
return 0;
fail:
- if (srq->umem)
- ib_umem_release(srq->umem);
+ ib_umem_release(srq->umem);
exit:
return rc;
}
@@ -2517,9 +2509,8 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
}
/* Completion Queues */
-int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
- int rc;
struct bnxt_re_cq *cq;
struct bnxt_qplib_nq *nq;
struct bnxt_re_dev *rdev;
@@ -2528,29 +2519,20 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
rdev = cq->rdev;
nq = cq->qplib_cq.nq;
- rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
- if (rc) {
- dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");
- return rc;
- }
- if (!IS_ERR_OR_NULL(cq->umem))
- ib_umem_release(cq->umem);
+ bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
+ ib_umem_release(cq->umem);
atomic_dec(&rdev->cq_count);
nq->budget--;
kfree(cq->cql);
- kfree(cq);
-
- return 0;
}
-struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
- struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
- struct bnxt_re_cq *cq = NULL;
+ struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
int rc, entries;
int cqe = attr->cqe;
struct bnxt_qplib_nq *nq = NULL;
@@ -2559,11 +2541,8 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
/* Validate CQ fields */
if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
cq->rdev = rdev;
cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
@@ -2641,15 +2620,13 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
}
}
- return &cq->ib_cq;
+ return 0;
c2fail:
- if (udata)
- ib_umem_release(cq->umem);
+ ib_umem_release(cq->umem);
fail:
kfree(cq->cql);
- kfree(cq);
- return ERR_PTR(rc);
+ return rc;
}
static u8 __req_to_ib_wc_status(u8 qstatus)
@@ -3353,8 +3330,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
mr->npages = 0;
mr->pages = NULL;
}
- if (!IS_ERR_OR_NULL(mr->ib_umem))
- ib_umem_release(mr->ib_umem);
+ ib_umem_release(mr->ib_umem);
kfree(mr);
atomic_dec(&rdev->mr_count);
@@ -3630,10 +3606,10 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
u32 chip_met_rev_num = 0;
int rc;
- dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",
- ibdev->uverbs_abi_ver);
+ dev_dbg(rdev_to_dev(rdev), "ABI version requested %u",
+ ibdev->ops.uverbs_abi_ver);
- if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
+ if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",
BNXT_RE_ABI_VERSION);
return -EPERM;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 09a33049e42f..31662b1ee35a 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -94,11 +94,11 @@ struct bnxt_re_qp {
};
struct bnxt_re_cq {
+ struct ib_cq ib_cq;
struct bnxt_re_dev *rdev;
spinlock_t cq_lock; /* protect cq */
u16 cq_count;
u16 cq_period;
- struct ib_cq ib_cq;
struct bnxt_qplib_cq qplib_cq;
struct bnxt_qplib_cqe *cql;
#define MAX_CQL_PER_POLL 1024
@@ -190,10 +190,9 @@ int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr,
const struct ib_send_wr **bad_send_wr);
int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
const struct ib_recv_wr **bad_recv_wr);
-struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 814f959c7db9..029babe713f3 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -596,6 +596,10 @@ static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)
}
static const struct ib_device_ops bnxt_re_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_BNXT_RE,
+ .uverbs_abi_ver = BNXT_RE_ABI_VERSION,
+
.add_gid = bnxt_re_add_gid,
.alloc_hw_stats = bnxt_re_ib_alloc_hw_stats,
.alloc_mr = bnxt_re_alloc_mr,
@@ -637,6 +641,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
.reg_user_mr = bnxt_re_reg_user_mr,
.req_notify_cq = bnxt_re_req_notify_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
@@ -648,7 +653,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
int ret;
/* ib device init */
- ibdev->owner = THIS_MODULE;
ibdev->node_type = RDMA_NODE_IB_CA;
strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
strlen(BNXT_RE_DESC) + 5);
@@ -661,7 +665,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
/* User space */
- ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
ibdev->uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -691,7 +694,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group);
- ibdev->driver_id = RDMA_DRIVER_BNXT_RE;
ib_set_device_ops(ibdev, &bnxt_re_dev_ops);
ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
if (ret)
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c
index 8ac72ac7cbac..37ee93824349 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.c
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c
@@ -187,20 +187,6 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)
return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
}
-#ifdef notyet
-int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
-{
- struct rdma_cq_setup setup;
- setup.id = cq->cqid;
- setup.base_addr = (u64) (cq->dma_addr);
- setup.size = 1UL << cq->size_log2;
- setup.credits = setup.size;
- setup.credit_thres = setup.size; /* TBD: overflow recovery */
- setup.ovfl_mode = 1;
- return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
-}
-#endif
-
static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
{
struct cxio_qpid_list *entry;
@@ -219,7 +205,7 @@ static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
if (!qpid)
goto out;
for (i = qpid+1; i & rdev_p->qpmask; i++) {
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
break;
entry->qpid = i;
@@ -237,7 +223,7 @@ static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
{
struct cxio_qpid_list *entry;
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return;
pr_debug("%s qpid 0x%x\n", __func__, qpid);
@@ -317,17 +303,15 @@ err1:
return -ENOMEM;
}
-int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
+void cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
{
- int err;
- err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
+ cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
kfree(cq->sw_queue);
dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),
(1UL << (cq->size_log2))
* sizeof(struct t3_cqe) + 1, cq->queue,
dma_unmap_addr(cq, mapping));
cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
- return err;
}
int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
@@ -565,9 +549,9 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
wqe->sge_cmd = cpu_to_be64(sge_cmd);
wqe->ctx1 = cpu_to_be64(ctx1);
wqe->ctx0 = cpu_to_be64(ctx0);
- pr_debug("CtrlQP dma_addr 0x%llx workq %p size %d\n",
- (unsigned long long)rdev_p->ctrl_qp.dma_addr,
- rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
+ pr_debug("CtrlQP dma_addr %pad workq %p size %d\n",
+ &rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq,
+ 1 << T3_CTRL_QP_SIZE_LOG2);
skb->priority = CPL_PRIORITY_CONTROL;
return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
err:
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h
index c64e50b5a548..40c029ffa425 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h
@@ -158,8 +158,7 @@ void cxio_rdev_close(struct cxio_rdev *rdev);
int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq,
enum t3_cq_opcode op, u32 credit);
int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq, int kernel);
-int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
-int cxio_resize_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
+void cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);
void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);
int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq,
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index 1c90c86fc8b8..0bca72cb4d9a 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -170,7 +170,7 @@ static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)
{
struct cpl_tid_release *req;
- skb = get_skb(skb, sizeof *req, GFP_KERNEL);
+ skb = get_skb(skb, sizeof(*req), GFP_KERNEL);
if (!skb)
return;
req = skb_put(skb, sizeof(*req));
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 3a481dfb1607..e775c1a1a450 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -88,7 +88,7 @@ static int iwch_alloc_ucontext(struct ib_ucontext *ucontext,
return 0;
}
-static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+static void iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
struct iwch_cq *chp;
@@ -100,17 +100,16 @@ static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
wait_event(chp->wait, !atomic_read(&chp->refcnt));
cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
- kfree(chp);
- return 0;
}
-static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+static int iwch_create_cq(struct ib_cq *ibcq,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
- struct iwch_dev *rhp;
- struct iwch_cq *chp;
+ struct iwch_dev *rhp = to_iwch_dev(ibcq->device);
+ struct iwch_cq *chp = to_iwch_cq(ibcq);
struct iwch_create_cq_resp uresp;
struct iwch_create_cq_req ureq;
static int warned;
@@ -118,19 +117,13 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- rhp = to_iwch_dev(ibdev);
- chp = kzalloc(sizeof(*chp), GFP_KERNEL);
- if (!chp)
- return ERR_PTR(-ENOMEM);
+ return -EINVAL;
if (udata) {
if (!t3a_device(rhp)) {
- if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
- kfree(chp);
- return ERR_PTR(-EFAULT);
- }
+ if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
+ return -EFAULT;
+
chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
}
}
@@ -151,10 +144,9 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
entries = roundup_pow_of_two(entries);
chp->cq.size_log2 = ilog2(entries);
- if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) {
- kfree(chp);
- return ERR_PTR(-ENOMEM);
- }
+ if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata))
+ return -ENOMEM;
+
chp->rhp = rhp;
chp->ibcq.cqe = 1 << chp->cq.size_log2;
spin_lock_init(&chp->lock);
@@ -163,8 +155,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
init_waitqueue_head(&chp->wait);
if (xa_store_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL)) {
cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
- kfree(chp);
- return ERR_PTR(-ENOMEM);
+ return -ENOMEM;
}
if (udata) {
@@ -172,10 +163,10 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
struct iwch_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct iwch_ucontext, ibucontext);
- mm = kmalloc(sizeof *mm, GFP_KERNEL);
+ mm = kmalloc(sizeof(*mm), GFP_KERNEL);
if (!mm) {
iwch_destroy_cq(&chp->ibcq, udata);
- return ERR_PTR(-ENOMEM);
+ return -ENOMEM;
}
uresp.cqid = chp->cq.cqid;
uresp.size_log2 = chp->cq.size_log2;
@@ -185,7 +176,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
spin_unlock(&ucontext->mmap_lock);
mm->key = uresp.key;
mm->addr = virt_to_phys(chp->cq.queue);
- if (udata->outlen < sizeof uresp) {
+ if (udata->outlen < sizeof(uresp)) {
if (!warned++)
pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");
mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
@@ -196,86 +187,19 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
sizeof(struct t3_cqe));
uresp.memsize = mm->len;
uresp.reserved = 0;
- resplen = sizeof uresp;
+ resplen = sizeof(uresp);
}
if (ib_copy_to_udata(udata, &uresp, resplen)) {
kfree(mm);
iwch_destroy_cq(&chp->ibcq, udata);
- return ERR_PTR(-EFAULT);
+ return -EFAULT;
}
insert_mmap(ucontext, mm);
}
- pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
+ pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr %pad\n",
chp->cq.cqid, chp, (1 << chp->cq.size_log2),
- (unsigned long long)chp->cq.dma_addr);
- return &chp->ibcq;
-}
-
-static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
-{
-#ifdef notyet
- struct iwch_cq *chp = to_iwch_cq(cq);
- struct t3_cq oldcq, newcq;
- int ret;
-
- pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
-
- /* We don't downsize... */
- if (cqe <= cq->cqe)
- return 0;
-
- /* create new t3_cq with new size */
- cqe = roundup_pow_of_two(cqe+1);
- newcq.size_log2 = ilog2(cqe);
-
- /* Dont allow resize to less than the current wce count */
- if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
- return -ENOMEM;
- }
-
- /* Quiesce all QPs using this CQ */
- ret = iwch_quiesce_qps(chp);
- if (ret) {
- return ret;
- }
-
- ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
- if (ret) {
- return ret;
- }
-
- /* copy CQEs */
- memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
- sizeof(struct t3_cqe));
-
- /* old iwch_qp gets new t3_cq but keeps old cqid */
- oldcq = chp->cq;
- chp->cq = newcq;
- chp->cq.cqid = oldcq.cqid;
-
- /* resize new t3_cq to update the HW context */
- ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
- if (ret) {
- chp->cq = oldcq;
- return ret;
- }
- chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
-
- /* destroy old t3_cq */
- oldcq.cqid = newcq.cqid;
- ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
- if (ret) {
- pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);
- }
-
- /* add user hooks here */
-
- /* resume qps */
- ret = iwch_resume_qps(chp);
- return ret;
-#else
- return -ENOSYS;
-#endif
+ &chp->cq.dma_addr);
+ return 0;
}
static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
@@ -422,8 +346,7 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
xa_erase_irq(&rhp->mrs, mmid);
if (mhp->kva)
kfree((void *) (unsigned long) mhp->kva);
- if (mhp->umem)
- ib_umem_release(mhp->umem);
+ ib_umem_release(mhp->umem);
pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
kfree(mhp);
return 0;
@@ -553,7 +476,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) {
pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter));
- if (i == PAGE_SIZE / sizeof *pages) {
+ if (i == PAGE_SIZE / sizeof(*pages)) {
err = iwch_write_pbl(mhp, pages, i, n);
if (err)
goto pbl_done;
@@ -587,7 +510,7 @@ pbl_done:
pr_debug("%s user resp pbl_addr 0x%x\n", __func__,
uresp.pbl_addr);
- if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
+ if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
iwch_dereg_mr(&mhp->ibmr, udata);
err = -EFAULT;
goto err;
@@ -880,13 +803,13 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
struct iwch_mm_entry *mm1, *mm2;
- mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
+ mm1 = kmalloc(sizeof(*mm1), GFP_KERNEL);
if (!mm1) {
iwch_destroy_qp(&qhp->ibqp, udata);
return ERR_PTR(-ENOMEM);
}
- mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
+ mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL);
if (!mm2) {
kfree(mm1);
iwch_destroy_qp(&qhp->ibqp, udata);
@@ -903,7 +826,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
uresp.db_key = ucontext->key;
ucontext->key += PAGE_SIZE;
spin_unlock(&ucontext->mmap_lock);
- if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
+ if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
kfree(mm1);
kfree(mm2);
iwch_destroy_qp(&qhp->ibqp, udata);
@@ -911,7 +834,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
}
mm1->key = uresp.key;
mm1->addr = virt_to_phys(qhp->wq.queue);
- mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
+ mm1->len = PAGE_ALIGN(wqsize * sizeof(union t3_wr));
insert_mmap(ucontext, mm1);
mm2->key = uresp.db_key;
mm2->addr = qhp->wq.udb & PAGE_MASK;
@@ -919,10 +842,11 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
insert_mmap(ucontext, mm2);
}
qhp->ibqp.qp_num = qhp->wq.qpid;
- pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
- __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
- qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr,
- 1 << qhp->wq.size_log2, qhp->wq.rq_addr);
+ pr_debug(
+ "%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr %pad size %d rq_addr 0x%x\n",
+ __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
+ qhp->wq.qpid, qhp, &qhp->wq.dma_addr, 1 << qhp->wq.size_log2,
+ qhp->wq.rq_addr);
return &qhp->ibqp;
}
@@ -932,7 +856,7 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
struct iwch_dev *rhp;
struct iwch_qp *qhp;
enum iwch_qp_attr_mask mask = 0;
- struct iwch_qp_attributes attrs;
+ struct iwch_qp_attributes attrs = {};
pr_debug("%s ib_qp %p\n", __func__, ibqp);
@@ -944,7 +868,6 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (!attr_mask)
return 0;
- memset(&attrs, 0, sizeof attrs);
qhp = to_iwch_qp(ibqp);
rhp = qhp->rhp;
@@ -1040,7 +963,6 @@ static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
return -EINVAL;
dev = to_iwch_dev(ibdev);
- memset(props, 0, sizeof *props);
memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
props->hw_ver = dev->rdev.t3cdev_p->type;
props->fw_ver = fw_vers_string_to_u64(dev);
@@ -1304,6 +1226,11 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
}
static const struct ib_device_ops iwch_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_CXGB3,
+ .uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION,
+ .uverbs_no_driver_id_binding = 1,
+
.alloc_hw_stats = iwch_alloc_stats,
.alloc_mr = iwch_alloc_mr,
.alloc_mw = iwch_alloc_mw,
@@ -1341,8 +1268,8 @@ static const struct ib_device_ops iwch_dev_ops = {
.query_port = iwch_query_port,
.reg_user_mr = iwch_reg_user_mr,
.req_notify_cq = iwch_arm_cq,
- .resize_cq = iwch_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd),
+ INIT_RDMA_OBJ_SIZE(ib_cq, iwch_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext),
};
@@ -1351,7 +1278,6 @@ int iwch_register_device(struct iwch_dev *dev)
pr_debug("%s iwch_dev %p\n", __func__, dev);
memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
- dev->ibdev.owner = THIS_MODULE;
dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
IB_DEVICE_MEM_WINDOW |
IB_DEVICE_MEM_MGT_EXTENSIONS;
@@ -1383,12 +1309,10 @@ int iwch_register_device(struct iwch_dev *dev)
dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
dev->ibdev.num_comp_vectors = 1;
dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
- dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name,
sizeof(dev->ibdev.iw_ifname));
- dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
return ib_register_device(&dev->ibdev, "cxgb3_%d");
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 0f3b1193d5f8..0147c407ac6c 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -953,7 +953,7 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb,
mpalen = sizeof(*mpa) + ep->plen;
if (mpa_rev_to_use == 2)
mpalen += sizeof(struct mpa_v2_conn_params);
- wrlen = roundup(mpalen + sizeof *req, 16);
+ wrlen = roundup(mpalen + sizeof(*req), 16);
skb = get_skb(skb, wrlen, GFP_KERNEL);
if (!skb) {
connect_reply_upcall(ep, -ENOMEM);
@@ -997,8 +997,9 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb,
}
if (mpa_rev_to_use == 2) {
- mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
- sizeof (struct mpa_v2_conn_params));
+ mpa->private_data_size =
+ htons(ntohs(mpa->private_data_size) +
+ sizeof(struct mpa_v2_conn_params));
pr_debug("initiator ird %u ord %u\n", ep->ird,
ep->ord);
mpa_v2_params.ird = htons((u16)ep->ird);
@@ -1057,7 +1058,7 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
mpalen = sizeof(*mpa) + plen;
if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn)
mpalen += sizeof(struct mpa_v2_conn_params);
- wrlen = roundup(mpalen + sizeof *req, 16);
+ wrlen = roundup(mpalen + sizeof(*req), 16);
skb = get_skb(NULL, wrlen, GFP_KERNEL);
if (!skb) {
@@ -1088,8 +1089,9 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
mpa->flags |= MPA_ENHANCED_RDMA_CONN;
- mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
- sizeof (struct mpa_v2_conn_params));
+ mpa->private_data_size =
+ htons(ntohs(mpa->private_data_size) +
+ sizeof(struct mpa_v2_conn_params));
mpa_v2_params.ird = htons(((u16)ep->ird) |
(peer2peer ? MPA_V2_PEER2PEER_MODEL :
0));
@@ -1136,7 +1138,7 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
mpalen = sizeof(*mpa) + plen;
if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn)
mpalen += sizeof(struct mpa_v2_conn_params);
- wrlen = roundup(mpalen + sizeof *req, 16);
+ wrlen = roundup(mpalen + sizeof(*req), 16);
skb = get_skb(NULL, wrlen, GFP_KERNEL);
if (!skb) {
@@ -1171,8 +1173,9 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
mpa->flags |= MPA_ENHANCED_RDMA_CONN;
- mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
- sizeof (struct mpa_v2_conn_params));
+ mpa->private_data_size =
+ htons(ntohs(mpa->private_data_size) +
+ sizeof(struct mpa_v2_conn_params));
mpa_v2_params.ird = htons((u16)ep->ird);
mpa_v2_params.ord = htons((u16)ep->ord);
if (peer2peer && (ep->mpa_attr.p2p_type !=
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index 52ce586621c6..3cc4d3331a3f 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -34,16 +34,15 @@
#include "iw_cxgb4.h"
-static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
- struct c4iw_dev_ucontext *uctx, struct sk_buff *skb,
- struct c4iw_wr_wait *wr_waitp)
+static void destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
+ struct c4iw_dev_ucontext *uctx, struct sk_buff *skb,
+ struct c4iw_wr_wait *wr_waitp)
{
struct fw_ri_res_wr *res_wr;
struct fw_ri_res *res;
int wr_len;
- int ret;
- wr_len = sizeof *res_wr + sizeof *res;
+ wr_len = sizeof(*res_wr) + sizeof(*res);
set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
res_wr = __skb_put_zero(skb, wr_len);
@@ -59,14 +58,13 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
res->u.cq.iqid = cpu_to_be32(cq->cqid);
c4iw_init_wr_wait(wr_waitp);
- ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);
+ c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);
kfree(cq->sw_queue);
dma_free_coherent(&(rdev->lldi.pdev->dev),
cq->memsize, cq->queue,
dma_unmap_addr(cq, mapping));
c4iw_put_cqid(rdev, cq->cqid, uctx);
- return ret;
}
static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
@@ -117,7 +115,7 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
}
/* build fw_ri_res_wr */
- wr_len = sizeof *res_wr + sizeof *res;
+ wr_len = sizeof(*res_wr) + sizeof(*res);
skb = alloc_skb(wr_len, GFP_KERNEL);
if (!skb) {
@@ -970,7 +968,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
return !err || err == -ENODATA ? npolled : err;
}
-int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
struct c4iw_cq *chp;
struct c4iw_ucontext *ucontext;
@@ -988,18 +986,16 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx,
chp->destroy_skb, chp->wr_waitp);
c4iw_put_wr_wait(chp->wr_waitp);
- kfree(chp);
- return 0;
}
-struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
- struct c4iw_dev *rhp;
- struct c4iw_cq *chp;
+ struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device);
+ struct c4iw_cq *chp = to_c4iw_cq(ibcq);
struct c4iw_create_cq ucmd;
struct c4iw_create_cq_resp uresp;
int ret, wr_len;
@@ -1010,22 +1006,16 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
pr_debug("ib_dev %p entries %d\n", ibdev, entries);
if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- rhp = to_c4iw_dev(ibdev);
+ return -EINVAL;
if (vector >= rhp->rdev.lldi.nciq)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (udata) {
if (udata->inlen < sizeof(ucmd))
ucontext->is_32b_cqe = 1;
}
- chp = kzalloc(sizeof(*chp), GFP_KERNEL);
- if (!chp)
- return ERR_PTR(-ENOMEM);
-
chp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
if (!chp->wr_waitp) {
ret = -ENOMEM;
@@ -1095,10 +1085,10 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
if (ucontext) {
ret = -ENOMEM;
- mm = kmalloc(sizeof *mm, GFP_KERNEL);
+ mm = kmalloc(sizeof(*mm), GFP_KERNEL);
if (!mm)
goto err_remove_handle;
- mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
+ mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL);
if (!mm2)
goto err_free_mm;
@@ -1135,10 +1125,11 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
mm2->len = PAGE_SIZE;
insert_mmap(ucontext, mm2);
}
- pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n",
- chp->cq.cqid, chp, chp->cq.size,
- chp->cq.memsize, (unsigned long long)chp->cq.dma_addr);
- return &chp->ibcq;
+
+ pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr %pad\n",
+ chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize,
+ &chp->cq.dma_addr);
+ return 0;
err_free_mm2:
kfree(mm2);
err_free_mm:
@@ -1154,8 +1145,7 @@ err_free_skb:
err_free_wr_wait:
c4iw_put_wr_wait(chp->wr_waitp);
err_free_chp:
- kfree(chp);
- return ERR_PTR(ret);
+ return ret;
}
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 4c0d925c5ff5..a8b9548bd1a2 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -327,7 +327,7 @@ static int qp_open(struct inode *inode, struct file *file)
unsigned long index;
int count = 1;
- qpd = kmalloc(sizeof *qpd, GFP_KERNEL);
+ qpd = kmalloc(sizeof(*qpd), GFP_KERNEL);
if (!qpd)
return -ENOMEM;
@@ -421,7 +421,7 @@ static int stag_open(struct inode *inode, struct file *file)
int ret = 0;
int count = 1;
- stagd = kmalloc(sizeof *stagd, GFP_KERNEL);
+ stagd = kmalloc(sizeof(*stagd), GFP_KERNEL);
if (!stagd) {
ret = -ENOMEM;
goto out;
@@ -1075,7 +1075,7 @@ static void *c4iw_uld_add(const struct cxgb4_lld_info *infop)
pr_info("Chelsio T4/T5 RDMA Driver - version %s\n",
DRV_VERSION);
- ctx = kzalloc(sizeof *ctx, GFP_KERNEL);
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx) {
ctx = ERR_PTR(-ENOMEM);
goto out;
@@ -1243,10 +1243,9 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)
case CXGB4_STATE_START_RECOVERY:
pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev));
if (ctx->dev) {
- struct ib_event event;
+ struct ib_event event = {};
ctx->dev->rdev.flags |= T4_FATAL_ERROR;
- memset(&event, 0, sizeof event);
event.event = IB_EVENT_DEVICE_FATAL;
event.device = &ctx->dev->ibdev;
ib_dispatch_event(&event);
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 916ef982172e..7d06b0f8d49a 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -490,13 +490,13 @@ struct c4iw_qp {
struct t4_wq wq;
spinlock_t lock;
struct mutex mutex;
- struct kref kref;
wait_queue_head_t wait;
int sq_sig_all;
struct c4iw_srq *srq;
- struct work_struct free_work;
struct c4iw_ucontext *ucontext;
struct c4iw_wr_wait *wr_waitp;
+ struct completion qp_rel_comp;
+ refcount_t qp_refcnt;
};
static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
@@ -992,10 +992,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
struct ib_udata *udata);
struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
-int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
-struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
+int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask srq_attr_mask,
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 811c0c8c5b16..aa772ee0706f 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -130,8 +130,9 @@ static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len,
copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE :
len;
- wr_len = roundup(sizeof *req + sizeof *sc +
- roundup(copy_len, T4_ULPTX_MIN_IO), 16);
+ wr_len = roundup(sizeof(*req) + sizeof(*sc) +
+ roundup(copy_len, T4_ULPTX_MIN_IO),
+ 16);
if (!skb) {
skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL);
@@ -807,8 +808,7 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
mhp->attr.pbl_size << 3);
if (mhp->kva)
kfree((void *) (unsigned long) mhp->kva);
- if (mhp->umem)
- ib_umem_release(mhp->umem);
+ ib_umem_release(mhp->umem);
pr_debug("mmid 0x%x ptr %p\n", mmid, mhp);
c4iw_put_wr_wait(mhp->wr_waitp);
kfree(mhp);
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 74b795642fca..5e59c5708729 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -271,7 +271,6 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
return -EINVAL;
dev = to_c4iw_dev(ibdev);
- memset(props, 0, sizeof *props);
memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);
props->fw_ver = dev->rdev.lldi.fw_vers;
@@ -490,6 +489,10 @@ static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
}
static const struct ib_device_ops c4iw_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_CXGB4,
+ .uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION,
+
.alloc_hw_stats = c4iw_alloc_stats,
.alloc_mr = c4iw_alloc_mr,
.alloc_mw = c4iw_alloc_mw,
@@ -534,6 +537,7 @@ static const struct ib_device_ops c4iw_dev_ops = {
.reg_user_mr = c4iw_reg_user_mr,
.req_notify_cq = c4iw_arm_cq,
INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
+ INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),
};
@@ -561,7 +565,6 @@ void c4iw_register_device(struct work_struct *work)
pr_debug("c4iw_dev %p\n", dev);
memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
- dev->ibdev.owner = THIS_MODULE;
dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
if (fastreg_support)
dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
@@ -594,13 +597,11 @@ void c4iw_register_device(struct work_struct *work)
dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports;
dev->ibdev.num_comp_vectors = dev->rdev.lldi.nciq;
dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;
- dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;
memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name,
sizeof(dev->ibdev.iw_ifname));
rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group);
- dev->ibdev.driver_id = RDMA_DRIVER_CXGB4;
ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
ret = set_netdevs(&dev->ibdev, &dev->rdev);
if (ret)
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index e92b9544357a..9523986d62b1 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -303,7 +303,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
wq->rq.msn = 1;
/* build fw_ri_res_wr */
- wr_len = sizeof *res_wr + 2 * sizeof *res;
+ wr_len = sizeof(*res_wr) + 2 * sizeof(*res);
if (need_rq)
wr_len += sizeof(*res);
skb = alloc_skb(wr_len, GFP_KERNEL);
@@ -439,7 +439,7 @@ static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,
rem -= len;
}
}
- len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);
+ len = roundup(plen + sizeof(*immdp), 16) - (plen + sizeof(*immdp));
if (len)
memset(dstp, 0, len);
immdp->op = FW_RI_DATA_IMMD;
@@ -528,7 +528,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
T4_MAX_SEND_INLINE, &plen);
if (ret)
return ret;
- size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
+ size = sizeof(wqe->send) + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl((__be64 *)sq->queue,
@@ -537,7 +537,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
- size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
+ size = sizeof(wqe->send) + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof(struct fw_ri_sge);
}
} else {
@@ -545,7 +545,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
wqe->send.u.immd_src[0].r1 = 0;
wqe->send.u.immd_src[0].r2 = 0;
wqe->send.u.immd_src[0].immdlen = 0;
- size = sizeof wqe->send + sizeof(struct fw_ri_immd);
+ size = sizeof(wqe->send) + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
@@ -579,7 +579,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
T4_MAX_WRITE_INLINE, &plen);
if (ret)
return ret;
- size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
+ size = sizeof(wqe->write) + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl((__be64 *)sq->queue,
@@ -588,7 +588,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
- size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
+ size = sizeof(wqe->write) + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof(struct fw_ri_sge);
}
} else {
@@ -596,7 +596,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
wqe->write.u.immd_src[0].r1 = 0;
wqe->write.u.immd_src[0].r2 = 0;
wqe->write.u.immd_src[0].immdlen = 0;
- size = sizeof wqe->write + sizeof(struct fw_ri_immd);
+ size = sizeof(wqe->write) + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
@@ -683,7 +683,7 @@ static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr,
}
wqe->read.r2 = 0;
wqe->read.r5 = 0;
- *len16 = DIV_ROUND_UP(sizeof wqe->read, 16);
+ *len16 = DIV_ROUND_UP(sizeof(wqe->read), 16);
return 0;
}
@@ -766,8 +766,8 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
&wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL);
if (ret)
return ret;
- *len16 = DIV_ROUND_UP(sizeof wqe->recv +
- wr->num_sge * sizeof(struct fw_ri_sge), 16);
+ *len16 = DIV_ROUND_UP(
+ sizeof(wqe->recv) + wr->num_sge * sizeof(struct fw_ri_sge), 16);
return 0;
}
@@ -886,47 +886,21 @@ static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr,
{
wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
wqe->inv.r2 = 0;
- *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
+ *len16 = DIV_ROUND_UP(sizeof(wqe->inv), 16);
return 0;
}
-static void free_qp_work(struct work_struct *work)
-{
- struct c4iw_ucontext *ucontext;
- struct c4iw_qp *qhp;
- struct c4iw_dev *rhp;
-
- qhp = container_of(work, struct c4iw_qp, free_work);
- ucontext = qhp->ucontext;
- rhp = qhp->rhp;
-
- pr_debug("qhp %p ucontext %p\n", qhp, ucontext);
- destroy_qp(&rhp->rdev, &qhp->wq,
- ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq);
-
- c4iw_put_wr_wait(qhp->wr_waitp);
- kfree(qhp);
-}
-
-static void queue_qp_free(struct kref *kref)
-{
- struct c4iw_qp *qhp;
-
- qhp = container_of(kref, struct c4iw_qp, kref);
- pr_debug("qhp %p\n", qhp);
- queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work);
-}
-
void c4iw_qp_add_ref(struct ib_qp *qp)
{
pr_debug("ib_qp %p\n", qp);
- kref_get(&to_c4iw_qp(qp)->kref);
+ refcount_inc(&to_c4iw_qp(qp)->qp_refcnt);
}
void c4iw_qp_rem_ref(struct ib_qp *qp)
{
pr_debug("ib_qp %p\n", qp);
- kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free);
+ if (refcount_dec_and_test(&to_c4iw_qp(qp)->qp_refcnt))
+ complete(&to_c4iw_qp(qp)->qp_rel_comp);
}
static void add_to_fc_list(struct list_head *head, struct list_head *entry)
@@ -1606,7 +1580,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
wqe->u.terminate.type = FW_RI_TYPE_TERMINATE;
- wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term);
+ wqe->u.terminate.immdlen = cpu_to_be32(sizeof(*term));
term = (struct terminate_message *)wqe->u.terminate.termmsg;
if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) {
term->layer_etype = qhp->attr.layer_etype;
@@ -1751,16 +1725,15 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
{
pr_debug("p2p_type = %d\n", p2p_type);
- memset(&init->u, 0, sizeof init->u);
+ memset(&init->u, 0, sizeof(init->u));
switch (p2p_type) {
case FW_RI_INIT_P2PTYPE_RDMA_WRITE:
init->u.write.opcode = FW_RI_RDMA_WRITE_WR;
init->u.write.stag_sink = cpu_to_be32(1);
init->u.write.to_sink = cpu_to_be64(1);
init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;
- init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write +
- sizeof(struct fw_ri_immd),
- 16);
+ init->u.write.len16 = DIV_ROUND_UP(
+ sizeof(init->u.write) + sizeof(struct fw_ri_immd), 16);
break;
case FW_RI_INIT_P2PTYPE_READ_REQ:
init->u.write.opcode = FW_RI_RDMA_READ_WR;
@@ -1768,7 +1741,7 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
init->u.read.to_src_lo = cpu_to_be32(1);
init->u.read.stag_sink = cpu_to_be32(1);
init->u.read.to_sink_lo = cpu_to_be32(1);
- init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16);
+ init->u.read.len16 = DIV_ROUND_UP(sizeof(init->u.read), 16);
break;
}
}
@@ -1782,7 +1755,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp,
qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord);
- skb = alloc_skb(sizeof *wqe, GFP_KERNEL);
+ skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);
if (!skb) {
ret = -ENOMEM;
goto out;
@@ -2099,10 +2072,12 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
{
struct c4iw_dev *rhp;
struct c4iw_qp *qhp;
+ struct c4iw_ucontext *ucontext;
struct c4iw_qp_attributes attrs;
qhp = to_c4iw_qp(ib_qp);
rhp = qhp->rhp;
+ ucontext = qhp->ucontext;
attrs.next_state = C4IW_QP_STATE_ERROR;
if (qhp->attr.state == C4IW_QP_STATE_TERMINATE)
@@ -2120,7 +2095,17 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
c4iw_qp_rem_ref(ib_qp);
+ wait_for_completion(&qhp->qp_rel_comp);
+
pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp, qhp->wq.sq.qid);
+ pr_debug("qhp %p ucontext %p\n", qhp, ucontext);
+
+ destroy_qp(&rhp->rdev, &qhp->wq,
+ ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq);
+
+ c4iw_put_wr_wait(qhp->wr_waitp);
+
+ kfree(qhp);
return 0;
}
@@ -2230,8 +2215,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
spin_lock_init(&qhp->lock);
mutex_init(&qhp->mutex);
init_waitqueue_head(&qhp->wait);
- kref_init(&qhp->kref);
- INIT_WORK(&qhp->free_work, free_qp_work);
+ init_completion(&qhp->qp_rel_comp);
+ refcount_set(&qhp->qp_refcnt, 1);
ret = xa_insert_irq(&rhp->qps, qhp->wq.sq.qid, qhp, GFP_KERNEL);
if (ret)
@@ -2302,7 +2287,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
ucontext->key += PAGE_SIZE;
}
spin_unlock(&ucontext->mmap_lock);
- ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
+ ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
if (ret)
goto err_free_ma_sync_key;
sq_key_mm->key = uresp.sq_key;
@@ -2386,7 +2371,7 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
struct c4iw_dev *rhp;
struct c4iw_qp *qhp;
enum c4iw_qp_attr_mask mask = 0;
- struct c4iw_qp_attributes attrs;
+ struct c4iw_qp_attributes attrs = {};
pr_debug("ib_qp %p\n", ibqp);
@@ -2398,7 +2383,6 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (!attr_mask)
return 0;
- memset(&attrs, 0, sizeof attrs);
qhp = to_c4iw_qp(ibqp);
rhp = qhp->rhp;
@@ -2482,8 +2466,8 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
{
struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
- memset(attr, 0, sizeof *attr);
- memset(init_attr, 0, sizeof *init_attr);
+ memset(attr, 0, sizeof(*attr));
+ memset(init_attr, 0, sizeof(*init_attr));
attr->qp_state = to_ib_qp_state(qhp->attr.state);
init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c
index 57ed26b3cc21..5c95c789f302 100644
--- a/drivers/infiniband/hw/cxgb4/resource.c
+++ b/drivers/infiniband/hw/cxgb4/resource.c
@@ -126,7 +126,7 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
rdev->stats.qid.cur += rdev->qpmask + 1;
mutex_unlock(&rdev->stats.lock);
for (i = qid+1; i & rdev->qpmask; i++) {
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = i;
@@ -137,13 +137,13 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
* now put the same ids on the qp list since they all
* map to the same db/gts page.
*/
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = qid;
list_add_tail(&entry->entry, &uctx->qpids);
for (i = qid+1; i & rdev->qpmask; i++) {
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = i;
@@ -165,7 +165,7 @@ void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
{
struct c4iw_qid_list *entry;
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return;
pr_debug("qid 0x%x\n", qid);
@@ -200,7 +200,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
rdev->stats.qid.cur += rdev->qpmask + 1;
mutex_unlock(&rdev->stats.lock);
for (i = qid+1; i & rdev->qpmask; i++) {
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = i;
@@ -211,13 +211,13 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
* now put the same ids on the cq list since they all
* map to the same db/gts page.
*/
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = qid;
list_add_tail(&entry->entry, &uctx->cqids);
for (i = qid; i & rdev->qpmask; i++) {
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out;
entry->qid = i;
@@ -239,7 +239,7 @@ void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
{
struct c4iw_qid_list *entry;
- entry = kmalloc(sizeof *entry, GFP_KERNEL);
+ entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return;
pr_debug("qid 0x%x\n", qid);
diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index 9e3cc3239c13..119f8efec564 100644
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -7,10 +7,8 @@
#define _EFA_H_
#include <linux/bitops.h>
-#include <linux/idr.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
-#include <linux/sched.h>
#include <rdma/efa-abi.h>
#include <rdma/ib_verbs.h>
@@ -136,10 +134,9 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata);
-int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
-struct ib_cq *efa_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index a5c788741a04..ec04ced9fd2b 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -39,8 +39,6 @@
enum efa_cmd_status {
EFA_CMD_SUBMITTED,
EFA_CMD_COMPLETED,
- /* Abort - canceled by the driver */
- EFA_CMD_ABORTED,
};
struct efa_comp_ctx {
@@ -532,16 +530,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c
msleep(aq->poll_interval);
}
- if (comp_ctx->status == EFA_CMD_ABORTED) {
- ibdev_err(aq->efa_dev, "Command was aborted\n");
- atomic64_inc(&aq->stats.aborted_cmd);
- err = -ENODEV;
- goto out;
- }
-
- WARN_ONCE(comp_ctx->status != EFA_CMD_COMPLETED,
- "Invalid completion status %d\n", comp_ctx->status);
-
err = efa_com_comp_status_to_errno(comp_ctx->comp_status);
out:
efa_com_put_comp_ctx(aq, comp_ctx);
@@ -666,66 +654,6 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
}
/**
- * efa_com_abort_admin_commands - Abort all the outstanding admin commands.
- * @edev: EFA communication layer struct
- *
- * This method aborts all the outstanding admin commands.
- * The caller should then call efa_com_wait_for_abort_completion to make sure
- * all the commands were completed.
- */
-static void efa_com_abort_admin_commands(struct efa_com_dev *edev)
-{
- struct efa_com_admin_queue *aq = &edev->aq;
- struct efa_comp_ctx *comp_ctx;
- unsigned long flags;
- u16 i;
-
- spin_lock(&aq->sq.lock);
- spin_lock_irqsave(&aq->cq.lock, flags);
- for (i = 0; i < aq->depth; i++) {
- comp_ctx = efa_com_get_comp_ctx(aq, i, false);
- if (!comp_ctx)
- break;
-
- comp_ctx->status = EFA_CMD_ABORTED;
-
- complete(&comp_ctx->wait_event);
- }
- spin_unlock_irqrestore(&aq->cq.lock, flags);
- spin_unlock(&aq->sq.lock);
-}
-
-/**
- * efa_com_wait_for_abort_completion - Wait for admin commands abort.
- * @edev: EFA communication layer struct
- *
- * This method wait until all the outstanding admin commands will be completed.
- */
-static void efa_com_wait_for_abort_completion(struct efa_com_dev *edev)
-{
- struct efa_com_admin_queue *aq = &edev->aq;
- int i;
-
- /* all mine */
- for (i = 0; i < aq->depth; i++)
- down(&aq->avail_cmds);
-
- /* let it go */
- for (i = 0; i < aq->depth; i++)
- up(&aq->avail_cmds);
-}
-
-static void efa_com_admin_flush(struct efa_com_dev *edev)
-{
- struct efa_com_admin_queue *aq = &edev->aq;
-
- clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
-
- efa_com_abort_admin_commands(edev);
- efa_com_wait_for_abort_completion(edev);
-}
-
-/**
* efa_com_admin_destroy - Destroy the admin and the async events queues.
* @edev: EFA communication layer struct
*/
@@ -737,7 +665,7 @@ void efa_com_admin_destroy(struct efa_com_dev *edev)
struct efa_com_admin_sq *sq = &aq->sq;
u16 size;
- efa_com_admin_flush(edev);
+ clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
devm_kfree(edev->dmadev, aq->comp_ctx_pool);
devm_kfree(edev->dmadev, aq->comp_ctx);
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
index 84d96724a74b..c67dd8109d1c 100644
--- a/drivers/infiniband/hw/efa/efa_com.h
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -45,7 +45,6 @@ struct efa_com_admin_sq {
/* Don't use anything other than atomic64 */
struct efa_com_stats_admin {
- atomic64_t aborted_cmd;
atomic64_t submitted_cmd;
atomic64_t completed_cmd;
atomic64_t no_completion;
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index c0016648804c..62345d8abf3c 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -3,7 +3,6 @@
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
-#include "efa.h"
#include "efa_com.h"
#include "efa_com_cmd.h"
@@ -57,7 +56,7 @@ int efa_com_create_qp(struct efa_com_dev *edev,
res->send_sub_cq_idx = cmd_completion.send_sub_cq_idx;
res->recv_sub_cq_idx = cmd_completion.recv_sub_cq_idx;
- return err;
+ return 0;
}
int efa_com_modify_qp(struct efa_com_dev *edev,
@@ -181,7 +180,7 @@ int efa_com_create_cq(struct efa_com_dev *edev,
result->cq_idx = cmd_completion.cq_idx;
result->actual_depth = params->cq_depth;
- return err;
+ return 0;
}
int efa_com_destroy_cq(struct efa_com_dev *edev,
@@ -307,7 +306,8 @@ int efa_com_create_ah(struct efa_com_dev *edev,
(struct efa_admin_acq_entry *)&cmd_completion,
sizeof(cmd_completion));
if (err) {
- ibdev_err(edev->efa_dev, "Failed to create ah [%d]\n", err);
+ ibdev_err(edev->efa_dev, "Failed to create ah for %pI6 [%d]\n",
+ ah_cmd.dest_addr, err);
return err;
}
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index db974caf1eb1..dd1c6d49466f 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -100,7 +100,7 @@ static int efa_request_mgmnt_irq(struct efa_dev *dev)
nr_cpumask_bits, &irq->affinity_hint_mask, irq->vector);
irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
- return err;
+ return 0;
}
static void efa_setup_mgmnt_irq(struct efa_dev *dev)
@@ -197,6 +197,10 @@ static void efa_stats_init(struct efa_dev *dev)
}
static const struct ib_device_ops efa_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_EFA,
+ .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,
+
.alloc_pd = efa_alloc_pd,
.alloc_ucontext = efa_alloc_ucontext,
.create_ah = efa_create_ah,
@@ -220,6 +224,7 @@ static const struct ib_device_ops efa_dev_ops = {
.reg_user_mr = efa_reg_mr,
INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
};
@@ -259,12 +264,10 @@ static int efa_ib_device_add(struct efa_dev *dev)
if (err)
goto err_release_doorbell_bar;
- dev->ibdev.owner = THIS_MODULE;
dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
dev->ibdev.phys_port_cnt = 1;
dev->ibdev.num_comp_vectors = 1;
dev->ibdev.dev.parent = &pdev->dev;
- dev->ibdev.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION;
dev->ibdev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
@@ -287,7 +290,6 @@ static int efa_ib_device_add(struct efa_dev *dev)
dev->ibdev.uverbs_ex_cmd_mask =
(1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
- dev->ibdev.driver_id = RDMA_DRIVER_EFA;
ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
err = ib_register_device(&dev->ibdev, "efa_%d");
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index fb6115244d4c..df77bc312a25 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -447,12 +447,6 @@ void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
struct efa_dev *dev = to_edev(ibpd->device);
struct efa_pd *pd = to_epd(ibpd);
- if (udata->inlen &&
- !ib_is_udata_cleared(udata, 0, udata->inlen)) {
- ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
- return;
- }
-
ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn);
efa_pd_dealloc(dev, pd->pdn);
}
@@ -470,12 +464,6 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
struct efa_qp *qp = to_eqp(ibqp);
int err;
- if (udata->inlen &&
- !ib_is_udata_cleared(udata, 0, udata->inlen)) {
- ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
- return -EINVAL;
- }
-
ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num);
err = efa_destroy_qp_handle(dev, qp->qp_handle);
if (err)
@@ -870,31 +858,18 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx)
return efa_com_destroy_cq(&dev->edev, &params);
}
-int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct efa_dev *dev = to_edev(ibcq->device);
struct efa_cq *cq = to_ecq(ibcq);
- int err;
-
- if (udata->inlen &&
- !ib_is_udata_cleared(udata, 0, udata->inlen)) {
- ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
- return -EINVAL;
- }
ibdev_dbg(&dev->ibdev,
"Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",
cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
- err = efa_destroy_cq_idx(dev, cq->cq_idx);
- if (err)
- return err;
-
+ efa_destroy_cq_idx(dev, cq->cq_idx);
dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
-
- kfree(cq);
- return 0;
}
static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
@@ -910,17 +885,20 @@ static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
return 0;
}
-static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,
- int vector, struct ib_ucontext *ibucontext,
- struct ib_udata *udata)
+int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct efa_ucontext *ucontext = rdma_udata_to_drv_context(
+ udata, struct efa_ucontext, ibucontext);
struct efa_ibv_create_cq_resp resp = {};
struct efa_com_create_cq_params params;
struct efa_com_create_cq_result result;
+ struct ib_device *ibdev = ibcq->device;
struct efa_dev *dev = to_edev(ibdev);
struct efa_ibv_create_cq cmd = {};
+ struct efa_cq *cq = to_ecq(ibcq);
bool cq_entry_inserted = false;
- struct efa_cq *cq;
+ int entries = attr->cqe;
int err;
ibdev_dbg(ibdev, "create_cq entries %d\n", entries);
@@ -978,19 +956,13 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,
goto err_out;
}
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq) {
- err = -ENOMEM;
- goto err_out;
- }
-
- cq->ucontext = to_eucontext(ibucontext);
+ cq->ucontext = ucontext;
cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);
cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (!cq->cpu_addr) {
err = -ENOMEM;
- goto err_free_cq;
+ goto err_out;
}
params.uarn = cq->ucontext->uarn;
@@ -1009,8 +981,8 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,
err = cq_mmap_entries_setup(dev, cq, &resp);
if (err) {
- ibdev_dbg(ibdev,
- "Could not setup cq[%u] mmap entries\n", cq->cq_idx);
+ ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n",
+ cq->cq_idx);
goto err_destroy_cq;
}
@@ -1026,11 +998,10 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,
}
}
- ibdev_dbg(ibdev,
- "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",
+ ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",
cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr);
- return &cq->ibcq;
+ return 0;
err_destroy_cq:
efa_destroy_cq_idx(dev, cq->cq_idx);
@@ -1039,23 +1010,9 @@ err_free_mapped:
DMA_FROM_DEVICE);
if (!cq_entry_inserted)
free_pages_exact(cq->cpu_addr, cq->size);
-err_free_cq:
- kfree(cq);
err_out:
atomic64_inc(&dev->stats.sw_stats.create_cq_err);
- return ERR_PTR(err);
-}
-
-struct ib_cq *efa_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
-{
- struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata,
- struct efa_ucontext,
- ibucontext);
-
- return do_create_cq(ibdev, attr->cqe, attr->comp_vector,
- &ucontext->ibucontext, udata);
+ return err;
}
static int umem_to_page_list(struct efa_dev *dev,
@@ -1065,21 +1022,15 @@ static int umem_to_page_list(struct efa_dev *dev,
u8 hp_shift)
{
u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT);
- struct sg_dma_page_iter sg_iter;
- unsigned int page_idx = 0;
+ struct ib_block_iter biter;
unsigned int hp_idx = 0;
ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n",
hp_cnt, pages_in_hp);
- for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
- if (page_idx % pages_in_hp == 0) {
- page_list[hp_idx] = sg_page_iter_dma_address(&sg_iter);
- hp_idx++;
- }
-
- page_idx++;
- }
+ rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap,
+ BIT(hp_shift))
+ page_list[hp_idx++] = rdma_block_iter_dma_address(&biter);
return 0;
}
@@ -1114,14 +1065,14 @@ err:
*/
static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl)
{
- unsigned int entry, payloads_in_sg, chunk_list_size, chunk_idx, payload_idx;
struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list;
int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages;
struct scatterlist *pages_sgl = pbl->phys.indirect.sgl;
+ unsigned int chunk_list_size, chunk_idx, payload_idx;
int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt;
struct efa_com_ctrl_buff_info *ctrl_buf;
u64 *cur_chunk_buf, *prev_chunk_buf;
- struct scatterlist *sg;
+ struct ib_block_iter biter;
dma_addr_t dma_addr;
int i;
@@ -1155,18 +1106,15 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl)
chunk_idx = 0;
payload_idx = 0;
cur_chunk_buf = chunk_list->chunks[0].buf;
- for_each_sg(pages_sgl, sg, sg_dma_cnt, entry) {
- payloads_in_sg = sg_dma_len(sg) >> EFA_CHUNK_PAYLOAD_SHIFT;
- for (i = 0; i < payloads_in_sg; i++) {
- cur_chunk_buf[payload_idx++] =
- (sg_dma_address(sg) & ~(EFA_CHUNK_PAYLOAD_SIZE - 1)) +
- (EFA_CHUNK_PAYLOAD_SIZE * i);
-
- if (payload_idx == EFA_PTRS_PER_CHUNK) {
- chunk_idx++;
- cur_chunk_buf = chunk_list->chunks[chunk_idx].buf;
- payload_idx = 0;
- }
+ rdma_for_each_block(pages_sgl, &biter, sg_dma_cnt,
+ EFA_CHUNK_PAYLOAD_SIZE) {
+ cur_chunk_buf[payload_idx++] =
+ rdma_block_iter_dma_address(&biter);
+
+ if (payload_idx == EFA_PTRS_PER_CHUNK) {
+ chunk_idx++;
+ cur_chunk_buf = chunk_list->chunks[chunk_idx].buf;
+ payload_idx = 0;
}
}
@@ -1314,30 +1262,30 @@ static int pbl_create(struct efa_dev *dev,
int err;
pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE;
- pbl->pbl_buf = kzalloc(pbl->pbl_buf_size_in_bytes,
- GFP_KERNEL | __GFP_NOWARN);
- if (pbl->pbl_buf) {
- pbl->physically_continuous = 1;
+ pbl->pbl_buf = kvzalloc(pbl->pbl_buf_size_in_bytes, GFP_KERNEL);
+ if (!pbl->pbl_buf)
+ return -ENOMEM;
+
+ if (is_vmalloc_addr(pbl->pbl_buf)) {
+ pbl->physically_continuous = 0;
err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt,
hp_shift);
if (err)
- goto err_continuous;
- err = pbl_continuous_initialize(dev, pbl);
+ goto err_free;
+
+ err = pbl_indirect_initialize(dev, pbl);
if (err)
- goto err_continuous;
+ goto err_free;
} else {
- pbl->physically_continuous = 0;
- pbl->pbl_buf = vzalloc(pbl->pbl_buf_size_in_bytes);
- if (!pbl->pbl_buf)
- return -ENOMEM;
-
+ pbl->physically_continuous = 1;
err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt,
hp_shift);
if (err)
- goto err_indirect;
- err = pbl_indirect_initialize(dev, pbl);
+ goto err_free;
+
+ err = pbl_continuous_initialize(dev, pbl);
if (err)
- goto err_indirect;
+ goto err_free;
}
ibdev_dbg(&dev->ibdev,
@@ -1346,24 +1294,20 @@ static int pbl_create(struct efa_dev *dev,
return 0;
-err_continuous:
- kfree(pbl->pbl_buf);
- return err;
-err_indirect:
- vfree(pbl->pbl_buf);
+err_free:
+ kvfree(pbl->pbl_buf);
return err;
}
static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl)
{
- if (pbl->physically_continuous) {
+ if (pbl->physically_continuous)
dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr,
pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE);
- kfree(pbl->pbl_buf);
- } else {
+ else
pbl_indirect_terminate(dev, pbl);
- vfree(pbl->pbl_buf);
- }
+
+ kvfree(pbl->pbl_buf);
}
static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr,
@@ -1417,56 +1361,6 @@ static int efa_create_pbl(struct efa_dev *dev,
return 0;
}
-static void efa_cont_pages(struct ib_umem *umem, u64 addr,
- unsigned long max_page_shift,
- int *count, u8 *shift, u32 *ncont)
-{
- struct scatterlist *sg;
- u64 base = ~0, p = 0;
- unsigned long tmp;
- unsigned long m;
- u64 len, pfn;
- int i = 0;
- int entry;
-
- addr = addr >> PAGE_SHIFT;
- tmp = (unsigned long)addr;
- m = find_first_bit(&tmp, BITS_PER_LONG);
- if (max_page_shift)
- m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
-
- for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
- len = DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE);
- pfn = sg_dma_address(sg) >> PAGE_SHIFT;
- if (base + p != pfn) {
- /*
- * If either the offset or the new
- * base are unaligned update m
- */
- tmp = (unsigned long)(pfn | p);
- if (!IS_ALIGNED(tmp, 1 << m))
- m = find_first_bit(&tmp, BITS_PER_LONG);
-
- base = pfn;
- p = 0;
- }
-
- p += len;
- i += len;
- }
-
- if (i) {
- m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
- *ncont = DIV_ROUND_UP(i, (1 << m));
- } else {
- m = 0;
- *ncont = 0;
- }
-
- *shift = PAGE_SHIFT + m;
- *count = i;
-}
-
struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_udata *udata)
@@ -1474,11 +1368,10 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
struct efa_dev *dev = to_edev(ibpd->device);
struct efa_com_reg_mr_params params = {};
struct efa_com_reg_mr_result result = {};
- unsigned long max_page_shift;
struct pbl_context pbl;
+ unsigned int pg_sz;
struct efa_mr *mr;
int inline_size;
- int npages;
int err;
if (udata->inlen &&
@@ -1515,13 +1408,24 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
params.iova = virt_addr;
params.mr_length_in_bytes = length;
params.permissions = access_flags & 0x1;
- max_page_shift = fls64(dev->dev_attr.page_size_cap);
- efa_cont_pages(mr->umem, start, max_page_shift, &npages,
- &params.page_shift, &params.page_num);
+ pg_sz = ib_umem_find_best_pgsz(mr->umem,
+ dev->dev_attr.page_size_cap,
+ virt_addr);
+ if (!pg_sz) {
+ err = -EOPNOTSUPP;
+ ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n",
+ dev->dev_attr.page_size_cap);
+ goto err_unmap;
+ }
+
+ params.page_shift = __ffs(pg_sz);
+ params.page_num = DIV_ROUND_UP(length + (start & (pg_sz - 1)),
+ pg_sz);
+
ibdev_dbg(&dev->ibdev,
- "start %#llx length %#llx npages %d params.page_shift %u params.page_num %u\n",
- start, length, npages, params.page_shift, params.page_num);
+ "start %#llx length %#llx params.page_shift %u params.page_num %u\n",
+ start, length, params.page_shift, params.page_num);
inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array);
if (params.page_num <= inline_size) {
@@ -1567,12 +1471,6 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
struct efa_mr *mr = to_emr(ibmr);
int err;
- if (udata->inlen &&
- !ib_is_udata_cleared(udata, 0, udata->inlen)) {
- ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
- return -EINVAL;
- }
-
ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey);
if (mr->umem) {
@@ -1580,8 +1478,8 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
err = efa_com_dereg_mr(&dev->edev, &params);
if (err)
return err;
- ib_umem_release(mr->umem);
}
+ ib_umem_release(mr->umem);
kfree(mr);
@@ -1707,13 +1605,15 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
err = -EINVAL;
}
- if (err)
+ if (err) {
ibdev_dbg(
&dev->ibdev,
"Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n",
entry->address, length, entry->mmap_flag, err);
+ return err;
+ }
- return err;
+ return 0;
}
int efa_mmap(struct ib_ucontext *ibucontext,
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 4228393e6c4c..184dba3c2828 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -2744,8 +2744,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
u16 link_width;
u16 link_speed;
- response_data_size = sizeof(struct opa_port_status_rsp) +
- num_vls * sizeof(struct _vls_pctrs);
+ response_data_size = struct_size(rsp, vls, num_vls);
if (response_data_size > sizeof(pmp->data)) {
pmp->mad_hdr.status |= OPA_PM_STATUS_REQUEST_TOO_LARGE;
return reply((struct ib_mad_hdr *)pmp);
@@ -3014,8 +3013,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
}
/* Sanity check */
- response_data_size = sizeof(struct opa_port_data_counters_msg) +
- num_vls * sizeof(struct _vls_dctrs);
+ response_data_size = struct_size(req, port[0].vls, num_vls);
if (response_data_size > sizeof(pmp->data)) {
pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
@@ -3232,8 +3230,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,
return reply((struct ib_mad_hdr *)pmp);
}
- response_data_size = sizeof(struct opa_port_error_counters64_msg) +
- num_vls * sizeof(struct _vls_ectrs);
+ response_data_size = struct_size(req, port[0].vls, num_vls);
if (response_data_size > sizeof(pmp->data)) {
pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 4e5c2d1b8cfa..79126b2b14ab 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -1594,9 +1594,8 @@ void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint)
else
sc_del_credit_return_intr(sc);
trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl);
- if (needint) {
+ if (needint)
sc_return_credits(sc);
- }
}
/**
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 7c8cfb149da0..235bdbc706ac 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -1830,23 +1830,14 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)
}
while (qp->s_last != qp->s_acked) {
- u32 s_last;
-
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 &&
cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
break;
trdma_clean_swqe(qp, wqe);
rvt_qp_wqe_unreserve(qp, wqe);
- s_last = qp->s_last;
- trace_hfi1_qp_send_completion(qp, wqe, s_last);
- if (++s_last >= qp->s_size)
- s_last = 0;
- qp->s_last = s_last;
- /* see post_send() */
- barrier();
- rvt_put_qp_swqe(qp, wqe);
- rvt_qp_swqe_complete(qp,
+ trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
+ rvt_qp_complete_swqe(qp,
wqe,
ib_hfi1_wc_opcode[wqe->wr.opcode],
IB_WC_SUCCESS);
@@ -1890,19 +1881,10 @@ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
trace_hfi1_rc_completion(qp, wqe->lpsn);
if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 ||
cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
- u32 s_last;
-
trdma_clean_swqe(qp, wqe);
- rvt_put_qp_swqe(qp, wqe);
rvt_qp_wqe_unreserve(qp, wqe);
- s_last = qp->s_last;
- trace_hfi1_qp_send_completion(qp, wqe, s_last);
- if (++s_last >= qp->s_size)
- s_last = 0;
- qp->s_last = s_last;
- /* see post_send() */
- barrier();
- rvt_qp_swqe_complete(qp,
+ trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
+ rvt_qp_complete_swqe(qp,
wqe,
ib_hfi1_wc_opcode[wqe->wr.opcode],
IB_WC_SUCCESS);
diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c
index aa9c8d3ef87b..92acccaaaa86 100644
--- a/drivers/infiniband/hw/hfi1/tid_rdma.c
+++ b/drivers/infiniband/hw/hfi1/tid_rdma.c
@@ -475,7 +475,7 @@ static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd,
* Must hold the qp s_lock and the exp_lock.
*
* Return:
- * false if either of the conditions below are statisfied:
+ * false if either of the conditions below are satisfied:
* 1. The list is empty or
* 2. The indicated qp is at the head of the list and the
* HFI1_S_WAIT_TID_SPACE bit is set in qp->s_flags.
@@ -2024,7 +2024,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,
trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req);
if (e->opcode == TID_OP(READ_REQ)) {
struct ib_reth *reth;
- u32 offset;
u32 len;
u32 rkey;
u64 vaddr;
@@ -2036,7 +2035,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,
* The requester always restarts from the start of the original
* request.
*/
- offset = delta_psn(psn, e->psn) * qp->pmtu;
len = be32_to_cpu(reth->length);
if (psn != e->psn || len != req->total_len)
goto unlock;
@@ -4550,7 +4548,7 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)
struct rvt_swqe *wqe;
struct tid_rdma_request *req;
struct tid_rdma_flow *flow;
- u32 aeth, psn, req_psn, ack_psn, fspsn, resync_psn, ack_kpsn;
+ u32 aeth, psn, req_psn, ack_psn, resync_psn, ack_kpsn;
unsigned long flags;
u16 fidx;
@@ -4754,7 +4752,6 @@ done:
IB_AETH_CREDIT_MASK) {
case 0: /* PSN sequence error */
flow = &req->flows[req->acked_tail];
- fspsn = full_flow_psn(flow, flow->flow_state.spsn);
trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail,
flow);
req->r_ack_psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c
index 02eee8eff1db..b89a9b9aef7a 100644
--- a/drivers/infiniband/hw/hfi1/user_pages.c
+++ b/drivers/infiniband/hw/hfi1/user_pages.c
@@ -118,13 +118,10 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np
void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,
size_t npages, bool dirty)
{
- size_t i;
-
- for (i = 0; i < npages; i++) {
- if (dirty)
- set_page_dirty_lock(p[i]);
- put_page(p[i]);
- }
+ if (dirty)
+ put_user_pages_dirty_lock(p, npages);
+ else
+ put_user_pages(p, npages);
if (mm) { /* during close after signal, mm can be NULL */
atomic64_sub(npages, &mm->pinned_vm);
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index bad3229bad37..c4b243f50c76 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1779,6 +1779,9 @@ static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
}
static const struct ib_device_ops hfi1_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_HFI1,
+
.alloc_hw_stats = alloc_hw_stats,
.alloc_rdma_netdev = hfi1_vnic_alloc_rn,
.get_dev_fw_str = hfi1_get_dev_fw_str,
@@ -1829,7 +1832,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
*/
if (!ib_hfi1_sys_image_guid)
ib_hfi1_sys_image_guid = ibdev->node_guid;
- ibdev->owner = THIS_MODULE;
ibdev->phys_port_cnt = dd->num_pports;
ibdev->dev.parent = &dd->pcidev->dev;
@@ -1923,7 +1925,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev,
&ib_hfi1_attr_group);
- ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_HFI1);
+ ret = rvt_register_device(&dd->verbs_dev.rdi);
if (ret)
goto err_verbs_txreq;
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig
index 61cda7d00627..8bf847bcd8d3 100644
--- a/drivers/infiniband/hw/hns/Kconfig
+++ b/drivers/infiniband/hw/hns/Kconfig
@@ -8,25 +8,24 @@ config INFINIBAND_HNS
is used in Hisilicon Hip06 and more further ICT SoC based on
platform device.
- To compile this driver as a module, choose M here: the module
- will be called hns-roce.
+ To compile HIP06 or HIP08 driver as module, choose M here.
config INFINIBAND_HNS_HIP06
- tristate "Hisilicon Hip06 Family RoCE support"
+ bool "Hisilicon Hip06 Family RoCE support"
depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET
---help---
RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and
Hip07 SoC. These RoCE engines are platform devices.
- To compile this driver as a module, choose M here: the module
- will be called hns-roce-hw-v1.
+ To compile this driver, choose Y here: if INFINIBAND_HNS is m, this
+ module will be called hns-roce-hw-v1
config INFINIBAND_HNS_HIP08
- tristate "Hisilicon Hip08 Family RoCE support"
+ bool "Hisilicon Hip08 Family RoCE support"
depends on INFINIBAND_HNS && PCI && HNS3
---help---
RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC.
The RoCE engine is a PCI device.
- To compile this driver as a module, choose M here: the module
- will be called hns-roce-hw-v2.
+ To compile this driver, choose Y here: if INFINIBAND_HNS is m, this
+ module will be called hns-roce-hw-v2.
diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
index f22d9922cfee..1c5a20af6992 100644
--- a/drivers/infiniband/hw/hns/Makefile
+++ b/drivers/infiniband/hw/hns/Makefile
@@ -5,11 +5,16 @@
ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3
-obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
-obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
+
+ifdef CONFIG_INFINIBAND_HNS_HIP06
hns-roce-hw-v1-objs := hns_roce_hw_v1.o
-obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
-hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o
+obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o $(hns-roce-objs)
+endif
+
+ifdef CONFIG_INFINIBAND_HNS_HIP08
+hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)
+obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o
+endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
index dac058d3df53..8c063c598d2a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_alloc.c
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -34,6 +34,7 @@
#include <linux/platform_device.h>
#include <linux/vmalloc.h>
#include "hns_roce_device.h"
+#include <rdma/ib_umem.h>
int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)
{
@@ -67,7 +68,6 @@ void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
{
hns_roce_bitmap_free_range(bitmap, obj, 1, rr);
}
-EXPORT_SYMBOL_GPL(hns_roce_bitmap_free);
int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
int align, unsigned long *obj)
@@ -174,7 +174,6 @@ void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
kfree(buf->page_list);
}
}
-EXPORT_SYMBOL_GPL(hns_roce_buf_free);
int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
struct hns_roce_buf *buf, u32 page_shift)
@@ -238,6 +237,104 @@ err_free:
return -ENOMEM;
}
+int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
+ int buf_cnt, int start, struct hns_roce_buf *buf)
+{
+ int i, end;
+ int total;
+
+ end = start + buf_cnt;
+ if (end > buf->npages) {
+ dev_err(hr_dev->dev,
+ "invalid kmem region,offset %d,buf_cnt %d,total %d!\n",
+ start, buf_cnt, buf->npages);
+ return -EINVAL;
+ }
+
+ total = 0;
+ for (i = start; i < end; i++)
+ if (buf->nbufs == 1)
+ bufs[total++] = buf->direct.map +
+ ((dma_addr_t)i << buf->page_shift);
+ else
+ bufs[total++] = buf->page_list[i].map;
+
+ return total;
+}
+
+int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
+ int buf_cnt, int start, struct ib_umem *umem,
+ int page_shift)
+{
+ struct ib_block_iter biter;
+ int total = 0;
+ int idx = 0;
+ u64 addr;
+
+ if (page_shift < PAGE_SHIFT) {
+ dev_err(hr_dev->dev, "invalid page shift %d!\n", page_shift);
+ return -EINVAL;
+ }
+
+ /* convert system page cnt to hw page cnt */
+ rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap,
+ 1 << page_shift) {
+ addr = rdma_block_iter_dma_address(&biter);
+ if (idx >= start) {
+ bufs[total++] = addr;
+ if (total >= buf_cnt)
+ goto done;
+ }
+ idx++;
+ }
+
+done:
+ return total;
+}
+
+void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum,
+ int offset, int buf_cnt)
+{
+ if (hopnum == HNS_ROCE_HOP_NUM_0)
+ region->hopnum = 0;
+ else
+ region->hopnum = hopnum;
+
+ region->offset = offset;
+ region->count = buf_cnt;
+}
+
+void hns_roce_free_buf_list(dma_addr_t **bufs, int region_cnt)
+{
+ int i;
+
+ for (i = 0; i < region_cnt; i++) {
+ kfree(bufs[i]);
+ bufs[i] = NULL;
+ }
+}
+
+int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions,
+ dma_addr_t **bufs, int region_cnt)
+{
+ struct hns_roce_buf_region *r;
+ int i;
+
+ for (i = 0; i < region_cnt; i++) {
+ r = &regions[i];
+ bufs[i] = kcalloc(r->count, sizeof(dma_addr_t), GFP_KERNEL);
+ if (!bufs[i])
+ goto err_alloc;
+ }
+
+ return 0;
+
+err_alloc:
+ hns_roce_free_buf_list(bufs, i);
+
+ return -ENOMEM;
+}
+
void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)
{
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c
index 2acf946d02e5..b83d5bd92329 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c
@@ -103,7 +103,6 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
context->out_param = out_param;
complete(&context->done);
}
-EXPORT_SYMBOL_GPL(hns_roce_cmd_event);
/* this should be called with "use_events" */
static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
@@ -204,7 +203,6 @@ int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
return ret;
}
-EXPORT_SYMBOL_GPL(hns_roce_cmd_mbox);
int hns_roce_cmd_init(struct hns_roce_dev *hr_dev)
{
@@ -291,7 +289,6 @@ struct hns_roce_cmd_mailbox
return mailbox;
}
-EXPORT_SYMBOL_GPL(hns_roce_alloc_cmd_mailbox);
void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox)
@@ -302,4 +299,3 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma);
kfree(mailbox);
}
-EXPORT_SYMBOL_GPL(hns_roce_free_cmd_mailbox);
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 9caf35061721..4e50c22a2da4 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -205,7 +205,6 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR);
}
-EXPORT_SYMBOL_GPL(hns_roce_free_cq);
static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev,
struct ib_udata *udata,
@@ -235,8 +234,7 @@ static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev,
&buf->hr_mtt);
} else {
ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem),
- (*umem)->page_shift,
- &buf->hr_mtt);
+ PAGE_SHIFT, &buf->hr_mtt);
}
if (ret)
goto err_buf;
@@ -300,15 +298,15 @@ static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev,
&buf->hr_buf);
}
-struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int hns_roce_ib_create_cq(struct ib_cq *ib_cq,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
- struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+ struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
struct device *dev = hr_dev->dev;
struct hns_roce_ib_create_cq ucmd;
struct hns_roce_ib_create_cq_resp resp = {};
- struct hns_roce_cq *hr_cq = NULL;
+ struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
struct hns_roce_uar *uar = NULL;
int vector = attr->comp_vector;
int cq_entries = attr->cqe;
@@ -319,13 +317,9 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) {
dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n",
cq_entries, hr_dev->caps.max_cqes);
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
- hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
- if (!hr_cq)
- return ERR_PTR(-ENOMEM);
-
if (hr_dev->caps.min_cqes)
cq_entries = max(cq_entries, hr_dev->caps.min_cqes);
@@ -416,7 +410,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
goto err_cqc;
}
- return &hr_cq->ib_cq;
+ return 0;
err_cqc:
hns_roce_free_cq(hr_dev, hr_cq);
@@ -428,9 +422,8 @@ err_dbmap:
err_mtt:
hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
- if (udata)
- ib_umem_release(hr_cq->umem);
- else
+ ib_umem_release(hr_cq->umem);
+ if (!udata)
hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
hr_cq->ib_cq.cqe);
@@ -439,47 +432,37 @@ err_db:
hns_roce_free_db(hr_dev, &hr_cq->db);
err_cq:
- kfree(hr_cq);
- return ERR_PTR(ret);
+ return ret;
}
-EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq);
-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
- int ret = 0;
if (hr_dev->hw->destroy_cq) {
- ret = hr_dev->hw->destroy_cq(ib_cq, udata);
- } else {
- hns_roce_free_cq(hr_dev, hr_cq);
- hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
-
- if (udata) {
- ib_umem_release(hr_cq->umem);
-
- if (hr_cq->db_en == 1)
- hns_roce_db_unmap_user(
- rdma_udata_to_drv_context(
- udata,
- struct hns_roce_ucontext,
- ibucontext),
- &hr_cq->db);
- } else {
- /* Free the buff of stored cq */
- hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
- ib_cq->cqe);
- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
- hns_roce_free_db(hr_dev, &hr_cq->db);
- }
-
- kfree(hr_cq);
+ hr_dev->hw->destroy_cq(ib_cq, udata);
+ return;
}
- return ret;
+ hns_roce_free_cq(hr_dev, hr_cq);
+ hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
+
+ ib_umem_release(hr_cq->umem);
+ if (udata) {
+ if (hr_cq->db_en == 1)
+ hns_roce_db_unmap_user(rdma_udata_to_drv_context(
+ udata,
+ struct hns_roce_ucontext,
+ ibucontext),
+ &hr_cq->db);
+ } else {
+ /* Free the buff of stored cq */
+ hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe);
+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
+ hns_roce_free_db(hr_dev, &hr_cq->db);
+ }
}
-EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq);
void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
{
@@ -495,7 +478,6 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
++cq->arm_sn;
cq->comp(cq);
}
-EXPORT_SYMBOL_GPL(hns_roce_cq_completion);
void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
{
@@ -517,7 +499,6 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
if (atomic_dec_and_test(&cq->refcount))
complete(&cq->free);
}
-EXPORT_SYMBOL_GPL(hns_roce_cq_event);
int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)
{
diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c
index 0c6c1fe87705..627aa46ef683 100644
--- a/drivers/infiniband/hw/hns/hns_roce_db.c
+++ b/drivers/infiniband/hw/hns/hns_roce_db.c
@@ -51,7 +51,6 @@ out:
return ret;
}
-EXPORT_SYMBOL(hns_roce_db_map_user);
void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
struct hns_roce_db *db)
@@ -67,7 +66,6 @@ void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
mutex_unlock(&context->page_mutex);
}
-EXPORT_SYMBOL(hns_roce_db_unmap_user);
static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
struct device *dma_device)
@@ -78,7 +76,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
if (!pgdir)
return NULL;
- bitmap_fill(pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2);
+ bitmap_fill(pgdir->order1,
+ HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
pgdir->bits[0] = pgdir->order0;
pgdir->bits[1] = pgdir->order1;
pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
@@ -116,7 +115,7 @@ found:
db->u.pgdir = pgdir;
db->index = i;
db->db_record = pgdir->page + db->index;
- db->dma = pgdir->db_dma + db->index * 4;
+ db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE;
db->order = order;
return 0;
@@ -150,7 +149,6 @@ out:
return ret;
}
-EXPORT_SYMBOL_GPL(hns_roce_alloc_db);
void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
{
@@ -170,7 +168,8 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
i >>= o;
set_bit(i, db->u.pgdir->bits[o]);
- if (bitmap_full(db->u.pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2)) {
+ if (bitmap_full(db->u.pgdir->order1,
+ HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
db->u.pgdir->db_dma);
list_del(&db->u.pgdir->list);
@@ -179,4 +178,3 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
mutex_unlock(&hr_dev->pgdir_mutex);
}
-EXPORT_SYMBOL_GPL(hns_roce_free_db);
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 563cf39df6d5..a548b28aab63 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -37,9 +37,12 @@
#define DRV_NAME "hns_roce"
+/* hip08 is a pci device, it includes two version according pci version id */
+#define PCI_REVISION_ID_HIP08_A 0x20
+#define PCI_REVISION_ID_HIP08_B 0x21
+
#define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6')
-#define MAC_ADDR_OCTET_NUM 6
#define HNS_ROCE_MAX_MSG_LEN 0x80000000
#define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))
@@ -48,6 +51,10 @@
#define HNS_ROCE_BA_SIZE (32 * 4096)
+#define BA_BYTE_LEN 8
+
+#define BITS_PER_BYTE 8
+
/* Hardware specification only for v1 engine */
#define HNS_ROCE_MIN_CQE_NUM 0x40
#define HNS_ROCE_MIN_WQE_NUM 0x20
@@ -55,6 +62,7 @@
/* Hardware specification only for v1 engine */
#define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7
#define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000
+#define HNS_ROCE_MAX_SGE_NUM 2
#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20
#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \
@@ -64,6 +72,9 @@
#define HNS_ROCE_MAX_IRQ_NUM 128
+#define HNS_ROCE_SGE_IN_WQE 2
+#define HNS_ROCE_SGE_SHIFT 4
+
#define EQ_ENABLE 1
#define EQ_DISABLE 0
@@ -81,6 +92,7 @@
#define HNS_ROCE_MAX_PORTS 6
#define HNS_ROCE_MAX_GID_NUM 16
#define HNS_ROCE_GID_SIZE 16
+#define HNS_ROCE_SGE_SIZE 16
#define HNS_ROCE_HOP_NUM_0 0xff
@@ -111,6 +123,8 @@
#define PAGES_SHIFT_24 24
#define PAGES_SHIFT_32 32
+#define HNS_ROCE_PCI_BAR_NUM 2
+
#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
#define SRQ_DB_REG 0x230
@@ -213,6 +227,9 @@ enum hns_roce_mtt_type {
MTT_TYPE_IDX
};
+#define HNS_ROCE_DB_TYPE_COUNT 2
+#define HNS_ROCE_DB_UNIT_SIZE 4
+
enum {
HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4
};
@@ -324,6 +341,29 @@ struct hns_roce_mtt {
enum hns_roce_mtt_type mtt_type;
};
+struct hns_roce_buf_region {
+ int offset; /* page offset */
+ u32 count; /* page count*/
+ int hopnum; /* addressing hop num */
+};
+
+#define HNS_ROCE_MAX_BT_REGION 3
+#define HNS_ROCE_MAX_BT_LEVEL 3
+struct hns_roce_hem_list {
+ struct list_head root_bt;
+ /* link all bt dma mem by hop config */
+ struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];
+ struct list_head btm_bt; /* link all bottom bt in @mid_bt */
+ dma_addr_t root_ba; /* pointer to the root ba table */
+ int bt_pg_shift;
+};
+
+/* memory translate region */
+struct hns_roce_mtr {
+ struct hns_roce_hem_list hem_list;
+ int buf_pg_shift;
+};
+
struct hns_roce_mw {
struct ib_mw ibmw;
u32 pdn;
@@ -413,8 +453,8 @@ struct hns_roce_buf {
struct hns_roce_db_pgdir {
struct list_head list;
DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);
- DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2);
- unsigned long *bits[2];
+ DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
+ unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT];
u32 *page;
dma_addr_t db_dma;
};
@@ -472,7 +512,7 @@ struct hns_roce_idx_que {
u32 buf_size;
struct ib_umem *umem;
struct hns_roce_mtt mtt;
- u64 *bitmap;
+ unsigned long *bitmap;
};
struct hns_roce_srq {
@@ -535,7 +575,7 @@ struct hns_roce_av {
u8 hop_limit;
__le32 sl_tclass_flowlabel;
u8 dgid[HNS_ROCE_GID_SIZE];
- u8 mac[6];
+ u8 mac[ETH_ALEN];
__le16 vlan;
bool vlan_en;
};
@@ -620,6 +660,14 @@ struct hns_roce_qp {
struct ib_umem *umem;
struct hns_roce_mtt mtt;
+ struct hns_roce_mtr mtr;
+
+ /* this define must less than HNS_ROCE_MAX_BT_REGION */
+#define HNS_ROCE_WQE_REGION_MAX 3
+ struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX];
+ int region_cnt;
+ int wqe_bt_pg_shift;
+
u32 buff_size;
struct mutex mutex;
u8 port;
@@ -830,6 +878,9 @@ struct hns_roce_caps {
u32 mtt_ba_pg_sz;
u32 mtt_buf_pg_sz;
u32 mtt_hop_num;
+ u32 wqe_sq_hop_num;
+ u32 wqe_sge_hop_num;
+ u32 wqe_rq_hop_num;
u32 sccc_ba_pg_sz;
u32 sccc_buf_pg_sz;
u32 sccc_hop_num;
@@ -921,7 +972,7 @@ struct hns_roce_hw {
int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
struct ib_udata *udata);
- int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata);
+ void (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata);
int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
int (*init_eq)(struct hns_roce_dev *hr_dev);
void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
@@ -940,6 +991,16 @@ struct hns_roce_hw {
const struct ib_device_ops *hns_roce_dev_srq_ops;
};
+enum hns_phy_state {
+ HNS_ROCE_PHY_SLEEP = 1,
+ HNS_ROCE_PHY_POLLING = 2,
+ HNS_ROCE_PHY_DISABLED = 3,
+ HNS_ROCE_PHY_TRAINING = 4,
+ HNS_ROCE_PHY_LINKUP = 5,
+ HNS_ROCE_PHY_LINKERR = 6,
+ HNS_ROCE_PHY_TEST = 7
+};
+
struct hns_roce_dev {
struct ib_device ib_dev;
struct platform_device *pdev;
@@ -962,7 +1023,7 @@ struct hns_roce_dev {
struct hns_roce_caps caps;
struct xarray qp_table_xa;
- unsigned char dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
+ unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];
u64 sys_image_guid;
u32 vendor_id;
u32 vendor_part_id;
@@ -1084,6 +1145,19 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
struct hns_roce_mtt *mtt, struct hns_roce_buf *buf);
+void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift,
+ int buf_pg_shift);
+int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+ dma_addr_t **bufs, struct hns_roce_buf_region *regions,
+ int region_cnt);
+void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev,
+ struct hns_roce_mtr *mtr);
+
+/* hns roce hw need current block and next block addr from mtt */
+#define MTT_MIN_COUNT 2
+int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+ int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr);
+
int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);
@@ -1148,6 +1222,18 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
struct hns_roce_mtt *mtt, struct ib_umem *umem);
+void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum,
+ int offset, int buf_cnt);
+int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions,
+ dma_addr_t **bufs, int count);
+void hns_roce_free_buf_list(dma_addr_t **bufs, int count);
+
+int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
+ int buf_cnt, int start, struct hns_roce_buf *buf);
+int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
+ int buf_cnt, int start, struct ib_umem *umem,
+ int page_shift);
+
int hns_roce_create_srq(struct ib_srq *srq,
struct ib_srq_init_attr *srq_init_attr,
struct ib_udata *udata);
@@ -1178,11 +1264,11 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
__be32 send_ieth(const struct ib_send_wr *wr);
int to_hr_qp_type(int qp_type);
-struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+int hns_roce_ib_create_cq(struct ib_cq *ib_cq,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
+void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
int hns_roce_db_map_user(struct hns_roce_ucontext *context,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 8e29dbb5b5fb..f4da5bd2884f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -56,7 +56,6 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
return false;
}
-EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
u32 bt_chunk_num)
@@ -165,7 +164,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
+ PAGE_SHIFT);
mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
+ PAGE_SHIFT);
- mhop->ba_l0_num = mhop->bt_chunk_size / 8;
+ mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
mhop->hop_num = hr_dev->caps.mtt_hop_num;
break;
case HEM_TYPE_CQE:
@@ -173,7 +172,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
+ PAGE_SHIFT);
mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
+ PAGE_SHIFT);
- mhop->ba_l0_num = mhop->bt_chunk_size / 8;
+ mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
mhop->hop_num = hr_dev->caps.cqe_hop_num;
break;
case HEM_TYPE_SRQWQE:
@@ -181,7 +180,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
+ PAGE_SHIFT);
mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
+ PAGE_SHIFT);
- mhop->ba_l0_num = mhop->bt_chunk_size / 8;
+ mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
break;
case HEM_TYPE_IDX:
@@ -189,7 +188,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
+ PAGE_SHIFT);
mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
+ PAGE_SHIFT);
- mhop->ba_l0_num = mhop->bt_chunk_size / 8;
+ mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
mhop->hop_num = hr_dev->caps.idx_hop_num;
break;
default:
@@ -206,7 +205,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
* MTT/CQE alloc hem for bt pages.
*/
bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
- chunk_ba_num = mhop->bt_chunk_size / 8;
+ chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
mhop->bt_chunk_size;
table_idx = (*obj & (table->num_obj - 1)) /
@@ -234,7 +233,6 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
return 0;
}
-EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
int npages,
@@ -376,18 +374,19 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
- end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
- while (1) {
- if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
- if (!(time_before(jiffies, end))) {
- dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
- spin_unlock_irqrestore(lock, flags);
- return -EBUSY;
- }
- } else {
+ end = HW_SYNC_TIMEOUT_MSECS;
+ while (end) {
+ if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
break;
- }
+
mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
+ end -= HW_SYNC_SLEEP_TIME_INTERVAL;
+ }
+
+ if (end <= 0) {
+ dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
+ spin_unlock_irqrestore(lock, flags);
+ return -EBUSY;
}
bt_cmd_l = (u32)bt_ba;
@@ -435,7 +434,7 @@ static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
buf_chunk_size = mhop.buf_chunk_size;
bt_chunk_size = mhop.bt_chunk_size;
hop_num = mhop.hop_num;
- chunk_ba_num = bt_chunk_size / 8;
+ chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
bt_num = hns_roce_get_bt_num(table->type, hop_num);
switch (bt_num) {
@@ -620,7 +619,6 @@ out:
mutex_unlock(&table->mutex);
return ret;
}
-EXPORT_SYMBOL_GPL(hns_roce_table_get);
static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
@@ -645,7 +643,7 @@ static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
bt_chunk_size = mhop.bt_chunk_size;
hop_num = mhop.hop_num;
- chunk_ba_num = bt_chunk_size / 8;
+ chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
bt_num = hns_roce_get_bt_num(table->type, hop_num);
switch (bt_num) {
@@ -763,7 +761,6 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev,
mutex_unlock(&table->mutex);
}
-EXPORT_SYMBOL_GPL(hns_roce_table_put);
void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
@@ -799,7 +796,7 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
i = mhop.l0_idx;
j = mhop.l1_idx;
if (mhop.hop_num == 2)
- hem_idx = i * (mhop.bt_chunk_size / 8) + j;
+ hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
else if (mhop.hop_num == 1 ||
mhop.hop_num == HNS_ROCE_HOP_NUM_0)
hem_idx = i;
@@ -836,7 +833,6 @@ out:
mutex_unlock(&table->mutex);
return addr;
}
-EXPORT_SYMBOL_GPL(hns_roce_table_find);
int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
@@ -999,7 +995,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
}
obj_per_chunk = buf_chunk_size / obj_size;
num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
- bt_chunk_num = bt_chunk_size / 8;
+ bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
if (type >= HEM_TYPE_MTT)
num_bt_l0 = bt_chunk_num;
@@ -1156,3 +1152,463 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
&hr_dev->mr_table.mtt_cqe_table);
hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
}
+
+struct roce_hem_item {
+ struct list_head list; /* link all hems in the same bt level */
+ struct list_head sibling; /* link all hems in last hop for mtt */
+ void *addr;
+ dma_addr_t dma_addr;
+ size_t count; /* max ba numbers */
+ int start; /* start buf offset in this hem */
+ int end; /* end buf offset in this hem */
+};
+
+static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
+ int start, int end,
+ int count, bool exist_bt,
+ int bt_level)
+{
+ struct roce_hem_item *hem;
+
+ hem = kzalloc(sizeof(*hem), GFP_KERNEL);
+ if (!hem)
+ return NULL;
+
+ if (exist_bt) {
+ hem->addr = dma_alloc_coherent(hr_dev->dev,
+ count * BA_BYTE_LEN,
+ &hem->dma_addr, GFP_KERNEL);
+ if (!hem->addr) {
+ kfree(hem);
+ return NULL;
+ }
+ }
+
+ hem->count = count;
+ hem->start = start;
+ hem->end = end;
+ INIT_LIST_HEAD(&hem->list);
+ INIT_LIST_HEAD(&hem->sibling);
+
+ return hem;
+}
+
+static void hem_list_free_item(struct hns_roce_dev *hr_dev,
+ struct roce_hem_item *hem, bool exist_bt)
+{
+ if (exist_bt)
+ dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
+ hem->addr, hem->dma_addr);
+ kfree(hem);
+}
+
+static void hem_list_free_all(struct hns_roce_dev *hr_dev,
+ struct list_head *head, bool exist_bt)
+{
+ struct roce_hem_item *hem, *temp_hem;
+
+ list_for_each_entry_safe(hem, temp_hem, head, list) {
+ list_del(&hem->list);
+ hem_list_free_item(hr_dev, hem, exist_bt);
+ }
+}
+
+static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
+ u64 table_addr)
+{
+ *(u64 *)(base_addr) = table_addr;
+}
+
+/* assign L0 table address to hem from root bt */
+static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
+ struct roce_hem_item *hem, void *cpu_addr,
+ u64 phy_addr)
+{
+ hem->addr = cpu_addr;
+ hem->dma_addr = (dma_addr_t)phy_addr;
+}
+
+static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
+ int offset)
+{
+ return (hem->start <= offset && offset <= hem->end);
+}
+
+static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
+ int page_offset)
+{
+ struct roce_hem_item *hem, *temp_hem;
+ struct roce_hem_item *found = NULL;
+
+ list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
+ if (hem_list_page_is_in_range(hem, page_offset)) {
+ found = hem;
+ break;
+ }
+ }
+
+ return found;
+}
+
+static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
+{
+ /*
+ * hopnum base address table levels
+ * 0 L0(buf)
+ * 1 L0 -> buf
+ * 2 L0 -> L1 -> buf
+ * 3 L0 -> L1 -> L2 -> buf
+ */
+ return bt_level >= (hopnum ? hopnum - 1 : hopnum);
+}
+
+/**
+ * calc base address entries num
+ * @hopnum: num of mutihop addressing
+ * @bt_level: base address table level
+ * @unit: ba entries per bt page
+ */
+static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
+{
+ u32 step;
+ int max;
+ int i;
+
+ if (hopnum <= bt_level)
+ return 0;
+ /*
+ * hopnum bt_level range
+ * 1 0 unit
+ * ------------
+ * 2 0 unit * unit
+ * 2 1 unit
+ * ------------
+ * 3 0 unit * unit * unit
+ * 3 1 unit * unit
+ * 3 2 unit
+ */
+ step = 1;
+ max = hopnum - bt_level;
+ for (i = 0; i < max; i++)
+ step = step * unit;
+
+ return step;
+}
+
+/**
+ * calc the root ba entries which could cover all regions
+ * @regions: buf region array
+ * @region_cnt: array size of @regions
+ * @unit: ba entries per bt page
+ */
+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
+ int region_cnt, int unit)
+{
+ struct hns_roce_buf_region *r;
+ int total = 0;
+ int step;
+ int i;
+
+ for (i = 0; i < region_cnt; i++) {
+ r = (struct hns_roce_buf_region *)&regions[i];
+ if (r->hopnum > 1) {
+ step = hem_list_calc_ba_range(r->hopnum, 1, unit);
+ if (step > 0)
+ total += (r->count + step - 1) / step;
+ } else {
+ total += r->count;
+ }
+ }
+
+ return total;
+}
+
+static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
+ const struct hns_roce_buf_region *r, int unit,
+ int offset, struct list_head *mid_bt,
+ struct list_head *btm_bt)
+{
+ struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
+ struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
+ struct roce_hem_item *cur, *pre;
+ const int hopnum = r->hopnum;
+ int start_aligned;
+ int distance;
+ int ret = 0;
+ int max_ofs;
+ int level;
+ u32 step;
+ int end;
+
+ if (hopnum <= 1)
+ return 0;
+
+ if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
+ dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
+ return -EINVAL;
+ }
+
+ if (offset < r->offset) {
+ dev_err(hr_dev->dev, "invalid offset %d,min %d!\n",
+ offset, r->offset);
+ return -EINVAL;
+ }
+
+ distance = offset - r->offset;
+ max_ofs = r->offset + r->count - 1;
+ for (level = 0; level < hopnum; level++)
+ INIT_LIST_HEAD(&temp_list[level]);
+
+ /* config L1 bt to last bt and link them to corresponding parent */
+ for (level = 1; level < hopnum; level++) {
+ cur = hem_list_search_item(&mid_bt[level], offset);
+ if (cur) {
+ hem_ptrs[level] = cur;
+ continue;
+ }
+
+ step = hem_list_calc_ba_range(hopnum, level, unit);
+ if (step < 1) {
+ ret = -EINVAL;
+ goto err_exit;
+ }
+
+ start_aligned = (distance / step) * step + r->offset;
+ end = min_t(int, start_aligned + step - 1, max_ofs);
+ cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
+ true, level);
+ if (!cur) {
+ ret = -ENOMEM;
+ goto err_exit;
+ }
+ hem_ptrs[level] = cur;
+ list_add(&cur->list, &temp_list[level]);
+ if (hem_list_is_bottom_bt(hopnum, level))
+ list_add(&cur->sibling, &temp_list[0]);
+
+ /* link bt to parent bt */
+ if (level > 1) {
+ pre = hem_ptrs[level - 1];
+ step = (cur->start - pre->start) / step * BA_BYTE_LEN;
+ hem_list_link_bt(hr_dev, pre->addr + step,
+ cur->dma_addr);
+ }
+ }
+
+ list_splice(&temp_list[0], btm_bt);
+ for (level = 1; level < hopnum; level++)
+ list_splice(&temp_list[level], &mid_bt[level]);
+
+ return 0;
+
+err_exit:
+ for (level = 1; level < hopnum; level++)
+ hem_list_free_all(hr_dev, &temp_list[level], true);
+
+ return ret;
+}
+
+static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list, int unit,
+ const struct hns_roce_buf_region *regions,
+ int region_cnt)
+{
+ struct roce_hem_item *hem, *temp_hem, *root_hem;
+ struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
+ const struct hns_roce_buf_region *r;
+ struct list_head temp_root;
+ struct list_head temp_btm;
+ void *cpu_base;
+ u64 phy_base;
+ int ret = 0;
+ int offset;
+ int total;
+ int step;
+ int i;
+
+ r = &regions[0];
+ root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
+ if (root_hem)
+ return 0;
+
+ INIT_LIST_HEAD(&temp_root);
+ total = r->offset;
+ /* indicate to last region */
+ r = &regions[region_cnt - 1];
+ root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1,
+ unit, true, 0);
+ if (!root_hem)
+ return -ENOMEM;
+ list_add(&root_hem->list, &temp_root);
+
+ hem_list->root_ba = root_hem->dma_addr;
+
+ INIT_LIST_HEAD(&temp_btm);
+ for (i = 0; i < region_cnt; i++)
+ INIT_LIST_HEAD(&temp_list[i]);
+
+ total = 0;
+ for (i = 0; i < region_cnt && total < unit; i++) {
+ r = &regions[i];
+ if (!r->count)
+ continue;
+
+ /* all regions's mid[x][0] shared the root_bt's trunk */
+ cpu_base = root_hem->addr + total * BA_BYTE_LEN;
+ phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
+
+ /* if hopnum is 0 or 1, cut a new fake hem from the root bt
+ * which's address share to all regions.
+ */
+ if (hem_list_is_bottom_bt(r->hopnum, 0)) {
+ hem = hem_list_alloc_item(hr_dev, r->offset,
+ r->offset + r->count - 1,
+ r->count, false, 0);
+ if (!hem) {
+ ret = -ENOMEM;
+ goto err_exit;
+ }
+ hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
+ list_add(&hem->list, &temp_list[i]);
+ list_add(&hem->sibling, &temp_btm);
+ total += r->count;
+ } else {
+ step = hem_list_calc_ba_range(r->hopnum, 1, unit);
+ if (step < 1) {
+ ret = -EINVAL;
+ goto err_exit;
+ }
+ /* if exist mid bt, link L1 to L0 */
+ list_for_each_entry_safe(hem, temp_hem,
+ &hem_list->mid_bt[i][1], list) {
+ offset = hem->start / step * BA_BYTE_LEN;
+ hem_list_link_bt(hr_dev, cpu_base + offset,
+ hem->dma_addr);
+ total++;
+ }
+ }
+ }
+
+ list_splice(&temp_btm, &hem_list->btm_bt);
+ list_splice(&temp_root, &hem_list->root_bt);
+ for (i = 0; i < region_cnt; i++)
+ list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
+
+ return 0;
+
+err_exit:
+ for (i = 0; i < region_cnt; i++)
+ hem_list_free_all(hr_dev, &temp_list[i], false);
+
+ hem_list_free_all(hr_dev, &temp_root, true);
+
+ return ret;
+}
+
+/* construct the base address table and link them by address hop config */
+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list,
+ const struct hns_roce_buf_region *regions,
+ int region_cnt)
+{
+ const struct hns_roce_buf_region *r;
+ int ofs, end;
+ int ret = 0;
+ int unit;
+ int i;
+
+ if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
+ dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
+ region_cnt);
+ return -EINVAL;
+ }
+
+ unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN;
+ for (i = 0; i < region_cnt; i++) {
+ r = &regions[i];
+ if (!r->count)
+ continue;
+
+ end = r->offset + r->count;
+ for (ofs = r->offset; ofs < end; ofs += unit) {
+ ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
+ hem_list->mid_bt[i],
+ &hem_list->btm_bt);
+ if (ret) {
+ dev_err(hr_dev->dev,
+ "alloc hem trunk fail ret=%d!\n", ret);
+ goto err_alloc;
+ }
+ }
+ }
+
+ ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
+ region_cnt);
+ if (ret)
+ dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
+ else
+ return 0;
+
+err_alloc:
+ hns_roce_hem_list_release(hr_dev, hem_list);
+
+ return ret;
+}
+
+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list)
+{
+ int i, j;
+
+ for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
+ for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
+ hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
+ j != 0);
+
+ hem_list_free_all(hr_dev, &hem_list->root_bt, true);
+ INIT_LIST_HEAD(&hem_list->btm_bt);
+ hem_list->root_ba = 0;
+}
+
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
+ int bt_page_order)
+{
+ int i, j;
+
+ INIT_LIST_HEAD(&hem_list->root_bt);
+ INIT_LIST_HEAD(&hem_list->btm_bt);
+ for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
+ for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
+ INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
+
+ hem_list->bt_pg_shift = bt_page_order;
+}
+
+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list,
+ int offset, int *mtt_cnt, u64 *phy_addr)
+{
+ struct list_head *head = &hem_list->btm_bt;
+ struct roce_hem_item *hem, *temp_hem;
+ void *cpu_base = NULL;
+ u64 phy_base = 0;
+ int nr = 0;
+
+ list_for_each_entry_safe(hem, temp_hem, head, sibling) {
+ if (hem_list_page_is_in_range(hem, offset)) {
+ nr = offset - hem->start;
+ cpu_base = hem->addr + nr * BA_BYTE_LEN;
+ phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
+ nr = hem->end + 1 - offset;
+ break;
+ }
+ }
+
+ if (mtt_cnt)
+ *mtt_cnt = nr;
+
+ if (phy_addr)
+ *phy_addr = phy_base;
+
+ return cpu_base;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h
index d9d668992e49..f1ccb8f35fe5 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.h
@@ -34,8 +34,8 @@
#ifndef _HNS_ROCE_HEM_H
#define _HNS_ROCE_HEM_H
-#define HW_SYNC_TIMEOUT_MSECS 500
#define HW_SYNC_SLEEP_TIME_INTERVAL 20
+#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL)
#define BT_CMD_SYNC_SHIFT 31
enum {
@@ -133,6 +133,20 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_mhop *mhop);
bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type);
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
+ int bt_page_order);
+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
+ int region_cnt, int unit);
+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list,
+ const struct hns_roce_buf_region *regions,
+ int region_cnt);
+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list);
+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
+ struct hns_roce_hem_list *hem_list,
+ int offset, int *mtt_cnt, u64 *phy_addr);
+
static inline void hns_roce_hem_first(struct hns_roce_hem *hem,
struct hns_roce_hem_iter *iter)
{
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index e068a02122f5..056a6873df7a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -717,7 +717,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
union ib_gid dgid;
u64 subnet_prefix;
int attr_mask = 0;
- int ret = -ENOMEM;
+ int ret;
int i, j;
u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };
u8 phy_port;
@@ -730,10 +730,16 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
/* Reserved cq for loop qp */
cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2;
cq_init_attr.comp_vector = 0;
- cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL);
- if (IS_ERR(cq)) {
- dev_err(dev, "Create cq for reserved loop qp failed!");
+
+ ibdev = &hr_dev->ib_dev;
+ cq = rdma_zalloc_drv_obj(ibdev, ib_cq);
+ if (!cq)
return -ENOMEM;
+
+ ret = hns_roce_ib_create_cq(cq, &cq_init_attr, NULL);
+ if (ret) {
+ dev_err(dev, "Create cq for reserved loop qp failed!");
+ goto alloc_cq_failed;
}
free_mr->mr_free_cq = to_hr_cq(cq);
free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev;
@@ -743,7 +749,6 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
free_mr->mr_free_cq->ib_cq.cq_context = NULL;
atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0);
- ibdev = &hr_dev->ib_dev;
pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
if (!pd)
goto alloc_mem_failed;
@@ -818,7 +823,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
attr.dest_qp_num = hr_qp->qpn;
memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr),
hr_dev->dev_addr[port],
- MAC_ADDR_OCTET_NUM);
+ ETH_ALEN);
memcpy(&dgid.raw, &subnet_prefix, sizeof(u64));
memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3);
@@ -865,9 +870,9 @@ alloc_pd_failed:
kfree(pd);
alloc_mem_failed:
- if (hns_roce_ib_destroy_cq(cq, NULL))
- dev_err(dev, "Destroy cq for create_lp_qp failed!\n");
-
+ hns_roce_ib_destroy_cq(cq, NULL);
+alloc_cq_failed:
+ kfree(cq);
return ret;
}
@@ -894,10 +899,8 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
i, ret);
}
- ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL);
- if (ret)
- dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret);
-
+ hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL);
+ kfree(&free_mr->mr_free_cq->ib_cq);
hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL);
kfree(&free_mr->mr_free_pd->ibpd);
}
@@ -966,8 +969,7 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
struct hns_roce_free_mr *free_mr;
struct hns_roce_v1_priv *priv;
struct completion comp;
- unsigned long end =
- msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies;
+ unsigned long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS;
priv = (struct hns_roce_v1_priv *)hr_dev->priv;
free_mr = &priv->free_mr;
@@ -987,10 +989,11 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
queue_work(free_mr->free_mr_wq, &(lp_qp_work->work));
- while (time_before_eq(jiffies, end)) {
+ while (end) {
if (try_wait_for_completion(&comp))
return 0;
msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);
+ end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE;
}
lp_qp_work->comp_flag = 0;
@@ -1104,8 +1107,7 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
struct hns_roce_free_mr *free_mr;
struct hns_roce_v1_priv *priv;
struct completion comp;
- unsigned long end =
- msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
+ unsigned long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS;
unsigned long start = jiffies;
int npages;
int ret = 0;
@@ -1135,10 +1137,11 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
queue_work(free_mr->free_mr_wq, &(mr_work->work));
- while (time_before_eq(jiffies, end)) {
+ while (end) {
if (try_wait_for_completion(&comp))
goto free_mr;
msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
+ end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE;
}
mr_work->comp_flag = 0;
@@ -1161,8 +1164,7 @@ free_mr:
hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
key_to_hw_index(mr->key), 0);
- if (mr->umem)
- ib_umem_release(mr->umem);
+ ib_umem_release(mr->umem);
kfree(mr);
@@ -1742,11 +1744,14 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port,
int gid_index, const union ib_gid *gid,
const struct ib_gid_attr *attr)
{
+ unsigned long flags;
u32 *p = NULL;
u8 gid_idx = 0;
gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
+ spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+
p = (u32 *)&gid->raw[0];
roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
(HNS_ROCE_V1_GID_NUM * gid_idx));
@@ -1763,6 +1768,8 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port,
roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
(HNS_ROCE_V1_GID_NUM * gid_idx));
+ spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
return 0;
}
@@ -2458,10 +2465,10 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
- end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
+ end = HW_SYNC_TIMEOUT_MSECS;
while (1) {
if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
- if (!(time_before(jiffies, end))) {
+ if (!end) {
dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
flags);
@@ -2470,7 +2477,8 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
} else {
break;
}
- msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
+ mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
+ end -= HW_SYNC_SLEEP_TIME_INTERVAL;
}
bt_cmd_val[0] = (__le32)bt_ba;
@@ -3633,9 +3641,8 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
- if (udata)
- ib_umem_release(hr_qp->umem);
- else {
+ ib_umem_release(hr_qp->umem);
+ if (!udata) {
kfree(hr_qp->sq.wrid);
kfree(hr_qp->rq.wrid);
@@ -3649,7 +3656,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
return 0;
}
-static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
@@ -3658,7 +3665,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
u32 cqe_cnt_cur;
u32 cq_buf_size;
int wait_time = 0;
- int ret = 0;
hns_roce_free_cq(hr_dev, hr_cq);
@@ -3680,7 +3686,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) {
dev_warn(dev, "Destroy cq 0x%lx timeout!\n",
hr_cq->cqn);
- ret = -ETIMEDOUT;
break;
}
wait_time++;
@@ -3688,17 +3693,12 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
- if (ibcq->uobject)
- ib_umem_release(hr_cq->umem);
- else {
+ ib_umem_release(hr_cq->umem);
+ if (!udata) {
/* Free the buff of stored cq */
cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz;
hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf);
}
-
- kfree(hr_cq);
-
- return ret;
}
static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not)
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index b5392cb5b20f..7f2da5e10e67 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -1098,7 +1098,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
if (ret == CMD_RST_PRC_SUCCESS)
return 0;
if (ret == CMD_RST_PRC_EBUSY)
- return ret;
+ return -EBUSY;
ret = __hns_roce_cmq_send(hr_dev, desc, num);
if (ret) {
@@ -1106,7 +1106,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
if (retval == CMD_RST_PRC_SUCCESS)
return 0;
else if (retval == CMD_RST_PRC_EBUSY)
- return retval;
+ return -EBUSY;
}
return ret;
@@ -1130,6 +1130,47 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
return 0;
}
+static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
+{
+ bool fclr_write_fail_flag = false;
+ struct hns_roce_func_clear *resp;
+ struct hns_roce_cmq_desc desc;
+ unsigned long end;
+ int ret;
+
+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
+ resp = (struct hns_roce_func_clear *)desc.data;
+
+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+ if (ret) {
+ fclr_write_fail_flag = true;
+ dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
+ ret);
+ return;
+ }
+
+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
+ end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
+ while (end) {
+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
+ end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
+
+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
+ true);
+
+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+ if (ret)
+ continue;
+
+ if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
+ hr_dev->is_reset = true;
+ return;
+ }
+ }
+
+ dev_err(hr_dev->dev, "Func clear fail.\n");
+}
+
static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
{
struct hns_roce_query_fw_info *resp;
@@ -1574,6 +1615,9 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
caps->mtt_ba_pg_sz = 0;
caps->mtt_buf_pg_sz = 0;
caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
+ caps->wqe_sq_hop_num = 2;
+ caps->wqe_sge_hop_num = 1;
+ caps->wqe_rq_hop_num = 2;
caps->cqe_ba_pg_sz = 0;
caps->cqe_buf_pg_sz = 0;
caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
@@ -1891,6 +1935,9 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
+ if (hr_dev->pci_dev->revision == 0x21)
+ hns_roce_function_clear(hr_dev);
+
hns_roce_free_link_table(hr_dev, &priv->tpq);
hns_roce_free_link_table(hr_dev, &priv->tsq);
}
@@ -2340,15 +2387,10 @@ static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
{
- u32 bitmap_num;
- int bit_num;
-
/* always called with interrupts disabled. */
spin_lock(&srq->lock);
- bitmap_num = wqe_index / (sizeof(u64) * 8);
- bit_num = wqe_index % (sizeof(u64) * 8);
- srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num);
+ bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
srq->tail++;
spin_unlock(&srq->lock);
@@ -3026,7 +3068,6 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
}
static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
- struct hns_roce_mtt *mtt,
enum ib_qp_state cur_state,
enum ib_qp_state new_state,
struct hns_roce_v2_qp_context *context,
@@ -3426,7 +3467,9 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
else
roce_set_field(context->byte_4_sqpn_tst,
V2_QPC_BYTE_4_SGE_SHIFT_M,
- V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
+ V2_QPC_BYTE_4_SGE_SHIFT_S,
+ hr_qp->sq.max_gs >
+ HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
@@ -3520,6 +3563,31 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
}
}
+static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
+ struct hns_roce_qp *hr_qp, int mtt_cnt,
+ u32 page_size)
+{
+ struct device *dev = hr_dev->dev;
+
+ if (hr_qp->rq.wqe_cnt < 1)
+ return true;
+
+ if (mtt_cnt < 1) {
+ dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
+ hr_qp->qpn);
+ return false;
+ }
+
+ if (mtt_cnt < MTT_MIN_COUNT &&
+ (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
+ dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
+ hr_qp->qpn);
+ return false;
+ }
+
+ return true;
+}
+
static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
const struct ib_qp_attr *attr, int attr_mask,
struct hns_roce_v2_qp_context *context,
@@ -3529,25 +3597,27 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
struct device *dev = hr_dev->dev;
+ u64 mtts[MTT_MIN_COUNT] = { 0 };
dma_addr_t dma_handle_3;
dma_addr_t dma_handle_2;
- dma_addr_t dma_handle;
+ u64 wqe_sge_ba;
u32 page_size;
u8 port_num;
u64 *mtts_3;
u64 *mtts_2;
- u64 *mtts;
+ int count;
u8 *dmac;
u8 *smac;
int port;
/* Search qp buf's mtts */
- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
- hr_qp->mtt.first_seg, &dma_handle);
- if (!mtts) {
- dev_err(dev, "qp buf pa find failed\n");
- return -EINVAL;
- }
+ page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
+ hr_qp->rq.offset / page_size, mtts,
+ MTT_MIN_COUNT, &wqe_sge_ba);
+ if (!ibqp->srq)
+ if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
+ return -EINVAL;
/* Search IRRL's mtts */
mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
@@ -3571,7 +3641,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
}
dmac = (u8 *)attr->ah_attr.roce.dmac;
- context->wqe_sge_ba = (u32)(dma_handle >> 3);
+ context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3);
qpc_mask->wqe_sge_ba = 0;
/*
@@ -3581,22 +3651,23 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
* 0 at the same time, else set them to 0x1.
*/
roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
- V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
+ V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
V2_QPC_BYTE_12_SQ_HOP_NUM_S,
- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
- 0 : hr_dev->caps.mtt_hop_num);
+ hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
+ 0 : hr_dev->caps.wqe_sq_hop_num);
roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
roce_set_field(context->byte_20_smac_sgid_idx,
V2_QPC_BYTE_20_SGE_HOP_NUM_M,
V2_QPC_BYTE_20_SGE_HOP_NUM_S,
- ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
- hr_dev->caps.mtt_hop_num : 0);
+ ((ibqp->qp_type == IB_QPT_GSI) ||
+ hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
+ hr_dev->caps.wqe_sge_hop_num : 0);
roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
V2_QPC_BYTE_20_SGE_HOP_NUM_M,
V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
@@ -3604,8 +3675,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
roce_set_field(context->byte_20_smac_sgid_idx,
V2_QPC_BYTE_20_RQ_HOP_NUM_M,
V2_QPC_BYTE_20_RQ_HOP_NUM_S,
- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
- 0 : hr_dev->caps.mtt_hop_num);
+ hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
+ 0 : hr_dev->caps.wqe_rq_hop_num);
roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
V2_QPC_BYTE_20_RQ_HOP_NUM_M,
V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
@@ -3613,7 +3684,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
roce_set_field(context->byte_16_buf_ba_pg_sz,
V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
- hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);
+ hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
@@ -3626,29 +3697,24 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
- context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
- >> PAGE_ADDR_SHIFT);
+ context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
qpc_mask->rq_cur_blk_addr = 0;
roce_set_field(context->byte_92_srq_info,
V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
- mtts[hr_qp->rq.offset / page_size]
- >> (32 + PAGE_ADDR_SHIFT));
+ mtts[0] >> (32 + PAGE_ADDR_SHIFT));
roce_set_field(qpc_mask->byte_92_srq_info,
V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
- context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
- >> PAGE_ADDR_SHIFT);
+ context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
qpc_mask->rq_nxt_blk_addr = 0;
roce_set_field(context->byte_104_rq_sge,
V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
- mtts[hr_qp->rq.offset / page_size + 1]
- >> (32 + PAGE_ADDR_SHIFT));
+ mtts[1] >> (32 + PAGE_ADDR_SHIFT));
roce_set_field(qpc_mask->byte_104_rq_sge,
V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
@@ -3708,13 +3774,14 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
V2_QPC_BYTE_20_SGID_IDX_M,
V2_QPC_BYTE_20_SGID_IDX_S, 0);
- memcpy(&(context->dmac), dmac, 4);
+ memcpy(&(context->dmac), dmac, sizeof(u32));
roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
qpc_mask->dmac = 0;
roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
V2_QPC_BYTE_52_DMAC_S, 0);
+ /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
@@ -3756,6 +3823,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
+ /* rocee send 2^lp_sgen_ini segs every time */
roce_set_field(context->byte_168_irrl_idx,
V2_QPC_BYTE_168_LP_SGEN_INI_M,
V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
@@ -3774,18 +3842,30 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
struct device *dev = hr_dev->dev;
- dma_addr_t dma_handle;
+ u64 sge_cur_blk = 0;
+ u64 sq_cur_blk = 0;
u32 page_size;
- u64 *mtts;
+ int count;
/* Search qp buf's mtts */
- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
- hr_qp->mtt.first_seg, &dma_handle);
- if (!mtts) {
- dev_err(dev, "qp buf pa find failed\n");
+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
+ if (count < 1) {
+ dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
return -EINVAL;
}
+ if (hr_qp->sge.offset) {
+ page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
+ hr_qp->sge.offset / page_size,
+ &sge_cur_blk, 1, NULL);
+ if (count < 1) {
+ dev_err(dev, "qp(0x%lx) sge pa find failed\n",
+ hr_qp->qpn);
+ return -EINVAL;
+ }
+ }
+
/* Not support alternate path and path migration */
if ((attr_mask & IB_QP_ALT_PATH) ||
(attr_mask & IB_QP_PATH_MIG_STATE)) {
@@ -3799,37 +3879,37 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
* we should set all bits of the relevant fields in context mask to
* 0 at the same time, else set them to 0x1.
*/
- context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
+ context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
roce_set_field(context->byte_168_irrl_idx,
V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
+ sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
qpc_mask->sq_cur_blk_addr = 0;
roce_set_field(qpc_mask->byte_168_irrl_idx,
V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
- context->sq_cur_sge_blk_addr =
- ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
- ((u32)(mtts[hr_qp->sge.offset / page_size]
- >> PAGE_ADDR_SHIFT)) : 0;
+ context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
+ hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
+ ((u32)(sge_cur_blk >>
+ PAGE_ADDR_SHIFT)) : 0;
roce_set_field(context->byte_184_irrl_idx,
V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
- ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
- (mtts[hr_qp->sge.offset / page_size] >>
+ ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
+ HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
+ (sge_cur_blk >>
(32 + PAGE_ADDR_SHIFT)) : 0);
qpc_mask->sq_cur_sge_blk_addr = 0;
roce_set_field(qpc_mask->byte_184_irrl_idx,
V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
- context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
+ context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
roce_set_field(context->byte_232_irrl_sge,
V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
+ sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
qpc_mask->rx_sq_cur_blk_addr = 0;
roce_set_field(qpc_mask->byte_232_irrl_sge,
V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
@@ -4144,7 +4224,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
roce_set_field(context->byte_224_retry_msg,
V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
- attr->sq_psn >> 16);
+ attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
roce_set_field(qpc_mask->byte_224_retry_msg,
V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
@@ -4230,7 +4310,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
V2_QPC_BYTE_60_QP_ST_S, 0);
/* SW pass context to HW */
- ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state,
+ ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state,
context, hr_qp);
if (ret) {
dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
@@ -4374,11 +4454,12 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
V2_QPC_BYTE_56_DQPN_M,
V2_QPC_BYTE_56_DQPN_S);
qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
- V2_QPC_BYTE_76_RRE_S)) << 2) |
- ((roce_get_bit(context->byte_76_srqn_op_en,
- V2_QPC_BYTE_76_RWE_S)) << 1) |
- ((roce_get_bit(context->byte_76_srqn_op_en,
- V2_QPC_BYTE_76_ATE_S)) << 3);
+ V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) |
+ ((roce_get_bit(context->byte_76_srqn_op_en,
+ V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) |
+ ((roce_get_bit(context->byte_76_srqn_op_en,
+ V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
+
if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
hr_qp->ibqp.qp_type == IB_QPT_UC) {
struct ib_global_route *grh =
@@ -4501,7 +4582,6 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
hns_roce_db_unmap_user(context, &hr_qp->rdb);
- ib_umem_release(hr_qp->umem);
} else {
kfree(hr_qp->sq.wrid);
kfree(hr_qp->rq.wrid);
@@ -4509,6 +4589,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
if (hr_qp->rq.wqe_cnt)
hns_roce_free_db(hr_dev, &hr_qp->rdb);
}
+ ib_umem_release(hr_qp->umem);
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
hr_qp->rq.wqe_cnt) {
@@ -4682,7 +4763,6 @@ static void hns_roce_irq_work_handle(struct work_struct *work)
dev_warn(dev, "Path migration failed.\n");
break;
case HNS_ROCE_EVENT_TYPE_COMM_EST:
- dev_info(dev, "Communication established.\n");
break;
case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
dev_warn(dev, "Send queue drained.\n");
@@ -5151,8 +5231,8 @@ static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
eq->l1_dma[i]);
- for (j = 0; j < bt_chk_sz / 8; j++) {
- idx = i * (bt_chk_sz / 8) + j;
+ for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
+ idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
if ((i == eq->l0_last_num - 1)
&& j == eq->l1_last_num - 1) {
eqe_alloc = (buf_chk_sz / eq->eqe_size)
@@ -5368,9 +5448,9 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
- ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1)
- / buf_chk_sz;
- bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);
+ ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
+ buf_chk_sz);
+ bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
/* hop_num = 0 */
if (mhop_num == HNS_ROCE_HOP_NUM_0) {
@@ -5415,12 +5495,12 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
goto err_dma_alloc_l0;
if (mhop_num == 1) {
- if (ba_num > (bt_chk_sz / 8))
+ if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
dev_err(dev, "ba_num %d is too large for 1 hop\n",
ba_num);
/* alloc buf */
- for (i = 0; i < bt_chk_sz / 8; i++) {
+ for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
if (eq_buf_cnt + 1 < ba_num) {
size = buf_chk_sz;
} else {
@@ -5444,7 +5524,7 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
} else if (mhop_num == 2) {
/* alloc L1 BT and buf */
- for (i = 0; i < bt_chk_sz / 8; i++) {
+ for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
&(eq->l1_dma[i]),
GFP_KERNEL);
@@ -5452,8 +5532,8 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
goto err_dma_alloc_l1;
*(eq->bt_l0 + i) = eq->l1_dma[i];
- for (j = 0; j < bt_chk_sz / 8; j++) {
- idx = i * bt_chk_sz / 8 + j;
+ for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
+ idx = i * bt_chk_sz / BA_BYTE_LEN + j;
if (eq_buf_cnt + 1 < ba_num) {
size = buf_chk_sz;
} else {
@@ -5498,8 +5578,8 @@ err_dma_alloc_l1:
dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
eq->l1_dma[i]);
- for (j = 0; j < bt_chk_sz / 8; j++) {
- idx = i * bt_chk_sz / 8 + j;
+ for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
+ idx = i * bt_chk_sz / BA_BYTE_LEN + j;
dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
eq->buf_dma[idx]);
}
@@ -5522,11 +5602,11 @@ err_dma_alloc_buf:
dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
eq->l1_dma[i]);
- for (j = 0; j < bt_chk_sz / 8; j++) {
+ for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
if (i == record_i && j >= record_j)
break;
- idx = i * bt_chk_sz / 8 + j;
+ idx = i * bt_chk_sz / BA_BYTE_LEN + j;
dma_free_coherent(dev, buf_chk_sz,
eq->buf[idx],
eq->buf_dma[idx]);
@@ -5972,18 +6052,19 @@ out:
return ret;
}
-static int find_empty_entry(struct hns_roce_idx_que *idx_que)
+static int find_empty_entry(struct hns_roce_idx_que *idx_que,
+ unsigned long size)
{
- int bit_num;
- int i;
+ int wqe_idx;
- /* bitmap[i] is set zero if all bits are allocated */
- for (i = 0; idx_que->bitmap[i] == 0; ++i)
- ;
- bit_num = ffs(idx_que->bitmap[i]);
- idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1));
+ if (unlikely(bitmap_full(idx_que->bitmap, size)))
+ return -ENOSPC;
- return i * sizeof(u64) * 8 + (bit_num - 1);
+ wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
+
+ bitmap_set(idx_que->bitmap, wqe_idx, 1);
+
+ return wqe_idx;
}
static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
@@ -6029,7 +6110,13 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
break;
}
- wqe_idx = find_empty_entry(&srq->idx_que);
+ wqe_idx = find_empty_entry(&srq->idx_que, srq->max);
+ if (wqe_idx < 0) {
+ ret = -ENOMEM;
+ *bad_wr = wr;
+ break;
+ }
+
fill_idx_queue(&srq->idx_que, ind, wqe_idx);
wqe = get_srq_wqe(srq, wqe_idx);
dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
@@ -6041,9 +6128,9 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
}
if (i < srq->max_gs) {
- dseg->len = 0;
- dseg->lkey = cpu_to_le32(0x100);
- dseg->addr = 0;
+ dseg[i].len = 0;
+ dseg[i].lkey = cpu_to_le32(0x100);
+ dseg[i].addr = 0;
}
srq->wrid[wqe_idx] = wr->wr_id;
@@ -6059,7 +6146,8 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
*/
wmb();
- srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << 24 | srq->srqn;
+ srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
+ (srq->srqn & V2_DB_BYTE_4_TAG_M);
srq_db.parameter = srq->head;
hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index edfdbe2ce0db..478f5a5b7aa1 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -54,7 +54,7 @@
#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000
#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100
#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000
-#define HNS_ROCE_V2_MAX_CQE_NUM 0x10000
+#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000
#define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000
#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 0x100
#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 0xff
@@ -241,6 +241,7 @@ enum hns_roce_opcode_type {
HNS_ROCE_OPC_POST_MB = 0x8504,
HNS_ROCE_OPC_QUERY_MB_ST = 0x8505,
HNS_ROCE_OPC_CFG_BT_ATTR = 0x8506,
+ HNS_ROCE_OPC_FUNC_CLEAR = 0x8508,
HNS_ROCE_OPC_CLR_SCCC = 0x8509,
HNS_ROCE_OPC_QUERY_SCCC = 0x850a,
HNS_ROCE_OPC_RESET_SCCC = 0x850b,
@@ -886,6 +887,10 @@ struct hns_roce_v2_qp_context {
#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16
#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16)
+#define V2_QP_RWE_S 1 /* rdma write enable */
+#define V2_QP_RRE_S 2 /* rdma read enable */
+#define V2_QP_ATE_S 3 /* rdma atomic enable */
+
struct hns_roce_v2_cqe {
__le32 byte_4;
union {
@@ -1226,6 +1231,22 @@ struct hns_roce_query_fw_info {
__le32 rsv[5];
};
+struct hns_roce_func_clear {
+ __le32 rst_funcid_en;
+ __le32 func_done;
+ __le32 rsv[4];
+};
+
+#define FUNC_CLEAR_RST_FUN_DONE_S 0
+/* Each physical function manages up to 248 virtual functionsï¼›
+ * it takes up to 100ms for each function to execute clearï¼›
+ * if an abnormal reset occurs, it is executed twice at most;
+ * so it takes up to 249 * 2 * 100ms.
+ */
+#define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100)
+#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL 40
+#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20
+
struct hns_roce_cfg_llm_a {
__le32 base_addr_l;
__le32 base_addr_h;
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 8da5f18bf820..9f83acec6001 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -57,17 +57,16 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
{
return gid_index * hr_dev->caps.num_ports + port;
}
-EXPORT_SYMBOL_GPL(hns_get_gid_index);
static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
{
u8 phy_port;
u32 i = 0;
- if (!memcmp(hr_dev->dev_addr[port], addr, MAC_ADDR_OCTET_NUM))
+ if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN))
return 0;
- for (i = 0; i < MAC_ADDR_OCTET_NUM; i++)
+ for (i = 0; i < ETH_ALEN; i++)
hr_dev->dev_addr[port][i] = addr[i];
phy_port = hr_dev->iboe.phy_port[port];
@@ -78,18 +77,13 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context)
{
struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
u8 port = attr->port_num - 1;
- unsigned long flags;
int ret;
if (port >= hr_dev->caps.num_ports)
return -EINVAL;
- spin_lock_irqsave(&hr_dev->iboe.lock, flags);
-
ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr);
- spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
-
return ret;
}
@@ -98,18 +92,13 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
struct ib_gid_attr zattr = { };
u8 port = attr->port_num - 1;
- unsigned long flags;
int ret;
if (port >= hr_dev->caps.num_ports)
return -EINVAL;
- spin_lock_irqsave(&hr_dev->iboe.lock, flags);
-
ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr);
- spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
-
return ret;
}
@@ -272,7 +261,8 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
IB_PORT_ACTIVE : IB_PORT_DOWN;
- props->phys_state = (props->state == IB_PORT_ACTIVE) ? 5 : 3;
+ props->phys_state = (props->state == IB_PORT_ACTIVE) ?
+ HNS_ROCE_PHY_LINKUP : HNS_ROCE_PHY_DISABLED;
spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
@@ -423,6 +413,11 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
}
static const struct ib_device_ops hns_roce_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_HNS,
+ .uverbs_abi_ver = 1,
+ .uverbs_no_driver_id_binding = 1,
+
.add_gid = hns_roce_add_gid,
.alloc_pd = hns_roce_alloc_pd,
.alloc_ucontext = hns_roce_alloc_ucontext,
@@ -451,6 +446,7 @@ static const struct ib_device_ops hns_roce_dev_ops = {
.reg_user_mr = hns_roce_reg_user_mr,
INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext),
};
@@ -489,14 +485,12 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
ib_dev = &hr_dev->ib_dev;
- ib_dev->owner = THIS_MODULE;
ib_dev->node_type = RDMA_NODE_IB_CA;
ib_dev->dev.parent = dev;
ib_dev->phys_port_cnt = hr_dev->caps.num_ports;
ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey;
ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors;
- ib_dev->uverbs_abi_ver = 1;
ib_dev->uverbs_cmd_mask =
(1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -545,7 +539,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops);
}
- ib_dev->driver_id = RDMA_DRIVER_HNS;
ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
for (i = 0; i < hr_dev->caps.num_ports; i++) {
@@ -980,7 +973,6 @@ error_failed_cmq_init:
return ret;
}
-EXPORT_SYMBOL_GPL(hns_roce_init);
void hns_roce_exit(struct hns_roce_dev *hr_dev)
{
@@ -1001,7 +993,6 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev)
if (hr_dev->hw->reset)
hr_dev->hw->reset(hr_dev, false);
}
-EXPORT_SYMBOL_GPL(hns_roce_exit);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 6110ec408626..549e1a38dfe0 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -47,7 +47,6 @@ unsigned long key_to_hw_index(u32 key)
{
return (key << 24) | (key >> 8);
}
-EXPORT_SYMBOL_GPL(key_to_hw_index);
static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox,
@@ -66,7 +65,6 @@ int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
HNS_ROCE_CMD_TIMEOUT_MSECS);
}
-EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt);
static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
unsigned long *seg)
@@ -293,7 +291,6 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
break;
}
}
-EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
struct hns_roce_mr *mr, int err_loop_index,
@@ -314,11 +311,11 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
mr->pbl_l1_dma_addr[i]);
- for (j = 0; j < pbl_bt_sz / 8; j++) {
+ for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
if (i == loop_i && j >= loop_j)
break;
- bt_idx = i * pbl_bt_sz / 8 + j;
+ bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
dma_free_coherent(dev, pbl_bt_sz,
mr->pbl_bt_l2[bt_idx],
mr->pbl_l2_dma_addr[bt_idx]);
@@ -329,8 +326,8 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
mr->pbl_l1_dma_addr[i]);
- for (j = 0; j < pbl_bt_sz / 8; j++) {
- bt_idx = i * pbl_bt_sz / 8 + j;
+ for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
+ bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
dma_free_coherent(dev, pbl_bt_sz,
mr->pbl_bt_l2[bt_idx],
mr->pbl_l2_dma_addr[bt_idx]);
@@ -533,7 +530,7 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
{
struct device *dev = hr_dev->dev;
unsigned long index = 0;
- int ret = 0;
+ int ret;
/* Allocate a key for mr from mr_table */
ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
@@ -559,7 +556,8 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
mr->pbl_l0_dma_addr = 0;
} else {
if (!hr_dev->caps.pbl_hop_num) {
- mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
+ mr->pbl_buf = dma_alloc_coherent(dev,
+ npages * BA_BYTE_LEN,
&(mr->pbl_dma_addr),
GFP_KERNEL);
if (!mr->pbl_buf)
@@ -590,9 +588,8 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
if (mhop_num == HNS_ROCE_HOP_NUM_0)
return;
- /* hop_num = 1 */
if (mhop_num == 1) {
- dma_free_coherent(dev, (unsigned int)(npages * 8),
+ dma_free_coherent(dev, (unsigned int)(npages * BA_BYTE_LEN),
mr->pbl_buf, mr->pbl_dma_addr);
return;
}
@@ -603,12 +600,13 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
if (mhop_num == 2) {
for (i = 0; i < mr->l0_chunk_last_num; i++) {
if (i == mr->l0_chunk_last_num - 1) {
- npages_allocated = i * (pbl_bt_sz / 8);
+ npages_allocated =
+ i * (pbl_bt_sz / BA_BYTE_LEN);
dma_free_coherent(dev,
- (npages - npages_allocated) * 8,
- mr->pbl_bt_l1[i],
- mr->pbl_l1_dma_addr[i]);
+ (npages - npages_allocated) * BA_BYTE_LEN,
+ mr->pbl_bt_l1[i],
+ mr->pbl_l1_dma_addr[i]);
break;
}
@@ -621,16 +619,17 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
mr->pbl_l1_dma_addr[i]);
- for (j = 0; j < pbl_bt_sz / 8; j++) {
- bt_idx = i * (pbl_bt_sz / 8) + j;
+ for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
+ bt_idx = i * (pbl_bt_sz / BA_BYTE_LEN) + j;
if ((i == mr->l0_chunk_last_num - 1)
&& j == mr->l1_chunk_last_num - 1) {
npages_allocated = bt_idx *
- (pbl_bt_sz / 8);
+ (pbl_bt_sz / BA_BYTE_LEN);
dma_free_coherent(dev,
- (npages - npages_allocated) * 8,
+ (npages - npages_allocated) *
+ BA_BYTE_LEN,
mr->pbl_bt_l2[bt_idx],
mr->pbl_l2_dma_addr[bt_idx]);
@@ -675,7 +674,8 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
npages = ib_umem_page_count(mr->umem);
if (!hr_dev->caps.pbl_hop_num)
- dma_free_coherent(dev, (unsigned int)(npages * 8),
+ dma_free_coherent(dev,
+ (unsigned int)(npages * BA_BYTE_LEN),
mr->pbl_buf, mr->pbl_dma_addr);
else
hns_roce_mhop_free(hr_dev, mr);
@@ -1059,6 +1059,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
page_addr = sg_page_iter_dma_address(&sg_iter);
if (!hr_dev->caps.pbl_hop_num) {
+ /* for hip06, page addr is aligned to 4K */
mr->pbl_buf[i++] = page_addr >> 12;
} else if (hr_dev->caps.pbl_hop_num == 1) {
mr->pbl_buf[i++] = page_addr;
@@ -1069,7 +1070,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
mr->pbl_bt_l2[i][j] = page_addr;
j++;
- if (j >= (pbl_bt_sz / 8)) {
+ if (j >= (pbl_bt_sz / BA_BYTE_LEN)) {
i++;
j = 0;
}
@@ -1117,7 +1118,8 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
} else {
u64 pbl_size = 1;
- bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8;
+ bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) /
+ BA_BYTE_LEN;
for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
pbl_size *= bt_size;
if (n > pbl_size) {
@@ -1293,9 +1295,7 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
} else {
hns_roce_mr_free(hr_dev, mr);
- if (mr->umem)
- ib_umem_release(mr->umem);
-
+ ib_umem_release(mr->umem);
kfree(mr);
}
@@ -1491,3 +1491,119 @@ int hns_roce_dealloc_mw(struct ib_mw *ibmw)
return 0;
}
+
+void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift,
+ int buf_pg_shift)
+{
+ hns_roce_hem_list_init(&mtr->hem_list, bt_pg_shift);
+ mtr->buf_pg_shift = buf_pg_shift;
+}
+
+void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev,
+ struct hns_roce_mtr *mtr)
+{
+ hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
+}
+
+static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev,
+ struct hns_roce_mtr *mtr, dma_addr_t *bufs,
+ struct hns_roce_buf_region *r)
+{
+ int offset;
+ int count;
+ int npage;
+ u64 *mtts;
+ int end;
+ int i;
+
+ offset = r->offset;
+ end = offset + r->count;
+ npage = 0;
+ while (offset < end) {
+ mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
+ offset, &count, NULL);
+ if (!mtts)
+ return -ENOBUFS;
+
+ /* Save page addr, low 12 bits : 0 */
+ for (i = 0; i < count; i++) {
+ if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
+ mtts[i] = cpu_to_le64(bufs[npage] >>
+ PAGE_ADDR_SHIFT);
+ else
+ mtts[i] = cpu_to_le64(bufs[npage]);
+
+ npage++;
+ }
+ offset += count;
+ }
+
+ return 0;
+}
+
+int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+ dma_addr_t **bufs, struct hns_roce_buf_region *regions,
+ int region_cnt)
+{
+ struct hns_roce_buf_region *r;
+ int ret;
+ int i;
+
+ ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions,
+ region_cnt);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < region_cnt; i++) {
+ r = &regions[i];
+ ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r);
+ if (ret) {
+ dev_err(hr_dev->dev,
+ "write mtr[%d/%d] err %d,offset=%d.\n",
+ i, region_cnt, ret, r->offset);
+ goto err_write;
+ }
+ }
+
+ return 0;
+
+err_write:
+ hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
+
+ return ret;
+}
+
+int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+ int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
+{
+ u64 *mtts = mtt_buf;
+ int mtt_count;
+ int total = 0;
+ u64 *addr;
+ int npage;
+ int left;
+
+ if (mtts == NULL || mtt_max < 1)
+ goto done;
+
+ left = mtt_max;
+ while (left > 0) {
+ mtt_count = 0;
+ addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
+ offset + total,
+ &mtt_count, NULL);
+ if (!addr || !mtt_count)
+ goto done;
+
+ npage = min(mtt_count, left);
+ memcpy(&mtts[total], addr, BA_BYTE_LEN * npage);
+ left -= npage;
+ total += npage;
+ }
+
+done:
+ if (base_addr)
+ *base_addr = mtr->hem_list.root_ba;
+
+ return total;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
index 813401384d78..920ca76b5db1 100644
--- a/drivers/infiniband/hw/hns/hns_roce_pd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -83,13 +83,11 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
return 0;
}
-EXPORT_SYMBOL_GPL(hns_roce_alloc_pd);
void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
{
hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn);
}
-EXPORT_SYMBOL_GPL(hns_roce_dealloc_pd);
int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
{
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 8db2817a249e..4f693cded74e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -64,7 +64,6 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
if (atomic_dec_and_test(&qp->refcount))
complete(&qp->free);
}
-EXPORT_SYMBOL_GPL(hns_roce_qp_event);
static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
enum hns_roce_event type)
@@ -139,7 +138,6 @@ enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)
return HNS_ROCE_QP_NUM_STATE;
}
}
-EXPORT_SYMBOL_GPL(to_hns_roce_state);
static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
struct hns_roce_qp *hr_qp)
@@ -242,7 +240,6 @@ void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
__xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1));
xa_unlock_irqrestore(xa, flags);
}
-EXPORT_SYMBOL_GPL(hns_roce_qp_remove);
void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
{
@@ -260,7 +257,6 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
}
}
-EXPORT_SYMBOL_GPL(hns_roce_qp_free);
void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
int cnt)
@@ -272,7 +268,6 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR);
}
-EXPORT_SYMBOL_GPL(hns_roce_release_range_qp);
static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
struct ib_qp_cap *cap, bool is_user, int has_rq,
@@ -422,6 +417,91 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev,
return 0;
}
+static int split_wqe_buf_region(struct hns_roce_dev *hr_dev,
+ struct hns_roce_qp *hr_qp,
+ struct hns_roce_buf_region *regions,
+ int region_max, int page_shift)
+{
+ int page_size = 1 << page_shift;
+ bool is_extend_sge;
+ int region_cnt = 0;
+ int buf_size;
+ int buf_cnt;
+
+ if (hr_qp->buff_size < 1 || region_max < 1)
+ return region_cnt;
+
+ if (hr_qp->sge.sge_cnt > 0)
+ is_extend_sge = true;
+ else
+ is_extend_sge = false;
+
+ /* sq region */
+ if (is_extend_sge)
+ buf_size = hr_qp->sge.offset - hr_qp->sq.offset;
+ else
+ buf_size = hr_qp->rq.offset - hr_qp->sq.offset;
+
+ if (buf_size > 0 && region_cnt < region_max) {
+ buf_cnt = DIV_ROUND_UP(buf_size, page_size);
+ hns_roce_init_buf_region(&regions[region_cnt],
+ hr_dev->caps.wqe_sq_hop_num,
+ hr_qp->sq.offset / page_size,
+ buf_cnt);
+ region_cnt++;
+ }
+
+ /* sge region */
+ if (is_extend_sge) {
+ buf_size = hr_qp->rq.offset - hr_qp->sge.offset;
+ if (buf_size > 0 && region_cnt < region_max) {
+ buf_cnt = DIV_ROUND_UP(buf_size, page_size);
+ hns_roce_init_buf_region(&regions[region_cnt],
+ hr_dev->caps.wqe_sge_hop_num,
+ hr_qp->sge.offset / page_size,
+ buf_cnt);
+ region_cnt++;
+ }
+ }
+
+ /* rq region */
+ buf_size = hr_qp->buff_size - hr_qp->rq.offset;
+ if (buf_size > 0) {
+ buf_cnt = DIV_ROUND_UP(buf_size, page_size);
+ hns_roce_init_buf_region(&regions[region_cnt],
+ hr_dev->caps.wqe_rq_hop_num,
+ hr_qp->rq.offset / page_size,
+ buf_cnt);
+ region_cnt++;
+ }
+
+ return region_cnt;
+}
+
+static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev,
+ struct hns_roce_buf_region *regions,
+ int region_cnt)
+{
+ int bt_pg_shift;
+ int ba_num;
+ int ret;
+
+ bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz;
+
+ /* all root ba entries must in one bt page */
+ do {
+ ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN;
+ ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt,
+ ba_num);
+ if (ret <= ba_num)
+ break;
+
+ bt_pg_shift++;
+ } while (ret > ba_num);
+
+ return bt_pg_shift - PAGE_SHIFT;
+}
+
static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
struct ib_qp_cap *cap,
struct hns_roce_qp *hr_qp)
@@ -534,15 +614,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
struct ib_udata *udata, unsigned long sqpn,
struct hns_roce_qp *hr_qp)
{
+ dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { 0 };
struct device *dev = hr_dev->dev;
struct hns_roce_ib_create_qp ucmd;
struct hns_roce_ib_create_qp_resp resp = {};
struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
udata, struct hns_roce_ucontext, ibucontext);
+ struct hns_roce_buf_region *r;
unsigned long qpn = 0;
- int ret = 0;
u32 page_shift;
- u32 npages;
+ int buf_count;
+ int ret;
int i;
mutex_init(&hr_qp->mutex);
@@ -596,6 +678,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
init_attr->cap.max_recv_sge];
}
+ page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
if (udata) {
if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
dev_err(dev, "ib_copy_from_udata error for create qp\n");
@@ -617,32 +700,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
ret = PTR_ERR(hr_qp->umem);
goto err_rq_sge_list;
}
-
- hr_qp->mtt.mtt_type = MTT_TYPE_WQE;
- page_shift = PAGE_SHIFT;
- if (hr_dev->caps.mtt_buf_pg_sz) {
- npages = (ib_umem_page_count(hr_qp->umem) +
- (1 << hr_dev->caps.mtt_buf_pg_sz) - 1) /
- (1 << hr_dev->caps.mtt_buf_pg_sz);
- page_shift += hr_dev->caps.mtt_buf_pg_sz;
- ret = hns_roce_mtt_init(hr_dev, npages,
- page_shift,
- &hr_qp->mtt);
- } else {
- ret = hns_roce_mtt_init(hr_dev,
- ib_umem_page_count(hr_qp->umem),
- page_shift, &hr_qp->mtt);
- }
+ hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
+ hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
+ page_shift);
+ ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
+ hr_qp->region_cnt);
if (ret) {
- dev_err(dev, "hns_roce_mtt_init error for create qp\n");
- goto err_buf;
+ dev_err(dev, "alloc buf_list error for create qp\n");
+ goto err_alloc_list;
}
- ret = hns_roce_ib_umem_write_mtt(hr_dev, &hr_qp->mtt,
- hr_qp->umem);
- if (ret) {
- dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n");
- goto err_mtt;
+ for (i = 0; i < hr_qp->region_cnt; i++) {
+ r = &hr_qp->regions[i];
+ buf_count = hns_roce_get_umem_bufs(hr_dev,
+ buf_list[i], r->count, r->offset,
+ hr_qp->umem, page_shift);
+ if (buf_count != r->count) {
+ dev_err(dev,
+ "get umem buf err, expect %d,ret %d.\n",
+ r->count, buf_count);
+ ret = -ENOBUFS;
+ goto err_get_bufs;
+ }
}
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
@@ -653,7 +732,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
&hr_qp->sdb);
if (ret) {
dev_err(dev, "sq record doorbell map failed!\n");
- goto err_mtt;
+ goto err_get_bufs;
}
/* indicate kernel supports sq record db */
@@ -715,7 +794,6 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
}
/* Allocate QP buf */
- page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size,
(1 << page_shift) * 2,
&hr_qp->hr_buf, page_shift)) {
@@ -723,21 +801,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
ret = -ENOMEM;
goto err_db;
}
-
- hr_qp->mtt.mtt_type = MTT_TYPE_WQE;
- /* Write MTT */
- ret = hns_roce_mtt_init(hr_dev, hr_qp->hr_buf.npages,
- hr_qp->hr_buf.page_shift, &hr_qp->mtt);
+ hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
+ hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
+ page_shift);
+ ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
+ hr_qp->region_cnt);
if (ret) {
- dev_err(dev, "hns_roce_mtt_init error for kernel create qp\n");
- goto err_buf;
+ dev_err(dev, "alloc buf_list error for create qp!\n");
+ goto err_alloc_list;
}
- ret = hns_roce_buf_write_mtt(hr_dev, &hr_qp->mtt,
- &hr_qp->hr_buf);
- if (ret) {
- dev_err(dev, "hns_roce_buf_write_mtt error for kernel create qp\n");
- goto err_mtt;
+ for (i = 0; i < hr_qp->region_cnt; i++) {
+ r = &hr_qp->regions[i];
+ buf_count = hns_roce_get_kmem_bufs(hr_dev,
+ buf_list[i], r->count, r->offset,
+ &hr_qp->hr_buf);
+ if (buf_count != r->count) {
+ dev_err(dev,
+ "get kmem buf err, expect %d,ret %d.\n",
+ r->count, buf_count);
+ ret = -ENOBUFS;
+ goto err_get_bufs;
+ }
}
hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64),
@@ -761,6 +846,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
}
}
+ hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions,
+ hr_qp->region_cnt);
+ hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift,
+ page_shift);
+ ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list,
+ hr_qp->regions, hr_qp->region_cnt);
+ if (ret) {
+ dev_err(dev, "mtr attach error for create qp\n");
+ goto err_mtr;
+ }
+
if (init_attr->qp_type == IB_QPT_GSI &&
hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
/* In v1 engine, GSI QP context in RoCE engine's register */
@@ -796,6 +892,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
}
hr_qp->event = hns_roce_ib_qp_event;
+ hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
return 0;
@@ -810,6 +907,9 @@ err_qpn:
if (!sqpn)
hns_roce_release_range_qp(hr_dev, qpn, 1);
+err_mtr:
+ hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
+
err_wrid:
if (udata) {
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
@@ -829,14 +929,13 @@ err_sq_dbmap:
hns_roce_qp_has_sq(init_attr))
hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
-err_mtt:
- hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
+err_get_bufs:
+ hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
-err_buf:
- if (hr_qp->umem)
- ib_umem_release(hr_qp->umem);
- else
+err_alloc_list:
+ if (!hr_qp->umem)
hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
+ ib_umem_release(hr_qp->umem);
err_db:
if (!udata && hns_roce_qp_has_rq(init_attr) &&
@@ -923,7 +1022,6 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
return &hr_qp->ibqp;
}
-EXPORT_SYMBOL_GPL(hns_roce_create_qp);
int to_hr_qp_type(int qp_type)
{
@@ -942,7 +1040,6 @@ int to_hr_qp_type(int qp_type)
return transport_type;
}
-EXPORT_SYMBOL_GPL(to_hr_qp_type);
int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
@@ -1062,7 +1159,6 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
}
}
-EXPORT_SYMBOL_GPL(hns_roce_lock_cqs);
void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
@@ -1079,7 +1175,6 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
spin_unlock_irq(&recv_cq->lock);
}
}
-EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs);
static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
{
@@ -1091,20 +1186,17 @@ void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
{
return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
}
-EXPORT_SYMBOL_GPL(get_recv_wqe);
void *get_send_wqe(struct hns_roce_qp *hr_qp, int n)
{
return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
}
-EXPORT_SYMBOL_GPL(get_send_wqe);
void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n)
{
return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset +
(n << hr_qp->sge.sge_shift));
}
-EXPORT_SYMBOL_GPL(get_send_extend_sge);
bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
struct ib_cq *ib_cq)
@@ -1123,7 +1215,6 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
return cur + nreq >= hr_wq->max_post;
}
-EXPORT_SYMBOL_GPL(hns_roce_wq_overflow);
int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
{
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index b3421b1f21e0..38bb548eaa6d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -30,7 +30,6 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
if (atomic_dec_and_test(&srq->refcount))
complete(&srq->free);
}
-EXPORT_SYMBOL_GPL(hns_roce_srq_event);
static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
enum hns_roce_event event_type)
@@ -181,28 +180,19 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
{
struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
struct hns_roce_idx_que *idx_que = &srq->idx_que;
- u32 bitmap_num;
- int i;
- bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64));
-
- idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL);
+ idx_que->bitmap = bitmap_zalloc(srq->max, GFP_KERNEL);
if (!idx_que->bitmap)
return -ENOMEM;
- bitmap_num = bitmap_num / (8 * sizeof(u64));
-
idx_que->buf_size = srq->idx_que.buf_size;
if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2,
&idx_que->idx_buf, page_shift)) {
- kfree(idx_que->bitmap);
+ bitmap_free(idx_que->bitmap);
return -ENOMEM;
}
- for (i = 0; i < bitmap_num; i++)
- idx_que->bitmap[i] = ~(0UL);
-
return 0;
}
@@ -264,8 +254,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
} else
ret = hns_roce_mtt_init(hr_dev,
ib_umem_page_count(srq->umem),
- srq->umem->page_shift,
- &srq->mtt);
+ PAGE_SHIFT, &srq->mtt);
if (ret)
goto err_buf;
@@ -291,10 +280,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
ret = hns_roce_mtt_init(hr_dev, npages,
page_shift, &srq->idx_que.mtt);
} else {
- ret = hns_roce_mtt_init(hr_dev,
- ib_umem_page_count(srq->idx_que.umem),
- srq->idx_que.umem->page_shift,
- &srq->idx_que.mtt);
+ ret = hns_roce_mtt_init(
+ hr_dev, ib_umem_page_count(srq->idx_que.umem),
+ PAGE_SHIFT, &srq->idx_que.mtt);
}
if (ret) {
@@ -391,21 +379,19 @@ err_idx_buf:
hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
err_idx_mtt:
- if (udata)
- ib_umem_release(srq->idx_que.umem);
+ ib_umem_release(srq->idx_que.umem);
err_create_idx:
hns_roce_buf_free(hr_dev, srq->idx_que.buf_size,
&srq->idx_que.idx_buf);
- kfree(srq->idx_que.bitmap);
+ bitmap_free(srq->idx_que.bitmap);
err_srq_mtt:
hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
err_buf:
- if (udata)
- ib_umem_release(srq->umem);
- else
+ ib_umem_release(srq->umem);
+ if (!udata)
hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
return ret;
@@ -419,15 +405,15 @@ void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
hns_roce_srq_free(hr_dev, srq);
hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
- if (ibsrq->uobject) {
+ if (udata) {
hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
- ib_umem_release(srq->idx_que.umem);
- ib_umem_release(srq->umem);
} else {
kvfree(srq->wrid);
hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift,
&srq->buf);
}
+ ib_umem_release(srq->idx_que.umem);
+ ib_umem_release(srq->umem);
}
int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 5689d742bafb..3c0c6aabc64e 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -1064,44 +1064,38 @@ void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
* @ib_cq: cq pointer
* @udata: user data or NULL for kernel object
*/
-static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
struct i40iw_cq *iwcq;
struct i40iw_device *iwdev;
struct i40iw_sc_cq *cq;
- if (!ib_cq) {
- i40iw_pr_err("ib_cq == NULL\n");
- return 0;
- }
-
iwcq = to_iwcq(ib_cq);
iwdev = to_iwdev(ib_cq->device);
cq = &iwcq->sc_cq;
i40iw_cq_wq_destroy(iwdev, cq);
cq_free_resources(iwdev, iwcq);
- kfree(iwcq);
i40iw_rem_devusecount(iwdev);
- return 0;
}
/**
* i40iw_create_cq - create cq
- * @ibdev: device pointer from stack
+ * @ibcq: CQ allocated
* @attr: attributes for cq
* @udata: user data
*/
-static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+static int i40iw_create_cq(struct ib_cq *ibcq,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
struct i40iw_device *iwdev = to_iwdev(ibdev);
- struct i40iw_cq *iwcq;
+ struct i40iw_cq *iwcq = to_iwcq(ibcq);
struct i40iw_pbl *iwpbl;
u32 cq_num = 0;
struct i40iw_sc_cq *cq;
struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_cq_init_info info;
+ struct i40iw_cq_init_info info = {};
enum i40iw_status_code status;
struct i40iw_cqp_request *cqp_request;
struct cqp_commands_info *cqp_info;
@@ -1111,22 +1105,16 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
int entries = attr->cqe;
if (iwdev->closing)
- return ERR_PTR(-ENODEV);
+ return -ENODEV;
if (entries > iwdev->max_cqe)
- return ERR_PTR(-EINVAL);
-
- iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
- if (!iwcq)
- return ERR_PTR(-ENOMEM);
-
- memset(&info, 0, sizeof(info));
+ return -EINVAL;
err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs,
iwdev->max_cq, &cq_num,
&iwdev->next_cq);
if (err_code)
- goto error;
+ return err_code;
cq = &iwcq->sc_cq;
cq->back_cq = (void *)iwcq;
@@ -1233,15 +1221,13 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
}
i40iw_add_devusecount(iwdev);
- return (struct ib_cq *)iwcq;
+ return 0;
cq_destroy:
i40iw_cq_wq_destroy(iwdev, cq);
cq_free_resources:
cq_free_resources(iwdev, iwcq);
-error:
- kfree(iwcq);
- return ERR_PTR(err_code);
+ return err_code;
}
/**
@@ -2018,8 +2004,7 @@ static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
struct cqp_commands_info *cqp_info;
u32 stag_idx;
- if (iwmr->region)
- ib_umem_release(iwmr->region);
+ ib_umem_release(iwmr->region);
if (iwmr->type != IW_MEMREG_TYPE_MEM) {
/* region is released. only test for userness. */
@@ -2655,6 +2640,11 @@ static int i40iw_query_pkey(struct ib_device *ibdev,
}
static const struct ib_device_ops i40iw_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_I40IW,
+ /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */
+ .uverbs_abi_ver = I40IW_ABI_VER,
+
.alloc_hw_stats = i40iw_alloc_hw_stats,
.alloc_mr = i40iw_alloc_mr,
.alloc_pd = i40iw_alloc_pd,
@@ -2694,6 +2684,7 @@ static const struct ib_device_ops i40iw_dev_ops = {
.reg_user_mr = i40iw_reg_user_mr,
.req_notify_cq = i40iw_req_notify_cq,
INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),
+ INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),
};
@@ -2712,7 +2703,6 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev
i40iw_pr_err("iwdev == NULL\n");
return NULL;
}
- iwibdev->ibdev.owner = THIS_MODULE;
iwdev->iwibdev = iwibdev;
iwibdev->iwdev = iwdev;
@@ -2771,9 +2761,6 @@ void i40iw_port_ibevent(struct i40iw_device *iwdev)
*/
void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
{
- if (!iwibdev)
- return;
-
ib_unregister_device(&iwibdev->ibdev);
wait_event_timeout(iwibdev->iwdev->close_wq,
!atomic64_read(&iwibdev->iwdev->use_count),
@@ -2795,7 +2782,6 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev)
return -ENOMEM;
iwibdev = iwdev->iwibdev;
rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group);
- iwibdev->ibdev.driver_id = RDMA_DRIVER_I40IW;
ret = ib_register_device(&iwibdev->ibdev, "i40iw%d");
if (ret)
goto error;
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 022a0b4ea452..a7d238d312f0 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -172,14 +172,14 @@ err_buf:
}
#define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
-struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx4_ib_dev *dev = to_mdev(ibdev);
- struct mlx4_ib_cq *cq;
+ struct mlx4_ib_cq *cq = to_mcq(ibcq);
struct mlx4_uar *uar;
void *buf_addr;
int err;
@@ -187,14 +187,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
udata, struct mlx4_ib_ucontext, ibucontext);
if (entries < 1 || entries > dev->dev->caps.max_cqes)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
- return ERR_PTR(-EINVAL);
-
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
+ return -EINVAL;
entries = roundup_pow_of_two(entries + 1);
cq->ibcq.cqe = entries - 1;
@@ -269,7 +265,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
goto err_cq_free;
}
- return &cq->ibcq;
+ return 0;
err_cq_free:
mlx4_cq_free(dev->dev, &cq->mcq);
@@ -281,19 +277,15 @@ err_dbmap:
err_mtt:
mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
- if (udata)
- ib_umem_release(cq->umem);
- else
+ ib_umem_release(cq->umem);
+ if (!udata)
mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
err_db:
if (!udata)
mlx4_db_free(dev->dev, &cq->db);
-
err_cq:
- kfree(cq);
-
- return ERR_PTR(err);
+ return err;
}
static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
@@ -475,18 +467,15 @@ err_buf:
kfree(cq->resize_buf);
cq->resize_buf = NULL;
- if (cq->resize_umem) {
- ib_umem_release(cq->resize_umem);
- cq->resize_umem = NULL;
- }
-
+ ib_umem_release(cq->resize_umem);
+ cq->resize_umem = NULL;
out:
mutex_unlock(&cq->resize_mutex);
return err;
}
-int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(cq->device);
struct mlx4_ib_cq *mcq = to_mcq(cq);
@@ -501,15 +490,11 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
struct mlx4_ib_ucontext,
ibucontext),
&mcq->db);
- ib_umem_release(mcq->umem);
} else {
mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
mlx4_db_free(dev->dev, &mcq->db);
}
-
- kfree(mcq);
-
- return 0;
+ ib_umem_release(mcq->umem);
}
static void dump_cqe(void *cqe)
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 25d09d53b51c..8790101facb7 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -1089,7 +1089,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
if (!dev->ib_active)
return -EAGAIN;
- if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
+ if (ibdev->ops.uverbs_abi_ver ==
+ MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
resp_v3.qp_tab_size = dev->dev->caps.num_qps;
resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size;
resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
@@ -1111,7 +1112,7 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
INIT_LIST_HEAD(&context->wqn_ranges_list);
mutex_init(&context->wqn_ranges_mutex);
- if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
+ if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
else
err = ib_copy_to_udata(udata, &resp, sizeof(resp));
@@ -2509,6 +2510,10 @@ static void get_fw_ver_str(struct ib_device *device, char *str)
}
static const struct ib_device_ops mlx4_ib_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_MLX4,
+ .uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION,
+
.add_gid = mlx4_ib_add_gid,
.alloc_mr = mlx4_ib_alloc_mr,
.alloc_pd = mlx4_ib_alloc_pd,
@@ -2560,6 +2565,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = {
.resize_cq = mlx4_ib_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, mlx4_ib_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext),
@@ -2642,7 +2648,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->dev = dev;
ibdev->bond_next_port = 0;
- ibdev->ib_dev.owner = THIS_MODULE;
ibdev->ib_dev.node_type = RDMA_NODE_IB_CA;
ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey;
ibdev->num_ports = num_ports;
@@ -2651,11 +2656,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
ibdev->ib_dev.dev.parent = &dev->persist->pdev->dev;
- if (dev->caps.userspace_caps)
- ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;
- else
- ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;
-
ibdev->ib_dev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -2729,6 +2729,10 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_fs_ops);
}
+ if (!dev->caps.userspace_caps)
+ ibdev->ib_dev.ops.uverbs_abi_ver =
+ MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;
+
mlx4_ib_alloc_eqs(dev, ibdev);
spin_lock_init(&iboe->lock);
@@ -2839,7 +2843,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
goto err_steer_free_bitmap;
rdma_set_device_sysfs_group(&ibdev->ib_dev, &mlx4_attr_group);
- ibdev->ib_dev.driver_id = RDMA_DRIVER_MLX4;
if (ib_register_device(&ibdev->ib_dev, "mlx4_%d"))
goto err_diag_counters;
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 26897102057d..eb53bb4c0c91 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -743,10 +743,9 @@ int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
-struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
@@ -907,7 +906,7 @@ void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port);
struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
struct ib_wq_init_attr *init_attr,
struct ib_udata *udata);
-int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
+void mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
u32 wq_attr_mask, struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 355205a28544..753479285ce9 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -258,7 +258,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va,
int *num_of_mtts)
{
u64 block_shift = MLX4_MAX_MTT_SHIFT;
- u64 min_shift = umem->page_shift;
+ u64 min_shift = PAGE_SHIFT;
u64 last_block_aligned_end = 0;
u64 current_block_start = 0;
u64 first_block_start = 0;
@@ -295,8 +295,8 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va,
* in access to the wrong data.
*/
misalignment_bits =
- (start_va & (~(((u64)(BIT(umem->page_shift))) - 1ULL)))
- ^ current_block_start;
+ (start_va & (~(((u64)(PAGE_SIZE)) - 1ULL))) ^
+ current_block_start;
block_shift = min(alignment_of(misalignment_bits),
block_shift);
}
@@ -368,8 +368,7 @@ end:
}
static struct ib_umem *mlx4_get_umem_mr(struct ib_udata *udata, u64 start,
- u64 length, u64 virt_addr,
- int access_flags)
+ u64 length, int access_flags)
{
/*
* Force registering the memory as writable if the underlying pages
@@ -415,8 +414,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
if (!mr)
return ERR_PTR(-ENOMEM);
- mr->umem =
- mlx4_get_umem_mr(udata, start, length, virt_addr, access_flags);
+ mr->umem = mlx4_get_umem_mr(udata, start, length, access_flags);
if (IS_ERR(mr->umem)) {
err = PTR_ERR(mr->umem);
goto err_free;
@@ -505,7 +503,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
ib_umem_release(mmr->umem);
- mmr->umem = mlx4_get_umem_mr(udata, start, length, virt_addr,
+ mmr->umem = mlx4_get_umem_mr(udata, start, length,
mr_access_flags);
if (IS_ERR(mmr->umem)) {
err = PTR_ERR(mmr->umem);
@@ -514,7 +512,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
goto release_mpt_entry;
}
n = ib_umem_page_count(mmr->umem);
- shift = mmr->umem->page_shift;
+ shift = PAGE_SHIFT;
err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr,
virt_addr, length, n, shift,
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 5221c0794d1d..82aff2f2fdc2 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1207,10 +1207,9 @@ err_mtt:
mlx4_mtt_cleanup(dev->dev, &qp->mtt);
err_buf:
- if (qp->umem)
- ib_umem_release(qp->umem);
- else
+ if (!qp->umem)
mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
+ ib_umem_release(qp->umem);
err_db:
if (!udata && qp_has_rq(init_attr))
@@ -1421,7 +1420,6 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
mlx4_ib_db_unmap_user(mcontext, &qp->db);
}
- ib_umem_release(qp->umem);
} else {
kvfree(qp->sq.wrid);
kvfree(qp->rq.wrid);
@@ -1432,6 +1430,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
if (qp->rq.wqe_cnt)
mlx4_db_free(dev->dev, &qp->db);
}
+ ib_umem_release(qp->umem);
del_gid_entries(qp);
}
@@ -4248,7 +4247,7 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
return err;
}
-int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
+void mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
@@ -4259,8 +4258,6 @@ int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);
kfree(qp);
-
- return 0;
}
struct ib_rwq_ind_table
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 4bf2946b9759..848db7264cc9 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -115,7 +115,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,
return PTR_ERR(srq->umem);
err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),
- srq->umem->page_shift, &srq->mtt);
+ PAGE_SHIFT, &srq->mtt);
if (err)
goto err_buf;
@@ -204,10 +204,9 @@ err_mtt:
mlx4_mtt_cleanup(dev->dev, &srq->mtt);
err_buf:
- if (srq->umem)
- ib_umem_release(srq->umem);
- else
+ if (!srq->umem)
mlx4_buf_free(dev->dev, buf_size, &srq->buf);
+ ib_umem_release(srq->umem);
err_db:
if (!udata)
@@ -275,13 +274,13 @@ void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
struct mlx4_ib_ucontext,
ibucontext),
&msrq->db);
- ib_umem_release(msrq->umem);
} else {
kvfree(msrq->wrid);
mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
&msrq->buf);
mlx4_db_free(dev->dev, &msrq->db);
}
+ ib_umem_release(msrq->umem);
}
void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 2e2e65f00257..22230fd7d741 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -884,14 +884,14 @@ static void notify_soft_wc_handler(struct work_struct *work)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
-struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx5_ib_dev *dev = to_mdev(ibdev);
- struct mlx5_ib_cq *cq;
+ struct mlx5_ib_cq *cq = to_mcq(ibcq);
int uninitialized_var(index);
int uninitialized_var(inlen);
u32 *cqb = NULL;
@@ -903,18 +903,14 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
if (entries < 0 ||
(entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (check_cq_create_flags(attr->flags))
- return ERR_PTR(-EOPNOTSUPP);
+ return -EOPNOTSUPP;
entries = roundup_pow_of_two(entries + 1);
if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
- return ERR_PTR(-EINVAL);
-
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
+ return -EINVAL;
cq->ibcq.cqe = entries - 1;
mutex_init(&cq->resize_mutex);
@@ -929,13 +925,13 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
&index, &inlen);
if (err)
- goto err_create;
+ return err;
} else {
cqe_size = cache_line_size() == 128 ? 128 : 64;
err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
&index, &inlen);
if (err)
- goto err_create;
+ return err;
INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
}
@@ -980,7 +976,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
kvfree(cqb);
- return &cq->ibcq;
+ return 0;
err_cmd:
mlx5_core_destroy_cq(dev->mdev, &cq->mcq);
@@ -991,14 +987,10 @@ err_cqb:
destroy_cq_user(cq, udata);
else
destroy_cq_kernel(dev, cq);
-
-err_create:
- kfree(cq);
-
- return ERR_PTR(err);
+ return err;
}
-int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(cq->device);
struct mlx5_ib_cq *mcq = to_mcq(cq);
@@ -1008,10 +1000,6 @@ int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
destroy_cq_user(mcq, udata);
else
destroy_cq_kernel(dev, mcq);
-
- kfree(mcq);
-
- return 0;
}
static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)
@@ -1137,11 +1125,6 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
return 0;
}
-static void un_resize_user(struct mlx5_ib_cq *cq)
-{
- ib_umem_release(cq->resize_umem);
-}
-
static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
int entries, int cqe_size)
{
@@ -1164,12 +1147,6 @@ ex:
return err;
}
-static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
-{
- free_cq_buf(dev, cq->resize_buf);
- cq->resize_buf = NULL;
-}
-
static int copy_resize_cqes(struct mlx5_ib_cq *cq)
{
struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
@@ -1350,10 +1327,11 @@ ex_alloc:
kvfree(in);
ex_resize:
- if (udata)
- un_resize_user(cq);
- else
- un_resize_kernel(dev, cq);
+ ib_umem_release(cq->resize_umem);
+ if (!udata) {
+ free_cq_buf(dev, cq->resize_buf);
+ cq->resize_buf = NULL;
+ }
ex:
mutex_unlock(&cq->resize_mutex);
return err;
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c
index 269b24a3baa1..aa9acebfcc23 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.c
+++ b/drivers/infiniband/hw/mlx5/ib_rep.c
@@ -60,7 +60,7 @@ mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
if (!__mlx5_ib_add(ibdev, profile))
return -EINVAL;
- rep->rep_if[REP_IB].priv = ibdev;
+ rep->rep_data[REP_IB].priv = ibdev;
return 0;
}
@@ -70,13 +70,13 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
{
struct mlx5_ib_dev *dev;
- if (!rep->rep_if[REP_IB].priv ||
+ if (!rep->rep_data[REP_IB].priv ||
rep->vport != MLX5_VPORT_UPLINK)
return;
dev = mlx5_ib_rep_to_dev(rep);
__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
- rep->rep_if[REP_IB].priv = NULL;
+ rep->rep_data[REP_IB].priv = NULL;
}
static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep)
@@ -84,16 +84,17 @@ static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep)
return mlx5_ib_rep_to_dev(rep);
}
+static const struct mlx5_eswitch_rep_ops rep_ops = {
+ .load = mlx5_ib_vport_rep_load,
+ .unload = mlx5_ib_vport_rep_unload,
+ .get_proto_dev = mlx5_ib_vport_get_proto_dev,
+};
+
void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev)
{
struct mlx5_eswitch *esw = mdev->priv.eswitch;
- struct mlx5_eswitch_rep_if rep_if = {};
-
- rep_if.load = mlx5_ib_vport_rep_load;
- rep_if.unload = mlx5_ib_vport_rep_unload;
- rep_if.get_proto_dev = mlx5_ib_vport_get_proto_dev;
- mlx5_eswitch_register_vport_reps(esw, &rep_if, REP_IB);
+ mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_IB);
}
void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev)
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h
index 8336e0517a5c..7a917e6d5c09 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.h
+++ b/drivers/infiniband/hw/mlx5/ib_rep.h
@@ -72,6 +72,6 @@ struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,
static inline
struct mlx5_ib_dev *mlx5_ib_rep_to_dev(struct mlx5_eswitch_rep *rep)
{
- return (struct mlx5_ib_dev *)rep->rep_if[REP_IB].priv;
+ return rep->rep_data[REP_IB].priv;
}
#endif /* __MLX5_IB_REP_H__ */
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 340290b883fe..05d2bfcb3d60 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -52,6 +52,7 @@
#include <linux/mlx5/port.h>
#include <linux/mlx5/vport.h>
#include <linux/mlx5/fs.h>
+#include <linux/mlx5/eswitch.h>
#include <linux/list.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
@@ -888,7 +889,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
if (MLX5_CAP_GEN(mdev, sho)) {
- props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
+ props->device_cap_flags |= IB_DEVICE_INTEGRITY_HANDOVER;
/* At this stage no support for signature handover */
props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
IB_PROT_T10DIF_TYPE_2 |
@@ -1008,6 +1009,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
props->max_srq_sge = max_rq_sg - 1;
props->max_fast_reg_page_list_len =
1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
+ props->max_pi_fast_reg_page_list_len =
+ props->max_fast_reg_page_list_len / 2;
get_atomic_caps_qp(dev, props);
props->masked_atomic_cap = IB_ATOMIC_NONE;
props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
@@ -3252,11 +3255,14 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
int max_table_size;
int num_entries;
int num_groups;
+ bool esw_encap;
u32 flags = 0;
int priority;
max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
log_max_ft_size));
+ esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
enum mlx5_flow_namespace_type fn_type;
@@ -3269,10 +3275,10 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
if (ft_type == MLX5_IB_FT_RX) {
fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
prio = &dev->flow_db->prios[priority];
- if (!dev->is_rep &&
+ if (!dev->is_rep && !esw_encap &&
MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
- if (!dev->is_rep &&
+ if (!dev->is_rep && !esw_encap &&
MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
reformat_l3_tunnel_to_l2))
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
@@ -3282,7 +3288,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
log_max_ft_size));
fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
prio = &dev->flow_db->egress_prios[priority];
- if (!dev->is_rep &&
+ if (!dev->is_rep && !esw_encap &&
MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
}
@@ -3892,6 +3898,7 @@ _get_flow_table(struct mlx5_ib_dev *dev,
struct mlx5_flow_namespace *ns = NULL;
struct mlx5_ib_flow_prio *prio = NULL;
int max_table_size = 0;
+ bool esw_encap;
u32 flags = 0;
int priority;
@@ -3900,22 +3907,30 @@ _get_flow_table(struct mlx5_ib_dev *dev,
else
priority = ib_prio_to_core_prio(fs_matcher->priority, false);
+ esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
log_max_ft_size));
- if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
+ if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- reformat_l3_tunnel_to_l2))
+ reformat_l3_tunnel_to_l2) &&
+ !esw_encap)
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
max_table_size = BIT(
MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
- if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
max_table_size = BIT(
MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
+ esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
priority = FDB_BYPASS_PATH;
}
@@ -4891,18 +4906,19 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
if (ret)
goto error0;
- devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL);
- if (IS_ERR(devr->c0)) {
- ret = PTR_ERR(devr->c0);
+ devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
+ if (!devr->c0) {
+ ret = -ENOMEM;
goto error1;
}
- devr->c0->device = &dev->ib_dev;
- devr->c0->uobject = NULL;
- devr->c0->comp_handler = NULL;
- devr->c0->event_handler = NULL;
- devr->c0->cq_context = NULL;
+
+ devr->c0->device = &dev->ib_dev;
atomic_set(&devr->c0->usecnt, 0);
+ ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
+ if (ret)
+ goto err_create_cq;
+
devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
if (IS_ERR(devr->x0)) {
ret = PTR_ERR(devr->x0);
@@ -4994,6 +5010,8 @@ error3:
mlx5_ib_dealloc_xrcd(devr->x0, NULL);
error2:
mlx5_ib_destroy_cq(devr->c0, NULL);
+err_create_cq:
+ kfree(devr->c0);
error1:
mlx5_ib_dealloc_pd(devr->p0, NULL);
error0:
@@ -5012,6 +5030,7 @@ static void destroy_dev_resources(struct mlx5_ib_resources *devr)
mlx5_ib_dealloc_xrcd(devr->x0, NULL);
mlx5_ib_dealloc_xrcd(devr->x1, NULL);
mlx5_ib_destroy_cq(devr->c0, NULL);
+ kfree(devr->c0);
mlx5_ib_dealloc_pd(devr->p0, NULL);
kfree(devr->p0);
@@ -6044,7 +6063,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
if (mlx5_use_mad_ifc(dev))
get_ext_port_caps(dev);
- dev->ib_dev.owner = THIS_MODULE;
dev->ib_dev.node_type = RDMA_NODE_IB_CA;
dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
dev->ib_dev.phys_port_cnt = dev->num_ports;
@@ -6124,8 +6142,13 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
}
static const struct ib_device_ops mlx5_ib_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_MLX5,
+ .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
+
.add_gid = mlx5_ib_add_gid,
.alloc_mr = mlx5_ib_alloc_mr,
+ .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
.alloc_pd = mlx5_ib_alloc_pd,
.alloc_ucontext = mlx5_ib_alloc_ucontext,
.attach_mcast = mlx5_ib_mcg_attach,
@@ -6155,6 +6178,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
.get_dma_mr = mlx5_ib_get_dma_mr,
.get_link_layer = mlx5_ib_port_link_layer,
.map_mr_sg = mlx5_ib_map_mr_sg,
+ .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
.mmap = mlx5_ib_mmap,
.modify_cq = mlx5_ib_modify_cq,
.modify_device = mlx5_ib_modify_device,
@@ -6179,6 +6203,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
.resize_cq = mlx5_ib_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
@@ -6221,7 +6246,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
struct mlx5_core_dev *mdev = dev->mdev;
int err;
- dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
dev->ib_dev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -6290,7 +6314,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
if (mlx5_accel_ipsec_device_caps(dev->mdev) &
MLX5_ACCEL_IPSEC_CAP_DEVICE)
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops);
- dev->ib_dev.driver_id = RDMA_DRIVER_MLX5;
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index 9f90be296ee0..fe1a76d8531c 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -55,9 +55,10 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
int i = 0;
struct scatterlist *sg;
int entry;
- unsigned long page_shift = umem->page_shift;
if (umem->is_odp) {
+ unsigned int page_shift = to_ib_umem_odp(umem)->page_shift;
+
*ncont = ib_umem_page_count(umem);
*count = *ncont << (page_shift - PAGE_SHIFT);
*shift = page_shift;
@@ -67,15 +68,15 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
return;
}
- addr = addr >> page_shift;
+ addr = addr >> PAGE_SHIFT;
tmp = (unsigned long)addr;
m = find_first_bit(&tmp, BITS_PER_LONG);
if (max_page_shift)
- m = min_t(unsigned long, max_page_shift - page_shift, m);
+ m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
- len = sg_dma_len(sg) >> page_shift;
- pfn = sg_dma_address(sg) >> page_shift;
+ len = sg_dma_len(sg) >> PAGE_SHIFT;
+ pfn = sg_dma_address(sg) >> PAGE_SHIFT;
if (base + p != pfn) {
/* If either the offset or the new
* base are unaligned update m
@@ -107,7 +108,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
*ncont = 0;
}
- *shift = page_shift + m;
+ *shift = PAGE_SHIFT + m;
*count = i;
}
@@ -140,8 +141,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
int page_shift, size_t offset, size_t num_pages,
__be64 *pas, int access_flags)
{
- unsigned long umem_page_shift = umem->page_shift;
- int shift = page_shift - umem_page_shift;
+ int shift = page_shift - PAGE_SHIFT;
int mask = (1 << shift) - 1;
int i, k, idx;
u64 cur = 0;
@@ -165,7 +165,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
i = 0;
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
- len = sg_dma_len(sg) >> umem_page_shift;
+ len = sg_dma_len(sg) >> PAGE_SHIFT;
base = sg_dma_address(sg);
/* Skip elements below offset */
@@ -184,7 +184,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
for (; k < len; k++) {
if (!(i & mask)) {
- cur = base + (k << umem_page_shift);
+ cur = base + (k << PAGE_SHIFT);
cur |= access_flags;
idx = (i >> shift) - offset;
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 40eb8be482e4..bdb83fc85f94 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -431,9 +431,6 @@ struct mlx5_ib_qp {
int create_type;
- /* Store signature errors */
- bool signature_en;
-
struct list_head qps_list;
struct list_head cq_recv_list;
struct list_head cq_send_list;
@@ -587,6 +584,9 @@ struct mlx5_ib_mr {
void *descs;
dma_addr_t desc_map;
int ndescs;
+ int data_length;
+ int meta_ndescs;
+ int meta_length;
int max_descs;
int desc_size;
int access_mode;
@@ -605,6 +605,13 @@ struct mlx5_ib_mr {
int access_flags; /* Needed for rereg MR */
struct mlx5_ib_mr *parent;
+ /* Needed for IB_MR_TYPE_INTEGRITY */
+ struct mlx5_ib_mr *pi_mr;
+ struct mlx5_ib_mr *klm_mr;
+ struct mlx5_ib_mr *mtt_mr;
+ u64 data_iova;
+ u64 pi_iova;
+
atomic_t num_leaf_free;
wait_queue_head_t q_leaf_free;
struct mlx5_async_work cb_work;
@@ -920,6 +927,7 @@ struct mlx5_ib_lb_state {
};
struct mlx5_ib_pf_eq {
+ struct notifier_block irq_nb;
struct mlx5_ib_dev *dev;
struct mlx5_eq *core;
struct work_struct work;
@@ -1115,10 +1123,9 @@ int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
int buflen, size_t *bc);
int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
void *buffer, int buflen, size_t *bc);
-struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
@@ -1148,8 +1155,15 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_sg,
+ u32 max_num_meta_sg);
int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset);
int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const struct ib_mad_hdr *in, size_t in_mad_size,
@@ -1201,7 +1215,7 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
struct ib_wq_init_attr *init_attr,
struct ib_udata *udata);
-int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
+void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
u32 wq_attr_mask, struct ib_udata *udata);
struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 5f09699fab98..6ac77e09a34a 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1507,10 +1507,9 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
return 0;
err:
- if (mr->umem) {
- ib_umem_release(mr->umem);
- mr->umem = NULL;
- }
+ ib_umem_release(mr->umem);
+ mr->umem = NULL;
+
clean_mr(dev, mr);
return err;
}
@@ -1606,8 +1605,9 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
synchronize_srcu(&dev->mr_srcu);
/* Destroy all page mappings */
if (umem_odp->page_list)
- mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem),
- ib_umem_end(umem));
+ mlx5_ib_invalidate_range(umem_odp,
+ ib_umem_start(umem_odp),
+ ib_umem_end(umem_odp));
else
mlx5_ib_free_implicit_mr(mr);
/*
@@ -1629,28 +1629,85 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
* remove the DMA mapping.
*/
mlx5_mr_cache_free(dev, mr);
- if (umem) {
- ib_umem_release(umem);
+ ib_umem_release(umem);
+ if (umem)
atomic_sub(npages, &dev->mdev->priv.reg_pages);
- }
+
if (!mr->allocated_from_cache)
kfree(mr);
}
int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{
- dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr));
+ struct mlx5_ib_mr *mmr = to_mmr(ibmr);
+
+ if (ibmr->type == IB_MR_TYPE_INTEGRITY) {
+ dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr);
+ dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
+ }
+
+ dereg_mr(to_mdev(ibmr->device), mmr);
+
return 0;
}
-struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,
+ int access_mode, int page_shift)
+{
+ void *mkc;
+
+ mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+
+ MLX5_SET(mkc, mkc, free, 1);
+ MLX5_SET(mkc, mkc, qpn, 0xffffff);
+ MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+ MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
+ MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
+ MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
+ MLX5_SET(mkc, mkc, umr_en, 1);
+ MLX5_SET(mkc, mkc, log_page_size, page_shift);
+}
+
+static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
+ int ndescs, int desc_size, int page_shift,
+ int access_mode, u32 *in, int inlen)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ int err;
+
+ mr->access_mode = access_mode;
+ mr->desc_size = desc_size;
+ mr->max_descs = ndescs;
+
+ err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);
+ if (err)
+ return err;
+
+ mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);
+
+ err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
+ if (err)
+ goto err_free_descs;
+
+ mr->mmkey.type = MLX5_MKEY_MR;
+ mr->ibmr.lkey = mr->mmkey.key;
+ mr->ibmr.rkey = mr->mmkey.key;
+
+ return 0;
+
+err_free_descs:
+ mlx5_free_priv_descs(mr);
+ return err;
+}
+
+static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
+ u32 max_num_sg, u32 max_num_meta_sg,
+ int desc_size, int access_mode)
+{
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
- int ndescs = ALIGN(max_num_sg, 4);
+ int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
+ int page_shift = 0;
struct mlx5_ib_mr *mr;
- void *mkc;
u32 *in;
int err;
@@ -1658,99 +1715,168 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
if (!mr)
return ERR_PTR(-ENOMEM);
+ mr->ibmr.pd = pd;
+ mr->ibmr.device = pd->device;
+
in = kzalloc(inlen, GFP_KERNEL);
if (!in) {
err = -ENOMEM;
goto err_free;
}
+ if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
+ page_shift = PAGE_SHIFT;
+
+ err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,
+ access_mode, in, inlen);
+ if (err)
+ goto err_free_in;
+
+ mr->umem = NULL;
+ kfree(in);
+
+ return mr;
+
+err_free_in:
+ kfree(in);
+err_free:
+ kfree(mr);
+ return ERR_PTR(err);
+}
+
+static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
+ int ndescs, u32 *in, int inlen)
+{
+ return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),
+ PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,
+ inlen);
+}
+
+static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
+ int ndescs, u32 *in, int inlen)
+{
+ return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),
+ 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
+}
+
+static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
+ int max_num_sg, int max_num_meta_sg,
+ u32 *in, int inlen)
+{
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ u32 psv_index[2];
+ void *mkc;
+ int err;
+
+ mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
+ if (!mr->sig)
+ return -ENOMEM;
+
+ /* create mem & wire PSVs */
+ err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);
+ if (err)
+ goto err_free_sig;
+
+ mr->sig->psv_memory.psv_idx = psv_index[0];
+ mr->sig->psv_wire.psv_idx = psv_index[1];
+
+ mr->sig->sig_status_checked = true;
+ mr->sig->sig_err_exists = false;
+ /* Next UMR, Arm SIGERR */
+ ++mr->sig->sigerr_count;
+ mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
+ sizeof(struct mlx5_klm),
+ MLX5_MKC_ACCESS_MODE_KLMS);
+ if (IS_ERR(mr->klm_mr)) {
+ err = PTR_ERR(mr->klm_mr);
+ goto err_destroy_psv;
+ }
+ mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
+ sizeof(struct mlx5_mtt),
+ MLX5_MKC_ACCESS_MODE_MTT);
+ if (IS_ERR(mr->mtt_mr)) {
+ err = PTR_ERR(mr->mtt_mr);
+ goto err_free_klm_mr;
+ }
+
+ /* Set bsf descriptors for mkey */
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
- MLX5_SET(mkc, mkc, free, 1);
- MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
- MLX5_SET(mkc, mkc, qpn, 0xffffff);
- MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+ MLX5_SET(mkc, mkc, bsf_en, 1);
+ MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
- if (mr_type == IB_MR_TYPE_MEM_REG) {
- mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
- MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
- err = mlx5_alloc_priv_descs(pd->device, mr,
- ndescs, sizeof(struct mlx5_mtt));
- if (err)
- goto err_free_in;
+ err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,
+ MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
+ if (err)
+ goto err_free_mtt_mr;
- mr->desc_size = sizeof(struct mlx5_mtt);
- mr->max_descs = ndescs;
- } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
- mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
+ return 0;
- err = mlx5_alloc_priv_descs(pd->device, mr,
- ndescs, sizeof(struct mlx5_klm));
- if (err)
- goto err_free_in;
- mr->desc_size = sizeof(struct mlx5_klm);
- mr->max_descs = ndescs;
- } else if (mr_type == IB_MR_TYPE_SIGNATURE) {
- u32 psv_index[2];
-
- MLX5_SET(mkc, mkc, bsf_en, 1);
- MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
- mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
- if (!mr->sig) {
- err = -ENOMEM;
- goto err_free_in;
- }
+err_free_mtt_mr:
+ dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
+ mr->mtt_mr = NULL;
+err_free_klm_mr:
+ dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);
+ mr->klm_mr = NULL;
+err_destroy_psv:
+ if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))
+ mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
+ mr->sig->psv_memory.psv_idx);
+ if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))
+ mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
+ mr->sig->psv_wire.psv_idx);
+err_free_sig:
+ kfree(mr->sig);
- /* create mem & wire PSVs */
- err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,
- 2, psv_index);
- if (err)
- goto err_free_sig;
+ return err;
+}
+
+static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
+ enum ib_mr_type mr_type, u32 max_num_sg,
+ u32 max_num_meta_sg)
+{
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+ int ndescs = ALIGN(max_num_sg, 4);
+ struct mlx5_ib_mr *mr;
+ u32 *in;
+ int err;
- mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
- mr->sig->psv_memory.psv_idx = psv_index[0];
- mr->sig->psv_wire.psv_idx = psv_index[1];
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
- mr->sig->sig_status_checked = true;
- mr->sig->sig_err_exists = false;
- /* Next UMR, Arm SIGERR */
- ++mr->sig->sigerr_count;
- } else {
+ in = kzalloc(inlen, GFP_KERNEL);
+ if (!in) {
+ err = -ENOMEM;
+ goto err_free;
+ }
+
+ mr->ibmr.device = pd->device;
+ mr->umem = NULL;
+
+ switch (mr_type) {
+ case IB_MR_TYPE_MEM_REG:
+ err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);
+ break;
+ case IB_MR_TYPE_SG_GAPS:
+ err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);
+ break;
+ case IB_MR_TYPE_INTEGRITY:
+ err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,
+ max_num_meta_sg, in, inlen);
+ break;
+ default:
mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
err = -EINVAL;
- goto err_free_in;
}
- MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);
- MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);
- MLX5_SET(mkc, mkc, umr_en, 1);
-
- mr->ibmr.device = pd->device;
- err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
if (err)
- goto err_destroy_psv;
+ goto err_free_in;
- mr->mmkey.type = MLX5_MKEY_MR;
- mr->ibmr.lkey = mr->mmkey.key;
- mr->ibmr.rkey = mr->mmkey.key;
- mr->umem = NULL;
kfree(in);
return &mr->ibmr;
-err_destroy_psv:
- if (mr->sig) {
- if (mlx5_core_destroy_psv(dev->mdev,
- mr->sig->psv_memory.psv_idx))
- mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
- mr->sig->psv_memory.psv_idx);
- if (mlx5_core_destroy_psv(dev->mdev,
- mr->sig->psv_wire.psv_idx))
- mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
- mr->sig->psv_wire.psv_idx);
- }
- mlx5_free_priv_descs(mr);
-err_free_sig:
- kfree(mr->sig);
err_free_in:
kfree(in);
err_free:
@@ -1758,6 +1884,19 @@ err_free:
return ERR_PTR(err);
}
+struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+ u32 max_num_sg, struct ib_udata *udata)
+{
+ return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
+}
+
+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_sg, u32 max_num_meta_sg)
+{
+ return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,
+ max_num_meta_sg);
+}
+
struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{
@@ -1887,16 +2026,53 @@ done:
}
static int
+mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ unsigned int sg_offset = 0;
+ int n = 0;
+
+ mr->meta_length = 0;
+ if (data_sg_nents == 1) {
+ n++;
+ mr->ndescs = 1;
+ if (data_sg_offset)
+ sg_offset = *data_sg_offset;
+ mr->data_length = sg_dma_len(data_sg) - sg_offset;
+ mr->data_iova = sg_dma_address(data_sg) + sg_offset;
+ if (meta_sg_nents == 1) {
+ n++;
+ mr->meta_ndescs = 1;
+ if (meta_sg_offset)
+ sg_offset = *meta_sg_offset;
+ else
+ sg_offset = 0;
+ mr->meta_length = sg_dma_len(meta_sg) - sg_offset;
+ mr->pi_iova = sg_dma_address(meta_sg) + sg_offset;
+ }
+ ibmr->length = mr->data_length + mr->meta_length;
+ }
+
+ return n;
+}
+
+static int
mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
struct scatterlist *sgl,
unsigned short sg_nents,
- unsigned int *sg_offset_p)
+ unsigned int *sg_offset_p,
+ struct scatterlist *meta_sgl,
+ unsigned short meta_sg_nents,
+ unsigned int *meta_sg_offset_p)
{
struct scatterlist *sg = sgl;
struct mlx5_klm *klms = mr->descs;
unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
u32 lkey = mr->ibmr.pd->local_dma_lkey;
- int i;
+ int i, j = 0;
mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
mr->ibmr.length = 0;
@@ -1911,12 +2087,36 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
sg_offset = 0;
}
- mr->ndescs = i;
if (sg_offset_p)
*sg_offset_p = sg_offset;
- return i;
+ mr->ndescs = i;
+ mr->data_length = mr->ibmr.length;
+
+ if (meta_sg_nents) {
+ sg = meta_sgl;
+ sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;
+ for_each_sg(meta_sgl, sg, meta_sg_nents, j) {
+ if (unlikely(i + j >= mr->max_descs))
+ break;
+ klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
+ sg_offset);
+ klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
+ sg_offset);
+ klms[i + j].key = cpu_to_be32(lkey);
+ mr->ibmr.length += sg_dma_len(sg) - sg_offset;
+
+ sg_offset = 0;
+ }
+ if (meta_sg_offset_p)
+ *meta_sg_offset_p = sg_offset;
+
+ mr->meta_ndescs = j;
+ mr->meta_length = mr->ibmr.length - mr->data_length;
+ }
+
+ return i + j;
}
static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
@@ -1933,6 +2133,181 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
return 0;
}
+static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ __be64 *descs;
+
+ if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs))
+ return -ENOMEM;
+
+ descs = mr->descs;
+ descs[mr->ndescs + mr->meta_ndescs++] =
+ cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
+
+ return 0;
+}
+
+static int
+mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
+ int n;
+
+ pi_mr->ndescs = 0;
+ pi_mr->meta_ndescs = 0;
+ pi_mr->meta_length = 0;
+
+ ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
+ pi_mr->desc_size * pi_mr->max_descs,
+ DMA_TO_DEVICE);
+
+ pi_mr->ibmr.page_size = ibmr->page_size;
+ n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,
+ mlx5_set_page);
+ if (n != data_sg_nents)
+ return n;
+
+ pi_mr->data_iova = pi_mr->ibmr.iova;
+ pi_mr->data_length = pi_mr->ibmr.length;
+ pi_mr->ibmr.length = pi_mr->data_length;
+ ibmr->length = pi_mr->data_length;
+
+ if (meta_sg_nents) {
+ u64 page_mask = ~((u64)ibmr->page_size - 1);
+ u64 iova = pi_mr->data_iova;
+
+ n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,
+ meta_sg_offset, mlx5_set_page_pi);
+
+ pi_mr->meta_length = pi_mr->ibmr.length;
+ /*
+ * PI address for the HW is the offset of the metadata address
+ * relative to the first data page address.
+ * It equals to first data page address + size of data pages +
+ * metadata offset at the first metadata page
+ */
+ pi_mr->pi_iova = (iova & page_mask) +
+ pi_mr->ndescs * ibmr->page_size +
+ (pi_mr->ibmr.iova & ~page_mask);
+ /*
+ * In order to use one MTT MR for data and metadata, we register
+ * also the gaps between the end of the data and the start of
+ * the metadata (the sig MR will verify that the HW will access
+ * to right addresses). This mapping is safe because we use
+ * internal mkey for the registration.
+ */
+ pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;
+ pi_mr->ibmr.iova = iova;
+ ibmr->length += pi_mr->meta_length;
+ }
+
+ ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
+ pi_mr->desc_size * pi_mr->max_descs,
+ DMA_TO_DEVICE);
+
+ return n;
+}
+
+static int
+mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ struct mlx5_ib_mr *pi_mr = mr->klm_mr;
+ int n;
+
+ pi_mr->ndescs = 0;
+ pi_mr->meta_ndescs = 0;
+ pi_mr->meta_length = 0;
+
+ ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
+ pi_mr->desc_size * pi_mr->max_descs,
+ DMA_TO_DEVICE);
+
+ n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,
+ meta_sg, meta_sg_nents, meta_sg_offset);
+
+ ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
+ pi_mr->desc_size * pi_mr->max_descs,
+ DMA_TO_DEVICE);
+
+ /* This is zero-based memory region */
+ pi_mr->data_iova = 0;
+ pi_mr->ibmr.iova = 0;
+ pi_mr->pi_iova = pi_mr->data_length;
+ ibmr->length = pi_mr->ibmr.length;
+
+ return n;
+}
+
+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ struct mlx5_ib_mr *pi_mr = NULL;
+ int n;
+
+ WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
+
+ mr->ndescs = 0;
+ mr->data_length = 0;
+ mr->data_iova = 0;
+ mr->meta_ndescs = 0;
+ mr->pi_iova = 0;
+ /*
+ * As a performance optimization, if possible, there is no need to
+ * perform UMR operation to register the data/metadata buffers.
+ * First try to map the sg lists to PA descriptors with local_dma_lkey.
+ * Fallback to UMR only in case of a failure.
+ */
+ n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,
+ data_sg_offset, meta_sg, meta_sg_nents,
+ meta_sg_offset);
+ if (n == data_sg_nents + meta_sg_nents)
+ goto out;
+ /*
+ * As a performance optimization, if possible, there is no need to map
+ * the sg lists to KLM descriptors. First try to map the sg lists to MTT
+ * descriptors and fallback to KLM only in case of a failure.
+ * It's more efficient for the HW to work with MTT descriptors
+ * (especially in high load).
+ * Use KLM (indirect access) only if it's mandatory.
+ */
+ pi_mr = mr->mtt_mr;
+ n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,
+ data_sg_offset, meta_sg, meta_sg_nents,
+ meta_sg_offset);
+ if (n == data_sg_nents + meta_sg_nents)
+ goto out;
+
+ pi_mr = mr->klm_mr;
+ n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,
+ data_sg_offset, meta_sg, meta_sg_nents,
+ meta_sg_offset);
+ if (unlikely(n != data_sg_nents + meta_sg_nents))
+ return -ENOMEM;
+
+out:
+ /* This is zero-based memory region */
+ ibmr->iova = 0;
+ mr->pi_mr = pi_mr;
+ if (pi_mr)
+ ibmr->sig_attrs->meta_length = pi_mr->meta_length;
+ else
+ ibmr->sig_attrs->meta_length = mr->meta_length;
+
+ return 0;
+}
+
int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset)
{
@@ -1946,7 +2321,8 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
DMA_TO_DEVICE);
if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
- n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset);
+ n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,
+ NULL);
else
n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
mlx5_set_page);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 91507a2e9290..3d18b6ea9efa 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -150,7 +150,7 @@ static struct ib_umem_odp *odp_lookup(u64 start, u64 length,
if (!rb)
goto not_found;
odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);
- if (ib_umem_start(&odp->umem) > start + length)
+ if (ib_umem_start(odp) > start + length)
goto not_found;
}
not_found:
@@ -200,7 +200,7 @@ void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
static void mr_leaf_free_action(struct work_struct *work)
{
struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work);
- int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT;
+ int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent;
mr->parent = NULL;
@@ -224,7 +224,6 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT /
sizeof(struct mlx5_mtt)) - 1;
u64 idx = 0, blk_start_idx = 0;
- struct ib_umem *umem;
int in_block = 0;
u64 addr;
@@ -232,15 +231,14 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
pr_err("invalidation called on NULL umem or non-ODP umem\n");
return;
}
- umem = &umem_odp->umem;
mr = umem_odp->private;
if (!mr || !mr->ibmr.pd)
return;
- start = max_t(u64, ib_umem_start(umem), start);
- end = min_t(u64, ib_umem_end(umem), end);
+ start = max_t(u64, ib_umem_start(umem_odp), start);
+ end = min_t(u64, ib_umem_end(umem_odp), end);
/*
* Iteration one - zap the HW's MTTs. The notifiers_count ensures that
@@ -249,8 +247,8 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
* but they will write 0s as well, so no difference in the end result.
*/
- for (addr = start; addr < end; addr += BIT(umem->page_shift)) {
- idx = (addr - ib_umem_start(umem)) >> umem->page_shift;
+ for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
+ idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
/*
* Strive to write the MTTs in chunks, but avoid overwriting
* non-existing MTTs. The huristic here can be improved to
@@ -544,13 +542,12 @@ static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end,
void *cookie)
{
struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie;
- struct ib_umem *umem = &umem_odp->umem;
if (mr->parent != imr)
return 0;
- ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem),
- ib_umem_end(umem));
+ ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
+ ib_umem_end(umem_odp));
if (umem_odp->dying)
return 0;
@@ -602,9 +599,9 @@ static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
}
next_mr:
- size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt);
+ size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt);
- page_shift = mr->umem->page_shift;
+ page_shift = odp->page_shift;
page_mask = ~(BIT(page_shift) - 1);
start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift;
access_mask = ODP_READ_ALLOWED_BIT;
@@ -1488,9 +1485,11 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)
mlx5_eq_update_ci(eq->core, cc, 1);
}
-static irqreturn_t mlx5_ib_eq_pf_int(int irq, void *eq_ptr)
+static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type,
+ void *data)
{
- struct mlx5_ib_pf_eq *eq = eq_ptr;
+ struct mlx5_ib_pf_eq *eq =
+ container_of(nb, struct mlx5_ib_pf_eq, irq_nb);
unsigned long flags;
if (spin_trylock_irqsave(&eq->lock, flags)) {
@@ -1553,20 +1552,26 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
goto err_mempool;
}
+ eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
param = (struct mlx5_eq_param) {
- .index = MLX5_EQ_PFAULT_IDX,
+ .irq_index = 0,
.mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT,
.nent = MLX5_IB_NUM_PF_EQE,
- .context = eq,
- .handler = mlx5_ib_eq_pf_int
};
- eq->core = mlx5_eq_create_generic(dev->mdev, "mlx5_ib_page_fault_eq", &param);
+ eq->core = mlx5_eq_create_generic(dev->mdev, &param);
if (IS_ERR(eq->core)) {
err = PTR_ERR(eq->core);
goto err_wq;
}
+ err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb);
+ if (err) {
+ mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err);
+ goto err_eq;
+ }
return 0;
+err_eq:
+ mlx5_eq_destroy_generic(dev->mdev, eq->core);
err_wq:
destroy_workqueue(eq->wq);
err_mempool:
@@ -1579,6 +1584,7 @@ mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
{
int err;
+ mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb);
err = mlx5_eq_destroy_generic(dev->mdev, eq->core);
cancel_work_sync(&eq->work);
destroy_workqueue(eq->wq);
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index f6623c77443a..4fbf60fed374 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -442,9 +442,9 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)
}
size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
+ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
- return MLX5_SIG_WQE_SIZE;
+ return MLX5_SIG_WQE_SIZE;
else
return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
}
@@ -496,9 +496,6 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
sizeof(struct mlx5_wqe_inline_seg);
attr->cap.max_inline_data = qp->max_inline_data;
- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
- qp->signature_en = true;
-
wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
@@ -790,8 +787,7 @@ static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
atomic_dec(&dev->delay_drop.rqs_cnt);
mlx5_ib_db_unmap_user(context, &rwq->db);
- if (rwq->umem)
- ib_umem_release(rwq->umem);
+ ib_umem_release(rwq->umem);
}
static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
@@ -977,8 +973,7 @@ err_free:
kvfree(*in);
err_umem:
- if (ubuffer->umem)
- ib_umem_release(ubuffer->umem);
+ ib_umem_release(ubuffer->umem);
err_bfreg:
if (bfregn != MLX5_IB_INVALID_BFREG)
@@ -997,8 +992,7 @@ static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
ibucontext);
mlx5_ib_db_unmap_user(context, &qp->db);
- if (base->ubuffer.umem)
- ib_umem_release(base->ubuffer.umem);
+ ib_umem_release(base->ubuffer.umem);
/*
* Free only the BFREGs which are handled by the kernel.
@@ -1042,7 +1036,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
void *qpc;
int err;
- if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
+ if (init_attr->create_flags & ~(IB_QP_CREATE_INTEGRITY_EN |
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
IB_QP_CREATE_IPOIB_UD_LSO |
IB_QP_CREATE_NETIF_QP |
@@ -4170,15 +4164,13 @@ static __be64 sig_mkey_mask(void)
}
static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
- struct mlx5_ib_mr *mr, bool umr_inline)
+ struct mlx5_ib_mr *mr, u8 flags)
{
- int size = mr->ndescs * mr->desc_size;
+ int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
memset(umr, 0, sizeof(*umr));
- umr->flags = MLX5_UMR_CHECK_NOT_FREE;
- if (umr_inline)
- umr->flags |= MLX5_UMR_INLINE;
+ umr->flags = flags;
umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
umr->mkey_mask = frwr_mkey_mask();
}
@@ -4305,7 +4297,7 @@ static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
struct mlx5_ib_mr *mr,
u32 key, int access)
{
- int ndescs = ALIGN(mr->ndescs, 8) >> 1;
+ int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
memset(seg, 0, sizeof(*seg));
@@ -4356,7 +4348,7 @@ static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
struct mlx5_ib_mr *mr,
struct mlx5_ib_pd *pd)
{
- int bcount = mr->desc_size * mr->ndescs;
+ int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
dseg->addr = cpu_to_be64(mr->desc_map);
dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
@@ -4549,23 +4541,37 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr,
return 0;
}
-static int set_sig_data_segment(const struct ib_sig_handover_wr *wr,
- struct mlx5_ib_qp *qp, void **seg,
- int *size, void **cur_edge)
+static int set_sig_data_segment(const struct ib_send_wr *send_wr,
+ struct ib_mr *sig_mr,
+ struct ib_sig_attrs *sig_attrs,
+ struct mlx5_ib_qp *qp, void **seg, int *size,
+ void **cur_edge)
{
- struct ib_sig_attrs *sig_attrs = wr->sig_attrs;
- struct ib_mr *sig_mr = wr->sig_mr;
struct mlx5_bsf *bsf;
- u32 data_len = wr->wr.sg_list->length;
- u32 data_key = wr->wr.sg_list->lkey;
- u64 data_va = wr->wr.sg_list->addr;
+ u32 data_len;
+ u32 data_key;
+ u64 data_va;
+ u32 prot_len = 0;
+ u32 prot_key = 0;
+ u64 prot_va = 0;
+ bool prot = false;
int ret;
int wqe_size;
+ struct mlx5_ib_mr *mr = to_mmr(sig_mr);
+ struct mlx5_ib_mr *pi_mr = mr->pi_mr;
+
+ data_len = pi_mr->data_length;
+ data_key = pi_mr->ibmr.lkey;
+ data_va = pi_mr->data_iova;
+ if (pi_mr->meta_ndescs) {
+ prot_len = pi_mr->meta_length;
+ prot_key = pi_mr->ibmr.lkey;
+ prot_va = pi_mr->pi_iova;
+ prot = true;
+ }
- if (!wr->prot ||
- (data_key == wr->prot->lkey &&
- data_va == wr->prot->addr &&
- data_len == wr->prot->length)) {
+ if (!prot || (data_key == prot_key && data_va == prot_va &&
+ data_len == prot_len)) {
/**
* Source domain doesn't contain signature information
* or data and protection are interleaved in memory.
@@ -4599,8 +4605,6 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr,
struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
struct mlx5_stride_block_entry *data_sentry;
struct mlx5_stride_block_entry *prot_sentry;
- u32 prot_key = wr->prot->lkey;
- u64 prot_va = wr->prot->addr;
u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
int prot_size;
@@ -4650,17 +4654,15 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr,
}
static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
- const struct ib_sig_handover_wr *wr, u32 size,
- u32 length, u32 pdn)
+ struct ib_mr *sig_mr, int access_flags,
+ u32 size, u32 length, u32 pdn)
{
- struct ib_mr *sig_mr = wr->sig_mr;
u32 sig_key = sig_mr->rkey;
u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
memset(seg, 0, sizeof(*seg));
- seg->flags = get_umr_flags(wr->access_flags) |
- MLX5_MKC_ACCESS_MODE_KLMS;
+ seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
MLX5_MKEY_BSF_EN | pdn);
@@ -4680,49 +4682,50 @@ static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
umr->mkey_mask = sig_mkey_mask();
}
-
-static int set_sig_umr_wr(const struct ib_send_wr *send_wr,
- struct mlx5_ib_qp *qp, void **seg, int *size,
- void **cur_edge)
+static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
+ struct mlx5_ib_qp *qp, void **seg, int *size,
+ void **cur_edge)
{
- const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);
- struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);
+ const struct ib_reg_wr *wr = reg_wr(send_wr);
+ struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
+ struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
+ struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
u32 pdn = get_pd(qp)->pdn;
u32 xlt_size;
int region_len, ret;
- if (unlikely(wr->wr.num_sge != 1) ||
- unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) ||
- unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) ||
+ if (unlikely(send_wr->num_sge != 0) ||
+ unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
+ unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
unlikely(!sig_mr->sig->sig_status_checked))
return -EINVAL;
/* length of the protected region, data + protection */
- region_len = wr->wr.sg_list->length;
- if (wr->prot &&
- (wr->prot->lkey != wr->wr.sg_list->lkey ||
- wr->prot->addr != wr->wr.sg_list->addr ||
- wr->prot->length != wr->wr.sg_list->length))
- region_len += wr->prot->length;
+ region_len = pi_mr->ibmr.length;
/**
* KLM octoword size - if protection was provided
* then we use strided block format (3 octowords),
* else we use single KLM (1 octoword)
**/
- xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm);
+ if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
+ xlt_size = 0x30;
+ else
+ xlt_size = sizeof(struct mlx5_klm);
set_sig_umr_segment(*seg, xlt_size);
*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
- set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn);
+ set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
+ pdn);
*seg += sizeof(struct mlx5_mkey_seg);
*size += sizeof(struct mlx5_mkey_seg) / 16;
handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
- ret = set_sig_data_segment(wr, qp, seg, size, cur_edge);
+ ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
+ cur_edge);
if (ret)
return ret;
@@ -4759,12 +4762,14 @@ static int set_psv_wr(struct ib_sig_domain *domain,
static int set_reg_wr(struct mlx5_ib_qp *qp,
const struct ib_reg_wr *wr,
- void **seg, int *size, void **cur_edge)
+ void **seg, int *size, void **cur_edge,
+ bool check_not_free)
{
struct mlx5_ib_mr *mr = to_mmr(wr->mr);
struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
- size_t mr_list_size = mr->ndescs * mr->desc_size;
+ int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
+ u8 flags = 0;
if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
mlx5_ib_warn(to_mdev(qp->ibqp.device),
@@ -4772,7 +4777,12 @@ static int set_reg_wr(struct mlx5_ib_qp *qp,
return -EINVAL;
}
- set_reg_umr_seg(*seg, mr, umr_inline);
+ if (check_not_free)
+ flags |= MLX5_UMR_CHECK_NOT_FREE;
+ if (umr_inline)
+ flags |= MLX5_UMR_INLINE;
+
+ set_reg_umr_seg(*seg, mr, flags);
*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
@@ -4898,8 +4908,12 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
struct mlx5_core_dev *mdev = dev->mdev;
+ struct ib_reg_wr reg_pi_wr;
struct mlx5_ib_qp *qp;
struct mlx5_ib_mr *mr;
+ struct mlx5_ib_mr *pi_mr;
+ struct mlx5_ib_mr pa_pi_mr;
+ struct ib_sig_attrs *sig_attrs;
struct mlx5_wqe_xrc_seg *xrc;
struct mlx5_bf *bf;
void *cur_edge;
@@ -4953,7 +4967,8 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
goto out;
}
- if (wr->opcode == IB_WR_REG_MR) {
+ if (wr->opcode == IB_WR_REG_MR ||
+ wr->opcode == IB_WR_REG_MR_INTEGRITY) {
fence = dev->umr_fence;
next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
} else {
@@ -5003,7 +5018,7 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
qp->sq.wr_data[idx] = IB_WR_REG_MR;
ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
err = set_reg_wr(qp, reg_wr(wr), &seg, &size,
- &cur_edge);
+ &cur_edge, true);
if (err) {
*bad_wr = wr;
goto out;
@@ -5011,26 +5026,82 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
num_sge = 0;
break;
- case IB_WR_REG_SIG_MR:
- qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
- mr = to_mmr(sig_handover_wr(wr)->sig_mr);
-
+ case IB_WR_REG_MR_INTEGRITY:
+ qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY;
+
+ mr = to_mmr(reg_wr(wr)->mr);
+ pi_mr = mr->pi_mr;
+
+ if (pi_mr) {
+ memset(&reg_pi_wr, 0,
+ sizeof(struct ib_reg_wr));
+
+ reg_pi_wr.mr = &pi_mr->ibmr;
+ reg_pi_wr.access = reg_wr(wr)->access;
+ reg_pi_wr.key = pi_mr->ibmr.rkey;
+
+ ctrl->imm = cpu_to_be32(reg_pi_wr.key);
+ /* UMR for data + prot registration */
+ err = set_reg_wr(qp, &reg_pi_wr, &seg,
+ &size, &cur_edge,
+ false);
+ if (err) {
+ *bad_wr = wr;
+ goto out;
+ }
+ finish_wqe(qp, ctrl, seg, size,
+ cur_edge, idx, wr->wr_id,
+ nreq, fence,
+ MLX5_OPCODE_UMR);
+
+ err = begin_wqe(qp, &seg, &ctrl, wr,
+ &idx, &size, &cur_edge,
+ nreq);
+ if (err) {
+ mlx5_ib_warn(dev, "\n");
+ err = -ENOMEM;
+ *bad_wr = wr;
+ goto out;
+ }
+ } else {
+ memset(&pa_pi_mr, 0,
+ sizeof(struct mlx5_ib_mr));
+ /* No UMR, use local_dma_lkey */
+ pa_pi_mr.ibmr.lkey =
+ mr->ibmr.pd->local_dma_lkey;
+
+ pa_pi_mr.ndescs = mr->ndescs;
+ pa_pi_mr.data_length = mr->data_length;
+ pa_pi_mr.data_iova = mr->data_iova;
+ if (mr->meta_ndescs) {
+ pa_pi_mr.meta_ndescs =
+ mr->meta_ndescs;
+ pa_pi_mr.meta_length =
+ mr->meta_length;
+ pa_pi_mr.pi_iova = mr->pi_iova;
+ }
+
+ pa_pi_mr.ibmr.length = mr->ibmr.length;
+ mr->pi_mr = &pa_pi_mr;
+ }
ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
- err = set_sig_umr_wr(wr, qp, &seg, &size,
- &cur_edge);
+ /* UMR for sig MR */
+ err = set_pi_umr_wr(wr, qp, &seg, &size,
+ &cur_edge);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
-
finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
wr->wr_id, nreq, fence,
MLX5_OPCODE_UMR);
+
/*
* SET_PSV WQEs are not signaled and solicited
* on error
*/
+ sig_attrs = mr->ibmr.sig_attrs;
err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
&size, &cur_edge, nreq, false,
true);
@@ -5040,19 +5111,18 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
*bad_wr = wr;
goto out;
}
-
- err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,
- mr->sig->psv_memory.psv_idx, &seg,
- &size);
+ err = set_psv_wr(&sig_attrs->mem,
+ mr->sig->psv_memory.psv_idx,
+ &seg, &size);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
-
finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
- wr->wr_id, nreq, fence,
+ wr->wr_id, nreq, next_fence,
MLX5_OPCODE_SET_PSV);
+
err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
&size, &cur_edge, nreq, false,
true);
@@ -5062,20 +5132,20 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
*bad_wr = wr;
goto out;
}
-
- err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
- mr->sig->psv_wire.psv_idx, &seg,
- &size);
+ err = set_psv_wr(&sig_attrs->wire,
+ mr->sig->psv_wire.psv_idx,
+ &seg, &size);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
-
finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
- wr->wr_id, nreq, fence,
+ wr->wr_id, nreq, next_fence,
MLX5_OPCODE_SET_PSV);
- qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
+
+ qp->next_fence =
+ MLX5_FENCE_MODE_INITIATOR_SMALL;
num_sge = 0;
goto skip_psv;
@@ -6047,7 +6117,7 @@ err:
return ERR_PTR(err);
}
-int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
+void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(wq->device);
struct mlx5_ib_rwq *rwq = to_mrwq(wq);
@@ -6055,8 +6125,6 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
destroy_user_rq(dev, wq->pd, rwq, udata);
kfree(rwq);
-
- return 0;
}
struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c
index 8ff0e90d7564..edccfd6e178f 100644
--- a/drivers/infiniband/hw/mthca/mthca_memfree.c
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.c
@@ -482,7 +482,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
if (ret < 0) {
- put_page(pages[0]);
+ put_user_page(pages[0]);
goto out;
}
@@ -490,7 +490,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
mthca_uarc_virt(dev, uar, i));
if (ret) {
pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
- put_page(sg_page(&db_tab->page[i].mem));
+ put_user_page(sg_page(&db_tab->page[i].mem));
goto out;
}
@@ -556,7 +556,7 @@ void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
if (db_tab->page[i].uvirt) {
mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1);
pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
- put_page(sg_page(&db_tab->page[i].mem));
+ put_user_page(sg_page(&db_tab->page[i].mem));
}
}
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 4f40dfedf920..23554d8bf241 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -601,10 +601,11 @@ static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
return 0;
}
-static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+static int mthca_create_cq(struct ib_cq *ibcq,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
struct mthca_create_cq ucmd;
struct mthca_cq *cq;
@@ -614,20 +615,20 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
udata, struct mthca_ucontext, ibucontext);
if (attr->flags)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (udata) {
- if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
- return ERR_PTR(-EFAULT);
+ if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+ return -EFAULT;
err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.set_db_index,
ucmd.set_db_page);
if (err)
- return ERR_PTR(err);
+ return err;
err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.arm_db_index,
@@ -636,11 +637,7 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
goto err_unmap_set;
}
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq) {
- err = -ENOMEM;
- goto err_unmap_arm;
- }
+ cq = to_mcq(ibcq);
if (udata) {
cq->buf.mr.ibmr.lkey = ucmd.lkey;
@@ -655,20 +652,17 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
cq);
if (err)
- goto err_free;
+ goto err_unmap_arm;
if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
mthca_free_cq(to_mdev(ibdev), cq);
err = -EFAULT;
- goto err_free;
+ goto err_unmap_arm;
}
cq->resize_buf = NULL;
- return &cq->ibcq;
-
-err_free:
- kfree(cq);
+ return 0;
err_unmap_arm:
if (udata)
@@ -680,7 +674,7 @@ err_unmap_set:
mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.set_db_index);
- return ERR_PTR(err);
+ return err;
}
static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq,
@@ -804,7 +798,7 @@ out:
return ret;
}
-static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
if (udata) {
struct mthca_ucontext *context =
@@ -823,9 +817,6 @@ static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
to_mcq(cq)->set_ci_db_index);
}
mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
- kfree(cq);
-
- return 0;
}
static inline u32 convert_access(int acc)
@@ -962,8 +953,7 @@ static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
struct mthca_mr *mmr = to_mmr(mr);
mthca_free_mr(to_mdev(mr->device), mmr);
- if (mmr->umem)
- ib_umem_release(mmr->umem);
+ ib_umem_release(mmr->umem);
kfree(mmr);
return 0;
@@ -1153,6 +1143,11 @@ static void get_dev_fw_str(struct ib_device *device, char *str)
}
static const struct ib_device_ops mthca_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_MTHCA,
+ .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION,
+ .uverbs_no_driver_id_binding = 1,
+
.alloc_pd = mthca_alloc_pd,
.alloc_ucontext = mthca_alloc_ucontext,
.attach_mcast = mthca_multicast_attach,
@@ -1185,6 +1180,7 @@ static const struct ib_device_ops mthca_dev_ops = {
.resize_cq = mthca_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext),
};
@@ -1243,9 +1239,6 @@ int mthca_register_device(struct mthca_dev *dev)
if (ret)
return ret;
- dev->ib_dev.owner = THIS_MODULE;
-
- dev->ib_dev.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION;
dev->ib_dev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -1303,7 +1296,6 @@ int mthca_register_device(struct mthca_dev *dev)
mutex_init(&dev->cap_mask_mutex);
rdma_set_device_sysfs_group(&dev->ib_dev, &mthca_attr_group);
- dev->ib_dev.driver_id = RDMA_DRIVER_MTHCA;
ret = ib_register_device(&dev->ib_dev, "mthca%d");
if (ret)
return ret;
diff --git a/drivers/infiniband/hw/nes/Kconfig b/drivers/infiniband/hw/nes/Kconfig
deleted file mode 100644
index 8245353e8106..000000000000
--- a/drivers/infiniband/hw/nes/Kconfig
+++ /dev/null
@@ -1,16 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config INFINIBAND_NES
- tristate "NetEffect RNIC Driver"
- depends on PCI && INET
- select LIBCRC32C
- ---help---
- This is the RDMA Network Interface Card (RNIC) driver for
- NetEffect Ethernet Cluster Server Adapters.
-
-config INFINIBAND_NES_DEBUG
- bool "Verbose debugging output"
- depends on INFINIBAND_NES
- default n
- ---help---
- This option enables debug messages from the NetEffect RNIC
- driver. Select this if you are diagnosing a problem.
diff --git a/drivers/infiniband/hw/nes/Makefile b/drivers/infiniband/hw/nes/Makefile
deleted file mode 100644
index 239689ad1f9e..000000000000
--- a/drivers/infiniband/hw/nes/Makefile
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_INFINIBAND_NES) += iw_nes.o
-
-iw_nes-objs := nes.o nes_hw.o nes_nic.o nes_utils.o nes_verbs.o nes_cm.o nes_mgt.o
diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c
deleted file mode 100644
index e00add6d78ec..000000000000
--- a/drivers/infiniband/hw/nes/nes.c
+++ /dev/null
@@ -1,1205 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
-#include <linux/mii.h>
-#include <linux/if_vlan.h>
-#include <linux/crc32.h>
-#include <linux/in.h>
-#include <linux/fs.h>
-#include <linux/init.h>
-#include <linux/if_arp.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/byteorder.h>
-#include <rdma/ib_smi.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/ib_pack.h>
-#include <rdma/iw_cm.h>
-
-#include "nes.h"
-
-#include <net/netevent.h>
-#include <net/neighbour.h>
-#include <linux/route.h>
-#include <net/ip_fib.h>
-
-MODULE_AUTHOR("NetEffect");
-MODULE_DESCRIPTION("NetEffect RNIC Low-level iWARP Driver");
-MODULE_LICENSE("Dual BSD/GPL");
-
-int interrupt_mod_interval = 0;
-
-/* Interoperability */
-int mpa_version = 1;
-module_param(mpa_version, int, 0644);
-MODULE_PARM_DESC(mpa_version, "MPA version to be used int MPA Req/Resp (0 or 1)");
-
-/* Interoperability */
-int disable_mpa_crc = 0;
-module_param(disable_mpa_crc, int, 0644);
-MODULE_PARM_DESC(disable_mpa_crc, "Disable checking of MPA CRC");
-
-unsigned int nes_drv_opt = NES_DRV_OPT_DISABLE_INT_MOD | NES_DRV_OPT_ENABLE_PAU;
-module_param(nes_drv_opt, int, 0644);
-MODULE_PARM_DESC(nes_drv_opt, "Driver option parameters");
-
-unsigned int nes_debug_level = 0;
-module_param_named(debug_level, nes_debug_level, uint, 0644);
-MODULE_PARM_DESC(debug_level, "Enable debug output level");
-
-unsigned int wqm_quanta = 0x10000;
-module_param(wqm_quanta, int, 0644);
-MODULE_PARM_DESC(wqm_quanta, "WQM quanta");
-
-static bool limit_maxrdreqsz;
-module_param(limit_maxrdreqsz, bool, 0644);
-MODULE_PARM_DESC(limit_maxrdreqsz, "Limit max read request size to 256 Bytes");
-
-LIST_HEAD(nes_adapter_list);
-static LIST_HEAD(nes_dev_list);
-
-atomic_t qps_destroyed;
-
-static unsigned int ee_flsh_adapter;
-static unsigned int sysfs_nonidx_addr;
-static unsigned int sysfs_idx_addr;
-
-static const struct pci_device_id nes_pci_table[] = {
- { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020), },
- { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR), },
- {0}
-};
-
-MODULE_DEVICE_TABLE(pci, nes_pci_table);
-
-static int nes_inetaddr_event(struct notifier_block *, unsigned long, void *);
-static int nes_net_event(struct notifier_block *, unsigned long, void *);
-static int nes_notifiers_registered;
-
-
-static struct notifier_block nes_inetaddr_notifier = {
- .notifier_call = nes_inetaddr_event
-};
-
-static struct notifier_block nes_net_notifier = {
- .notifier_call = nes_net_event
-};
-
-/**
- * nes_inetaddr_event
- */
-static int nes_inetaddr_event(struct notifier_block *notifier,
- unsigned long event, void *ptr)
-{
- struct in_ifaddr *ifa = ptr;
- struct net_device *event_netdev = ifa->ifa_dev->dev;
- struct nes_device *nesdev;
- struct net_device *netdev;
- struct net_device *upper_dev;
- struct nes_vnic *nesvnic;
- unsigned int is_bonded;
-
- nes_debug(NES_DBG_NETDEV, "nes_inetaddr_event: ip address %pI4, netmask %pI4.\n",
- &ifa->ifa_address, &ifa->ifa_mask);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p. (%s)\n",
- nesdev, nesdev->netdev[0]->name);
- netdev = nesdev->netdev[0];
- nesvnic = netdev_priv(netdev);
- upper_dev = netdev_master_upper_dev_get(netdev);
- is_bonded = netif_is_bond_slave(netdev) &&
- (upper_dev == event_netdev);
- if ((netdev == event_netdev) || is_bonded) {
- if (nesvnic->rdma_enabled == 0) {
- nes_debug(NES_DBG_NETDEV, "Returning without processing event for %s since"
- " RDMA is not enabled.\n",
- netdev->name);
- return NOTIFY_OK;
- }
- /* we have ifa->ifa_address/mask here if we need it */
- switch (event) {
- case NETDEV_DOWN:
- nes_debug(NES_DBG_NETDEV, "event:DOWN\n");
- nes_write_indexed(nesdev,
- NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)), 0);
-
- nes_manage_arp_cache(netdev, netdev->dev_addr,
- ntohl(nesvnic->local_ipaddr), NES_ARP_DELETE);
- nesvnic->local_ipaddr = 0;
- if (is_bonded)
- continue;
- else
- return NOTIFY_OK;
- break;
- case NETDEV_UP:
- nes_debug(NES_DBG_NETDEV, "event:UP\n");
-
- if (nesvnic->local_ipaddr != 0) {
- nes_debug(NES_DBG_NETDEV, "Interface already has local_ipaddr\n");
- return NOTIFY_OK;
- }
- /* fall through */
- case NETDEV_CHANGEADDR:
- /* Add the address to the IP table */
- if (upper_dev) {
- struct in_device *in;
-
- rcu_read_lock();
- in = __in_dev_get_rcu(upper_dev);
- nesvnic->local_ipaddr = in->ifa_list->ifa_address;
- rcu_read_unlock();
- } else {
- nesvnic->local_ipaddr = ifa->ifa_address;
- }
-
- nes_write_indexed(nesdev,
- NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)),
- ntohl(nesvnic->local_ipaddr));
- nes_manage_arp_cache(netdev, netdev->dev_addr,
- ntohl(nesvnic->local_ipaddr), NES_ARP_ADD);
- if (is_bonded)
- continue;
- else
- return NOTIFY_OK;
- break;
- default:
- break;
- }
- }
- }
-
- return NOTIFY_DONE;
-}
-
-
-/**
- * nes_net_event
- */
-static int nes_net_event(struct notifier_block *notifier,
- unsigned long event, void *ptr)
-{
- struct neighbour *neigh = ptr;
- struct nes_device *nesdev;
- struct net_device *netdev;
- struct nes_vnic *nesvnic;
-
- switch (event) {
- case NETEVENT_NEIGH_UPDATE:
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- /* nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p.\n", nesdev); */
- netdev = nesdev->netdev[0];
- nesvnic = netdev_priv(netdev);
- if (netdev == neigh->dev) {
- if (nesvnic->rdma_enabled == 0) {
- nes_debug(NES_DBG_NETDEV, "Skipping device %s since no RDMA\n",
- netdev->name);
- } else {
- if (neigh->nud_state & NUD_VALID) {
- nes_manage_arp_cache(neigh->dev, neigh->ha,
- ntohl(*(__be32 *)neigh->primary_key), NES_ARP_ADD);
- } else {
- nes_manage_arp_cache(neigh->dev, neigh->ha,
- ntohl(*(__be32 *)neigh->primary_key), NES_ARP_DELETE);
- }
- }
- return NOTIFY_OK;
- }
- }
- break;
- default:
- nes_debug(NES_DBG_NETDEV, "NETEVENT_ %lu undefined\n", event);
- break;
- }
-
- return NOTIFY_DONE;
-}
-
-
-/**
- * nes_add_ref
- */
-void nes_add_ref(struct ib_qp *ibqp)
-{
- struct nes_qp *nesqp;
-
- nesqp = to_nesqp(ibqp);
- nes_debug(NES_DBG_QP, "Bumping refcount for QP%u. Pre-inc value = %u\n",
- ibqp->qp_num, atomic_read(&nesqp->refcount));
- atomic_inc(&nesqp->refcount);
-}
-
-static void nes_cqp_rem_ref_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request)
-{
- unsigned long flags;
- struct nes_qp *nesqp = cqp_request->cqp_callback_pointer;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- atomic_inc(&qps_destroyed);
-
- /* Free the control structures */
-
- if (nesqp->pbl_vbase) {
- pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,
- nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_256pbl++;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);
- nesqp->pbl_vbase = NULL;
-
- } else {
- pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,
- nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);
- }
- nes_free_resource(nesadapter, nesadapter->allocated_qps, nesqp->hwqp.qp_id);
-
- nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = NULL;
- kfree(nesqp->allocated_buffer);
-
-}
-
-/**
- * nes_rem_ref
- */
-void nes_rem_ref(struct ib_qp *ibqp)
-{
- u64 u64temp;
- struct nes_qp *nesqp;
- struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- u32 opcode;
-
- nesqp = to_nesqp(ibqp);
-
- if (atomic_read(&nesqp->refcount) == 0) {
- printk(KERN_INFO PFX "%s: Reference count already 0 for QP%d, last aeq = 0x%04X.\n",
- __func__, ibqp->qp_num, nesqp->last_aeq);
- BUG();
- }
-
- if (atomic_dec_and_test(&nesqp->refcount)) {
- if (nesqp->pau_mode)
- nes_destroy_pau_qp(nesdev, nesqp);
-
- /* Destroy the QP */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
- return;
- }
- cqp_request->waiting = 0;
- cqp_request->callback = 1;
- cqp_request->cqp_callback = nes_cqp_rem_ref_callback;
- cqp_request->cqp_callback_pointer = nesqp;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- opcode = NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_IWARP;
-
- if (nesqp->hte_added) {
- opcode |= NES_CQP_QP_DEL_HTE;
- nesqp->hte_added = 0;
- }
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
- u64temp = (u64)nesqp->nesqp_context_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
- nes_post_cqp_request(nesdev, cqp_request);
- }
-}
-
-
-/**
- * nes_get_qp
- */
-struct ib_qp *nes_get_qp(struct ib_device *device, int qpn)
-{
- struct nes_vnic *nesvnic = to_nesvnic(device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- if ((qpn < NES_FIRST_QPN) || (qpn >= (NES_FIRST_QPN + nesadapter->max_qp)))
- return NULL;
-
- return &nesadapter->qp_table[qpn - NES_FIRST_QPN]->ibqp;
-}
-
-
-/**
- * nes_print_macaddr
- */
-static void nes_print_macaddr(struct net_device *netdev)
-{
- nes_debug(NES_DBG_INIT, "%s: %pM, IRQ %u\n",
- netdev->name, netdev->dev_addr, netdev->irq);
-}
-
-/**
- * nes_interrupt - handle interrupts
- */
-static irqreturn_t nes_interrupt(int irq, void *dev_id)
-{
- struct nes_device *nesdev = (struct nes_device *)dev_id;
- int handled = 0;
- u32 int_mask;
- u32 int_req;
- u32 int_stat;
- u32 intf_int_stat;
- u32 timer_stat;
-
- if (nesdev->msi_enabled) {
- /* No need to read the interrupt pending register if msi is enabled */
- handled = 1;
- } else {
- if (unlikely(nesdev->nesadapter->hw_rev == NE020_REV)) {
- /* Master interrupt enable provides synchronization for kicking off bottom half
- when interrupt sharing is going on */
- int_mask = nes_read32(nesdev->regs + NES_INT_MASK);
- if (int_mask & 0x80000000) {
- /* Check interrupt status to see if this might be ours */
- int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
- int_req = nesdev->int_req;
- if (int_stat&int_req) {
- /* if interesting CEQ or AEQ is pending, claim the interrupt */
- if ((int_stat&int_req) & (~(NES_INT_TIMER|NES_INT_INTF))) {
- handled = 1;
- } else {
- if (((int_stat & int_req) & NES_INT_TIMER) == NES_INT_TIMER) {
- /* Timer might be running but might be for another function */
- timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
- if ((timer_stat & nesdev->timer_int_req) != 0) {
- handled = 1;
- }
- }
- if ((((int_stat & int_req) & NES_INT_INTF) == NES_INT_INTF) &&
- (handled == 0)) {
- intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
- if ((intf_int_stat & nesdev->intf_int_req) != 0) {
- handled = 1;
- }
- }
- }
- if (handled) {
- nes_write32(nesdev->regs+NES_INT_MASK, int_mask & (~0x80000000));
- int_mask = nes_read32(nesdev->regs+NES_INT_MASK);
- /* Save off the status to save an additional read */
- nesdev->int_stat = int_stat;
- nesdev->napi_isr_ran = 1;
- }
- }
- }
- } else {
- handled = nes_read32(nesdev->regs+NES_INT_PENDING);
- }
- }
-
- if (handled) {
-
- if (nes_napi_isr(nesdev) == 0) {
- tasklet_schedule(&nesdev->dpc_tasklet);
-
- }
- return IRQ_HANDLED;
- } else {
- return IRQ_NONE;
- }
-}
-
-
-/**
- * nes_probe - Device initialization
- */
-static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent)
-{
- struct net_device *netdev = NULL;
- struct nes_device *nesdev = NULL;
- int ret = 0;
- void __iomem *mmio_regs = NULL;
- u8 hw_rev;
-
- printk(KERN_INFO PFX "NetEffect RNIC driver v%s loading. (%s)\n",
- DRV_VERSION, pci_name(pcidev));
-
- ret = pci_enable_device(pcidev);
- if (ret) {
- printk(KERN_ERR PFX "Unable to enable PCI device. (%s)\n", pci_name(pcidev));
- goto bail0;
- }
-
- nes_debug(NES_DBG_INIT, "BAR0 (@0x%08lX) size = 0x%lX bytes\n",
- (long unsigned int)pci_resource_start(pcidev, BAR_0),
- (long unsigned int)pci_resource_len(pcidev, BAR_0));
- nes_debug(NES_DBG_INIT, "BAR1 (@0x%08lX) size = 0x%lX bytes\n",
- (long unsigned int)pci_resource_start(pcidev, BAR_1),
- (long unsigned int)pci_resource_len(pcidev, BAR_1));
-
- /* Make sure PCI base addr are MMIO */
- if (!(pci_resource_flags(pcidev, BAR_0) & IORESOURCE_MEM) ||
- !(pci_resource_flags(pcidev, BAR_1) & IORESOURCE_MEM)) {
- printk(KERN_ERR PFX "PCI regions not an MMIO resource\n");
- ret = -ENODEV;
- goto bail1;
- }
-
- /* Reserve PCI I/O and memory resources */
- ret = pci_request_regions(pcidev, DRV_NAME);
- if (ret) {
- printk(KERN_ERR PFX "Unable to request regions. (%s)\n", pci_name(pcidev));
- goto bail1;
- }
-
- if ((sizeof(dma_addr_t) > 4)) {
- ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64));
- if (ret < 0) {
- printk(KERN_ERR PFX "64b DMA mask configuration failed\n");
- goto bail2;
- }
- ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64));
- if (ret) {
- printk(KERN_ERR PFX "64b DMA consistent mask configuration failed\n");
- goto bail2;
- }
- } else {
- ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32));
- if (ret < 0) {
- printk(KERN_ERR PFX "32b DMA mask configuration failed\n");
- goto bail2;
- }
- ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
- if (ret) {
- printk(KERN_ERR PFX "32b DMA consistent mask configuration failed\n");
- goto bail2;
- }
- }
-
- pci_set_master(pcidev);
-
- /* Allocate hardware structure */
- nesdev = kzalloc(sizeof(struct nes_device), GFP_KERNEL);
- if (!nesdev) {
- ret = -ENOMEM;
- goto bail2;
- }
-
- nes_debug(NES_DBG_INIT, "Allocated nes device at %p\n", nesdev);
- nesdev->pcidev = pcidev;
- pci_set_drvdata(pcidev, nesdev);
-
- pci_read_config_byte(pcidev, 0x0008, &hw_rev);
- nes_debug(NES_DBG_INIT, "hw_rev=%u\n", hw_rev);
-
- spin_lock_init(&nesdev->indexed_regs_lock);
-
- /* Remap the PCI registers in adapter BAR0 to kernel VA space */
- mmio_regs = ioremap_nocache(pci_resource_start(pcidev, BAR_0),
- pci_resource_len(pcidev, BAR_0));
- if (mmio_regs == NULL) {
- printk(KERN_ERR PFX "Unable to remap BAR0\n");
- ret = -EIO;
- goto bail3;
- }
- nesdev->regs = mmio_regs;
- nesdev->index_reg = 0x50 + (PCI_FUNC(pcidev->devfn)*8) + mmio_regs;
-
- /* Ensure interrupts are disabled */
- nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
-
- if (nes_drv_opt & NES_DRV_OPT_ENABLE_MSI) {
- if (!pci_enable_msi(nesdev->pcidev)) {
- nesdev->msi_enabled = 1;
- nes_debug(NES_DBG_INIT, "MSI is enabled for device %s\n",
- pci_name(pcidev));
- } else {
- nes_debug(NES_DBG_INIT, "MSI is disabled by linux for device %s\n",
- pci_name(pcidev));
- }
- } else {
- nes_debug(NES_DBG_INIT, "MSI not requested due to driver options for device %s\n",
- pci_name(pcidev));
- }
-
- nesdev->csr_start = pci_resource_start(nesdev->pcidev, BAR_0);
- nesdev->doorbell_region = pci_resource_start(nesdev->pcidev, BAR_1);
-
- /* Init the adapter */
- nesdev->nesadapter = nes_init_adapter(nesdev, hw_rev);
- if (!nesdev->nesadapter) {
- printk(KERN_ERR PFX "Unable to initialize adapter.\n");
- ret = -ENOMEM;
- goto bail5;
- }
- nesdev->nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
- nesdev->nesadapter->wqm_quanta = wqm_quanta;
-
- /* nesdev->base_doorbell_index =
- nesdev->nesadapter->pd_config_base[PCI_FUNC(nesdev->pcidev->devfn)]; */
- nesdev->base_doorbell_index = 1;
- nesdev->doorbell_start = nesdev->nesadapter->doorbell_start;
- if (nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
- switch (PCI_FUNC(nesdev->pcidev->devfn) %
- nesdev->nesadapter->port_count) {
- case 1:
- nesdev->mac_index = 2;
- break;
- case 2:
- nesdev->mac_index = 1;
- break;
- case 3:
- nesdev->mac_index = 3;
- break;
- case 0:
- default:
- nesdev->mac_index = 0;
- }
- } else {
- nesdev->mac_index = PCI_FUNC(nesdev->pcidev->devfn) %
- nesdev->nesadapter->port_count;
- }
-
- if ((limit_maxrdreqsz ||
- ((nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_GLADIUS) &&
- (hw_rev == NE020_REV1))) &&
- (pcie_get_readrq(pcidev) > 256)) {
- if (pcie_set_readrq(pcidev, 256))
- printk(KERN_ERR PFX "Unable to set max read request"
- " to 256 bytes\n");
- else
- nes_debug(NES_DBG_INIT, "Max read request size set"
- " to 256 bytes\n");
- }
-
- tasklet_init(&nesdev->dpc_tasklet, nes_dpc, (unsigned long)nesdev);
-
- /* bring up the Control QP */
- if (nes_init_cqp(nesdev)) {
- ret = -ENODEV;
- goto bail6;
- }
-
- /* Arm the CCQ */
- nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- PCI_FUNC(nesdev->pcidev->devfn));
- nes_read32(nesdev->regs+NES_CQE_ALLOC);
-
- /* Enable the interrupts */
- nesdev->int_req = (0x101 << PCI_FUNC(nesdev->pcidev->devfn)) |
- (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16));
- if (PCI_FUNC(nesdev->pcidev->devfn) < 4) {
- nesdev->int_req |= (1 << (PCI_FUNC(nesdev->mac_index)+24));
- }
-
- /* TODO: This really should be the first driver to load, not function 0 */
- if (PCI_FUNC(nesdev->pcidev->devfn) == 0) {
- /* pick up PCI and critical errors if the first driver to load */
- nesdev->intf_int_req = NES_INTF_INT_PCIERR | NES_INTF_INT_CRITERR;
- nesdev->int_req |= NES_INT_INTF;
- } else {
- nesdev->intf_int_req = 0;
- }
- nesdev->intf_int_req |= (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16));
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0, 0);
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 0);
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS2, 0x00001265);
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS4, 0x18021804);
-
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS3, 0x17801790);
-
- /* deal with both periodic and one_shot */
- nesdev->timer_int_req = 0x101 << PCI_FUNC(nesdev->pcidev->devfn);
- nesdev->nesadapter->timer_int_req |= nesdev->timer_int_req;
- nes_debug(NES_DBG_INIT, "setting int_req for function %u, nesdev = 0x%04X, adapter = 0x%04X\n",
- PCI_FUNC(nesdev->pcidev->devfn),
- nesdev->timer_int_req, nesdev->nesadapter->timer_int_req);
-
- nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
-
- list_add_tail(&nesdev->list, &nes_dev_list);
-
- /* Request an interrupt line for the driver */
- ret = request_irq(pcidev->irq, nes_interrupt, IRQF_SHARED, DRV_NAME, nesdev);
- if (ret) {
- printk(KERN_ERR PFX "%s: requested IRQ %u is busy\n",
- pci_name(pcidev), pcidev->irq);
- goto bail65;
- }
-
- nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
-
- if (nes_notifiers_registered == 0) {
- register_inetaddr_notifier(&nes_inetaddr_notifier);
- register_netevent_notifier(&nes_net_notifier);
- }
- nes_notifiers_registered++;
-
- INIT_DELAYED_WORK(&nesdev->work, nes_recheck_link_status);
-
- /* Initialize network devices */
- netdev = nes_netdev_init(nesdev, mmio_regs);
- if (netdev == NULL) {
- ret = -ENOMEM;
- goto bail7;
- }
-
- /* Register network device */
- ret = register_netdev(netdev);
- if (ret) {
- printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n", ret);
- nes_netdev_destroy(netdev);
- goto bail7;
- }
-
- nes_print_macaddr(netdev);
-
- nesdev->netdev_count++;
- nesdev->nesadapter->netdev_count++;
-
- printk(KERN_INFO PFX "%s: NetEffect RNIC driver successfully loaded.\n",
- pci_name(pcidev));
- return 0;
-
- bail7:
- printk(KERN_ERR PFX "bail7\n");
- while (nesdev->netdev_count > 0) {
- nesdev->netdev_count--;
- nesdev->nesadapter->netdev_count--;
-
- unregister_netdev(nesdev->netdev[nesdev->netdev_count]);
- nes_netdev_destroy(nesdev->netdev[nesdev->netdev_count]);
- }
-
- nes_debug(NES_DBG_INIT, "netdev_count=%d, nesadapter->netdev_count=%d\n",
- nesdev->netdev_count, nesdev->nesadapter->netdev_count);
-
- nes_notifiers_registered--;
- if (nes_notifiers_registered == 0) {
- unregister_netevent_notifier(&nes_net_notifier);
- unregister_inetaddr_notifier(&nes_inetaddr_notifier);
- }
-
- list_del(&nesdev->list);
- nes_destroy_cqp(nesdev);
-
- bail65:
- printk(KERN_ERR PFX "bail65\n");
- free_irq(pcidev->irq, nesdev);
- if (nesdev->msi_enabled) {
- pci_disable_msi(pcidev);
- }
- bail6:
- printk(KERN_ERR PFX "bail6\n");
- tasklet_kill(&nesdev->dpc_tasklet);
- /* Deallocate the Adapter Structure */
- nes_destroy_adapter(nesdev->nesadapter);
-
- bail5:
- printk(KERN_ERR PFX "bail5\n");
- iounmap(nesdev->regs);
-
- bail3:
- printk(KERN_ERR PFX "bail3\n");
- kfree(nesdev);
-
- bail2:
- pci_release_regions(pcidev);
-
- bail1:
- pci_disable_device(pcidev);
-
- bail0:
- return ret;
-}
-
-
-/**
- * nes_remove - unload from kernel
- */
-static void nes_remove(struct pci_dev *pcidev)
-{
- struct nes_device *nesdev = pci_get_drvdata(pcidev);
- struct net_device *netdev;
- int netdev_index = 0;
- unsigned long flags;
-
- if (nesdev->netdev_count) {
- netdev = nesdev->netdev[netdev_index];
- if (netdev) {
- netif_stop_queue(netdev);
- unregister_netdev(netdev);
- nes_netdev_destroy(netdev);
-
- nesdev->netdev[netdev_index] = NULL;
- nesdev->netdev_count--;
- nesdev->nesadapter->netdev_count--;
- }
- }
-
- nes_notifiers_registered--;
- if (nes_notifiers_registered == 0) {
- unregister_netevent_notifier(&nes_net_notifier);
- unregister_inetaddr_notifier(&nes_inetaddr_notifier);
- }
-
- list_del(&nesdev->list);
- nes_destroy_cqp(nesdev);
-
- free_irq(pcidev->irq, nesdev);
- tasklet_kill(&nesdev->dpc_tasklet);
-
- spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
- if (nesdev->link_recheck) {
- spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
- cancel_delayed_work_sync(&nesdev->work);
- } else {
- spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
- }
-
- /* Deallocate the Adapter Structure */
- nes_destroy_adapter(nesdev->nesadapter);
-
- if (nesdev->msi_enabled) {
- pci_disable_msi(pcidev);
- }
-
- iounmap(nesdev->regs);
- kfree(nesdev);
-
- /* nes_debug(NES_DBG_SHUTDOWN, "calling pci_release_regions.\n"); */
- pci_release_regions(pcidev);
- pci_disable_device(pcidev);
- pci_set_drvdata(pcidev, NULL);
-}
-
-
-static ssize_t adapter_show(struct device_driver *ddp, char *buf)
-{
- unsigned int devfn = 0xffffffff;
- unsigned char bus_number = 0xff;
- unsigned int i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- devfn = nesdev->pcidev->devfn;
- bus_number = nesdev->pcidev->bus->number;
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "%x:%x\n", bus_number, devfn);
-}
-
-static ssize_t adapter_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
-
- ee_flsh_adapter = simple_strtoul(p, &p, 10);
- return strnlen(buf, count);
-}
-
-static ssize_t eeprom_cmd_show(struct device_driver *ddp, char *buf)
-{
- u32 eeprom_cmd = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- eeprom_cmd = nes_read32(nesdev->regs + NES_EEPROM_COMMAND);
- break;
- }
- i++;
- }
- return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_cmd);
-}
-
-static ssize_t eeprom_cmd_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write32(nesdev->regs + NES_EEPROM_COMMAND, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t eeprom_data_show(struct device_driver *ddp, char *buf)
-{
- u32 eeprom_data = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- eeprom_data = nes_read32(nesdev->regs + NES_EEPROM_DATA);
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_data);
-}
-
-static ssize_t eeprom_data_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write32(nesdev->regs + NES_EEPROM_DATA, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t flash_cmd_show(struct device_driver *ddp, char *buf)
-{
- u32 flash_cmd = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- flash_cmd = nes_read32(nesdev->regs + NES_FLASH_COMMAND);
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_cmd);
-}
-
-static ssize_t flash_cmd_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write32(nesdev->regs + NES_FLASH_COMMAND, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t flash_data_show(struct device_driver *ddp, char *buf)
-{
- u32 flash_data = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- flash_data = nes_read32(nesdev->regs + NES_FLASH_DATA);
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_data);
-}
-
-static ssize_t flash_data_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write32(nesdev->regs + NES_FLASH_DATA, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t nonidx_addr_show(struct device_driver *ddp, char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_nonidx_addr);
-}
-
-static ssize_t nonidx_addr_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X')
- sysfs_nonidx_addr = simple_strtoul(p, &p, 16);
-
- return strnlen(buf, count);
-}
-
-static ssize_t nonidx_data_show(struct device_driver *ddp, char *buf)
-{
- u32 nonidx_data = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nonidx_data = nes_read32(nesdev->regs + sysfs_nonidx_addr);
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%x\n", nonidx_data);
-}
-
-static ssize_t nonidx_data_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write32(nesdev->regs + sysfs_nonidx_addr, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t idx_addr_show(struct device_driver *ddp, char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_idx_addr);
-}
-
-static ssize_t idx_addr_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X')
- sysfs_idx_addr = simple_strtoul(p, &p, 16);
-
- return strnlen(buf, count);
-}
-
-static ssize_t idx_data_show(struct device_driver *ddp, char *buf)
-{
- u32 idx_data = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- idx_data = nes_read_indexed(nesdev, sysfs_idx_addr);
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%x\n", idx_data);
-}
-
-static ssize_t idx_data_store(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- char *p = (char *)buf;
- u32 val;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') {
- val = simple_strtoul(p, &p, 16);
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nes_write_indexed(nesdev, sysfs_idx_addr, val);
- break;
- }
- i++;
- }
- }
- return strnlen(buf, count);
-}
-
-static ssize_t wqm_quanta_show(struct device_driver *ddp, char *buf)
-{
- u32 wqm_quanta_value = 0xdead;
- u32 i = 0;
- struct nes_device *nesdev;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- wqm_quanta_value = nesdev->nesadapter->wqm_quanta;
- break;
- }
- i++;
- }
-
- return snprintf(buf, PAGE_SIZE, "0x%X\n", wqm_quanta_value);
-}
-
-static ssize_t wqm_quanta_store(struct device_driver *ddp, const char *buf,
- size_t count)
-{
- unsigned long wqm_quanta_value;
- u32 wqm_config1;
- u32 i = 0;
- struct nes_device *nesdev;
-
- if (kstrtoul(buf, 0, &wqm_quanta_value) < 0)
- return -EINVAL;
-
- list_for_each_entry(nesdev, &nes_dev_list, list) {
- if (i == ee_flsh_adapter) {
- nesdev->nesadapter->wqm_quanta = wqm_quanta_value;
- wqm_config1 = nes_read_indexed(nesdev,
- NES_IDX_WQM_CONFIG1);
- nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1,
- ((wqm_quanta_value << 1) |
- (wqm_config1 & 0x00000001)));
- break;
- }
- i++;
- }
- return strnlen(buf, count);
-}
-
-static DRIVER_ATTR_RW(adapter);
-static DRIVER_ATTR_RW(eeprom_cmd);
-static DRIVER_ATTR_RW(eeprom_data);
-static DRIVER_ATTR_RW(flash_cmd);
-static DRIVER_ATTR_RW(flash_data);
-static DRIVER_ATTR_RW(nonidx_addr);
-static DRIVER_ATTR_RW(nonidx_data);
-static DRIVER_ATTR_RW(idx_addr);
-static DRIVER_ATTR_RW(idx_data);
-static DRIVER_ATTR_RW(wqm_quanta);
-
-static struct attribute *nes_attrs[] = {
- &driver_attr_adapter.attr,
- &driver_attr_eeprom_cmd.attr,
- &driver_attr_eeprom_data.attr,
- &driver_attr_flash_cmd.attr,
- &driver_attr_flash_data.attr,
- &driver_attr_nonidx_addr.attr,
- &driver_attr_nonidx_data.attr,
- &driver_attr_idx_addr.attr,
- &driver_attr_idx_data.attr,
- &driver_attr_wqm_quanta.attr,
- NULL,
-};
-ATTRIBUTE_GROUPS(nes);
-
-static struct pci_driver nes_pci_driver = {
- .name = DRV_NAME,
- .id_table = nes_pci_table,
- .probe = nes_probe,
- .remove = nes_remove,
- .groups = nes_groups,
-};
-
-
-/**
- * nes_init_module - module initialization entry point
- */
-static int __init nes_init_module(void)
-{
- int retval;
-
- retval = nes_cm_start();
- if (retval) {
- printk(KERN_ERR PFX "Unable to start NetEffect iWARP CM.\n");
- return retval;
- }
- return pci_register_driver(&nes_pci_driver);
-}
-
-
-/**
- * nes_exit_module - module unload entry point
- */
-static void __exit nes_exit_module(void)
-{
- nes_cm_stop();
-
- pci_unregister_driver(&nes_pci_driver);
-}
-
-
-module_init(nes_init_module);
-module_exit(nes_exit_module);
diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h
deleted file mode 100644
index a895fe980d10..000000000000
--- a/drivers/infiniband/hw/nes/nes.h
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef __NES_H
-#define __NES_H
-
-#include <linux/netdevice.h>
-#include <linux/inetdevice.h>
-#include <linux/interrupt.h>
-#include <linux/spinlock.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <linux/dma-mapping.h>
-#include <linux/workqueue.h>
-#include <linux/slab.h>
-#include <asm/io.h>
-#include <linux/crc32c.h>
-
-#include <rdma/ib_smi.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/ib_pack.h>
-#include <rdma/rdma_cm.h>
-#include <rdma/iw_cm.h>
-#include <rdma/rdma_netlink.h>
-#include <rdma/iw_portmap.h>
-
-#define NES_SEND_FIRST_WRITE
-
-#define QUEUE_DISCONNECTS
-
-#define DRV_NAME "iw_nes"
-#define DRV_VERSION "1.5.0.1"
-#define PFX DRV_NAME ": "
-
-/*
- * NetEffect PCI vendor id and NE010 PCI device id.
- */
-#ifndef PCI_VENDOR_ID_NETEFFECT /* not in pci.ids yet */
-#define PCI_VENDOR_ID_NETEFFECT 0x1678
-#define PCI_DEVICE_ID_NETEFFECT_NE020 0x0100
-#define PCI_DEVICE_ID_NETEFFECT_NE020_KR 0x0110
-#endif
-
-#define NE020_REV 4
-#define NE020_REV1 5
-
-#define BAR_0 0
-#define BAR_1 2
-
-#define RX_BUF_SIZE (1536 + 8)
-#define NES_REG0_SIZE (4 * 1024)
-#define NES_TX_TIMEOUT (6*HZ)
-#define NES_FIRST_QPN 64
-#define NES_SW_CONTEXT_ALIGN 1024
-
-#define NES_MAX_MTU 9000
-
-#define NES_NIC_MAX_NICS 16
-#define NES_MAX_ARP_TABLE_SIZE 4096
-
-#define NES_NIC_CEQ_SIZE 8
-/* NICs will be on a separate CQ */
-#define NES_CCEQ_SIZE ((nesadapter->max_cq / nesadapter->port_count) - 32)
-
-#define NES_MAX_PORT_COUNT 4
-
-#define MAX_DPC_ITERATIONS 128
-
-#define NES_DRV_OPT_ENABLE_MPA_VER_0 0x00000001
-#define NES_DRV_OPT_DISABLE_MPA_CRC 0x00000002
-#define NES_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004
-#define NES_DRV_OPT_DISABLE_INTF 0x00000008
-#define NES_DRV_OPT_ENABLE_MSI 0x00000010
-#define NES_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020
-#define NES_DRV_OPT_SUPRESS_OPTION_BC 0x00000040
-#define NES_DRV_OPT_NO_INLINE_DATA 0x00000080
-#define NES_DRV_OPT_DISABLE_INT_MOD 0x00000100
-#define NES_DRV_OPT_DISABLE_VIRT_WQ 0x00000200
-#define NES_DRV_OPT_ENABLE_PAU 0x00000400
-
-#define NES_AEQ_EVENT_TIMEOUT 2500
-#define NES_DISCONNECT_EVENT_TIMEOUT 2000
-
-/* debug levels */
-/* must match userspace */
-#define NES_DBG_HW 0x00000001
-#define NES_DBG_INIT 0x00000002
-#define NES_DBG_ISR 0x00000004
-#define NES_DBG_PHY 0x00000008
-#define NES_DBG_NETDEV 0x00000010
-#define NES_DBG_CM 0x00000020
-#define NES_DBG_CM1 0x00000040
-#define NES_DBG_NIC_RX 0x00000080
-#define NES_DBG_NIC_TX 0x00000100
-#define NES_DBG_CQP 0x00000200
-#define NES_DBG_MMAP 0x00000400
-#define NES_DBG_MR 0x00000800
-#define NES_DBG_PD 0x00001000
-#define NES_DBG_CQ 0x00002000
-#define NES_DBG_QP 0x00004000
-#define NES_DBG_MOD_QP 0x00008000
-#define NES_DBG_AEQ 0x00010000
-#define NES_DBG_IW_RX 0x00020000
-#define NES_DBG_IW_TX 0x00040000
-#define NES_DBG_SHUTDOWN 0x00080000
-#define NES_DBG_PAU 0x00100000
-#define NES_DBG_NLMSG 0x00200000
-#define NES_DBG_RSVD1 0x10000000
-#define NES_DBG_RSVD2 0x20000000
-#define NES_DBG_RSVD3 0x40000000
-#define NES_DBG_RSVD4 0x80000000
-#define NES_DBG_ALL 0xffffffff
-
-#ifdef CONFIG_INFINIBAND_NES_DEBUG
-#define nes_debug(level, fmt, args...) \
-do { \
- if (level & nes_debug_level) \
- printk(KERN_ERR PFX "%s[%u]: " fmt, __func__, __LINE__, ##args); \
-} while (0)
-
-#define NES_EVENT_TIMEOUT 1200000
-#else
-#define nes_debug(level, fmt, args...) no_printk(fmt, ##args)
-
-#define NES_EVENT_TIMEOUT 100000
-#endif
-
-#include "nes_hw.h"
-#include "nes_verbs.h"
-#include "nes_context.h"
-#include <rdma/nes-abi.h>
-#include "nes_cm.h"
-#include "nes_mgt.h"
-
-extern int interrupt_mod_interval;
-extern int nes_if_count;
-extern int mpa_version;
-extern int disable_mpa_crc;
-extern unsigned int nes_drv_opt;
-extern unsigned int nes_debug_level;
-extern unsigned int wqm_quanta;
-extern struct list_head nes_adapter_list;
-
-extern atomic_t cm_connects;
-extern atomic_t cm_accepts;
-extern atomic_t cm_disconnects;
-extern atomic_t cm_closes;
-extern atomic_t cm_connecteds;
-extern atomic_t cm_connect_reqs;
-extern atomic_t cm_rejects;
-extern atomic_t mod_qp_timouts;
-extern atomic_t qps_created;
-extern atomic_t qps_destroyed;
-extern atomic_t sw_qps_destroyed;
-extern u32 mh_detected;
-extern u32 mh_pauses_sent;
-extern u32 cm_packets_sent;
-extern u32 cm_packets_bounced;
-extern u32 cm_packets_created;
-extern u32 cm_packets_received;
-extern u32 cm_packets_dropped;
-extern u32 cm_packets_retrans;
-extern atomic_t cm_listens_created;
-extern atomic_t cm_listens_destroyed;
-extern u32 cm_backlog_drops;
-extern atomic_t cm_loopbacks;
-extern atomic_t cm_nodes_created;
-extern atomic_t cm_nodes_destroyed;
-extern atomic_t cm_accel_dropped_pkts;
-extern atomic_t cm_resets_recvd;
-extern atomic_t pau_qps_created;
-extern atomic_t pau_qps_destroyed;
-
-extern u32 int_mod_timer_init;
-extern u32 int_mod_cq_depth_256;
-extern u32 int_mod_cq_depth_128;
-extern u32 int_mod_cq_depth_32;
-extern u32 int_mod_cq_depth_24;
-extern u32 int_mod_cq_depth_16;
-extern u32 int_mod_cq_depth_4;
-extern u32 int_mod_cq_depth_1;
-
-struct nes_device {
- struct nes_adapter *nesadapter;
- void __iomem *regs;
- void __iomem *index_reg;
- struct pci_dev *pcidev;
- struct net_device *netdev[NES_NIC_MAX_NICS];
- u64 link_status_interrupts;
- struct tasklet_struct dpc_tasklet;
- spinlock_t indexed_regs_lock;
- unsigned long csr_start;
- unsigned long doorbell_region;
- unsigned long doorbell_start;
- unsigned long mac_tx_errors;
- unsigned long mac_pause_frames_sent;
- unsigned long mac_pause_frames_received;
- unsigned long mac_rx_errors;
- unsigned long mac_rx_crc_errors;
- unsigned long mac_rx_symbol_err_frames;
- unsigned long mac_rx_jabber_frames;
- unsigned long mac_rx_oversized_frames;
- unsigned long mac_rx_short_frames;
- unsigned long port_rx_discards;
- unsigned long port_tx_discards;
- unsigned int mac_index;
- unsigned int nes_stack_start;
-
- /* Control Structures */
- void *cqp_vbase;
- dma_addr_t cqp_pbase;
- u32 cqp_mem_size;
- u8 ceq_index;
- u8 nic_ceq_index;
- struct nes_hw_cqp cqp;
- struct nes_hw_cq ccq;
- struct list_head cqp_avail_reqs;
- struct list_head cqp_pending_reqs;
- struct nes_cqp_request *nes_cqp_requests;
-
- u32 int_req;
- u32 int_stat;
- u32 timer_int_req;
- u32 timer_only_int_count;
- u32 intf_int_req;
- u32 last_mac_tx_pauses;
- u32 last_used_chunks_tx;
- struct list_head list;
-
- u16 base_doorbell_index;
- u16 currcq_count;
- u16 deepcq_count;
- u8 iw_status;
- u8 msi_enabled;
- u8 netdev_count;
- u8 napi_isr_ran;
- u8 disable_rx_flow_control;
- u8 disable_tx_flow_control;
-
- struct delayed_work work;
- u8 link_recheck;
-};
-
-/* Receive skb private area - must fit in skb->cb area */
-struct nes_rskb_cb {
- u64 busaddr;
- u32 maplen;
- u32 seqnum;
- u8 *data_start;
- struct nes_qp *nesqp;
-};
-
-static inline __le32 get_crc_value(struct nes_v4_quad *nes_quad)
-{
- u32 crc_value;
- crc_value = crc32c(~0, (void *)nes_quad, sizeof (struct nes_v4_quad));
-
- /*
- * With commit ef19454b ("[LIB] crc32c: Keep intermediate crc
- * state in cpu order"), behavior of crc32c changes on
- * big-endian platforms. Our algorithm expects the previous
- * behavior; otherwise we have RDMA connection establishment
- * issue on big-endian.
- */
- return cpu_to_le32(crc_value);
-}
-
-static inline void
-set_wqe_64bit_value(__le32 *wqe_words, u32 index, u64 value)
-{
- wqe_words[index] = cpu_to_le32((u32) value);
- wqe_words[index + 1] = cpu_to_le32(upper_32_bits(value));
-}
-
-static inline void
-set_wqe_32bit_value(__le32 *wqe_words, u32 index, u32 value)
-{
- wqe_words[index] = cpu_to_le32(value);
-}
-
-static inline void
-nes_fill_init_cqp_wqe(struct nes_hw_cqp_wqe *cqp_wqe, struct nes_device *nesdev)
-{
- cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_LOW_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_LEN_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_LOW_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_LOW_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_HIGH_IDX] = 0;
-}
-
-static inline void
-nes_fill_init_qp_wqe(struct nes_hw_qp_wqe *wqe, struct nes_qp *nesqp, u32 head)
-{
- u32 value;
- value = ((u32)((unsigned long) nesqp)) | head;
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX,
- (u32)(upper_32_bits((unsigned long)(nesqp))));
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, value);
-}
-
-/* Read from memory-mapped device */
-static inline u32 nes_read_indexed(struct nes_device *nesdev, u32 reg_index)
-{
- unsigned long flags;
- void __iomem *addr = nesdev->index_reg;
- u32 value;
-
- spin_lock_irqsave(&nesdev->indexed_regs_lock, flags);
-
- writel(reg_index, addr);
- value = readl((void __iomem *)addr + 4);
-
- spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags);
- return value;
-}
-
-static inline u32 nes_read32(const void __iomem *addr)
-{
- return readl(addr);
-}
-
-static inline u16 nes_read16(const void __iomem *addr)
-{
- return readw(addr);
-}
-
-static inline u8 nes_read8(const void __iomem *addr)
-{
- return readb(addr);
-}
-
-/* Write to memory-mapped device */
-static inline void nes_write_indexed(struct nes_device *nesdev, u32 reg_index, u32 val)
-{
- unsigned long flags;
- void __iomem *addr = nesdev->index_reg;
-
- spin_lock_irqsave(&nesdev->indexed_regs_lock, flags);
-
- writel(reg_index, addr);
- writel(val, (void __iomem *)addr + 4);
-
- spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags);
-}
-
-static inline void nes_write32(void __iomem *addr, u32 val)
-{
- writel(val, addr);
-}
-
-static inline void nes_write16(void __iomem *addr, u16 val)
-{
- writew(val, addr);
-}
-
-static inline void nes_write8(void __iomem *addr, u8 val)
-{
- writeb(val, addr);
-}
-
-enum nes_resource {
- NES_RESOURCE_MW = 1,
- NES_RESOURCE_FAST_MR,
- NES_RESOURCE_PHYS_MR,
- NES_RESOURCE_USER_MR,
- NES_RESOURCE_PD,
- NES_RESOURCE_QP,
- NES_RESOURCE_CQ,
- NES_RESOURCE_ARP
-};
-
-static inline int nes_alloc_resource(struct nes_adapter *nesadapter,
- unsigned long *resource_array, u32 max_resources,
- u32 *req_resource_num, u32 *next, enum nes_resource resource_type)
-{
- unsigned long flags;
- u32 resource_num;
-
- spin_lock_irqsave(&nesadapter->resource_lock, flags);
-
- resource_num = find_next_zero_bit(resource_array, max_resources, *next);
- if (resource_num >= max_resources) {
- resource_num = find_first_zero_bit(resource_array, max_resources);
- if (resource_num >= max_resources) {
- printk(KERN_ERR PFX "%s: No available resources [type=%u].\n", __func__, resource_type);
- spin_unlock_irqrestore(&nesadapter->resource_lock, flags);
- return -EMFILE;
- }
- }
- set_bit(resource_num, resource_array);
- *next = resource_num+1;
- if (*next == max_resources) {
- *next = 0;
- }
- spin_unlock_irqrestore(&nesadapter->resource_lock, flags);
- *req_resource_num = resource_num;
-
- return 0;
-}
-
-static inline int nes_is_resource_allocated(struct nes_adapter *nesadapter,
- unsigned long *resource_array, u32 resource_num)
-{
- unsigned long flags;
- int bit_is_set;
-
- spin_lock_irqsave(&nesadapter->resource_lock, flags);
-
- bit_is_set = test_bit(resource_num, resource_array);
- nes_debug(NES_DBG_HW, "resource_num %u is%s allocated.\n",
- resource_num, (bit_is_set ? "": " not"));
- spin_unlock_irqrestore(&nesadapter->resource_lock, flags);
-
- return bit_is_set;
-}
-
-static inline void nes_free_resource(struct nes_adapter *nesadapter,
- unsigned long *resource_array, u32 resource_num)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&nesadapter->resource_lock, flags);
- clear_bit(resource_num, resource_array);
- spin_unlock_irqrestore(&nesadapter->resource_lock, flags);
-}
-
-static inline struct nes_vnic *to_nesvnic(struct ib_device *ibdev)
-{
- return container_of(ibdev, struct nes_ib_device, ibdev)->nesvnic;
-}
-
-static inline struct nes_pd *to_nespd(struct ib_pd *ibpd)
-{
- return container_of(ibpd, struct nes_pd, ibpd);
-}
-
-static inline struct nes_ucontext *to_nesucontext(struct ib_ucontext *ibucontext)
-{
- return container_of(ibucontext, struct nes_ucontext, ibucontext);
-}
-
-static inline struct nes_mr *to_nesmr(struct ib_mr *ibmr)
-{
- return container_of(ibmr, struct nes_mr, ibmr);
-}
-
-static inline struct nes_mr *to_nesmr_from_ibfmr(struct ib_fmr *ibfmr)
-{
- return container_of(ibfmr, struct nes_mr, ibfmr);
-}
-
-static inline struct nes_mr *to_nesmw(struct ib_mw *ibmw)
-{
- return container_of(ibmw, struct nes_mr, ibmw);
-}
-
-static inline struct nes_fmr *to_nesfmr(struct nes_mr *nesmr)
-{
- return container_of(nesmr, struct nes_fmr, nesmr);
-}
-
-static inline struct nes_cq *to_nescq(struct ib_cq *ibcq)
-{
- return container_of(ibcq, struct nes_cq, ibcq);
-}
-
-static inline struct nes_qp *to_nesqp(struct ib_qp *ibqp)
-{
- return container_of(ibqp, struct nes_qp, ibqp);
-}
-
-
-
-/* nes.c */
-void nes_add_ref(struct ib_qp *);
-void nes_rem_ref(struct ib_qp *);
-struct ib_qp *nes_get_qp(struct ib_device *, int);
-
-
-/* nes_hw.c */
-struct nes_adapter *nes_init_adapter(struct nes_device *, u8);
-void nes_nic_init_timer_defaults(struct nes_device *, u8);
-void nes_destroy_adapter(struct nes_adapter *);
-int nes_init_cqp(struct nes_device *);
-int nes_init_phy(struct nes_device *);
-int nes_init_nic_qp(struct nes_device *, struct net_device *);
-void nes_destroy_nic_qp(struct nes_vnic *);
-int nes_napi_isr(struct nes_device *);
-void nes_dpc(unsigned long);
-void nes_nic_ce_handler(struct nes_device *, struct nes_hw_nic_cq *);
-void nes_iwarp_ce_handler(struct nes_device *, struct nes_hw_cq *);
-int nes_destroy_cqp(struct nes_device *);
-int nes_nic_cm_xmit(struct sk_buff *, struct net_device *);
-void nes_recheck_link_status(struct work_struct *work);
-void nes_terminate_timeout(struct timer_list *t);
-
-/* nes_nic.c */
-struct net_device *nes_netdev_init(struct nes_device *, void __iomem *);
-void nes_netdev_destroy(struct net_device *);
-int nes_nic_cm_xmit(struct sk_buff *, struct net_device *);
-
-/* nes_cm.c */
-void *nes_cm_create(struct net_device *);
-int nes_cm_recv(struct sk_buff *, struct net_device *);
-void nes_update_arp(unsigned char *, u32, u32, u16, u16);
-void nes_manage_arp_cache(struct net_device *, unsigned char *, u32, u32);
-void nes_sock_release(struct nes_qp *, unsigned long *);
-void flush_wqes(struct nes_device *nesdev, struct nes_qp *, u32, u32);
-int nes_manage_apbvt(struct nes_vnic *, u32, u32, u32);
-int nes_cm_disconn(struct nes_qp *);
-void nes_cm_disconn_worker(void *);
-
-/* nes_verbs.c */
-int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32, u32);
-int nes_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *);
-struct nes_ib_device *nes_init_ofa_device(struct net_device *);
-void nes_port_ibevent(struct nes_vnic *nesvnic);
-void nes_destroy_ofa_device(struct nes_ib_device *);
-int nes_register_ofa_device(struct nes_ib_device *);
-
-/* nes_util.c */
-int nes_read_eeprom_values(struct nes_device *, struct nes_adapter *);
-void nes_write_1G_phy_reg(struct nes_device *, u8, u8, u16);
-void nes_read_1G_phy_reg(struct nes_device *, u8, u8, u16 *);
-void nes_write_10G_phy_reg(struct nes_device *, u16, u8, u16, u16);
-void nes_read_10G_phy_reg(struct nes_device *, u8, u8, u16);
-struct nes_cqp_request *nes_get_cqp_request(struct nes_device *);
-void nes_free_cqp_request(struct nes_device *nesdev,
- struct nes_cqp_request *cqp_request);
-void nes_put_cqp_request(struct nes_device *nesdev,
- struct nes_cqp_request *cqp_request);
-void nes_post_cqp_request(struct nes_device *, struct nes_cqp_request *);
-int nes_arp_table(struct nes_device *, u32, u8 *, u32);
-void nes_mh_fix(struct timer_list *t);
-void nes_clc(struct timer_list *t);
-void nes_dump_mem(unsigned int, void *, int);
-u32 nes_crc32(u32, u32, u32, u32, u8 *, u32, u32, u32);
-
-#endif /* __NES_H */
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
deleted file mode 100644
index 62bf986eba67..000000000000
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ /dev/null
@@ -1,3992 +0,0 @@
-/*
- * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-
-#define TCPOPT_TIMESTAMP 8
-
-#include <linux/atomic.h>
-#include <linux/skbuff.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/init.h>
-#include <linux/if_arp.h>
-#include <linux/if_vlan.h>
-#include <linux/notifier.h>
-#include <linux/net.h>
-#include <linux/types.h>
-#include <linux/timer.h>
-#include <linux/time.h>
-#include <linux/delay.h>
-#include <linux/etherdevice.h>
-#include <linux/netdevice.h>
-#include <linux/random.h>
-#include <linux/list.h>
-#include <linux/threads.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>
-#include <net/arp.h>
-#include <net/neighbour.h>
-#include <net/route.h>
-#include <net/ip_fib.h>
-#include <net/secure_seq.h>
-#include <net/tcp.h>
-#include <linux/fcntl.h>
-
-#include "nes.h"
-
-u32 cm_packets_sent;
-u32 cm_packets_bounced;
-u32 cm_packets_dropped;
-u32 cm_packets_retrans;
-u32 cm_packets_created;
-u32 cm_packets_received;
-atomic_t cm_listens_created;
-atomic_t cm_listens_destroyed;
-u32 cm_backlog_drops;
-atomic_t cm_loopbacks;
-atomic_t cm_nodes_created;
-atomic_t cm_nodes_destroyed;
-atomic_t cm_accel_dropped_pkts;
-atomic_t cm_resets_recvd;
-
-static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *);
-static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *);
-static int mini_cm_del_listen(struct nes_cm_core *, struct nes_cm_listener *);
-static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *, struct nes_vnic *, u16, void *, struct nes_cm_info *);
-static int mini_cm_close(struct nes_cm_core *, struct nes_cm_node *);
-static int mini_cm_accept(struct nes_cm_core *, struct nes_cm_node *);
-static int mini_cm_reject(struct nes_cm_core *, struct nes_cm_node *);
-static int mini_cm_recv_pkt(struct nes_cm_core *, struct nes_vnic *, struct sk_buff *);
-static int mini_cm_dealloc_core(struct nes_cm_core *);
-static int mini_cm_get(struct nes_cm_core *);
-static int mini_cm_set(struct nes_cm_core *, u32, u32);
-
-static void form_cm_frame(struct sk_buff *, struct nes_cm_node *, void *, u32, void *, u32, u8);
-static int add_ref_cm_node(struct nes_cm_node *);
-static int rem_ref_cm_node(struct nes_cm_core *, struct nes_cm_node *);
-
-static int nes_cm_disconn_true(struct nes_qp *);
-static int nes_cm_post_event(struct nes_cm_event *event);
-static int nes_disconnect(struct nes_qp *nesqp, int abrupt);
-static void nes_disconnect_worker(struct work_struct *work);
-
-static int send_mpa_request(struct nes_cm_node *, struct sk_buff *);
-static int send_mpa_reject(struct nes_cm_node *);
-static int send_syn(struct nes_cm_node *, u32, struct sk_buff *);
-static int send_reset(struct nes_cm_node *, struct sk_buff *);
-static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb);
-static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb);
-static void process_packet(struct nes_cm_node *, struct sk_buff *, struct nes_cm_core *);
-
-static void active_open_err(struct nes_cm_node *, struct sk_buff *, int);
-static void passive_open_err(struct nes_cm_node *, struct sk_buff *, int);
-static void cleanup_retrans_entry(struct nes_cm_node *);
-static void handle_rcv_mpa(struct nes_cm_node *, struct sk_buff *);
-static void free_retrans_entry(struct nes_cm_node *cm_node);
-static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, struct sk_buff *skb, int optionsize, int passive);
-
-/* CM event handler functions */
-static void cm_event_connected(struct nes_cm_event *);
-static void cm_event_connect_error(struct nes_cm_event *);
-static void cm_event_reset(struct nes_cm_event *);
-static void cm_event_mpa_req(struct nes_cm_event *);
-static void cm_event_mpa_reject(struct nes_cm_event *);
-static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node);
-
-/* MPA build functions */
-static int cm_build_mpa_frame(struct nes_cm_node *, u8 **, u16 *, u8 *, u8);
-static void build_mpa_v2(struct nes_cm_node *, void *, u8);
-static void build_mpa_v1(struct nes_cm_node *, void *, u8);
-static void build_rdma0_msg(struct nes_cm_node *, struct nes_qp **);
-
-static void print_core(struct nes_cm_core *core);
-static void record_ird_ord(struct nes_cm_node *, u16, u16);
-
-/* External CM API Interface */
-/* instance of function pointers for client API */
-/* set address of this instance to cm_core->cm_ops at cm_core alloc */
-static const struct nes_cm_ops nes_cm_api = {
- .accelerated = mini_cm_accelerated,
- .listen = mini_cm_listen,
- .stop_listener = mini_cm_del_listen,
- .connect = mini_cm_connect,
- .close = mini_cm_close,
- .accept = mini_cm_accept,
- .reject = mini_cm_reject,
- .recv_pkt = mini_cm_recv_pkt,
- .destroy_cm_core = mini_cm_dealloc_core,
- .get = mini_cm_get,
- .set = mini_cm_set
-};
-
-static struct nes_cm_core *g_cm_core;
-
-atomic_t cm_connects;
-atomic_t cm_accepts;
-atomic_t cm_disconnects;
-atomic_t cm_closes;
-atomic_t cm_connecteds;
-atomic_t cm_connect_reqs;
-atomic_t cm_rejects;
-
-int nes_add_ref_cm_node(struct nes_cm_node *cm_node)
-{
- return add_ref_cm_node(cm_node);
-}
-
-int nes_rem_ref_cm_node(struct nes_cm_node *cm_node)
-{
- return rem_ref_cm_node(cm_node->cm_core, cm_node);
-}
-/**
- * create_event
- */
-static struct nes_cm_event *create_event(struct nes_cm_node * cm_node,
- enum nes_cm_event_type type)
-{
- struct nes_cm_event *event;
-
- if (!cm_node->cm_id)
- return NULL;
-
- /* allocate an empty event */
- event = kzalloc(sizeof(*event), GFP_ATOMIC);
-
- if (!event)
- return NULL;
-
- event->type = type;
- event->cm_node = cm_node;
- event->cm_info.rem_addr = cm_node->rem_addr;
- event->cm_info.loc_addr = cm_node->loc_addr;
- event->cm_info.rem_port = cm_node->rem_port;
- event->cm_info.loc_port = cm_node->loc_port;
- event->cm_info.cm_id = cm_node->cm_id;
-
- nes_debug(NES_DBG_CM, "cm_node=%p Created event=%p, type=%u, "
- "dst_addr=%08x[%x], src_addr=%08x[%x]\n",
- cm_node, event, type, event->cm_info.loc_addr,
- event->cm_info.loc_port, event->cm_info.rem_addr,
- event->cm_info.rem_port);
-
- nes_cm_post_event(event);
- return event;
-}
-
-
-/**
- * send_mpa_request
- */
-static int send_mpa_request(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- u8 start_addr = 0;
- u8 *start_ptr = &start_addr;
- u8 **start_buff = &start_ptr;
- u16 buff_len = 0;
-
- if (!skb) {
- nes_debug(NES_DBG_CM, "skb set to NULL\n");
- return -1;
- }
-
- /* send an MPA Request frame */
- cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REQUEST);
- form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK);
-
- return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
-}
-
-
-
-static int send_mpa_reject(struct nes_cm_node *cm_node)
-{
- struct sk_buff *skb = NULL;
- u8 start_addr = 0;
- u8 *start_ptr = &start_addr;
- u8 **start_buff = &start_ptr;
- u16 buff_len = 0;
- struct ietf_mpa_v1 *mpa_frame;
-
- skb = dev_alloc_skb(MAX_CM_BUFFER);
- if (!skb) {
- nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n");
- return -ENOMEM;
- }
-
- /* send an MPA reject frame */
- cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REPLY);
- mpa_frame = (struct ietf_mpa_v1 *)*start_buff;
- mpa_frame->flags |= IETF_MPA_FLAGS_REJECT;
- form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK | SET_FIN);
-
- cm_node->state = NES_CM_STATE_FIN_WAIT1;
- return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
-}
-
-
-/**
- * recv_mpa - process a received TCP pkt, we are expecting an
- * IETF MPA frame
- */
-static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type,
- u32 len)
-{
- struct ietf_mpa_v1 *mpa_frame;
- struct ietf_mpa_v2 *mpa_v2_frame;
- struct ietf_rtr_msg *rtr_msg;
- int mpa_hdr_len;
- int priv_data_len;
-
- *type = NES_MPA_REQUEST_ACCEPT;
-
- /* assume req frame is in tcp data payload */
- if (len < sizeof(struct ietf_mpa_v1)) {
- nes_debug(NES_DBG_CM, "The received ietf buffer was too small (%x)\n", len);
- return -EINVAL;
- }
-
- /* points to the beginning of the frame, which could be MPA V1 or V2 */
- mpa_frame = (struct ietf_mpa_v1 *)buffer;
- mpa_hdr_len = sizeof(struct ietf_mpa_v1);
- priv_data_len = ntohs(mpa_frame->priv_data_len);
-
- /* make sure mpa private data len is less than 512 bytes */
- if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
- nes_debug(NES_DBG_CM, "The received Length of Private"
- " Data field exceeds 512 octets\n");
- return -EINVAL;
- }
- /*
- * make sure MPA receiver interoperate with the
- * received MPA version and MPA key information
- *
- */
- if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
- nes_debug(NES_DBG_CM, "The received mpa version"
- " is not supported\n");
- return -EINVAL;
- }
- /*
- * backwards compatibility only
- */
- if (mpa_frame->rev > cm_node->mpa_frame_rev) {
- nes_debug(NES_DBG_CM, "The received mpa version"
- " can not be interoperated\n");
- return -EINVAL;
- } else {
- cm_node->mpa_frame_rev = mpa_frame->rev;
- }
-
- if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) {
- if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) {
- nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n");
- return -EINVAL;
- }
- } else {
- if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) {
- nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n");
- return -EINVAL;
- }
- }
-
- if (priv_data_len + mpa_hdr_len != len) {
- nes_debug(NES_DBG_CM, "The received ietf buffer was not right"
- " complete (%x + %x != %x)\n",
- priv_data_len, mpa_hdr_len, len);
- return -EINVAL;
- }
- /* make sure it does not exceed the max size */
- if (len > MAX_CM_BUFFER) {
- nes_debug(NES_DBG_CM, "The received ietf buffer was too large"
- " (%x + %x != %x)\n",
- priv_data_len, mpa_hdr_len, len);
- return -EINVAL;
- }
-
- cm_node->mpa_frame_size = priv_data_len;
-
- switch (mpa_frame->rev) {
- case IETF_MPA_V2: {
- u16 ird_size;
- u16 ord_size;
- u16 rtr_ctrl_ird;
- u16 rtr_ctrl_ord;
-
- mpa_v2_frame = (struct ietf_mpa_v2 *)buffer;
- mpa_hdr_len += IETF_RTR_MSG_SIZE;
- cm_node->mpa_frame_size -= IETF_RTR_MSG_SIZE;
- rtr_msg = &mpa_v2_frame->rtr_msg;
-
- /* parse rtr message */
- rtr_ctrl_ird = ntohs(rtr_msg->ctrl_ird);
- rtr_ctrl_ord = ntohs(rtr_msg->ctrl_ord);
- ird_size = rtr_ctrl_ird & IETF_NO_IRD_ORD;
- ord_size = rtr_ctrl_ord & IETF_NO_IRD_ORD;
-
- if (!(rtr_ctrl_ird & IETF_PEER_TO_PEER)) {
- /* send reset */
- return -EINVAL;
- }
- if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD)
- cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
-
- if (cm_node->mpav2_ird_ord != IETF_NO_IRD_ORD) {
- /* responder */
- if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) {
- /* we are still negotiating */
- if (ord_size > NES_MAX_IRD) {
- cm_node->ird_size = NES_MAX_IRD;
- } else {
- cm_node->ird_size = ord_size;
- if (ord_size == 0 &&
- (rtr_ctrl_ord & IETF_RDMA0_READ)) {
- cm_node->ird_size = 1;
- nes_debug(NES_DBG_CM,
- "%s: Remote peer doesn't support RDMA0_READ (ord=%u)\n",
- __func__, ord_size);
- }
- }
- if (ird_size > NES_MAX_ORD)
- cm_node->ord_size = NES_MAX_ORD;
- else
- cm_node->ord_size = ird_size;
- } else { /* initiator */
- if (ord_size > NES_MAX_IRD) {
- nes_debug(NES_DBG_CM,
- "%s: Unable to support the requested (ord =%u)\n",
- __func__, ord_size);
- return -EINVAL;
- }
- cm_node->ird_size = ord_size;
-
- if (ird_size > NES_MAX_ORD) {
- cm_node->ord_size = NES_MAX_ORD;
- } else {
- if (ird_size == 0 &&
- (rtr_ctrl_ord & IETF_RDMA0_READ)) {
- nes_debug(NES_DBG_CM,
- "%s: Remote peer doesn't support RDMA0_READ (ird=%u)\n",
- __func__, ird_size);
- return -EINVAL;
- } else {
- cm_node->ord_size = ird_size;
- }
- }
- }
- }
-
- if (rtr_ctrl_ord & IETF_RDMA0_READ) {
- cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
-
- } else if (rtr_ctrl_ord & IETF_RDMA0_WRITE) {
- cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
- } else { /* Not supported RDMA0 operation */
- return -EINVAL;
- }
- break;
- }
- case IETF_MPA_V1:
- default:
- break;
- }
-
- /* copy entire MPA frame to our cm_node's frame */
- memcpy(cm_node->mpa_frame_buf, buffer + mpa_hdr_len, cm_node->mpa_frame_size);
-
- if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
- *type = NES_MPA_REQUEST_REJECT;
- return 0;
-}
-
-
-/**
- * form_cm_frame - get a free packet and build empty frame Use
- * node info to build.
- */
-static void form_cm_frame(struct sk_buff *skb,
- struct nes_cm_node *cm_node, void *options, u32 optionsize,
- void *data, u32 datasize, u8 flags)
-{
- struct tcphdr *tcph;
- struct iphdr *iph;
- struct ethhdr *ethh;
- u8 *buf;
- u16 packetsize = sizeof(*iph);
-
- packetsize += sizeof(*tcph);
- packetsize += optionsize + datasize;
-
- skb_trim(skb, 0);
- memset(skb->data, 0x00, ETH_HLEN + sizeof(*iph) + sizeof(*tcph));
-
- buf = skb_put(skb, packetsize + ETH_HLEN);
-
- ethh = (struct ethhdr *)buf;
- buf += ETH_HLEN;
-
- iph = (struct iphdr *)buf;
- buf += sizeof(*iph);
- tcph = (struct tcphdr *)buf;
- skb_reset_mac_header(skb);
- skb_set_network_header(skb, ETH_HLEN);
- skb_set_transport_header(skb, ETH_HLEN + sizeof(*iph));
- buf += sizeof(*tcph);
-
- skb->ip_summed = CHECKSUM_PARTIAL;
- if (!(cm_node->netdev->features & NETIF_F_IP_CSUM))
- skb->ip_summed = CHECKSUM_NONE;
- skb->protocol = htons(0x800);
- skb->data_len = 0;
- skb->mac_len = ETH_HLEN;
-
- memcpy(ethh->h_dest, cm_node->rem_mac, ETH_ALEN);
- memcpy(ethh->h_source, cm_node->loc_mac, ETH_ALEN);
- ethh->h_proto = htons(0x0800);
-
- iph->version = IPVERSION;
- iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
- iph->tos = 0;
- iph->tot_len = htons(packetsize);
- iph->id = htons(++cm_node->tcp_cntxt.loc_id);
-
- iph->frag_off = htons(0x4000);
- iph->ttl = 0x40;
- iph->protocol = 0x06; /* IPPROTO_TCP */
-
- iph->saddr = htonl(cm_node->loc_addr);
- iph->daddr = htonl(cm_node->rem_addr);
-
- tcph->source = htons(cm_node->loc_port);
- tcph->dest = htons(cm_node->rem_port);
- tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
-
- if (flags & SET_ACK) {
- cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
- tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
- tcph->ack = 1;
- } else {
- tcph->ack_seq = 0;
- }
-
- if (flags & SET_SYN) {
- cm_node->tcp_cntxt.loc_seq_num++;
- tcph->syn = 1;
- } else {
- cm_node->tcp_cntxt.loc_seq_num += datasize;
- }
-
- if (flags & SET_FIN) {
- cm_node->tcp_cntxt.loc_seq_num++;
- tcph->fin = 1;
- }
-
- if (flags & SET_RST)
- tcph->rst = 1;
-
- tcph->doff = (u16)((sizeof(*tcph) + optionsize + 3) >> 2);
- tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
- tcph->urg_ptr = 0;
- if (optionsize)
- memcpy(buf, options, optionsize);
- buf += optionsize;
- if (datasize)
- memcpy(buf, data, datasize);
-
- skb_shinfo(skb)->nr_frags = 0;
- cm_packets_created++;
-}
-
-/**
- * print_core - dump a cm core
- */
-static void print_core(struct nes_cm_core *core)
-{
- nes_debug(NES_DBG_CM, "---------------------------------------------\n");
- nes_debug(NES_DBG_CM, "CM Core -- (core = %p )\n", core);
- if (!core)
- return;
- nes_debug(NES_DBG_CM, "---------------------------------------------\n");
-
- nes_debug(NES_DBG_CM, "State : %u \n", core->state);
-
- nes_debug(NES_DBG_CM, "Listen Nodes : %u \n", atomic_read(&core->listen_node_cnt));
- nes_debug(NES_DBG_CM, "Active Nodes : %u \n", atomic_read(&core->node_cnt));
-
- nes_debug(NES_DBG_CM, "core : %p \n", core);
-
- nes_debug(NES_DBG_CM, "-------------- end core ---------------\n");
-}
-
-static void record_ird_ord(struct nes_cm_node *cm_node,
- u16 conn_ird, u16 conn_ord)
-{
- if (conn_ird > NES_MAX_IRD)
- conn_ird = NES_MAX_IRD;
-
- if (conn_ord > NES_MAX_ORD)
- conn_ord = NES_MAX_ORD;
-
- cm_node->ird_size = conn_ird;
- cm_node->ord_size = conn_ord;
-}
-
-/**
- * cm_build_mpa_frame - build a MPA V1 frame or MPA V2 frame
- */
-static int cm_build_mpa_frame(struct nes_cm_node *cm_node, u8 **start_buff,
- u16 *buff_len, u8 *pci_mem, u8 mpa_key)
-{
- int ret = 0;
-
- *start_buff = (pci_mem) ? pci_mem : &cm_node->mpa_frame_buf[0];
-
- switch (cm_node->mpa_frame_rev) {
- case IETF_MPA_V1:
- *start_buff = (u8 *)*start_buff + sizeof(struct ietf_rtr_msg);
- *buff_len = sizeof(struct ietf_mpa_v1) + cm_node->mpa_frame_size;
- build_mpa_v1(cm_node, *start_buff, mpa_key);
- break;
- case IETF_MPA_V2:
- *buff_len = sizeof(struct ietf_mpa_v2) + cm_node->mpa_frame_size;
- build_mpa_v2(cm_node, *start_buff, mpa_key);
- break;
- default:
- ret = -EINVAL;
- }
- return ret;
-}
-
-/**
- * build_mpa_v2 - build a MPA V2 frame
- */
-static void build_mpa_v2(struct nes_cm_node *cm_node,
- void *start_addr, u8 mpa_key)
-{
- struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr;
- struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
- u16 ctrl_ird;
- u16 ctrl_ord;
-
- /* initialize the upper 5 bytes of the frame */
- build_mpa_v1(cm_node, start_addr, mpa_key);
- mpa_frame->flags |= IETF_MPA_V2_FLAG; /* set a bit to indicate MPA V2 */
- mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE);
-
- /* initialize RTR msg */
- if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
- ctrl_ird = IETF_NO_IRD_ORD;
- ctrl_ord = IETF_NO_IRD_ORD;
- } else {
- ctrl_ird = cm_node->ird_size & IETF_NO_IRD_ORD;
- ctrl_ord = cm_node->ord_size & IETF_NO_IRD_ORD;
- }
- ctrl_ird |= IETF_PEER_TO_PEER;
-
- switch (mpa_key) {
- case MPA_KEY_REQUEST:
- ctrl_ord |= IETF_RDMA0_WRITE;
- ctrl_ord |= IETF_RDMA0_READ;
- break;
- case MPA_KEY_REPLY:
- switch (cm_node->send_rdma0_op) {
- case SEND_RDMA_WRITE_ZERO:
- ctrl_ord |= IETF_RDMA0_WRITE;
- break;
- case SEND_RDMA_READ_ZERO:
- ctrl_ord |= IETF_RDMA0_READ;
- break;
- }
- }
- rtr_msg->ctrl_ird = htons(ctrl_ird);
- rtr_msg->ctrl_ord = htons(ctrl_ord);
-}
-
-/**
- * build_mpa_v1 - build a MPA V1 frame
- */
-static void build_mpa_v1(struct nes_cm_node *cm_node, void *start_addr, u8 mpa_key)
-{
- struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr;
-
- switch (mpa_key) {
- case MPA_KEY_REQUEST:
- memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
- break;
- case MPA_KEY_REPLY:
- memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
- break;
- }
- mpa_frame->flags = IETF_MPA_FLAGS_CRC;
- mpa_frame->rev = cm_node->mpa_frame_rev;
- mpa_frame->priv_data_len = htons(cm_node->mpa_frame_size);
-}
-
-static void build_rdma0_msg(struct nes_cm_node *cm_node, struct nes_qp **nesqp_addr)
-{
- u64 u64temp;
- struct nes_qp *nesqp = *nesqp_addr;
- struct nes_hw_qp_wqe *wqe = &nesqp->hwqp.sq_vbase[0];
-
- u64temp = (unsigned long)nesqp->nesuqp_addr;
- u64temp |= NES_SW_CONTEXT_ALIGN >> 1;
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, u64temp);
-
- wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = 0;
- wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = 0;
-
- switch (cm_node->send_rdma0_op) {
- case SEND_RDMA_WRITE_ZERO:
- nes_debug(NES_DBG_CM, "Sending first write.\n");
- wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] =
- cpu_to_le32(NES_IWARP_SQ_OP_RDMAW);
- wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = 0;
- wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = 0;
- wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 0;
- break;
-
- case SEND_RDMA_READ_ZERO:
- default:
- if (cm_node->send_rdma0_op != SEND_RDMA_READ_ZERO)
- WARN(1, "Unsupported RDMA0 len operation=%u\n",
- cm_node->send_rdma0_op);
- nes_debug(NES_DBG_CM, "Sending first rdma operation.\n");
- wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] =
- cpu_to_le32(NES_IWARP_SQ_OP_RDMAR);
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = 1;
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = 0;
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 0;
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = 1;
- wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 1;
- break;
- }
-
- if (nesqp->sq_kmapped) {
- nesqp->sq_kmapped = 0;
- kunmap(nesqp->page);
- }
-
- /*use the reserved spot on the WQ for the extra first WQE*/
- nesqp->nesqp_context->ird_ord_sizes &= cpu_to_le32(~(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT |
- NES_QPCONTEXT_ORDIRD_WRPDU |
- NES_QPCONTEXT_ORDIRD_ALSMM));
- nesqp->skip_lsmm = 1;
- nesqp->hwqp.sq_tail = 0;
-}
-
-/**
- * schedule_nes_timer
- * note - cm_node needs to be protected before calling this. Encase in:
- * rem_ref_cm_node(cm_core, cm_node);add_ref_cm_node(cm_node);
- */
-int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
- enum nes_timer_type type, int send_retrans,
- int close_when_complete)
-{
- unsigned long flags;
- struct nes_cm_core *cm_core = cm_node->cm_core;
- struct nes_timer_entry *new_send;
- int ret = 0;
-
- new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
- if (!new_send)
- return -ENOMEM;
-
- /* new_send->timetosend = currenttime */
- new_send->retrycount = NES_DEFAULT_RETRYS;
- new_send->retranscount = NES_DEFAULT_RETRANS;
- new_send->skb = skb;
- new_send->timetosend = jiffies;
- new_send->type = type;
- new_send->netdev = cm_node->netdev;
- new_send->send_retrans = send_retrans;
- new_send->close_when_complete = close_when_complete;
-
- if (type == NES_TIMER_TYPE_CLOSE) {
- new_send->timetosend += (HZ / 10);
- if (cm_node->recv_entry) {
- kfree(new_send);
- WARN_ON(1);
- return -EINVAL;
- }
- cm_node->recv_entry = new_send;
- }
-
- if (type == NES_TIMER_TYPE_SEND) {
- new_send->seq_num = ntohl(tcp_hdr(skb)->seq);
- refcount_inc(&new_send->skb->users);
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- cm_node->send_entry = new_send;
- add_ref_cm_node(cm_node);
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
- new_send->timetosend = jiffies + NES_RETRY_TIMEOUT;
-
- ret = nes_nic_cm_xmit(new_send->skb, cm_node->netdev);
- if (ret != NETDEV_TX_OK) {
- nes_debug(NES_DBG_CM, "Error sending packet %p "
- "(jiffies = %lu)\n", new_send, jiffies);
- new_send->timetosend = jiffies;
- ret = NETDEV_TX_OK;
- } else {
- cm_packets_sent++;
- if (!send_retrans) {
- cleanup_retrans_entry(cm_node);
- if (close_when_complete)
- rem_ref_cm_node(cm_core, cm_node);
- return ret;
- }
- }
- }
-
- if (!timer_pending(&cm_core->tcp_timer))
- mod_timer(&cm_core->tcp_timer, new_send->timetosend);
-
- return ret;
-}
-
-static void nes_retrans_expired(struct nes_cm_node *cm_node)
-{
- struct iw_cm_id *cm_id = cm_node->cm_id;
- enum nes_cm_node_state state = cm_node->state;
- cm_node->state = NES_CM_STATE_CLOSED;
-
- switch (state) {
- case NES_CM_STATE_SYN_RCVD:
- case NES_CM_STATE_CLOSING:
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- break;
- case NES_CM_STATE_LAST_ACK:
- case NES_CM_STATE_FIN_WAIT1:
- if (cm_node->cm_id)
- cm_id->rem_ref(cm_id);
- send_reset(cm_node, NULL);
- break;
- default:
- add_ref_cm_node(cm_node);
- send_reset(cm_node, NULL);
- create_event(cm_node, NES_CM_EVENT_ABORTED);
- }
-}
-
-static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node)
-{
- struct nes_timer_entry *recv_entry = cm_node->recv_entry;
- struct iw_cm_id *cm_id = cm_node->cm_id;
- struct nes_qp *nesqp;
- unsigned long qplockflags;
-
- if (!recv_entry)
- return;
- nesqp = (struct nes_qp *)recv_entry->skb;
- if (nesqp) {
- spin_lock_irqsave(&nesqp->lock, qplockflags);
- if (nesqp->cm_id) {
- nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, "
- "refcount = %d: HIT A "
- "NES_TIMER_TYPE_CLOSE with something "
- "to do!!!\n", nesqp->hwqp.qp_id, cm_id,
- atomic_read(&nesqp->refcount));
- nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
- nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;
- nesqp->ibqp_state = IB_QPS_ERR;
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- nes_cm_disconn(nesqp);
- } else {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, "
- "refcount = %d: HIT A "
- "NES_TIMER_TYPE_CLOSE with nothing "
- "to do!!!\n", nesqp->hwqp.qp_id, cm_id,
- atomic_read(&nesqp->refcount));
- }
- } else if (rem_node) {
- /* TIME_WAIT state */
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- }
- if (cm_node->cm_id)
- cm_id->rem_ref(cm_id);
- kfree(recv_entry);
- cm_node->recv_entry = NULL;
-}
-
-/**
- * nes_cm_timer_tick
- */
-static void nes_cm_timer_tick(struct timer_list *unused)
-{
- unsigned long flags;
- unsigned long nexttimeout = jiffies + NES_LONG_TIME;
- struct nes_cm_node *cm_node;
- struct nes_timer_entry *send_entry, *recv_entry;
- struct list_head *list_core_temp;
- struct list_head *list_node;
- struct nes_cm_core *cm_core = g_cm_core;
- u32 settimer = 0;
- unsigned long timetosend;
- int ret = NETDEV_TX_OK;
-
- struct list_head timer_list;
-
- INIT_LIST_HEAD(&timer_list);
- spin_lock_irqsave(&cm_core->ht_lock, flags);
-
- list_for_each_safe(list_node, list_core_temp,
- &cm_core->connected_nodes) {
- cm_node = container_of(list_node, struct nes_cm_node, list);
- if ((cm_node->recv_entry) || (cm_node->send_entry)) {
- add_ref_cm_node(cm_node);
- list_add(&cm_node->timer_entry, &timer_list);
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- list_for_each_safe(list_node, list_core_temp, &timer_list) {
- cm_node = container_of(list_node, struct nes_cm_node,
- timer_entry);
- recv_entry = cm_node->recv_entry;
-
- if (recv_entry) {
- if (time_after(recv_entry->timetosend, jiffies)) {
- if (nexttimeout > recv_entry->timetosend ||
- !settimer) {
- nexttimeout = recv_entry->timetosend;
- settimer = 1;
- }
- } else {
- handle_recv_entry(cm_node, 1);
- }
- }
-
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- do {
- send_entry = cm_node->send_entry;
- if (!send_entry)
- break;
- if (time_after(send_entry->timetosend, jiffies)) {
- if (cm_node->state != NES_CM_STATE_TSA) {
- if ((nexttimeout >
- send_entry->timetosend) ||
- !settimer) {
- nexttimeout =
- send_entry->timetosend;
- settimer = 1;
- }
- } else {
- free_retrans_entry(cm_node);
- }
- break;
- }
-
- if ((cm_node->state == NES_CM_STATE_TSA) ||
- (cm_node->state == NES_CM_STATE_CLOSED)) {
- free_retrans_entry(cm_node);
- break;
- }
-
- if (!send_entry->retranscount ||
- !send_entry->retrycount) {
- cm_packets_dropped++;
- free_retrans_entry(cm_node);
-
- spin_unlock_irqrestore(
- &cm_node->retrans_list_lock, flags);
- nes_retrans_expired(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- spin_lock_irqsave(&cm_node->retrans_list_lock,
- flags);
- break;
- }
- refcount_inc(&send_entry->skb->users);
- cm_packets_retrans++;
- nes_debug(NES_DBG_CM, "Retransmitting send_entry %p "
- "for node %p, jiffies = %lu, time to send = "
- "%lu, retranscount = %u, send_entry->seq_num = "
- "0x%08X, cm_node->tcp_cntxt.rem_ack_num = "
- "0x%08X\n", send_entry, cm_node, jiffies,
- send_entry->timetosend,
- send_entry->retranscount,
- send_entry->seq_num,
- cm_node->tcp_cntxt.rem_ack_num);
-
- spin_unlock_irqrestore(&cm_node->retrans_list_lock,
- flags);
- ret = nes_nic_cm_xmit(send_entry->skb, cm_node->netdev);
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- if (ret != NETDEV_TX_OK) {
- nes_debug(NES_DBG_CM, "rexmit failed for "
- "node=%p\n", cm_node);
- cm_packets_bounced++;
- send_entry->retrycount--;
- nexttimeout = jiffies + NES_SHORT_TIME;
- settimer = 1;
- break;
- } else {
- cm_packets_sent++;
- }
- nes_debug(NES_DBG_CM, "Packet Sent: retrans count = "
- "%u, retry count = %u.\n",
- send_entry->retranscount,
- send_entry->retrycount);
- if (send_entry->send_retrans) {
- send_entry->retranscount--;
- timetosend = (NES_RETRY_TIMEOUT <<
- (NES_DEFAULT_RETRANS - send_entry->retranscount));
-
- send_entry->timetosend = jiffies +
- min(timetosend, NES_MAX_TIMEOUT);
- if (nexttimeout > send_entry->timetosend ||
- !settimer) {
- nexttimeout = send_entry->timetosend;
- settimer = 1;
- }
- } else {
- int close_when_complete;
- close_when_complete =
- send_entry->close_when_complete;
- nes_debug(NES_DBG_CM, "cm_node=%p state=%d\n",
- cm_node, cm_node->state);
- free_retrans_entry(cm_node);
- if (close_when_complete)
- rem_ref_cm_node(cm_node->cm_core,
- cm_node);
- }
- } while (0);
-
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- }
-
- if (settimer) {
- if (!timer_pending(&cm_core->tcp_timer))
- mod_timer(&cm_core->tcp_timer, nexttimeout);
- }
-}
-
-
-/**
- * send_syn
- */
-static int send_syn(struct nes_cm_node *cm_node, u32 sendack,
- struct sk_buff *skb)
-{
- int ret;
- int flags = SET_SYN;
- char optionsbuffer[sizeof(struct option_mss) +
- sizeof(struct option_windowscale) + sizeof(struct option_base) +
- TCP_OPTIONS_PADDING];
-
- int optionssize = 0;
- /* Sending MSS option */
- union all_known_options *options;
-
- if (!cm_node)
- return -EINVAL;
-
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_mss.optionnum = OPTION_NUMBER_MSS;
- options->as_mss.length = sizeof(struct option_mss);
- options->as_mss.mss = htons(cm_node->tcp_cntxt.mss);
- optionssize += sizeof(struct option_mss);
-
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE;
- options->as_windowscale.length = sizeof(struct option_windowscale);
- options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
- optionssize += sizeof(struct option_windowscale);
-
- if (sendack && !(NES_DRV_OPT_SUPRESS_OPTION_BC & nes_drv_opt)) {
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_base.optionnum = OPTION_NUMBER_WRITE0;
- options->as_base.length = sizeof(struct option_base);
- optionssize += sizeof(struct option_base);
- /* we need the size to be a multiple of 4 */
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_end = 1;
- optionssize += 1;
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_end = 1;
- optionssize += 1;
- }
-
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_end = OPTION_NUMBER_END;
- optionssize += 1;
-
- if (!skb)
- skb = dev_alloc_skb(MAX_CM_BUFFER);
- if (!skb) {
- nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n");
- return -1;
- }
-
- if (sendack)
- flags |= SET_ACK;
-
- form_cm_frame(skb, cm_node, optionsbuffer, optionssize, NULL, 0, flags);
- ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
-
- return ret;
-}
-
-
-/**
- * send_reset
- */
-static int send_reset(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- int ret;
- int flags = SET_RST | SET_ACK;
-
- if (!skb)
- skb = dev_alloc_skb(MAX_CM_BUFFER);
- if (!skb) {
- nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n");
- return -ENOMEM;
- }
-
- form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, flags);
- ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 1);
-
- return ret;
-}
-
-
-/**
- * send_ack
- */
-static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- int ret;
-
- if (!skb)
- skb = dev_alloc_skb(MAX_CM_BUFFER);
-
- if (!skb) {
- nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n");
- return -1;
- }
-
- form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK);
- ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 0);
-
- return ret;
-}
-
-
-/**
- * send_fin
- */
-static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- int ret;
-
- /* if we didn't get a frame get one */
- if (!skb)
- skb = dev_alloc_skb(MAX_CM_BUFFER);
-
- if (!skb) {
- nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n");
- return -1;
- }
-
- form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK | SET_FIN);
- ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
-
- return ret;
-}
-
-
-/**
- * find_node - find a cm node that matches the reference cm node
- */
-static struct nes_cm_node *find_node(struct nes_cm_core *cm_core,
- u16 rem_port, nes_addr_t rem_addr, u16 loc_port, nes_addr_t loc_addr)
-{
- unsigned long flags;
- struct list_head *hte;
- struct nes_cm_node *cm_node;
-
- /* get a handle on the hte */
- hte = &cm_core->connected_nodes;
-
- /* walk list and find cm_node associated with this session ID */
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_for_each_entry(cm_node, hte, list) {
- /* compare quad, return node handle if a match */
- nes_debug(NES_DBG_CM, "finding node %x:%x =? %x:%x ^ %x:%x =? %x:%x\n",
- cm_node->loc_addr, cm_node->loc_port,
- loc_addr, loc_port,
- cm_node->rem_addr, cm_node->rem_port,
- rem_addr, rem_port);
- if ((cm_node->loc_addr == loc_addr) &&
- (cm_node->loc_port == loc_port) &&
- (cm_node->rem_addr == rem_addr) &&
- (cm_node->rem_port == rem_port)) {
- add_ref_cm_node(cm_node);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- return cm_node;
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- /* no owner node */
- return NULL;
-}
-
-
-/**
- * find_listener - find a cm node listening on this addr-port pair
- */
-static struct nes_cm_listener *find_listener(struct nes_cm_core *cm_core,
- nes_addr_t dst_addr, u16 dst_port,
- enum nes_cm_listener_state listener_state)
-{
- unsigned long flags;
- struct nes_cm_listener *listen_node;
- nes_addr_t listen_addr;
- u16 listen_port;
-
- /* walk list and find cm_node associated with this session ID */
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_for_each_entry(listen_node, &cm_core->listen_list.list, list) {
- listen_addr = listen_node->loc_addr;
- listen_port = listen_node->loc_port;
-
- /* compare node pair, return node handle if a match */
- if (((listen_addr == dst_addr) ||
- listen_addr == 0x00000000) &&
- (listen_port == dst_port) &&
- (listener_state & listen_node->listener_state)) {
- atomic_inc(&listen_node->ref_count);
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- return listen_node;
- }
- }
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
-
- /* no listener */
- return NULL;
-}
-
-/**
- * add_hte_node - add a cm node to the hash table
- */
-static int add_hte_node(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node)
-{
- unsigned long flags;
- struct list_head *hte;
-
- if (!cm_node || !cm_core)
- return -EINVAL;
-
- nes_debug(NES_DBG_CM, "Adding Node %p to Active Connection HT\n",
- cm_node);
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
-
- /* get a handle on the hash table element (list head for this slot) */
- hte = &cm_core->connected_nodes;
- list_add_tail(&cm_node->list, hte);
- atomic_inc(&cm_core->ht_node_cnt);
-
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- return 0;
-}
-
-
-/**
- * mini_cm_dec_refcnt_listen
- */
-static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
- struct nes_cm_listener *listener, int free_hanging_nodes)
-{
- int ret = -EINVAL;
- int err = 0;
- unsigned long flags;
- struct list_head *list_pos = NULL;
- struct list_head *list_temp = NULL;
- struct nes_cm_node *cm_node = NULL;
- struct list_head reset_list;
-
- nes_debug(NES_DBG_CM, "attempting listener= %p free_nodes= %d, "
- "refcnt=%d\n", listener, free_hanging_nodes,
- atomic_read(&listener->ref_count));
- /* free non-accelerated child nodes for this listener */
- INIT_LIST_HEAD(&reset_list);
- if (free_hanging_nodes) {
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_for_each_safe(list_pos, list_temp,
- &g_cm_core->connected_nodes) {
- cm_node = container_of(list_pos, struct nes_cm_node,
- list);
- if ((cm_node->listener == listener) &&
- (!cm_node->accelerated)) {
- add_ref_cm_node(cm_node);
- list_add(&cm_node->reset_entry, &reset_list);
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- }
-
- list_for_each_safe(list_pos, list_temp, &reset_list) {
- cm_node = container_of(list_pos, struct nes_cm_node,
- reset_entry);
- {
- struct nes_cm_node *loopback = cm_node->loopbackpartner;
- enum nes_cm_node_state old_state;
- if (NES_CM_STATE_FIN_WAIT1 <= cm_node->state) {
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- } else {
- if (!loopback) {
- cleanup_retrans_entry(cm_node);
- err = send_reset(cm_node, NULL);
- if (err) {
- cm_node->state =
- NES_CM_STATE_CLOSED;
- WARN_ON(1);
- } else {
- old_state = cm_node->state;
- cm_node->state = NES_CM_STATE_LISTENER_DESTROYED;
- if (old_state != NES_CM_STATE_MPAREQ_RCVD)
- rem_ref_cm_node(
- cm_node->cm_core,
- cm_node);
- }
- } else {
- struct nes_cm_event event;
-
- event.cm_node = loopback;
- event.cm_info.rem_addr =
- loopback->rem_addr;
- event.cm_info.loc_addr =
- loopback->loc_addr;
- event.cm_info.rem_port =
- loopback->rem_port;
- event.cm_info.loc_port =
- loopback->loc_port;
- event.cm_info.cm_id = loopback->cm_id;
- add_ref_cm_node(loopback);
- loopback->state = NES_CM_STATE_CLOSED;
- cm_event_connect_error(&event);
- cm_node->state = NES_CM_STATE_LISTENER_DESTROYED;
-
- rem_ref_cm_node(cm_node->cm_core,
- cm_node);
-
- }
- }
- }
- }
-
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- if (!atomic_dec_return(&listener->ref_count)) {
- list_del(&listener->list);
-
- /* decrement our listen node count */
- atomic_dec(&cm_core->listen_node_cnt);
-
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
-
- if (listener->nesvnic) {
- nes_manage_apbvt(listener->nesvnic,
- listener->loc_port,
- PCI_FUNC(listener->nesvnic->nesdev->pcidev->devfn),
- NES_MANAGE_APBVT_DEL);
-
- nes_debug(NES_DBG_NLMSG,
- "Delete APBVT loc_port = %04X\n",
- listener->loc_port);
- }
-
- nes_debug(NES_DBG_CM, "destroying listener (%p)\n", listener);
-
- kfree(listener);
- listener = NULL;
- ret = 0;
- atomic_inc(&cm_listens_destroyed);
- } else {
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- }
- if (listener) {
- if (atomic_read(&listener->pend_accepts_cnt) > 0)
- nes_debug(NES_DBG_CM, "destroying listener (%p)"
- " with non-zero pending accepts=%u\n",
- listener, atomic_read(&listener->pend_accepts_cnt));
- }
-
- return ret;
-}
-
-
-/**
- * mini_cm_del_listen
- */
-static int mini_cm_del_listen(struct nes_cm_core *cm_core,
- struct nes_cm_listener *listener)
-{
- listener->listener_state = NES_CM_LISTENER_PASSIVE_STATE;
- listener->cm_id = NULL; /* going to be destroyed pretty soon */
- return mini_cm_dec_refcnt_listen(cm_core, listener, 1);
-}
-
-
-/**
- * mini_cm_accelerated
- */
-static inline int mini_cm_accelerated(struct nes_cm_core *cm_core,
- struct nes_cm_node *cm_node)
-{
- cm_node->accelerated = true;
-
- if (cm_node->accept_pend) {
- BUG_ON(!cm_node->listener);
- atomic_dec(&cm_node->listener->pend_accepts_cnt);
- cm_node->accept_pend = 0;
- BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0);
- }
-
- if (!timer_pending(&cm_core->tcp_timer))
- mod_timer(&cm_core->tcp_timer, (jiffies + NES_SHORT_TIME));
-
- return 0;
-}
-
-
-/**
- * nes_addr_resolve_neigh
- */
-static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpindex)
-{
- struct rtable *rt;
- struct neighbour *neigh;
- int rc = arpindex;
- struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter;
- __be32 dst_ipaddr = htonl(dst_ip);
-
- rt = ip_route_output(&init_net, dst_ipaddr, nesvnic->local_ipaddr, 0, 0);
- if (IS_ERR(rt)) {
- printk(KERN_ERR "%s: ip_route_output_key failed for 0x%08X\n",
- __func__, dst_ip);
- return rc;
- }
-
- neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);
-
- rcu_read_lock();
- if (neigh) {
- if (neigh->nud_state & NUD_VALID) {
- nes_debug(NES_DBG_CM, "Neighbor MAC address for 0x%08X"
- " is %pM, Gateway is 0x%08X \n", dst_ip,
- neigh->ha, ntohl(rt->rt_gw4));
-
- if (arpindex >= 0) {
- if (ether_addr_equal(nesadapter->arp_table[arpindex].mac_addr, neigh->ha)) {
- /* Mac address same as in nes_arp_table */
- goto out;
- }
-
- nes_manage_arp_cache(nesvnic->netdev,
- nesadapter->arp_table[arpindex].mac_addr,
- dst_ip, NES_ARP_DELETE);
- }
-
- nes_manage_arp_cache(nesvnic->netdev, neigh->ha,
- dst_ip, NES_ARP_ADD);
- rc = nes_arp_table(nesvnic->nesdev, dst_ip, NULL,
- NES_ARP_RESOLVE);
- } else {
- neigh_event_send(neigh, NULL);
- }
- }
-out:
- rcu_read_unlock();
-
- if (neigh)
- neigh_release(neigh);
-
- ip_rt_put(rt);
- return rc;
-}
-
-/**
- * make_cm_node - create a new instance of a cm node
- */
-static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core,
- struct nes_vnic *nesvnic, struct nes_cm_info *cm_info,
- struct nes_cm_listener *listener)
-{
- struct nes_cm_node *cm_node;
- int oldarpindex = 0;
- int arpindex = 0;
- struct nes_device *nesdev;
- struct nes_adapter *nesadapter;
-
- /* create an hte and cm_node for this instance */
- cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
- if (!cm_node)
- return NULL;
-
- /* set our node specific transport info */
- if (listener) {
- cm_node->loc_addr = listener->loc_addr;
- cm_node->loc_port = listener->loc_port;
- } else {
- cm_node->loc_addr = cm_info->loc_addr;
- cm_node->loc_port = cm_info->loc_port;
- }
- cm_node->rem_addr = cm_info->rem_addr;
- cm_node->rem_port = cm_info->rem_port;
-
- cm_node->mpa_frame_rev = mpa_version;
- cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
- cm_node->mpav2_ird_ord = 0;
- cm_node->ird_size = 0;
- cm_node->ord_size = 0;
-
- nes_debug(NES_DBG_CM, "Make node addresses : loc = %pI4:%x, rem = %pI4:%x\n",
- &cm_node->loc_addr, cm_node->loc_port,
- &cm_node->rem_addr, cm_node->rem_port);
- cm_node->listener = listener;
- if (listener)
- cm_node->tos = listener->tos;
- cm_node->netdev = nesvnic->netdev;
- cm_node->cm_id = cm_info->cm_id;
- memcpy(cm_node->loc_mac, nesvnic->netdev->dev_addr, ETH_ALEN);
-
- nes_debug(NES_DBG_CM, "listener=%p, cm_id=%p\n", cm_node->listener,
- cm_node->cm_id);
-
- spin_lock_init(&cm_node->retrans_list_lock);
-
- cm_node->loopbackpartner = NULL;
- atomic_set(&cm_node->ref_count, 1);
- /* associate our parent CM core */
- cm_node->cm_core = cm_core;
- cm_node->tcp_cntxt.loc_id = NES_CM_DEF_LOCAL_ID;
- cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE;
- cm_node->tcp_cntxt.rcv_wnd = NES_CM_DEFAULT_RCV_WND_SCALED >>
- NES_CM_DEFAULT_RCV_WND_SCALE;
- cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr),
- htonl(cm_node->rem_addr),
- htons(cm_node->loc_port),
- htons(cm_node->rem_port));
- cm_node->tcp_cntxt.mss = nesvnic->max_frame_size - sizeof(struct iphdr) -
- sizeof(struct tcphdr) - ETH_HLEN - VLAN_HLEN;
- cm_node->tcp_cntxt.rcv_nxt = 0;
- /* get a unique session ID , add thread_id to an upcounter to handle race */
- atomic_inc(&cm_core->node_cnt);
- cm_node->conn_type = cm_info->conn_type;
- cm_node->apbvt_set = 0;
- cm_node->accept_pend = 0;
-
- cm_node->nesvnic = nesvnic;
- /* get some device handles, for arp lookup */
- nesdev = nesvnic->nesdev;
- nesadapter = nesdev->nesadapter;
-
- cm_node->loopbackpartner = NULL;
-
- /* get the mac addr for the remote node */
- oldarpindex = nes_arp_table(nesdev, cm_node->rem_addr,
- NULL, NES_ARP_RESOLVE);
- arpindex = nes_addr_resolve_neigh(nesvnic, cm_node->rem_addr,
- oldarpindex);
- if (arpindex < 0) {
- kfree(cm_node);
- return NULL;
- }
-
- /* copy the mac addr to node context */
- memcpy(cm_node->rem_mac, nesadapter->arp_table[arpindex].mac_addr, ETH_ALEN);
- nes_debug(NES_DBG_CM, "Remote mac addr from arp table: %pM\n",
- cm_node->rem_mac);
-
- add_hte_node(cm_core, cm_node);
- atomic_inc(&cm_nodes_created);
-
- return cm_node;
-}
-
-
-/**
- * add_ref_cm_node - destroy an instance of a cm node
- */
-static int add_ref_cm_node(struct nes_cm_node *cm_node)
-{
- atomic_inc(&cm_node->ref_count);
- return 0;
-}
-
-
-/**
- * rem_ref_cm_node - destroy an instance of a cm node
- */
-static int rem_ref_cm_node(struct nes_cm_core *cm_core,
- struct nes_cm_node *cm_node)
-{
- unsigned long flags;
- struct nes_qp *nesqp;
-
- if (!cm_node)
- return -EINVAL;
-
- spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags);
- if (atomic_dec_return(&cm_node->ref_count)) {
- spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
- return 0;
- }
- list_del(&cm_node->list);
- atomic_dec(&cm_core->ht_node_cnt);
- spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
-
- /* if the node is destroyed before connection was accelerated */
- if (!cm_node->accelerated && cm_node->accept_pend) {
- BUG_ON(!cm_node->listener);
- atomic_dec(&cm_node->listener->pend_accepts_cnt);
- BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0);
- }
- WARN_ON(cm_node->send_entry);
- if (cm_node->recv_entry)
- handle_recv_entry(cm_node, 0);
- if (cm_node->listener) {
- mini_cm_dec_refcnt_listen(cm_core, cm_node->listener, 0);
- } else {
- if (cm_node->apbvt_set && cm_node->nesvnic) {
- nes_manage_apbvt(cm_node->nesvnic, cm_node->loc_port,
- PCI_FUNC(cm_node->nesvnic->nesdev->pcidev->devfn),
- NES_MANAGE_APBVT_DEL);
- }
- nes_debug(NES_DBG_NLMSG, "Delete APBVT loc_port = %04X\n",
- cm_node->loc_port);
- }
-
- atomic_dec(&cm_core->node_cnt);
- atomic_inc(&cm_nodes_destroyed);
- nesqp = cm_node->nesqp;
- if (nesqp) {
- nesqp->cm_node = NULL;
- nes_rem_ref(&nesqp->ibqp);
- cm_node->nesqp = NULL;
- }
-
- kfree(cm_node);
- return 0;
-}
-
-/**
- * process_options
- */
-static int process_options(struct nes_cm_node *cm_node, u8 *optionsloc,
- u32 optionsize, u32 syn_packet)
-{
- u32 tmp;
- u32 offset = 0;
- union all_known_options *all_options;
- char got_mss_option = 0;
-
- while (offset < optionsize) {
- all_options = (union all_known_options *)(optionsloc + offset);
- switch (all_options->as_base.optionnum) {
- case OPTION_NUMBER_END:
- offset = optionsize;
- break;
- case OPTION_NUMBER_NONE:
- offset += 1;
- continue;
- case OPTION_NUMBER_MSS:
- nes_debug(NES_DBG_CM, "%s: MSS Length: %d Offset: %d "
- "Size: %d\n", __func__,
- all_options->as_mss.length, offset, optionsize);
- got_mss_option = 1;
- if (all_options->as_mss.length != 4) {
- return 1;
- } else {
- tmp = ntohs(all_options->as_mss.mss);
- if (tmp > 0 && tmp <
- cm_node->tcp_cntxt.mss)
- cm_node->tcp_cntxt.mss = tmp;
- }
- break;
- case OPTION_NUMBER_WINDOW_SCALE:
- cm_node->tcp_cntxt.snd_wscale =
- all_options->as_windowscale.shiftcount;
- break;
- default:
- nes_debug(NES_DBG_CM, "TCP Option not understood: %x\n",
- all_options->as_base.optionnum);
- break;
- }
- offset += all_options->as_base.length;
- }
- if ((!got_mss_option) && (syn_packet))
- cm_node->tcp_cntxt.mss = NES_CM_DEFAULT_MSS;
- return 0;
-}
-
-static void drop_packet(struct sk_buff *skb)
-{
- atomic_inc(&cm_accel_dropped_pkts);
- dev_kfree_skb_any(skb);
-}
-
-static void handle_fin_pkt(struct nes_cm_node *cm_node)
-{
- nes_debug(NES_DBG_CM, "Received FIN, cm_node = %p, state = %u. "
- "refcnt=%d\n", cm_node, cm_node->state,
- atomic_read(&cm_node->ref_count));
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_RCVD:
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_MPAREJ_RCVD:
- cm_node->tcp_cntxt.rcv_nxt++;
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_LAST_ACK;
- send_fin(cm_node, NULL);
- break;
- case NES_CM_STATE_MPAREQ_SENT:
- create_event(cm_node, NES_CM_EVENT_ABORTED);
- cm_node->tcp_cntxt.rcv_nxt++;
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- add_ref_cm_node(cm_node);
- send_reset(cm_node, NULL);
- break;
- case NES_CM_STATE_FIN_WAIT1:
- cm_node->tcp_cntxt.rcv_nxt++;
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSING;
- send_ack(cm_node, NULL);
- /* Wait for ACK as this is simultaneous close..
- * After we receive ACK, do not send anything..
- * Just rm the node.. Done.. */
- break;
- case NES_CM_STATE_FIN_WAIT2:
- cm_node->tcp_cntxt.rcv_nxt++;
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_TIME_WAIT;
- send_ack(cm_node, NULL);
- schedule_nes_timer(cm_node, NULL, NES_TIMER_TYPE_CLOSE, 1, 0);
- break;
- case NES_CM_STATE_TIME_WAIT:
- cm_node->tcp_cntxt.rcv_nxt++;
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- break;
- case NES_CM_STATE_TSA:
- default:
- nes_debug(NES_DBG_CM, "Error Rcvd FIN for node-%p state = %d\n",
- cm_node, cm_node->state);
- break;
- }
-}
-
-
-static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb,
- struct tcphdr *tcph)
-{
-
- int reset = 0; /* whether to send reset in case of err.. */
- atomic_inc(&cm_resets_recvd);
- nes_debug(NES_DBG_CM, "Received Reset, cm_node = %p, state = %u."
- " refcnt=%d\n", cm_node, cm_node->state,
- atomic_read(&cm_node->ref_count));
- cleanup_retrans_entry(cm_node);
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_MPAREQ_SENT:
- nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p "
- "listener=%p state=%d\n", __func__, __LINE__, cm_node,
- cm_node->listener, cm_node->state);
- switch (cm_node->mpa_frame_rev) {
- case IETF_MPA_V2:
- cm_node->mpa_frame_rev = IETF_MPA_V1;
- /* send a syn and goto syn sent state */
- cm_node->state = NES_CM_STATE_SYN_SENT;
- if (send_syn(cm_node, 0, NULL)) {
- active_open_err(cm_node, skb, reset);
- }
- break;
- case IETF_MPA_V1:
- default:
- active_open_err(cm_node, skb, reset);
- break;
- }
- break;
- case NES_CM_STATE_MPAREQ_RCVD:
- atomic_inc(&cm_node->passive_state);
- dev_kfree_skb_any(skb);
- break;
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_SYN_RCVD:
- case NES_CM_STATE_LISTENING:
- nes_debug(NES_DBG_CM, "Bad state %s[%u]\n", __func__, __LINE__);
- passive_open_err(cm_node, skb, reset);
- break;
- case NES_CM_STATE_TSA:
- active_open_err(cm_node, skb, reset);
- break;
- case NES_CM_STATE_CLOSED:
- drop_packet(skb);
- break;
- case NES_CM_STATE_FIN_WAIT2:
- case NES_CM_STATE_FIN_WAIT1:
- case NES_CM_STATE_LAST_ACK:
- cm_node->cm_id->rem_ref(cm_node->cm_id);
- /* fall through */
- case NES_CM_STATE_TIME_WAIT:
- cm_node->state = NES_CM_STATE_CLOSED;
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- drop_packet(skb);
- break;
- default:
- drop_packet(skb);
- break;
- }
-}
-
-
-static void handle_rcv_mpa(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- int ret = 0;
- int datasize = skb->len;
- u8 *dataloc = skb->data;
-
- enum nes_cm_event_type type = NES_CM_EVENT_UNKNOWN;
- u32 res_type;
-
- ret = parse_mpa(cm_node, dataloc, &res_type, datasize);
- if (ret) {
- nes_debug(NES_DBG_CM, "didn't like MPA Request\n");
- if (cm_node->state == NES_CM_STATE_MPAREQ_SENT) {
- nes_debug(NES_DBG_CM, "%s[%u] create abort for "
- "cm_node=%p listener=%p state=%d\n", __func__,
- __LINE__, cm_node, cm_node->listener,
- cm_node->state);
- active_open_err(cm_node, skb, 1);
- } else {
- passive_open_err(cm_node, skb, 1);
- }
- return;
- }
-
- switch (cm_node->state) {
- case NES_CM_STATE_ESTABLISHED:
- if (res_type == NES_MPA_REQUEST_REJECT)
- /*BIG problem as we are receiving the MPA.. So should
- * not be REJECT.. This is Passive Open.. We can
- * only receive it Reject for Active Open...*/
- WARN_ON(1);
- cm_node->state = NES_CM_STATE_MPAREQ_RCVD;
- type = NES_CM_EVENT_MPA_REQ;
- atomic_set(&cm_node->passive_state,
- NES_PASSIVE_STATE_INDICATED);
- break;
- case NES_CM_STATE_MPAREQ_SENT:
- cleanup_retrans_entry(cm_node);
- if (res_type == NES_MPA_REQUEST_REJECT) {
- type = NES_CM_EVENT_MPA_REJECT;
- cm_node->state = NES_CM_STATE_MPAREJ_RCVD;
- } else {
- type = NES_CM_EVENT_CONNECTED;
- cm_node->state = NES_CM_STATE_TSA;
- }
- send_ack(cm_node, NULL);
- break;
- default:
- WARN_ON(1);
- break;
- }
- dev_kfree_skb_any(skb);
- create_event(cm_node, type);
-}
-
-static void indicate_pkt_err(struct nes_cm_node *cm_node, struct sk_buff *skb)
-{
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_MPAREQ_SENT:
- nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p "
- "listener=%p state=%d\n", __func__, __LINE__, cm_node,
- cm_node->listener, cm_node->state);
- active_open_err(cm_node, skb, 1);
- break;
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_SYN_RCVD:
- passive_open_err(cm_node, skb, 1);
- break;
- case NES_CM_STATE_TSA:
- default:
- drop_packet(skb);
- }
-}
-
-static int check_syn(struct nes_cm_node *cm_node, struct tcphdr *tcph,
- struct sk_buff *skb)
-{
- int err;
-
- err = ((ntohl(tcph->ack_seq) == cm_node->tcp_cntxt.loc_seq_num)) ? 0 : 1;
- if (err)
- active_open_err(cm_node, skb, 1);
-
- return err;
-}
-
-static int check_seq(struct nes_cm_node *cm_node, struct tcphdr *tcph,
- struct sk_buff *skb)
-{
- int err = 0;
- u32 seq;
- u32 ack_seq;
- u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
- u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
- u32 rcv_wnd;
-
- seq = ntohl(tcph->seq);
- ack_seq = ntohl(tcph->ack_seq);
- rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
- if (ack_seq != loc_seq_num)
- err = 1;
- else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
- err = 1;
- if (err) {
- nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p "
- "listener=%p state=%d\n", __func__, __LINE__, cm_node,
- cm_node->listener, cm_node->state);
- indicate_pkt_err(cm_node, skb);
- nes_debug(NES_DBG_CM, "seq ERROR cm_node =%p seq=0x%08X "
- "rcv_nxt=0x%08X rcv_wnd=0x%x\n", cm_node, seq, rcv_nxt,
- rcv_wnd);
- }
- return err;
-}
-
-/*
- * handle_syn_pkt() is for Passive node. The syn packet is received when a node
- * is created with a listener or it may comein as rexmitted packet which in
- * that case will be just dropped.
- */
-static void handle_syn_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb,
- struct tcphdr *tcph)
-{
- int ret;
- u32 inc_sequence;
- int optionsize;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
- skb_trim(skb, 0);
- inc_sequence = ntohl(tcph->seq);
-
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_MPAREQ_SENT:
- /* Rcvd syn on active open connection*/
- active_open_err(cm_node, skb, 1);
- break;
- case NES_CM_STATE_LISTENING:
- /* Passive OPEN */
- if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
- cm_node->listener->backlog) {
- nes_debug(NES_DBG_CM, "drop syn due to backlog "
- "pressure \n");
- cm_backlog_drops++;
- passive_open_err(cm_node, skb, 0);
- break;
- }
- ret = handle_tcp_options(cm_node, tcph, skb, optionsize,
- 1);
- if (ret) {
- passive_open_err(cm_node, skb, 0);
- /* drop pkt */
- break;
- }
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
- BUG_ON(cm_node->send_entry);
- cm_node->accept_pend = 1;
- atomic_inc(&cm_node->listener->pend_accepts_cnt);
-
- cm_node->state = NES_CM_STATE_SYN_RCVD;
- send_syn(cm_node, 1, skb);
- break;
- case NES_CM_STATE_CLOSED:
- cleanup_retrans_entry(cm_node);
- add_ref_cm_node(cm_node);
- send_reset(cm_node, skb);
- break;
- case NES_CM_STATE_TSA:
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_FIN_WAIT1:
- case NES_CM_STATE_FIN_WAIT2:
- case NES_CM_STATE_MPAREQ_RCVD:
- case NES_CM_STATE_LAST_ACK:
- case NES_CM_STATE_CLOSING:
- case NES_CM_STATE_UNKNOWN:
- default:
- drop_packet(skb);
- break;
- }
-}
-
-static void handle_synack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb,
- struct tcphdr *tcph)
-{
- int ret;
- u32 inc_sequence;
- int optionsize;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
- skb_trim(skb, 0);
- inc_sequence = ntohl(tcph->seq);
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_SENT:
- cleanup_retrans_entry(cm_node);
- /* active open */
- if (check_syn(cm_node, tcph, skb))
- return;
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- /* setup options */
- ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 0);
- if (ret) {
- nes_debug(NES_DBG_CM, "cm_node=%p tcp_options failed\n",
- cm_node);
- break;
- }
- cleanup_retrans_entry(cm_node);
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
- send_mpa_request(cm_node, skb);
- cm_node->state = NES_CM_STATE_MPAREQ_SENT;
- break;
- case NES_CM_STATE_MPAREQ_RCVD:
- /* passive open, so should not be here */
- passive_open_err(cm_node, skb, 1);
- break;
- case NES_CM_STATE_LISTENING:
- cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- send_reset(cm_node, skb);
- break;
- case NES_CM_STATE_CLOSED:
- cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
- cleanup_retrans_entry(cm_node);
- add_ref_cm_node(cm_node);
- send_reset(cm_node, skb);
- break;
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_FIN_WAIT1:
- case NES_CM_STATE_FIN_WAIT2:
- case NES_CM_STATE_LAST_ACK:
- case NES_CM_STATE_TSA:
- case NES_CM_STATE_CLOSING:
- case NES_CM_STATE_UNKNOWN:
- case NES_CM_STATE_MPAREQ_SENT:
- default:
- drop_packet(skb);
- break;
- }
-}
-
-static int handle_ack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb,
- struct tcphdr *tcph)
-{
- int datasize = 0;
- u32 inc_sequence;
- int ret = 0;
- int optionsize;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
-
- if (check_seq(cm_node, tcph, skb))
- return -EINVAL;
-
- skb_pull(skb, tcph->doff << 2);
- inc_sequence = ntohl(tcph->seq);
- datasize = skb->len;
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_RCVD:
- /* Passive OPEN */
- cleanup_retrans_entry(cm_node);
- ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 1);
- if (ret)
- break;
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- cm_node->state = NES_CM_STATE_ESTABLISHED;
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- handle_rcv_mpa(cm_node, skb);
- } else { /* rcvd ACK only */
- dev_kfree_skb_any(skb);
- }
- break;
- case NES_CM_STATE_ESTABLISHED:
- /* Passive OPEN */
- cleanup_retrans_entry(cm_node);
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- handle_rcv_mpa(cm_node, skb);
- } else {
- drop_packet(skb);
- }
- break;
- case NES_CM_STATE_MPAREQ_SENT:
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- handle_rcv_mpa(cm_node, skb);
- } else { /* Could be just an ack pkt.. */
- dev_kfree_skb_any(skb);
- }
- break;
- case NES_CM_STATE_LISTENING:
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- send_reset(cm_node, skb);
- break;
- case NES_CM_STATE_CLOSED:
- cleanup_retrans_entry(cm_node);
- add_ref_cm_node(cm_node);
- send_reset(cm_node, skb);
- break;
- case NES_CM_STATE_LAST_ACK:
- case NES_CM_STATE_CLOSING:
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- cm_node->cm_id->rem_ref(cm_node->cm_id);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- drop_packet(skb);
- break;
- case NES_CM_STATE_FIN_WAIT1:
- cleanup_retrans_entry(cm_node);
- drop_packet(skb);
- cm_node->state = NES_CM_STATE_FIN_WAIT2;
- break;
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_FIN_WAIT2:
- case NES_CM_STATE_TSA:
- case NES_CM_STATE_MPAREQ_RCVD:
- case NES_CM_STATE_UNKNOWN:
- default:
- cleanup_retrans_entry(cm_node);
- drop_packet(skb);
- break;
- }
- return ret;
-}
-
-
-
-static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph,
- struct sk_buff *skb, int optionsize, int passive)
-{
- u8 *optionsloc = (u8 *)&tcph[1];
-
- if (optionsize) {
- if (process_options(cm_node, optionsloc, optionsize,
- (u32)tcph->syn)) {
- nes_debug(NES_DBG_CM, "%s: Node %p, Sending RESET\n",
- __func__, cm_node);
- if (passive)
- passive_open_err(cm_node, skb, 1);
- else
- active_open_err(cm_node, skb, 1);
- return 1;
- }
- }
-
- cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) <<
- cm_node->tcp_cntxt.snd_wscale;
-
- if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
- cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
- return 0;
-}
-
-/*
- * active_open_err() will send reset() if flag set..
- * It will also send ABORT event.
- */
-static void active_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb,
- int reset)
-{
- cleanup_retrans_entry(cm_node);
- if (reset) {
- nes_debug(NES_DBG_CM, "ERROR active err called for cm_node=%p, "
- "state=%d\n", cm_node, cm_node->state);
- add_ref_cm_node(cm_node);
- send_reset(cm_node, skb);
- } else {
- dev_kfree_skb_any(skb);
- }
-
- cm_node->state = NES_CM_STATE_CLOSED;
- create_event(cm_node, NES_CM_EVENT_ABORTED);
-}
-
-/*
- * passive_open_err() will either do a reset() or will free up the skb and
- * remove the cm_node.
- */
-static void passive_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb,
- int reset)
-{
- cleanup_retrans_entry(cm_node);
- cm_node->state = NES_CM_STATE_CLOSED;
- if (reset) {
- nes_debug(NES_DBG_CM, "passive_open_err sending RST for "
- "cm_node=%p state =%d\n", cm_node, cm_node->state);
- send_reset(cm_node, skb);
- } else {
- dev_kfree_skb_any(skb);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- }
-}
-
-/*
- * free_retrans_entry() routines assumes that the retrans_list_lock has
- * been acquired before calling.
- */
-static void free_retrans_entry(struct nes_cm_node *cm_node)
-{
- struct nes_timer_entry *send_entry;
-
- send_entry = cm_node->send_entry;
- if (send_entry) {
- cm_node->send_entry = NULL;
- dev_kfree_skb_any(send_entry->skb);
- kfree(send_entry);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- }
-}
-
-static void cleanup_retrans_entry(struct nes_cm_node *cm_node)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- free_retrans_entry(cm_node);
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
-}
-
-/**
- * process_packet
- * Returns skb if to be freed, else it will return NULL if already used..
- */
-static void process_packet(struct nes_cm_node *cm_node, struct sk_buff *skb,
- struct nes_cm_core *cm_core)
-{
- enum nes_tcpip_pkt_type pkt_type = NES_PKT_TYPE_UNKNOWN;
- struct tcphdr *tcph = tcp_hdr(skb);
- u32 fin_set = 0;
- int ret = 0;
-
- skb_pull(skb, ip_hdr(skb)->ihl << 2);
-
- nes_debug(NES_DBG_CM, "process_packet: cm_node=%p state =%d syn=%d "
- "ack=%d rst=%d fin=%d\n", cm_node, cm_node->state, tcph->syn,
- tcph->ack, tcph->rst, tcph->fin);
-
- if (tcph->rst) {
- pkt_type = NES_PKT_TYPE_RST;
- } else if (tcph->syn) {
- pkt_type = NES_PKT_TYPE_SYN;
- if (tcph->ack)
- pkt_type = NES_PKT_TYPE_SYNACK;
- } else if (tcph->ack) {
- pkt_type = NES_PKT_TYPE_ACK;
- }
- if (tcph->fin)
- fin_set = 1;
-
- switch (pkt_type) {
- case NES_PKT_TYPE_SYN:
- handle_syn_pkt(cm_node, skb, tcph);
- break;
- case NES_PKT_TYPE_SYNACK:
- handle_synack_pkt(cm_node, skb, tcph);
- break;
- case NES_PKT_TYPE_ACK:
- ret = handle_ack_pkt(cm_node, skb, tcph);
- if (fin_set && !ret)
- handle_fin_pkt(cm_node);
- break;
- case NES_PKT_TYPE_RST:
- handle_rst_pkt(cm_node, skb, tcph);
- break;
- default:
- if ((fin_set) && (!check_seq(cm_node, tcph, skb)))
- handle_fin_pkt(cm_node);
- drop_packet(skb);
- break;
- }
-}
-
-/**
- * mini_cm_listen - create a listen node with params
- */
-static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *cm_core,
- struct nes_vnic *nesvnic, struct nes_cm_info *cm_info)
-{
- struct nes_cm_listener *listener;
- unsigned long flags;
-
- nes_debug(NES_DBG_CM, "Search for 0x%08x : 0x%04x\n",
- cm_info->loc_addr, cm_info->loc_port);
-
- /* cannot have multiple matching listeners */
- listener = find_listener(cm_core, cm_info->loc_addr, cm_info->loc_port,
- NES_CM_LISTENER_EITHER_STATE);
-
- if (listener && listener->listener_state == NES_CM_LISTENER_ACTIVE_STATE) {
- /* find automatically incs ref count ??? */
- atomic_dec(&listener->ref_count);
- nes_debug(NES_DBG_CM, "Not creating listener since it already exists\n");
- return NULL;
- }
-
- if (!listener) {
- /* create a CM listen node (1/2 node to compare incoming traffic to) */
- listener = kzalloc(sizeof(*listener), GFP_ATOMIC);
- if (!listener)
- return NULL;
-
- listener->loc_addr = cm_info->loc_addr;
- listener->loc_port = cm_info->loc_port;
- listener->reused_node = 0;
-
- atomic_set(&listener->ref_count, 1);
- }
- /* pasive case */
- /* find already inc'ed the ref count */
- else {
- listener->reused_node = 1;
- }
-
- listener->cm_id = cm_info->cm_id;
- atomic_set(&listener->pend_accepts_cnt, 0);
- listener->cm_core = cm_core;
- listener->nesvnic = nesvnic;
- atomic_inc(&cm_core->node_cnt);
-
- listener->conn_type = cm_info->conn_type;
- listener->backlog = cm_info->backlog;
- listener->listener_state = NES_CM_LISTENER_ACTIVE_STATE;
-
- if (!listener->reused_node) {
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_add(&listener->list, &cm_core->listen_list.list);
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- atomic_inc(&cm_core->listen_node_cnt);
- }
-
- nes_debug(NES_DBG_CM, "Api - listen(): addr=0x%08X, port=0x%04x,"
- " listener = %p, backlog = %d, cm_id = %p.\n",
- cm_info->loc_addr, cm_info->loc_port,
- listener, listener->backlog, listener->cm_id);
-
- return listener;
-}
-
-
-/**
- * mini_cm_connect - make a connection node with params
- */
-static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *cm_core,
- struct nes_vnic *nesvnic, u16 private_data_len,
- void *private_data, struct nes_cm_info *cm_info)
-{
- int ret = 0;
- struct nes_cm_node *cm_node;
- struct nes_cm_listener *loopbackremotelistener;
- struct nes_cm_node *loopbackremotenode;
- struct nes_cm_info loopback_cm_info;
- u8 *start_buff;
-
- /* create a CM connection node */
- cm_node = make_cm_node(cm_core, nesvnic, cm_info, NULL);
- if (!cm_node)
- return NULL;
-
- /* set our node side to client (active) side */
- cm_node->tcp_cntxt.client = 1;
- cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE;
-
- if (cm_info->loc_addr == cm_info->rem_addr) {
- loopbackremotelistener = find_listener(cm_core,
- cm_node->loc_addr, cm_node->rem_port,
- NES_CM_LISTENER_ACTIVE_STATE);
- if (loopbackremotelistener == NULL) {
- create_event(cm_node, NES_CM_EVENT_ABORTED);
- } else {
- loopback_cm_info = *cm_info;
- loopback_cm_info.loc_port = cm_info->rem_port;
- loopback_cm_info.rem_port = cm_info->loc_port;
- loopback_cm_info.loc_port =
- cm_info->rem_port;
- loopback_cm_info.rem_port =
- cm_info->loc_port;
- loopback_cm_info.cm_id = loopbackremotelistener->cm_id;
- loopbackremotenode = make_cm_node(cm_core, nesvnic,
- &loopback_cm_info, loopbackremotelistener);
- if (!loopbackremotenode) {
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- return NULL;
- }
- atomic_inc(&cm_loopbacks);
- loopbackremotenode->loopbackpartner = cm_node;
- loopbackremotenode->tcp_cntxt.rcv_wscale =
- NES_CM_DEFAULT_RCV_WND_SCALE;
- cm_node->loopbackpartner = loopbackremotenode;
- memcpy(loopbackremotenode->mpa_frame_buf, private_data,
- private_data_len);
- loopbackremotenode->mpa_frame_size = private_data_len;
-
- /* we are done handling this state. */
- /* set node to a TSA state */
- cm_node->state = NES_CM_STATE_TSA;
- cm_node->tcp_cntxt.rcv_nxt =
- loopbackremotenode->tcp_cntxt.loc_seq_num;
- loopbackremotenode->tcp_cntxt.rcv_nxt =
- cm_node->tcp_cntxt.loc_seq_num;
- cm_node->tcp_cntxt.max_snd_wnd =
- loopbackremotenode->tcp_cntxt.rcv_wnd;
- loopbackremotenode->tcp_cntxt.max_snd_wnd =
- cm_node->tcp_cntxt.rcv_wnd;
- cm_node->tcp_cntxt.snd_wnd =
- loopbackremotenode->tcp_cntxt.rcv_wnd;
- loopbackremotenode->tcp_cntxt.snd_wnd =
- cm_node->tcp_cntxt.rcv_wnd;
- cm_node->tcp_cntxt.snd_wscale =
- loopbackremotenode->tcp_cntxt.rcv_wscale;
- loopbackremotenode->tcp_cntxt.snd_wscale =
- cm_node->tcp_cntxt.rcv_wscale;
- loopbackremotenode->state = NES_CM_STATE_MPAREQ_RCVD;
- create_event(loopbackremotenode, NES_CM_EVENT_MPA_REQ);
- }
- return cm_node;
- }
-
- start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2);
- cm_node->mpa_frame_size = private_data_len;
-
- memcpy(start_buff, private_data, private_data_len);
-
- /* send a syn and goto syn sent state */
- cm_node->state = NES_CM_STATE_SYN_SENT;
- ret = send_syn(cm_node, 0, NULL);
-
- if (ret) {
- /* error in sending the syn free up the cm_node struct */
- nes_debug(NES_DBG_CM, "Api - connect() FAILED: dest "
- "addr=0x%08X, port=0x%04x, cm_node=%p, cm_id = %p.\n",
- cm_node->rem_addr, cm_node->rem_port, cm_node,
- cm_node->cm_id);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- cm_node = NULL;
- }
-
- if (cm_node) {
- nes_debug(NES_DBG_CM, "Api - connect(): dest addr=0x%08X,"
- "port=0x%04x, cm_node=%p, cm_id = %p.\n",
- cm_node->rem_addr, cm_node->rem_port, cm_node,
- cm_node->cm_id);
- }
-
- return cm_node;
-}
-
-
-/**
- * mini_cm_accept - accept a connection
- * This function is never called
- */
-static int mini_cm_accept(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node)
-{
- return 0;
-}
-
-
-/**
- * mini_cm_reject - reject and teardown a connection
- */
-static int mini_cm_reject(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node)
-{
- int ret = 0;
- int err = 0;
- int passive_state;
- struct nes_cm_event event;
- struct iw_cm_id *cm_id = cm_node->cm_id;
- struct nes_cm_node *loopback = cm_node->loopbackpartner;
-
- nes_debug(NES_DBG_CM, "%s cm_node=%p type=%d state=%d\n",
- __func__, cm_node, cm_node->tcp_cntxt.client, cm_node->state);
-
- if (cm_node->tcp_cntxt.client)
- return ret;
- cleanup_retrans_entry(cm_node);
-
- if (!loopback) {
- passive_state = atomic_add_return(1, &cm_node->passive_state);
- if (passive_state == NES_SEND_RESET_EVENT) {
- cm_node->state = NES_CM_STATE_CLOSED;
- rem_ref_cm_node(cm_core, cm_node);
- } else {
- if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) {
- rem_ref_cm_node(cm_core, cm_node);
- } else {
- ret = send_mpa_reject(cm_node);
- if (ret) {
- cm_node->state = NES_CM_STATE_CLOSED;
- err = send_reset(cm_node, NULL);
- if (err)
- WARN_ON(1);
- } else {
- cm_id->add_ref(cm_id);
- }
- }
- }
- } else {
- cm_node->cm_id = NULL;
- if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) {
- rem_ref_cm_node(cm_core, cm_node);
- rem_ref_cm_node(cm_core, loopback);
- } else {
- event.cm_node = loopback;
- event.cm_info.rem_addr = loopback->rem_addr;
- event.cm_info.loc_addr = loopback->loc_addr;
- event.cm_info.rem_port = loopback->rem_port;
- event.cm_info.loc_port = loopback->loc_port;
- event.cm_info.cm_id = loopback->cm_id;
- cm_event_mpa_reject(&event);
- rem_ref_cm_node(cm_core, cm_node);
- loopback->state = NES_CM_STATE_CLOSING;
-
- cm_id = loopback->cm_id;
- rem_ref_cm_node(cm_core, loopback);
- cm_id->rem_ref(cm_id);
- }
- }
-
- return ret;
-}
-
-
-/**
- * mini_cm_close
- */
-static int mini_cm_close(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node)
-{
- int ret = 0;
-
- if (!cm_core || !cm_node)
- return -EINVAL;
-
- switch (cm_node->state) {
- case NES_CM_STATE_SYN_RCVD:
- case NES_CM_STATE_SYN_SENT:
- case NES_CM_STATE_ONE_SIDE_ESTABLISHED:
- case NES_CM_STATE_ESTABLISHED:
- case NES_CM_STATE_ACCEPTING:
- case NES_CM_STATE_MPAREQ_SENT:
- case NES_CM_STATE_MPAREQ_RCVD:
- cleanup_retrans_entry(cm_node);
- send_reset(cm_node, NULL);
- break;
- case NES_CM_STATE_CLOSE_WAIT:
- cm_node->state = NES_CM_STATE_LAST_ACK;
- send_fin(cm_node, NULL);
- break;
- case NES_CM_STATE_FIN_WAIT1:
- case NES_CM_STATE_FIN_WAIT2:
- case NES_CM_STATE_LAST_ACK:
- case NES_CM_STATE_TIME_WAIT:
- case NES_CM_STATE_CLOSING:
- ret = -1;
- break;
- case NES_CM_STATE_LISTENING:
- cleanup_retrans_entry(cm_node);
- send_reset(cm_node, NULL);
- break;
- case NES_CM_STATE_MPAREJ_RCVD:
- case NES_CM_STATE_UNKNOWN:
- case NES_CM_STATE_INITED:
- case NES_CM_STATE_CLOSED:
- case NES_CM_STATE_LISTENER_DESTROYED:
- ret = rem_ref_cm_node(cm_core, cm_node);
- break;
- case NES_CM_STATE_TSA:
- if (cm_node->send_entry)
- printk(KERN_ERR "ERROR Close got called from STATE_TSA "
- "send_entry=%p\n", cm_node->send_entry);
- ret = rem_ref_cm_node(cm_core, cm_node);
- break;
- }
- return ret;
-}
-
-
-/**
- * recv_pkt - recv an ETHERNET packet, and process it through CM
- * node state machine
- */
-static int mini_cm_recv_pkt(struct nes_cm_core *cm_core,
- struct nes_vnic *nesvnic, struct sk_buff *skb)
-{
- struct nes_cm_node *cm_node = NULL;
- struct nes_cm_listener *listener = NULL;
- struct iphdr *iph;
- struct tcphdr *tcph;
- struct nes_cm_info nfo;
- int skb_handled = 1;
- __be32 tmp_daddr, tmp_saddr;
-
- if (!skb)
- return 0;
- if (skb->len < sizeof(struct iphdr) + sizeof(struct tcphdr))
- return 0;
-
- iph = (struct iphdr *)skb->data;
- tcph = (struct tcphdr *)(skb->data + sizeof(struct iphdr));
-
- nfo.loc_addr = ntohl(iph->daddr);
- nfo.loc_port = ntohs(tcph->dest);
- nfo.rem_addr = ntohl(iph->saddr);
- nfo.rem_port = ntohs(tcph->source);
-
- tmp_daddr = cpu_to_be32(iph->daddr);
- tmp_saddr = cpu_to_be32(iph->saddr);
-
- nes_debug(NES_DBG_CM, "Received packet: dest=%pI4:0x%04X src=%pI4:0x%04X\n",
- &tmp_daddr, tcph->dest, &tmp_saddr, tcph->source);
-
- do {
- cm_node = find_node(cm_core,
- nfo.rem_port, nfo.rem_addr,
- nfo.loc_port, nfo.loc_addr);
-
- if (!cm_node) {
- /* Only type of packet accepted are for */
- /* the PASSIVE open (syn only) */
- if ((!tcph->syn) || (tcph->ack)) {
- skb_handled = 0;
- break;
- }
- listener = find_listener(cm_core, nfo.loc_addr,
- nfo.loc_port,
- NES_CM_LISTENER_ACTIVE_STATE);
- if (!listener) {
- nfo.cm_id = NULL;
- nfo.conn_type = 0;
- nes_debug(NES_DBG_CM, "Unable to find listener for the pkt\n");
- skb_handled = 0;
- break;
- }
- nfo.cm_id = listener->cm_id;
- nfo.conn_type = listener->conn_type;
- cm_node = make_cm_node(cm_core, nesvnic, &nfo,
- listener);
- if (!cm_node) {
- nes_debug(NES_DBG_CM, "Unable to allocate "
- "node\n");
- cm_packets_dropped++;
- atomic_dec(&listener->ref_count);
- dev_kfree_skb_any(skb);
- break;
- }
- if (!tcph->rst && !tcph->fin) {
- cm_node->state = NES_CM_STATE_LISTENING;
- } else {
- cm_packets_dropped++;
- rem_ref_cm_node(cm_core, cm_node);
- dev_kfree_skb_any(skb);
- break;
- }
- add_ref_cm_node(cm_node);
- } else if (cm_node->state == NES_CM_STATE_TSA) {
- if (cm_node->nesqp->pau_mode)
- nes_queue_mgt_skbs(skb, nesvnic, cm_node->nesqp);
- else {
- rem_ref_cm_node(cm_core, cm_node);
- atomic_inc(&cm_accel_dropped_pkts);
- dev_kfree_skb_any(skb);
- }
- break;
- }
- skb_reset_network_header(skb);
- skb_set_transport_header(skb, sizeof(*tcph));
- skb->len = ntohs(iph->tot_len);
- process_packet(cm_node, skb, cm_core);
- rem_ref_cm_node(cm_core, cm_node);
- } while (0);
- return skb_handled;
-}
-
-
-/**
- * nes_cm_alloc_core - allocate a top level instance of a cm core
- */
-static struct nes_cm_core *nes_cm_alloc_core(void)
-{
- struct nes_cm_core *cm_core;
-
- /* setup the CM core */
- /* alloc top level core control structure */
- cm_core = kzalloc(sizeof(*cm_core), GFP_KERNEL);
- if (!cm_core)
- return NULL;
-
- INIT_LIST_HEAD(&cm_core->connected_nodes);
- timer_setup(&cm_core->tcp_timer, nes_cm_timer_tick, 0);
-
- cm_core->mtu = NES_CM_DEFAULT_MTU;
- cm_core->state = NES_CM_STATE_INITED;
- cm_core->free_tx_pkt_max = NES_CM_DEFAULT_FREE_PKTS;
-
- atomic_set(&cm_core->events_posted, 0);
-
- cm_core->api = &nes_cm_api;
-
- spin_lock_init(&cm_core->ht_lock);
- spin_lock_init(&cm_core->listen_list_lock);
-
- INIT_LIST_HEAD(&cm_core->listen_list.list);
-
- nes_debug(NES_DBG_CM, "Init CM Core completed -- cm_core=%p\n", cm_core);
-
- nes_debug(NES_DBG_CM, "Enable QUEUE EVENTS\n");
- cm_core->event_wq = alloc_ordered_workqueue("nesewq", 0);
- if (!cm_core->event_wq)
- goto out_free_cmcore;
- cm_core->post_event = nes_cm_post_event;
- nes_debug(NES_DBG_CM, "Enable QUEUE DISCONNECTS\n");
- cm_core->disconn_wq = alloc_ordered_workqueue("nesdwq", 0);
- if (!cm_core->disconn_wq)
- goto out_free_wq;
-
- print_core(cm_core);
- return cm_core;
-
-out_free_wq:
- destroy_workqueue(cm_core->event_wq);
-out_free_cmcore:
- kfree(cm_core);
- return NULL;
-}
-
-
-/**
- * mini_cm_dealloc_core - deallocate a top level instance of a cm core
- */
-static int mini_cm_dealloc_core(struct nes_cm_core *cm_core)
-{
- nes_debug(NES_DBG_CM, "De-Alloc CM Core (%p)\n", cm_core);
-
- if (!cm_core)
- return -EINVAL;
-
- barrier();
-
- if (timer_pending(&cm_core->tcp_timer))
- del_timer(&cm_core->tcp_timer);
-
- destroy_workqueue(cm_core->event_wq);
- destroy_workqueue(cm_core->disconn_wq);
- nes_debug(NES_DBG_CM, "\n");
- kfree(cm_core);
-
- return 0;
-}
-
-
-/**
- * mini_cm_get
- */
-static int mini_cm_get(struct nes_cm_core *cm_core)
-{
- return cm_core->state;
-}
-
-
-/**
- * mini_cm_set
- */
-static int mini_cm_set(struct nes_cm_core *cm_core, u32 type, u32 value)
-{
- int ret = 0;
-
- switch (type) {
- case NES_CM_SET_PKT_SIZE:
- cm_core->mtu = value;
- break;
- case NES_CM_SET_FREE_PKT_Q_SIZE:
- cm_core->free_tx_pkt_max = value;
- break;
- default:
- /* unknown set option */
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-
-/**
- * nes_cm_init_tsa_conn setup HW; MPA frames must be
- * successfully exchanged when this is called
- */
-static int nes_cm_init_tsa_conn(struct nes_qp *nesqp, struct nes_cm_node *cm_node)
-{
- int ret = 0;
-
- if (!nesqp)
- return -EINVAL;
-
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_IPV4 |
- NES_QPCONTEXT_MISC_NO_NAGLE | NES_QPCONTEXT_MISC_DO_NOT_FRAG |
- NES_QPCONTEXT_MISC_DROS);
-
- if (cm_node->tcp_cntxt.snd_wscale || cm_node->tcp_cntxt.rcv_wscale)
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WSCALE);
-
- nesqp->nesqp_context->misc2 |= cpu_to_le32(64 << NES_QPCONTEXT_MISC2_TTL_SHIFT);
-
- nesqp->nesqp_context->misc2 |= cpu_to_le32(
- cm_node->tos << NES_QPCONTEXT_MISC2_TOS_SHIFT);
-
- nesqp->nesqp_context->mss |= cpu_to_le32(((u32)cm_node->tcp_cntxt.mss) << 16);
-
- nesqp->nesqp_context->tcp_state_flow_label |= cpu_to_le32(
- (u32)NES_QPCONTEXT_TCPSTATE_EST << NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT);
-
- nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32(
- (cm_node->tcp_cntxt.snd_wscale << NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT) &
- NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK);
-
- nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32(
- (cm_node->tcp_cntxt.rcv_wscale << NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT) &
- NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK);
-
- nesqp->nesqp_context->keepalive = cpu_to_le32(0x80);
- nesqp->nesqp_context->ts_recent = 0;
- nesqp->nesqp_context->ts_age = 0;
- nesqp->nesqp_context->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- nesqp->nesqp_context->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd);
- nesqp->nesqp_context->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
- nesqp->nesqp_context->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd <<
- cm_node->tcp_cntxt.rcv_wscale);
- nesqp->nesqp_context->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- nesqp->nesqp_context->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- nesqp->nesqp_context->srtt = 0;
- nesqp->nesqp_context->rttvar = cpu_to_le32(0x6);
- nesqp->nesqp_context->ssthresh = cpu_to_le32(0x3FFFC000);
- nesqp->nesqp_context->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss);
- nesqp->nesqp_context->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
- nesqp->nesqp_context->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- nesqp->nesqp_context->max_snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd);
-
- nes_debug(NES_DBG_CM, "QP%u: rcv_nxt = 0x%08X, snd_nxt = 0x%08X,"
- " Setting MSS to %u, PDWscale = 0x%08X, rcv_wnd = %u, context misc = 0x%08X.\n",
- nesqp->hwqp.qp_id, le32_to_cpu(nesqp->nesqp_context->rcv_nxt),
- le32_to_cpu(nesqp->nesqp_context->snd_nxt),
- cm_node->tcp_cntxt.mss, le32_to_cpu(nesqp->nesqp_context->pd_index_wscale),
- le32_to_cpu(nesqp->nesqp_context->rcv_wnd),
- le32_to_cpu(nesqp->nesqp_context->misc));
- nes_debug(NES_DBG_CM, " snd_wnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->snd_wnd));
- nes_debug(NES_DBG_CM, " snd_cwnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->cwnd));
- nes_debug(NES_DBG_CM, " max_swnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->max_snd_wnd));
-
- nes_debug(NES_DBG_CM, "Change cm_node state to TSA\n");
- cm_node->state = NES_CM_STATE_TSA;
-
- return ret;
-}
-
-
-/**
- * nes_cm_disconn
- */
-int nes_cm_disconn(struct nes_qp *nesqp)
-{
- struct disconn_work *work;
-
- work = kzalloc(sizeof *work, GFP_ATOMIC);
- if (!work)
- return -ENOMEM; /* Timer will clean up */
-
- nes_add_ref(&nesqp->ibqp);
- work->nesqp = nesqp;
- INIT_WORK(&work->work, nes_disconnect_worker);
- queue_work(g_cm_core->disconn_wq, &work->work);
- return 0;
-}
-
-
-/**
- * nes_disconnect_worker
- */
-static void nes_disconnect_worker(struct work_struct *work)
-{
- struct disconn_work *dwork = container_of(work, struct disconn_work, work);
- struct nes_qp *nesqp = dwork->nesqp;
-
- kfree(dwork);
- nes_debug(NES_DBG_CM, "processing AEQE id 0x%04X for QP%u.\n",
- nesqp->last_aeq, nesqp->hwqp.qp_id);
- nes_cm_disconn_true(nesqp);
- nes_rem_ref(&nesqp->ibqp);
-}
-
-
-/**
- * nes_cm_disconn_true
- */
-static int nes_cm_disconn_true(struct nes_qp *nesqp)
-{
- unsigned long flags;
- int ret = 0;
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- struct nes_vnic *nesvnic;
- u16 last_ae;
- u8 original_hw_tcp_state;
- u8 original_ibqp_state;
- int disconn_status = 0;
- int issue_disconn = 0;
- int issue_close = 0;
- int issue_flush = 0;
- u32 flush_q = NES_CQP_FLUSH_RQ;
- struct ib_event ibevent;
-
- if (!nesqp) {
- nes_debug(NES_DBG_CM, "disconnect_worker nesqp is NULL\n");
- return -1;
- }
-
- spin_lock_irqsave(&nesqp->lock, flags);
- cm_id = nesqp->cm_id;
- /* make sure we havent already closed this connection */
- if (!cm_id) {
- nes_debug(NES_DBG_CM, "QP%u disconnect_worker cmid is NULL\n",
- nesqp->hwqp.qp_id);
- spin_unlock_irqrestore(&nesqp->lock, flags);
- return -1;
- }
-
- nesvnic = to_nesvnic(nesqp->ibqp.device);
- nes_debug(NES_DBG_CM, "Disconnecting QP%u\n", nesqp->hwqp.qp_id);
-
- original_hw_tcp_state = nesqp->hw_tcp_state;
- original_ibqp_state = nesqp->ibqp_state;
- last_ae = nesqp->last_aeq;
-
- if (nesqp->term_flags) {
- issue_disconn = 1;
- issue_close = 1;
- nesqp->cm_id = NULL;
- del_timer(&nesqp->terminate_timer);
- if (nesqp->flush_issued == 0) {
- nesqp->flush_issued = 1;
- issue_flush = 1;
- }
- } else if ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
- ((original_ibqp_state == IB_QPS_RTS) &&
- (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
- issue_disconn = 1;
- if (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET)
- disconn_status = -ECONNRESET;
- }
-
- if (((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
- (original_hw_tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT) ||
- (last_ae == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) ||
- (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
- issue_close = 1;
- nesqp->cm_id = NULL;
- if (nesqp->flush_issued == 0) {
- nesqp->flush_issued = 1;
- issue_flush = 1;
- }
- }
-
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
- if ((issue_flush) && (nesqp->destroyed == 0)) {
- /* Flush the queue(s) */
- if (nesqp->hw_iwarp_state >= NES_AEQE_IWARP_STATE_TERMINATE)
- flush_q |= NES_CQP_FLUSH_SQ;
- flush_wqes(nesvnic->nesdev, nesqp, flush_q, 1);
-
- if (nesqp->term_flags) {
- ibevent.device = nesqp->ibqp.device;
- ibevent.event = nesqp->terminate_eventtype;
- ibevent.element.qp = &nesqp->ibqp;
- if (nesqp->ibqp.event_handler)
- nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
- }
- }
-
- if ((cm_id) && (cm_id->event_handler)) {
- if (issue_disconn) {
- atomic_inc(&cm_disconnects);
- cm_event.event = IW_CM_EVENT_DISCONNECT;
- cm_event.status = disconn_status;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
-
- nes_debug(NES_DBG_CM, "Generating a CM Disconnect Event"
- " for QP%u, SQ Head = %u, SQ Tail = %u. "
- "cm_id = %p, refcount = %u.\n",
- nesqp->hwqp.qp_id, nesqp->hwqp.sq_head,
- nesqp->hwqp.sq_tail, cm_id,
- atomic_read(&nesqp->refcount));
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- if (ret)
- nes_debug(NES_DBG_CM, "OFA CM event_handler "
- "returned, ret=%d\n", ret);
- }
-
- if (issue_close) {
- atomic_inc(&cm_closes);
- nes_disconnect(nesqp, 1);
-
- cm_id->provider_data = nesqp;
- /* Send up the close complete event */
- cm_event.event = IW_CM_EVENT_CLOSE;
- cm_event.status = 0;
- cm_event.provider_data = cm_id->provider_data;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- if (ret)
- nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
-
- cm_id->rem_ref(cm_id);
- }
- }
-
- return 0;
-}
-
-
-/**
- * nes_disconnect
- */
-static int nes_disconnect(struct nes_qp *nesqp, int abrupt)
-{
- int ret = 0;
- struct nes_vnic *nesvnic;
- struct nes_device *nesdev;
- struct nes_ib_device *nesibdev;
-
- nesvnic = to_nesvnic(nesqp->ibqp.device);
- if (!nesvnic)
- return -EINVAL;
-
- nesdev = nesvnic->nesdev;
- nesibdev = nesvnic->nesibdev;
-
- nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n",
- netdev_refcnt_read(nesvnic->netdev));
-
- if (nesqp->active_conn) {
-
- /* indicate this connection is NOT active */
- nesqp->active_conn = 0;
- } else {
- /* Need to free the Last Streaming Mode Message */
- if (nesqp->ietf_frame) {
- if (nesqp->lsmm_mr)
- nesibdev->ibdev.ops.dereg_mr(nesqp->lsmm_mr,
- NULL);
- pci_free_consistent(nesdev->pcidev,
- nesqp->private_data_len + nesqp->ietf_frame_size,
- nesqp->ietf_frame, nesqp->ietf_frame_pbase);
- }
- }
-
- /* close the CM node down if it is still active */
- if (nesqp->cm_node) {
- nes_debug(NES_DBG_CM, "Call close API\n");
-
- g_cm_core->api->close(g_cm_core, nesqp->cm_node);
- }
-
- return ret;
-}
-
-
-/**
- * nes_accept
- */
-int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
-{
- u64 u64temp;
- struct ib_qp *ibqp;
- struct nes_qp *nesqp;
- struct nes_vnic *nesvnic;
- struct nes_device *nesdev;
- struct nes_cm_node *cm_node;
- struct nes_adapter *adapter;
- struct ib_qp_attr attr;
- struct iw_cm_event cm_event;
- struct nes_hw_qp_wqe *wqe;
- struct nes_v4_quad nes_quad;
- u32 crc_value;
- int ret;
- int passive_state;
- struct ib_mr *ibmr = NULL;
- struct nes_pd *nespd;
- u64 tagged_offset;
- u8 mpa_frame_offset = 0;
- struct ietf_mpa_v2 *mpa_v2_frame;
- u8 start_addr = 0;
- u8 *start_ptr = &start_addr;
- u8 **start_buff = &start_ptr;
- u16 buff_len = 0;
- struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
- struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
-
- ibqp = nes_get_qp(cm_id->device, conn_param->qpn);
- if (!ibqp)
- return -EINVAL;
-
- /* get all our handles */
- nesqp = to_nesqp(ibqp);
- nesvnic = to_nesvnic(nesqp->ibqp.device);
- nesdev = nesvnic->nesdev;
- adapter = nesdev->nesadapter;
-
- cm_node = (struct nes_cm_node *)cm_id->provider_data;
- nes_debug(NES_DBG_CM, "nes_accept: cm_node= %p nesvnic=%p, netdev=%p,"
- "%s\n", cm_node, nesvnic, nesvnic->netdev,
- nesvnic->netdev->name);
-
- if (NES_CM_STATE_LISTENER_DESTROYED == cm_node->state) {
- if (cm_node->loopbackpartner)
- rem_ref_cm_node(cm_node->cm_core, cm_node->loopbackpartner);
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- return -EINVAL;
- }
-
- passive_state = atomic_add_return(1, &cm_node->passive_state);
- if (passive_state == NES_SEND_RESET_EVENT) {
- rem_ref_cm_node(cm_node->cm_core, cm_node);
- return -ECONNRESET;
- }
- /* associate the node with the QP */
- nesqp->cm_node = (void *)cm_node;
- cm_node->nesqp = nesqp;
-
-
- nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n",
- nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener);
- atomic_inc(&cm_accepts);
-
- nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n",
- netdev_refcnt_read(nesvnic->netdev));
-
- nesqp->ietf_frame_size = sizeof(struct ietf_mpa_v2);
- /* allocate the ietf frame and space for private data */
- nesqp->ietf_frame = pci_alloc_consistent(nesdev->pcidev,
- nesqp->ietf_frame_size + conn_param->private_data_len,
- &nesqp->ietf_frame_pbase);
-
- if (!nesqp->ietf_frame) {
- nes_debug(NES_DBG_CM, "Unable to allocate memory for private data\n");
- return -ENOMEM;
- }
- mpa_v2_frame = (struct ietf_mpa_v2 *)nesqp->ietf_frame;
-
- if (cm_node->mpa_frame_rev == IETF_MPA_V1)
- mpa_frame_offset = 4;
-
- if (cm_node->mpa_frame_rev == IETF_MPA_V1 ||
- cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
- record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord);
- }
-
- memcpy(mpa_v2_frame->priv_data, conn_param->private_data,
- conn_param->private_data_len);
-
- cm_build_mpa_frame(cm_node, start_buff, &buff_len, nesqp->ietf_frame, MPA_KEY_REPLY);
- nesqp->private_data_len = conn_param->private_data_len;
-
- /* setup our first outgoing iWarp send WQE (the IETF frame response) */
- wqe = &nesqp->hwqp.sq_vbase[0];
-
- if (raddr->sin_addr.s_addr != laddr->sin_addr.s_addr) {
- u64temp = (unsigned long)nesqp;
- nespd = nesqp->nespd;
- tagged_offset = (u64)(unsigned long)*start_buff;
- ibmr = nes_reg_phys_mr(&nespd->ibpd,
- nesqp->ietf_frame_pbase + mpa_frame_offset,
- buff_len, IB_ACCESS_LOCAL_WRITE,
- &tagged_offset);
- if (IS_ERR(ibmr)) {
- nes_debug(NES_DBG_CM, "Unable to register memory region"
- "for lSMM for cm_node = %p \n",
- cm_node);
- pci_free_consistent(nesdev->pcidev,
- nesqp->private_data_len + nesqp->ietf_frame_size,
- nesqp->ietf_frame, nesqp->ietf_frame_pbase);
- return PTR_ERR(ibmr);
- }
-
- ibmr->pd = &nespd->ibpd;
- ibmr->device = nespd->ibpd.device;
- nesqp->lsmm_mr = ibmr;
-
- u64temp |= NES_SW_CONTEXT_ALIGN >> 1;
- set_wqe_64bit_value(wqe->wqe_words,
- NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX,
- u64temp);
- wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] =
- cpu_to_le32(NES_IWARP_SQ_WQE_STREAMING |
- NES_IWARP_SQ_WQE_WRPDU);
- wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] =
- cpu_to_le32(buff_len);
- set_wqe_64bit_value(wqe->wqe_words,
- NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
- (u64)(unsigned long)(*start_buff));
- wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] =
- cpu_to_le32(buff_len);
- wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = ibmr->lkey;
- if (nesqp->sq_kmapped) {
- nesqp->sq_kmapped = 0;
- kunmap(nesqp->page);
- }
-
- nesqp->nesqp_context->ird_ord_sizes |=
- cpu_to_le32(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT |
- NES_QPCONTEXT_ORDIRD_WRPDU);
- } else {
- nesqp->nesqp_context->ird_ord_sizes |=
- cpu_to_le32(NES_QPCONTEXT_ORDIRD_WRPDU);
- }
- nesqp->skip_lsmm = 1;
-
- /* Cache the cm_id in the qp */
- nesqp->cm_id = cm_id;
- cm_node->cm_id = cm_id;
-
- /* nesqp->cm_node = (void *)cm_id->provider_data; */
- cm_id->provider_data = nesqp;
- nesqp->active_conn = 0;
-
- if (cm_node->state == NES_CM_STATE_TSA)
- nes_debug(NES_DBG_CM, "Already state = TSA for cm_node=%p\n",
- cm_node);
-
- nes_cm_init_tsa_conn(nesqp, cm_node);
-
- nesqp->nesqp_context->tcpPorts[0] =
- cpu_to_le16(cm_node->loc_port);
- nesqp->nesqp_context->tcpPorts[1] =
- cpu_to_le16(cm_node->rem_port);
-
- nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr);
-
- nesqp->nesqp_context->misc2 |= cpu_to_le32(
- (u32)PCI_FUNC(nesdev->pcidev->devfn) <<
- NES_QPCONTEXT_MISC2_SRC_IP_SHIFT);
-
- nesqp->nesqp_context->arp_index_vlan |=
- cpu_to_le32(nes_arp_table(nesdev,
- le32_to_cpu(nesqp->nesqp_context->ip0), NULL,
- NES_ARP_RESOLVE) << 16);
-
- nesqp->nesqp_context->ts_val_delta = cpu_to_le32(
- jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW));
-
- nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id);
-
- nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(
- ((u32)1 << NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT));
- nesqp->nesqp_context->ird_ord_sizes |=
- cpu_to_le32((u32)cm_node->ord_size);
-
- memset(&nes_quad, 0, sizeof(nes_quad));
- nes_quad.DstIpAdrIndex =
- cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
- nes_quad.SrcIpadr = htonl(cm_node->rem_addr);
- nes_quad.TcpPorts[0] = htons(cm_node->rem_port);
- nes_quad.TcpPorts[1] = htons(cm_node->loc_port);
-
- /* Produce hash key */
- crc_value = get_crc_value(&nes_quad);
- nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff);
- nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, CRC = 0x%08X\n",
- nesqp->hte_index, nesqp->hte_index & adapter->hte_index_mask);
-
- nesqp->hte_index &= adapter->hte_index_mask;
- nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index);
-
- cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node);
-
- nes_debug(NES_DBG_CM, "QP%u, Destination IP = 0x%08X:0x%04X, local = "
- "0x%08X:0x%04X, rcv_nxt=0x%08X, snd_nxt=0x%08X, mpa + "
- "private data length=%u.\n", nesqp->hwqp.qp_id,
- ntohl(raddr->sin_addr.s_addr), ntohs(raddr->sin_port),
- ntohl(laddr->sin_addr.s_addr), ntohs(laddr->sin_port),
- le32_to_cpu(nesqp->nesqp_context->rcv_nxt),
- le32_to_cpu(nesqp->nesqp_context->snd_nxt),
- buff_len);
-
- /* notify OF layer that accept event was successful */
- cm_id->add_ref(cm_id);
- nes_add_ref(&nesqp->ibqp);
-
- cm_event.event = IW_CM_EVENT_ESTABLISHED;
- cm_event.status = 0;
- cm_event.provider_data = (void *)nesqp;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
- cm_event.ird = cm_node->ird_size;
- cm_event.ord = cm_node->ord_size;
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- attr.qp_state = IB_QPS_RTS;
- nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);
- if (cm_node->loopbackpartner) {
- cm_node->loopbackpartner->mpa_frame_size =
- nesqp->private_data_len;
- /* copy entire MPA frame to our cm_node's frame */
- memcpy(cm_node->loopbackpartner->mpa_frame_buf,
- conn_param->private_data, conn_param->private_data_len);
- create_event(cm_node->loopbackpartner, NES_CM_EVENT_CONNECTED);
- }
- if (ret)
- printk(KERN_ERR "%s[%u] OFA CM event_handler returned, "
- "ret=%d\n", __func__, __LINE__, ret);
-
- return 0;
-}
-
-
-/**
- * nes_reject
- */
-int nes_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
-{
- struct nes_cm_node *cm_node;
- struct nes_cm_node *loopback;
- struct nes_cm_core *cm_core;
- u8 *start_buff;
-
- atomic_inc(&cm_rejects);
- cm_node = (struct nes_cm_node *)cm_id->provider_data;
- loopback = cm_node->loopbackpartner;
- cm_core = cm_node->cm_core;
- cm_node->cm_id = cm_id;
-
- if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER)
- return -EINVAL;
-
- if (loopback) {
- memcpy(&loopback->mpa_frame.priv_data, pdata, pdata_len);
- loopback->mpa_frame.priv_data_len = pdata_len;
- loopback->mpa_frame_size = pdata_len;
- } else {
- start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2);
- cm_node->mpa_frame_size = pdata_len;
- memcpy(start_buff, pdata, pdata_len);
- }
- return cm_core->api->reject(cm_core, cm_node);
-}
-
-
-/**
- * nes_connect
- * setup and launch cm connect node
- */
-int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
-{
- struct ib_qp *ibqp;
- struct nes_qp *nesqp;
- struct nes_vnic *nesvnic;
- struct nes_device *nesdev;
- struct nes_cm_node *cm_node;
- struct nes_cm_info cm_info;
- int apbvt_set = 0;
- struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
- struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
-
- if (cm_id->remote_addr.ss_family != AF_INET)
- return -ENOSYS;
- ibqp = nes_get_qp(cm_id->device, conn_param->qpn);
- if (!ibqp)
- return -EINVAL;
- nesqp = to_nesqp(ibqp);
- if (!nesqp)
- return -EINVAL;
- nesvnic = to_nesvnic(nesqp->ibqp.device);
- if (!nesvnic)
- return -EINVAL;
- nesdev = nesvnic->nesdev;
- if (!nesdev)
- return -EINVAL;
-
- if (!laddr->sin_port || !raddr->sin_port)
- return -EINVAL;
-
- nes_debug(NES_DBG_CM, "QP%u, current IP = 0x%08X, Destination IP = "
- "0x%08X:0x%04X, local = 0x%08X:0x%04X.\n", nesqp->hwqp.qp_id,
- ntohl(nesvnic->local_ipaddr), ntohl(raddr->sin_addr.s_addr),
- ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr),
- ntohs(laddr->sin_port));
-
- atomic_inc(&cm_connects);
- nesqp->active_conn = 1;
-
- /* cache the cm_id in the qp */
- nesqp->cm_id = cm_id;
- cm_id->provider_data = nesqp;
- nesqp->private_data_len = conn_param->private_data_len;
-
- nes_debug(NES_DBG_CM, "requested ord = 0x%08X.\n", (u32)conn_param->ord);
- nes_debug(NES_DBG_CM, "mpa private data len =%u\n",
- conn_param->private_data_len);
-
- /* set up the connection params for the node */
- cm_info.loc_addr = ntohl(laddr->sin_addr.s_addr);
- cm_info.loc_port = ntohs(laddr->sin_port);
- cm_info.rem_addr = ntohl(raddr->sin_addr.s_addr);
- cm_info.rem_port = ntohs(raddr->sin_port);
- cm_info.cm_id = cm_id;
- cm_info.conn_type = NES_CM_IWARP_CONN_TYPE;
-
- if (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr) {
- nes_manage_apbvt(nesvnic, cm_info.loc_port,
- PCI_FUNC(nesdev->pcidev->devfn),
- NES_MANAGE_APBVT_ADD);
- apbvt_set = 1;
- }
-
- cm_id->add_ref(cm_id);
-
- /* create a connect CM node connection */
- cm_node = g_cm_core->api->connect(g_cm_core, nesvnic,
- conn_param->private_data_len, (void *)conn_param->private_data,
- &cm_info);
- if (!cm_node) {
- if (apbvt_set)
- nes_manage_apbvt(nesvnic, cm_info.loc_port,
- PCI_FUNC(nesdev->pcidev->devfn),
- NES_MANAGE_APBVT_DEL);
-
- nes_debug(NES_DBG_NLMSG, "Delete loc_port = %04X\n",
- cm_info.loc_port);
- cm_id->rem_ref(cm_id);
- return -ENOMEM;
- }
-
- record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord);
- if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO &&
- cm_node->ord_size == 0)
- cm_node->ord_size = 1;
-
- cm_node->apbvt_set = apbvt_set;
- cm_node->tos = cm_id->tos;
- nesqp->cm_node = cm_node;
- cm_node->nesqp = nesqp;
- nes_add_ref(&nesqp->ibqp);
-
- return 0;
-}
-
-
-/**
- * nes_create_listen
- */
-int nes_create_listen(struct iw_cm_id *cm_id, int backlog)
-{
- struct nes_vnic *nesvnic;
- struct nes_cm_listener *cm_node;
- struct nes_cm_info cm_info;
- int err;
- struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
-
- nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n",
- cm_id, ntohs(laddr->sin_port));
-
- if (cm_id->m_local_addr.ss_family != AF_INET)
- return -ENOSYS;
- nesvnic = to_nesvnic(cm_id->device);
- if (!nesvnic)
- return -EINVAL;
-
- nes_debug(NES_DBG_CM, "nesvnic=%p, netdev=%p, %s\n",
- nesvnic, nesvnic->netdev, nesvnic->netdev->name);
-
- nes_debug(NES_DBG_CM, "nesvnic->local_ipaddr=0x%08x, sin_addr.s_addr=0x%08x\n",
- nesvnic->local_ipaddr, laddr->sin_addr.s_addr);
-
- /* setup listen params in our api call struct */
- cm_info.loc_addr = ntohl(nesvnic->local_ipaddr);
- cm_info.loc_port = ntohs(laddr->sin_port);
- cm_info.backlog = backlog;
- cm_info.cm_id = cm_id;
-
- cm_info.conn_type = NES_CM_IWARP_CONN_TYPE;
-
- cm_node = g_cm_core->api->listen(g_cm_core, nesvnic, &cm_info);
- if (!cm_node) {
- printk(KERN_ERR "%s[%u] Error returned from listen API call\n",
- __func__, __LINE__);
- return -ENOMEM;
- }
-
- cm_id->provider_data = cm_node;
- cm_node->tos = cm_id->tos;
-
- if (!cm_node->reused_node) {
- err = nes_manage_apbvt(nesvnic, cm_node->loc_port,
- PCI_FUNC(nesvnic->nesdev->pcidev->devfn),
- NES_MANAGE_APBVT_ADD);
- if (err) {
- printk(KERN_ERR "nes_manage_apbvt call returned %d.\n",
- err);
- g_cm_core->api->stop_listener(g_cm_core, (void *)cm_node);
- return err;
- }
- atomic_inc(&cm_listens_created);
- }
-
- cm_id->add_ref(cm_id);
- cm_id->provider_data = (void *)cm_node;
-
-
- return 0;
-}
-
-
-/**
- * nes_destroy_listen
- */
-int nes_destroy_listen(struct iw_cm_id *cm_id)
-{
- if (cm_id->provider_data)
- g_cm_core->api->stop_listener(g_cm_core, cm_id->provider_data);
- else
- nes_debug(NES_DBG_CM, "cm_id->provider_data was NULL\n");
-
- cm_id->rem_ref(cm_id);
-
- return 0;
-}
-
-
-/**
- * nes_cm_recv
- */
-int nes_cm_recv(struct sk_buff *skb, struct net_device *netdevice)
-{
- int rc = 0;
-
- cm_packets_received++;
- if ((g_cm_core) && (g_cm_core->api))
- rc = g_cm_core->api->recv_pkt(g_cm_core, netdev_priv(netdevice), skb);
- else
- nes_debug(NES_DBG_CM, "Unable to process packet for CM,"
- " cm is not setup properly.\n");
-
- return rc;
-}
-
-
-/**
- * nes_cm_start
- * Start and init a cm core module
- */
-int nes_cm_start(void)
-{
- nes_debug(NES_DBG_CM, "\n");
- /* create the primary CM core, pass this handle to subsequent core inits */
- g_cm_core = nes_cm_alloc_core();
- if (g_cm_core)
- return 0;
- else
- return -ENOMEM;
-}
-
-
-/**
- * nes_cm_stop
- * stop and dealloc all cm core instances
- */
-int nes_cm_stop(void)
-{
- g_cm_core->api->destroy_cm_core(g_cm_core);
- return 0;
-}
-
-
-/**
- * cm_event_connected
- * handle a connected event, setup QPs and HW
- */
-static void cm_event_connected(struct nes_cm_event *event)
-{
- struct nes_qp *nesqp;
- struct nes_vnic *nesvnic;
- struct nes_device *nesdev;
- struct nes_cm_node *cm_node;
- struct nes_adapter *nesadapter;
- struct ib_qp_attr attr;
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- struct nes_v4_quad nes_quad;
- u32 crc_value;
- int ret;
- struct sockaddr_in *laddr;
- struct sockaddr_in *raddr;
- struct sockaddr_in *cm_event_laddr;
-
- /* get all our handles */
- cm_node = event->cm_node;
- cm_id = cm_node->cm_id;
- nes_debug(NES_DBG_CM, "cm_event_connected - %p - cm_id = %p\n", cm_node, cm_id);
- nesqp = (struct nes_qp *)cm_id->provider_data;
- nesvnic = to_nesvnic(nesqp->ibqp.device);
- nesdev = nesvnic->nesdev;
- nesadapter = nesdev->nesadapter;
- laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
- raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
- cm_event_laddr = (struct sockaddr_in *)&cm_event.local_addr;
-
- if (nesqp->destroyed)
- return;
- atomic_inc(&cm_connecteds);
- nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on"
- " local port 0x%04X. jiffies = %lu.\n",
- nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr),
- ntohs(raddr->sin_port), ntohs(laddr->sin_port), jiffies);
-
- nes_cm_init_tsa_conn(nesqp, cm_node);
-
- /* set the QP tsa context */
- nesqp->nesqp_context->tcpPorts[0] =
- cpu_to_le16(cm_node->loc_port);
- nesqp->nesqp_context->tcpPorts[1] =
- cpu_to_le16(cm_node->rem_port);
- nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr);
-
- nesqp->nesqp_context->misc2 |= cpu_to_le32(
- (u32)PCI_FUNC(nesdev->pcidev->devfn) <<
- NES_QPCONTEXT_MISC2_SRC_IP_SHIFT);
- nesqp->nesqp_context->arp_index_vlan |= cpu_to_le32(
- nes_arp_table(nesdev,
- le32_to_cpu(nesqp->nesqp_context->ip0),
- NULL, NES_ARP_RESOLVE) << 16);
- nesqp->nesqp_context->ts_val_delta = cpu_to_le32(
- jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW));
- nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id);
- nesqp->nesqp_context->ird_ord_sizes |=
- cpu_to_le32((u32)1 <<
- NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT);
- nesqp->nesqp_context->ird_ord_sizes |=
- cpu_to_le32((u32)cm_node->ord_size);
-
- /* Adjust tail for not having a LSMM */
- /*nesqp->hwqp.sq_tail = 1;*/
-
- build_rdma0_msg(cm_node, &nesqp);
-
- nes_write32(nesdev->regs + NES_WQE_ALLOC,
- (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id);
-
- memset(&nes_quad, 0, sizeof(nes_quad));
-
- nes_quad.DstIpAdrIndex =
- cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
- nes_quad.SrcIpadr = htonl(cm_node->rem_addr);
- nes_quad.TcpPorts[0] = htons(cm_node->rem_port);
- nes_quad.TcpPorts[1] = htons(cm_node->loc_port);
-
- /* Produce hash key */
- crc_value = get_crc_value(&nes_quad);
- nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff);
- nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, After CRC = 0x%08X\n",
- nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask);
-
- nesqp->hte_index &= nesadapter->hte_index_mask;
- nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index);
-
- nesqp->ietf_frame = &cm_node->mpa_frame;
- nesqp->private_data_len = (u8)cm_node->mpa_frame_size;
- cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node);
-
- /* notify OF layer we successfully created the requested connection */
- cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
- cm_event.status = 0;
- cm_event.provider_data = cm_id->provider_data;
- cm_event_laddr->sin_family = AF_INET;
- cm_event_laddr->sin_port = laddr->sin_port;
- cm_event.remote_addr = cm_id->m_remote_addr;
-
- cm_event.private_data = (void *)event->cm_node->mpa_frame_buf;
- cm_event.private_data_len = (u8)event->cm_node->mpa_frame_size;
- cm_event.ird = cm_node->ird_size;
- cm_event.ord = cm_node->ord_size;
-
- cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr);
- ret = cm_id->event_handler(cm_id, &cm_event);
- nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
-
- if (ret)
- printk(KERN_ERR "%s[%u] OFA CM event_handler returned, "
- "ret=%d\n", __func__, __LINE__, ret);
- attr.qp_state = IB_QPS_RTS;
- nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);
-
- nes_debug(NES_DBG_CM, "Exiting connect thread for QP%u. jiffies = "
- "%lu\n", nesqp->hwqp.qp_id, jiffies);
-
- return;
-}
-
-
-/**
- * cm_event_connect_error
- */
-static void cm_event_connect_error(struct nes_cm_event *event)
-{
- struct nes_qp *nesqp;
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- /* struct nes_cm_info cm_info; */
- int ret;
-
- if (!event->cm_node)
- return;
-
- cm_id = event->cm_node->cm_id;
- if (!cm_id)
- return;
-
- nes_debug(NES_DBG_CM, "cm_node=%p, cm_id=%p\n", event->cm_node, cm_id);
- nesqp = cm_id->provider_data;
-
- if (!nesqp)
- return;
-
- /* notify OF layer about this connection error event */
- /* cm_id->rem_ref(cm_id); */
- nesqp->cm_id = NULL;
- cm_id->provider_data = NULL;
- cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
- cm_event.status = -ECONNRESET;
- cm_event.provider_data = cm_id->provider_data;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
-
-#ifdef CONFIG_INFINIBAND_NES_DEBUG
- {
- struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
- &cm_event.local_addr;
- struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
- &cm_event.remote_addr;
- nes_debug(NES_DBG_CM, "call CM_EVENT REJECTED, local_addr=%08x, remote_addr=%08x\n",
- cm_event_laddr->sin_addr.s_addr, cm_event_raddr->sin_addr.s_addr);
- }
-#endif
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
- if (ret)
- printk(KERN_ERR "%s[%u] OFA CM event_handler returned, "
- "ret=%d\n", __func__, __LINE__, ret);
- cm_id->rem_ref(cm_id);
-
- rem_ref_cm_node(event->cm_node->cm_core, event->cm_node);
- return;
-}
-
-
-/**
- * cm_event_reset
- */
-static void cm_event_reset(struct nes_cm_event *event)
-{
- struct nes_qp *nesqp;
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- /* struct nes_cm_info cm_info; */
- int ret;
-
- if (!event->cm_node)
- return;
-
- if (!event->cm_node->cm_id)
- return;
-
- cm_id = event->cm_node->cm_id;
-
- nes_debug(NES_DBG_CM, "%p - cm_id = %p\n", event->cm_node, cm_id);
- nesqp = cm_id->provider_data;
- if (!nesqp)
- return;
-
- nesqp->cm_id = NULL;
- /* cm_id->provider_data = NULL; */
- cm_event.event = IW_CM_EVENT_DISCONNECT;
- cm_event.status = -ECONNRESET;
- cm_event.provider_data = cm_id->provider_data;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
-
- cm_id->add_ref(cm_id);
- ret = cm_id->event_handler(cm_id, &cm_event);
- atomic_inc(&cm_closes);
- cm_event.event = IW_CM_EVENT_CLOSE;
- cm_event.status = 0;
- cm_event.provider_data = cm_id->provider_data;
- cm_event.local_addr = cm_id->m_local_addr;
- cm_event.remote_addr = cm_id->m_remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
- nes_debug(NES_DBG_CM, "NODE %p Generating CLOSE\n", event->cm_node);
- ret = cm_id->event_handler(cm_id, &cm_event);
-
- nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
-
-
- /* notify OF layer about this connection error event */
- cm_id->rem_ref(cm_id);
-
- return;
-}
-
-
-/**
- * cm_event_mpa_req
- */
-static void cm_event_mpa_req(struct nes_cm_event *event)
-{
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- int ret;
- struct nes_cm_node *cm_node;
- struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
- &cm_event.local_addr;
- struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
- &cm_event.remote_addr;
-
- cm_node = event->cm_node;
- if (!cm_node)
- return;
- cm_id = cm_node->cm_id;
-
- atomic_inc(&cm_connect_reqs);
- nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
- cm_node, cm_id, jiffies);
-
- cm_event.event = IW_CM_EVENT_CONNECT_REQUEST;
- cm_event.status = 0;
- cm_event.provider_data = (void *)cm_node;
-
- cm_event_laddr->sin_family = AF_INET;
- cm_event_laddr->sin_port = htons(event->cm_info.loc_port);
- cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr);
-
- cm_event_raddr->sin_family = AF_INET;
- cm_event_raddr->sin_port = htons(event->cm_info.rem_port);
- cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr);
- cm_event.private_data = cm_node->mpa_frame_buf;
- cm_event.private_data_len = (u8)cm_node->mpa_frame_size;
- if (cm_node->mpa_frame_rev == IETF_MPA_V1) {
- cm_event.ird = NES_MAX_IRD;
- cm_event.ord = NES_MAX_ORD;
- } else {
- cm_event.ird = cm_node->ird_size;
- cm_event.ord = cm_node->ord_size;
- }
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- if (ret)
- printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n",
- __func__, __LINE__, ret);
- return;
-}
-
-
-static void cm_event_mpa_reject(struct nes_cm_event *event)
-{
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- struct nes_cm_node *cm_node;
- int ret;
- struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
- &cm_event.local_addr;
- struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
- &cm_event.remote_addr;
-
- cm_node = event->cm_node;
- if (!cm_node)
- return;
- cm_id = cm_node->cm_id;
-
- atomic_inc(&cm_connect_reqs);
- nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
- cm_node, cm_id, jiffies);
-
- cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
- cm_event.status = -ECONNREFUSED;
- cm_event.provider_data = cm_id->provider_data;
-
- cm_event_laddr->sin_family = AF_INET;
- cm_event_laddr->sin_port = htons(event->cm_info.loc_port);
- cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr);
-
- cm_event_raddr->sin_family = AF_INET;
- cm_event_raddr->sin_port = htons(event->cm_info.rem_port);
- cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr);
-
- cm_event.private_data = cm_node->mpa_frame_buf;
- cm_event.private_data_len = (u8)cm_node->mpa_frame_size;
-
- nes_debug(NES_DBG_CM, "call CM_EVENT_MPA_REJECTED, local_addr=%08x, "
- "remove_addr=%08x\n",
- cm_event_laddr->sin_addr.s_addr,
- cm_event_raddr->sin_addr.s_addr);
-
- ret = cm_id->event_handler(cm_id, &cm_event);
- if (ret)
- printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n",
- __func__, __LINE__, ret);
-
- return;
-}
-
-
-static void nes_cm_event_handler(struct work_struct *);
-
-/**
- * nes_cm_post_event
- * post an event to the cm event handler
- */
-static int nes_cm_post_event(struct nes_cm_event *event)
-{
- atomic_inc(&event->cm_node->cm_core->events_posted);
- add_ref_cm_node(event->cm_node);
- event->cm_info.cm_id->add_ref(event->cm_info.cm_id);
- INIT_WORK(&event->event_work, nes_cm_event_handler);
- nes_debug(NES_DBG_CM, "cm_node=%p queue_work, event=%p\n",
- event->cm_node, event);
-
- queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
-
- nes_debug(NES_DBG_CM, "Exit\n");
- return 0;
-}
-
-
-/**
- * nes_cm_event_handler
- * worker function to handle cm events
- * will free instance of nes_cm_event
- */
-static void nes_cm_event_handler(struct work_struct *work)
-{
- struct nes_cm_event *event = container_of(work, struct nes_cm_event,
- event_work);
- struct nes_cm_core *cm_core;
-
- if ((!event) || (!event->cm_node) || (!event->cm_node->cm_core))
- return;
-
- cm_core = event->cm_node->cm_core;
- nes_debug(NES_DBG_CM, "event=%p, event->type=%u, events posted=%u\n",
- event, event->type, atomic_read(&cm_core->events_posted));
-
- switch (event->type) {
- case NES_CM_EVENT_MPA_REQ:
- cm_event_mpa_req(event);
- nes_debug(NES_DBG_CM, "cm_node=%p CM Event: MPA REQUEST\n",
- event->cm_node);
- break;
- case NES_CM_EVENT_RESET:
- nes_debug(NES_DBG_CM, "cm_node = %p CM Event: RESET\n",
- event->cm_node);
- cm_event_reset(event);
- break;
- case NES_CM_EVENT_CONNECTED:
- if ((!event->cm_node->cm_id) ||
- (event->cm_node->state != NES_CM_STATE_TSA))
- break;
- cm_event_connected(event);
- nes_debug(NES_DBG_CM, "CM Event: CONNECTED\n");
- break;
- case NES_CM_EVENT_MPA_REJECT:
- if ((!event->cm_node->cm_id) ||
- (event->cm_node->state == NES_CM_STATE_TSA))
- break;
- cm_event_mpa_reject(event);
- nes_debug(NES_DBG_CM, "CM Event: REJECT\n");
- break;
-
- case NES_CM_EVENT_ABORTED:
- if ((!event->cm_node->cm_id) ||
- (event->cm_node->state == NES_CM_STATE_TSA))
- break;
- cm_event_connect_error(event);
- nes_debug(NES_DBG_CM, "CM Event: ABORTED\n");
- break;
- case NES_CM_EVENT_DROPPED_PKT:
- nes_debug(NES_DBG_CM, "CM Event: DROPPED PKT\n");
- break;
- default:
- nes_debug(NES_DBG_CM, "CM Event: UNKNOWN EVENT TYPE\n");
- break;
- }
-
- atomic_dec(&cm_core->events_posted);
- event->cm_info.cm_id->rem_ref(event->cm_info.cm_id);
- rem_ref_cm_node(cm_core, event->cm_node);
- kfree(event);
-
- return;
-}
diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h
deleted file mode 100644
index b9cc02b4e8d5..000000000000
--- a/drivers/infiniband/hw/nes/nes_cm.h
+++ /dev/null
@@ -1,470 +0,0 @@
-/*
- * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#ifndef NES_CM_H
-#define NES_CM_H
-
-#define QUEUE_EVENTS
-
-#define NES_MANAGE_APBVT_DEL 0
-#define NES_MANAGE_APBVT_ADD 1
-
-#define NES_MPA_REQUEST_ACCEPT 1
-#define NES_MPA_REQUEST_REJECT 2
-
-/* IETF MPA -- defines, enums, structs */
-#define IEFT_MPA_KEY_REQ "MPA ID Req Frame"
-#define IEFT_MPA_KEY_REP "MPA ID Rep Frame"
-#define IETF_MPA_KEY_SIZE 16
-#define IETF_MPA_VERSION 1
-#define IETF_MAX_PRIV_DATA_LEN 512
-#define IETF_MPA_FRAME_SIZE 20
-#define IETF_RTR_MSG_SIZE 4
-#define IETF_MPA_V2_FLAG 0x10
-
-/* IETF RTR MSG Fields */
-#define IETF_PEER_TO_PEER 0x8000
-#define IETF_FLPDU_ZERO_LEN 0x4000
-#define IETF_RDMA0_WRITE 0x8000
-#define IETF_RDMA0_READ 0x4000
-#define IETF_NO_IRD_ORD 0x3FFF
-#define NES_MAX_IRD 0x40
-#define NES_MAX_ORD 0x7F
-
-enum ietf_mpa_flags {
- IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */
- IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */
- IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */
-};
-
-struct ietf_mpa_v1 {
- u8 key[IETF_MPA_KEY_SIZE];
- u8 flags;
- u8 rev;
- __be16 priv_data_len;
- u8 priv_data[0];
-};
-
-#define ietf_mpa_req_resp_frame ietf_mpa_frame
-
-struct ietf_rtr_msg {
- __be16 ctrl_ird;
- __be16 ctrl_ord;
-};
-
-struct ietf_mpa_v2 {
- u8 key[IETF_MPA_KEY_SIZE];
- u8 flags;
- u8 rev;
- __be16 priv_data_len;
- struct ietf_rtr_msg rtr_msg;
- u8 priv_data[0];
-};
-
-struct nes_v4_quad {
- u32 rsvd0;
- __le32 DstIpAdrIndex; /* Only most significant 5 bits are valid */
- __be32 SrcIpadr;
- __be16 TcpPorts[2]; /* src is low, dest is high */
-};
-
-struct nes_cm_node;
-enum nes_timer_type {
- NES_TIMER_TYPE_SEND,
- NES_TIMER_TYPE_RECV,
- NES_TIMER_NODE_CLEANUP,
- NES_TIMER_TYPE_CLOSE,
-};
-
-#define NES_PASSIVE_STATE_INDICATED 0
-#define NES_DO_NOT_SEND_RESET_EVENT 1
-#define NES_SEND_RESET_EVENT 2
-
-#define MAX_NES_IFS 4
-
-#define SET_ACK 1
-#define SET_SYN 2
-#define SET_FIN 4
-#define SET_RST 8
-
-#define TCP_OPTIONS_PADDING 3
-
-struct option_base {
- u8 optionnum;
- u8 length;
-};
-
-enum option_numbers {
- OPTION_NUMBER_END,
- OPTION_NUMBER_NONE,
- OPTION_NUMBER_MSS,
- OPTION_NUMBER_WINDOW_SCALE,
- OPTION_NUMBER_SACK_PERM,
- OPTION_NUMBER_SACK,
- OPTION_NUMBER_WRITE0 = 0xbc
-};
-
-struct option_mss {
- u8 optionnum;
- u8 length;
- __be16 mss;
-};
-
-struct option_windowscale {
- u8 optionnum;
- u8 length;
- u8 shiftcount;
-};
-
-union all_known_options {
- char as_end;
- struct option_base as_base;
- struct option_mss as_mss;
- struct option_windowscale as_windowscale;
-};
-
-struct nes_timer_entry {
- struct list_head list;
- unsigned long timetosend; /* jiffies */
- struct sk_buff *skb;
- u32 type;
- u32 retrycount;
- u32 retranscount;
- u32 context;
- u32 seq_num;
- u32 send_retrans;
- int close_when_complete;
- struct net_device *netdev;
-};
-
-#define NES_DEFAULT_RETRYS 64
-#define NES_DEFAULT_RETRANS 8
-#ifdef CONFIG_INFINIBAND_NES_DEBUG
-#define NES_RETRY_TIMEOUT (1000*HZ/1000)
-#else
-#define NES_RETRY_TIMEOUT (3000*HZ/1000)
-#endif
-#define NES_SHORT_TIME (10)
-#define NES_LONG_TIME (2000*HZ/1000)
-#define NES_MAX_TIMEOUT ((unsigned long) (12*HZ))
-
-#define NES_CM_HASHTABLE_SIZE 1024
-#define NES_CM_TCP_TIMER_INTERVAL 3000
-#define NES_CM_DEFAULT_MTU 1540
-#define NES_CM_DEFAULT_FRAME_CNT 10
-#define NES_CM_THREAD_STACK_SIZE 256
-#define NES_CM_DEFAULT_RCV_WND 64240 // before we know that window scaling is allowed
-#define NES_CM_DEFAULT_RCV_WND_SCALED 256960 // after we know that window scaling is allowed
-#define NES_CM_DEFAULT_RCV_WND_SCALE 2
-#define NES_CM_DEFAULT_FREE_PKTS 0x000A
-#define NES_CM_FREE_PKT_LO_WATERMARK 2
-
-#define NES_CM_DEFAULT_MSS 536
-
-#define NES_CM_DEF_SEQ 0x159bf75f
-#define NES_CM_DEF_LOCAL_ID 0x3b47
-
-#define NES_CM_DEF_SEQ2 0x18ed5740
-#define NES_CM_DEF_LOCAL_ID2 0xb807
-#define MAX_CM_BUFFER (IETF_MPA_FRAME_SIZE + IETF_RTR_MSG_SIZE + IETF_MAX_PRIV_DATA_LEN)
-
-typedef u32 nes_addr_t;
-
-#define nes_cm_tsa_context nes_qp_context
-
-struct nes_qp;
-
-/* cm node transition states */
-enum nes_cm_node_state {
- NES_CM_STATE_UNKNOWN,
- NES_CM_STATE_INITED,
- NES_CM_STATE_LISTENING,
- NES_CM_STATE_SYN_RCVD,
- NES_CM_STATE_SYN_SENT,
- NES_CM_STATE_ONE_SIDE_ESTABLISHED,
- NES_CM_STATE_ESTABLISHED,
- NES_CM_STATE_ACCEPTING,
- NES_CM_STATE_MPAREQ_SENT,
- NES_CM_STATE_MPAREQ_RCVD,
- NES_CM_STATE_MPAREJ_RCVD,
- NES_CM_STATE_TSA,
- NES_CM_STATE_FIN_WAIT1,
- NES_CM_STATE_FIN_WAIT2,
- NES_CM_STATE_CLOSE_WAIT,
- NES_CM_STATE_TIME_WAIT,
- NES_CM_STATE_LAST_ACK,
- NES_CM_STATE_CLOSING,
- NES_CM_STATE_LISTENER_DESTROYED,
- NES_CM_STATE_CLOSED
-};
-
-enum mpa_frame_version {
- IETF_MPA_V1 = 1,
- IETF_MPA_V2 = 2
-};
-
-enum mpa_frame_key {
- MPA_KEY_REQUEST,
- MPA_KEY_REPLY
-};
-
-enum send_rdma0 {
- SEND_RDMA_READ_ZERO = 1,
- SEND_RDMA_WRITE_ZERO = 2
-};
-
-enum nes_tcpip_pkt_type {
- NES_PKT_TYPE_UNKNOWN,
- NES_PKT_TYPE_SYN,
- NES_PKT_TYPE_SYNACK,
- NES_PKT_TYPE_ACK,
- NES_PKT_TYPE_FIN,
- NES_PKT_TYPE_RST
-};
-
-
-/* type of nes connection */
-enum nes_cm_conn_type {
- NES_CM_IWARP_CONN_TYPE,
-};
-
-/* CM context params */
-struct nes_cm_tcp_context {
- u8 client;
-
- u32 loc_seq_num;
- u32 loc_ack_num;
- u32 rem_ack_num;
- u32 rcv_nxt;
-
- u32 loc_id;
- u32 rem_id;
-
- u32 snd_wnd;
- u32 max_snd_wnd;
-
- u32 rcv_wnd;
- u32 mss;
- u8 snd_wscale;
- u8 rcv_wscale;
-
- struct nes_cm_tsa_context tsa_cntxt;
-};
-
-
-enum nes_cm_listener_state {
- NES_CM_LISTENER_PASSIVE_STATE = 1,
- NES_CM_LISTENER_ACTIVE_STATE = 2,
- NES_CM_LISTENER_EITHER_STATE = 3
-};
-
-struct nes_cm_listener {
- struct list_head list;
- struct nes_cm_core *cm_core;
- u8 loc_mac[ETH_ALEN];
- nes_addr_t loc_addr;
- u16 loc_port;
- struct iw_cm_id *cm_id;
- enum nes_cm_conn_type conn_type;
- atomic_t ref_count;
- struct nes_vnic *nesvnic;
- atomic_t pend_accepts_cnt;
- int backlog;
- enum nes_cm_listener_state listener_state;
- u32 reused_node;
- u8 tos;
-};
-
-/* per connection node and node state information */
-struct nes_cm_node {
- nes_addr_t loc_addr, rem_addr;
- u16 loc_port, rem_port;
-
- u8 loc_mac[ETH_ALEN];
- u8 rem_mac[ETH_ALEN];
-
- enum nes_cm_node_state state;
- struct nes_cm_tcp_context tcp_cntxt;
- struct nes_cm_core *cm_core;
- struct sk_buff_head resend_list;
- atomic_t ref_count;
- struct net_device *netdev;
-
- struct nes_cm_node *loopbackpartner;
-
- struct nes_timer_entry *send_entry;
- struct nes_timer_entry *recv_entry;
- spinlock_t retrans_list_lock;
- enum send_rdma0 send_rdma0_op;
-
- union {
- struct ietf_mpa_v1 mpa_frame;
- struct ietf_mpa_v2 mpa_v2_frame;
- u8 mpa_frame_buf[MAX_CM_BUFFER];
- };
- enum mpa_frame_version mpa_frame_rev;
- u16 ird_size;
- u16 ord_size;
- u16 mpav2_ird_ord;
-
- u16 mpa_frame_size;
- struct iw_cm_id *cm_id;
- struct list_head list;
- bool accelerated;
- struct nes_cm_listener *listener;
- enum nes_cm_conn_type conn_type;
- struct nes_vnic *nesvnic;
- int apbvt_set;
- int accept_pend;
- struct list_head timer_entry;
- struct list_head reset_entry;
- struct nes_qp *nesqp;
- atomic_t passive_state;
- u8 tos;
-};
-
-/* structure for client or CM to fill when making CM api calls. */
-/* - only need to set relevant data, based on op. */
-struct nes_cm_info {
- union {
- struct iw_cm_id *cm_id;
- struct net_device *netdev;
- };
-
- u16 loc_port;
- u16 rem_port;
- nes_addr_t loc_addr;
- nes_addr_t rem_addr;
- enum nes_cm_conn_type conn_type;
- int backlog;
-};
-
-/* CM event codes */
-enum nes_cm_event_type {
- NES_CM_EVENT_UNKNOWN,
- NES_CM_EVENT_ESTABLISHED,
- NES_CM_EVENT_MPA_REQ,
- NES_CM_EVENT_MPA_CONNECT,
- NES_CM_EVENT_MPA_ACCEPT,
- NES_CM_EVENT_MPA_REJECT,
- NES_CM_EVENT_MPA_ESTABLISHED,
- NES_CM_EVENT_CONNECTED,
- NES_CM_EVENT_CLOSED,
- NES_CM_EVENT_RESET,
- NES_CM_EVENT_DROPPED_PKT,
- NES_CM_EVENT_CLOSE_IMMED,
- NES_CM_EVENT_CLOSE_HARD,
- NES_CM_EVENT_CLOSE_CLEAN,
- NES_CM_EVENT_ABORTED,
- NES_CM_EVENT_SEND_FIRST
-};
-
-/* event to post to CM event handler */
-struct nes_cm_event {
- enum nes_cm_event_type type;
-
- struct nes_cm_info cm_info;
- struct work_struct event_work;
- struct nes_cm_node *cm_node;
-};
-
-struct nes_cm_core {
- enum nes_cm_node_state state;
-
- atomic_t listen_node_cnt;
- struct nes_cm_node listen_list;
- spinlock_t listen_list_lock;
-
- u32 mtu;
- u32 free_tx_pkt_max;
- u32 rx_pkt_posted;
- atomic_t ht_node_cnt;
- struct list_head connected_nodes;
- /* struct list_head hashtable[NES_CM_HASHTABLE_SIZE]; */
- spinlock_t ht_lock;
-
- struct timer_list tcp_timer;
-
- const struct nes_cm_ops *api;
-
- int (*post_event)(struct nes_cm_event *event);
- atomic_t events_posted;
- struct workqueue_struct *event_wq;
- struct workqueue_struct *disconn_wq;
-
- atomic_t node_cnt;
- u64 aborted_connects;
- u32 options;
-
- struct nes_cm_node *current_listen_node;
-};
-
-
-#define NES_CM_SET_PKT_SIZE (1 << 1)
-#define NES_CM_SET_FREE_PKT_Q_SIZE (1 << 2)
-
-/* CM ops/API for client interface */
-struct nes_cm_ops {
- int (*accelerated)(struct nes_cm_core *, struct nes_cm_node *);
- struct nes_cm_listener * (*listen)(struct nes_cm_core *, struct nes_vnic *,
- struct nes_cm_info *);
- int (*stop_listener)(struct nes_cm_core *, struct nes_cm_listener *);
- struct nes_cm_node * (*connect)(struct nes_cm_core *,
- struct nes_vnic *, u16, void *,
- struct nes_cm_info *);
- int (*close)(struct nes_cm_core *, struct nes_cm_node *);
- int (*accept)(struct nes_cm_core *, struct nes_cm_node *);
- int (*reject)(struct nes_cm_core *, struct nes_cm_node *);
- int (*recv_pkt)(struct nes_cm_core *, struct nes_vnic *,
- struct sk_buff *);
- int (*destroy_cm_core)(struct nes_cm_core *);
- int (*get)(struct nes_cm_core *);
- int (*set)(struct nes_cm_core *, u32, u32);
-};
-
-int schedule_nes_timer(struct nes_cm_node *, struct sk_buff *,
- enum nes_timer_type, int, int);
-
-int nes_accept(struct iw_cm_id *, struct iw_cm_conn_param *);
-int nes_reject(struct iw_cm_id *, const void *, u8);
-int nes_connect(struct iw_cm_id *, struct iw_cm_conn_param *);
-int nes_create_listen(struct iw_cm_id *, int);
-int nes_destroy_listen(struct iw_cm_id *);
-
-int nes_cm_recv(struct sk_buff *, struct net_device *);
-int nes_cm_start(void);
-int nes_cm_stop(void);
-int nes_add_ref_cm_node(struct nes_cm_node *cm_node);
-int nes_rem_ref_cm_node(struct nes_cm_node *cm_node);
-
-#endif /* NES_CM_H */
diff --git a/drivers/infiniband/hw/nes/nes_context.h b/drivers/infiniband/hw/nes/nes_context.h
deleted file mode 100644
index a69eef16d72d..000000000000
--- a/drivers/infiniband/hw/nes/nes_context.h
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef NES_CONTEXT_H
-#define NES_CONTEXT_H
-
-struct nes_qp_context {
- __le32 misc;
- __le32 cqs;
- __le32 sq_addr_low;
- __le32 sq_addr_high;
- __le32 rq_addr_low;
- __le32 rq_addr_high;
- __le32 misc2;
- __le16 tcpPorts[2];
- __le32 ip0;
- __le32 ip1;
- __le32 ip2;
- __le32 ip3;
- __le32 mss;
- __le32 arp_index_vlan;
- __le32 tcp_state_flow_label;
- __le32 pd_index_wscale;
- __le32 keepalive;
- u32 ts_recent;
- u32 ts_age;
- __le32 snd_nxt;
- __le32 snd_wnd;
- __le32 rcv_nxt;
- __le32 rcv_wnd;
- __le32 snd_max;
- __le32 snd_una;
- u32 srtt;
- __le32 rttvar;
- __le32 ssthresh;
- __le32 cwnd;
- __le32 snd_wl1;
- __le32 snd_wl2;
- __le32 max_snd_wnd;
- __le32 ts_val_delta;
- u32 retransmit;
- u32 probe_cnt;
- u32 hte_index;
- __le32 q2_addr_low;
- __le32 q2_addr_high;
- __le32 ird_index;
- u32 Rsvd3;
- __le32 ird_ord_sizes;
- u32 mrkr_offset;
- __le32 aeq_token_low;
- __le32 aeq_token_high;
-};
-
-/* QP Context Misc Field */
-
-#define NES_QPCONTEXT_MISC_IWARP_VER_MASK 0x00000003
-#define NES_QPCONTEXT_MISC_IWARP_VER_SHIFT 0
-#define NES_QPCONTEXT_MISC_EFB_SIZE_MASK 0x000000C0
-#define NES_QPCONTEXT_MISC_EFB_SIZE_SHIFT 6
-#define NES_QPCONTEXT_MISC_RQ_SIZE_MASK 0x00000300
-#define NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT 8
-#define NES_QPCONTEXT_MISC_SQ_SIZE_MASK 0x00000c00
-#define NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT 10
-#define NES_QPCONTEXT_MISC_PCI_FCN_MASK 0x00007000
-#define NES_QPCONTEXT_MISC_PCI_FCN_SHIFT 12
-#define NES_QPCONTEXT_MISC_DUP_ACKS_MASK 0x00070000
-#define NES_QPCONTEXT_MISC_DUP_ACKS_SHIFT 16
-
-enum nes_qp_context_misc_bits {
- NES_QPCONTEXT_MISC_RX_WQE_SIZE = 0x00000004,
- NES_QPCONTEXT_MISC_IPV4 = 0x00000008,
- NES_QPCONTEXT_MISC_DO_NOT_FRAG = 0x00000010,
- NES_QPCONTEXT_MISC_INSERT_VLAN = 0x00000020,
- NES_QPCONTEXT_MISC_DROS = 0x00008000,
- NES_QPCONTEXT_MISC_WSCALE = 0x00080000,
- NES_QPCONTEXT_MISC_KEEPALIVE = 0x00100000,
- NES_QPCONTEXT_MISC_TIMESTAMP = 0x00200000,
- NES_QPCONTEXT_MISC_SACK = 0x00400000,
- NES_QPCONTEXT_MISC_RDMA_WRITE_EN = 0x00800000,
- NES_QPCONTEXT_MISC_RDMA_READ_EN = 0x01000000,
- NES_QPCONTEXT_MISC_WBIND_EN = 0x10000000,
- NES_QPCONTEXT_MISC_FAST_REGISTER_EN = 0x20000000,
- NES_QPCONTEXT_MISC_PRIV_EN = 0x40000000,
- NES_QPCONTEXT_MISC_NO_NAGLE = 0x80000000
-};
-
-enum nes_qp_acc_wq_sizes {
- HCONTEXT_TSA_WQ_SIZE_4 = 0,
- HCONTEXT_TSA_WQ_SIZE_32 = 1,
- HCONTEXT_TSA_WQ_SIZE_128 = 2,
- HCONTEXT_TSA_WQ_SIZE_512 = 3
-};
-
-/* QP Context Misc2 Fields */
-#define NES_QPCONTEXT_MISC2_TTL_MASK 0x000000ff
-#define NES_QPCONTEXT_MISC2_TTL_SHIFT 0
-#define NES_QPCONTEXT_MISC2_HOP_LIMIT_MASK 0x000000ff
-#define NES_QPCONTEXT_MISC2_HOP_LIMIT_SHIFT 0
-#define NES_QPCONTEXT_MISC2_LIMIT_MASK 0x00000300
-#define NES_QPCONTEXT_MISC2_LIMIT_SHIFT 8
-#define NES_QPCONTEXT_MISC2_NIC_INDEX_MASK 0x0000fc00
-#define NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT 10
-#define NES_QPCONTEXT_MISC2_SRC_IP_MASK 0x001f0000
-#define NES_QPCONTEXT_MISC2_SRC_IP_SHIFT 16
-#define NES_QPCONTEXT_MISC2_TOS_MASK 0xff000000
-#define NES_QPCONTEXT_MISC2_TOS_SHIFT 24
-#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_MASK 0xff000000
-#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_SHIFT 24
-
-/* QP Context Tcp State/Flow Label Fields */
-#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_MASK 0x000fffff
-#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_SHIFT 0
-#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_MASK 0xf0000000
-#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT 28
-
-enum nes_qp_tcp_state {
- NES_QPCONTEXT_TCPSTATE_CLOSED = 1,
- NES_QPCONTEXT_TCPSTATE_EST = 5,
- NES_QPCONTEXT_TCPSTATE_TIME_WAIT = 11,
-};
-
-/* QP Context PD Index/wscale Fields */
-#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK 0x0000000f
-#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT 0
-#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK 0x00000f00
-#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT 8
-#define NES_QPCONTEXT_PDWSCALE_PDINDEX_MASK 0xffff0000
-#define NES_QPCONTEXT_PDWSCALE_PDINDEX_SHIFT 16
-
-/* QP Context Keepalive Fields */
-#define NES_QPCONTEXT_KEEPALIVE_DELTA_MASK 0x0000ffff
-#define NES_QPCONTEXT_KEEPALIVE_DELTA_SHIFT 0
-#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_MASK 0x00ff0000
-#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_SHIFT 16
-#define NES_QPCONTEXT_KEEPALIVE_INTV_MASK 0xff000000
-#define NES_QPCONTEXT_KEEPALIVE_INTV_SHIFT 24
-
-/* QP Context ORD/IRD Fields */
-#define NES_QPCONTEXT_ORDIRD_ORDSIZE_MASK 0x0000007f
-#define NES_QPCONTEXT_ORDIRD_ORDSIZE_SHIFT 0
-#define NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK 0x00030000
-#define NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT 16
-#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_MASK 0x30000000
-#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT 28
-
-enum nes_ord_ird_bits {
- NES_QPCONTEXT_ORDIRD_WRPDU = 0x02000000,
- NES_QPCONTEXT_ORDIRD_LSMM_PRESENT = 0x04000000,
- NES_QPCONTEXT_ORDIRD_ALSMM = 0x08000000,
- NES_QPCONTEXT_ORDIRD_AAH = 0x40000000,
- NES_QPCONTEXT_ORDIRD_RNMC = 0x80000000
-};
-
-enum nes_iwarp_qp_state {
- NES_QPCONTEXT_IWARP_STATE_NONEXIST = 0,
- NES_QPCONTEXT_IWARP_STATE_IDLE = 1,
- NES_QPCONTEXT_IWARP_STATE_RTS = 2,
- NES_QPCONTEXT_IWARP_STATE_CLOSING = 3,
- NES_QPCONTEXT_IWARP_STATE_TERMINATE = 5,
- NES_QPCONTEXT_IWARP_STATE_ERROR = 6
-};
-
-
-#endif /* NES_CONTEXT_H */
diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c
deleted file mode 100644
index 5517e392bc01..000000000000
--- a/drivers/infiniband/hw/nes/nes_hw.c
+++ /dev/null
@@ -1,3887 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/if_vlan.h>
-#include <linux/slab.h>
-
-#include "nes.h"
-
-static int wide_ppm_offset;
-module_param(wide_ppm_offset, int, 0644);
-MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
-
-static u32 crit_err_count;
-u32 int_mod_timer_init;
-u32 int_mod_cq_depth_256;
-u32 int_mod_cq_depth_128;
-u32 int_mod_cq_depth_32;
-u32 int_mod_cq_depth_24;
-u32 int_mod_cq_depth_16;
-u32 int_mod_cq_depth_4;
-u32 int_mod_cq_depth_1;
-static const u8 nes_max_critical_error_count = 100;
-#include "nes_cm.h"
-
-static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
-static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
-static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
- struct nes_adapter *nesadapter, u8 OneG_Mode);
-static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
-static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
-static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
-static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
- struct nes_hw_aeqe *aeqe);
-static void process_critical_error(struct nes_device *nesdev);
-static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
-static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
-static void nes_terminate_start_timer(struct nes_qp *nesqp);
-
-static const char *const nes_iwarp_state_str[] = {
- "Non-Existent",
- "Idle",
- "RTS",
- "Closing",
- "RSVD1",
- "Terminate",
- "Error",
- "RSVD2",
-};
-
-static const char *const nes_tcp_state_str[] = {
- "Non-Existent",
- "Closed",
- "Listen",
- "SYN Sent",
- "SYN Rcvd",
- "Established",
- "Close Wait",
- "FIN Wait 1",
- "Closing",
- "Last Ack",
- "FIN Wait 2",
- "Time Wait",
- "RSVD1",
- "RSVD2",
- "RSVD3",
- "RSVD4",
-};
-
-static inline void print_ip(struct nes_cm_node *cm_node)
-{
- unsigned char *rem_addr;
- if (cm_node) {
- rem_addr = (unsigned char *)&cm_node->rem_addr;
- printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
- }
-}
-
-/**
- * nes_nic_init_timer_defaults
- */
-void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
-{
- unsigned long flags;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
-
- spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
-
- shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
- shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
- if (jumbomode) {
- shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW;
- shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
- shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH;
- } else {
- shared_timer->threshold_low = DEFAULT_NES_QL_LOW;
- shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
- shared_timer->threshold_high = DEFAULT_NES_QL_HIGH;
- }
-
- /* todo use netdev->mtu to set thresholds */
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
-}
-
-
-/**
- * nes_nic_init_timer
- */
-static void nes_nic_init_timer(struct nes_device *nesdev)
-{
- unsigned long flags;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
-
- spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
-
- if (shared_timer->timer_in_use_old == 0) {
- nesdev->deepcq_count = 0;
- shared_timer->timer_direction_upward = 0;
- shared_timer->timer_direction_downward = 0;
- shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
- shared_timer->timer_in_use_old = 0;
-
- }
- if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
- shared_timer->timer_in_use_old = shared_timer->timer_in_use;
- nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
- 0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
- }
- /* todo use netdev->mtu to set thresholds */
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
-}
-
-
-/**
- * nes_nic_tune_timer
- */
-static void nes_nic_tune_timer(struct nes_device *nesdev)
-{
- unsigned long flags;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
- u16 cq_count = nesdev->currcq_count;
-
- spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
-
- if (shared_timer->cq_count_old <= cq_count)
- shared_timer->cq_direction_downward = 0;
- else
- shared_timer->cq_direction_downward++;
- shared_timer->cq_count_old = cq_count;
- if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
- if (cq_count <= shared_timer->threshold_low &&
- shared_timer->threshold_low > 4) {
- shared_timer->threshold_low = shared_timer->threshold_low/2;
- shared_timer->cq_direction_downward=0;
- nesdev->currcq_count = 0;
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
- return;
- }
- }
-
- if (cq_count > 1) {
- nesdev->deepcq_count += cq_count;
- if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */
- shared_timer->timer_direction_upward++;
- shared_timer->timer_direction_downward = 0;
- } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
- shared_timer->timer_direction_upward = 0;
- shared_timer->timer_direction_downward = 0;
- } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */
- shared_timer->timer_direction_downward++;
- shared_timer->timer_direction_upward = 0;
- } else if (cq_count <= (shared_timer->threshold_high) * 2) {
- shared_timer->timer_in_use -= 2;
- shared_timer->timer_direction_upward = 0;
- shared_timer->timer_direction_downward++;
- } else {
- shared_timer->timer_in_use -= 4;
- shared_timer->timer_direction_upward = 0;
- shared_timer->timer_direction_downward++;
- }
-
- if (shared_timer->timer_direction_upward > 3 ) { /* using history */
- shared_timer->timer_in_use += 3;
- shared_timer->timer_direction_upward = 0;
- shared_timer->timer_direction_downward = 0;
- }
- if (shared_timer->timer_direction_downward > 5) { /* using history */
- shared_timer->timer_in_use -= 4 ;
- shared_timer->timer_direction_downward = 0;
- shared_timer->timer_direction_upward = 0;
- }
- }
-
- /* boundary checking */
- if (shared_timer->timer_in_use > shared_timer->threshold_high)
- shared_timer->timer_in_use = shared_timer->threshold_high;
- else if (shared_timer->timer_in_use < shared_timer->threshold_low)
- shared_timer->timer_in_use = shared_timer->threshold_low;
-
- nesdev->currcq_count = 0;
-
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
-}
-
-
-/**
- * nes_init_adapter - initialize adapter
- */
-struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
- struct nes_adapter *nesadapter = NULL;
- unsigned long num_pds;
- u32 u32temp;
- u32 port_count;
- u16 max_rq_wrs;
- u16 max_sq_wrs;
- u32 max_mr;
- u32 max_256pbl;
- u32 max_4kpbl;
- u32 max_qp;
- u32 max_irrq;
- u32 max_cq;
- u32 hte_index_mask;
- u32 adapter_size;
- u32 arp_table_size;
- u16 vendor_id;
- u16 device_id;
- u8 OneG_Mode;
- u8 func_index;
-
- /* search the list of existing adapters */
- list_for_each_entry(nesadapter, &nes_adapter_list, list) {
- nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
- " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
- nesdev->pcidev->devfn,
- PCI_SLOT(nesadapter->devfn),
- nesadapter->bus_number,
- PCI_SLOT(nesdev->pcidev->devfn),
- nesdev->pcidev->bus->number );
- if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
- (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
- nesadapter->ref_count++;
- return nesadapter;
- }
- }
-
- /* no adapter found */
- num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
- if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
- nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
- hw_rev);
- return NULL;
- }
-
- nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
- nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
- nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
- nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
- nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
-
- nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
-
-
- if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
- return NULL;
-
- max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
- nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
- if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
- nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
- max_qp, u32temp);
- max_qp = (u32)1 << (u32temp & 0x001f);
- }
-
- hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
- nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
- max_qp, hte_index_mask);
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
-
- max_irrq = 1 << (u32temp & 0x001f);
-
- if (max_qp > max_irrq) {
- max_qp = max_irrq;
- nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
- max_qp);
- }
-
- /* there should be no reason to allocate more pds than qps */
- if (num_pds > max_qp)
- num_pds = max_qp;
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
- max_mr = (u32)8192 << (u32temp & 0x7);
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
- max_256pbl = (u32)1 << (u32temp & 0x0000001f);
- max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
- max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
- arp_table_size = 1 << u32temp;
-
- adapter_size = (sizeof(struct nes_adapter) +
- (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
- adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
- adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
- adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
- adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
- adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
- adapter_size += sizeof(struct nes_qp **) * max_qp;
-
- /* allocate a new adapter struct */
- nesadapter = kzalloc(adapter_size, GFP_KERNEL);
- if (!nesadapter)
- return NULL;
-
- nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
- nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
-
- if (nes_read_eeprom_values(nesdev, nesadapter)) {
- printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
- kfree(nesadapter);
- return NULL;
- }
-
- nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
- (nesadapter->mac_addr_low >> 24);
-
- pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
- PCI_DEVICE_ID, &device_id);
- nesadapter->vendor_part_id = device_id;
-
- if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
- OneG_Mode)) {
- kfree(nesadapter);
- return NULL;
- }
- nes_init_csr_ne020(nesdev, hw_rev, port_count);
-
- memset(nesadapter->pft_mcast_map, 255,
- sizeof nesadapter->pft_mcast_map);
-
- /* populate the new nesadapter */
- nesadapter->nesdev = nesdev;
- nesadapter->devfn = nesdev->pcidev->devfn;
- nesadapter->bus_number = nesdev->pcidev->bus->number;
- nesadapter->ref_count = 1;
- nesadapter->timer_int_req = 0xffff0000;
- nesadapter->OneG_Mode = OneG_Mode;
- nesadapter->doorbell_start = nesdev->doorbell_region;
-
- /* nesadapter->tick_delta = clk_divisor; */
- nesadapter->hw_rev = hw_rev;
- nesadapter->port_count = port_count;
-
- nesadapter->max_qp = max_qp;
- nesadapter->hte_index_mask = hte_index_mask;
- nesadapter->max_irrq = max_irrq;
- nesadapter->max_mr = max_mr;
- nesadapter->max_256pbl = max_256pbl - 1;
- nesadapter->max_4kpbl = max_4kpbl - 1;
- nesadapter->max_cq = max_cq;
- nesadapter->free_256pbl = max_256pbl - 1;
- nesadapter->free_4kpbl = max_4kpbl - 1;
- nesadapter->max_pd = num_pds;
- nesadapter->arp_table_size = arp_table_size;
-
- nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
- if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
- nesadapter->et_use_adaptive_rx_coalesce = 0;
- nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
- nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
- } else {
- nesadapter->et_use_adaptive_rx_coalesce = 1;
- nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
- nesadapter->et_rx_coalesce_usecs_irq = 0;
- printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
- }
- /* Setup and enable the periodic timer */
- if (nesadapter->et_rx_coalesce_usecs_irq)
- nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
- ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
- else
- nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
-
- nesadapter->base_pd = 1;
-
- nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
- IB_DEVICE_MEM_WINDOW |
- IB_DEVICE_MEM_MGT_EXTENSIONS;
-
- nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
- [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
- nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
- nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
- nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
- nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
- nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
-
-
- /* mark the usual suspect QPs, MR and CQs as in use */
- for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
- set_bit(u32temp, nesadapter->allocated_qps);
- set_bit(u32temp, nesadapter->allocated_cqs);
- }
- set_bit(0, nesadapter->allocated_mrs);
-
- for (u32temp = 0; u32temp < 20; u32temp++)
- set_bit(u32temp, nesadapter->allocated_pds);
- u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
-
- max_rq_wrs = ((u32temp >> 8) & 3);
- switch (max_rq_wrs) {
- case 0:
- max_rq_wrs = 4;
- break;
- case 1:
- max_rq_wrs = 16;
- break;
- case 2:
- max_rq_wrs = 32;
- break;
- case 3:
- max_rq_wrs = 512;
- break;
- }
-
- max_sq_wrs = (u32temp & 3);
- switch (max_sq_wrs) {
- case 0:
- max_sq_wrs = 4;
- break;
- case 1:
- max_sq_wrs = 16;
- break;
- case 2:
- max_sq_wrs = 32;
- break;
- case 3:
- max_sq_wrs = 512;
- break;
- }
- nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
- nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
-
- nesadapter->max_sge = 4;
- nesadapter->max_cqe = 32766;
-
- if (nes_read_eeprom_values(nesdev, nesadapter)) {
- printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
- kfree(nesadapter);
- return NULL;
- }
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
- nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
- (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
-
- /* setup port configuration */
- if (nesadapter->port_count == 1) {
- nesadapter->log_port = 0x00000000;
- if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
- nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
- else
- nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
- } else {
- if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
- nesadapter->log_port = 0x000000D8;
- } else {
- if (nesadapter->port_count == 2)
- nesadapter->log_port = 0x00000044;
- else
- nesadapter->log_port = 0x000000e4;
- }
- nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
- }
-
- nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
- nesadapter->log_port);
- nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
- nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
-
- spin_lock_init(&nesadapter->resource_lock);
- spin_lock_init(&nesadapter->phy_lock);
- spin_lock_init(&nesadapter->pbl_lock);
- spin_lock_init(&nesadapter->periodic_timer_lock);
-
- INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
- INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
- INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
- INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
-
- if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
- u32 pcs_control_status0, pcs_control_status1;
- u32 reset_value;
- u32 i = 0;
- u32 int_cnt = 0;
- u32 ext_cnt = 0;
- unsigned long flags;
- u32 j = 0;
-
- pcs_control_status0 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0);
- pcs_control_status1 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
-
- for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
- pcs_control_status0 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0);
- pcs_control_status1 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
- if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
- || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
- int_cnt++;
- usleep_range(1000, 2000);
- }
- if (int_cnt > 1) {
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
- mh_detected++;
- reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
- reset_value |= 0x0000003d;
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
-
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
- & 0x00000040) != 0x00000040) && (j++ < 5000));
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
-
- pcs_control_status0 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0);
- pcs_control_status1 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
-
- for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
- pcs_control_status0 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0);
- pcs_control_status1 = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
- if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
- || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
- if (++ext_cnt > int_cnt) {
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
- 0x0000F088);
- mh_detected++;
- reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
- reset_value |= 0x0000003d;
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
-
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
- & 0x00000040) != 0x00000040) && (j++ < 5000));
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- break;
- }
- }
- usleep_range(1000, 2000);
- }
- }
- }
-
- if (nesadapter->hw_rev == NE020_REV) {
- timer_setup(&nesadapter->mh_timer, nes_mh_fix, 0);
- nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */
- add_timer(&nesadapter->mh_timer);
- } else {
- nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
- }
-
- timer_setup(&nesadapter->lc_timer, nes_clc, 0);
- nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
- add_timer(&nesadapter->lc_timer);
-
- list_add_tail(&nesadapter->list, &nes_adapter_list);
-
- for (func_index = 0; func_index < 8; func_index++) {
- pci_bus_read_config_word(nesdev->pcidev->bus,
- PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
- func_index), 0, &vendor_id);
- if (vendor_id == 0xffff)
- break;
- }
- nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
- func_index, pci_name(nesdev->pcidev));
- nesadapter->adapter_fcn_count = func_index;
-
- return nesadapter;
-}
-
-
-/**
- * nes_reset_adapter_ne020
- */
-static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
-{
- u32 port_count;
- u32 u32temp;
- u32 i;
-
- u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
- port_count = ((u32temp & 0x00000300) >> 8) + 1;
- /* TODO: assuming that both SERDES are set the same for now */
- *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
- nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
- u32temp, port_count);
- if (*OneG_Mode)
- nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
- u32temp &= 0xff00ffc0;
- switch (port_count) {
- case 1:
- u32temp |= 0x00ee0000;
- break;
- case 2:
- u32temp |= 0x00cc0000;
- break;
- case 4:
- u32temp |= 0x00000000;
- break;
- default:
- return 0;
- break;
- }
-
- /* check and do full reset if needed */
- if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
- nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
-
- i = 0;
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
- mdelay(1);
- if (i > 10000) {
- nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
- return 0;
- }
-
- i = 0;
- while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
- mdelay(1);
- if (i > 10000) {
- printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
- nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
- return 0;
- }
- }
-
- /* port reset */
- switch (port_count) {
- case 1:
- u32temp |= 0x00ee0010;
- break;
- case 2:
- u32temp |= 0x00cc0030;
- break;
- case 4:
- u32temp |= 0x00000030;
- break;
- }
-
- nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
-
- i = 0;
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
- mdelay(1);
- if (i > 10000) {
- nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
- return 0;
- }
-
- /* serdes 0 */
- i = 0;
- while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
- & 0x0000000f)) != 0x0000000f) && i++ < 5000)
- mdelay(1);
- if (i > 5000) {
- nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
- return 0;
- }
-
- /* serdes 1 */
- if (port_count > 1) {
- i = 0;
- while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
- & 0x0000000f)) != 0x0000000f) && i++ < 5000)
- mdelay(1);
- if (i > 5000) {
- nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
- return 0;
- }
- }
-
- return port_count;
-}
-
-
-/**
- * nes_init_serdes
- */
-static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
- struct nes_adapter *nesadapter, u8 OneG_Mode)
-{
- int i;
- u32 u32temp;
- u32 sds;
-
- if (hw_rev != NE020_REV) {
- /* init serdes 0 */
- switch (nesadapter->phy_type[0]) {
- case NES_PHY_TYPE_CX4:
- if (wide_ppm_offset)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
- else
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
- break;
- case NES_PHY_TYPE_KR:
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
- break;
- case NES_PHY_TYPE_PUMA_1G:
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
- sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
- sds |= 0x00000100;
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
- break;
- default:
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
- break;
- }
-
- if (!OneG_Mode)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
-
- if (port_count < 2)
- return 0;
-
- /* init serdes 1 */
- if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
-
- switch (nesadapter->phy_type[1]) {
- case NES_PHY_TYPE_ARGUS:
- case NES_PHY_TYPE_SFP_D:
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
- break;
- case NES_PHY_TYPE_CX4:
- if (wide_ppm_offset)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
- break;
- case NES_PHY_TYPE_KR:
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
- break;
- case NES_PHY_TYPE_PUMA_1G:
- sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
- sds |= 0x000000100;
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
- }
- if (!OneG_Mode) {
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
- sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
- sds &= 0xFFFFFFBF;
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
- }
- } else {
- /* init serdes 0 */
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
- i = 0;
- while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
- & 0x0000000f)) != 0x0000000f) && i++ < 5000)
- mdelay(1);
- if (i > 5000) {
- nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
- return 1;
- }
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
- if (OneG_Mode)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
- else
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
-
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
- if (port_count > 1) {
- /* init serdes 1 */
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
- i = 0;
- while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
- & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
- mdelay(1);
- if (i > 5000) {
- printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
- /* return 1; */
- }
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
- }
- }
- return 0;
-}
-
-
-/**
- * nes_init_csr_ne020
- * Initialize registers for ne020 hardware
- */
-static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
-{
- u32 u32temp;
-
- nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
-
- nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
- /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
- nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
- nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
- /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
- nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
- nes_write_indexed(nesdev, 0x00000600, 0x55555555);
- nes_write_indexed(nesdev, 0x00000604, 0x55555555);
-
- /* TODO: move these MAC register settings to NIC bringup */
- nes_write_indexed(nesdev, 0x00002000, 0x00000001);
- nes_write_indexed(nesdev, 0x00002004, 0x00000001);
- nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
- nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
- nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
- nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
- if (port_count > 1) {
- nes_write_indexed(nesdev, 0x00002200, 0x00000001);
- nes_write_indexed(nesdev, 0x00002204, 0x00000001);
- nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
- nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
- nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
- nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
- nes_write_indexed(nesdev, 0x00000908, 0x20000001);
- }
- if (port_count > 2) {
- nes_write_indexed(nesdev, 0x00002400, 0x00000001);
- nes_write_indexed(nesdev, 0x00002404, 0x00000001);
- nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
- nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
- nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
- nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
- nes_write_indexed(nesdev, 0x00000910, 0x20000001);
-
- nes_write_indexed(nesdev, 0x00002600, 0x00000001);
- nes_write_indexed(nesdev, 0x00002604, 0x00000001);
- nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
- nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
- nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
- nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
- nes_write_indexed(nesdev, 0x00000918, 0x20000001);
- }
-
- nes_write_indexed(nesdev, 0x00005000, 0x00018000);
- /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
- nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
- 0x00000001);
- nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
- nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
- nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
- nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
- nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
-
- /* TODO: move this to code, get from EEPROM */
- nes_write_indexed(nesdev, 0x00000900, 0x20000001);
- nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
- nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
-
- nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
- /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
-
- if (hw_rev != NE020_REV) {
- u32temp = nes_read_indexed(nesdev, 0x000008e8);
- u32temp |= 0x80000000;
- nes_write_indexed(nesdev, 0x000008e8, u32temp);
- u32temp = nes_read_indexed(nesdev, 0x000021f8);
- u32temp &= 0x7fffffff;
- u32temp |= 0x7fff0010;
- nes_write_indexed(nesdev, 0x000021f8, u32temp);
- if (port_count > 1) {
- u32temp = nes_read_indexed(nesdev, 0x000023f8);
- u32temp &= 0x7fffffff;
- u32temp |= 0x7fff0010;
- nes_write_indexed(nesdev, 0x000023f8, u32temp);
- }
- }
-}
-
-
-/**
- * nes_destroy_adapter - destroy the adapter structure
- */
-void nes_destroy_adapter(struct nes_adapter *nesadapter)
-{
- struct nes_adapter *tmp_adapter;
-
- list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
- nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
- tmp_adapter);
- }
-
- nesadapter->ref_count--;
- if (!nesadapter->ref_count) {
- if (nesadapter->hw_rev == NE020_REV) {
- del_timer(&nesadapter->mh_timer);
- }
- del_timer(&nesadapter->lc_timer);
-
- list_del(&nesadapter->list);
- kfree(nesadapter);
- }
-}
-
-
-/**
- * nes_init_cqp
- */
-int nes_init_cqp(struct nes_device *nesdev)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_cqp_qp_context *cqp_qp_context;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_hw_ceq *ceq;
- struct nes_hw_ceq *nic_ceq;
- struct nes_hw_aeq *aeq;
- void *vmem;
- dma_addr_t pmem;
- u32 count=0;
- u32 cqp_head;
- u64 u64temp;
- u32 u32temp;
-
- /* allocate CQP memory */
- /* Need to add max_cq to the aeq size once cq overflow checking is added back */
- /* SQ is 512 byte aligned, others are 256 byte aligned */
- nesdev->cqp_mem_size = 512 +
- (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
- (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
- max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
- max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
- (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
- sizeof(struct nes_hw_cqp_qp_context);
-
- nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
- nesdev->cqp_mem_size,
- &nesdev->cqp_pbase);
- if (!nesdev->cqp_vbase) {
- nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
- return -ENOMEM;
- }
-
- /* Allocate a twice the number of CQP requests as the SQ size */
- nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
- 2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
- if (!nesdev->nes_cqp_requests) {
- pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
- nesdev->cqp.sq_pbase);
- return -ENOMEM;
- }
-
- nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
- nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
-
- spin_lock_init(&nesdev->cqp.lock);
- init_waitqueue_head(&nesdev->cqp.waitq);
-
- /* Setup Various Structures */
- vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
- ~(unsigned long)(512 - 1));
- pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
- ~(unsigned long long)(512 - 1));
-
- nesdev->cqp.sq_vbase = vmem;
- nesdev->cqp.sq_pbase = pmem;
- nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
- nesdev->cqp.sq_head = 0;
- nesdev->cqp.sq_tail = 0;
- nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
-
- vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
- pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
-
- nesdev->ccq.cq_vbase = vmem;
- nesdev->ccq.cq_pbase = pmem;
- nesdev->ccq.cq_size = NES_CCQ_SIZE;
- nesdev->ccq.cq_head = 0;
- nesdev->ccq.ce_handler = nes_cqp_ce_handler;
- nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
-
- vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
- pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
-
- nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
- ceq = &nesadapter->ceq[nesdev->ceq_index];
- ceq->ceq_vbase = vmem;
- ceq->ceq_pbase = pmem;
- ceq->ceq_size = NES_CCEQ_SIZE;
- ceq->ceq_head = 0;
-
- vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
- pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
-
- nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
- nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
- nic_ceq->ceq_vbase = vmem;
- nic_ceq->ceq_pbase = pmem;
- nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
- nic_ceq->ceq_head = 0;
-
- vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
- pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
-
- aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
- aeq->aeq_vbase = vmem;
- aeq->aeq_pbase = pmem;
- aeq->aeq_size = nesadapter->max_qp;
- aeq->aeq_head = 0;
-
- /* Setup QP Context */
- vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
- pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
-
- cqp_qp_context = vmem;
- cqp_qp_context->context_words[0] =
- cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
- cqp_qp_context->context_words[1] = 0;
- cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
- cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
-
-
- /* Write the address to Create CQP */
- if ((sizeof(dma_addr_t) > 4)) {
- nes_write_indexed(nesdev,
- NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
- ((u64)pmem) >> 32);
- } else {
- nes_write_indexed(nesdev,
- NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
- }
- nes_write_indexed(nesdev,
- NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
- (u32)pmem);
-
- INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
- INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
-
- for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
- init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
- list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
- }
-
- /* Write Create CCQ WQE */
- cqp_head = nesdev->cqp.sq_head++;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
- NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- (nesdev->ccq.cq_number |
- ((u32)nesdev->ceq_index << 16)));
- u64temp = (u64)nesdev->ccq.cq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
- u64temp = (unsigned long)&nesdev->ccq;
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
- cpu_to_le32((u32)(u64temp >> 1));
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
- cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
-
- /* Write Create CEQ WQE */
- cqp_head = nesdev->cqp.sq_head++;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
- u64temp = (u64)ceq->ceq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
-
- /* Write Create AEQ WQE */
- cqp_head = nesdev->cqp.sq_head++;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
- u64temp = (u64)aeq->aeq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
-
- /* Write Create NIC CEQ WQE */
- cqp_head = nesdev->cqp.sq_head++;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
- u64temp = (u64)nic_ceq->ceq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
-
- /* Poll until CCQP done */
- count = 0;
- do {
- if (count++ > 1000) {
- printk(KERN_ERR PFX "Error creating CQP\n");
- pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
- nesdev->cqp_vbase, nesdev->cqp_pbase);
- return -1;
- }
- udelay(10);
- } while (!(nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
-
- nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
-
- u32temp = 0x04800000;
- nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
-
- /* wait for the CCQ, CEQ, and AEQ to get created */
- count = 0;
- do {
- if (count++ > 1000) {
- printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
- pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
- nesdev->cqp_vbase, nesdev->cqp_pbase);
- return -1;
- }
- udelay(10);
- } while (((nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
-
- /* dump the QP status value */
- nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
-
- nesdev->cqp.sq_tail++;
-
- return 0;
-}
-
-
-/**
- * nes_destroy_cqp
- */
-int nes_destroy_cqp(struct nes_device *nesdev)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- u32 count = 0;
- u32 cqp_head;
- unsigned long flags;
-
- do {
- if (count++ > 1000)
- break;
- udelay(10);
- } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
-
- /* Reset CCQ */
- nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
- nesdev->ccq.cq_number);
-
- /* Disable device interrupts */
- nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
-
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
-
- /* Destroy the AEQ */
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
- ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
- cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
-
- /* Destroy the NIC CEQ */
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
- ((u32)nesdev->nic_ceq_index << 8));
-
- /* Destroy the CEQ */
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
- (nesdev->ceq_index << 8));
-
- /* Destroy the CCQ */
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
- ((u32)nesdev->ceq_index << 16));
-
- /* Destroy CQP */
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
- NES_CQP_QP_TYPE_CQP);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
-
- barrier();
- /* Ring doorbell (5 WQEs) */
- nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
-
- /* wait for the CCQ, CEQ, and AEQ to get destroyed */
- count = 0;
- do {
- if (count++ > 1000) {
- printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
- PCI_FUNC(nesdev->pcidev->devfn));
- break;
- }
- udelay(10);
- } while (((nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
-
- /* dump the QP status value */
- nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
- PCI_FUNC(nesdev->pcidev->devfn),
- nes_read_indexed(nesdev,
- NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
-
- kfree(nesdev->nes_cqp_requests);
-
- /* Free the control structures */
- pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
- nesdev->cqp.sq_pbase);
-
- return 0;
-}
-
-
-/**
- * nes_init_1g_phy
- */
-static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
-{
- u32 counter = 0;
- u16 phy_data;
- int ret = 0;
-
- nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
- nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
-
- /* Reset the PHY */
- nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
- udelay(100);
- counter = 0;
- do {
- nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
- if (counter++ > 100) {
- ret = -1;
- break;
- }
- } while (phy_data & 0x8000);
-
- /* Setting no phy loopback */
- phy_data &= 0xbfff;
- phy_data |= 0x1140;
- nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data);
- nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
- nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
- nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
-
- /* Setting the interrupt mask */
- nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
- nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
- nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
-
- /* turning on flow control */
- nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
- nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
- nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
-
- /* Clear Half duplex */
- nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
- nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
- nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
-
- nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
- nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
-
- return ret;
-}
-
-
-/**
- * nes_init_2025_phy
- */
-static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
-{
- u32 temp_phy_data = 0;
- u32 temp_phy_data2 = 0;
- u32 counter = 0;
- u32 sds;
- u32 mac_index = nesdev->mac_index;
- int ret = 0;
- unsigned int first_attempt = 1;
-
- /* Check firmware heartbeat */
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- udelay(1500);
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
- temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
-
- if (temp_phy_data != temp_phy_data2) {
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- if ((temp_phy_data & 0xff) > 0x20)
- return 0;
- printk(PFX "Reinitialize external PHY\n");
- }
-
- /* no heartbeat, configure the PHY */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
-
- switch (phy_type) {
- case NES_PHY_TYPE_ARGUS:
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
-
- /* setup LEDs */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
- break;
-
- case NES_PHY_TYPE_SFP_D:
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
-
- /* setup LEDs */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
- break;
-
- case NES_PHY_TYPE_KR:
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
-
- /* setup LEDs */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
-
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
- break;
- }
-
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
-
- /* Bring PHY out of reset */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
-
- /* Check for heartbeat */
- counter = 0;
- mdelay(690);
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- do {
- if (counter++ > 150) {
- printk(PFX "No PHY heartbeat\n");
- break;
- }
- mdelay(1);
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
- temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- } while (temp_phy_data2 == temp_phy_data);
-
- /* wait for tracking */
- counter = 0;
- do {
- nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- if (counter++ > 300) {
- if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
- first_attempt = 0;
- counter = 0;
- /* reset AMCC PHY and try again */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
- continue;
- } else {
- ret = 1;
- break;
- }
- }
- mdelay(10);
- } while ((temp_phy_data & 0xff) < 0x30);
-
- /* setup signal integrity */
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
- if (phy_type == NES_PHY_TYPE_KR) {
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
- } else {
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
- nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
- }
-
- /* reset serdes */
- sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
- sds |= 0x1;
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
- sds &= 0xfffffffe;
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
-
- counter = 0;
- while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
- && (counter++ < 5000))
- ;
-
- return ret;
-}
-
-
-/**
- * nes_init_phy
- */
-int nes_init_phy(struct nes_device *nesdev)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 mac_index = nesdev->mac_index;
- u32 tx_config = 0;
- unsigned long flags;
- u8 phy_type = nesadapter->phy_type[mac_index];
- u8 phy_index = nesadapter->phy_index[mac_index];
- int ret = 0;
-
- tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
- if (phy_type == NES_PHY_TYPE_1G) {
- /* setup 1G MDIO operation */
- tx_config &= 0xFFFFFFE3;
- tx_config |= 0x04;
- } else {
- /* setup 10G MDIO operation */
- tx_config &= 0xFFFFFFE3;
- tx_config |= 0x1D;
- }
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
-
- spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
-
- switch (phy_type) {
- case NES_PHY_TYPE_1G:
- ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
- break;
- case NES_PHY_TYPE_ARGUS:
- case NES_PHY_TYPE_SFP_D:
- case NES_PHY_TYPE_KR:
- ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
- break;
- }
-
- spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
-
- return ret;
-}
-
-
-/**
- * nes_replenish_nic_rq
- */
-static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
-{
- unsigned long flags;
- dma_addr_t bus_address;
- struct sk_buff *skb;
- struct nes_hw_nic_rq_wqe *nic_rqe;
- struct nes_hw_nic *nesnic;
- struct nes_device *nesdev;
- struct nes_rskb_cb *cb;
- u32 rx_wqes_posted = 0;
-
- nesnic = &nesvnic->nic;
- nesdev = nesvnic->nesdev;
- spin_lock_irqsave(&nesnic->rq_lock, flags);
- if (nesnic->replenishing_rq !=0) {
- if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
- (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
- atomic_set(&nesvnic->rx_skb_timer_running, 1);
- spin_unlock_irqrestore(&nesnic->rq_lock, flags);
- nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
- add_timer(&nesvnic->rq_wqes_timer);
- } else
- spin_unlock_irqrestore(&nesnic->rq_lock, flags);
- return;
- }
- nesnic->replenishing_rq = 1;
- spin_unlock_irqrestore(&nesnic->rq_lock, flags);
- do {
- skb = dev_alloc_skb(nesvnic->max_frame_size);
- if (skb) {
- skb->dev = nesvnic->netdev;
-
- bus_address = pci_map_single(nesdev->pcidev,
- skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->busaddr = bus_address;
- cb->maplen = nesvnic->max_frame_size;
-
- nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
- cpu_to_le32(nesvnic->max_frame_size);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
- cpu_to_le32((u32)bus_address);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
- cpu_to_le32((u32)((u64)bus_address >> 32));
- nesnic->rx_skb[nesnic->rq_head] = skb;
- nesnic->rq_head++;
- nesnic->rq_head &= nesnic->rq_size - 1;
- atomic_dec(&nesvnic->rx_skbs_needed);
- barrier();
- if (++rx_wqes_posted == 255) {
- nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
- rx_wqes_posted = 0;
- }
- } else {
- spin_lock_irqsave(&nesnic->rq_lock, flags);
- if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
- (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
- atomic_set(&nesvnic->rx_skb_timer_running, 1);
- spin_unlock_irqrestore(&nesnic->rq_lock, flags);
- nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
- add_timer(&nesvnic->rq_wqes_timer);
- } else
- spin_unlock_irqrestore(&nesnic->rq_lock, flags);
- break;
- }
- } while (atomic_read(&nesvnic->rx_skbs_needed));
- barrier();
- if (rx_wqes_posted)
- nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
- nesnic->replenishing_rq = 0;
-}
-
-
-/**
- * nes_rq_wqes_timeout
- */
-static void nes_rq_wqes_timeout(struct timer_list *t)
-{
- struct nes_vnic *nesvnic = from_timer(nesvnic, t, rq_wqes_timer);
- printk("%s: Timer fired.\n", __func__);
- atomic_set(&nesvnic->rx_skb_timer_running, 0);
- if (atomic_read(&nesvnic->rx_skbs_needed))
- nes_replenish_nic_rq(nesvnic);
-}
-
-
-/**
- * nes_init_nic_qp
- */
-int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_hw_nic_sq_wqe *nic_sqe;
- struct nes_hw_nic_qp_context *nic_context;
- struct sk_buff *skb;
- struct nes_hw_nic_rq_wqe *nic_rqe;
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- unsigned long flags;
- void *vmem;
- dma_addr_t pmem;
- u64 u64temp;
- int ret;
- u32 cqp_head;
- u32 counter;
- u32 wqe_count;
- struct nes_rskb_cb *cb;
- u8 jumbomode=0;
-
- /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
- nesvnic->nic_mem_size = 256 +
- (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
- (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
- (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
- (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
- sizeof(struct nes_hw_nic_qp_context);
-
- nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
- nesvnic->nic_mem_size,
- &nesvnic->nic_pbase);
- if (!nesvnic->nic_vbase) {
- nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
- return -ENOMEM;
- }
- nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
- nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
-
- vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
- ~(unsigned long)(256 - 1));
- pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
- ~(unsigned long long)(256 - 1));
-
- /* Setup the first Fragment buffers */
- nesvnic->nic.first_frag_vbase = vmem;
-
- for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
- nesvnic->nic.frag_paddr[counter] = pmem;
- pmem += sizeof(struct nes_first_frag);
- }
-
- /* setup the SQ */
- vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
-
- nesvnic->nic.sq_vbase = (void *)vmem;
- nesvnic->nic.sq_pbase = pmem;
- nesvnic->nic.sq_head = 0;
- nesvnic->nic.sq_tail = 0;
- nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
- for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
- nic_sqe = &nesvnic->nic.sq_vbase[counter];
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
- cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
- NES_NIC_SQ_WQE_COMPLETION);
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
- cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
- cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
- cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
- }
-
- nesvnic->get_cqp_request = nes_get_cqp_request;
- nesvnic->post_cqp_request = nes_post_cqp_request;
- nesvnic->mcrq_mcast_filter = NULL;
-
- spin_lock_init(&nesvnic->nic.rq_lock);
-
- /* setup the RQ */
- vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
- pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
-
-
- nesvnic->nic.rq_vbase = vmem;
- nesvnic->nic.rq_pbase = pmem;
- nesvnic->nic.rq_head = 0;
- nesvnic->nic.rq_tail = 0;
- nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
-
- /* setup the CQ */
- vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
- pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
-
- if (nesdev->nesadapter->netdev_count > 2)
- nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
- else
- nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
-
- nesvnic->nic_cq.cq_vbase = vmem;
- nesvnic->nic_cq.cq_pbase = pmem;
- nesvnic->nic_cq.cq_head = 0;
- nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
-
- nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
-
- /* Send CreateCQ request to CQP */
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- cqp_head = nesdev->cqp.sq_head;
-
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
- NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
- ((u32)nesvnic->nic_cq.cq_size << 16));
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
- nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
- u64temp = (u64)nesvnic->nic_cq.cq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
- u64temp = (unsigned long)&nesvnic->nic_cq;
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
- cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- /* Send CreateQP request to CQP */
- nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
- nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
- cpu_to_le32((u32)NES_NIC_CTX_SIZE |
- ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
- nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
- nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
- nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
- if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
- nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
- }
-
- u64temp = (u64)nesvnic->nic.sq_pbase;
- nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
- nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
- u64temp = (u64)nesvnic->nic.rq_pbase;
- nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
- nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
- NES_CQP_QP_TYPE_NIC);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
- u64temp = (u64)nesvnic->nic_cq.cq_pbase +
- (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
- nesdev->cqp.sq_head = cqp_head;
-
- barrier();
-
- /* Ring doorbell (2 WQEs) */
- nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
- nesvnic->nic.qp_id);
-
- ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
- nesvnic->nic.qp_id, ret);
- if (!ret) {
- nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
- pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
- nesvnic->nic_pbase);
- return -EIO;
- }
-
- /* Populate the RQ */
- for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
- skb = dev_alloc_skb(nesvnic->max_frame_size);
- if (!skb) {
- nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
-
- nes_destroy_nic_qp(nesvnic);
- return -ENOMEM;
- }
-
- skb->dev = netdev;
-
- pmem = pci_map_single(nesdev->pcidev, skb->data,
- nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->busaddr = pmem;
- cb->maplen = nesvnic->max_frame_size;
-
- nic_rqe = &nesvnic->nic.rq_vbase[counter];
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
- nesvnic->nic.rx_skb[counter] = skb;
- }
-
- wqe_count = NES_NIC_WQ_SIZE - 1;
- nesvnic->nic.rq_head = wqe_count;
- barrier();
- do {
- counter = min(wqe_count, ((u32)255));
- wqe_count -= counter;
- nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
- } while (wqe_count);
- timer_setup(&nesvnic->rq_wqes_timer, nes_rq_wqes_timeout, 0);
- nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
- if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
- {
- nes_nic_init_timer(nesdev);
- if (netdev->mtu > 1500)
- jumbomode = 1;
- nes_nic_init_timer_defaults(nesdev, jumbomode);
- }
- if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
- (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
- nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n",
- netdev->name);
- nes_destroy_nic_qp(nesvnic);
- return -ENOMEM;
- }
-
- return 0;
-}
-
-
-/**
- * nes_destroy_nic_qp
- */
-void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
-{
- u64 u64temp;
- dma_addr_t bus_address;
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_hw_nic_sq_wqe *nic_sqe;
- __le16 *wqe_fragment_length;
- u16 wqe_fragment_index;
- u32 cqp_head;
- u32 wqm_cfg0;
- unsigned long flags;
- struct sk_buff *rx_skb;
- struct nes_rskb_cb *cb;
- int ret;
-
- if (nesdev->nesadapter->allow_unaligned_fpdus)
- nes_destroy_mgt(nesvnic);
-
- /* clear wqe stall before destroying NIC QP */
- wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
- nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
-
- /* Free remaining NIC receive buffers */
- while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
- rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
- cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
- pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
- PCI_DMA_FROMDEVICE);
-
- dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
- nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
- }
-
- /* Free remaining NIC transmit buffers */
- while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
- nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
- wqe_fragment_index = 1;
- wqe_fragment_length = (__le16 *)
- &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
- /* bump past the vlan tag */
- wqe_fragment_length++;
- if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
- u64temp = (u64)le32_to_cpu(
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
- wqe_fragment_index*2]);
- u64temp += ((u64)le32_to_cpu(
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
- + wqe_fragment_index*2]))<<32;
- bus_address = (dma_addr_t)u64temp;
- if (test_and_clear_bit(nesvnic->nic.sq_tail,
- nesvnic->nic.first_frag_overflow)) {
- pci_unmap_single(nesdev->pcidev,
- bus_address,
- le16_to_cpu(wqe_fragment_length[
- wqe_fragment_index++]),
- PCI_DMA_TODEVICE);
- }
- for (; wqe_fragment_index < 5; wqe_fragment_index++) {
- if (wqe_fragment_length[wqe_fragment_index]) {
- u64temp = le32_to_cpu(
- nic_sqe->wqe_words[
- NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
- wqe_fragment_index*2]);
- u64temp += ((u64)le32_to_cpu(
- nic_sqe->wqe_words[
- NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
- wqe_fragment_index*2]))<<32;
- bus_address = (dma_addr_t)u64temp;
- pci_unmap_page(nesdev->pcidev,
- bus_address,
- le16_to_cpu(
- wqe_fragment_length[
- wqe_fragment_index]),
- PCI_DMA_TODEVICE);
- } else
- break;
- }
- }
- if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
- dev_kfree_skb(
- nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
-
- nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
- & (nesvnic->nic.sq_size - 1);
- }
-
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
-
- /* Destroy NIC QP */
- cqp_head = nesdev->cqp.sq_head;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- nesvnic->nic.qp_id);
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
-
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
-
- /* Destroy NIC CQ */
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
-
- nesdev->cqp.sq_head = cqp_head;
- barrier();
-
- /* Ring doorbell (2 WQEs) */
- nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
- " cqp.sq_tail=%u, cqp.sq_size=%u\n",
- cqp_head, nesdev->cqp.sq_head,
- nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
-
- ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
- NES_EVENT_TIMEOUT);
-
- nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
- " cqp.sq_head=%u, cqp.sq_tail=%u\n",
- ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
- if (!ret) {
- nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
- nesvnic->nic.qp_id);
- }
-
- pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
- nesvnic->nic_pbase);
-
- /* restore old wqm_cfg0 value */
- nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
-}
-
-/**
- * nes_napi_isr
- */
-int nes_napi_isr(struct nes_device *nesdev)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 int_stat;
-
- if (nesdev->napi_isr_ran) {
- /* interrupt status has already been read in ISR */
- int_stat = nesdev->int_stat;
- } else {
- int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
- nesdev->int_stat = int_stat;
- nesdev->napi_isr_ran = 1;
- }
-
- int_stat &= nesdev->int_req;
- /* iff NIC, process here, else wait for DPC */
- if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
- nesdev->napi_isr_ran = 0;
- nes_write32(nesdev->regs + NES_INT_STAT,
- (int_stat &
- ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
-
- /* Process the CEQs */
- nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
-
- if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
- (!nesadapter->et_use_adaptive_rx_coalesce)) ||
- ((nesadapter->et_use_adaptive_rx_coalesce) &&
- (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
- if ((nesdev->int_req & NES_INT_TIMER) == 0) {
- /* Enable Periodic timer interrupts */
- nesdev->int_req |= NES_INT_TIMER;
- /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
- /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
- nes_write32(nesdev->regs+NES_TIMER_STAT,
- nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
- nes_write32(nesdev->regs+NES_INTF_INT_MASK,
- ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
- }
-
- if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
- {
- nes_nic_init_timer(nesdev);
- }
- /* Enable interrupts, except CEQs */
- nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
- } else {
- /* Enable interrupts, make sure timer is off */
- nesdev->int_req &= ~NES_INT_TIMER;
- nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
- nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
- }
- nesdev->deepcq_count = 0;
- return 1;
- } else {
- return 0;
- }
-}
-
-static void process_critical_error(struct nes_device *nesdev)
-{
- u32 debug_error;
- u32 nes_idx_debug_error_masks0 = 0;
- u16 error_module = 0;
-
- debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
- printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
- (u16)debug_error);
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
- 0x01010000 | (debug_error & 0x0000ffff));
- if (crit_err_count++ > 10)
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
- error_module = (u16) (debug_error & 0x1F00) >> 8;
- if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
- nes_max_critical_error_count) {
- printk(KERN_ERR PFX "Masking off critical error for module "
- "0x%02X\n", (u16)error_module);
- nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
- NES_IDX_DEBUG_ERROR_MASKS0);
- nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
- nes_idx_debug_error_masks0 | (1 << error_module));
- }
-}
-/**
- * nes_dpc
- */
-void nes_dpc(unsigned long param)
-{
- struct nes_device *nesdev = (struct nes_device *)param;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 counter;
- u32 loop_counter = 0;
- u32 int_status_bit;
- u32 int_stat;
- u32 timer_stat;
- u32 temp_int_stat;
- u32 intf_int_stat;
- u32 processed_intf_int = 0;
- u16 processed_timer_int = 0;
- u16 completion_ints = 0;
- u16 timer_ints = 0;
-
- /* nes_debug(NES_DBG_ISR, "\n"); */
-
- do {
- timer_stat = 0;
- if (nesdev->napi_isr_ran) {
- nesdev->napi_isr_ran = 0;
- int_stat = nesdev->int_stat;
- } else
- int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
- if (processed_intf_int != 0)
- int_stat &= nesdev->int_req & ~NES_INT_INTF;
- else
- int_stat &= nesdev->int_req;
- if (processed_timer_int == 0) {
- processed_timer_int = 1;
- if (int_stat & NES_INT_TIMER) {
- timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
- if ((timer_stat & nesdev->timer_int_req) == 0) {
- int_stat &= ~NES_INT_TIMER;
- }
- }
- } else {
- int_stat &= ~NES_INT_TIMER;
- }
-
- if (int_stat) {
- if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
- NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
- /* Ack the interrupts */
- nes_write32(nesdev->regs+NES_INT_STAT,
- (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
- NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
- }
-
- temp_int_stat = int_stat;
- for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
- if (int_stat & int_status_bit) {
- nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
- temp_int_stat &= ~int_status_bit;
- completion_ints = 1;
- }
- if (!(temp_int_stat & 0x0000ffff))
- break;
- int_status_bit <<= 1;
- }
-
- /* Process the AEQ for this pci function */
- int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
- if (int_stat & int_status_bit) {
- nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
- }
-
- /* Process the MAC interrupt for this pci function */
- int_status_bit = 1 << (24 + nesdev->mac_index);
- if (int_stat & int_status_bit) {
- nes_process_mac_intr(nesdev, nesdev->mac_index);
- }
-
- if (int_stat & NES_INT_TIMER) {
- if (timer_stat & nesdev->timer_int_req) {
- nes_write32(nesdev->regs + NES_TIMER_STAT,
- (timer_stat & nesdev->timer_int_req) |
- ~(nesdev->nesadapter->timer_int_req));
- timer_ints = 1;
- }
- }
-
- if (int_stat & NES_INT_INTF) {
- processed_intf_int = 1;
- intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
- intf_int_stat &= nesdev->intf_int_req;
- if (NES_INTF_INT_CRITERR & intf_int_stat) {
- process_critical_error(nesdev);
- }
- if (NES_INTF_INT_PCIERR & intf_int_stat) {
- printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
- BUG();
- }
- if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
- printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
- BUG();
- }
- nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
- }
-
- if (int_stat & NES_INT_TSW) {
- }
- }
- /* Don't use the interface interrupt bit stay in loop */
- int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
- NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
- } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
-
- if (timer_ints == 1) {
- if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
- if (completion_ints == 0) {
- nesdev->timer_only_int_count++;
- if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
- nesdev->timer_only_int_count = 0;
- nesdev->int_req &= ~NES_INT_TIMER;
- nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
- nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
- } else {
- nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
- }
- } else {
- if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
- {
- nes_nic_init_timer(nesdev);
- }
- nesdev->timer_only_int_count = 0;
- nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
- }
- } else {
- nesdev->timer_only_int_count = 0;
- nesdev->int_req &= ~NES_INT_TIMER;
- nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
- nes_write32(nesdev->regs+NES_TIMER_STAT,
- nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
- nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
- }
- } else {
- if ( (completion_ints == 1) &&
- (((nesadapter->et_rx_coalesce_usecs_irq) &&
- (!nesadapter->et_use_adaptive_rx_coalesce)) ||
- ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
- (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
- /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
- nesdev->timer_only_int_count = 0;
- nesdev->int_req |= NES_INT_TIMER;
- nes_write32(nesdev->regs+NES_TIMER_STAT,
- nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
- nes_write32(nesdev->regs+NES_INTF_INT_MASK,
- ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
- nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
- } else {
- nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
- }
- }
- nesdev->deepcq_count = 0;
-}
-
-
-/**
- * nes_process_ceq
- */
-static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
-{
- u64 u64temp;
- struct nes_hw_cq *cq;
- u32 head;
- u32 ceq_size;
-
- /* nes_debug(NES_DBG_CQ, "\n"); */
- head = ceq->ceq_head;
- ceq_size = ceq->ceq_size;
-
- do {
- if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
- NES_CEQE_VALID) {
- u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
- ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
- u64temp <<= 1;
- cq = *((struct nes_hw_cq **)&u64temp);
- /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
- barrier();
- ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
-
- /* call the event handler */
- cq->ce_handler(nesdev, cq);
-
- if (++head >= ceq_size)
- head = 0;
- } else {
- break;
- }
-
- } while (1);
-
- ceq->ceq_head = head;
-}
-
-
-/**
- * nes_process_aeq
- */
-static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
-{
- /* u64 u64temp; */
- u32 head;
- u32 aeq_size;
- u32 aeqe_misc;
- u32 aeqe_cq_id;
- struct nes_hw_aeqe volatile *aeqe;
-
- head = aeq->aeq_head;
- aeq_size = aeq->aeq_size;
-
- do {
- aeqe = &aeq->aeq_vbase[head];
- if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
- break;
- aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
- aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
- if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
- if (aeqe_cq_id >= NES_FIRST_QPN) {
- /* dealing with an accelerated QP related AE */
- /*
- * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
- * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
- */
- nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
- } else {
- /* TODO: dealing with a CQP related AE */
- nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
- (u16)(aeqe_misc >> 16));
- }
- }
-
- aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
-
- if (++head >= aeq_size)
- head = 0;
-
- nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
- }
- while (1);
- aeq->aeq_head = head;
-}
-
-static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 reset_value;
- u32 i=0;
- u32 u32temp;
-
- if (nesadapter->hw_rev == NE020_REV) {
- return;
- }
- mh_detected++;
-
- reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
-
- if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
- reset_value |= 0x0000001d;
- else
- reset_value |= 0x0000002d;
-
- if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
- if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
- nesadapter->link_interrupt_count[0] = 0;
- nesadapter->link_interrupt_count[1] = 0;
- u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
- if (0x00000040 & u32temp)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
- else
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
-
- reset_value |= 0x0000003d;
- }
- nesadapter->link_interrupt_count[mac_index] = 0;
- }
-
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
-
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
- & 0x00000040) != 0x00000040) && (i++ < 5000));
-
- if (0x0000003d == (reset_value & 0x0000003d)) {
- u32 pcs_control_status0, pcs_control_status1;
-
- for (i = 0; i < 10; i++) {
- pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
- pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
- if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
- && (pcs_control_status0 & 0x00100000))
- || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
- && (pcs_control_status1 & 0x00100000)))
- continue;
- else
- break;
- }
- if (10 == i) {
- u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
- if (0x00000040 & u32temp)
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
- else
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
-
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
-
- while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
- & 0x00000040) != 0x00000040) && (i++ < 5000));
- }
- }
-}
-
-/**
- * nes_process_mac_intr
- */
-static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
-{
- unsigned long flags;
- u32 pcs_control_status;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_vnic *nesvnic;
- u32 mac_status;
- u32 mac_index = nesdev->mac_index;
- u32 u32temp;
- u16 phy_data;
- u16 temp_phy_data;
- u32 pcs_val = 0x0f0f0000;
- u32 pcs_mask = 0x0f1f0000;
- u32 cdr_ctrl;
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- return;
- }
- nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
-
- /* ack the MAC interrupt */
- mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
- /* Clear the interrupt */
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
-
- nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
-
- if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
- nesdev->link_status_interrupts++;
- if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
- nes_reset_link(nesdev, mac_index);
-
- /* read the PHY interrupt status register */
- if ((nesadapter->OneG_Mode) &&
- (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
- do {
- nes_read_1G_phy_reg(nesdev, 0x1a,
- nesadapter->phy_index[mac_index], &phy_data);
- nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
- nesadapter->phy_index[mac_index], phy_data);
- } while (phy_data&0x8000);
-
- temp_phy_data = 0;
- do {
- nes_read_1G_phy_reg(nesdev, 0x11,
- nesadapter->phy_index[mac_index], &phy_data);
- nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
- nesadapter->phy_index[mac_index], phy_data);
- if (temp_phy_data == phy_data)
- break;
- temp_phy_data = phy_data;
- } while (1);
-
- nes_read_1G_phy_reg(nesdev, 0x1e,
- nesadapter->phy_index[mac_index], &phy_data);
- nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
- nesadapter->phy_index[mac_index], phy_data);
-
- nes_read_1G_phy_reg(nesdev, 1,
- nesadapter->phy_index[mac_index], &phy_data);
- nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
- nesadapter->phy_index[mac_index], phy_data);
-
- if (temp_phy_data & 0x1000) {
- nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
- phy_data = 4;
- } else {
- nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
- }
- }
- nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
- nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
- nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
-
- if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
- switch (mac_index) {
- case 1:
- case 3:
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
- break;
- default:
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0);
- break;
- }
- } else {
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
- pcs_control_status = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
- }
-
- nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
- mac_index, pcs_control_status);
- if ((nesadapter->OneG_Mode) &&
- (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
- u32temp = 0x01010000;
- if (nesadapter->port_count > 2) {
- u32temp |= 0x02020000;
- }
- if ((pcs_control_status & u32temp)!= u32temp) {
- phy_data = 0;
- nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
- }
- } else {
- switch (nesadapter->phy_type[mac_index]) {
- case NES_PHY_TYPE_ARGUS:
- case NES_PHY_TYPE_SFP_D:
- case NES_PHY_TYPE_KR:
- /* clear the alarms */
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
- /* check link status */
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
-
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
- nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
- phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
-
- phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
-
- nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
- __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
- break;
-
- case NES_PHY_TYPE_PUMA_1G:
- if (mac_index < 2)
- pcs_val = pcs_mask = 0x01010000;
- else
- pcs_val = pcs_mask = 0x02020000;
- /* fall through */
- default:
- phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
- break;
- }
- }
-
- if (phy_data & 0x0004) {
- if (wide_ppm_offset &&
- (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
- (nesadapter->hw_rev != NE020_REV)) {
- cdr_ctrl = nes_read_indexed(nesdev,
- NES_IDX_ETH_SERDES_CDR_CONTROL0 +
- mac_index * 0x200);
- nes_write_indexed(nesdev,
- NES_IDX_ETH_SERDES_CDR_CONTROL0 +
- mac_index * 0x200,
- cdr_ctrl | 0x000F0000);
- }
- nesadapter->mac_link_down[mac_index] = 0;
- list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
- nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
- nesvnic->linkup);
- if (nesvnic->linkup == 0) {
- printk(PFX "The Link is now up for port %s, netdev %p.\n",
- nesvnic->netdev->name, nesvnic->netdev);
- if (netif_queue_stopped(nesvnic->netdev))
- netif_start_queue(nesvnic->netdev);
- nesvnic->linkup = 1;
- netif_carrier_on(nesvnic->netdev);
-
- spin_lock(&nesvnic->port_ibevent_lock);
- if (nesvnic->of_device_registered) {
- if (nesdev->iw_status == 0) {
- nesdev->iw_status = 1;
- nes_port_ibevent(nesvnic);
- }
- }
- spin_unlock(&nesvnic->port_ibevent_lock);
- }
- }
- } else {
- if (wide_ppm_offset &&
- (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
- (nesadapter->hw_rev != NE020_REV)) {
- cdr_ctrl = nes_read_indexed(nesdev,
- NES_IDX_ETH_SERDES_CDR_CONTROL0 +
- mac_index * 0x200);
- nes_write_indexed(nesdev,
- NES_IDX_ETH_SERDES_CDR_CONTROL0 +
- mac_index * 0x200,
- cdr_ctrl & 0xFFF0FFFF);
- }
- nesadapter->mac_link_down[mac_index] = 1;
- list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
- nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
- nesvnic->linkup);
- if (nesvnic->linkup == 1) {
- printk(PFX "The Link is now down for port %s, netdev %p.\n",
- nesvnic->netdev->name, nesvnic->netdev);
- if (!(netif_queue_stopped(nesvnic->netdev)))
- netif_stop_queue(nesvnic->netdev);
- nesvnic->linkup = 0;
- netif_carrier_off(nesvnic->netdev);
-
- spin_lock(&nesvnic->port_ibevent_lock);
- if (nesvnic->of_device_registered) {
- if (nesdev->iw_status == 1) {
- nesdev->iw_status = 0;
- nes_port_ibevent(nesvnic);
- }
- }
- spin_unlock(&nesvnic->port_ibevent_lock);
- }
- }
- }
- if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
- nesdev->link_recheck = 1;
- mod_delayed_work(system_wq, &nesdev->work,
- NES_LINK_RECHECK_DELAY);
- }
- }
-
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
-
- nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
-}
-
-void nes_recheck_link_status(struct work_struct *work)
-{
- unsigned long flags;
- struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_vnic *nesvnic;
- u32 mac_index = nesdev->mac_index;
- u16 phy_data;
- u16 temp_phy_data;
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
-
- /* check link status */
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
-
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
- nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
- phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
-
- phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
-
- nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
- __func__, phy_data,
- nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
-
- if (phy_data & 0x0004) {
- nesadapter->mac_link_down[mac_index] = 0;
- list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
- if (nesvnic->linkup == 0) {
- printk(PFX "The Link is now up for port %s, netdev %p.\n",
- nesvnic->netdev->name, nesvnic->netdev);
- if (netif_queue_stopped(nesvnic->netdev))
- netif_start_queue(nesvnic->netdev);
- nesvnic->linkup = 1;
- netif_carrier_on(nesvnic->netdev);
-
- spin_lock(&nesvnic->port_ibevent_lock);
- if (nesvnic->of_device_registered) {
- if (nesdev->iw_status == 0) {
- nesdev->iw_status = 1;
- nes_port_ibevent(nesvnic);
- }
- }
- spin_unlock(&nesvnic->port_ibevent_lock);
- }
- }
-
- } else {
- nesadapter->mac_link_down[mac_index] = 1;
- list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
- if (nesvnic->linkup == 1) {
- printk(PFX "The Link is now down for port %s, netdev %p.\n",
- nesvnic->netdev->name, nesvnic->netdev);
- if (!(netif_queue_stopped(nesvnic->netdev)))
- netif_stop_queue(nesvnic->netdev);
- nesvnic->linkup = 0;
- netif_carrier_off(nesvnic->netdev);
-
- spin_lock(&nesvnic->port_ibevent_lock);
- if (nesvnic->of_device_registered) {
- if (nesdev->iw_status == 1) {
- nesdev->iw_status = 0;
- nes_port_ibevent(nesvnic);
- }
- }
- spin_unlock(&nesvnic->port_ibevent_lock);
- }
- }
- }
- if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
- schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
- else
- nesdev->link_recheck = 0;
-
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
-}
-
-
-static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
-{
- struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
-
- napi_schedule(&nesvnic->napi);
-}
-
-
-/* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
-* getting out of nic_ce_handler
-*/
-#define MAX_RQES_TO_PROCESS 384
-
-/**
- * nes_nic_ce_handler
- */
-void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
-{
- u64 u64temp;
- dma_addr_t bus_address;
- struct nes_hw_nic *nesnic;
- struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_nic_rq_wqe *nic_rqe;
- struct nes_hw_nic_sq_wqe *nic_sqe;
- struct sk_buff *skb;
- struct sk_buff *rx_skb;
- struct nes_rskb_cb *cb;
- __le16 *wqe_fragment_length;
- u32 head;
- u32 cq_size;
- u32 rx_pkt_size;
- u32 cqe_count=0;
- u32 cqe_errv;
- u32 cqe_misc;
- u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
- u16 vlan_tag;
- u16 pkt_type;
- u16 rqes_processed = 0;
- u8 sq_cqes = 0;
-
- head = cq->cq_head;
- cq_size = cq->cq_size;
- cq->cqes_pending = 1;
- do {
- if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
- NES_NIC_CQE_VALID) {
- nesnic = &nesvnic->nic;
- cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
- if (cqe_misc & NES_NIC_CQE_SQ) {
- sq_cqes++;
- wqe_fragment_index = 1;
- nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
- skb = nesnic->tx_skb[nesnic->sq_tail];
- wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
- /* bump past the vlan tag */
- wqe_fragment_length++;
- if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
- u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
- wqe_fragment_index * 2]);
- u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
- wqe_fragment_index * 2])) << 32;
- bus_address = (dma_addr_t)u64temp;
- if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
- pci_unmap_single(nesdev->pcidev,
- bus_address,
- le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
- PCI_DMA_TODEVICE);
- }
- for (; wqe_fragment_index < 5; wqe_fragment_index++) {
- if (wqe_fragment_length[wqe_fragment_index]) {
- u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
- wqe_fragment_index * 2]);
- u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
- + wqe_fragment_index * 2])) <<32;
- bus_address = (dma_addr_t)u64temp;
- pci_unmap_page(nesdev->pcidev,
- bus_address,
- le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
- PCI_DMA_TODEVICE);
- } else
- break;
- }
- }
- if (skb)
- dev_kfree_skb_any(skb);
- nesnic->sq_tail++;
- nesnic->sq_tail &= nesnic->sq_size-1;
- if (sq_cqes > 128) {
- barrier();
- /* restart the queue if it had been stopped */
- if (netif_queue_stopped(nesvnic->netdev))
- netif_wake_queue(nesvnic->netdev);
- sq_cqes = 0;
- }
- } else {
- rqes_processed ++;
-
- cq->rx_cqes_completed++;
- cq->rx_pkts_indicated++;
- rx_pkt_size = cqe_misc & 0x0000ffff;
- nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
- /* Get the skb */
- rx_skb = nesnic->rx_skb[nesnic->rq_tail];
- nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
- bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
- bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
- pci_unmap_single(nesdev->pcidev, bus_address,
- nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
- cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
- cb->busaddr = 0;
- /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
- /* rx_skb->len = rx_pkt_size; */
- rx_skb->len = 0; /* TODO: see if this is necessary */
- skb_put(rx_skb, rx_pkt_size);
- rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
- nesnic->rq_tail++;
- nesnic->rq_tail &= nesnic->rq_size - 1;
-
- atomic_inc(&nesvnic->rx_skbs_needed);
- if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
- nes_write32(nesdev->regs+NES_CQE_ALLOC,
- cq->cq_number | (cqe_count << 16));
- /* nesadapter->tune_timer.cq_count += cqe_count; */
- nesdev->currcq_count += cqe_count;
- cqe_count = 0;
- nes_replenish_nic_rq(nesvnic);
- }
- pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
- cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
- rx_skb->ip_summed = CHECKSUM_NONE;
-
- if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
- (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
- if ((cqe_errv &
- (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
- NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
- if (nesvnic->netdev->features & NETIF_F_RXCSUM)
- rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
- } else
- nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
- " errv = 0x%X, pkt_type = 0x%X.\n",
- nesvnic->netdev->name, cqe_errv, pkt_type);
-
- } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
- if ((cqe_errv &
- (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
- NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
- if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
- rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
- /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
- nesvnic->netdev->name); */
- }
- } else
- nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
- " errv = 0x%X, pkt_type = 0x%X.\n",
- nesvnic->netdev->name, cqe_errv, pkt_type);
- }
- /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
- pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
-
- if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
- if (nes_cm_recv(rx_skb, nesvnic->netdev))
- rx_skb = NULL;
- }
- if (rx_skb == NULL)
- goto skip_rx_indicate0;
-
-
- if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
- vlan_tag = (u16)(le32_to_cpu(
- cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
- >> 16);
- nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
- nesvnic->netdev->name, vlan_tag);
-
- __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
- }
- napi_gro_receive(&nesvnic->napi, rx_skb);
-
-skip_rx_indicate0:
- ;
- /* nesvnic->netstats.rx_packets++; */
- /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
- }
-
- cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
- /* Accounting... */
- cqe_count++;
- if (++head >= cq_size)
- head = 0;
- if (cqe_count == 255) {
- /* Replenish Nic CQ */
- nes_write32(nesdev->regs+NES_CQE_ALLOC,
- cq->cq_number | (cqe_count << 16));
- /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
- nesdev->currcq_count += cqe_count;
- cqe_count = 0;
- }
-
- if (cq->rx_cqes_completed >= nesvnic->budget)
- break;
- } else {
- cq->cqes_pending = 0;
- break;
- }
-
- } while (1);
-
- if (sq_cqes) {
- barrier();
- /* restart the queue if it had been stopped */
- if (netif_queue_stopped(nesvnic->netdev))
- netif_wake_queue(nesvnic->netdev);
- }
- cq->cq_head = head;
- /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
- cq->cq_number, cqe_count, cq->cq_head); */
- cq->cqe_allocs_pending = cqe_count;
- if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
- {
- /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
- nesdev->currcq_count += cqe_count;
- nes_nic_tune_timer(nesdev);
- }
- if (atomic_read(&nesvnic->rx_skbs_needed))
- nes_replenish_nic_rq(nesvnic);
-}
-
-
-
-/**
- * nes_cqp_ce_handler
- */
-static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
-{
- u64 u64temp;
- unsigned long flags;
- struct nes_hw_cqp *cqp = NULL;
- struct nes_cqp_request *cqp_request;
- struct nes_hw_cqp_wqe *cqp_wqe;
- u32 head;
- u32 cq_size;
- u32 cqe_count=0;
- u32 error_code;
- u32 opcode;
- u32 ctx_index;
- /* u32 counter; */
-
- head = cq->cq_head;
- cq_size = cq->cq_size;
-
- do {
- /* process the CQE */
- /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
- le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
-
- opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
- if (opcode & NES_CQE_VALID) {
- cqp = &nesdev->cqp;
-
- error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
- if (error_code) {
- nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
- " Major/Minor codes = 0x%04X:%04X.\n",
- le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
- (u16)(error_code >> 16),
- (u16)error_code);
- }
-
- u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
- cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
- ((u64)(le32_to_cpu(cq->cq_vbase[head].
- cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
-
- cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
- if (cqp_request) {
- if (cqp_request->waiting) {
- /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
- cqp_request->major_code = (u16)(error_code >> 16);
- cqp_request->minor_code = (u16)error_code;
- barrier();
- cqp_request->request_done = 1;
- wake_up(&cqp_request->waitq);
- nes_put_cqp_request(nesdev, cqp_request);
- } else {
- if (cqp_request->callback)
- cqp_request->cqp_callback(nesdev, cqp_request);
- nes_free_cqp_request(nesdev, cqp_request);
- }
- } else {
- wake_up(&nesdev->cqp.waitq);
- }
-
- cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
- nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
- if (++cqp->sq_tail >= cqp->sq_size)
- cqp->sq_tail = 0;
-
- /* Accounting... */
- cqe_count++;
- if (++head >= cq_size)
- head = 0;
- } else {
- break;
- }
- } while (1);
- cq->cq_head = head;
-
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
- ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
- (nesdev->cqp.sq_size - 1)) != 1)) {
- cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
- struct nes_cqp_request, list);
- list_del_init(&cqp_request->list);
- head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[head];
- memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
- barrier();
-
- opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]);
- if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
- ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
- else
- ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
- cqp_wqe->wqe_words[ctx_index] =
- cpu_to_le32((u32)((unsigned long)cqp_request));
- cqp_wqe->wqe_words[ctx_index + 1] =
- cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
- nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
- cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
- /* Ring doorbell (1 WQEs) */
- barrier();
- nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
- }
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
-
- /* Arm the CCQ */
- nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- cq->cq_number);
- nes_read32(nesdev->regs+NES_CQE_ALLOC);
-}
-
-static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
-{
- if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
- /* skip over ethernet header */
- pkt += ETH_HLEN;
-
- /* Skip over IP and TCP headers */
- pkt += 4 * (pkt[0] & 0x0f);
- pkt += 4 * ((pkt[12] >> 4) & 0x0f);
- }
- return pkt;
-}
-
-/* Determine if incoming error pkt is rdma layer */
-static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
-{
- u8 *pkt;
- u16 *mpa;
- u32 opcode = 0xffffffff;
-
- if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
- pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
- mpa = (u16 *)locate_mpa(pkt, aeq_info);
- opcode = be16_to_cpu(mpa[1]) & 0xf;
- }
-
- return opcode;
-}
-
-/* Build iWARP terminate header */
-static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
-{
- u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
- u16 ddp_seg_len;
- int copy_len = 0;
- u8 is_tagged = 0;
- u8 flush_code = 0;
- struct nes_terminate_hdr *termhdr;
-
- termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
- memset(termhdr, 0, 64);
-
- if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
-
- /* Use data from offending packet to fill in ddp & rdma hdrs */
- pkt = locate_mpa(pkt, aeq_info);
- ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
- if (ddp_seg_len) {
- copy_len = 2;
- termhdr->hdrct = DDP_LEN_FLAG;
- if (pkt[2] & 0x80) {
- is_tagged = 1;
- if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
- copy_len += TERM_DDP_LEN_TAGGED;
- termhdr->hdrct |= DDP_HDR_FLAG;
- }
- } else {
- if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
- copy_len += TERM_DDP_LEN_UNTAGGED;
- termhdr->hdrct |= DDP_HDR_FLAG;
- }
-
- if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
- if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
- copy_len += TERM_RDMA_LEN;
- termhdr->hdrct |= RDMA_HDR_FLAG;
- }
- }
- }
- }
- }
-
- switch (async_event_id) {
- case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
- switch (iwarp_opcode(nesqp, aeq_info)) {
- case IWARP_OPCODE_WRITE:
- flush_code = IB_WC_LOC_PROT_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
- termhdr->error_code = DDP_TAGGED_INV_STAG;
- break;
- default:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_INV_STAG;
- }
- break;
- case NES_AEQE_AEID_AMP_INVALID_STAG:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_INV_STAG;
- break;
- case NES_AEQE_AEID_AMP_BAD_QP:
- flush_code = IB_WC_LOC_QP_OP_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_QN;
- break;
- case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
- case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
- switch (iwarp_opcode(nesqp, aeq_info)) {
- case IWARP_OPCODE_SEND_INV:
- case IWARP_OPCODE_SEND_SE_INV:
- flush_code = IB_WC_REM_OP_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
- termhdr->error_code = RDMAP_CANT_INV_STAG;
- break;
- default:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_INV_STAG;
- }
- break;
- case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
- if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
- flush_code = IB_WC_LOC_PROT_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
- termhdr->error_code = DDP_TAGGED_BOUNDS;
- } else {
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_INV_BOUNDS;
- }
- break;
- case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
- case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
- case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_ACCESS;
- break;
- case NES_AEQE_AEID_AMP_TO_WRAP:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_TO_WRAP;
- break;
- case NES_AEQE_AEID_AMP_BAD_PD:
- switch (iwarp_opcode(nesqp, aeq_info)) {
- case IWARP_OPCODE_WRITE:
- flush_code = IB_WC_LOC_PROT_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
- termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
- break;
- case IWARP_OPCODE_SEND_INV:
- case IWARP_OPCODE_SEND_SE_INV:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_CANT_INV_STAG;
- break;
- default:
- flush_code = IB_WC_REM_ACCESS_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
- termhdr->error_code = RDMAP_UNASSOC_STAG;
- }
- break;
- case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
- flush_code = IB_WC_LOC_LEN_ERR;
- termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
- termhdr->error_code = MPA_MARKER;
- break;
- case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
- flush_code = IB_WC_GENERAL_ERR;
- termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
- termhdr->error_code = MPA_CRC;
- break;
- case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
- case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
- flush_code = IB_WC_LOC_LEN_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
- termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
- break;
- case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
- case NES_AEQE_AEID_DDP_NO_L_BIT:
- flush_code = IB_WC_FATAL_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
- termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
- break;
- case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
- case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
- flush_code = IB_WC_GENERAL_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
- break;
- case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
- flush_code = IB_WC_LOC_LEN_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
- break;
- case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
- flush_code = IB_WC_GENERAL_ERR;
- if (is_tagged) {
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
- termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
- } else {
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
- }
- break;
- case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
- flush_code = IB_WC_GENERAL_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_MO;
- break;
- case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
- flush_code = IB_WC_REM_OP_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
- break;
- case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
- flush_code = IB_WC_GENERAL_ERR;
- termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
- termhdr->error_code = DDP_UNTAGGED_INV_QN;
- break;
- case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
- flush_code = IB_WC_GENERAL_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
- termhdr->error_code = RDMAP_INV_RDMAP_VER;
- break;
- case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
- flush_code = IB_WC_LOC_QP_OP_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
- termhdr->error_code = RDMAP_UNEXPECTED_OP;
- break;
- default:
- flush_code = IB_WC_FATAL_ERR;
- termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
- termhdr->error_code = RDMAP_UNSPECIFIED;
- break;
- }
-
- if (copy_len)
- memcpy(termhdr + 1, pkt, copy_len);
-
- if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
- if (aeq_info & NES_AEQE_SQ)
- nesqp->term_sq_flush_code = flush_code;
- else
- nesqp->term_rq_flush_code = flush_code;
- }
-
- return sizeof(struct nes_terminate_hdr) + copy_len;
-}
-
-static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
- struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
-{
- u64 context;
- unsigned long flags;
- u32 aeq_info;
- u16 async_event_id;
- u8 tcp_state;
- u8 iwarp_state;
- u32 termlen = 0;
- u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
- NES_CQP_QP_TERM_DONT_SEND_FIN;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- if (nesqp->term_flags & NES_TERM_SENT)
- return; /* Sanity check */
-
- aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
- tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
- iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
- async_event_id = (u16)aeq_info;
-
- context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
- aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
- if (!context) {
- WARN_ON(!context);
- return;
- }
-
- nesqp = (struct nes_qp *)(unsigned long)context;
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- nesqp->terminate_eventtype = eventtype;
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
- if (nesadapter->send_term_ok)
- termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
- else
- mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
-
- if (!nesdev->iw_status) {
- nesqp->term_flags = NES_TERM_DONE;
- nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
- nes_cm_disconn(nesqp);
- } else {
- nes_terminate_start_timer(nesqp);
- nesqp->term_flags |= NES_TERM_SENT;
- nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
- }
-}
-
-static void nes_terminate_send_fin(struct nes_device *nesdev,
- struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
-{
- u32 aeq_info;
- u16 async_event_id;
- u8 tcp_state;
- u8 iwarp_state;
- unsigned long flags;
-
- aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
- tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
- iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
- async_event_id = (u16)aeq_info;
-
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
- /* Send the fin only */
- nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
- NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
-}
-
-/* Cleanup after a terminate sent or received */
-static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
-{
- u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
- unsigned long flags;
- struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
- struct nes_device *nesdev = nesvnic->nesdev;
- u8 first_time = 0;
-
- spin_lock_irqsave(&nesqp->lock, flags);
- if (nesqp->hte_added) {
- nesqp->hte_added = 0;
- next_iwarp_state |= NES_CQP_QP_DEL_HTE;
- }
-
- first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
- nesqp->term_flags |= NES_TERM_DONE;
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
- /* Make sure we go through this only once */
- if (first_time) {
- if (timeout_occurred == 0)
- del_timer(&nesqp->terminate_timer);
- else
- next_iwarp_state |= NES_CQP_QP_RESET;
-
- nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
- nes_cm_disconn(nesqp);
- }
-}
-
-static void nes_terminate_received(struct nes_device *nesdev,
- struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
-{
- u32 aeq_info;
- u8 *pkt;
- u32 *mpa;
- u8 ddp_ctl;
- u8 rdma_ctl;
- u16 aeq_id = 0;
-
- aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
- if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
- /* Terminate is not a performance path so the silicon */
- /* did not validate the frame - do it now */
- pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
- mpa = (u32 *)locate_mpa(pkt, aeq_info);
- ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
- rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
- if ((ddp_ctl & 0xc0) != 0x40)
- aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
- else if ((ddp_ctl & 0x03) != 1)
- aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
- else if (be32_to_cpu(mpa[2]) != 2)
- aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
- else if (be32_to_cpu(mpa[3]) != 1)
- aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
- else if (be32_to_cpu(mpa[4]) != 0)
- aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
- else if ((rdma_ctl & 0xc0) != 0x40)
- aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
-
- if (aeq_id) {
- /* Bad terminate recvd - send back a terminate */
- aeq_info = (aeq_info & 0xffff0000) | aeq_id;
- aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
- nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
- return;
- }
- }
-
- nesqp->term_flags |= NES_TERM_RCVD;
- nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
- nes_terminate_start_timer(nesqp);
- nes_terminate_send_fin(nesdev, nesqp, aeqe);
-}
-
-/* Timeout routine in case terminate fails to complete */
-void nes_terminate_timeout(struct timer_list *t)
-{
- struct nes_qp *nesqp = from_timer(nesqp, t, terminate_timer);
-
- nes_terminate_done(nesqp, 1);
-}
-
-/* Set a timer in case hw cannot complete the terminate sequence */
-static void nes_terminate_start_timer(struct nes_qp *nesqp)
-{
- mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
-}
-
-/**
- * nes_process_iwarp_aeqe
- */
-static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
- struct nes_hw_aeqe *aeqe)
-{
- u64 context;
- unsigned long flags;
- struct nes_qp *nesqp;
- struct nes_hw_cq *hw_cq;
- struct nes_cq *nescq;
- int resource_allocated;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 aeq_info;
- u32 next_iwarp_state = 0;
- u32 aeqe_cq_id;
- u16 async_event_id;
- u8 tcp_state;
- u8 iwarp_state;
- struct ib_event ibevent;
-
- nes_debug(NES_DBG_AEQ, "\n");
- aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
- if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
- context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
- context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
- } else {
- context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
- aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
- BUG_ON(!context);
- }
-
- /* context is nesqp unless async_event_id == CQ ERROR */
- nesqp = (struct nes_qp *)(unsigned long)context;
- async_event_id = (u16)aeq_info;
- tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
- iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
- nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
- " Tcp state = %s, iWARP state = %s\n",
- async_event_id,
- le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
- nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
-
- aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
- if (aeq_info & NES_AEQE_QP) {
- if (!nes_is_resource_allocated(nesadapter,
- nesadapter->allocated_qps,
- aeqe_cq_id))
- return;
- }
-
- switch (async_event_id) {
- case NES_AEQE_AEID_LLP_FIN_RECEIVED:
- if (nesqp->term_flags)
- return; /* Ignore it, wait for close complete */
-
- if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
- if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
- (nesqp->ibqp_state == IB_QPS_RTS)) {
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
- nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
- spin_unlock_irqrestore(&nesqp->lock, flags);
- nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
- nes_cm_disconn(nesqp);
- }
- nesqp->cm_id->add_ref(nesqp->cm_id);
- schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
- NES_TIMER_TYPE_CLOSE, 1, 0);
- nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
- " need ae to finish up, original_last_aeq = 0x%04X."
- " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- async_event_id, nesqp->last_aeq, tcp_state);
- }
- break;
- case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- spin_unlock_irqrestore(&nesqp->lock, flags);
- nes_cm_disconn(nesqp);
- break;
-
- case NES_AEQE_AEID_RESET_SENT:
- tcp_state = NES_AEQE_TCP_STATE_CLOSED;
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- nesqp->hte_added = 0;
- spin_unlock_irqrestore(&nesqp->lock, flags);
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
- nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
- nes_cm_disconn(nesqp);
- break;
-
- case NES_AEQE_AEID_LLP_CONNECTION_RESET:
- if (atomic_read(&nesqp->close_timer_started))
- return;
- spin_lock_irqsave(&nesqp->lock, flags);
- nesqp->hw_iwarp_state = iwarp_state;
- nesqp->hw_tcp_state = tcp_state;
- nesqp->last_aeq = async_event_id;
- spin_unlock_irqrestore(&nesqp->lock, flags);
- nes_cm_disconn(nesqp);
- break;
-
- case NES_AEQE_AEID_TERMINATE_SENT:
- nes_terminate_send_fin(nesdev, nesqp, aeqe);
- break;
-
- case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
- nes_terminate_received(nesdev, nesqp, aeqe);
- break;
-
- case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
- case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
- case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
- case NES_AEQE_AEID_AMP_INVALID_STAG:
- case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
- case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
- case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
- case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
- case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
- case NES_AEQE_AEID_AMP_TO_WRAP:
- printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
- nesqp->hwqp.qp_id, async_event_id);
- nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
- break;
-
- case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
- case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
- case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
- case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
- if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
- aeq_info &= 0xffff0000;
- aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
- aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
- }
- /* fall through */
- case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
- case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
- case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
- case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
- case NES_AEQE_AEID_AMP_BAD_QP:
- case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
- case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
- case NES_AEQE_AEID_DDP_NO_L_BIT:
- case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
- case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
- case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
- case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
- case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
- case NES_AEQE_AEID_AMP_BAD_PD:
- case NES_AEQE_AEID_AMP_FASTREG_SHARED:
- case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
- case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
- case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
- case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
- case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
- case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
- case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
- case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
- case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
- case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
- case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
- case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
- case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
- case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
- case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
- case NES_AEQE_AEID_BAD_CLOSE:
- case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
- case NES_AEQE_AEID_STAG_ZERO_INVALID:
- case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
- case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
- printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
- nesqp->hwqp.qp_id, async_event_id);
- print_ip(nesqp->cm_node);
- if (!atomic_read(&nesqp->close_timer_started))
- nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
- break;
-
- case NES_AEQE_AEID_CQ_OPERATION_ERROR:
- context <<= 1;
- nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
- le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
- resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
- le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
- if (resource_allocated) {
- printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
- __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
- hw_cq = (struct nes_hw_cq *)(unsigned long)context;
- if (hw_cq) {
- nescq = container_of(hw_cq, struct nes_cq, hw_cq);
- if (nescq->ibcq.event_handler) {
- ibevent.device = nescq->ibcq.device;
- ibevent.event = IB_EVENT_CQ_ERR;
- ibevent.element.cq = &nescq->ibcq;
- nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
- }
- }
- }
- break;
-
- default:
- nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
- async_event_id);
- break;
- }
-
-}
-
-/**
- * nes_iwarp_ce_handler
- */
-void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
-{
- struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
-
- /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
- nescq->hw_cq.cq_number); */
- nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
-
- if (nescq->ibcq.comp_handler)
- nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
-
- return;
-}
-
-
-/**
- * nes_manage_apbvt()
- */
-int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
- u32 nic_index, u32 add_port)
-{
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- int ret = 0;
- u16 major_code;
-
- /* Send manage APBVT request to CQP */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
- (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
- accel_local_port, accel_local_port, nic_index);
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
- ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
-
- nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- if (add_port == NES_MANAGE_APBVT_ADD)
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n",
- ret, cqp_request->major_code, cqp_request->minor_code);
- major_code = cqp_request->major_code;
-
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (!ret)
- return -ETIME;
- else if (major_code)
- return -EIO;
- else
- return 0;
-}
-
-
-/**
- * nes_manage_arp_cache
- */
-void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
- u32 ip_addr, u32 action)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev;
- struct nes_cqp_request *cqp_request;
- int arp_index;
-
- nesdev = nesvnic->nesdev;
- arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
- if (arp_index == -1) {
- return;
- }
-
- /* update the ARP entry */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
- return;
- }
- cqp_request->waiting = 0;
- cqp_wqe = &cqp_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
- NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
- (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
-
- if (action == NES_ARP_ADD) {
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
- cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
- (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
- (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
- cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
- (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
- } else {
- cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
- cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
- }
-
- nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
- nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
-
- atomic_set(&cqp_request->refcount, 1);
- nes_post_cqp_request(nesdev, cqp_request);
-}
-
-
-/**
- * flush_wqes
- */
-void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
- u32 which_wq, u32 wait_completion)
-{
- struct nes_cqp_request *cqp_request;
- struct nes_hw_cqp_wqe *cqp_wqe;
- u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
- u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
- int ret;
-
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
- return;
- }
- if (wait_completion) {
- cqp_request->waiting = 1;
- atomic_set(&cqp_request->refcount, 2);
- } else {
- cqp_request->waiting = 0;
- }
- cqp_wqe = &cqp_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- /* If wqe in error was identified, set code to be put into cqe */
- if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
- which_wq |= NES_CQP_FLUSH_MAJ_MIN;
- sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
- nesqp->term_sq_flush_code = 0;
- }
-
- if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
- which_wq |= NES_CQP_FLUSH_MAJ_MIN;
- rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
- nesqp->term_rq_flush_code = 0;
- }
-
- if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
- cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
- cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
- }
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
- cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
-
- nes_post_cqp_request(nesdev, cqp_request);
-
- if (wait_completion) {
- /* Wait for CQP */
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X\n",
- ret, cqp_request->major_code, cqp_request->minor_code);
- nes_put_cqp_request(nesdev, cqp_request);
- }
-}
diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h
deleted file mode 100644
index 3c56470816a8..000000000000
--- a/drivers/infiniband/hw/nes/nes_hw.h
+++ /dev/null
@@ -1,1380 +0,0 @@
-/*
-* Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenIB.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*/
-
-#ifndef __NES_HW_H
-#define __NES_HW_H
-
-#define NES_PHY_TYPE_CX4 1
-#define NES_PHY_TYPE_1G 2
-#define NES_PHY_TYPE_ARGUS 4
-#define NES_PHY_TYPE_PUMA_1G 5
-#define NES_PHY_TYPE_PUMA_10G 6
-#define NES_PHY_TYPE_GLADIUS 7
-#define NES_PHY_TYPE_SFP_D 8
-#define NES_PHY_TYPE_KR 9
-
-#define NES_MULTICAST_PF_MAX 8
-#define NES_A0 3
-
-#define NES_ENABLE_PAU 0x07000001
-#define NES_DISABLE_PAU 0x07000000
-#define NES_PAU_COUNTER 10
-#define NES_CQP_OPCODE_MASK 0x3f
-
-enum pci_regs {
- NES_INT_STAT = 0x0000,
- NES_INT_MASK = 0x0004,
- NES_INT_PENDING = 0x0008,
- NES_INTF_INT_STAT = 0x000C,
- NES_INTF_INT_MASK = 0x0010,
- NES_TIMER_STAT = 0x0014,
- NES_PERIODIC_CONTROL = 0x0018,
- NES_ONE_SHOT_CONTROL = 0x001C,
- NES_EEPROM_COMMAND = 0x0020,
- NES_EEPROM_DATA = 0x0024,
- NES_FLASH_COMMAND = 0x0028,
- NES_FLASH_DATA = 0x002C,
- NES_SOFTWARE_RESET = 0x0030,
- NES_CQ_ACK = 0x0034,
- NES_WQE_ALLOC = 0x0040,
- NES_CQE_ALLOC = 0x0044,
- NES_AEQ_ALLOC = 0x0048
-};
-
-enum indexed_regs {
- NES_IDX_CREATE_CQP_LOW = 0x0000,
- NES_IDX_CREATE_CQP_HIGH = 0x0004,
- NES_IDX_QP_CONTROL = 0x0040,
- NES_IDX_FLM_CONTROL = 0x0080,
- NES_IDX_INT_CPU_STATUS = 0x00a0,
- NES_IDX_GPR_TRIGGER = 0x00bc,
- NES_IDX_GPIO_CONTROL = 0x00f0,
- NES_IDX_GPIO_DATA = 0x00f4,
- NES_IDX_GPR2 = 0x010c,
- NES_IDX_TCP_CONFIG0 = 0x01e4,
- NES_IDX_TCP_TIMER_CONFIG = 0x01ec,
- NES_IDX_TCP_NOW = 0x01f0,
- NES_IDX_QP_MAX_CFG_SIZES = 0x0200,
- NES_IDX_QP_CTX_SIZE = 0x0218,
- NES_IDX_TCP_TIMER_SIZE0 = 0x0238,
- NES_IDX_TCP_TIMER_SIZE1 = 0x0240,
- NES_IDX_ARP_CACHE_SIZE = 0x0258,
- NES_IDX_CQ_CTX_SIZE = 0x0260,
- NES_IDX_MRT_SIZE = 0x0278,
- NES_IDX_PBL_REGION_SIZE = 0x0280,
- NES_IDX_IRRQ_COUNT = 0x02b0,
- NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x02f0,
- NES_IDX_RX_WINDOW_BUFFER_SIZE = 0x0300,
- NES_IDX_DST_IP_ADDR = 0x0400,
- NES_IDX_PCIX_DIAG = 0x08e8,
- NES_IDX_MPP_DEBUG = 0x0a00,
- NES_IDX_PORT_RX_DISCARDS = 0x0a30,
- NES_IDX_PORT_TX_DISCARDS = 0x0a34,
- NES_IDX_MPP_LB_DEBUG = 0x0b00,
- NES_IDX_DENALI_CTL_22 = 0x1058,
- NES_IDX_MAC_TX_CONTROL = 0x2000,
- NES_IDX_MAC_TX_CONFIG = 0x2004,
- NES_IDX_MAC_TX_PAUSE_QUANTA = 0x2008,
- NES_IDX_MAC_RX_CONTROL = 0x200c,
- NES_IDX_MAC_RX_CONFIG = 0x2010,
- NES_IDX_MAC_EXACT_MATCH_BOTTOM = 0x201c,
- NES_IDX_MAC_MDIO_CONTROL = 0x2084,
- NES_IDX_MAC_TX_OCTETS_LOW = 0x2100,
- NES_IDX_MAC_TX_OCTETS_HIGH = 0x2104,
- NES_IDX_MAC_TX_FRAMES_LOW = 0x2108,
- NES_IDX_MAC_TX_FRAMES_HIGH = 0x210c,
- NES_IDX_MAC_TX_PAUSE_FRAMES = 0x2118,
- NES_IDX_MAC_TX_ERRORS = 0x2138,
- NES_IDX_MAC_RX_OCTETS_LOW = 0x213c,
- NES_IDX_MAC_RX_OCTETS_HIGH = 0x2140,
- NES_IDX_MAC_RX_FRAMES_LOW = 0x2144,
- NES_IDX_MAC_RX_FRAMES_HIGH = 0x2148,
- NES_IDX_MAC_RX_BC_FRAMES_LOW = 0x214c,
- NES_IDX_MAC_RX_MC_FRAMES_HIGH = 0x2150,
- NES_IDX_MAC_RX_PAUSE_FRAMES = 0x2154,
- NES_IDX_MAC_RX_SHORT_FRAMES = 0x2174,
- NES_IDX_MAC_RX_OVERSIZED_FRAMES = 0x2178,
- NES_IDX_MAC_RX_JABBER_FRAMES = 0x217c,
- NES_IDX_MAC_RX_CRC_ERR_FRAMES = 0x2180,
- NES_IDX_MAC_RX_LENGTH_ERR_FRAMES = 0x2184,
- NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES = 0x2188,
- NES_IDX_MAC_INT_STATUS = 0x21f0,
- NES_IDX_MAC_INT_MASK = 0x21f4,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 = 0x2800,
- NES_IDX_PHY_PCS_CONTROL_STATUS1 = 0x2a00,
- NES_IDX_ETH_SERDES_COMMON_CONTROL0 = 0x2808,
- NES_IDX_ETH_SERDES_COMMON_CONTROL1 = 0x2a08,
- NES_IDX_ETH_SERDES_COMMON_STATUS0 = 0x280c,
- NES_IDX_ETH_SERDES_COMMON_STATUS1 = 0x2a0c,
- NES_IDX_ETH_SERDES_TX_EMP0 = 0x2810,
- NES_IDX_ETH_SERDES_TX_EMP1 = 0x2a10,
- NES_IDX_ETH_SERDES_TX_DRIVE0 = 0x2814,
- NES_IDX_ETH_SERDES_TX_DRIVE1 = 0x2a14,
- NES_IDX_ETH_SERDES_RX_MODE0 = 0x2818,
- NES_IDX_ETH_SERDES_RX_MODE1 = 0x2a18,
- NES_IDX_ETH_SERDES_RX_SIGDET0 = 0x281c,
- NES_IDX_ETH_SERDES_RX_SIGDET1 = 0x2a1c,
- NES_IDX_ETH_SERDES_BYPASS0 = 0x2820,
- NES_IDX_ETH_SERDES_BYPASS1 = 0x2a20,
- NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0 = 0x2824,
- NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1 = 0x2a24,
- NES_IDX_ETH_SERDES_RX_EQ_CONTROL0 = 0x2828,
- NES_IDX_ETH_SERDES_RX_EQ_CONTROL1 = 0x2a28,
- NES_IDX_ETH_SERDES_RX_EQ_STATUS0 = 0x282c,
- NES_IDX_ETH_SERDES_RX_EQ_STATUS1 = 0x2a2c,
- NES_IDX_ETH_SERDES_CDR_RESET0 = 0x2830,
- NES_IDX_ETH_SERDES_CDR_RESET1 = 0x2a30,
- NES_IDX_ETH_SERDES_CDR_CONTROL0 = 0x2834,
- NES_IDX_ETH_SERDES_CDR_CONTROL1 = 0x2a34,
- NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0 = 0x2838,
- NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1 = 0x2a38,
- NES_IDX_ENDNODE0_NSTAT_RX_DISCARD = 0x3080,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO = 0x3000,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI = 0x3004,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO = 0x3008,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI = 0x300c,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO = 0x7000,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI = 0x7004,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO = 0x7008,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI = 0x700c,
- NES_IDX_WQM_CONFIG0 = 0x5000,
- NES_IDX_WQM_CONFIG1 = 0x5004,
- NES_IDX_CM_CONFIG = 0x5100,
- NES_IDX_NIC_LOGPORT_TO_PHYPORT = 0x6000,
- NES_IDX_NIC_PHYPORT_TO_USW = 0x6008,
- NES_IDX_NIC_ACTIVE = 0x6010,
- NES_IDX_NIC_UNICAST_ALL = 0x6018,
- NES_IDX_NIC_MULTICAST_ALL = 0x6020,
- NES_IDX_NIC_MULTICAST_ENABLE = 0x6028,
- NES_IDX_NIC_BROADCAST_ON = 0x6030,
- NES_IDX_USED_CHUNKS_TX = 0x60b0,
- NES_IDX_TX_POOL_SIZE = 0x60b8,
- NES_IDX_QUAD_HASH_TABLE_SIZE = 0x6148,
- NES_IDX_PERFECT_FILTER_LOW = 0x6200,
- NES_IDX_PERFECT_FILTER_HIGH = 0x6204,
- NES_IDX_IPV4_TCP_REXMITS = 0x7080,
- NES_IDX_DEBUG_ERROR_CONTROL_STATUS = 0x913c,
- NES_IDX_DEBUG_ERROR_MASKS0 = 0x9140,
- NES_IDX_DEBUG_ERROR_MASKS1 = 0x9144,
- NES_IDX_DEBUG_ERROR_MASKS2 = 0x9148,
- NES_IDX_DEBUG_ERROR_MASKS3 = 0x914c,
- NES_IDX_DEBUG_ERROR_MASKS4 = 0x9150,
- NES_IDX_DEBUG_ERROR_MASKS5 = 0x9154,
-};
-
-#define NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE 1
-#define NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE (1 << 17)
-
-enum nes_cqp_opcodes {
- NES_CQP_CREATE_QP = 0x00,
- NES_CQP_MODIFY_QP = 0x01,
- NES_CQP_DESTROY_QP = 0x02,
- NES_CQP_CREATE_CQ = 0x03,
- NES_CQP_MODIFY_CQ = 0x04,
- NES_CQP_DESTROY_CQ = 0x05,
- NES_CQP_ALLOCATE_STAG = 0x09,
- NES_CQP_REGISTER_STAG = 0x0a,
- NES_CQP_QUERY_STAG = 0x0b,
- NES_CQP_REGISTER_SHARED_STAG = 0x0c,
- NES_CQP_DEALLOCATE_STAG = 0x0d,
- NES_CQP_MANAGE_ARP_CACHE = 0x0f,
- NES_CQP_DOWNLOAD_SEGMENT = 0x10,
- NES_CQP_SUSPEND_QPS = 0x11,
- NES_CQP_UPLOAD_CONTEXT = 0x13,
- NES_CQP_CREATE_CEQ = 0x16,
- NES_CQP_DESTROY_CEQ = 0x18,
- NES_CQP_CREATE_AEQ = 0x19,
- NES_CQP_DESTROY_AEQ = 0x1b,
- NES_CQP_LMI_ACCESS = 0x20,
- NES_CQP_FLUSH_WQES = 0x22,
- NES_CQP_MANAGE_APBVT = 0x23,
- NES_CQP_MANAGE_QUAD_HASH = 0x25
-};
-
-enum nes_cqp_wqe_word_idx {
- NES_CQP_WQE_OPCODE_IDX = 0,
- NES_CQP_WQE_ID_IDX = 1,
- NES_CQP_WQE_COMP_CTX_LOW_IDX = 2,
- NES_CQP_WQE_COMP_CTX_HIGH_IDX = 3,
- NES_CQP_WQE_COMP_SCRATCH_LOW_IDX = 4,
- NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX = 5,
-};
-
-enum nes_cqp_wqe_word_download_idx { /* format differs from other cqp ops */
- NES_CQP_WQE_DL_OPCODE_IDX = 0,
- NES_CQP_WQE_DL_COMP_CTX_LOW_IDX = 1,
- NES_CQP_WQE_DL_COMP_CTX_HIGH_IDX = 2,
- NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX = 3
- /* For index values 4-15 use NES_NIC_SQ_WQE_ values */
-};
-
-enum nes_cqp_cq_wqeword_idx {
- NES_CQP_CQ_WQE_PBL_LOW_IDX = 6,
- NES_CQP_CQ_WQE_PBL_HIGH_IDX = 7,
- NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX = 8,
- NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX = 9,
- NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX = 10,
-};
-
-enum nes_cqp_stag_wqeword_idx {
- NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX = 1,
- NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX = 6,
- NES_CQP_STAG_WQE_LEN_LOW_IDX = 7,
- NES_CQP_STAG_WQE_STAG_IDX = 8,
- NES_CQP_STAG_WQE_VA_LOW_IDX = 10,
- NES_CQP_STAG_WQE_VA_HIGH_IDX = 11,
- NES_CQP_STAG_WQE_PA_LOW_IDX = 12,
- NES_CQP_STAG_WQE_PA_HIGH_IDX = 13,
- NES_CQP_STAG_WQE_PBL_LEN_IDX = 14
-};
-
-#define NES_CQP_OP_LOGICAL_PORT_SHIFT 26
-#define NES_CQP_OP_IWARP_STATE_SHIFT 28
-#define NES_CQP_OP_TERMLEN_SHIFT 28
-
-enum nes_cqp_qp_bits {
- NES_CQP_QP_ARP_VALID = (1<<8),
- NES_CQP_QP_WINBUF_VALID = (1<<9),
- NES_CQP_QP_CONTEXT_VALID = (1<<10),
- NES_CQP_QP_ORD_VALID = (1<<11),
- NES_CQP_QP_WINBUF_DATAIND_EN = (1<<12),
- NES_CQP_QP_VIRT_WQS = (1<<13),
- NES_CQP_QP_DEL_HTE = (1<<14),
- NES_CQP_QP_CQS_VALID = (1<<15),
- NES_CQP_QP_TYPE_TSA = 0,
- NES_CQP_QP_TYPE_IWARP = (1<<16),
- NES_CQP_QP_TYPE_CQP = (4<<16),
- NES_CQP_QP_TYPE_NIC = (5<<16),
- NES_CQP_QP_MSS_CHG = (1<<20),
- NES_CQP_QP_STATIC_RESOURCES = (1<<21),
- NES_CQP_QP_IGNORE_MW_BOUND = (1<<22),
- NES_CQP_QP_VWQ_USE_LMI = (1<<23),
- NES_CQP_QP_IWARP_STATE_IDLE = (1<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_IWARP_STATE_RTS = (2<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_IWARP_STATE_CLOSING = (3<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_IWARP_STATE_TERMINATE = (5<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_IWARP_STATE_ERROR = (6<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_IWARP_STATE_MASK = (7<<NES_CQP_OP_IWARP_STATE_SHIFT),
- NES_CQP_QP_TERM_DONT_SEND_FIN = (1<<24),
- NES_CQP_QP_TERM_DONT_SEND_TERM_MSG = (1<<25),
- NES_CQP_QP_RESET = (1<<31),
-};
-
-enum nes_cqp_qp_wqe_word_idx {
- NES_CQP_QP_WQE_CONTEXT_LOW_IDX = 6,
- NES_CQP_QP_WQE_CONTEXT_HIGH_IDX = 7,
- NES_CQP_QP_WQE_FLUSH_SQ_CODE = 8,
- NES_CQP_QP_WQE_FLUSH_RQ_CODE = 9,
- NES_CQP_QP_WQE_NEW_MSS_IDX = 15,
-};
-
-enum nes_nic_ctx_bits {
- NES_NIC_CTX_RQ_SIZE_32 = (3<<8),
- NES_NIC_CTX_RQ_SIZE_512 = (3<<8),
- NES_NIC_CTX_SQ_SIZE_32 = (1<<10),
- NES_NIC_CTX_SQ_SIZE_512 = (3<<10),
-};
-
-enum nes_nic_qp_ctx_word_idx {
- NES_NIC_CTX_MISC_IDX = 0,
- NES_NIC_CTX_SQ_LOW_IDX = 2,
- NES_NIC_CTX_SQ_HIGH_IDX = 3,
- NES_NIC_CTX_RQ_LOW_IDX = 4,
- NES_NIC_CTX_RQ_HIGH_IDX = 5,
-};
-
-enum nes_cqp_cq_bits {
- NES_CQP_CQ_CEQE_MASK = (1<<9),
- NES_CQP_CQ_CEQ_VALID = (1<<10),
- NES_CQP_CQ_RESIZE = (1<<11),
- NES_CQP_CQ_CHK_OVERFLOW = (1<<12),
- NES_CQP_CQ_4KB_CHUNK = (1<<14),
- NES_CQP_CQ_VIRT = (1<<15),
-};
-
-enum nes_cqp_stag_bits {
- NES_CQP_STAG_VA_TO = (1<<9),
- NES_CQP_STAG_DEALLOC_PBLS = (1<<10),
- NES_CQP_STAG_PBL_BLK_SIZE = (1<<11),
- NES_CQP_STAG_MR = (1<<13),
- NES_CQP_STAG_RIGHTS_LOCAL_READ = (1<<16),
- NES_CQP_STAG_RIGHTS_LOCAL_WRITE = (1<<17),
- NES_CQP_STAG_RIGHTS_REMOTE_READ = (1<<18),
- NES_CQP_STAG_RIGHTS_REMOTE_WRITE = (1<<19),
- NES_CQP_STAG_RIGHTS_WINDOW_BIND = (1<<20),
- NES_CQP_STAG_REM_ACC_EN = (1<<21),
- NES_CQP_STAG_LEAVE_PENDING = (1<<31),
-};
-
-enum nes_cqp_ceq_wqeword_idx {
- NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX = 1,
- NES_CQP_CEQ_WQE_PBL_LOW_IDX = 6,
- NES_CQP_CEQ_WQE_PBL_HIGH_IDX = 7,
-};
-
-enum nes_cqp_ceq_bits {
- NES_CQP_CEQ_4KB_CHUNK = (1<<14),
- NES_CQP_CEQ_VIRT = (1<<15),
-};
-
-enum nes_cqp_aeq_wqeword_idx {
- NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX = 1,
- NES_CQP_AEQ_WQE_PBL_LOW_IDX = 6,
- NES_CQP_AEQ_WQE_PBL_HIGH_IDX = 7,
-};
-
-enum nes_cqp_aeq_bits {
- NES_CQP_AEQ_4KB_CHUNK = (1<<14),
- NES_CQP_AEQ_VIRT = (1<<15),
-};
-
-enum nes_cqp_lmi_wqeword_idx {
- NES_CQP_LMI_WQE_LMI_OFFSET_IDX = 1,
- NES_CQP_LMI_WQE_FRAG_LOW_IDX = 8,
- NES_CQP_LMI_WQE_FRAG_HIGH_IDX = 9,
- NES_CQP_LMI_WQE_FRAG_LEN_IDX = 10,
-};
-
-enum nes_cqp_arp_wqeword_idx {
- NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX = 6,
- NES_CQP_ARP_WQE_MAC_HIGH_IDX = 7,
- NES_CQP_ARP_WQE_REACHABILITY_MAX_IDX = 1,
-};
-
-enum nes_cqp_upload_wqeword_idx {
- NES_CQP_UPLOAD_WQE_CTXT_LOW_IDX = 6,
- NES_CQP_UPLOAD_WQE_CTXT_HIGH_IDX = 7,
- NES_CQP_UPLOAD_WQE_HTE_IDX = 8,
-};
-
-enum nes_cqp_arp_bits {
- NES_CQP_ARP_VALID = (1<<8),
- NES_CQP_ARP_PERM = (1<<9),
-};
-
-enum nes_cqp_flush_bits {
- NES_CQP_FLUSH_SQ = (1<<30),
- NES_CQP_FLUSH_RQ = (1<<31),
- NES_CQP_FLUSH_MAJ_MIN = (1<<28),
-};
-
-enum nes_cqe_opcode_bits {
- NES_CQE_STAG_VALID = (1<<6),
- NES_CQE_ERROR = (1<<7),
- NES_CQE_SQ = (1<<8),
- NES_CQE_SE = (1<<9),
- NES_CQE_PSH = (1<<29),
- NES_CQE_FIN = (1<<30),
- NES_CQE_VALID = (1<<31),
-};
-
-
-enum nes_cqe_word_idx {
- NES_CQE_PAYLOAD_LENGTH_IDX = 0,
- NES_CQE_COMP_COMP_CTX_LOW_IDX = 2,
- NES_CQE_COMP_COMP_CTX_HIGH_IDX = 3,
- NES_CQE_INV_STAG_IDX = 4,
- NES_CQE_QP_ID_IDX = 5,
- NES_CQE_ERROR_CODE_IDX = 6,
- NES_CQE_OPCODE_IDX = 7,
-};
-
-enum nes_ceqe_word_idx {
- NES_CEQE_CQ_CTX_LOW_IDX = 0,
- NES_CEQE_CQ_CTX_HIGH_IDX = 1,
-};
-
-enum nes_ceqe_status_bit {
- NES_CEQE_VALID = (1<<31),
-};
-
-enum nes_int_bits {
- NES_INT_CEQ0 = (1<<0),
- NES_INT_CEQ1 = (1<<1),
- NES_INT_CEQ2 = (1<<2),
- NES_INT_CEQ3 = (1<<3),
- NES_INT_CEQ4 = (1<<4),
- NES_INT_CEQ5 = (1<<5),
- NES_INT_CEQ6 = (1<<6),
- NES_INT_CEQ7 = (1<<7),
- NES_INT_CEQ8 = (1<<8),
- NES_INT_CEQ9 = (1<<9),
- NES_INT_CEQ10 = (1<<10),
- NES_INT_CEQ11 = (1<<11),
- NES_INT_CEQ12 = (1<<12),
- NES_INT_CEQ13 = (1<<13),
- NES_INT_CEQ14 = (1<<14),
- NES_INT_CEQ15 = (1<<15),
- NES_INT_AEQ0 = (1<<16),
- NES_INT_AEQ1 = (1<<17),
- NES_INT_AEQ2 = (1<<18),
- NES_INT_AEQ3 = (1<<19),
- NES_INT_AEQ4 = (1<<20),
- NES_INT_AEQ5 = (1<<21),
- NES_INT_AEQ6 = (1<<22),
- NES_INT_AEQ7 = (1<<23),
- NES_INT_MAC0 = (1<<24),
- NES_INT_MAC1 = (1<<25),
- NES_INT_MAC2 = (1<<26),
- NES_INT_MAC3 = (1<<27),
- NES_INT_TSW = (1<<28),
- NES_INT_TIMER = (1<<29),
- NES_INT_INTF = (1<<30),
-};
-
-enum nes_intf_int_bits {
- NES_INTF_INT_PCIERR = (1<<0),
- NES_INTF_PERIODIC_TIMER = (1<<2),
- NES_INTF_ONE_SHOT_TIMER = (1<<3),
- NES_INTF_INT_CRITERR = (1<<14),
- NES_INTF_INT_AEQ0_OFLOW = (1<<16),
- NES_INTF_INT_AEQ1_OFLOW = (1<<17),
- NES_INTF_INT_AEQ2_OFLOW = (1<<18),
- NES_INTF_INT_AEQ3_OFLOW = (1<<19),
- NES_INTF_INT_AEQ4_OFLOW = (1<<20),
- NES_INTF_INT_AEQ5_OFLOW = (1<<21),
- NES_INTF_INT_AEQ6_OFLOW = (1<<22),
- NES_INTF_INT_AEQ7_OFLOW = (1<<23),
- NES_INTF_INT_AEQ_OFLOW = (0xff<<16),
-};
-
-enum nes_mac_int_bits {
- NES_MAC_INT_LINK_STAT_CHG = (1<<1),
- NES_MAC_INT_XGMII_EXT = (1<<2),
- NES_MAC_INT_TX_UNDERFLOW = (1<<6),
- NES_MAC_INT_TX_ERROR = (1<<7),
-};
-
-enum nes_cqe_allocate_bits {
- NES_CQE_ALLOC_INC_SELECT = (1<<28),
- NES_CQE_ALLOC_NOTIFY_NEXT = (1<<29),
- NES_CQE_ALLOC_NOTIFY_SE = (1<<30),
- NES_CQE_ALLOC_RESET = (1<<31),
-};
-
-enum nes_nic_rq_wqe_word_idx {
- NES_NIC_RQ_WQE_LENGTH_1_0_IDX = 0,
- NES_NIC_RQ_WQE_LENGTH_3_2_IDX = 1,
- NES_NIC_RQ_WQE_FRAG0_LOW_IDX = 2,
- NES_NIC_RQ_WQE_FRAG0_HIGH_IDX = 3,
- NES_NIC_RQ_WQE_FRAG1_LOW_IDX = 4,
- NES_NIC_RQ_WQE_FRAG1_HIGH_IDX = 5,
- NES_NIC_RQ_WQE_FRAG2_LOW_IDX = 6,
- NES_NIC_RQ_WQE_FRAG2_HIGH_IDX = 7,
- NES_NIC_RQ_WQE_FRAG3_LOW_IDX = 8,
- NES_NIC_RQ_WQE_FRAG3_HIGH_IDX = 9,
-};
-
-enum nes_nic_sq_wqe_word_idx {
- NES_NIC_SQ_WQE_MISC_IDX = 0,
- NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX = 1,
- NES_NIC_SQ_WQE_LSO_INFO_IDX = 2,
- NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX = 3,
- NES_NIC_SQ_WQE_LENGTH_2_1_IDX = 4,
- NES_NIC_SQ_WQE_LENGTH_4_3_IDX = 5,
- NES_NIC_SQ_WQE_FRAG0_LOW_IDX = 6,
- NES_NIC_SQ_WQE_FRAG0_HIGH_IDX = 7,
- NES_NIC_SQ_WQE_FRAG1_LOW_IDX = 8,
- NES_NIC_SQ_WQE_FRAG1_HIGH_IDX = 9,
- NES_NIC_SQ_WQE_FRAG2_LOW_IDX = 10,
- NES_NIC_SQ_WQE_FRAG2_HIGH_IDX = 11,
- NES_NIC_SQ_WQE_FRAG3_LOW_IDX = 12,
- NES_NIC_SQ_WQE_FRAG3_HIGH_IDX = 13,
- NES_NIC_SQ_WQE_FRAG4_LOW_IDX = 14,
- NES_NIC_SQ_WQE_FRAG4_HIGH_IDX = 15,
-};
-
-enum nes_iwarp_sq_wqe_word_idx {
- NES_IWARP_SQ_WQE_MISC_IDX = 0,
- NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX = 1,
- NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX = 2,
- NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX = 3,
- NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
- NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
- NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX = 7,
- NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX = 8,
- NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX = 9,
- NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX = 10,
- NES_IWARP_SQ_WQE_RDMA_STAG_IDX = 11,
- NES_IWARP_SQ_WQE_IMM_DATA_START_IDX = 12,
- NES_IWARP_SQ_WQE_FRAG0_LOW_IDX = 16,
- NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX = 17,
- NES_IWARP_SQ_WQE_LENGTH0_IDX = 18,
- NES_IWARP_SQ_WQE_STAG0_IDX = 19,
- NES_IWARP_SQ_WQE_FRAG1_LOW_IDX = 20,
- NES_IWARP_SQ_WQE_FRAG1_HIGH_IDX = 21,
- NES_IWARP_SQ_WQE_LENGTH1_IDX = 22,
- NES_IWARP_SQ_WQE_STAG1_IDX = 23,
- NES_IWARP_SQ_WQE_FRAG2_LOW_IDX = 24,
- NES_IWARP_SQ_WQE_FRAG2_HIGH_IDX = 25,
- NES_IWARP_SQ_WQE_LENGTH2_IDX = 26,
- NES_IWARP_SQ_WQE_STAG2_IDX = 27,
- NES_IWARP_SQ_WQE_FRAG3_LOW_IDX = 28,
- NES_IWARP_SQ_WQE_FRAG3_HIGH_IDX = 29,
- NES_IWARP_SQ_WQE_LENGTH3_IDX = 30,
- NES_IWARP_SQ_WQE_STAG3_IDX = 31,
-};
-
-enum nes_iwarp_sq_bind_wqe_word_idx {
- NES_IWARP_SQ_BIND_WQE_MR_IDX = 6,
- NES_IWARP_SQ_BIND_WQE_MW_IDX = 7,
- NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX = 8,
- NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX = 9,
- NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX = 10,
- NES_IWARP_SQ_BIND_WQE_VA_FBO_HIGH_IDX = 11,
-};
-
-enum nes_iwarp_sq_fmr_wqe_word_idx {
- NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX = 7,
- NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX = 8,
- NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX = 9,
- NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX = 10,
- NES_IWARP_SQ_FMR_WQE_VA_FBO_HIGH_IDX = 11,
- NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX = 12,
- NES_IWARP_SQ_FMR_WQE_PBL_ADDR_HIGH_IDX = 13,
- NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX = 14,
-};
-
-enum nes_iwarp_sq_fmr_opcodes {
- NES_IWARP_SQ_FMR_WQE_ZERO_BASED = (1<<6),
- NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K = (0<<7),
- NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M = (1<<7),
- NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ = (1<<16),
- NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE = (1<<17),
- NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ = (1<<18),
- NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE = (1<<19),
- NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND = (1<<20),
-};
-
-#define NES_IWARP_SQ_FMR_WQE_MR_LENGTH_HIGH_MASK 0xFF;
-
-enum nes_iwarp_sq_locinv_wqe_word_idx {
- NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX = 6,
-};
-
-enum nes_iwarp_rq_wqe_word_idx {
- NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX = 1,
- NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX = 2,
- NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX = 3,
- NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
- NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
- NES_IWARP_RQ_WQE_FRAG0_LOW_IDX = 8,
- NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX = 9,
- NES_IWARP_RQ_WQE_LENGTH0_IDX = 10,
- NES_IWARP_RQ_WQE_STAG0_IDX = 11,
- NES_IWARP_RQ_WQE_FRAG1_LOW_IDX = 12,
- NES_IWARP_RQ_WQE_FRAG1_HIGH_IDX = 13,
- NES_IWARP_RQ_WQE_LENGTH1_IDX = 14,
- NES_IWARP_RQ_WQE_STAG1_IDX = 15,
- NES_IWARP_RQ_WQE_FRAG2_LOW_IDX = 16,
- NES_IWARP_RQ_WQE_FRAG2_HIGH_IDX = 17,
- NES_IWARP_RQ_WQE_LENGTH2_IDX = 18,
- NES_IWARP_RQ_WQE_STAG2_IDX = 19,
- NES_IWARP_RQ_WQE_FRAG3_LOW_IDX = 20,
- NES_IWARP_RQ_WQE_FRAG3_HIGH_IDX = 21,
- NES_IWARP_RQ_WQE_LENGTH3_IDX = 22,
- NES_IWARP_RQ_WQE_STAG3_IDX = 23,
-};
-
-enum nes_nic_sq_wqe_bits {
- NES_NIC_SQ_WQE_PHDR_CS_READY = (1<<21),
- NES_NIC_SQ_WQE_LSO_ENABLE = (1<<22),
- NES_NIC_SQ_WQE_TAGVALUE_ENABLE = (1<<23),
- NES_NIC_SQ_WQE_DISABLE_CHKSUM = (1<<30),
- NES_NIC_SQ_WQE_COMPLETION = (1<<31),
-};
-
-enum nes_nic_cqe_word_idx {
- NES_NIC_CQE_ACCQP_ID_IDX = 0,
- NES_NIC_CQE_HASH_RCVNXT = 1,
- NES_NIC_CQE_TAG_PKT_TYPE_IDX = 2,
- NES_NIC_CQE_MISC_IDX = 3,
-};
-
-#define NES_PKT_TYPE_APBVT_BITS 0xC112
-#define NES_PKT_TYPE_APBVT_MASK 0xff3e
-
-#define NES_PKT_TYPE_PVALID_BITS 0x10000000
-#define NES_PKT_TYPE_PVALID_MASK 0x30000000
-
-#define NES_PKT_TYPE_TCPV4_BITS 0x0110
-#define NES_PKT_TYPE_TCPV4_MASK 0x3f30
-
-#define NES_PKT_TYPE_UDPV4_BITS 0x0210
-#define NES_PKT_TYPE_UDPV4_MASK 0x3f30
-
-#define NES_PKT_TYPE_IPV4_BITS 0x0010
-#define NES_PKT_TYPE_IPV4_MASK 0x3f30
-
-#define NES_PKT_TYPE_OTHER_BITS 0x0000
-#define NES_PKT_TYPE_OTHER_MASK 0x0030
-
-#define NES_NIC_CQE_ERRV_SHIFT 16
-enum nes_nic_ev_bits {
- NES_NIC_ERRV_BITS_MODE = (1<<0),
- NES_NIC_ERRV_BITS_IPV4_CSUM_ERR = (1<<1),
- NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR = (1<<2),
- NES_NIC_ERRV_BITS_WQE_OVERRUN = (1<<3),
- NES_NIC_ERRV_BITS_IPH_ERR = (1<<4),
-};
-
-enum nes_nic_cqe_bits {
- NES_NIC_CQE_ERRV_MASK = (0xff<<NES_NIC_CQE_ERRV_SHIFT),
- NES_NIC_CQE_SQ = (1<<24),
- NES_NIC_CQE_ACCQP_PORT = (1<<28),
- NES_NIC_CQE_ACCQP_VALID = (1<<29),
- NES_NIC_CQE_TAG_VALID = (1<<30),
- NES_NIC_CQE_VALID = (1<<31),
-};
-
-enum nes_aeqe_word_idx {
- NES_AEQE_COMP_CTXT_LOW_IDX = 0,
- NES_AEQE_COMP_CTXT_HIGH_IDX = 1,
- NES_AEQE_COMP_QP_CQ_ID_IDX = 2,
- NES_AEQE_MISC_IDX = 3,
-};
-
-enum nes_aeqe_bits {
- NES_AEQE_QP = (1<<16),
- NES_AEQE_CQ = (1<<17),
- NES_AEQE_SQ = (1<<18),
- NES_AEQE_INBOUND_RDMA = (1<<19),
- NES_AEQE_IWARP_STATE_MASK = (7<<20),
- NES_AEQE_TCP_STATE_MASK = (0xf<<24),
- NES_AEQE_Q2_DATA_WRITTEN = (0x3<<28),
- NES_AEQE_VALID = (1<<31),
-};
-
-#define NES_AEQE_IWARP_STATE_SHIFT 20
-#define NES_AEQE_TCP_STATE_SHIFT 24
-#define NES_AEQE_Q2_DATA_ETHERNET (1<<28)
-#define NES_AEQE_Q2_DATA_MPA (1<<29)
-
-enum nes_aeqe_iwarp_state {
- NES_AEQE_IWARP_STATE_NON_EXISTANT = 0,
- NES_AEQE_IWARP_STATE_IDLE = 1,
- NES_AEQE_IWARP_STATE_RTS = 2,
- NES_AEQE_IWARP_STATE_CLOSING = 3,
- NES_AEQE_IWARP_STATE_TERMINATE = 5,
- NES_AEQE_IWARP_STATE_ERROR = 6
-};
-
-enum nes_aeqe_tcp_state {
- NES_AEQE_TCP_STATE_NON_EXISTANT = 0,
- NES_AEQE_TCP_STATE_CLOSED = 1,
- NES_AEQE_TCP_STATE_LISTEN = 2,
- NES_AEQE_TCP_STATE_SYN_SENT = 3,
- NES_AEQE_TCP_STATE_SYN_RCVD = 4,
- NES_AEQE_TCP_STATE_ESTABLISHED = 5,
- NES_AEQE_TCP_STATE_CLOSE_WAIT = 6,
- NES_AEQE_TCP_STATE_FIN_WAIT_1 = 7,
- NES_AEQE_TCP_STATE_CLOSING = 8,
- NES_AEQE_TCP_STATE_LAST_ACK = 9,
- NES_AEQE_TCP_STATE_FIN_WAIT_2 = 10,
- NES_AEQE_TCP_STATE_TIME_WAIT = 11
-};
-
-enum nes_aeqe_aeid {
- NES_AEQE_AEID_AMP_UNALLOCATED_STAG = 0x0102,
- NES_AEQE_AEID_AMP_INVALID_STAG = 0x0103,
- NES_AEQE_AEID_AMP_BAD_QP = 0x0104,
- NES_AEQE_AEID_AMP_BAD_PD = 0x0105,
- NES_AEQE_AEID_AMP_BAD_STAG_KEY = 0x0106,
- NES_AEQE_AEID_AMP_BAD_STAG_INDEX = 0x0107,
- NES_AEQE_AEID_AMP_BOUNDS_VIOLATION = 0x0108,
- NES_AEQE_AEID_AMP_RIGHTS_VIOLATION = 0x0109,
- NES_AEQE_AEID_AMP_TO_WRAP = 0x010a,
- NES_AEQE_AEID_AMP_FASTREG_SHARED = 0x010b,
- NES_AEQE_AEID_AMP_FASTREG_VALID_STAG = 0x010c,
- NES_AEQE_AEID_AMP_FASTREG_MW_STAG = 0x010d,
- NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS = 0x010e,
- NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW = 0x010f,
- NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH = 0x0110,
- NES_AEQE_AEID_AMP_INVALIDATE_SHARED = 0x0111,
- NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS = 0x0112,
- NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS = 0x0113,
- NES_AEQE_AEID_AMP_MWBIND_VALID_STAG = 0x0114,
- NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG = 0x0115,
- NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG = 0x0116,
- NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG = 0x0117,
- NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS = 0x0118,
- NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS = 0x0119,
- NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT = 0x011a,
- NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED = 0x011b,
- NES_AEQE_AEID_BAD_CLOSE = 0x0201,
- NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE = 0x0202,
- NES_AEQE_AEID_CQ_OPERATION_ERROR = 0x0203,
- NES_AEQE_AEID_PRIV_OPERATION_DENIED = 0x0204,
- NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO = 0x0205,
- NES_AEQE_AEID_STAG_ZERO_INVALID = 0x0206,
- NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN = 0x0301,
- NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID = 0x0302,
- NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER = 0x0303,
- NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION = 0x0304,
- NES_AEQE_AEID_DDP_UBE_INVALID_MO = 0x0305,
- NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE = 0x0306,
- NES_AEQE_AEID_DDP_UBE_INVALID_QN = 0x0307,
- NES_AEQE_AEID_DDP_NO_L_BIT = 0x0308,
- NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION = 0x0311,
- NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE = 0x0312,
- NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST = 0x0313,
- NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP = 0x0314,
- NES_AEQE_AEID_INVALID_ARP_ENTRY = 0x0401,
- NES_AEQE_AEID_INVALID_TCP_OPTION_RCVD = 0x0402,
- NES_AEQE_AEID_STALE_ARP_ENTRY = 0x0403,
- NES_AEQE_AEID_LLP_CLOSE_COMPLETE = 0x0501,
- NES_AEQE_AEID_LLP_CONNECTION_RESET = 0x0502,
- NES_AEQE_AEID_LLP_FIN_RECEIVED = 0x0503,
- NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH = 0x0504,
- NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR = 0x0505,
- NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE = 0x0506,
- NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL = 0x0507,
- NES_AEQE_AEID_LLP_SYN_RECEIVED = 0x0508,
- NES_AEQE_AEID_LLP_TERMINATE_RECEIVED = 0x0509,
- NES_AEQE_AEID_LLP_TOO_MANY_RETRIES = 0x050a,
- NES_AEQE_AEID_LLP_TOO_MANY_KEEPALIVE_RETRIES = 0x050b,
- NES_AEQE_AEID_RESET_SENT = 0x0601,
- NES_AEQE_AEID_TERMINATE_SENT = 0x0602,
- NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC = 0x0700
-};
-
-enum nes_iwarp_sq_opcodes {
- NES_IWARP_SQ_WQE_WRPDU = (1<<15),
- NES_IWARP_SQ_WQE_PSH = (1<<21),
- NES_IWARP_SQ_WQE_STREAMING = (1<<23),
- NES_IWARP_SQ_WQE_IMM_DATA = (1<<28),
- NES_IWARP_SQ_WQE_READ_FENCE = (1<<29),
- NES_IWARP_SQ_WQE_LOCAL_FENCE = (1<<30),
- NES_IWARP_SQ_WQE_SIGNALED_COMPL = (1<<31),
-};
-
-enum nes_iwarp_sq_wqe_bits {
- NES_IWARP_SQ_OP_RDMAW = 0,
- NES_IWARP_SQ_OP_RDMAR = 1,
- NES_IWARP_SQ_OP_SEND = 3,
- NES_IWARP_SQ_OP_SENDINV = 4,
- NES_IWARP_SQ_OP_SENDSE = 5,
- NES_IWARP_SQ_OP_SENDSEINV = 6,
- NES_IWARP_SQ_OP_BIND = 8,
- NES_IWARP_SQ_OP_FAST_REG = 9,
- NES_IWARP_SQ_OP_LOCINV = 10,
- NES_IWARP_SQ_OP_RDMAR_LOCINV = 11,
- NES_IWARP_SQ_OP_NOP = 12,
-};
-
-enum nes_iwarp_cqe_major_code {
- NES_IWARP_CQE_MAJOR_FLUSH = 1,
- NES_IWARP_CQE_MAJOR_DRV = 0x8000
-};
-
-enum nes_iwarp_cqe_minor_code {
- NES_IWARP_CQE_MINOR_FLUSH = 1
-};
-
-#define NES_EEPROM_READ_REQUEST (1<<16)
-#define NES_MAC_ADDR_VALID (1<<20)
-
-/*
- * NES index registers init values.
- */
-struct nes_init_values {
- u32 index;
- u32 data;
- u8 wrt;
-};
-
-/*
- * NES registers in BAR0.
- */
-struct nes_pci_regs {
- u32 int_status;
- u32 int_mask;
- u32 int_pending;
- u32 intf_int_status;
- u32 intf_int_mask;
- u32 other_regs[59]; /* pad out to 256 bytes for now */
-};
-
-#define NES_CQP_SQ_SIZE 128
-#define NES_CCQ_SIZE 128
-#define NES_NIC_WQ_SIZE 512
-#define NES_NIC_CTX_SIZE ((NES_NIC_CTX_RQ_SIZE_512) | (NES_NIC_CTX_SQ_SIZE_512))
-#define NES_NIC_BACK_STORE 0x00038000
-
-struct nes_device;
-
-struct nes_hw_nic_qp_context {
- __le32 context_words[6];
-};
-
-struct nes_hw_nic_sq_wqe {
- __le32 wqe_words[16];
-};
-
-struct nes_hw_nic_rq_wqe {
- __le32 wqe_words[16];
-};
-
-struct nes_hw_nic_cqe {
- __le32 cqe_words[4];
-};
-
-struct nes_hw_cqp_qp_context {
- __le32 context_words[4];
-};
-
-struct nes_hw_cqp_wqe {
- __le32 wqe_words[16];
-};
-
-struct nes_hw_qp_wqe {
- __le32 wqe_words[32];
-};
-
-struct nes_hw_cqe {
- __le32 cqe_words[8];
-};
-
-struct nes_hw_ceqe {
- __le32 ceqe_words[2];
-};
-
-struct nes_hw_aeqe {
- __le32 aeqe_words[4];
-};
-
-struct nes_cqp_request {
- union {
- u64 cqp_callback_context;
- void *cqp_callback_pointer;
- };
- wait_queue_head_t waitq;
- struct nes_hw_cqp_wqe cqp_wqe;
- struct list_head list;
- atomic_t refcount;
- void (*cqp_callback)(struct nes_device *nesdev, struct nes_cqp_request *cqp_request);
- u16 major_code;
- u16 minor_code;
- u8 waiting;
- u8 request_done;
- u8 dynamic;
- u8 callback;
-};
-
-struct nes_hw_cqp {
- struct nes_hw_cqp_wqe *sq_vbase;
- dma_addr_t sq_pbase;
- spinlock_t lock;
- wait_queue_head_t waitq;
- u16 qp_id;
- u16 sq_head;
- u16 sq_tail;
- u16 sq_size;
-};
-
-#define NES_FIRST_FRAG_SIZE 128
-struct nes_first_frag {
- u8 buffer[NES_FIRST_FRAG_SIZE];
-};
-
-struct nes_hw_nic {
- struct nes_first_frag *first_frag_vbase; /* virtual address of first frags */
- struct nes_hw_nic_sq_wqe *sq_vbase; /* virtual address of sq */
- struct nes_hw_nic_rq_wqe *rq_vbase; /* virtual address of rq */
- struct sk_buff *tx_skb[NES_NIC_WQ_SIZE];
- struct sk_buff *rx_skb[NES_NIC_WQ_SIZE];
- dma_addr_t frag_paddr[NES_NIC_WQ_SIZE];
- unsigned long first_frag_overflow[BITS_TO_LONGS(NES_NIC_WQ_SIZE)];
- dma_addr_t sq_pbase; /* PCI memory for host rings */
- dma_addr_t rq_pbase; /* PCI memory for host rings */
-
- u16 qp_id;
- u16 sq_head;
- u16 sq_tail;
- u16 sq_size;
- u16 rq_head;
- u16 rq_tail;
- u16 rq_size;
- u8 replenishing_rq;
- u8 reserved;
-
- spinlock_t rq_lock;
-};
-
-struct nes_hw_nic_cq {
- struct nes_hw_nic_cqe volatile *cq_vbase; /* PCI memory for host rings */
- void (*ce_handler)(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
- dma_addr_t cq_pbase; /* PCI memory for host rings */
- int rx_cqes_completed;
- int cqe_allocs_pending;
- int rx_pkts_indicated;
- u16 cq_head;
- u16 cq_size;
- u16 cq_number;
- u8 cqes_pending;
-};
-
-struct nes_hw_qp {
- struct nes_hw_qp_wqe *sq_vbase; /* PCI memory for host rings */
- struct nes_hw_qp_wqe *rq_vbase; /* PCI memory for host rings */
- void *q2_vbase; /* PCI memory for host rings */
- dma_addr_t sq_pbase; /* PCI memory for host rings */
- dma_addr_t rq_pbase; /* PCI memory for host rings */
- dma_addr_t q2_pbase; /* PCI memory for host rings */
- u32 qp_id;
- u16 sq_head;
- u16 sq_tail;
- u16 sq_size;
- u16 rq_head;
- u16 rq_tail;
- u16 rq_size;
- u8 rq_encoded_size;
- u8 sq_encoded_size;
-};
-
-struct nes_hw_cq {
- struct nes_hw_cqe *cq_vbase; /* PCI memory for host rings */
- void (*ce_handler)(struct nes_device *nesdev, struct nes_hw_cq *cq);
- dma_addr_t cq_pbase; /* PCI memory for host rings */
- u16 cq_head;
- u16 cq_size;
- u16 cq_number;
-};
-
-struct nes_hw_ceq {
- struct nes_hw_ceqe volatile *ceq_vbase; /* PCI memory for host rings */
- dma_addr_t ceq_pbase; /* PCI memory for host rings */
- u16 ceq_head;
- u16 ceq_size;
-};
-
-struct nes_hw_aeq {
- struct nes_hw_aeqe volatile *aeq_vbase; /* PCI memory for host rings */
- dma_addr_t aeq_pbase; /* PCI memory for host rings */
- u16 aeq_head;
- u16 aeq_size;
-};
-
-struct nic_qp_map {
- u8 qpid;
- u8 nic_index;
- u8 logical_port;
- u8 is_hnic;
-};
-
-#define NES_CQP_ARP_AEQ_INDEX_MASK 0x000f0000
-#define NES_CQP_ARP_AEQ_INDEX_SHIFT 16
-
-#define NES_CQP_APBVT_ADD 0x00008000
-#define NES_CQP_APBVT_NIC_SHIFT 16
-
-#define NES_ARP_ADD 1
-#define NES_ARP_DELETE 2
-#define NES_ARP_RESOLVE 3
-
-#define NES_MAC_SW_IDLE 0
-#define NES_MAC_SW_INTERRUPT 1
-#define NES_MAC_SW_MH 2
-
-struct nes_arp_entry {
- u32 ip_addr;
- u8 mac_addr[ETH_ALEN];
-};
-
-#define NES_NIC_FAST_TIMER 96
-#define NES_NIC_FAST_TIMER_LOW 40
-#define NES_NIC_FAST_TIMER_HIGH 1000
-#define DEFAULT_NES_QL_HIGH 256
-#define DEFAULT_NES_QL_LOW 16
-#define DEFAULT_NES_QL_TARGET 64
-#define DEFAULT_JUMBO_NES_QL_LOW 12
-#define DEFAULT_JUMBO_NES_QL_TARGET 40
-#define DEFAULT_JUMBO_NES_QL_HIGH 128
-#define NES_NIC_CQ_DOWNWARD_TREND 16
-#define NES_PFT_SIZE 48
-
-#define NES_MGT_WQ_COUNT 32
-#define NES_MGT_CTX_SIZE ((NES_NIC_CTX_RQ_SIZE_32) | (NES_NIC_CTX_SQ_SIZE_32))
-#define NES_MGT_QP_OFFSET 36
-#define NES_MGT_QP_COUNT 4
-
-struct nes_hw_tune_timer {
- /* u16 cq_count; */
- u16 threshold_low;
- u16 threshold_target;
- u16 threshold_high;
- u16 timer_in_use;
- u16 timer_in_use_old;
- u16 timer_in_use_min;
- u16 timer_in_use_max;
- u8 timer_direction_upward;
- u8 timer_direction_downward;
- u16 cq_count_old;
- u8 cq_direction_downward;
-};
-
-#define NES_TIMER_INT_LIMIT 2
-#define NES_TIMER_INT_LIMIT_DYNAMIC 10
-#define NES_TIMER_ENABLE_LIMIT 4
-#define NES_MAX_LINK_INTERRUPTS 128
-#define NES_MAX_LINK_CHECK 200
-
-struct nes_adapter {
- u64 fw_ver;
- unsigned long *allocated_qps;
- unsigned long *allocated_cqs;
- unsigned long *allocated_mrs;
- unsigned long *allocated_pds;
- unsigned long *allocated_arps;
- struct nes_qp **qp_table;
- struct workqueue_struct *work_q;
-
- struct list_head list;
- struct list_head active_listeners;
- /* list of the netdev's associated with each logical port */
- struct list_head nesvnic_list[4];
-
- struct timer_list mh_timer;
- struct timer_list lc_timer;
- struct work_struct work;
- spinlock_t resource_lock;
- spinlock_t phy_lock;
- spinlock_t pbl_lock;
- spinlock_t periodic_timer_lock;
-
- struct nes_arp_entry arp_table[NES_MAX_ARP_TABLE_SIZE];
-
- /* Adapter CEQ and AEQs */
- struct nes_hw_ceq ceq[16];
- struct nes_hw_aeq aeq[8];
-
- struct nes_hw_tune_timer tune_timer;
-
- unsigned long doorbell_start;
-
- u32 hw_rev;
- u32 vendor_id;
- u32 vendor_part_id;
- u32 device_cap_flags;
- u32 tick_delta;
- u32 timer_int_req;
- u32 arp_table_size;
- u32 next_arp_index;
-
- u32 max_mr;
- u32 max_256pbl;
- u32 max_4kpbl;
- u32 free_256pbl;
- u32 free_4kpbl;
- u32 max_mr_size;
- u32 max_qp;
- u32 next_qp;
- u32 max_irrq;
- u32 max_qp_wr;
- u32 max_sge;
- u32 max_cq;
- u32 next_cq;
- u32 max_cqe;
- u32 max_pd;
- u32 base_pd;
- u32 next_pd;
- u32 hte_index_mask;
-
- /* EEPROM information */
- u32 rx_pool_size;
- u32 tx_pool_size;
- u32 rx_threshold;
- u32 tcp_timer_core_clk_divisor;
- u32 iwarp_config;
- u32 cm_config;
- u32 sws_timer_config;
- u32 tcp_config1;
- u32 wqm_wat;
- u32 core_clock;
- u32 firmware_version;
- u32 eeprom_version;
-
- u32 nic_rx_eth_route_err;
-
- u32 et_rx_coalesce_usecs;
- u32 et_rx_max_coalesced_frames;
- u32 et_rx_coalesce_usecs_irq;
- u32 et_rx_max_coalesced_frames_irq;
- u32 et_pkt_rate_low;
- u32 et_rx_coalesce_usecs_low;
- u32 et_rx_max_coalesced_frames_low;
- u32 et_pkt_rate_high;
- u32 et_rx_coalesce_usecs_high;
- u32 et_rx_max_coalesced_frames_high;
- u32 et_rate_sample_interval;
- u32 timer_int_limit;
- u32 wqm_quanta;
- u8 allow_unaligned_fpdus;
-
- /* Adapter base MAC address */
- u32 mac_addr_low;
- u16 mac_addr_high;
-
- u16 firmware_eeprom_offset;
- u16 software_eeprom_offset;
-
- u16 max_irrq_wr;
-
- /* pd config for each port */
- u16 pd_config_size[4];
- u16 pd_config_base[4];
-
- u16 link_interrupt_count[4];
- u8 crit_error_count[32];
-
- /* the phy index for each port */
- u8 phy_index[4];
- u8 mac_sw_state[4];
- u8 mac_link_down[4];
- u8 phy_type[4];
- u8 log_port;
-
- /* PCI information */
- struct nes_device *nesdev;
- unsigned int devfn;
- unsigned char bus_number;
- unsigned char OneG_Mode;
-
- unsigned char ref_count;
- u8 netdev_count;
- u8 netdev_max; /* from host nic address count in EEPROM */
- u8 port_count;
- u8 virtwq;
- u8 send_term_ok;
- u8 et_use_adaptive_rx_coalesce;
- u8 adapter_fcn_count;
- u8 pft_mcast_map[NES_PFT_SIZE];
-};
-
-struct nes_pbl {
- u64 *pbl_vbase;
- dma_addr_t pbl_pbase;
- struct page *page;
- unsigned long user_base;
- u32 pbl_size;
- struct list_head list;
- /* TODO: need to add list for two level tables */
-};
-
-#define NES_4K_PBL_CHUNK_SIZE 4096
-
-struct nes_fast_mr_wqe_pbl {
- u64 *kva;
- dma_addr_t paddr;
-};
-
-struct nes_listener {
- struct work_struct work;
- struct workqueue_struct *wq;
- struct nes_vnic *nesvnic;
- struct iw_cm_id *cm_id;
- struct list_head list;
- unsigned long socket;
- u8 accept_failed;
-};
-
-struct nes_ib_device;
-
-#define NES_EVENT_DELAY msecs_to_jiffies(100)
-
-struct nes_vnic {
- struct nes_ib_device *nesibdev;
- u64 sq_full;
- u64 tso_requests;
- u64 segmented_tso_requests;
- u64 linearized_skbs;
- u64 tx_sw_dropped;
- u64 endnode_nstat_rx_discard;
- u64 endnode_nstat_rx_octets;
- u64 endnode_nstat_rx_frames;
- u64 endnode_nstat_tx_octets;
- u64 endnode_nstat_tx_frames;
- u64 endnode_ipv4_tcp_retransmits;
- /* void *mem; */
- struct nes_device *nesdev;
- struct net_device *netdev;
- atomic_t rx_skbs_needed;
- atomic_t rx_skb_timer_running;
- int budget;
- u32 msg_enable;
- /* u32 tx_avail; */
- __be32 local_ipaddr;
- struct napi_struct napi;
- spinlock_t tx_lock; /* could use netdev tx lock? */
- struct timer_list rq_wqes_timer;
- u32 nic_mem_size;
- void *nic_vbase;
- dma_addr_t nic_pbase;
- struct nes_hw_nic nic;
- struct nes_hw_nic_cq nic_cq;
- u32 mcrq_qp_id;
- struct nes_ucontext *mcrq_ucontext;
- struct nes_cqp_request* (*get_cqp_request)(struct nes_device *nesdev);
- void (*post_cqp_request)(struct nes_device*, struct nes_cqp_request *);
- int (*mcrq_mcast_filter)( struct nes_vnic* nesvnic, __u8* dmi_addr );
- struct net_device_stats netstats;
- /* used to put the netdev on the adapters logical port list */
- struct list_head list;
- u16 max_frame_size;
- u8 netdev_open;
- u8 linkup;
- u8 logical_port;
- u8 netdev_index; /* might not be needed, indexes nesdev->netdev */
- u8 perfect_filter_index;
- u8 nic_index;
- u8 qp_nic_index[4];
- u8 next_qp_nic_index;
- u8 of_device_registered;
- u8 rdma_enabled;
- struct timer_list event_timer;
- enum ib_event_type delayed_event;
- enum ib_event_type last_dispatched_event;
- spinlock_t port_ibevent_lock;
- u32 mgt_mem_size;
- void *mgt_vbase;
- dma_addr_t mgt_pbase;
- struct nes_vnic_mgt *mgtvnic[NES_MGT_QP_COUNT];
- struct task_struct *mgt_thread;
- wait_queue_head_t mgt_wait_queue;
- struct sk_buff_head mgt_skb_list;
-
-};
-
-struct nes_ib_device {
- struct ib_device ibdev;
- struct nes_vnic *nesvnic;
-
- /* Virtual RNIC Limits */
- u32 max_mr;
- u32 max_qp;
- u32 max_cq;
- u32 max_pd;
- u32 num_mr;
- u32 num_qp;
- u32 num_cq;
- u32 num_pd;
-};
-
-enum nes_hdrct_flags {
- DDP_LEN_FLAG = 0x80,
- DDP_HDR_FLAG = 0x40,
- RDMA_HDR_FLAG = 0x20
-};
-
-enum nes_term_layers {
- LAYER_RDMA = 0,
- LAYER_DDP = 1,
- LAYER_MPA = 2
-};
-
-enum nes_term_error_types {
- RDMAP_CATASTROPHIC = 0,
- RDMAP_REMOTE_PROT = 1,
- RDMAP_REMOTE_OP = 2,
- DDP_CATASTROPHIC = 0,
- DDP_TAGGED_BUFFER = 1,
- DDP_UNTAGGED_BUFFER = 2,
- DDP_LLP = 3
-};
-
-enum nes_term_rdma_errors {
- RDMAP_INV_STAG = 0x00,
- RDMAP_INV_BOUNDS = 0x01,
- RDMAP_ACCESS = 0x02,
- RDMAP_UNASSOC_STAG = 0x03,
- RDMAP_TO_WRAP = 0x04,
- RDMAP_INV_RDMAP_VER = 0x05,
- RDMAP_UNEXPECTED_OP = 0x06,
- RDMAP_CATASTROPHIC_LOCAL = 0x07,
- RDMAP_CATASTROPHIC_GLOBAL = 0x08,
- RDMAP_CANT_INV_STAG = 0x09,
- RDMAP_UNSPECIFIED = 0xff
-};
-
-enum nes_term_ddp_errors {
- DDP_CATASTROPHIC_LOCAL = 0x00,
- DDP_TAGGED_INV_STAG = 0x00,
- DDP_TAGGED_BOUNDS = 0x01,
- DDP_TAGGED_UNASSOC_STAG = 0x02,
- DDP_TAGGED_TO_WRAP = 0x03,
- DDP_TAGGED_INV_DDP_VER = 0x04,
- DDP_UNTAGGED_INV_QN = 0x01,
- DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
- DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
- DDP_UNTAGGED_INV_MO = 0x04,
- DDP_UNTAGGED_INV_TOO_LONG = 0x05,
- DDP_UNTAGGED_INV_DDP_VER = 0x06
-};
-
-enum nes_term_mpa_errors {
- MPA_CLOSED = 0x01,
- MPA_CRC = 0x02,
- MPA_MARKER = 0x03,
- MPA_REQ_RSP = 0x04,
-};
-
-struct nes_terminate_hdr {
- u8 layer_etype;
- u8 error_code;
- u8 hdrct;
- u8 rsvd;
-};
-
-/* Used to determine how to fill in terminate error codes */
-#define IWARP_OPCODE_WRITE 0
-#define IWARP_OPCODE_READREQ 1
-#define IWARP_OPCODE_READRSP 2
-#define IWARP_OPCODE_SEND 3
-#define IWARP_OPCODE_SEND_INV 4
-#define IWARP_OPCODE_SEND_SE 5
-#define IWARP_OPCODE_SEND_SE_INV 6
-#define IWARP_OPCODE_TERM 7
-
-/* These values are used only during terminate processing */
-#define TERM_DDP_LEN_TAGGED 14
-#define TERM_DDP_LEN_UNTAGGED 18
-#define TERM_RDMA_LEN 28
-#define RDMA_OPCODE_MASK 0x0f
-#define RDMA_READ_REQ_OPCODE 1
-#define BAD_FRAME_OFFSET 64
-#define CQE_MAJOR_DRV 0x8000
-
-/* Used for link status recheck after interrupt processing */
-#define NES_LINK_RECHECK_DELAY msecs_to_jiffies(50)
-#define NES_LINK_RECHECK_MAX 60
-
-#endif /* __NES_HW_H */
diff --git a/drivers/infiniband/hw/nes/nes_mgt.c b/drivers/infiniband/hw/nes/nes_mgt.c
deleted file mode 100644
index cc4dce5c3e5f..000000000000
--- a/drivers/infiniband/hw/nes/nes_mgt.c
+++ /dev/null
@@ -1,1155 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/skbuff.h>
-#include <linux/etherdevice.h>
-#include <linux/kthread.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <net/tcp.h>
-#include "nes.h"
-#include "nes_mgt.h"
-
-atomic_t pau_qps_created;
-atomic_t pau_qps_destroyed;
-
-static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic)
-{
- unsigned long flags;
- dma_addr_t bus_address;
- struct sk_buff *skb;
- struct nes_hw_nic_rq_wqe *nic_rqe;
- struct nes_hw_mgt *nesmgt;
- struct nes_device *nesdev;
- struct nes_rskb_cb *cb;
- u32 rx_wqes_posted = 0;
-
- nesmgt = &mgtvnic->mgt;
- nesdev = mgtvnic->nesvnic->nesdev;
- spin_lock_irqsave(&nesmgt->rq_lock, flags);
- if (nesmgt->replenishing_rq != 0) {
- if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) &&
- (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) {
- atomic_set(&mgtvnic->rx_skb_timer_running, 1);
- spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
- mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */
- add_timer(&mgtvnic->rq_wqes_timer);
- } else {
- spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
- }
- return;
- }
- nesmgt->replenishing_rq = 1;
- spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
- do {
- skb = dev_alloc_skb(mgtvnic->nesvnic->max_frame_size);
- if (skb) {
- skb->dev = mgtvnic->nesvnic->netdev;
-
- bus_address = pci_map_single(nesdev->pcidev,
- skb->data, mgtvnic->nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->busaddr = bus_address;
- cb->maplen = mgtvnic->nesvnic->max_frame_size;
-
- nic_rqe = &nesmgt->rq_vbase[mgtvnic->mgt.rq_head];
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
- cpu_to_le32(mgtvnic->nesvnic->max_frame_size);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
- cpu_to_le32((u32)bus_address);
- nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
- cpu_to_le32((u32)((u64)bus_address >> 32));
- nesmgt->rx_skb[nesmgt->rq_head] = skb;
- nesmgt->rq_head++;
- nesmgt->rq_head &= nesmgt->rq_size - 1;
- atomic_dec(&mgtvnic->rx_skbs_needed);
- barrier();
- if (++rx_wqes_posted == 255) {
- nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id);
- rx_wqes_posted = 0;
- }
- } else {
- spin_lock_irqsave(&nesmgt->rq_lock, flags);
- if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) &&
- (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) {
- atomic_set(&mgtvnic->rx_skb_timer_running, 1);
- spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
- mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */
- add_timer(&mgtvnic->rq_wqes_timer);
- } else {
- spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
- }
- break;
- }
- } while (atomic_read(&mgtvnic->rx_skbs_needed));
- barrier();
- if (rx_wqes_posted)
- nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id);
- nesmgt->replenishing_rq = 0;
-}
-
-/**
- * nes_mgt_rq_wqes_timeout
- */
-static void nes_mgt_rq_wqes_timeout(struct timer_list *t)
-{
- struct nes_vnic_mgt *mgtvnic = from_timer(mgtvnic, t,
- rq_wqes_timer);
-
- atomic_set(&mgtvnic->rx_skb_timer_running, 0);
- if (atomic_read(&mgtvnic->rx_skbs_needed))
- nes_replenish_mgt_rq(mgtvnic);
-}
-
-/**
- * nes_mgt_free_skb - unmap and free skb
- */
-static void nes_mgt_free_skb(struct nes_device *nesdev, struct sk_buff *skb, u32 dir)
-{
- struct nes_rskb_cb *cb;
-
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, dir);
- cb->busaddr = 0;
- dev_kfree_skb_any(skb);
-}
-
-/**
- * nes_download_callback - handle download completions
- */
-static void nes_download_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request)
-{
- struct pau_fpdu_info *fpdu_info = cqp_request->cqp_callback_pointer;
- struct nes_qp *nesqp = fpdu_info->nesqp;
- struct sk_buff *skb;
- int i;
-
- for (i = 0; i < fpdu_info->frag_cnt; i++) {
- skb = fpdu_info->frags[i].skb;
- if (fpdu_info->frags[i].cmplt) {
- nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE);
- nes_rem_ref_cm_node(nesqp->cm_node);
- }
- }
-
- if (fpdu_info->hdr_vbase)
- pci_free_consistent(nesdev->pcidev, fpdu_info->hdr_len,
- fpdu_info->hdr_vbase, fpdu_info->hdr_pbase);
- kfree(fpdu_info);
-}
-
-/**
- * nes_get_seq - Get the seq, ack_seq and window from the packet
- */
-static u32 nes_get_seq(struct sk_buff *skb, u32 *ack, u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd)
-{
- struct nes_rskb_cb *cb = (struct nes_rskb_cb *)&skb->cb[0];
- struct iphdr *iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
- struct tcphdr *tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
-
- *ack = be32_to_cpu(tcph->ack_seq);
- *wnd = be16_to_cpu(tcph->window);
- *fin_rcvd = tcph->fin;
- *rst_rcvd = tcph->rst;
- return be32_to_cpu(tcph->seq);
-}
-
-/**
- * nes_get_next_skb - Get the next skb based on where current skb is in the queue
- */
-static struct sk_buff *nes_get_next_skb(struct nes_device *nesdev, struct nes_qp *nesqp,
- struct sk_buff *skb, u32 nextseq, u32 *ack,
- u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd)
-{
- u32 seq;
- bool processacks;
- struct sk_buff *old_skb;
-
- if (skb) {
- /* Continue processing fpdu */
- skb = skb_peek_next(skb, &nesqp->pau_list);
- if (!skb)
- goto out;
- processacks = false;
- } else {
- /* Starting a new one */
- if (skb_queue_empty(&nesqp->pau_list))
- goto out;
- skb = skb_peek(&nesqp->pau_list);
- processacks = true;
- }
-
- while (1) {
- if (skb_queue_empty(&nesqp->pau_list))
- goto out;
-
- seq = nes_get_seq(skb, ack, wnd, fin_rcvd, rst_rcvd);
- if (seq == nextseq) {
- if (skb->len || processacks)
- break;
- } else if (after(seq, nextseq)) {
- goto out;
- }
-
- old_skb = skb;
- skb = skb_peek_next(skb, &nesqp->pau_list);
- skb_unlink(old_skb, &nesqp->pau_list);
- nes_mgt_free_skb(nesdev, old_skb, PCI_DMA_TODEVICE);
- nes_rem_ref_cm_node(nesqp->cm_node);
- if (!skb)
- goto out;
- }
- return skb;
-
-out:
- return NULL;
-}
-
-/**
- * get_fpdu_info - Find the next complete fpdu and return its fragments.
- */
-static int get_fpdu_info(struct nes_device *nesdev, struct nes_qp *nesqp,
- struct pau_fpdu_info **pau_fpdu_info)
-{
- struct sk_buff *skb;
- struct iphdr *iph;
- struct tcphdr *tcph;
- struct nes_rskb_cb *cb;
- struct pau_fpdu_info *fpdu_info = NULL;
- struct pau_fpdu_frag frags[MAX_FPDU_FRAGS];
- u32 fpdu_len = 0;
- u32 tmp_len;
- int frag_cnt = 0;
- u32 tot_len;
- u32 frag_tot;
- u32 ack;
- u32 fin_rcvd;
- u32 rst_rcvd;
- u16 wnd;
- int i;
- int rc = 0;
-
- *pau_fpdu_info = NULL;
-
- skb = nes_get_next_skb(nesdev, nesqp, NULL, nesqp->pau_rcv_nxt, &ack, &wnd, &fin_rcvd, &rst_rcvd);
- if (!skb)
- goto out;
-
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- if (skb->len) {
- fpdu_len = be16_to_cpu(*(__be16 *) skb->data) + MPA_FRAMING;
- fpdu_len = (fpdu_len + 3) & 0xfffffffc;
- tmp_len = fpdu_len;
-
- /* See if we have all of the fpdu */
- frag_tot = 0;
- memset(&frags, 0, sizeof frags);
- for (i = 0; i < MAX_FPDU_FRAGS; i++) {
- frags[i].physaddr = cb->busaddr;
- frags[i].physaddr += skb->data - cb->data_start;
- frags[i].frag_len = min(tmp_len, skb->len);
- frags[i].skb = skb;
- frags[i].cmplt = (skb->len == frags[i].frag_len);
- frag_tot += frags[i].frag_len;
- frag_cnt++;
-
- tmp_len -= frags[i].frag_len;
- if (tmp_len == 0)
- break;
-
- skb = nes_get_next_skb(nesdev, nesqp, skb,
- nesqp->pau_rcv_nxt + frag_tot, &ack, &wnd, &fin_rcvd, &rst_rcvd);
- if (!skb)
- goto out;
- if (rst_rcvd) {
- /* rst received in the middle of fpdu */
- for (; i >= 0; i--) {
- skb_unlink(frags[i].skb, &nesqp->pau_list);
- nes_mgt_free_skb(nesdev, frags[i].skb, PCI_DMA_TODEVICE);
- }
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- frags[0].physaddr = cb->busaddr;
- frags[0].physaddr += skb->data - cb->data_start;
- frags[0].frag_len = skb->len;
- frags[0].skb = skb;
- frags[0].cmplt = true;
- frag_cnt = 1;
- break;
- }
-
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- }
- } else {
- /* no data */
- frags[0].physaddr = cb->busaddr;
- frags[0].frag_len = 0;
- frags[0].skb = skb;
- frags[0].cmplt = true;
- frag_cnt = 1;
- }
-
- /* Found one */
- fpdu_info = kzalloc(sizeof(*fpdu_info), GFP_ATOMIC);
- if (!fpdu_info) {
- rc = -ENOMEM;
- goto out;
- }
-
- fpdu_info->cqp_request = nes_get_cqp_request(nesdev);
- if (fpdu_info->cqp_request == NULL) {
- nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
- rc = -ENOMEM;
- goto out;
- }
-
- cb = (struct nes_rskb_cb *)&frags[0].skb->cb[0];
- iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
- tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
- fpdu_info->hdr_len = (((unsigned char *)tcph) + 4 * (tcph->doff)) - cb->data_start;
- fpdu_info->data_len = fpdu_len;
- tot_len = fpdu_info->hdr_len + fpdu_len - ETH_HLEN;
-
- if (frags[0].cmplt) {
- fpdu_info->hdr_pbase = cb->busaddr;
- fpdu_info->hdr_vbase = NULL;
- } else {
- fpdu_info->hdr_vbase = pci_alloc_consistent(nesdev->pcidev,
- fpdu_info->hdr_len, &fpdu_info->hdr_pbase);
- if (!fpdu_info->hdr_vbase) {
- nes_debug(NES_DBG_PAU, "Unable to allocate memory for pau first frag\n");
- rc = -ENOMEM;
- goto out;
- }
-
- /* Copy hdrs, adjusting len and seqnum */
- memcpy(fpdu_info->hdr_vbase, cb->data_start, fpdu_info->hdr_len);
- iph = (struct iphdr *)(fpdu_info->hdr_vbase + ETH_HLEN);
- tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
- }
-
- iph->tot_len = cpu_to_be16(tot_len);
- iph->saddr = cpu_to_be32(0x7f000001);
-
- tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt);
- tcph->ack_seq = cpu_to_be32(ack);
- tcph->window = cpu_to_be16(wnd);
-
- nesqp->pau_rcv_nxt += fpdu_len + fin_rcvd;
-
- memcpy(fpdu_info->frags, frags, sizeof(fpdu_info->frags));
- fpdu_info->frag_cnt = frag_cnt;
- fpdu_info->nesqp = nesqp;
- *pau_fpdu_info = fpdu_info;
-
- /* Update skb's for next pass */
- for (i = 0; i < frag_cnt; i++) {
- cb = (struct nes_rskb_cb *)&frags[i].skb->cb[0];
- skb_pull(frags[i].skb, frags[i].frag_len);
-
- if (frags[i].skb->len == 0) {
- /* Pull skb off the list - it will be freed in the callback */
- if (!skb_queue_empty(&nesqp->pau_list))
- skb_unlink(frags[i].skb, &nesqp->pau_list);
- } else {
- /* Last skb still has data so update the seq */
- iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
- tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
- tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt);
- }
- }
-
-out:
- if (rc) {
- if (fpdu_info) {
- if (fpdu_info->cqp_request)
- nes_put_cqp_request(nesdev, fpdu_info->cqp_request);
- kfree(fpdu_info);
- }
- }
- return rc;
-}
-
-/**
- * forward_fpdu - send complete fpdus, one at a time
- */
-static int forward_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp)
-{
- struct nes_device *nesdev = nesvnic->nesdev;
- struct pau_fpdu_info *fpdu_info;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- unsigned long flags;
- u64 u64tmp;
- u32 u32tmp;
- int rc;
-
- while (1) {
- spin_lock_irqsave(&nesqp->pau_lock, flags);
- rc = get_fpdu_info(nesdev, nesqp, &fpdu_info);
- if (rc || (fpdu_info == NULL)) {
- spin_unlock_irqrestore(&nesqp->pau_lock, flags);
- return rc;
- }
-
- cqp_request = fpdu_info->cqp_request;
- cqp_wqe = &cqp_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_OPCODE_IDX,
- NES_CQP_DOWNLOAD_SEGMENT |
- (((u32)nesvnic->logical_port) << NES_CQP_OP_LOGICAL_PORT_SHIFT));
-
- u32tmp = fpdu_info->hdr_len << 16;
- u32tmp |= fpdu_info->hdr_len + (u32)fpdu_info->data_len;
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX,
- u32tmp);
-
- u32tmp = (fpdu_info->frags[1].frag_len << 16) | fpdu_info->frags[0].frag_len;
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_2_1_IDX,
- u32tmp);
-
- u32tmp = (fpdu_info->frags[3].frag_len << 16) | fpdu_info->frags[2].frag_len;
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_4_3_IDX,
- u32tmp);
-
- u64tmp = (u64)fpdu_info->hdr_pbase;
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX,
- lower_32_bits(u64tmp));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_HIGH_IDX,
- upper_32_bits(u64tmp));
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX,
- lower_32_bits(fpdu_info->frags[0].physaddr));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_HIGH_IDX,
- upper_32_bits(fpdu_info->frags[0].physaddr));
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_LOW_IDX,
- lower_32_bits(fpdu_info->frags[1].physaddr));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_HIGH_IDX,
- upper_32_bits(fpdu_info->frags[1].physaddr));
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_LOW_IDX,
- lower_32_bits(fpdu_info->frags[2].physaddr));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_HIGH_IDX,
- upper_32_bits(fpdu_info->frags[2].physaddr));
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_LOW_IDX,
- lower_32_bits(fpdu_info->frags[3].physaddr));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_HIGH_IDX,
- upper_32_bits(fpdu_info->frags[3].physaddr));
-
- cqp_request->cqp_callback_pointer = fpdu_info;
- cqp_request->callback = 1;
- cqp_request->cqp_callback = nes_download_callback;
-
- atomic_set(&cqp_request->refcount, 1);
- nes_post_cqp_request(nesdev, cqp_request);
- spin_unlock_irqrestore(&nesqp->pau_lock, flags);
- }
-
- return 0;
-}
-
-static void process_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp)
-{
- int again = 1;
- unsigned long flags;
-
- do {
- /* Ignore rc - if it failed, tcp retries will cause it to try again */
- forward_fpdus(nesvnic, nesqp);
-
- spin_lock_irqsave(&nesqp->pau_lock, flags);
- if (nesqp->pau_pending) {
- nesqp->pau_pending = 0;
- } else {
- nesqp->pau_busy = 0;
- again = 0;
- }
-
- spin_unlock_irqrestore(&nesqp->pau_lock, flags);
- } while (again);
-}
-
-/**
- * queue_fpdus - Handle fpdu's that hw passed up to sw
- */
-static void queue_fpdus(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp)
-{
- struct sk_buff *tmpskb;
- struct nes_rskb_cb *cb;
- struct iphdr *iph;
- struct tcphdr *tcph;
- unsigned char *tcph_end;
- u32 rcv_nxt;
- u32 rcv_wnd;
- u32 seqnum;
- u32 len;
- bool process_it = false;
- unsigned long flags;
-
- /* Move data ptr to after tcp header */
- iph = (struct iphdr *)skb->data;
- tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
- seqnum = be32_to_cpu(tcph->seq);
- tcph_end = (((char *)tcph) + (4 * tcph->doff));
-
- len = be16_to_cpu(iph->tot_len);
- if (skb->len > len)
- skb_trim(skb, len);
- skb_pull(skb, tcph_end - skb->data);
-
- /* Initialize tracking values */
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->seqnum = seqnum;
-
- /* Make sure data is in the receive window */
- rcv_nxt = nesqp->pau_rcv_nxt;
- rcv_wnd = le32_to_cpu(nesqp->nesqp_context->rcv_wnd);
- if (!between(seqnum, rcv_nxt, (rcv_nxt + rcv_wnd))) {
- nes_mgt_free_skb(nesvnic->nesdev, skb, PCI_DMA_TODEVICE);
- nes_rem_ref_cm_node(nesqp->cm_node);
- return;
- }
-
- spin_lock_irqsave(&nesqp->pau_lock, flags);
-
- if (nesqp->pau_busy)
- nesqp->pau_pending = 1;
- else
- nesqp->pau_busy = 1;
-
- /* Queue skb by sequence number */
- if (skb_queue_len(&nesqp->pau_list) == 0) {
- __skb_queue_head(&nesqp->pau_list, skb);
- } else {
- skb_queue_walk(&nesqp->pau_list, tmpskb) {
- cb = (struct nes_rskb_cb *)&tmpskb->cb[0];
- if (before(seqnum, cb->seqnum))
- break;
- }
- __skb_insert(skb, tmpskb->prev, tmpskb, &nesqp->pau_list);
- }
- if (nesqp->pau_state == PAU_READY)
- process_it = true;
- spin_unlock_irqrestore(&nesqp->pau_lock, flags);
-
- if (process_it)
- process_fpdus(nesvnic, nesqp);
-
- return;
-}
-
-/**
- * mgt_thread - Handle mgt skbs in a safe context
- */
-static int mgt_thread(void *context)
-{
- struct nes_vnic *nesvnic = context;
- struct sk_buff *skb;
- struct nes_rskb_cb *cb;
-
- while (!kthread_should_stop()) {
- wait_event_interruptible(nesvnic->mgt_wait_queue,
- skb_queue_len(&nesvnic->mgt_skb_list) || kthread_should_stop());
- while ((skb_queue_len(&nesvnic->mgt_skb_list)) && !kthread_should_stop()) {
- skb = skb_dequeue(&nesvnic->mgt_skb_list);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->data_start = skb->data - ETH_HLEN;
- cb->busaddr = pci_map_single(nesvnic->nesdev->pcidev, cb->data_start,
- nesvnic->max_frame_size, PCI_DMA_TODEVICE);
- queue_fpdus(skb, nesvnic, cb->nesqp);
- }
- }
-
- /* Closing down so delete any entries on the queue */
- while (skb_queue_len(&nesvnic->mgt_skb_list)) {
- skb = skb_dequeue(&nesvnic->mgt_skb_list);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- nes_rem_ref_cm_node(cb->nesqp->cm_node);
- dev_kfree_skb_any(skb);
- }
- return 0;
-}
-
-/**
- * nes_queue_skbs - Queue skb so it can be handled in a thread context
- */
-void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp)
-{
- struct nes_rskb_cb *cb;
-
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->nesqp = nesqp;
- skb_queue_tail(&nesvnic->mgt_skb_list, skb);
- wake_up_interruptible(&nesvnic->mgt_wait_queue);
-}
-
-void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp)
-{
- struct sk_buff *skb;
- unsigned long flags;
- atomic_inc(&pau_qps_destroyed);
-
- /* Free packets that have not yet been forwarded */
- /* Lock is acquired by skb_dequeue when removing the skb */
- spin_lock_irqsave(&nesqp->pau_lock, flags);
- while (skb_queue_len(&nesqp->pau_list)) {
- skb = skb_dequeue(&nesqp->pau_list);
- nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE);
- nes_rem_ref_cm_node(nesqp->cm_node);
- }
- spin_unlock_irqrestore(&nesqp->pau_lock, flags);
-}
-
-static void nes_chg_qh_handler(struct nes_device *nesdev, struct nes_cqp_request *cqp_request)
-{
- struct pau_qh_chg *qh_chg = cqp_request->cqp_callback_pointer;
- struct nes_cqp_request *new_request;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_adapter *nesadapter;
- struct nes_qp *nesqp;
- struct nes_v4_quad nes_quad;
- u32 crc_value;
- u64 u64temp;
-
- nesadapter = nesdev->nesadapter;
- nesqp = qh_chg->nesqp;
-
- /* Should we handle the bad completion */
- if (cqp_request->major_code)
- WARN(1, PFX "Invalid cqp_request major_code=0x%x\n",
- cqp_request->major_code);
-
- switch (nesqp->pau_state) {
- case PAU_DEL_QH:
- /* Old hash code deleted, now set the new one */
- nesqp->pau_state = PAU_ADD_LB_QH;
- new_request = nes_get_cqp_request(nesdev);
- if (new_request == NULL) {
- nes_debug(NES_DBG_PAU, "Failed to get a new_request.\n");
- WARN_ON(1);
- return;
- }
-
- memset(&nes_quad, 0, sizeof(nes_quad));
- nes_quad.DstIpAdrIndex =
- cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
- nes_quad.SrcIpadr = cpu_to_be32(0x7f000001);
- nes_quad.TcpPorts[0] = swab16(nesqp->nesqp_context->tcpPorts[1]);
- nes_quad.TcpPorts[1] = swab16(nesqp->nesqp_context->tcpPorts[0]);
-
- /* Produce hash key */
- crc_value = get_crc_value(&nes_quad);
- nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff);
- nes_debug(NES_DBG_PAU, "new HTE Index = 0x%08X, CRC = 0x%08X\n",
- nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask);
-
- nesqp->hte_index &= nesadapter->hte_index_mask;
- nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index);
- nesqp->nesqp_context->ip0 = cpu_to_le32(0x7f000001);
- nesqp->nesqp_context->rcv_nxt = cpu_to_le32(nesqp->pau_rcv_nxt);
-
- cqp_wqe = &new_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words,
- NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH |
- NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
- u64temp = (u64)nesqp->nesqp_context_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
-
- nes_debug(NES_DBG_PAU, "Waiting for CQP completion for adding the quad hash.\n");
-
- new_request->cqp_callback_pointer = qh_chg;
- new_request->callback = 1;
- new_request->cqp_callback = nes_chg_qh_handler;
- atomic_set(&new_request->refcount, 1);
- nes_post_cqp_request(nesdev, new_request);
- break;
-
- case PAU_ADD_LB_QH:
- /* Start processing the queued fpdu's */
- nesqp->pau_state = PAU_READY;
- process_fpdus(qh_chg->nesvnic, qh_chg->nesqp);
- kfree(qh_chg);
- break;
- }
-}
-
-/**
- * nes_change_quad_hash
- */
-static int nes_change_quad_hash(struct nes_device *nesdev,
- struct nes_vnic *nesvnic, struct nes_qp *nesqp)
-{
- struct nes_cqp_request *cqp_request = NULL;
- struct pau_qh_chg *qh_chg = NULL;
- u64 u64temp;
- struct nes_hw_cqp_wqe *cqp_wqe;
- int ret = 0;
-
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
- ret = -ENOMEM;
- goto chg_qh_err;
- }
-
- qh_chg = kmalloc(sizeof *qh_chg, GFP_ATOMIC);
- if (!qh_chg) {
- ret = -ENOMEM;
- goto chg_qh_err;
- }
- qh_chg->nesdev = nesdev;
- qh_chg->nesvnic = nesvnic;
- qh_chg->nesqp = nesqp;
- nesqp->pau_state = PAU_DEL_QH;
-
- cqp_wqe = &cqp_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words,
- NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | NES_CQP_QP_DEL_HTE |
- NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
- u64temp = (u64)nesqp->nesqp_context_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
-
- nes_debug(NES_DBG_PAU, "Waiting for CQP completion for deleting the quad hash.\n");
-
- cqp_request->cqp_callback_pointer = qh_chg;
- cqp_request->callback = 1;
- cqp_request->cqp_callback = nes_chg_qh_handler;
- atomic_set(&cqp_request->refcount, 1);
- nes_post_cqp_request(nesdev, cqp_request);
-
- return ret;
-
-chg_qh_err:
- kfree(qh_chg);
- if (cqp_request)
- nes_put_cqp_request(nesdev, cqp_request);
- return ret;
-}
-
-/**
- * nes_mgt_ce_handler
- * This management code deals with any packed and unaligned (pau) fpdu's
- * that the hardware cannot handle.
- */
-static void nes_mgt_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
-{
- struct nes_vnic_mgt *mgtvnic = container_of(cq, struct nes_vnic_mgt, mgt_cq);
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 head;
- u32 cq_size;
- u32 cqe_count = 0;
- u32 cqe_misc;
- u32 qp_id = 0;
- u32 skbs_needed;
- unsigned long context;
- struct nes_qp *nesqp;
- struct sk_buff *rx_skb;
- struct nes_rskb_cb *cb;
-
- head = cq->cq_head;
- cq_size = cq->cq_size;
-
- while (1) {
- cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
- if (!(cqe_misc & NES_NIC_CQE_VALID))
- break;
-
- nesqp = NULL;
- if (cqe_misc & NES_NIC_CQE_ACCQP_VALID) {
- qp_id = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_ACCQP_ID_IDX]);
- qp_id &= 0x001fffff;
- if (qp_id < nesadapter->max_qp) {
- context = (unsigned long)nesadapter->qp_table[qp_id - NES_FIRST_QPN];
- nesqp = (struct nes_qp *)context;
- }
- }
-
- if (nesqp) {
- if (nesqp->pau_mode == false) {
- nesqp->pau_mode = true; /* First time for this qp */
- nesqp->pau_rcv_nxt = le32_to_cpu(
- cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]);
- skb_queue_head_init(&nesqp->pau_list);
- spin_lock_init(&nesqp->pau_lock);
- atomic_inc(&pau_qps_created);
- nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp);
- }
-
- rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail];
- rx_skb->len = 0;
- skb_put(rx_skb, cqe_misc & 0x0000ffff);
- rx_skb->protocol = eth_type_trans(rx_skb, mgtvnic->nesvnic->netdev);
- cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
- pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, PCI_DMA_FROMDEVICE);
- cb->busaddr = 0;
- mgtvnic->mgt.rq_tail++;
- mgtvnic->mgt.rq_tail &= mgtvnic->mgt.rq_size - 1;
-
- nes_add_ref_cm_node(nesqp->cm_node);
- nes_queue_mgt_skbs(rx_skb, mgtvnic->nesvnic, nesqp);
- } else {
- printk(KERN_ERR PFX "Invalid QP %d for packed/unaligned handling\n", qp_id);
- }
-
- cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
- cqe_count++;
- if (++head >= cq_size)
- head = 0;
-
- if (cqe_count == 255) {
- /* Replenish mgt CQ */
- nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (cqe_count << 16));
- nesdev->currcq_count += cqe_count;
- cqe_count = 0;
- }
-
- skbs_needed = atomic_inc_return(&mgtvnic->rx_skbs_needed);
- if (skbs_needed > (mgtvnic->mgt.rq_size >> 1))
- nes_replenish_mgt_rq(mgtvnic);
- }
-
- cq->cq_head = head;
- nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- cq->cq_number | (cqe_count << 16));
- nes_read32(nesdev->regs + NES_CQE_ALLOC);
- nesdev->currcq_count += cqe_count;
-}
-
-/**
- * nes_init_mgt_qp
- */
-int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic)
-{
- struct nes_vnic_mgt *mgtvnic;
- u32 counter;
- void *vmem;
- dma_addr_t pmem;
- struct nes_hw_cqp_wqe *cqp_wqe;
- u32 cqp_head;
- unsigned long flags;
- struct nes_hw_nic_qp_context *mgt_context;
- u64 u64temp;
- struct nes_hw_nic_rq_wqe *mgt_rqe;
- struct sk_buff *skb;
- u32 wqe_count;
- struct nes_rskb_cb *cb;
- u32 mgt_mem_size;
- void *mgt_vbase;
- dma_addr_t mgt_pbase;
- int i;
- int ret;
-
- /* Allocate space the all mgt QPs once */
- mgtvnic = kcalloc(NES_MGT_QP_COUNT, sizeof(struct nes_vnic_mgt),
- GFP_KERNEL);
- if (!mgtvnic)
- return -ENOMEM;
-
- /* Allocate fragment, RQ, and CQ; Reuse CEQ based on the PCI function */
- /* We are not sending from this NIC so sq is not allocated */
- mgt_mem_size = 256 +
- (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)) +
- (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_cqe)) +
- sizeof(struct nes_hw_nic_qp_context);
- mgt_mem_size = (mgt_mem_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
- mgt_vbase = pci_alloc_consistent(nesdev->pcidev, NES_MGT_QP_COUNT * mgt_mem_size, &mgt_pbase);
- if (!mgt_vbase) {
- kfree(mgtvnic);
- nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt host descriptor rings\n");
- return -ENOMEM;
- }
-
- nesvnic->mgt_mem_size = NES_MGT_QP_COUNT * mgt_mem_size;
- nesvnic->mgt_vbase = mgt_vbase;
- nesvnic->mgt_pbase = mgt_pbase;
-
- skb_queue_head_init(&nesvnic->mgt_skb_list);
- init_waitqueue_head(&nesvnic->mgt_wait_queue);
- nesvnic->mgt_thread = kthread_run(mgt_thread, nesvnic, "nes_mgt_thread");
-
- for (i = 0; i < NES_MGT_QP_COUNT; i++) {
- mgtvnic->nesvnic = nesvnic;
- mgtvnic->mgt.qp_id = nesdev->mac_index + NES_MGT_QP_OFFSET + i;
- memset(mgt_vbase, 0, mgt_mem_size);
- nes_debug(NES_DBG_INIT, "Allocated mgt QP structures at %p (phys = %016lX), size = %u.\n",
- mgt_vbase, (unsigned long)mgt_pbase, mgt_mem_size);
-
- vmem = (void *)(((unsigned long)mgt_vbase + (256 - 1)) &
- ~(unsigned long)(256 - 1));
- pmem = (dma_addr_t)(((unsigned long long)mgt_pbase + (256 - 1)) &
- ~(unsigned long long)(256 - 1));
-
- spin_lock_init(&mgtvnic->mgt.rq_lock);
-
- /* setup the RQ */
- mgtvnic->mgt.rq_vbase = vmem;
- mgtvnic->mgt.rq_pbase = pmem;
- mgtvnic->mgt.rq_head = 0;
- mgtvnic->mgt.rq_tail = 0;
- mgtvnic->mgt.rq_size = NES_MGT_WQ_COUNT;
-
- /* setup the CQ */
- vmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe));
- pmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe));
-
- mgtvnic->mgt_cq.cq_number = mgtvnic->mgt.qp_id;
- mgtvnic->mgt_cq.cq_vbase = vmem;
- mgtvnic->mgt_cq.cq_pbase = pmem;
- mgtvnic->mgt_cq.cq_head = 0;
- mgtvnic->mgt_cq.cq_size = NES_MGT_WQ_COUNT;
-
- mgtvnic->mgt_cq.ce_handler = nes_mgt_ce_handler;
-
- /* Send CreateCQ request to CQP */
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- cqp_head = nesdev->cqp.sq_head;
-
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
- NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
- ((u32)mgtvnic->mgt_cq.cq_size << 16));
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
- mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16));
- u64temp = (u64)mgtvnic->mgt_cq.cq_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
- u64temp = (unsigned long)&mgtvnic->mgt_cq;
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
- cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- /* Send CreateQP request to CQP */
- mgt_context = (void *)(&mgtvnic->mgt_cq.cq_vbase[mgtvnic->mgt_cq.cq_size]);
- mgt_context->context_words[NES_NIC_CTX_MISC_IDX] =
- cpu_to_le32((u32)NES_MGT_CTX_SIZE |
- ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
- nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
- nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
- nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
- if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0)
- mgt_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
-
- u64temp = (u64)mgtvnic->mgt.rq_pbase;
- mgt_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
- mgt_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
- u64temp = (u64)mgtvnic->mgt.rq_pbase;
- mgt_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
- mgt_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
- NES_CQP_QP_TYPE_NIC);
- cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(mgtvnic->mgt.qp_id);
- u64temp = (u64)mgtvnic->mgt_cq.cq_pbase +
- (mgtvnic->mgt_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
- nesdev->cqp.sq_head = cqp_head;
-
- barrier();
-
- /* Ring doorbell (2 WQEs) */
- nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- nes_debug(NES_DBG_INIT, "Waiting for create MGT QP%u to complete.\n",
- mgtvnic->mgt.qp_id);
-
- ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_INIT, "Create MGT QP%u completed, wait_event_timeout ret = %u.\n",
- mgtvnic->mgt.qp_id, ret);
- if (!ret) {
- nes_debug(NES_DBG_INIT, "MGT QP%u create timeout expired\n", mgtvnic->mgt.qp_id);
- if (i == 0) {
- pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase,
- nesvnic->mgt_pbase);
- kfree(mgtvnic);
- } else {
- nes_destroy_mgt(nesvnic);
- }
- return -EIO;
- }
-
- /* Populate the RQ */
- for (counter = 0; counter < (NES_MGT_WQ_COUNT - 1); counter++) {
- skb = dev_alloc_skb(nesvnic->max_frame_size);
- if (!skb) {
- nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
- return -ENOMEM;
- }
-
- skb->dev = netdev;
-
- pmem = pci_map_single(nesdev->pcidev, skb->data,
- nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
- cb = (struct nes_rskb_cb *)&skb->cb[0];
- cb->busaddr = pmem;
- cb->maplen = nesvnic->max_frame_size;
-
- mgt_rqe = &mgtvnic->mgt.rq_vbase[counter];
- mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32((u32)nesvnic->max_frame_size);
- mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
- mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
- mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
- mgtvnic->mgt.rx_skb[counter] = skb;
- }
-
- timer_setup(&mgtvnic->rq_wqes_timer, nes_mgt_rq_wqes_timeout,
- 0);
-
- wqe_count = NES_MGT_WQ_COUNT - 1;
- mgtvnic->mgt.rq_head = wqe_count;
- barrier();
- do {
- counter = min(wqe_count, ((u32)255));
- wqe_count -= counter;
- nes_write32(nesdev->regs + NES_WQE_ALLOC, (counter << 24) | mgtvnic->mgt.qp_id);
- } while (wqe_count);
-
- nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- mgtvnic->mgt_cq.cq_number);
- nes_read32(nesdev->regs + NES_CQE_ALLOC);
-
- mgt_vbase += mgt_mem_size;
- mgt_pbase += mgt_mem_size;
- nesvnic->mgtvnic[i] = mgtvnic++;
- }
- return 0;
-}
-
-
-void nes_destroy_mgt(struct nes_vnic *nesvnic)
-{
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_vnic_mgt *mgtvnic;
- struct nes_vnic_mgt *first_mgtvnic;
- unsigned long flags;
- struct nes_hw_cqp_wqe *cqp_wqe;
- u32 cqp_head;
- struct sk_buff *rx_skb;
- int i;
- int ret;
-
- kthread_stop(nesvnic->mgt_thread);
-
- /* Free remaining NIC receive buffers */
- first_mgtvnic = nesvnic->mgtvnic[0];
- for (i = 0; i < NES_MGT_QP_COUNT; i++) {
- mgtvnic = nesvnic->mgtvnic[i];
- if (mgtvnic == NULL)
- continue;
-
- while (mgtvnic->mgt.rq_head != mgtvnic->mgt.rq_tail) {
- rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail];
- nes_mgt_free_skb(nesdev, rx_skb, PCI_DMA_FROMDEVICE);
- mgtvnic->mgt.rq_tail++;
- mgtvnic->mgt.rq_tail &= (mgtvnic->mgt.rq_size - 1);
- }
-
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
-
- /* Destroy NIC QP */
- cqp_head = nesdev->cqp.sq_head;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- mgtvnic->mgt.qp_id);
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
-
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
-
- /* Destroy NIC CQ */
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- (NES_CQP_DESTROY_CQ | ((u32)mgtvnic->mgt_cq.cq_size << 16)));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- (mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16)));
-
- if (++cqp_head >= nesdev->cqp.sq_size)
- cqp_head = 0;
-
- nesdev->cqp.sq_head = cqp_head;
- barrier();
-
- /* Ring doorbell (2 WQEs) */
- nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
- " cqp.sq_tail=%u, cqp.sq_size=%u\n",
- cqp_head, nesdev->cqp.sq_head,
- nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
-
- ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
- NES_EVENT_TIMEOUT);
-
- nes_debug(NES_DBG_SHUTDOWN, "Destroy MGT QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
- " cqp.sq_head=%u, cqp.sq_tail=%u\n",
- ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
- if (!ret)
- nes_debug(NES_DBG_SHUTDOWN, "MGT QP%u destroy timeout expired\n",
- mgtvnic->mgt.qp_id);
-
- nesvnic->mgtvnic[i] = NULL;
- }
-
- if (nesvnic->mgt_vbase) {
- pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase,
- nesvnic->mgt_pbase);
- nesvnic->mgt_vbase = NULL;
- nesvnic->mgt_pbase = 0;
- }
-
- kfree(first_mgtvnic);
-}
diff --git a/drivers/infiniband/hw/nes/nes_mgt.h b/drivers/infiniband/hw/nes/nes_mgt.h
deleted file mode 100644
index 4f7f701c4a81..000000000000
--- a/drivers/infiniband/hw/nes/nes_mgt.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
-* Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenIB.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*/
-
-#ifndef __NES_MGT_H
-#define __NES_MGT_H
-
-#define MPA_FRAMING 6 /* length is 2 bytes, crc is 4 bytes */
-
-int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic);
-void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp);
-void nes_destroy_mgt(struct nes_vnic *nesvnic);
-void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp);
-
-struct nes_hw_mgt {
- struct nes_hw_nic_rq_wqe *rq_vbase; /* virtual address of rq */
- dma_addr_t rq_pbase; /* PCI memory for host rings */
- struct sk_buff *rx_skb[NES_NIC_WQ_SIZE];
- u16 qp_id;
- u16 sq_head;
- u16 rq_head;
- u16 rq_tail;
- u16 rq_size;
- u8 replenishing_rq;
- u8 reserved;
- spinlock_t rq_lock;
-};
-
-struct nes_vnic_mgt {
- struct nes_vnic *nesvnic;
- struct nes_hw_mgt mgt;
- struct nes_hw_nic_cq mgt_cq;
- atomic_t rx_skbs_needed;
- struct timer_list rq_wqes_timer;
- atomic_t rx_skb_timer_running;
-};
-
-#define MAX_FPDU_FRAGS 4
-struct pau_fpdu_frag {
- struct sk_buff *skb;
- u64 physaddr;
- u32 frag_len;
- bool cmplt;
-};
-
-struct pau_fpdu_info {
- struct nes_qp *nesqp;
- struct nes_cqp_request *cqp_request;
- void *hdr_vbase;
- dma_addr_t hdr_pbase;
- int hdr_len;
- u16 data_len;
- u16 frag_cnt;
- struct pau_fpdu_frag frags[MAX_FPDU_FRAGS];
-};
-
-enum pau_qh_state {
- PAU_DEL_QH,
- PAU_ADD_LB_QH,
- PAU_READY
-};
-
-struct pau_qh_chg {
- struct nes_device *nesdev;
- struct nes_vnic *nesvnic;
- struct nes_qp *nesqp;
-};
-
-#endif /* __NES_MGT_H */
diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c
deleted file mode 100644
index 16f33454c198..000000000000
--- a/drivers/infiniband/hw/nes/nes_nic.c
+++ /dev/null
@@ -1,1870 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/if_arp.h>
-#include <linux/if_vlan.h>
-#include <linux/ethtool.h>
-#include <linux/slab.h>
-#include <net/tcp.h>
-
-#include <net/inet_common.h>
-#include <linux/inet.h>
-
-#include "nes.h"
-
-static struct nic_qp_map nic_qp_mapping_0[] = {
- {16,0,0,1},{24,4,0,0},{28,8,0,0},{32,12,0,0},
- {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0},
- {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0},
- {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_1[] = {
- {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0},
- {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_2[] = {
- {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_3[] = {
- {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_4[] = {
- {28,8,0,0},{32,12,0,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_5[] = {
- {29,9,1,0},{33,13,1,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_6[] = {
- {30,10,2,0},{34,14,2,0}
-};
-
-static struct nic_qp_map nic_qp_mapping_7[] = {
- {31,11,3,0},{35,15,3,0}
-};
-
-static struct nic_qp_map *nic_qp_mapping_per_function[] = {
- nic_qp_mapping_0, nic_qp_mapping_1, nic_qp_mapping_2, nic_qp_mapping_3,
- nic_qp_mapping_4, nic_qp_mapping_5, nic_qp_mapping_6, nic_qp_mapping_7
-};
-
-static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK
- | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN;
-static int debug = -1;
-static int nics_per_function = 1;
-
-/**
- * nes_netdev_poll
- */
-static int nes_netdev_poll(struct napi_struct *napi, int budget)
-{
- struct nes_vnic *nesvnic = container_of(napi, struct nes_vnic, napi);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_nic_cq *nescq = &nesvnic->nic_cq;
-
- nesvnic->budget = budget;
- nescq->cqes_pending = 0;
- nescq->rx_cqes_completed = 0;
- nescq->cqe_allocs_pending = 0;
- nescq->rx_pkts_indicated = 0;
-
- nes_nic_ce_handler(nesdev, nescq);
-
- if (nescq->cqes_pending == 0) {
- napi_complete(napi);
- /* clear out completed cqes and arm */
- nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- nescq->cq_number | (nescq->cqe_allocs_pending << 16));
- nes_read32(nesdev->regs+NES_CQE_ALLOC);
- } else {
- /* clear out completed cqes but don't arm */
- nes_write32(nesdev->regs+NES_CQE_ALLOC,
- nescq->cq_number | (nescq->cqe_allocs_pending << 16));
- nes_debug(NES_DBG_NETDEV, "%s: exiting with work pending\n",
- nesvnic->netdev->name);
- }
- return nescq->rx_pkts_indicated;
-}
-
-
-/**
- * nes_netdev_open - Activate the network interface; ifconfig
- * ethx up.
- */
-static int nes_netdev_open(struct net_device *netdev)
-{
- u32 macaddr_low;
- u16 macaddr_high;
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- int ret;
- int i;
- struct nes_vnic *first_nesvnic = NULL;
- u32 nic_active_bit;
- u32 nic_active;
- struct list_head *list_pos, *list_temp;
- unsigned long flags;
-
- if (nesvnic->netdev_open == 1)
- return 0;
-
- if (netif_msg_ifup(nesvnic))
- printk(KERN_INFO PFX "%s: enabling interface\n", netdev->name);
-
- ret = nes_init_nic_qp(nesdev, netdev);
- if (ret) {
- return ret;
- }
-
- netif_carrier_off(netdev);
- netif_stop_queue(netdev);
-
- if ((!nesvnic->of_device_registered) && (nesvnic->rdma_enabled)) {
- nesvnic->nesibdev = nes_init_ofa_device(netdev);
- if (nesvnic->nesibdev == NULL) {
- printk(KERN_ERR PFX "%s: nesvnic->nesibdev alloc failed", netdev->name);
- } else {
- nesvnic->nesibdev->nesvnic = nesvnic;
- ret = nes_register_ofa_device(nesvnic->nesibdev);
- if (ret) {
- printk(KERN_ERR PFX "%s: Unable to register RDMA device, ret = %d\n",
- netdev->name, ret);
- }
- }
- }
- /* Set packet filters */
- nic_active_bit = 1 << nesvnic->nic_index;
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active);
-
- macaddr_high = ((u16)netdev->dev_addr[0]) << 8;
- macaddr_high += (u16)netdev->dev_addr[1];
-
- macaddr_low = ((u32)netdev->dev_addr[2]) << 24;
- macaddr_low += ((u32)netdev->dev_addr[3]) << 16;
- macaddr_low += ((u32)netdev->dev_addr[4]) << 8;
- macaddr_low += (u32)netdev->dev_addr[5];
-
- /* Program the various MAC regs */
- for (i = 0; i < NES_MAX_PORT_COUNT; i++) {
- if (nesvnic->qp_nic_index[i] == 0xf) {
- break;
- }
- nes_debug(NES_DBG_NETDEV, "i=%d, perfect filter table index= %d, PERF FILTER LOW"
- " (Addr:%08X) = %08X, HIGH = %08X.\n",
- i, nesvnic->qp_nic_index[i],
- NES_IDX_PERFECT_FILTER_LOW+
- (nesvnic->qp_nic_index[i] * 8),
- macaddr_low,
- (u32)macaddr_high | NES_MAC_ADDR_VALID |
- ((((u32)nesvnic->nic_index) << 16)));
- nes_write_indexed(nesdev,
- NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8),
- macaddr_low);
- nes_write_indexed(nesdev,
- NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8),
- (u32)macaddr_high | NES_MAC_ADDR_VALID |
- ((((u32)nesvnic->nic_index) << 16)));
- }
-
-
- nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
- nesvnic->nic_cq.cq_number);
- nes_read32(nesdev->regs+NES_CQE_ALLOC);
- list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) {
- first_nesvnic = container_of(list_pos, struct nes_vnic, list);
- if (first_nesvnic->netdev_open == 1)
- break;
- }
- if (first_nesvnic->netdev_open == 0) {
- nes_debug(NES_DBG_INIT, "Setting up MAC interrupt mask.\n");
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK + (0x200 * nesdev->mac_index),
- ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT |
- NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR));
- first_nesvnic = nesvnic;
- }
-
- if (first_nesvnic->linkup) {
- /* Enable network packets */
- nesvnic->linkup = 1;
- netif_start_queue(netdev);
- netif_carrier_on(netdev);
- }
-
- spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
- if (nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_SFP_D) {
- nesdev->link_recheck = 1;
- mod_delayed_work(system_wq, &nesdev->work,
- NES_LINK_RECHECK_DELAY);
- }
- spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
-
- spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags);
- if (nesvnic->of_device_registered) {
- nesdev->nesadapter->send_term_ok = 1;
- if (nesvnic->linkup == 1) {
- if (nesdev->iw_status == 0) {
- nesdev->iw_status = 1;
- nes_port_ibevent(nesvnic);
- }
- } else {
- nesdev->iw_status = 0;
- }
- }
- spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags);
-
- napi_enable(&nesvnic->napi);
- nesvnic->netdev_open = 1;
-
- return 0;
-}
-
-
-/**
- * nes_netdev_stop
- */
-static int nes_netdev_stop(struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- u32 nic_active_mask;
- u32 nic_active;
- struct nes_vnic *first_nesvnic = NULL;
- struct list_head *list_pos, *list_temp;
- unsigned long flags;
-
- nes_debug(NES_DBG_SHUTDOWN, "nesvnic=%p, nesdev=%p, netdev=%p %s\n",
- nesvnic, nesdev, netdev, netdev->name);
- if (nesvnic->netdev_open == 0)
- return 0;
-
- if (netif_msg_ifdown(nesvnic))
- printk(KERN_INFO PFX "%s: disabling interface\n", netdev->name);
- netif_carrier_off(netdev);
-
- /* Disable network packets */
- napi_disable(&nesvnic->napi);
- netif_stop_queue(netdev);
- list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) {
- first_nesvnic = container_of(list_pos, struct nes_vnic, list);
- if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic))
- break;
- }
-
- if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic) &&
- (PCI_FUNC(first_nesvnic->nesdev->pcidev->devfn) !=
- PCI_FUNC(nesvnic->nesdev->pcidev->devfn))) {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+
- (0x200*nesdev->mac_index), 0xffffffff);
- nes_write_indexed(first_nesvnic->nesdev,
- NES_IDX_MAC_INT_MASK+
- (0x200*first_nesvnic->nesdev->mac_index),
- ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT |
- NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR));
- } else {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+(0x200*nesdev->mac_index), 0xffffffff);
- }
-
- nic_active_mask = ~((u32)(1 << nesvnic->nic_index));
- nes_write_indexed(nesdev, NES_IDX_PERFECT_FILTER_HIGH+
- (nesvnic->perfect_filter_index*8), 0);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE);
- nic_active &= nic_active_mask;
- nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL);
- nic_active &= nic_active_mask;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE);
- nic_active &= nic_active_mask;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL);
- nic_active &= nic_active_mask;
- nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON);
- nic_active &= nic_active_mask;
- nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active);
-
- spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags);
- if (nesvnic->of_device_registered) {
- nesdev->nesadapter->send_term_ok = 0;
- nesdev->iw_status = 0;
- if (nesvnic->linkup == 1)
- nes_port_ibevent(nesvnic);
- }
- del_timer_sync(&nesvnic->event_timer);
- nesvnic->event_timer.function = NULL;
- spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags);
-
- nes_destroy_nic_qp(nesvnic);
-
- nesvnic->netdev_open = 0;
-
- return 0;
-}
-
-
-/**
- * nes_nic_send
- */
-static bool nes_nic_send(struct sk_buff *skb, struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_nic *nesnic = &nesvnic->nic;
- struct nes_hw_nic_sq_wqe *nic_sqe;
- struct tcphdr *tcph;
- __le16 *wqe_fragment_length;
- u32 wqe_misc;
- u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
- u16 skb_fragment_index;
- dma_addr_t bus_address;
-
- nic_sqe = &nesnic->sq_vbase[nesnic->sq_head];
- wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
-
- /* setup the VLAN tag if present */
- if (skb_vlan_tag_present(skb)) {
- nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n",
- netdev->name, skb_vlan_tag_get(skb));
- wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE;
- wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb);
- } else
- wqe_misc = 0;
-
- /* bump past the vlan tag */
- wqe_fragment_length++;
- /* wqe_fragment_address = (u64 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX]; */
- wqe_misc |= NES_NIC_SQ_WQE_COMPLETION;
-
- if (skb->ip_summed == CHECKSUM_PARTIAL) {
- if (skb_is_gso(skb)) {
- tcph = tcp_hdr(skb);
- /* nes_debug(NES_DBG_NIC_TX, "%s: TSO request... is_gso = %u seg size = %u\n",
- netdev->name, skb_is_gso(skb), skb_shinfo(skb)->gso_size); */
- wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE | (u16)skb_shinfo(skb)->gso_size;
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX,
- ((u32)tcph->doff) |
- (((u32)(((unsigned char *)tcph) - skb->data)) << 4));
- }
- } else { /* CHECKSUM_HW */
- wqe_misc |= NES_NIC_SQ_WQE_DISABLE_CHKSUM;
- }
-
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX,
- skb->len);
- memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer,
- skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE), skb_headlen(skb)));
- wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE),
- skb_headlen(skb)));
- wqe_fragment_length[1] = 0;
- if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) {
- if ((skb_shinfo(skb)->nr_frags + 1) > 4) {
- nes_debug(NES_DBG_NIC_TX, "%s: Packet with %u fragments not sent, skb_headlen=%u\n",
- netdev->name, skb_shinfo(skb)->nr_frags + 2, skb_headlen(skb));
- kfree_skb(skb);
- nesvnic->tx_sw_dropped++;
- return false;
- }
- set_bit(nesnic->sq_head, nesnic->first_frag_overflow);
- bus_address = pci_map_single(nesdev->pcidev, skb->data + NES_FIRST_FRAG_SIZE,
- skb_headlen(skb) - NES_FIRST_FRAG_SIZE, PCI_DMA_TODEVICE);
- wqe_fragment_length[wqe_fragment_index++] =
- cpu_to_le16(skb_headlen(skb) - NES_FIRST_FRAG_SIZE);
- wqe_fragment_length[wqe_fragment_index] = 0;
- set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX,
- ((u64)(bus_address)));
- nesnic->tx_skb[nesnic->sq_head] = skb;
- }
-
- if (skb_headlen(skb) == skb->len) {
- if (skb_headlen(skb) <= NES_FIRST_FRAG_SIZE) {
- nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_2_1_IDX] = 0;
- nesnic->tx_skb[nesnic->sq_head] = skb;
- }
- } else {
- /* Deal with Fragments */
- nesnic->tx_skb[nesnic->sq_head] = skb;
- for (skb_fragment_index = 0; skb_fragment_index < skb_shinfo(skb)->nr_frags;
- skb_fragment_index++) {
- skb_frag_t *frag =
- &skb_shinfo(skb)->frags[skb_fragment_index];
- bus_address = skb_frag_dma_map(&nesdev->pcidev->dev,
- frag, 0, skb_frag_size(frag),
- DMA_TO_DEVICE);
- wqe_fragment_length[wqe_fragment_index] =
- cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[skb_fragment_index]));
- set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index),
- bus_address);
- wqe_fragment_index++;
- if (wqe_fragment_index < 5)
- wqe_fragment_length[wqe_fragment_index] = 0;
- }
- }
-
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX, wqe_misc);
- nesnic->sq_head++;
- nesnic->sq_head &= nesnic->sq_size - 1;
- return true;
-}
-
-
-/**
- * nes_netdev_start_xmit
- */
-static netdev_tx_t nes_netdev_start_xmit(struct sk_buff *skb, struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_hw_nic *nesnic = &nesvnic->nic;
- struct nes_hw_nic_sq_wqe *nic_sqe;
- struct tcphdr *tcph;
- /* struct udphdr *udph; */
-#define NES_MAX_TSO_FRAGS MAX_SKB_FRAGS
- /* 64K segment plus overflow on each side */
- dma_addr_t tso_bus_address[NES_MAX_TSO_FRAGS];
- dma_addr_t bus_address;
- u32 tso_frag_index;
- u32 tso_frag_count;
- u32 tso_wqe_length;
- u32 curr_tcp_seq;
- u32 wqe_count=1;
- struct iphdr *iph;
- __le16 *wqe_fragment_length;
- u32 nr_frags;
- u32 original_first_length;
- /* u64 *wqe_fragment_address; */
- /* first fragment (0) is used by copy buffer */
- u16 wqe_fragment_index=1;
- u16 hoffset;
- u16 nhoffset;
- u16 wqes_needed;
- u16 wqes_available;
- u32 wqe_misc;
-
- /*
- * nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u,"
- * " (%u frags), tso_size=%u\n",
- * netdev->name, skb->len, skb_headlen(skb),
- * skb_shinfo(skb)->nr_frags, skb_is_gso(skb));
- */
-
- if (netif_queue_stopped(netdev))
- return NETDEV_TX_BUSY;
-
- /* Check if SQ is full */
- if ((((nesnic->sq_tail+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) == 1) {
- if (!netif_queue_stopped(netdev)) {
- netif_stop_queue(netdev);
- barrier();
- if ((((((volatile u16)nesnic->sq_tail)+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) != 1) {
- netif_start_queue(netdev);
- goto sq_no_longer_full;
- }
- }
- nesvnic->sq_full++;
- return NETDEV_TX_BUSY;
- }
-
-sq_no_longer_full:
- nr_frags = skb_shinfo(skb)->nr_frags;
- if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) {
- nr_frags++;
- }
- /* Check if too many fragments */
- if (unlikely((nr_frags > 4))) {
- if (skb_is_gso(skb)) {
- nesvnic->segmented_tso_requests++;
- nesvnic->tso_requests++;
- /* Basically 4 fragments available per WQE with extended fragments */
- wqes_needed = nr_frags >> 2;
- wqes_needed += (nr_frags&3)?1:0;
- wqes_available = (((nesnic->sq_tail+nesnic->sq_size)-nesnic->sq_head) - 1) &
- (nesnic->sq_size - 1);
-
- if (unlikely(wqes_needed > wqes_available)) {
- if (!netif_queue_stopped(netdev)) {
- netif_stop_queue(netdev);
- barrier();
- wqes_available = (((((volatile u16)nesnic->sq_tail)+nesnic->sq_size)-nesnic->sq_head) - 1) &
- (nesnic->sq_size - 1);
- if (wqes_needed <= wqes_available) {
- netif_start_queue(netdev);
- goto tso_sq_no_longer_full;
- }
- }
- nesvnic->sq_full++;
- nes_debug(NES_DBG_NIC_TX, "%s: HNIC SQ full- TSO request has too many frags!\n",
- netdev->name);
- return NETDEV_TX_BUSY;
- }
-tso_sq_no_longer_full:
- /* Map all the buffers */
- for (tso_frag_count=0; tso_frag_count < skb_shinfo(skb)->nr_frags;
- tso_frag_count++) {
- skb_frag_t *frag =
- &skb_shinfo(skb)->frags[tso_frag_count];
- tso_bus_address[tso_frag_count] =
- skb_frag_dma_map(&nesdev->pcidev->dev,
- frag, 0, skb_frag_size(frag),
- DMA_TO_DEVICE);
- }
-
- tso_frag_index = 0;
- curr_tcp_seq = ntohl(tcp_hdr(skb)->seq);
- hoffset = skb_transport_header(skb) - skb->data;
- nhoffset = skb_network_header(skb) - skb->data;
- original_first_length = hoffset + ((((struct tcphdr *)skb_transport_header(skb))->doff)<<2);
-
- for (wqe_count=0; wqe_count<((u32)wqes_needed); wqe_count++) {
- tso_wqe_length = 0;
- nic_sqe = &nesnic->sq_vbase[nesnic->sq_head];
- wqe_fragment_length =
- (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
- /* setup the VLAN tag if present */
- if (skb_vlan_tag_present(skb)) {
- nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n",
- netdev->name,
- skb_vlan_tag_get(skb));
- wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE;
- wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb);
- } else
- wqe_misc = 0;
-
- /* bump past the vlan tag */
- wqe_fragment_length++;
-
- /* Assumes header totally fits in allocated buffer and is in first fragment */
- if (original_first_length > NES_FIRST_FRAG_SIZE) {
- nes_debug(NES_DBG_NIC_TX, "ERROR: SKB header too big, headlen=%u, FIRST_FRAG_SIZE=%u\n",
- original_first_length, NES_FIRST_FRAG_SIZE);
- nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u,"
- " (%u frags), is_gso = %u tso_size=%u\n",
- netdev->name,
- skb->len, skb_headlen(skb),
- skb_shinfo(skb)->nr_frags, skb_is_gso(skb), skb_shinfo(skb)->gso_size);
- }
- memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer,
- skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE),
- original_first_length));
- iph = (struct iphdr *)
- (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[nhoffset]);
- tcph = (struct tcphdr *)
- (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[hoffset]);
- if ((wqe_count+1)!=(u32)wqes_needed) {
- tcph->fin = 0;
- tcph->psh = 0;
- tcph->rst = 0;
- tcph->urg = 0;
- }
- if (wqe_count) {
- tcph->syn = 0;
- }
- tcph->seq = htonl(curr_tcp_seq);
- wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE),
- original_first_length));
-
- wqe_fragment_index = 1;
- if ((wqe_count==0) && (skb_headlen(skb) > original_first_length)) {
- set_bit(nesnic->sq_head, nesnic->first_frag_overflow);
- bus_address = pci_map_single(nesdev->pcidev, skb->data + original_first_length,
- skb_headlen(skb) - original_first_length, PCI_DMA_TODEVICE);
- wqe_fragment_length[wqe_fragment_index++] =
- cpu_to_le16(skb_headlen(skb) - original_first_length);
- wqe_fragment_length[wqe_fragment_index] = 0;
- set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX,
- bus_address);
- tso_wqe_length += skb_headlen(skb) -
- original_first_length;
- }
- while (wqe_fragment_index < 5) {
- wqe_fragment_length[wqe_fragment_index] =
- cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index]));
- set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index),
- (u64)tso_bus_address[tso_frag_index]);
- wqe_fragment_index++;
- tso_wqe_length += skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index++]);
- if (wqe_fragment_index < 5)
- wqe_fragment_length[wqe_fragment_index] = 0;
- if (tso_frag_index == tso_frag_count)
- break;
- }
- if ((wqe_count+1) == (u32)wqes_needed) {
- nesnic->tx_skb[nesnic->sq_head] = skb;
- } else {
- nesnic->tx_skb[nesnic->sq_head] = NULL;
- }
- wqe_misc |= NES_NIC_SQ_WQE_COMPLETION | (u16)skb_shinfo(skb)->gso_size;
- if ((tso_wqe_length + original_first_length) > skb_shinfo(skb)->gso_size) {
- wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE;
- } else {
- iph->tot_len = htons(tso_wqe_length + original_first_length - nhoffset);
- }
-
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX,
- wqe_misc);
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX,
- ((u32)tcph->doff) | (((u32)hoffset) << 4));
-
- set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX,
- tso_wqe_length + original_first_length);
- curr_tcp_seq += tso_wqe_length;
- nesnic->sq_head++;
- nesnic->sq_head &= nesnic->sq_size-1;
- }
- } else {
- hoffset = skb_transport_header(skb) - skb->data;
- nhoffset = skb_network_header(skb) - skb->data;
- if (skb_linearize(skb)) {
- nesvnic->tx_sw_dropped++;
- kfree_skb(skb);
- return NETDEV_TX_OK;
- }
- nesvnic->linearized_skbs++;
- skb_set_transport_header(skb, hoffset);
- skb_set_network_header(skb, nhoffset);
- if (!nes_nic_send(skb, netdev))
- return NETDEV_TX_OK;
- }
- } else {
- if (!nes_nic_send(skb, netdev))
- return NETDEV_TX_OK;
- }
-
- barrier();
-
- if (wqe_count)
- nes_write32(nesdev->regs+NES_WQE_ALLOC,
- (wqe_count << 24) | (1 << 23) | nesvnic->nic.qp_id);
-
- netif_trans_update(netdev);
-
- return NETDEV_TX_OK;
-}
-
-
-/**
- * nes_netdev_get_stats
- */
-static struct net_device_stats *nes_netdev_get_stats(struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- u64 u64temp;
- u32 u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_DISCARD + (nesvnic->nic_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->endnode_nstat_rx_discard += u32temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO + (nesvnic->nic_index*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32;
-
- nesvnic->endnode_nstat_rx_octets += u64temp;
- nesvnic->netstats.rx_bytes += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO + (nesvnic->nic_index*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32;
-
- nesvnic->endnode_nstat_rx_frames += u64temp;
- nesvnic->netstats.rx_packets += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO + (nesvnic->nic_index*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32;
-
- nesvnic->endnode_nstat_tx_octets += u64temp;
- nesvnic->netstats.tx_bytes += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO + (nesvnic->nic_index*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32;
-
- nesvnic->endnode_nstat_tx_frames += u64temp;
- nesvnic->netstats.tx_packets += u64temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_short_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_oversized_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_jabber_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_length_errors += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_crc_errors += u32temp;
- nesvnic->netstats.rx_crc_errors += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_tx_errors += u32temp;
- nesvnic->netstats.tx_errors += u32temp;
-
- return &nesvnic->netstats;
-}
-
-
-/**
- * nes_netdev_tx_timeout
- */
-static void nes_netdev_tx_timeout(struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
-
- if (netif_msg_timer(nesvnic))
- nes_debug(NES_DBG_NIC_TX, "%s: tx timeout\n", netdev->name);
-}
-
-
-/**
- * nes_netdev_set_mac_address
- */
-static int nes_netdev_set_mac_address(struct net_device *netdev, void *p)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct sockaddr *mac_addr = p;
- int i;
- u32 macaddr_low;
- u16 macaddr_high;
-
- if (!is_valid_ether_addr(mac_addr->sa_data))
- return -EADDRNOTAVAIL;
-
- memcpy(netdev->dev_addr, mac_addr->sa_data, netdev->addr_len);
- printk(PFX "%s: Address length = %d, Address = %pM\n",
- __func__, netdev->addr_len, mac_addr->sa_data);
- macaddr_high = ((u16)netdev->dev_addr[0]) << 8;
- macaddr_high += (u16)netdev->dev_addr[1];
- macaddr_low = ((u32)netdev->dev_addr[2]) << 24;
- macaddr_low += ((u32)netdev->dev_addr[3]) << 16;
- macaddr_low += ((u32)netdev->dev_addr[4]) << 8;
- macaddr_low += (u32)netdev->dev_addr[5];
-
- for (i = 0; i < NES_MAX_PORT_COUNT; i++) {
- if (nesvnic->qp_nic_index[i] == 0xf) {
- break;
- }
- nes_write_indexed(nesdev,
- NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8),
- macaddr_low);
- nes_write_indexed(nesdev,
- NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8),
- (u32)macaddr_high | NES_MAC_ADDR_VALID |
- ((((u32)nesvnic->nic_index) << 16)));
- }
- return 0;
-}
-
-
-static void set_allmulti(struct nes_device *nesdev, u32 nic_active_bit)
-{
- u32 nic_active;
-
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL);
- nic_active &= ~nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active);
-}
-
-#define get_addr(addrs, index) ((addrs) + (index) * ETH_ALEN)
-
-/**
- * nes_netdev_set_multicast_list
- */
-static void nes_netdev_set_multicast_list(struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter;
- u32 nic_active_bit;
- u32 nic_active;
- u32 perfect_filter_register_address;
- u32 macaddr_low;
- u16 macaddr_high;
- u8 mc_all_on = 0;
- u8 mc_index;
- int mc_nic_index = -1;
- u8 pft_entries_preallocated = max(nesadapter->adapter_fcn_count *
- nics_per_function, 4);
- u8 max_pft_entries_avaiable = NES_PFT_SIZE - pft_entries_preallocated;
- unsigned long flags;
- int mc_count = netdev_mc_count(netdev);
-
- spin_lock_irqsave(&nesadapter->resource_lock, flags);
- nic_active_bit = 1 << nesvnic->nic_index;
-
- if (netdev->flags & IFF_PROMISC) {
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL);
- nic_active |= nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active);
- mc_all_on = 1;
- } else if ((netdev->flags & IFF_ALLMULTI) ||
- (nesvnic->nic_index > 3)) {
- set_allmulti(nesdev, nic_active_bit);
- mc_all_on = 1;
- } else {
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL);
- nic_active &= ~nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active);
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL);
- nic_active &= ~nic_active_bit;
- nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active);
- }
-
- nes_debug(NES_DBG_NIC_RX, "Number of MC entries = %d, Promiscuous = %d, All Multicast = %d.\n",
- mc_count, !!(netdev->flags & IFF_PROMISC),
- !!(netdev->flags & IFF_ALLMULTI));
- if (!mc_all_on) {
- char *addrs;
- int i;
- struct netdev_hw_addr *ha;
-
- addrs = kmalloc_array(mc_count, ETH_ALEN, GFP_ATOMIC);
- if (!addrs) {
- set_allmulti(nesdev, nic_active_bit);
- goto unlock;
- }
- i = 0;
- netdev_for_each_mc_addr(ha, netdev)
- memcpy(get_addr(addrs, i++), ha->addr, ETH_ALEN);
-
- perfect_filter_register_address = NES_IDX_PERFECT_FILTER_LOW +
- pft_entries_preallocated * 0x8;
- for (i = 0, mc_index = 0; mc_index < max_pft_entries_avaiable;
- mc_index++) {
- while (i < mc_count && nesvnic->mcrq_mcast_filter &&
- ((mc_nic_index = nesvnic->mcrq_mcast_filter(nesvnic,
- get_addr(addrs, i++))) == 0));
- if (mc_nic_index < 0)
- mc_nic_index = nesvnic->nic_index;
- while (nesadapter->pft_mcast_map[mc_index] < 16 &&
- nesadapter->pft_mcast_map[mc_index] !=
- nesvnic->nic_index &&
- mc_index < max_pft_entries_avaiable) {
- nes_debug(NES_DBG_NIC_RX,
- "mc_index=%d skipping nic_index=%d, used for=%d\n",
- mc_index, nesvnic->nic_index,
- nesadapter->pft_mcast_map[mc_index]);
- mc_index++;
- }
- if (mc_index >= max_pft_entries_avaiable)
- break;
- if (i < mc_count) {
- char *addr = get_addr(addrs, i++);
-
- nes_debug(NES_DBG_NIC_RX, "Assigning MC Address %pM to register 0x%04X nic_idx=%d\n",
- addr,
- perfect_filter_register_address+(mc_index * 8),
- mc_nic_index);
- macaddr_high = ((u8) addr[0]) << 8;
- macaddr_high += (u8) addr[1];
- macaddr_low = ((u8) addr[2]) << 24;
- macaddr_low += ((u8) addr[3]) << 16;
- macaddr_low += ((u8) addr[4]) << 8;
- macaddr_low += (u8) addr[5];
-
- nes_write_indexed(nesdev,
- perfect_filter_register_address+(mc_index * 8),
- macaddr_low);
- nes_write_indexed(nesdev,
- perfect_filter_register_address+4+(mc_index * 8),
- (u32)macaddr_high | NES_MAC_ADDR_VALID |
- ((((u32)(1<<mc_nic_index)) << 16)));
- nesadapter->pft_mcast_map[mc_index] =
- nesvnic->nic_index;
- } else {
- nes_debug(NES_DBG_NIC_RX, "Clearing MC Address at register 0x%04X\n",
- perfect_filter_register_address+(mc_index * 8));
- nes_write_indexed(nesdev,
- perfect_filter_register_address+4+(mc_index * 8),
- 0);
- nesadapter->pft_mcast_map[mc_index] = 255;
- }
- }
- kfree(addrs);
- /* PFT is not large enough */
- if (i < mc_count)
- set_allmulti(nesdev, nic_active_bit);
- }
-
-unlock:
- spin_unlock_irqrestore(&nesadapter->resource_lock, flags);
-}
-
-
-/**
- * nes_netdev_change_mtu
- */
-static int nes_netdev_change_mtu(struct net_device *netdev, int new_mtu)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- u8 jumbomode = 0;
- u32 nic_active;
- u32 nic_active_bit;
- u32 uc_all_active;
- u32 mc_all_active;
-
- netdev->mtu = new_mtu;
- nesvnic->max_frame_size = new_mtu + VLAN_ETH_HLEN;
-
- if (netdev->mtu > ETH_DATA_LEN) {
- jumbomode=1;
- }
- nes_nic_init_timer_defaults(nesdev, jumbomode);
-
- if (netif_running(netdev)) {
- nic_active_bit = 1 << nesvnic->nic_index;
- mc_all_active = nes_read_indexed(nesdev,
- NES_IDX_NIC_MULTICAST_ALL) & nic_active_bit;
- uc_all_active = nes_read_indexed(nesdev,
- NES_IDX_NIC_UNICAST_ALL) & nic_active_bit;
-
- nes_netdev_stop(netdev);
- nes_netdev_open(netdev);
-
- nic_active = nes_read_indexed(nesdev,
- NES_IDX_NIC_MULTICAST_ALL);
- nic_active |= mc_all_active;
- nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL,
- nic_active);
-
- nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL);
- nic_active |= uc_all_active;
- nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active);
- }
-
- return 0;
-}
-
-
-static const char nes_ethtool_stringset[][ETH_GSTRING_LEN] = {
- "Link Change Interrupts",
- "Linearized SKBs",
- "T/GSO Requests",
- "Pause Frames Sent",
- "Pause Frames Received",
- "Internal Routing Errors",
- "SQ SW Dropped SKBs",
- "SQ Full",
- "Segmented TSO Requests",
- "Rx Symbol Errors",
- "Rx Jabber Errors",
- "Rx Oversized Frames",
- "Rx Short Frames",
- "Rx Length Errors",
- "Rx CRC Errors",
- "Rx Port Discard",
- "Endnode Rx Discards",
- "Endnode Rx Octets",
- "Endnode Rx Frames",
- "Endnode Tx Octets",
- "Endnode Tx Frames",
- "Tx Errors",
- "mh detected",
- "mh pauses",
- "Retransmission Count",
- "CM Connects",
- "CM Accepts",
- "Disconnects",
- "Connected Events",
- "Connect Requests",
- "CM Rejects",
- "ModifyQP Timeouts",
- "CreateQPs",
- "SW DestroyQPs",
- "DestroyQPs",
- "CM Closes",
- "CM Packets Sent",
- "CM Packets Bounced",
- "CM Packets Created",
- "CM Packets Rcvd",
- "CM Packets Dropped",
- "CM Packets Retrans",
- "CM Listens Created",
- "CM Listens Destroyed",
- "CM Backlog Drops",
- "CM Loopbacks",
- "CM Nodes Created",
- "CM Nodes Destroyed",
- "CM Accel Drops",
- "CM Resets Received",
- "Free 4Kpbls",
- "Free 256pbls",
- "Timer Inits",
- "PAU CreateQPs",
- "PAU DestroyQPs",
-};
-#define NES_ETHTOOL_STAT_COUNT ARRAY_SIZE(nes_ethtool_stringset)
-
-
-/**
- * nes_netdev_get_sset_count
- */
-static int nes_netdev_get_sset_count(struct net_device *netdev, int stringset)
-{
- if (stringset == ETH_SS_STATS)
- return NES_ETHTOOL_STAT_COUNT;
- else
- return -EINVAL;
-}
-
-
-/**
- * nes_netdev_get_strings
- */
-static void nes_netdev_get_strings(struct net_device *netdev, u32 stringset,
- u8 *ethtool_strings)
-{
- if (stringset == ETH_SS_STATS)
- memcpy(ethtool_strings,
- &nes_ethtool_stringset,
- sizeof(nes_ethtool_stringset));
-}
-
-
-/**
- * nes_netdev_get_ethtool_stats
- */
-
-static void nes_netdev_get_ethtool_stats(struct net_device *netdev,
- struct ethtool_stats *target_ethtool_stats, u64 *target_stat_values)
-{
- u64 u64temp;
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 nic_count;
- u32 u32temp;
- u32 index = 0;
-
- target_ethtool_stats->n_stats = NES_ETHTOOL_STAT_COUNT;
- target_stat_values[index] = nesvnic->nesdev->link_status_interrupts;
- target_stat_values[++index] = nesvnic->linearized_skbs;
- target_stat_values[++index] = nesvnic->tso_requests;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_TX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_pause_frames_sent += u32temp;
- target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_sent;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_pause_frames_received += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_PORT_RX_DISCARDS + (nesvnic->nesdev->mac_index*0x40));
- nesvnic->nesdev->port_rx_discards += u32temp;
- nesvnic->netstats.rx_dropped += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_PORT_TX_DISCARDS + (nesvnic->nesdev->mac_index*0x40));
- nesvnic->nesdev->port_tx_discards += u32temp;
- nesvnic->netstats.tx_dropped += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_short_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_oversized_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_jabber_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->netstats.rx_length_errors += u32temp;
- nesvnic->nesdev->mac_rx_errors += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_rx_errors += u32temp;
- nesvnic->nesdev->mac_rx_crc_errors += u32temp;
- nesvnic->netstats.rx_crc_errors += u32temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200));
- nesvnic->nesdev->mac_tx_errors += u32temp;
- nesvnic->netstats.tx_errors += u32temp;
-
- for (nic_count = 0; nic_count < NES_MAX_PORT_COUNT; nic_count++) {
- if (nesvnic->qp_nic_index[nic_count] == 0xf)
- break;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_DISCARD +
- (nesvnic->qp_nic_index[nic_count]*0x200));
- nesvnic->netstats.rx_dropped += u32temp;
- nesvnic->endnode_nstat_rx_discard += u32temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO +
- (nesvnic->qp_nic_index[nic_count]*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI +
- (nesvnic->qp_nic_index[nic_count]*0x200))) << 32;
-
- nesvnic->endnode_nstat_rx_octets += u64temp;
- nesvnic->netstats.rx_bytes += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO +
- (nesvnic->qp_nic_index[nic_count]*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI +
- (nesvnic->qp_nic_index[nic_count]*0x200))) << 32;
-
- nesvnic->endnode_nstat_rx_frames += u64temp;
- nesvnic->netstats.rx_packets += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO +
- (nesvnic->qp_nic_index[nic_count]*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI +
- (nesvnic->qp_nic_index[nic_count]*0x200))) << 32;
-
- nesvnic->endnode_nstat_tx_octets += u64temp;
- nesvnic->netstats.tx_bytes += u64temp;
-
- u64temp = (u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO +
- (nesvnic->qp_nic_index[nic_count]*0x200));
- u64temp += ((u64)nes_read_indexed(nesdev,
- NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI +
- (nesvnic->qp_nic_index[nic_count]*0x200))) << 32;
-
- nesvnic->endnode_nstat_tx_frames += u64temp;
- nesvnic->netstats.tx_packets += u64temp;
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_IPV4_TCP_REXMITS + (nesvnic->qp_nic_index[nic_count]*0x200));
- nesvnic->endnode_ipv4_tcp_retransmits += u32temp;
- }
-
- target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_received;
- target_stat_values[++index] = nesdev->nesadapter->nic_rx_eth_route_err;
- target_stat_values[++index] = nesvnic->tx_sw_dropped;
- target_stat_values[++index] = nesvnic->sq_full;
- target_stat_values[++index] = nesvnic->segmented_tso_requests;
- target_stat_values[++index] = nesvnic->nesdev->mac_rx_symbol_err_frames;
- target_stat_values[++index] = nesvnic->nesdev->mac_rx_jabber_frames;
- target_stat_values[++index] = nesvnic->nesdev->mac_rx_oversized_frames;
- target_stat_values[++index] = nesvnic->nesdev->mac_rx_short_frames;
- target_stat_values[++index] = nesvnic->netstats.rx_length_errors;
- target_stat_values[++index] = nesvnic->nesdev->mac_rx_crc_errors;
- target_stat_values[++index] = nesvnic->nesdev->port_rx_discards;
- target_stat_values[++index] = nesvnic->endnode_nstat_rx_discard;
- target_stat_values[++index] = nesvnic->endnode_nstat_rx_octets;
- target_stat_values[++index] = nesvnic->endnode_nstat_rx_frames;
- target_stat_values[++index] = nesvnic->endnode_nstat_tx_octets;
- target_stat_values[++index] = nesvnic->endnode_nstat_tx_frames;
- target_stat_values[++index] = nesvnic->nesdev->mac_tx_errors;
- target_stat_values[++index] = mh_detected;
- target_stat_values[++index] = mh_pauses_sent;
- target_stat_values[++index] = nesvnic->endnode_ipv4_tcp_retransmits;
- target_stat_values[++index] = atomic_read(&cm_connects);
- target_stat_values[++index] = atomic_read(&cm_accepts);
- target_stat_values[++index] = atomic_read(&cm_disconnects);
- target_stat_values[++index] = atomic_read(&cm_connecteds);
- target_stat_values[++index] = atomic_read(&cm_connect_reqs);
- target_stat_values[++index] = atomic_read(&cm_rejects);
- target_stat_values[++index] = atomic_read(&mod_qp_timouts);
- target_stat_values[++index] = atomic_read(&qps_created);
- target_stat_values[++index] = atomic_read(&sw_qps_destroyed);
- target_stat_values[++index] = atomic_read(&qps_destroyed);
- target_stat_values[++index] = atomic_read(&cm_closes);
- target_stat_values[++index] = cm_packets_sent;
- target_stat_values[++index] = cm_packets_bounced;
- target_stat_values[++index] = cm_packets_created;
- target_stat_values[++index] = cm_packets_received;
- target_stat_values[++index] = cm_packets_dropped;
- target_stat_values[++index] = cm_packets_retrans;
- target_stat_values[++index] = atomic_read(&cm_listens_created);
- target_stat_values[++index] = atomic_read(&cm_listens_destroyed);
- target_stat_values[++index] = cm_backlog_drops;
- target_stat_values[++index] = atomic_read(&cm_loopbacks);
- target_stat_values[++index] = atomic_read(&cm_nodes_created);
- target_stat_values[++index] = atomic_read(&cm_nodes_destroyed);
- target_stat_values[++index] = atomic_read(&cm_accel_dropped_pkts);
- target_stat_values[++index] = atomic_read(&cm_resets_recvd);
- target_stat_values[++index] = nesadapter->free_4kpbl;
- target_stat_values[++index] = nesadapter->free_256pbl;
- target_stat_values[++index] = int_mod_timer_init;
- target_stat_values[++index] = atomic_read(&pau_qps_created);
- target_stat_values[++index] = atomic_read(&pau_qps_destroyed);
-}
-
-/**
- * nes_netdev_get_drvinfo
- */
-static void nes_netdev_get_drvinfo(struct net_device *netdev,
- struct ethtool_drvinfo *drvinfo)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter;
-
- strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
- strlcpy(drvinfo->bus_info, pci_name(nesvnic->nesdev->pcidev),
- sizeof(drvinfo->bus_info));
- snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
- "%u.%u", nesadapter->firmware_version >> 16,
- nesadapter->firmware_version & 0x000000ff);
- strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
-}
-
-
-/**
- * nes_netdev_set_coalesce
- */
-static int nes_netdev_set_coalesce(struct net_device *netdev,
- struct ethtool_coalesce *et_coalesce)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
- unsigned long flags;
-
- spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
- if (et_coalesce->rx_max_coalesced_frames_low) {
- shared_timer->threshold_low = et_coalesce->rx_max_coalesced_frames_low;
- }
- if (et_coalesce->rx_max_coalesced_frames_irq) {
- shared_timer->threshold_target = et_coalesce->rx_max_coalesced_frames_irq;
- }
- if (et_coalesce->rx_max_coalesced_frames_high) {
- shared_timer->threshold_high = et_coalesce->rx_max_coalesced_frames_high;
- }
- if (et_coalesce->rx_coalesce_usecs_low) {
- shared_timer->timer_in_use_min = et_coalesce->rx_coalesce_usecs_low;
- }
- if (et_coalesce->rx_coalesce_usecs_high) {
- shared_timer->timer_in_use_max = et_coalesce->rx_coalesce_usecs_high;
- }
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
-
- /* using this to drive total interrupt moderation */
- nesadapter->et_rx_coalesce_usecs_irq = et_coalesce->rx_coalesce_usecs_irq;
- if (et_coalesce->use_adaptive_rx_coalesce) {
- nesadapter->et_use_adaptive_rx_coalesce = 1;
- nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
- nesadapter->et_rx_coalesce_usecs_irq = 0;
- if (et_coalesce->pkt_rate_low) {
- nesadapter->et_pkt_rate_low = et_coalesce->pkt_rate_low;
- }
- } else {
- nesadapter->et_use_adaptive_rx_coalesce = 0;
- nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
- if (nesadapter->et_rx_coalesce_usecs_irq) {
- nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
- 0x80000000 | ((u32)(nesadapter->et_rx_coalesce_usecs_irq*8)));
- }
- }
- return 0;
-}
-
-
-/**
- * nes_netdev_get_coalesce
- */
-static int nes_netdev_get_coalesce(struct net_device *netdev,
- struct ethtool_coalesce *et_coalesce)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct ethtool_coalesce temp_et_coalesce;
- struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
- unsigned long flags;
-
- memset(&temp_et_coalesce, 0, sizeof(temp_et_coalesce));
- temp_et_coalesce.rx_coalesce_usecs_irq = nesadapter->et_rx_coalesce_usecs_irq;
- temp_et_coalesce.use_adaptive_rx_coalesce = nesadapter->et_use_adaptive_rx_coalesce;
- temp_et_coalesce.rate_sample_interval = nesadapter->et_rate_sample_interval;
- temp_et_coalesce.pkt_rate_low = nesadapter->et_pkt_rate_low;
- spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
- temp_et_coalesce.rx_max_coalesced_frames_low = shared_timer->threshold_low;
- temp_et_coalesce.rx_max_coalesced_frames_irq = shared_timer->threshold_target;
- temp_et_coalesce.rx_max_coalesced_frames_high = shared_timer->threshold_high;
- temp_et_coalesce.rx_coalesce_usecs_low = shared_timer->timer_in_use_min;
- temp_et_coalesce.rx_coalesce_usecs_high = shared_timer->timer_in_use_max;
- if (nesadapter->et_use_adaptive_rx_coalesce) {
- temp_et_coalesce.rx_coalesce_usecs_irq = shared_timer->timer_in_use;
- }
- spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
- memcpy(et_coalesce, &temp_et_coalesce, sizeof(*et_coalesce));
- return 0;
-}
-
-
-/**
- * nes_netdev_get_pauseparam
- */
-static void nes_netdev_get_pauseparam(struct net_device *netdev,
- struct ethtool_pauseparam *et_pauseparam)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
-
- et_pauseparam->autoneg = 0;
- et_pauseparam->rx_pause = (nesvnic->nesdev->disable_rx_flow_control == 0) ? 1:0;
- et_pauseparam->tx_pause = (nesvnic->nesdev->disable_tx_flow_control == 0) ? 1:0;
-}
-
-
-/**
- * nes_netdev_set_pauseparam
- */
-static int nes_netdev_set_pauseparam(struct net_device *netdev,
- struct ethtool_pauseparam *et_pauseparam)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- u32 u32temp;
-
- if (et_pauseparam->autoneg) {
- /* TODO: should return unsupported */
- return 0;
- }
- if ((et_pauseparam->tx_pause == 1) && (nesdev->disable_tx_flow_control == 1)) {
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200));
- u32temp |= NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE;
- nes_write_indexed(nesdev,
- NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp);
- nesdev->disable_tx_flow_control = 0;
- } else if ((et_pauseparam->tx_pause == 0) && (nesdev->disable_tx_flow_control == 0)) {
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200));
- u32temp &= ~NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE;
- nes_write_indexed(nesdev,
- NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp);
- nesdev->disable_tx_flow_control = 1;
- }
- if ((et_pauseparam->rx_pause == 1) && (nesdev->disable_rx_flow_control == 1)) {
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40));
- u32temp &= ~NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE;
- nes_write_indexed(nesdev,
- NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp);
- nesdev->disable_rx_flow_control = 0;
- } else if ((et_pauseparam->rx_pause == 0) && (nesdev->disable_rx_flow_control == 0)) {
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40));
- u32temp |= NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE;
- nes_write_indexed(nesdev,
- NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp);
- nesdev->disable_rx_flow_control = 1;
- }
-
- return 0;
-}
-
-
-/**
- * nes_netdev_get_settings
- */
-static int nes_netdev_get_link_ksettings(struct net_device *netdev,
- struct ethtool_link_ksettings *cmd)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 mac_index = nesdev->mac_index;
- u8 phy_type = nesadapter->phy_type[mac_index];
- u8 phy_index = nesadapter->phy_index[mac_index];
- u16 phy_data;
- u32 supported, advertising;
-
- cmd->base.duplex = DUPLEX_FULL;
- cmd->base.port = PORT_MII;
-
- if (nesadapter->OneG_Mode) {
- cmd->base.speed = SPEED_1000;
- if (phy_type == NES_PHY_TYPE_PUMA_1G) {
- supported = SUPPORTED_1000baseT_Full;
- advertising = ADVERTISED_1000baseT_Full;
- cmd->base.autoneg = AUTONEG_DISABLE;
- cmd->base.phy_address = mac_index;
- } else {
- unsigned long flags;
-
- supported = SUPPORTED_1000baseT_Full
- | SUPPORTED_Autoneg;
- advertising = ADVERTISED_1000baseT_Full
- | ADVERTISED_Autoneg;
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- if (phy_data & 0x1000)
- cmd->base.autoneg = AUTONEG_ENABLE;
- else
- cmd->base.autoneg = AUTONEG_DISABLE;
- cmd->base.phy_address = phy_index;
- }
- ethtool_convert_legacy_u32_to_link_mode(
- cmd->link_modes.supported, supported);
- ethtool_convert_legacy_u32_to_link_mode(
- cmd->link_modes.advertising, advertising);
- return 0;
- }
- if ((phy_type == NES_PHY_TYPE_ARGUS) ||
- (phy_type == NES_PHY_TYPE_SFP_D) ||
- (phy_type == NES_PHY_TYPE_KR)) {
- cmd->base.port = PORT_FIBRE;
- supported = SUPPORTED_FIBRE;
- advertising = ADVERTISED_FIBRE;
- cmd->base.phy_address = phy_index;
- } else {
- supported = SUPPORTED_10000baseT_Full;
- advertising = ADVERTISED_10000baseT_Full;
- cmd->base.phy_address = mac_index;
- }
- cmd->base.speed = SPEED_10000;
- cmd->base.autoneg = AUTONEG_DISABLE;
- ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
- supported);
- ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
- advertising);
-
- return 0;
-}
-
-
-/**
- * nes_netdev_set_settings
- */
-static int
-nes_netdev_set_link_ksettings(struct net_device *netdev,
- const struct ethtool_link_ksettings *cmd)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- if ((nesadapter->OneG_Mode) &&
- (nesadapter->phy_type[nesdev->mac_index] != NES_PHY_TYPE_PUMA_1G)) {
- unsigned long flags;
- u16 phy_data;
- u8 phy_index = nesadapter->phy_index[nesdev->mac_index];
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
- if (cmd->base.autoneg) {
- /* Turn on Full duplex, Autoneg, and restart autonegotiation */
- phy_data |= 0x1300;
- } else {
- /* Turn off autoneg */
- phy_data &= ~0x1000;
- }
- nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data);
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- }
-
- return 0;
-}
-
-
-static const struct ethtool_ops nes_ethtool_ops = {
- .get_link = ethtool_op_get_link,
- .get_strings = nes_netdev_get_strings,
- .get_sset_count = nes_netdev_get_sset_count,
- .get_ethtool_stats = nes_netdev_get_ethtool_stats,
- .get_drvinfo = nes_netdev_get_drvinfo,
- .get_coalesce = nes_netdev_get_coalesce,
- .set_coalesce = nes_netdev_set_coalesce,
- .get_pauseparam = nes_netdev_get_pauseparam,
- .set_pauseparam = nes_netdev_set_pauseparam,
- .get_link_ksettings = nes_netdev_get_link_ksettings,
- .set_link_ksettings = nes_netdev_set_link_ksettings,
-};
-
-static void nes_vlan_mode(struct net_device *netdev, struct nes_device *nesdev, netdev_features_t features)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 u32temp;
- unsigned long flags;
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
-
- nes_debug(NES_DBG_NETDEV, "%s: %s\n", __func__, netdev->name);
-
- /* Enable/Disable VLAN Stripping */
- u32temp = nes_read_indexed(nesdev, NES_IDX_PCIX_DIAG);
- if (features & NETIF_F_HW_VLAN_CTAG_RX)
- u32temp &= 0xfdffffff;
- else
- u32temp |= 0x02000000;
-
- nes_write_indexed(nesdev, NES_IDX_PCIX_DIAG, u32temp);
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
-}
-
-static netdev_features_t nes_fix_features(struct net_device *netdev, netdev_features_t features)
-{
- /*
- * Since there is no support for separate rx/tx vlan accel
- * enable/disable make sure tx flag is always in same state as rx.
- */
- if (features & NETIF_F_HW_VLAN_CTAG_RX)
- features |= NETIF_F_HW_VLAN_CTAG_TX;
- else
- features &= ~NETIF_F_HW_VLAN_CTAG_TX;
-
- return features;
-}
-
-static int nes_set_features(struct net_device *netdev, netdev_features_t features)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- u32 changed = netdev->features ^ features;
-
- if (changed & NETIF_F_HW_VLAN_CTAG_RX)
- nes_vlan_mode(netdev, nesdev, features);
-
- return 0;
-}
-
-static const struct net_device_ops nes_netdev_ops = {
- .ndo_open = nes_netdev_open,
- .ndo_stop = nes_netdev_stop,
- .ndo_start_xmit = nes_netdev_start_xmit,
- .ndo_get_stats = nes_netdev_get_stats,
- .ndo_tx_timeout = nes_netdev_tx_timeout,
- .ndo_set_mac_address = nes_netdev_set_mac_address,
- .ndo_set_rx_mode = nes_netdev_set_multicast_list,
- .ndo_change_mtu = nes_netdev_change_mtu,
- .ndo_validate_addr = eth_validate_addr,
- .ndo_fix_features = nes_fix_features,
- .ndo_set_features = nes_set_features,
-};
-
-/**
- * nes_netdev_init - initialize network device
- */
-struct net_device *nes_netdev_init(struct nes_device *nesdev,
- void __iomem *mmio_addr)
-{
- u64 u64temp;
- struct nes_vnic *nesvnic;
- struct net_device *netdev;
- struct nic_qp_map *curr_qp_map;
- u8 phy_type = nesdev->nesadapter->phy_type[nesdev->mac_index];
-
- netdev = alloc_etherdev(sizeof(struct nes_vnic));
- if (!netdev) {
- printk(KERN_ERR PFX "nesvnic etherdev alloc failed");
- return NULL;
- }
- nesvnic = netdev_priv(netdev);
-
- nes_debug(NES_DBG_INIT, "netdev = %p, %s\n", netdev, netdev->name);
-
- SET_NETDEV_DEV(netdev, &nesdev->pcidev->dev);
-
- netdev->watchdog_timeo = NES_TX_TIMEOUT;
- netdev->irq = nesdev->pcidev->irq;
- netdev->max_mtu = NES_MAX_MTU;
- netdev->hard_header_len = ETH_HLEN;
- netdev->addr_len = ETH_ALEN;
- netdev->type = ARPHRD_ETHER;
- netdev->netdev_ops = &nes_netdev_ops;
- netdev->ethtool_ops = &nes_ethtool_ops;
- netif_napi_add(netdev, &nesvnic->napi, nes_netdev_poll, 128);
- nes_debug(NES_DBG_INIT, "Enabling VLAN Insert/Delete.\n");
-
- /* Fill in the port structure */
- nesvnic->netdev = netdev;
- nesvnic->nesdev = nesdev;
- nesvnic->msg_enable = netif_msg_init(debug, default_msg);
- nesvnic->netdev_index = nesdev->netdev_count;
- nesvnic->perfect_filter_index = nesdev->nesadapter->netdev_count;
- nesvnic->max_frame_size = netdev->mtu + netdev->hard_header_len + VLAN_HLEN;
-
- curr_qp_map = nic_qp_mapping_per_function[PCI_FUNC(nesdev->pcidev->devfn)];
- nesvnic->nic.qp_id = curr_qp_map[nesdev->netdev_count].qpid;
- nesvnic->nic_index = curr_qp_map[nesdev->netdev_count].nic_index;
- nesvnic->logical_port = curr_qp_map[nesdev->netdev_count].logical_port;
-
- /* Setup the burned in MAC address */
- u64temp = (u64)nesdev->nesadapter->mac_addr_low;
- u64temp += ((u64)nesdev->nesadapter->mac_addr_high) << 32;
- u64temp += nesvnic->nic_index;
- netdev->dev_addr[0] = (u8)(u64temp>>40);
- netdev->dev_addr[1] = (u8)(u64temp>>32);
- netdev->dev_addr[2] = (u8)(u64temp>>24);
- netdev->dev_addr[3] = (u8)(u64temp>>16);
- netdev->dev_addr[4] = (u8)(u64temp>>8);
- netdev->dev_addr[5] = (u8)u64temp;
-
- netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
- if ((nesvnic->logical_port < 2) || (nesdev->nesadapter->hw_rev != NE020_REV))
- netdev->hw_features |= NETIF_F_TSO;
-
- netdev->features = netdev->hw_features | NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX;
-
- nes_debug(NES_DBG_INIT, "nesvnic = %p, reported features = 0x%lX, QPid = %d,"
- " nic_index = %d, logical_port = %d, mac_index = %d.\n",
- nesvnic, (unsigned long)netdev->features, nesvnic->nic.qp_id,
- nesvnic->nic_index, nesvnic->logical_port, nesdev->mac_index);
-
- if (nesvnic->nesdev->nesadapter->port_count == 1 &&
- nesvnic->nesdev->nesadapter->adapter_fcn_count == 1) {
-
- nesvnic->qp_nic_index[0] = nesvnic->nic_index;
- nesvnic->qp_nic_index[1] = nesvnic->nic_index + 1;
- if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT) {
- nesvnic->qp_nic_index[2] = 0xf;
- nesvnic->qp_nic_index[3] = 0xf;
- } else {
- nesvnic->qp_nic_index[2] = nesvnic->nic_index + 2;
- nesvnic->qp_nic_index[3] = nesvnic->nic_index + 3;
- }
- } else {
- if (nesvnic->nesdev->nesadapter->port_count == 2 ||
- (nesvnic->nesdev->nesadapter->port_count == 1 &&
- nesvnic->nesdev->nesadapter->adapter_fcn_count == 2)) {
- nesvnic->qp_nic_index[0] = nesvnic->nic_index;
- nesvnic->qp_nic_index[1] = nesvnic->nic_index
- + 2;
- nesvnic->qp_nic_index[2] = 0xf;
- nesvnic->qp_nic_index[3] = 0xf;
- } else {
- nesvnic->qp_nic_index[0] = nesvnic->nic_index;
- nesvnic->qp_nic_index[1] = 0xf;
- nesvnic->qp_nic_index[2] = 0xf;
- nesvnic->qp_nic_index[3] = 0xf;
- }
- }
- nesvnic->next_qp_nic_index = 0;
-
- if (nesdev->netdev_count == 0) {
- nesvnic->rdma_enabled = 1;
- } else {
- nesvnic->rdma_enabled = 0;
- }
- nesvnic->nic_cq.cq_number = nesvnic->nic.qp_id;
- timer_setup(&nesvnic->event_timer, NULL, 0);
- spin_lock_init(&nesvnic->tx_lock);
- spin_lock_init(&nesvnic->port_ibevent_lock);
- nesdev->netdev[nesdev->netdev_count] = netdev;
-
- nes_debug(NES_DBG_INIT, "Adding nesvnic (%p) to the adapters nesvnic_list for MAC%d.\n",
- nesvnic, nesdev->mac_index);
- list_add_tail(&nesvnic->list, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]);
-
- if ((nesdev->netdev_count == 0) &&
- ((PCI_FUNC(nesdev->pcidev->devfn) == nesdev->mac_index) ||
- ((phy_type == NES_PHY_TYPE_PUMA_1G) &&
- (((PCI_FUNC(nesdev->pcidev->devfn) == 1) && (nesdev->mac_index == 2)) ||
- ((PCI_FUNC(nesdev->pcidev->devfn) == 2) && (nesdev->mac_index == 1)))))) {
- u32 u32temp;
- u32 link_mask = 0;
- u32 link_val = 0;
- u16 temp_phy_data;
- u16 phy_data = 0;
- unsigned long flags;
-
- u32temp = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200 * (nesdev->mac_index & 1)));
- if (phy_type != NES_PHY_TYPE_PUMA_1G) {
- u32temp |= 0x00200000;
- nes_write_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200 * (nesdev->mac_index & 1)), u32temp);
- }
-
- /* Check and set linkup here. This is for back to back */
- /* configuration where second port won't get link interrupt */
- switch (phy_type) {
- case NES_PHY_TYPE_PUMA_1G:
- if (nesdev->mac_index < 2) {
- link_mask = 0x01010000;
- link_val = 0x01010000;
- } else {
- link_mask = 0x02020000;
- link_val = 0x02020000;
- }
- break;
- case NES_PHY_TYPE_SFP_D:
- spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
- nes_read_10G_phy_reg(nesdev,
- nesdev->nesadapter->phy_index[nesdev->mac_index],
- 1, 0x9003);
- temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- nes_read_10G_phy_reg(nesdev,
- nesdev->nesadapter->phy_index[nesdev->mac_index],
- 3, 0x0021);
- nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- nes_read_10G_phy_reg(nesdev,
- nesdev->nesadapter->phy_index[nesdev->mac_index],
- 3, 0x0021);
- phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
- phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
- break;
- default:
- link_mask = 0x0f1f0000;
- link_val = 0x0f0f0000;
- break;
- }
-
- u32temp = nes_read_indexed(nesdev,
- NES_IDX_PHY_PCS_CONTROL_STATUS0 +
- (0x200 * (nesdev->mac_index & 1)));
-
- if (phy_type == NES_PHY_TYPE_SFP_D) {
- if (phy_data & 0x0004)
- nesvnic->linkup = 1;
- } else {
- if ((u32temp & link_mask) == link_val)
- nesvnic->linkup = 1;
- }
-
- /* clear the MAC interrupt status, assumes direct logical to physical mapping */
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index));
- nes_debug(NES_DBG_INIT, "Phy interrupt status = 0x%X.\n", u32temp);
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index), u32temp);
-
- nes_init_phy(nesdev);
- }
-
- nes_vlan_mode(netdev, nesdev, netdev->features);
-
- return netdev;
-}
-
-
-/**
- * nes_netdev_destroy - destroy network device structure
- */
-void nes_netdev_destroy(struct net_device *netdev)
-{
- struct nes_vnic *nesvnic = netdev_priv(netdev);
-
- /* make sure 'stop' method is called by Linux stack */
- /* nes_netdev_stop(netdev); */
-
- list_del(&nesvnic->list);
-
- if (nesvnic->of_device_registered) {
- nes_destroy_ofa_device(nesvnic->nesibdev);
- }
-
- free_netdev(netdev);
-}
-
-
-/**
- * nes_nic_cm_xmit -- CM calls this to send out pkts
- */
-int nes_nic_cm_xmit(struct sk_buff *skb, struct net_device *netdev)
-{
- int ret;
-
- skb->dev = netdev;
- ret = dev_queue_xmit(skb);
- if (ret) {
- nes_debug(NES_DBG_CM, "Bad return code from dev_queue_xmit %d\n", ret);
- }
-
- return ret;
-}
diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c
deleted file mode 100644
index 21b4a8373acf..000000000000
--- a/drivers/infiniband/hw/nes/nes_utils.c
+++ /dev/null
@@ -1,916 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
-#include <linux/mii.h>
-#include <linux/if_vlan.h>
-#include <linux/slab.h>
-#include <linux/crc32.h>
-#include <linux/in.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/byteorder.h>
-
-#include "nes.h"
-
-static u16 nes_read16_eeprom(void __iomem *addr, u16 offset);
-
-u32 mh_detected;
-u32 mh_pauses_sent;
-
-static u32 nes_set_pau(struct nes_device *nesdev)
-{
- u32 ret = 0;
- u32 counter;
-
- nes_write_indexed(nesdev, NES_IDX_GPR2, NES_ENABLE_PAU);
- nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1);
-
- for (counter = 0; counter < NES_PAU_COUNTER; counter++) {
- udelay(30);
- if (!nes_read_indexed(nesdev, NES_IDX_GPR2)) {
- printk(KERN_INFO PFX "PAU is supported.\n");
- break;
- }
- nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1);
- }
- if (counter == NES_PAU_COUNTER) {
- printk(KERN_INFO PFX "PAU is not supported.\n");
- return -EPERM;
- }
- return ret;
-}
-
-/**
- * nes_read_eeprom_values -
- */
-int nes_read_eeprom_values(struct nes_device *nesdev, struct nes_adapter *nesadapter)
-{
- u32 mac_addr_low;
- u16 mac_addr_high;
- u16 eeprom_data;
- u16 eeprom_offset;
- u16 next_section_address;
- u16 sw_section_ver;
- u8 major_ver = 0;
- u8 minor_ver = 0;
-
- /* TODO: deal with EEPROM endian issues */
- if (nesadapter->firmware_eeprom_offset == 0) {
- /* Read the EEPROM Parameters */
- eeprom_data = nes_read16_eeprom(nesdev->regs, 0);
- nes_debug(NES_DBG_HW, "EEPROM Offset 0 = 0x%04X\n", eeprom_data);
- eeprom_offset = 2 + (((eeprom_data & 0x007f) << 3) <<
- ((eeprom_data & 0x0080) >> 7));
- nes_debug(NES_DBG_HW, "Firmware Offset = 0x%04X\n", eeprom_offset);
- nesadapter->firmware_eeprom_offset = eeprom_offset;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4);
- if (eeprom_data != 0x5746) {
- nes_debug(NES_DBG_HW, "Not a valid Firmware Image = 0x%04X\n", eeprom_data);
- return -1;
- }
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- eeprom_offset += ((eeprom_data & 0x00ff) << 3) << ((eeprom_data & 0x0100) >> 8);
- nes_debug(NES_DBG_HW, "Software Offset = 0x%04X\n", eeprom_offset);
- nesadapter->software_eeprom_offset = eeprom_offset;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4);
- if (eeprom_data != 0x5753) {
- printk("Not a valid Software Image = 0x%04X\n", eeprom_data);
- return -1;
- }
- sw_section_ver = nes_read16_eeprom(nesdev->regs, nesadapter->software_eeprom_offset + 6);
- nes_debug(NES_DBG_HW, "Software section version number = 0x%04X\n",
- sw_section_ver);
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) <<
- ((eeprom_data & 0x0100) >> 8));
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x414d) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_offset = next_section_address;
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) <<
- ((eeprom_data & 0x0100) >> 8));
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x4f52) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x4f52 but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_offset = next_section_address;
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3);
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x5746) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5746 but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_offset = next_section_address;
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3);
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x5753) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5753 but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_offset = next_section_address;
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3);
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x414d) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_offset = next_section_address;
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2);
- nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n",
- eeprom_offset + 2, eeprom_data);
- next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3);
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4);
- if (eeprom_data != 0x464e) {
- nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x464e but was 0x%04X\n",
- eeprom_data);
- goto no_fw_rev;
- }
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 8);
- printk(PFX "Firmware version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data);
- major_ver = (u8)(eeprom_data >> 8);
- minor_ver = (u8)(eeprom_data);
-
- if (nes_drv_opt & NES_DRV_OPT_DISABLE_VIRT_WQ) {
- nes_debug(NES_DBG_HW, "Virtual WQs have been disabled\n");
- } else if (((major_ver == 2) && (minor_ver > 21)) || ((major_ver > 2) && (major_ver != 255))) {
- nesadapter->virtwq = 1;
- }
- if (((major_ver == 3) && (minor_ver >= 16)) || (major_ver > 3))
- nesadapter->send_term_ok = 1;
-
- if (nes_drv_opt & NES_DRV_OPT_ENABLE_PAU) {
- if (!nes_set_pau(nesdev))
- nesadapter->allow_unaligned_fpdus = 1;
- }
-
- nesadapter->firmware_version = (((u32)(u8)(eeprom_data>>8)) << 16) +
- (u32)((u8)eeprom_data);
-
- eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 10);
- printk(PFX "EEPROM version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data);
- nesadapter->eeprom_version = (((u32)(u8)(eeprom_data>>8)) << 16) +
- (u32)((u8)eeprom_data);
-
-no_fw_rev:
- /* eeprom is valid */
- eeprom_offset = nesadapter->software_eeprom_offset;
- eeprom_offset += 8;
- nesadapter->netdev_max = (u8)nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- mac_addr_high = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- mac_addr_low = (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- mac_addr_low <<= 16;
- mac_addr_low += (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "Base MAC Address = 0x%04X%08X\n",
- mac_addr_high, mac_addr_low);
- nes_debug(NES_DBG_HW, "MAC Address count = %u\n", nesadapter->netdev_max);
-
- nesadapter->mac_addr_low = mac_addr_low;
- nesadapter->mac_addr_high = mac_addr_high;
-
- /* Read the Phy Type array */
- eeprom_offset += 10;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->phy_type[0] = (u8)(eeprom_data >> 8);
- nesadapter->phy_type[1] = (u8)eeprom_data;
-
- /* Read the port array */
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->phy_type[2] = (u8)(eeprom_data >> 8);
- nesadapter->phy_type[3] = (u8)eeprom_data;
- /* port_count is set by soft reset reg */
- nes_debug(NES_DBG_HW, "port_count = %u, port 0 -> %u, port 1 -> %u,"
- " port 2 -> %u, port 3 -> %u\n",
- nesadapter->port_count,
- nesadapter->phy_type[0], nesadapter->phy_type[1],
- nesadapter->phy_type[2], nesadapter->phy_type[3]);
-
- /* Read PD config array */
- eeprom_offset += 10;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_size[0] = eeprom_data;
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_base[0] = eeprom_data;
- nes_debug(NES_DBG_HW, "PD0 config, size=0x%04x, base=0x%04x\n",
- nesadapter->pd_config_size[0], nesadapter->pd_config_base[0]);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_size[1] = eeprom_data;
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_base[1] = eeprom_data;
- nes_debug(NES_DBG_HW, "PD1 config, size=0x%04x, base=0x%04x\n",
- nesadapter->pd_config_size[1], nesadapter->pd_config_base[1]);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_size[2] = eeprom_data;
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_base[2] = eeprom_data;
- nes_debug(NES_DBG_HW, "PD2 config, size=0x%04x, base=0x%04x\n",
- nesadapter->pd_config_size[2], nesadapter->pd_config_base[2]);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_size[3] = eeprom_data;
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->pd_config_base[3] = eeprom_data;
- nes_debug(NES_DBG_HW, "PD3 config, size=0x%04x, base=0x%04x\n",
- nesadapter->pd_config_size[3], nesadapter->pd_config_base[3]);
-
- /* Read Rx Pool Size */
- eeprom_offset += 22; /* 46 */
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->rx_pool_size = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "rx_pool_size = 0x%08X\n", nesadapter->rx_pool_size);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->tx_pool_size = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "tx_pool_size = 0x%08X\n", nesadapter->tx_pool_size);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->rx_threshold = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "rx_threshold = 0x%08X\n", nesadapter->rx_threshold);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->tcp_timer_core_clk_divisor = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "tcp_timer_core_clk_divisor = 0x%08X\n",
- nesadapter->tcp_timer_core_clk_divisor);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->iwarp_config = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "iwarp_config = 0x%08X\n", nesadapter->iwarp_config);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->cm_config = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "cm_config = 0x%08X\n", nesadapter->cm_config);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->sws_timer_config = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "sws_timer_config = 0x%08X\n", nesadapter->sws_timer_config);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->tcp_config1 = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "tcp_config1 = 0x%08X\n", nesadapter->tcp_config1);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->wqm_wat = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "wqm_wat = 0x%08X\n", nesadapter->wqm_wat);
-
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- eeprom_offset += 2;
- nesadapter->core_clock = (((u32)eeprom_data) << 16) +
- nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nes_debug(NES_DBG_HW, "core_clock = 0x%08X\n", nesadapter->core_clock);
-
- if ((sw_section_ver) && (nesadapter->hw_rev != NE020_REV)) {
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->phy_index[0] = (eeprom_data & 0xff00)>>8;
- nesadapter->phy_index[1] = eeprom_data & 0x00ff;
- eeprom_offset += 2;
- eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset);
- nesadapter->phy_index[2] = (eeprom_data & 0xff00)>>8;
- nesadapter->phy_index[3] = eeprom_data & 0x00ff;
- } else {
- nesadapter->phy_index[0] = 4;
- nesadapter->phy_index[1] = 5;
- nesadapter->phy_index[2] = 6;
- nesadapter->phy_index[3] = 7;
- }
- nes_debug(NES_DBG_HW, "Phy address map = 0 > %u, 1 > %u, 2 > %u, 3 > %u\n",
- nesadapter->phy_index[0],nesadapter->phy_index[1],
- nesadapter->phy_index[2],nesadapter->phy_index[3]);
- }
-
- return 0;
-}
-
-
-/**
- * nes_read16_eeprom
- */
-static u16 nes_read16_eeprom(void __iomem *addr, u16 offset)
-{
- writel(NES_EEPROM_READ_REQUEST + (offset >> 1),
- (void __iomem *)addr + NES_EEPROM_COMMAND);
-
- do {
- } while (readl((void __iomem *)addr + NES_EEPROM_COMMAND) &
- NES_EEPROM_READ_REQUEST);
-
- return readw((void __iomem *)addr + NES_EEPROM_DATA);
-}
-
-
-/**
- * nes_write_1G_phy_reg
- */
-void nes_write_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 data)
-{
- u32 u32temp;
- u32 counter;
-
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x50020000 | data | ((u32)phy_reg << 18) | ((u32)phy_addr << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1))
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
-}
-
-
-/**
- * nes_read_1G_phy_reg
- * This routine only issues the read, the data must be read
- * separately.
- */
-void nes_read_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 *data)
-{
- u32 u32temp;
- u32 counter;
-
- /* nes_debug(NES_DBG_PHY, "phy addr = %d, mac_index = %d\n",
- phy_addr, nesdev->mac_index); */
-
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x60020000 | ((u32)phy_reg << 18) | ((u32)phy_addr << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1)) {
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
- *data = 0xffff;
- } else {
- *data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
- }
-}
-
-
-/**
- * nes_write_10G_phy_reg
- */
-void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_addr, u8 dev_addr, u16 phy_reg,
- u16 data)
-{
- u32 port_addr;
- u32 u32temp;
- u32 counter;
-
- port_addr = phy_addr;
-
- /* set address */
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1))
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
-
- /* set data */
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x10020000 | (u32)data | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1))
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
-}
-
-
-/**
- * nes_read_10G_phy_reg
- * This routine only issues the read, the data must be read
- * separately.
- */
-void nes_read_10G_phy_reg(struct nes_device *nesdev, u8 phy_addr, u8 dev_addr, u16 phy_reg)
-{
- u32 port_addr;
- u32 u32temp;
- u32 counter;
-
- port_addr = phy_addr;
-
- /* set address */
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1))
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
-
- /* issue read */
- nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL,
- 0x30020000 | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23));
- for (counter = 0; counter < 100 ; counter++) {
- udelay(30);
- u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS);
- if (u32temp & 1) {
- nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1);
- break;
- }
- }
- if (!(u32temp & 1))
- nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n",
- u32temp);
-}
-
-
-/**
- * nes_get_cqp_request
- */
-struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev)
-{
- unsigned long flags;
- struct nes_cqp_request *cqp_request = NULL;
-
- if (!list_empty(&nesdev->cqp_avail_reqs)) {
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- if (!list_empty(&nesdev->cqp_avail_reqs)) {
- cqp_request = list_entry(nesdev->cqp_avail_reqs.next,
- struct nes_cqp_request, list);
- list_del_init(&cqp_request->list);
- }
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- }
- if (cqp_request == NULL) {
- cqp_request = kzalloc(sizeof(struct nes_cqp_request), GFP_ATOMIC);
- if (cqp_request) {
- cqp_request->dynamic = 1;
- INIT_LIST_HEAD(&cqp_request->list);
- }
- }
-
- if (cqp_request) {
- init_waitqueue_head(&cqp_request->waitq);
- cqp_request->waiting = 0;
- cqp_request->request_done = 0;
- cqp_request->callback = 0;
- init_waitqueue_head(&cqp_request->waitq);
- nes_debug(NES_DBG_CQP, "Got cqp request %p from the available list \n",
- cqp_request);
- } else
- printk(KERN_ERR PFX "%s: Could not allocated a CQP request.\n",
- __func__);
-
- return cqp_request;
-}
-
-void nes_free_cqp_request(struct nes_device *nesdev,
- struct nes_cqp_request *cqp_request)
-{
- unsigned long flags;
-
- nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
- cqp_request,
- le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX]) & 0x3f);
-
- if (cqp_request->dynamic) {
- kfree(cqp_request);
- } else {
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
- }
-}
-
-void nes_put_cqp_request(struct nes_device *nesdev,
- struct nes_cqp_request *cqp_request)
-{
- if (atomic_dec_and_test(&cqp_request->refcount))
- nes_free_cqp_request(nesdev, cqp_request);
-}
-
-
-/**
- * nes_post_cqp_request
- */
-void nes_post_cqp_request(struct nes_device *nesdev,
- struct nes_cqp_request *cqp_request)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- unsigned long flags;
- u32 cqp_head;
- u64 u64temp;
- u32 opcode;
- int ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
-
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
-
- if (((((nesdev->cqp.sq_tail+(nesdev->cqp.sq_size*2))-nesdev->cqp.sq_head) &
- (nesdev->cqp.sq_size - 1)) != 1)
- && (list_empty(&nesdev->cqp_pending_reqs))) {
- cqp_head = nesdev->cqp.sq_head++;
- nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
- cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
- memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
- opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]);
- if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
- ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
- barrier();
- u64temp = (unsigned long)cqp_request;
- set_wqe_64bit_value(cqp_wqe->wqe_words, ctx_index, u64temp);
- nes_debug(NES_DBG_CQP, "CQP request (opcode 0x%02X), line 1 = 0x%08X put on CQPs SQ,"
- " request = %p, cqp_head = %u, cqp_tail = %u, cqp_size = %u,"
- " waiting = %d, refcount = %d.\n",
- opcode & NES_CQP_OPCODE_MASK,
- le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX]), cqp_request,
- nesdev->cqp.sq_head, nesdev->cqp.sq_tail, nesdev->cqp.sq_size,
- cqp_request->waiting, atomic_read(&cqp_request->refcount));
-
- barrier();
-
- /* Ring doorbell (1 WQEs) */
- nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
-
- barrier();
- } else {
- nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X), line 1 = 0x%08X"
- " put on the pending queue.\n",
- cqp_request,
- le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f,
- le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_ID_IDX]));
- list_add_tail(&cqp_request->list, &nesdev->cqp_pending_reqs);
- }
-
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
-
- return;
-}
-
-/**
- * nes_arp_table
- */
-int nes_arp_table(struct nes_device *nesdev, u32 ip_addr, u8 *mac_addr, u32 action)
-{
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- int arp_index;
- int err = 0;
- __be32 tmp_addr;
-
- for (arp_index = 0; (u32) arp_index < nesadapter->arp_table_size; arp_index++) {
- if (nesadapter->arp_table[arp_index].ip_addr == ip_addr)
- break;
- }
-
- if (action == NES_ARP_ADD) {
- if (arp_index != nesadapter->arp_table_size) {
- return -1;
- }
-
- arp_index = 0;
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_arps,
- nesadapter->arp_table_size, (u32 *)&arp_index, &nesadapter->next_arp_index, NES_RESOURCE_ARP);
- if (err) {
- nes_debug(NES_DBG_NETDEV, "nes_alloc_resource returned error = %u\n", err);
- return err;
- }
- nes_debug(NES_DBG_NETDEV, "ADD, arp_index=%d\n", arp_index);
-
- nesadapter->arp_table[arp_index].ip_addr = ip_addr;
- memcpy(nesadapter->arp_table[arp_index].mac_addr, mac_addr, ETH_ALEN);
- return arp_index;
- }
-
- /* DELETE or RESOLVE */
- if (arp_index == nesadapter->arp_table_size) {
- tmp_addr = cpu_to_be32(ip_addr);
- nes_debug(NES_DBG_NETDEV, "MAC for %pI4 not in ARP table - cannot %s\n",
- &tmp_addr, action == NES_ARP_RESOLVE ? "resolve" : "delete");
- return -1;
- }
-
- if (action == NES_ARP_RESOLVE) {
- nes_debug(NES_DBG_NETDEV, "RESOLVE, arp_index=%d\n", arp_index);
- return arp_index;
- }
-
- if (action == NES_ARP_DELETE) {
- nes_debug(NES_DBG_NETDEV, "DELETE, arp_index=%d\n", arp_index);
- nesadapter->arp_table[arp_index].ip_addr = 0;
- eth_zero_addr(nesadapter->arp_table[arp_index].mac_addr);
- nes_free_resource(nesadapter, nesadapter->allocated_arps, arp_index);
- return arp_index;
- }
-
- return -1;
-}
-
-
-/**
- * nes_mh_fix
- */
-void nes_mh_fix(struct timer_list *t)
-{
- struct nes_adapter *nesadapter = from_timer(nesadapter, t, mh_timer);
- struct nes_device *nesdev = nesadapter->nesdev;
- unsigned long flags;
- struct nes_vnic *nesvnic;
- u32 used_chunks_tx;
- u32 temp_used_chunks_tx;
- u32 temp_last_used_chunks_tx;
- u32 used_chunks_mask;
- u32 mac_tx_frames_low;
- u32 mac_tx_frames_high;
- u32 mac_tx_pauses;
- u32 reset_value;
- u32 tx_control;
- u32 tx_config;
- u32 tx_pause_quanta;
- u32 rx_control;
- u32 rx_config;
- u32 mac_exact_match;
- u32 mpp_debug;
- u32 i=0;
- u32 chunks_tx_progress = 0;
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- if ((nesadapter->mac_sw_state[0] != NES_MAC_SW_IDLE) || (nesadapter->mac_link_down[0])) {
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- goto no_mh_work;
- }
- nesadapter->mac_sw_state[0] = NES_MAC_SW_MH;
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
- do {
- mac_tx_frames_low = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_LOW);
- mac_tx_frames_high = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_HIGH);
- mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES);
- used_chunks_tx = nes_read_indexed(nesdev, NES_IDX_USED_CHUNKS_TX);
- nesdev->mac_pause_frames_sent += mac_tx_pauses;
- used_chunks_mask = 0;
- temp_used_chunks_tx = used_chunks_tx;
- temp_last_used_chunks_tx = nesdev->last_used_chunks_tx;
-
- if (nesdev->netdev[0]) {
- nesvnic = netdev_priv(nesdev->netdev[0]);
- } else {
- break;
- }
-
- for (i=0; i<4; i++) {
- used_chunks_mask <<= 8;
- if (nesvnic->qp_nic_index[i] != 0xff) {
- used_chunks_mask |= 0xff;
- if ((temp_used_chunks_tx&0xff)<(temp_last_used_chunks_tx&0xff)) {
- chunks_tx_progress = 1;
- }
- }
- temp_used_chunks_tx >>= 8;
- temp_last_used_chunks_tx >>= 8;
- }
- if ((mac_tx_frames_low) || (mac_tx_frames_high) ||
- (!(used_chunks_tx&used_chunks_mask)) ||
- (!(nesdev->last_used_chunks_tx&used_chunks_mask)) ||
- (chunks_tx_progress) ) {
- nesdev->last_used_chunks_tx = used_chunks_tx;
- break;
- }
- nesdev->last_used_chunks_tx = used_chunks_tx;
- barrier();
-
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000005);
- mh_pauses_sent++;
- mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES);
- if (mac_tx_pauses) {
- nesdev->mac_pause_frames_sent += mac_tx_pauses;
- break;
- }
-
- tx_control = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONTROL);
- tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
- tx_pause_quanta = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA);
- rx_control = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONTROL);
- rx_config = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONFIG);
- mac_exact_match = nes_read_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM);
- mpp_debug = nes_read_indexed(nesdev, NES_IDX_MPP_DEBUG);
-
- /* one last ditch effort to avoid a false positive */
- mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES);
- if (mac_tx_pauses) {
- nesdev->last_mac_tx_pauses = nesdev->mac_pause_frames_sent;
- nes_debug(NES_DBG_HW, "failsafe caught slow outbound pause\n");
- break;
- }
- mh_detected++;
-
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, 0x00000000);
- reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
-
- nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value | 0x0000001d);
-
- while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
- & 0x00000040) != 0x00000040) && (i++ < 5000)) {
- /* mdelay(1); */
- }
-
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
- nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0);
-
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
- if (nesadapter->OneG_Mode) {
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
- } else {
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
- }
- nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_STATUS0);
- nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
-
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, tx_control);
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
- nes_write_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA, tx_pause_quanta);
- nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONTROL, rx_control);
- nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONFIG, rx_config);
- nes_write_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM, mac_exact_match);
- nes_write_indexed(nesdev, NES_IDX_MPP_DEBUG, mpp_debug);
-
- } while (0);
-
- nesadapter->mac_sw_state[0] = NES_MAC_SW_IDLE;
-no_mh_work:
- nesdev->nesadapter->mh_timer.expires = jiffies + (HZ/5);
- add_timer(&nesdev->nesadapter->mh_timer);
-}
-
-/**
- * nes_clc
- */
-void nes_clc(struct timer_list *t)
-{
- struct nes_adapter *nesadapter = from_timer(nesadapter, t, lc_timer);
- unsigned long flags;
-
- spin_lock_irqsave(&nesadapter->phy_lock, flags);
- nesadapter->link_interrupt_count[0] = 0;
- nesadapter->link_interrupt_count[1] = 0;
- nesadapter->link_interrupt_count[2] = 0;
- nesadapter->link_interrupt_count[3] = 0;
- spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
-
- nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
- add_timer(&nesadapter->lc_timer);
-}
-
-
-/**
- * nes_dump_mem
- */
-void nes_dump_mem(unsigned int dump_debug_level, void *addr, int length)
-{
- if (!(nes_debug_level & dump_debug_level)) {
- return;
- }
-
- if (length > 0x100) {
- nes_debug(dump_debug_level, "Length truncated from %x to %x\n", length, 0x100);
- length = 0x100;
- }
- nes_debug(dump_debug_level, "Address=0x%p, length=0x%x (%d)\n", addr, length, length);
-
- print_hex_dump(KERN_ERR, PFX, DUMP_PREFIX_NONE, 16, 1, addr, length, true);
-}
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
deleted file mode 100644
index 49024326a518..000000000000
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ /dev/null
@@ -1,3759 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/random.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>
-#include <asm/byteorder.h>
-
-#include <rdma/ib_verbs.h>
-#include <rdma/iw_cm.h>
-#include <rdma/ib_user_verbs.h>
-#include <rdma/uverbs_ioctl.h>
-
-#include "nes.h"
-
-#include <rdma/ib_umem.h>
-
-atomic_t mod_qp_timouts;
-atomic_t qps_created;
-atomic_t sw_qps_destroyed;
-
-static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
-static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
-
-/**
- * nes_alloc_mw
- */
-static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type,
- struct ib_udata *udata)
-{
- struct nes_pd *nespd = to_nespd(ibpd);
- struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_cqp_request *cqp_request;
- struct nes_mr *nesmr;
- struct ib_mw *ibmw;
- struct nes_hw_cqp_wqe *cqp_wqe;
- int ret;
- u32 stag;
- u32 stag_index = 0;
- u32 next_stag_index = 0;
- u32 driver_key = 0;
- u8 stag_key = 0;
-
- if (type != IB_MW_TYPE_1)
- return ERR_PTR(-EINVAL);
-
- get_random_bytes(&next_stag_index, sizeof(next_stag_index));
- stag_key = (u8)next_stag_index;
-
- driver_key = 0;
-
- next_stag_index >>= 8;
- next_stag_index %= nesadapter->max_mr;
-
- ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
- nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW);
- if (ret) {
- return ERR_PTR(ret);
- }
-
- nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
- if (!nesmr) {
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
-
- stag = stag_index << 8;
- stag |= driver_key;
- stag += (u32)stag_key;
-
- nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n",
- stag, stag_index);
-
- /* Register the region with the adapter */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- kfree(nesmr);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
-
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
- cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ |
- NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO |
- NES_CQP_STAG_REM_ACC_EN);
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff));
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- stag, ret, cqp_request->major_code, cqp_request->minor_code);
- if ((!ret) || (cqp_request->major_code)) {
- nes_put_cqp_request(nesdev, cqp_request);
- kfree(nesmr);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- if (!ret) {
- return ERR_PTR(-ETIME);
- } else {
- return ERR_PTR(-ENOMEM);
- }
- }
- nes_put_cqp_request(nesdev, cqp_request);
-
- nesmr->ibmw.rkey = stag;
- nesmr->mode = IWNES_MEMREG_TYPE_MW;
- ibmw = &nesmr->ibmw;
- nesmr->pbl_4k = 0;
- nesmr->pbls_used = 0;
-
- return ibmw;
-}
-
-
-/**
- * nes_dealloc_mw
- */
-static int nes_dealloc_mw(struct ib_mw *ibmw)
-{
- struct nes_mr *nesmr = to_nesmw(ibmw);
- struct nes_vnic *nesvnic = to_nesvnic(ibmw->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- int err = 0;
- int ret;
-
- /* Deallocate the window with the adapter */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n",
- ibmw->rkey);
- ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- ret, cqp_request->major_code, cqp_request->minor_code);
- if (!ret)
- err = -ETIME;
- else if (cqp_request->major_code)
- err = -EIO;
-
- nes_put_cqp_request(nesdev, cqp_request);
-
- nes_free_resource(nesadapter, nesadapter->allocated_mrs,
- (ibmw->rkey & 0x0fffff00) >> 8);
- kfree(nesmr);
-
- return err;
-}
-
-
-/*
- * nes_alloc_fast_mr
- */
-static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
- u32 stag, u32 page_count)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- unsigned long flags;
- int ret;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 opcode = 0;
- u16 major_code;
- u64 region_length = page_count * PAGE_SIZE;
-
-
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, "
- "region_length = %llu\n",
- page_count, region_length);
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (nesadapter->free_4kpbl > 0) {
- nesadapter->free_4kpbl--;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- } else {
- /* No 4kpbl's available: */
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_debug(NES_DBG_MR, "Out of Pbls\n");
- nes_free_cqp_request(nesdev, cqp_request);
- return -ENOMEM;
- }
-
- opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR |
- NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO |
- NES_CQP_STAG_REM_ACC_EN;
- /*
- * The current OFED API does not support the zero based TO option.
- * If added then need to changed the NES_CQP_STAG_VA* option. Also,
- * the API does not support that ability to have the MR set for local
- * access only when created and not allow the SQ op to override. Given
- * this the remote enable must be set here.
- */
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1);
-
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
- cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
- cpu_to_le32(nespd->pd_id & 0x00007fff);
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0);
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8));
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
- barrier();
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- ret = wait_event_timeout(cqp_request->waitq,
- (0 != cqp_request->request_done),
- NES_EVENT_TIMEOUT);
-
- nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, "
- "wait_event_timeout ret = %u, CQP Major:Minor codes = "
- "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code,
- cqp_request->minor_code);
- major_code = cqp_request->major_code;
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (!ret || major_code) {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_4kpbl++;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- }
-
- if (!ret)
- return -ETIME;
- else if (major_code)
- return -EIO;
- return 0;
-}
-
-/*
- * nes_alloc_mr
- */
-static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
-{
- struct nes_pd *nespd = to_nespd(ibpd);
- struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- u32 next_stag_index;
- u8 stag_key = 0;
- u32 driver_key = 0;
- int err = 0;
- u32 stag_index = 0;
- struct nes_mr *nesmr;
- u32 stag;
- int ret;
- struct ib_mr *ibmr;
-
- if (mr_type != IB_MR_TYPE_MEM_REG)
- return ERR_PTR(-EINVAL);
-
- if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
- return ERR_PTR(-E2BIG);
-
-/*
- * Note: Set to always use a fixed length single page entry PBL. This is to allow
- * for the fast_reg_mr operation to always know the size of the PBL.
- */
- if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
- return ERR_PTR(-E2BIG);
-
- get_random_bytes(&next_stag_index, sizeof(next_stag_index));
- stag_key = (u8)next_stag_index;
- next_stag_index >>= 8;
- next_stag_index %= nesadapter->max_mr;
-
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
- nesadapter->max_mr, &stag_index,
- &next_stag_index, NES_RESOURCE_FAST_MR);
- if (err)
- return ERR_PTR(err);
-
- nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
- if (!nesmr) {
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
-
- stag = stag_index << 8;
- stag |= driver_key;
- stag += (u32)stag_key;
-
- nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n",
- stag, stag_index);
-
- ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg);
-
- if (ret == 0) {
- nesmr->ibmr.rkey = stag;
- nesmr->ibmr.lkey = stag;
- nesmr->mode = IWNES_MEMREG_TYPE_FMEM;
- ibmr = &nesmr->ibmr;
- } else {
- kfree(nesmr);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
-
- nesmr->pages = pci_alloc_consistent(nesdev->pcidev,
- max_num_sg * sizeof(u64),
- &nesmr->paddr);
- if (!nesmr->paddr)
- goto err;
-
- nesmr->max_pages = max_num_sg;
-
- return ibmr;
-
-err:
- nes_dereg_mr(ibmr, udata);
-
- return ERR_PTR(-ENOMEM);
-}
-
-static int nes_set_page(struct ib_mr *ibmr, u64 addr)
-{
- struct nes_mr *nesmr = to_nesmr(ibmr);
-
- if (unlikely(nesmr->npages == nesmr->max_pages))
- return -ENOMEM;
-
- nesmr->pages[nesmr->npages++] = cpu_to_le64(addr);
-
- return 0;
-}
-
-static int nes_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
- int sg_nents, unsigned int *sg_offset)
-{
- struct nes_mr *nesmr = to_nesmr(ibmr);
-
- nesmr->npages = 0;
-
- return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, nes_set_page);
-}
-
-/**
- * nes_query_device
- */
-static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
- struct ib_udata *uhw)
-{
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_ib_device *nesibdev = nesvnic->nesibdev;
-
- if (uhw->inlen || uhw->outlen)
- return -EINVAL;
-
- memset(props, 0, sizeof(*props));
- memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6);
-
- props->fw_ver = nesdev->nesadapter->firmware_version;
- props->device_cap_flags = nesdev->nesadapter->device_cap_flags;
- props->vendor_id = nesdev->nesadapter->vendor_id;
- props->vendor_part_id = nesdev->nesadapter->vendor_part_id;
- props->hw_ver = nesdev->nesadapter->hw_rev;
- props->max_mr_size = 0x80000000;
- props->max_qp = nesibdev->max_qp;
- props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
- props->max_send_sge = nesdev->nesadapter->max_sge;
- props->max_recv_sge = nesdev->nesadapter->max_sge;
- props->max_cq = nesibdev->max_cq;
- props->max_cqe = nesdev->nesadapter->max_cqe;
- props->max_mr = nesibdev->max_mr;
- props->max_mw = nesibdev->max_mr;
- props->max_pd = nesibdev->max_pd;
- props->max_sge_rd = 1;
- switch (nesdev->nesadapter->max_irrq_wr) {
- case 0:
- props->max_qp_rd_atom = 2;
- break;
- case 1:
- props->max_qp_rd_atom = 8;
- break;
- case 2:
- props->max_qp_rd_atom = 32;
- break;
- case 3:
- props->max_qp_rd_atom = 64;
- break;
- default:
- props->max_qp_rd_atom = 0;
- }
- props->max_qp_init_rd_atom = props->max_qp_rd_atom;
- props->atomic_cap = IB_ATOMIC_NONE;
- props->max_map_per_fmr = 1;
-
- return 0;
-}
-
-
-/**
- * nes_query_port
- */
-static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props)
-{
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
- struct net_device *netdev = nesvnic->netdev;
-
- /* props being zeroed by the caller, avoid zeroing it here */
-
- props->max_mtu = IB_MTU_4096;
- props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
-
- props->lid = 1;
- if (netif_queue_stopped(netdev))
- props->state = IB_PORT_DOWN;
- else if (nesvnic->linkup)
- props->state = IB_PORT_ACTIVE;
- else
- props->state = IB_PORT_DOWN;
- props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
- IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
- props->gid_tbl_len = 1;
- props->pkey_tbl_len = 1;
- props->active_width = IB_WIDTH_4X;
- props->active_speed = IB_SPEED_SDR;
- props->max_msg_sz = 0x80000000;
-
- return 0;
-}
-
-/**
- * nes_query_pkey
- */
-static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
-{
- *pkey = 0;
- return 0;
-}
-
-
-/**
- * nes_query_gid
- */
-static int nes_query_gid(struct ib_device *ibdev, u8 port,
- int index, union ib_gid *gid)
-{
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-
- memset(&(gid->raw[0]), 0, sizeof(gid->raw));
- memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6);
-
- return 0;
-}
-
-
-/**
- * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
- * of all objects associated with a particular user-mode client.
- */
-static int nes_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
-{
- struct ib_device *ibdev = uctx->device;
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_alloc_ucontext_req req;
- struct nes_alloc_ucontext_resp uresp = {};
- struct nes_ucontext *nes_ucontext = to_nesucontext(uctx);
- struct nes_ib_device *nesibdev = nesvnic->nesibdev;
-
-
- if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) {
- printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n");
- return -EINVAL;
- }
-
- if (req.userspace_ver != NES_ABI_USERSPACE_VER) {
- printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n",
- req.userspace_ver, NES_ABI_USERSPACE_VER);
- return -EINVAL;
- }
-
-
- uresp.max_qps = nesibdev->max_qp;
- uresp.max_pds = nesibdev->max_pd;
- uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2;
- uresp.virtwq = nesadapter->virtwq;
- uresp.kernel_ver = NES_ABI_KERNEL_VER;
-
- nes_ucontext->nesdev = nesdev;
- nes_ucontext->mmap_wq_offset = uresp.max_pds;
- nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset +
- ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) /
- PAGE_SIZE;
-
-
- if (ib_copy_to_udata(udata, &uresp, sizeof(uresp)))
- return -EFAULT;
-
- INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list);
- INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list);
- return 0;
-}
-
-/**
- * nes_dealloc_ucontext
- */
-static void nes_dealloc_ucontext(struct ib_ucontext *context)
-{
- return;
-}
-
-/**
- * nes_mmap
- */
-static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
-{
- unsigned long index;
- struct nes_vnic *nesvnic = to_nesvnic(context->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- /* struct nes_adapter *nesadapter = nesdev->nesadapter; */
- struct nes_ucontext *nes_ucontext;
- struct nes_qp *nesqp;
-
- nes_ucontext = to_nesucontext(context);
-
-
- if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) {
- index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE;
- index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) +
- PAGE_SIZE-1) & (~(PAGE_SIZE-1));
- if (!test_bit(index, nes_ucontext->allocated_wqs)) {
- nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index);
- return -EFAULT;
- }
- nesqp = nes_ucontext->mmap_nesqp[index];
- if (nesqp == NULL) {
- nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index);
- return -EFAULT;
- }
- if (remap_pfn_range(vma, vma->vm_start,
- virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT,
- vma->vm_end - vma->vm_start,
- vma->vm_page_prot)) {
- nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n");
- return -EAGAIN;
- }
- vma->vm_private_data = nesqp;
- return 0;
- } else {
- index = vma->vm_pgoff;
- if (!test_bit(index, nes_ucontext->allocated_doorbells))
- return -EFAULT;
-
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
- if (io_remap_pfn_range(vma, vma->vm_start,
- (nesdev->doorbell_start +
- ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096))
- >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot))
- return -EAGAIN;
- vma->vm_private_data = nes_ucontext;
- return 0;
- }
-
- return -ENOSYS;
-}
-
-
-/**
- * nes_alloc_pd
- */
-static int nes_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
-{
- struct ib_device *ibdev = pd->device;
- struct nes_pd *nespd = to_nespd(pd);
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_alloc_pd_resp uresp;
- u32 pd_num = 0;
- int err;
- struct nes_ucontext *nesucontext = rdma_udata_to_drv_context(
- udata, struct nes_ucontext, ibucontext);
-
- nes_debug(
- NES_DBG_PD,
- "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
- nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev,
- &nesucontext->ibucontext, netdev_refcnt_read(nesvnic->netdev));
-
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds,
- nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD);
- if (err)
- return err;
-
- nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n",
- nespd, dev_name(&nesvnic->nesibdev->ibdev.dev));
-
- nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd;
-
- if (udata) {
- nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells,
- NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db);
- nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
- nespd->mmap_db_index, nespd->pd_id);
- if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) {
- nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n");
- nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
- return -ENOMEM;
- }
-
- uresp.pd_id = nespd->pd_id;
- uresp.mmap_db_index = nespd->mmap_db_index;
- if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) {
- nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
- return -EFAULT;
- }
-
- set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
- nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id;
- nesucontext->first_free_db = nespd->mmap_db_index + 1;
- }
-
- nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd);
- return 0;
-}
-
-
-/**
- * nes_dealloc_pd
- */
-static void nes_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
-{
- struct nes_ucontext *nesucontext;
- struct nes_pd *nespd = to_nespd(ibpd);
- struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
-
- if (udata) {
- nesucontext =
- rdma_udata_to_drv_context(
- udata,
- struct nes_ucontext,
- ibucontext);
- nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n",
- nespd->mmap_db_index);
- clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
- nesucontext->mmap_db_index[nespd->mmap_db_index] = 0;
- if (nesucontext->first_free_db > nespd->mmap_db_index) {
- nesucontext->first_free_db = nespd->mmap_db_index;
- }
- }
-
- nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n",
- nespd->pd_id, nespd);
- nes_free_resource(nesadapter, nesadapter->allocated_pds,
- (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12));
-}
-
-
-/**
- * nes_get_encoded_size
- */
-static inline u8 nes_get_encoded_size(int *size)
-{
- u8 encoded_size = 0;
- if (*size <= 32) {
- *size = 32;
- encoded_size = 1;
- } else if (*size <= 128) {
- *size = 128;
- encoded_size = 2;
- } else if (*size <= 512) {
- *size = 512;
- encoded_size = 3;
- }
- return (encoded_size);
-}
-
-
-
-/**
- * nes_setup_virt_qp
- */
-static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
- struct nes_vnic *nesvnic, int sq_size, int rq_size)
-{
- unsigned long flags;
- void *mem;
- __le64 *pbl = NULL;
- __le64 *tpbl;
- __le64 *pblbuffer;
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- u32 pbl_entries;
- u8 rq_pbl_entries;
- u8 sq_pbl_entries;
-
- pbl_entries = nespbl->pbl_size >> 3;
- nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
- nespbl->pbl_size, pbl_entries,
- (void *)nespbl->pbl_vbase,
- (unsigned long) nespbl->pbl_pbase);
- pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */
- /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
- /* the first pbl to be fro the rq_vbase... */
- rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
- sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
- nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
- if (!nespbl->page) {
- nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n");
- kfree(nespbl);
- return -ENOMEM;
- }
-
- nesqp->hwqp.sq_vbase = kmap(nespbl->page);
- nesqp->page = nespbl->page;
- if (!nesqp->hwqp.sq_vbase) {
- nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n");
- kfree(nespbl);
- return -ENOMEM;
- }
-
- /* Now to get to sq.. we need to calculate how many */
- /* PBL entries were used by the rq.. */
- pbl += sq_pbl_entries;
- nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
- /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
- /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
-
- nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
- nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,
- nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (!nesadapter->free_256pbl) {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
- nespbl->pbl_pbase);
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- kunmap(nesqp->page);
- kfree(nespbl);
- return -ENOMEM;
- }
- nesadapter->free_256pbl--;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-
- nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase);
- pblbuffer = nesqp->pbl_vbase;
- if (!nesqp->pbl_vbase) {
- /* memory allocated during nes_reg_user_mr() */
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
- nespbl->pbl_pbase);
- kfree(nespbl);
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_256pbl++;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- kunmap(nesqp->page);
- return -ENOMEM;
- }
- memset(nesqp->pbl_vbase, 0, 256);
- /* fill in the page address in the pbl buffer.. */
- tpbl = pblbuffer + 16;
- pbl = (__le64 *)nespbl->pbl_vbase;
- while (sq_pbl_entries--)
- *tpbl++ = *pbl++;
- tpbl = pblbuffer;
- while (rq_pbl_entries--)
- *tpbl++ = *pbl++;
-
- /* done with memory allocated during nes_reg_user_mr() */
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
- nespbl->pbl_pbase);
- kfree(nespbl);
-
- nesqp->qp_mem_size =
- max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */
- /* Round up to a multiple of a page */
- nesqp->qp_mem_size += PAGE_SIZE - 1;
- nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
-
- mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
- &nesqp->hwqp.q2_pbase);
-
- if (!mem) {
- pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);
- nesqp->pbl_vbase = NULL;
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_256pbl++;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- kunmap(nesqp->page);
- return -ENOMEM;
- }
- nesqp->sq_kmapped = 1;
- nesqp->hwqp.q2_vbase = mem;
- mem += 256;
- memset(nesqp->hwqp.q2_vbase, 0, 256);
- nesqp->nesqp_context = mem;
- memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
- nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
-
- return 0;
-}
-
-
-/**
- * nes_setup_mmap_qp
- */
-static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic,
- int sq_size, int rq_size)
-{
- void *mem;
- struct nes_device *nesdev = nesvnic->nesdev;
-
- nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) +
- (sizeof(struct nes_hw_qp_wqe) * rq_size) +
- max((u32)sizeof(struct nes_qp_context), ((u32)256)) +
- 256; /* this is Q2 */
- /* Round up to a multiple of a page */
- nesqp->qp_mem_size += PAGE_SIZE - 1;
- nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
-
- mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
- &nesqp->hwqp.sq_pbase);
- if (!mem)
- return -ENOMEM;
- nes_debug(NES_DBG_QP, "PCI consistent memory for "
- "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
- mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size);
-
- memset(mem, 0, nesqp->qp_mem_size);
-
- nesqp->hwqp.sq_vbase = mem;
- mem += sizeof(struct nes_hw_qp_wqe) * sq_size;
-
- nesqp->hwqp.rq_vbase = mem;
- nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase +
- sizeof(struct nes_hw_qp_wqe) * sq_size;
- mem += sizeof(struct nes_hw_qp_wqe) * rq_size;
-
- nesqp->hwqp.q2_vbase = mem;
- nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase +
- sizeof(struct nes_hw_qp_wqe) * rq_size;
- mem += 256;
- memset(nesqp->hwqp.q2_vbase, 0, 256);
-
- nesqp->nesqp_context = mem;
- nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
- memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
- return 0;
-}
-
-
-/**
- * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
- */
-static void nes_free_qp_mem(struct nes_device *nesdev,
- struct nes_qp *nesqp, int virt_wqs)
-{
- unsigned long flags;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- if (!virt_wqs) {
- pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,
- nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);
- }else {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_256pbl++;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
- pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );
- nesqp->pbl_vbase = NULL;
- if (nesqp->sq_kmapped) {
- nesqp->sq_kmapped = 0;
- kunmap(nesqp->page);
- }
- }
-}
-
-
-/**
- * nes_create_qp
- */
-static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
- struct ib_qp_init_attr *init_attr, struct ib_udata *udata)
-{
- u64 u64temp= 0;
- u64 u64nesqp = 0;
- struct nes_pd *nespd = to_nespd(ibpd);
- struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_qp *nesqp;
- struct nes_cq *nescq;
- struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context(
- udata, struct nes_ucontext, ibucontext);
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- struct nes_create_qp_req req;
- struct nes_create_qp_resp uresp;
- struct nes_pbl *nespbl = NULL;
- u32 qp_num = 0;
- u32 opcode = 0;
- /* u32 counter = 0; */
- void *mem;
- unsigned long flags;
- int ret;
- int err;
- int virt_wqs = 0;
- int sq_size;
- int rq_size;
- u8 sq_encoded_size;
- u8 rq_encoded_size;
- /* int counter; */
-
- if (init_attr->create_flags)
- return ERR_PTR(-EINVAL);
-
- atomic_inc(&qps_created);
- switch (init_attr->qp_type) {
- case IB_QPT_RC:
- if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) {
- init_attr->cap.max_inline_data = 0;
- } else {
- init_attr->cap.max_inline_data = 64;
- }
- sq_size = init_attr->cap.max_send_wr;
- rq_size = init_attr->cap.max_recv_wr;
-
- /* check if the encoded sizes are OK or not... */
- sq_encoded_size = nes_get_encoded_size(&sq_size);
- rq_encoded_size = nes_get_encoded_size(&rq_size);
-
- if ((!sq_encoded_size) || (!rq_encoded_size)) {
- nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n",
- rq_size, sq_size);
- return ERR_PTR(-EINVAL);
- }
-
- init_attr->cap.max_send_wr = sq_size -2;
- init_attr->cap.max_recv_wr = rq_size -1;
- nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size);
-
- ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps,
- nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP);
- if (ret) {
- return ERR_PTR(ret);
- }
-
- /* Need 512 (actually now 1024) byte alignment on this structure */
- mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL);
- if (!mem) {
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- return ERR_PTR(-ENOMEM);
- }
- u64nesqp = (unsigned long)mem;
- u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1;
- u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1;
- u64nesqp &= ~u64temp;
- nesqp = (struct nes_qp *)(unsigned long)u64nesqp;
- /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n",
- nesqp, mem, NES_SW_CONTEXT_ALIGN); */
- nesqp->allocated_buffer = mem;
-
- if (udata) {
- if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) {
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- kfree(nesqp->allocated_buffer);
- nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n");
- return ERR_PTR(-EFAULT);
- }
- if (req.user_wqe_buffers) {
- virt_wqs = 1;
- }
- if (req.user_qp_buffer)
- nesqp->nesuqp_addr = req.user_qp_buffer;
-
- nesqp->user_mode = 1;
- if (virt_wqs) {
- err = 1;
- list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) {
- if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) {
- list_del(&nespbl->list);
- err = 0;
- nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
- nespbl, nespbl->user_base);
- break;
- }
- }
- if (err) {
- nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n",
- (long long unsigned int)req.user_wqe_buffers);
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- kfree(nesqp->allocated_buffer);
- return ERR_PTR(-EFAULT);
- }
- }
-
- nesqp->mmap_sq_db_index =
- find_next_zero_bit(nes_ucontext->allocated_wqs,
- NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq);
- /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
- nespd->mmap_db_index); */
- if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {
- nes_debug(NES_DBG_QP,
- "db index > max user regions, failing create QP\n");
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- if (virt_wqs) {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
- nespbl->pbl_pbase);
- kfree(nespbl);
- }
- kfree(nesqp->allocated_buffer);
- return ERR_PTR(-ENOMEM);
- }
- set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
- nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;
- nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;
- }
- err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) :
- nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size);
- if (err) {
- nes_debug(NES_DBG_QP,
- "error geting qp mem code = %d\n", err);
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- kfree(nesqp->allocated_buffer);
- return ERR_PTR(-ENOMEM);
- }
-
- nesqp->hwqp.sq_size = sq_size;
- nesqp->hwqp.sq_encoded_size = sq_encoded_size;
- nesqp->hwqp.sq_head = 1;
- nesqp->hwqp.rq_size = rq_size;
- nesqp->hwqp.rq_encoded_size = rq_encoded_size;
- /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n",
- (void *)nesqp->nesqp_context_pbase);
- */
- nesqp->hwqp.qp_id = qp_num;
- nesqp->ibqp.qp_num = nesqp->hwqp.qp_id;
- nesqp->nespd = nespd;
-
- nescq = to_nescq(init_attr->send_cq);
- nesqp->nesscq = nescq;
- nescq = to_nescq(init_attr->recv_cq);
- nesqp->nesrcq = nescq;
-
- nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) <<
- NES_QPCONTEXT_MISC_PCI_FCN_SHIFT);
- nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size <<
- NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT);
- nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size <<
- NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT);
- if (!udata) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN);
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN);
- }
- nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number +
- ((u32)nesqp->nesrcq->hw_cq.cq_number << 16));
- u64temp = (u64)nesqp->hwqp.sq_pbase;
- nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);
- nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
-
-
- if (!virt_wqs) {
- u64temp = (u64)nesqp->hwqp.sq_pbase;
- nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);
- nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
- u64temp = (u64)nesqp->hwqp.rq_pbase;
- nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);
- nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
- } else {
- u64temp = (u64)nesqp->pbl_pbase;
- nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);
- nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
- }
-
- /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n",
- nesvnic->next_qp_nic_index,
- nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */
- spin_lock_irqsave(&nesdev->cqp.lock, flags);
- nesqp->nesqp_context->misc2 |= cpu_to_le32(
- (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] <<
- NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT);
- nesvnic->next_qp_nic_index++;
- if ((nesvnic->next_qp_nic_index > 3) ||
- (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) {
- nesvnic->next_qp_nic_index = 0;
- }
- spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
-
- nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16);
- u64temp = (u64)nesqp->hwqp.q2_pbase;
- nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp);
- nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32));
- nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp)));
- nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp))));
- nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM |
- NES_QPCONTEXT_ORDIRD_AAH |
- ((((u32)nesadapter->max_irrq_wr) <<
- NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK));
- if (disable_mpa_crc) {
- nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n");
- nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC);
- }
-
-
- /* Create the QP */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n");
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- nes_free_qp_mem(nesdev, nesqp,virt_wqs);
- kfree(nesqp->allocated_buffer);
- return ERR_PTR(-ENOMEM);
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- if (!virt_wqs) {
- opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP |
- NES_CQP_QP_IWARP_STATE_IDLE;
- } else {
- opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS |
- NES_CQP_QP_IWARP_STATE_IDLE;
- }
- opcode |= NES_CQP_QP_CQS_VALID;
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
-
- u64temp = (u64)nesqp->nesqp_context_pbase;
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n",
- nesqp->hwqp.qp_id);
- ret = wait_event_timeout(cqp_request->waitq,
- (cqp_request->request_done != 0), NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u,"
- " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail,
- cqp_request->major_code, cqp_request->minor_code);
- if ((!ret) || (cqp_request->major_code)) {
- nes_put_cqp_request(nesdev, cqp_request);
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- nes_free_qp_mem(nesdev, nesqp,virt_wqs);
- kfree(nesqp->allocated_buffer);
- if (!ret) {
- return ERR_PTR(-ETIME);
- } else {
- return ERR_PTR(-EIO);
- }
- }
-
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (udata) {
- uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index;
- uresp.mmap_rq_db_index = 0;
- uresp.actual_sq_size = sq_size;
- uresp.actual_rq_size = rq_size;
- uresp.qp_id = nesqp->hwqp.qp_id;
- uresp.nes_drv_opt = nes_drv_opt;
- if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
- nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
- nes_free_qp_mem(nesdev, nesqp,virt_wqs);
- kfree(nesqp->allocated_buffer);
- return ERR_PTR(-EFAULT);
- }
- }
-
- nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n",
- nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));
- spin_lock_init(&nesqp->lock);
- nes_add_ref(&nesqp->ibqp);
- break;
- default:
- nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type);
- return ERR_PTR(-EINVAL);
- }
- init_completion(&nesqp->sq_drained);
- init_completion(&nesqp->rq_drained);
-
- nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
- timer_setup(&nesqp->terminate_timer, nes_terminate_timeout, 0);
-
- /* update the QP table */
- nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;
- nes_debug(NES_DBG_QP, "netdev refcnt=%u\n",
- netdev_refcnt_read(nesvnic->netdev));
-
- return &nesqp->ibqp;
-}
-
-/**
- * nes_clean_cq
- */
-static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq)
-{
- u32 cq_head;
- u32 lo;
- u32 hi;
- u64 u64temp;
- unsigned long flags = 0;
-
- spin_lock_irqsave(&nescq->lock, flags);
-
- cq_head = nescq->hw_cq.cq_head;
- while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
- rmb();
- lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
- hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]);
- u64temp = (((u64)hi) << 32) | ((u64)lo);
- u64temp &= ~(NES_SW_CONTEXT_ALIGN-1);
- if (u64temp == (u64)(unsigned long)nesqp) {
- /* Zero the context value so cqe will be ignored */
- nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0;
- nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0;
- }
-
- if (++cq_head >= nescq->hw_cq.cq_size)
- cq_head = 0;
- }
-
- spin_unlock_irqrestore(&nescq->lock, flags);
-}
-
-
-/**
- * nes_destroy_qp
- */
-static int nes_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
-{
- struct nes_qp *nesqp = to_nesqp(ibqp);
- struct nes_ucontext *nes_ucontext;
- struct ib_qp_attr attr;
- struct iw_cm_id *cm_id;
- struct iw_cm_event cm_event;
- int ret = 0;
-
- atomic_inc(&sw_qps_destroyed);
- nesqp->destroyed = 1;
-
- /* Blow away the connection if it exists. */
- if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) {
- /* if (nesqp->ibqp_state == IB_QPS_RTS) { */
- attr.qp_state = IB_QPS_ERR;
- nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);
- }
-
- if (((nesqp->ibqp_state == IB_QPS_INIT) ||
- (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) {
- cm_id = nesqp->cm_id;
- cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
- cm_event.status = -ETIMEDOUT;
- cm_event.local_addr = cm_id->local_addr;
- cm_event.remote_addr = cm_id->remote_addr;
- cm_event.private_data = NULL;
- cm_event.private_data_len = 0;
-
- nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for "
- "QP%u. cm_id = %p, refcount = %u. \n",
- nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount));
-
- cm_id->rem_ref(cm_id);
- ret = cm_id->event_handler(cm_id, &cm_event);
- if (ret)
- nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret);
- }
-
- if (nesqp->user_mode) {
- if (udata) {
- nes_ucontext =
- rdma_udata_to_drv_context(
- udata,
- struct nes_ucontext,
- ibucontext);
- clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
- nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL;
- if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) {
- nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;
- }
- }
- if (nesqp->pbl_pbase && nesqp->sq_kmapped) {
- nesqp->sq_kmapped = 0;
- kunmap(nesqp->page);
- }
- } else {
- /* Clean any pending completions from the cq(s) */
- if (nesqp->nesscq)
- nes_clean_cq(nesqp, nesqp->nesscq);
-
- if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq))
- nes_clean_cq(nesqp, nesqp->nesrcq);
- }
- nes_rem_ref(&nesqp->ibqp);
- return 0;
-}
-
-
-/**
- * nes_create_cq
- */
-static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
-{
- int entries = attr->cqe;
- u64 u64temp;
- struct nes_vnic *nesvnic = to_nesvnic(ibdev);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_cq *nescq;
- struct nes_ucontext *nes_ucontext = NULL;
- struct nes_cqp_request *cqp_request;
- void *mem = NULL;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_pbl *nespbl = NULL;
- struct nes_create_cq_req req;
- struct nes_create_cq_resp resp;
- u32 cq_num = 0;
- u32 opcode = 0;
- u32 pbl_entries = 1;
- int err;
- unsigned long flags;
- int ret;
-
- if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- if (entries > nesadapter->max_cqe)
- return ERR_PTR(-EINVAL);
-
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs,
- nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ);
- if (err) {
- return ERR_PTR(err);
- }
-
- nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL);
- if (!nescq) {
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- return ERR_PTR(-ENOMEM);
- }
-
- nescq->hw_cq.cq_size = max(entries + 1, 5);
- nescq->hw_cq.cq_number = cq_num;
- nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1;
-
- if (udata) {
- struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context(
- udata, struct nes_ucontext, ibucontext);
-
- if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) {
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-EFAULT);
- }
- nesvnic->mcrq_ucontext = nes_ucontext;
- nes_ucontext->mcrqf = req.mcrqf;
- if (nes_ucontext->mcrqf) {
- if (nes_ucontext->mcrqf & 0x80000000)
- nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 28 + 2 * ((nes_ucontext->mcrqf & 0xf) - 1);
- else if (nes_ucontext->mcrqf & 0x40000000)
- nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff;
- else
- nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1;
- nescq->mcrqf = nes_ucontext->mcrqf;
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- }
- nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n",
- (unsigned long)req.user_cq_buffer, entries);
- err = 1;
- list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) {
- if (nespbl->user_base == (unsigned long )req.user_cq_buffer) {
- list_del(&nespbl->list);
- err = 0;
- nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n",
- nespbl);
- break;
- }
- }
- if (err) {
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-EFAULT);
- }
-
- pbl_entries = nespbl->pbl_size >> 3;
- nescq->cq_mem_size = 0;
- } else {
- nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe);
- nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n",
- entries, nescq->cq_mem_size, nescq->hw_cq.cq_number);
-
- /* allocate the physical buffer space */
- mem = pci_zalloc_consistent(nesdev->pcidev, nescq->cq_mem_size,
- &nescq->hw_cq.cq_pbase);
- if (!mem) {
- printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n");
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-ENOMEM);
- }
-
- nescq->hw_cq.cq_vbase = mem;
- nescq->hw_cq.cq_head = 0;
- nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n",
- nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase,
- (u32)nescq->hw_cq.cq_pbase);
- }
-
- nescq->hw_cq.ce_handler = nes_iwarp_ce_handler;
- spin_lock_init(&nescq->lock);
-
- /* send CreateCQ request to CQP */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n");
- if (!udata)
- pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
- nescq->hw_cq.cq_pbase);
- else {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
- nespbl->pbl_vbase, nespbl->pbl_pbase);
- kfree(nespbl);
- }
-
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-ENOMEM);
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
- NES_CQP_CQ_CHK_OVERFLOW |
- NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16);
-
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-
- if (pbl_entries != 1) {
- if (pbl_entries > 32) {
- /* use 4k pbl */
- nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries);
- if (nesadapter->free_4kpbl == 0) {
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_free_cqp_request(nesdev, cqp_request);
- if (!udata)
- pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
- nescq->hw_cq.cq_pbase);
- else {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
- nespbl->pbl_vbase, nespbl->pbl_pbase);
- kfree(nespbl);
- }
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-ENOMEM);
- } else {
- opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK);
- nescq->virtual_cq = 2;
- nesadapter->free_4kpbl--;
- }
- } else {
- /* use 256 byte pbl */
- nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries);
- if (nesadapter->free_256pbl == 0) {
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_free_cqp_request(nesdev, cqp_request);
- if (!udata)
- pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
- nescq->hw_cq.cq_pbase);
- else {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
- nespbl->pbl_vbase, nespbl->pbl_pbase);
- kfree(nespbl);
- }
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-ENOMEM);
- } else {
- opcode |= NES_CQP_CQ_VIRT;
- nescq->virtual_cq = 1;
- nesadapter->free_256pbl--;
- }
- }
- }
-
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16)));
-
- if (udata) {
- if (pbl_entries != 1)
- u64temp = (u64)nespbl->pbl_pbase;
- else
- u64temp = le64_to_cpu(nespbl->pbl_vbase[0]);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX,
- nes_ucontext->mmap_db_index[0]);
- } else {
- u64temp = (u64)nescq->hw_cq.cq_pbase;
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
- }
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
- u64temp = (u64)(unsigned long)&nescq->hw_cq;
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
- cpu_to_le32((u32)(u64temp >> 1));
- cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
- cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n",
- nescq->hw_cq.cq_number);
- ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
- NES_EVENT_TIMEOUT * 2);
- nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n",
- nescq->hw_cq.cq_number, ret);
- if ((!ret) || (cqp_request->major_code)) {
- nes_put_cqp_request(nesdev, cqp_request);
- if (!udata)
- pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
- nescq->hw_cq.cq_pbase);
- else {
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
- nespbl->pbl_vbase, nespbl->pbl_pbase);
- kfree(nespbl);
- }
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-EIO);
- }
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (udata) {
- /* free the nespbl */
- pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
- nespbl->pbl_pbase);
- kfree(nespbl);
- resp.cq_id = nescq->hw_cq.cq_number;
- resp.cq_size = nescq->hw_cq.cq_size;
- resp.mmap_db_index = 0;
- if (ib_copy_to_udata(udata, &resp, sizeof resp - sizeof resp.reserved)) {
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
- kfree(nescq);
- return ERR_PTR(-EFAULT);
- }
- }
-
- return &nescq->ibcq;
-}
-
-
-/**
- * nes_destroy_cq
- */
-static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
-{
- struct nes_cq *nescq;
- struct nes_device *nesdev;
- struct nes_vnic *nesvnic;
- struct nes_adapter *nesadapter;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- unsigned long flags;
- u32 opcode = 0;
- int ret;
-
- if (ib_cq == NULL)
- return 0;
-
- nescq = to_nescq(ib_cq);
- nesvnic = to_nesvnic(ib_cq->device);
- nesdev = nesvnic->nesdev;
- nesadapter = nesdev->nesadapter;
-
- nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number);
-
- /* Send DestroyCQ request to CQP */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
- opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16);
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (nescq->virtual_cq == 1) {
- nesadapter->free_256pbl++;
- if (nesadapter->free_256pbl > nesadapter->max_256pbl) {
- printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n",
- __func__, nesadapter->free_256pbl, nesadapter->max_256pbl);
- }
- } else if (nescq->virtual_cq == 2) {
- nesadapter->free_4kpbl++;
- if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) {
- printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n",
- __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl);
- }
- opcode |= NES_CQP_CQ_4KB_CHUNK;
- }
-
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
- (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16)));
- if (!nescq->mcrqf)
- nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n",
- nescq->hw_cq.cq_number);
- ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- nescq->hw_cq.cq_number, ret, cqp_request->major_code,
- cqp_request->minor_code);
- if (!ret) {
- nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n",
- nescq->hw_cq.cq_number);
- ret = -ETIME;
- } else if (cqp_request->major_code) {
- nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n",
- nescq->hw_cq.cq_number);
- ret = -EIO;
- } else {
- ret = 0;
- }
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (nescq->cq_mem_size)
- pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size,
- nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase);
- kfree(nescq);
-
- return ret;
-}
-
-/**
- * root_256
- */
-static u32 root_256(struct nes_device *nesdev,
- struct nes_root_vpbl *root_vpbl,
- struct nes_root_vpbl *new_root,
- u16 pbl_count_4k)
-{
- u64 leaf_pbl;
- int i, j, k;
-
- if (pbl_count_4k == 1) {
- new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
- 512, &new_root->pbl_pbase);
-
- if (new_root->pbl_vbase == NULL)
- return 0;
-
- leaf_pbl = (u64)root_vpbl->pbl_pbase;
- for (i = 0; i < 16; i++) {
- new_root->pbl_vbase[i].pa_low =
- cpu_to_le32((u32)leaf_pbl);
- new_root->pbl_vbase[i].pa_high =
- cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
- leaf_pbl += 256;
- }
- } else {
- for (i = 3; i >= 0; i--) {
- j = i * 16;
- root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i];
- leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) +
- (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high))
- << 32);
- for (k = 1; k < 16; k++) {
- leaf_pbl += 256;
- root_vpbl->pbl_vbase[j + k].pa_low =
- cpu_to_le32((u32)leaf_pbl);
- root_vpbl->pbl_vbase[j + k].pa_high =
- cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
- }
- }
- }
-
- return 1;
-}
-
-
-/**
- * nes_reg_mr
- */
-static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
- u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
- dma_addr_t single_buffer, u16 pbl_count_4k,
- u16 residual_page_count_4k, int acc, u64 *iova_start,
- u16 *actual_pbl_cnt, u8 *used_4k_pbls)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- unsigned long flags;
- int ret;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- uint pg_cnt = 0;
- u16 pbl_count_256 = 0;
- u16 pbl_count = 0;
- u8 use_256_pbls = 0;
- u8 use_4k_pbls = 0;
- u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0;
- struct nes_root_vpbl new_root = { 0, NULL, NULL };
- u32 opcode = 0;
- u16 major_code;
-
- /* Register the region with the adapter */
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- if (pbl_count_4k) {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-
- pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k;
- pbl_count_256 = (pg_cnt + 31) / 32;
- if (pg_cnt <= 32) {
- if (pbl_count_256 <= nesadapter->free_256pbl)
- use_256_pbls = 1;
- else if (pbl_count_4k <= nesadapter->free_4kpbl)
- use_4k_pbls = 1;
- } else if (pg_cnt <= 2048) {
- if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) &&
- (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) {
- use_4k_pbls = 1;
- } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) {
- use_256_pbls = 1;
- use_two_level = 1;
- } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
- use_4k_pbls = 1;
- }
- } else {
- if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl)
- use_4k_pbls = 1;
- }
-
- if (use_256_pbls) {
- pbl_count = pbl_count_256;
- nesadapter->free_256pbl -= pbl_count + use_two_level;
- } else if (use_4k_pbls) {
- pbl_count = pbl_count_4k;
- nesadapter->free_4kpbl -= pbl_count + use_two_level;
- } else {
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_debug(NES_DBG_MR, "Out of Pbls\n");
- nes_free_cqp_request(nesdev, cqp_request);
- return -ENOMEM;
- }
-
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- }
-
- if (use_256_pbls && use_two_level) {
- if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k) == 1) {
- if (new_root.pbl_pbase != 0)
- root_vpbl = &new_root;
- } else {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- nesadapter->free_256pbl += pbl_count_256 + use_two_level;
- use_256_pbls = 0;
-
- if (pbl_count_4k == 1)
- use_two_level = 0;
- pbl_count = pbl_count_4k;
-
- if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
- nesadapter->free_4kpbl -= pbl_count + use_two_level;
- use_4k_pbls = 1;
- }
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-
- if (use_4k_pbls == 0)
- return -ENOMEM;
- }
- }
-
- opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
- NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
- if (acc & IB_ACCESS_LOCAL_WRITE)
- opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE;
- if (acc & IB_ACCESS_REMOTE_WRITE)
- opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN;
- if (acc & IB_ACCESS_REMOTE_READ)
- opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN;
- if (acc & IB_ACCESS_MW_BIND)
- opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN;
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length);
-
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
- cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
- cpu_to_le32(nespd->pd_id & 0x00007fff);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
-
- if (pbl_count == 0) {
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer);
- } else {
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8));
-
- if (use_4k_pbls)
- cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
- }
- barrier();
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- stag, ret, cqp_request->major_code, cqp_request->minor_code);
- major_code = cqp_request->major_code;
- nes_put_cqp_request(nesdev, cqp_request);
-
- if ((!ret || major_code) && pbl_count != 0) {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (use_256_pbls)
- nesadapter->free_256pbl += pbl_count + use_two_level;
- else if (use_4k_pbls)
- nesadapter->free_4kpbl += pbl_count + use_two_level;
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- }
- if (new_root.pbl_pbase)
- pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase,
- new_root.pbl_pbase);
-
- if (!ret)
- return -ETIME;
- else if (major_code)
- return -EIO;
-
- *actual_pbl_cnt = pbl_count + use_two_level;
- *used_4k_pbls = use_4k_pbls;
- return 0;
-}
-
-
-/**
- * nes_reg_phys_mr
- */
-struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
- int acc, u64 *iova_start)
-{
- u64 region_length;
- struct nes_pd *nespd = to_nespd(ib_pd);
- struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_mr *nesmr;
- struct ib_mr *ibmr;
- struct nes_vpbl vpbl;
- struct nes_root_vpbl root_vpbl;
- u32 stag;
- unsigned long mask;
- u32 stag_index = 0;
- u32 next_stag_index = 0;
- u32 driver_key = 0;
- int err = 0;
- int ret = 0;
- u16 pbl_count = 0;
- u8 single_page = 1;
- u8 stag_key = 0;
-
- region_length = 0;
- vpbl.pbl_vbase = NULL;
- root_vpbl.pbl_vbase = NULL;
- root_vpbl.pbl_pbase = 0;
-
- get_random_bytes(&next_stag_index, sizeof(next_stag_index));
- stag_key = (u8)next_stag_index;
-
- driver_key = 0;
-
- next_stag_index >>= 8;
- next_stag_index %= nesadapter->max_mr;
-
- if ((addr ^ *iova_start) & ~PAGE_MASK)
- return ERR_PTR(-EINVAL);
-
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr,
- &stag_index, &next_stag_index, NES_RESOURCE_PHYS_MR);
- if (err) {
- return ERR_PTR(err);
- }
-
- nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
- if (!nesmr) {
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
-
- /* Allocate a 4K buffer for the PBL */
- vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
- &vpbl.pbl_pbase);
- nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
- vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase);
- if (!vpbl.pbl_vbase) {
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- ibmr = ERR_PTR(-ENOMEM);
- kfree(nesmr);
- goto reg_phys_err;
- }
-
-
- mask = !size;
-
- if (mask & ~PAGE_MASK) {
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n");
- ibmr = ERR_PTR(-EINVAL);
- kfree(nesmr);
- goto reg_phys_err;
- }
-
- region_length += size;
- vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)addr & PAGE_MASK);
- vpbl.pbl_vbase[0].pa_high = cpu_to_le32((u32)((((u64)addr) >> 32)));
-
- stag = stag_index << 8;
- stag |= driver_key;
- stag += (u32)stag_key;
-
- nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX,"
- " length = 0x%016lX, index = 0x%08X\n",
- stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index);
-
- /* Make the leaf PBL the root if only one PBL */
- root_vpbl.pbl_pbase = vpbl.pbl_pbase;
-
- if (single_page) {
- pbl_count = 0;
- } else {
- pbl_count = 1;
- }
- ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
- addr, pbl_count, 1, acc, iova_start,
- &nesmr->pbls_used, &nesmr->pbl_4k);
-
- if (ret == 0) {
- nesmr->ibmr.rkey = stag;
- nesmr->ibmr.lkey = stag;
- nesmr->mode = IWNES_MEMREG_TYPE_MEM;
- ibmr = &nesmr->ibmr;
- } else {
- kfree(nesmr);
- ibmr = ERR_PTR(-ENOMEM);
- }
-
-reg_phys_err:
- /* single PBL case */
- pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase);
- return ibmr;
-}
-
-
-/**
- * nes_get_dma_mr
- */
-static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc)
-{
- u64 kva = 0;
-
- nes_debug(NES_DBG_MR, "\n");
-
- return nes_reg_phys_mr(pd, 0, 0xffffffffffULL, acc, &kva);
-}
-
-/**
- * nes_reg_user_mr
- */
-static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
- u64 virt, int acc, struct ib_udata *udata)
-{
- u64 iova_start;
- __le64 *pbl;
- u64 region_length;
- dma_addr_t last_dma_addr = 0;
- dma_addr_t first_dma_addr = 0;
- struct nes_pd *nespd = to_nespd(pd);
- struct nes_vnic *nesvnic = to_nesvnic(pd->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct ib_mr *ibmr = ERR_PTR(-EINVAL);
- struct sg_dma_page_iter dma_iter;
- struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context(
- udata, struct nes_ucontext, ibucontext);
- struct nes_pbl *nespbl;
- struct nes_mr *nesmr;
- struct ib_umem *region;
- struct nes_mem_reg_req req;
- struct nes_vpbl vpbl;
- struct nes_root_vpbl root_vpbl;
- int page_index;
- int page_count = 0;
- int err, pbl_depth = 0;
- int ret;
- u32 stag;
- u32 stag_index = 0;
- u32 next_stag_index;
- u32 driver_key;
- u32 root_pbl_index = 0;
- u32 cur_pbl_index = 0;
- u32 skip_pages;
- u16 pbl_count;
- u8 single_page = 1;
- u8 stag_key;
-
- region = ib_umem_get(udata, start, length, acc, 0);
- if (IS_ERR(region)) {
- return (struct ib_mr *)region;
- }
-
- nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u,"
- " offset = %u, page size = %lu.\n",
- (unsigned long int)start, (unsigned long int)virt, (u32)length,
- ib_umem_offset(region), BIT(region->page_shift));
-
- skip_pages = ((u32)ib_umem_offset(region)) >> 12;
-
- if (ib_copy_from_udata(&req, udata, sizeof(req))) {
- ib_umem_release(region);
- return ERR_PTR(-EFAULT);
- }
- nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type);
-
- switch (req.reg_type) {
- case IWNES_MEMREG_TYPE_MEM:
- pbl_depth = 0;
- region_length = 0;
- vpbl.pbl_vbase = NULL;
- root_vpbl.pbl_vbase = NULL;
- root_vpbl.pbl_pbase = 0;
-
- get_random_bytes(&next_stag_index, sizeof(next_stag_index));
- stag_key = (u8)next_stag_index;
-
- driver_key = next_stag_index & 0x70000000;
-
- next_stag_index >>= 8;
- next_stag_index %= nesadapter->max_mr;
-
- err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
- nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_USER_MR);
- if (err) {
- ib_umem_release(region);
- return ERR_PTR(err);
- }
-
- nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
- if (!nesmr) {
- ib_umem_release(region);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- return ERR_PTR(-ENOMEM);
- }
- nesmr->region = region;
-
- for_each_sg_dma_page (region->sg_head.sgl, &dma_iter, region->nmap, 0) {
-
- region_length += PAGE_SIZE;
- region_length -= skip_pages << 12;
- skip_pages = 0;
- if ((page_count != 0) && (page_count << 12) - (ib_umem_offset(region) & (4096 - 1)) >= region->length)
- goto enough_pages;
- if ((page_count & 0x01FF) == 0) {
- if (page_count >= 1024 * 512) {
- ib_umem_release(region);
- nes_free_resource(nesadapter,
- nesadapter->allocated_mrs, stag_index);
- kfree(nesmr);
- ibmr = ERR_PTR(-E2BIG);
- goto reg_user_mr_err;
- }
- if (root_pbl_index == 1) {
- root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
- 8192, &root_vpbl.pbl_pbase);
- nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
- root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
- if (!root_vpbl.pbl_vbase) {
- ib_umem_release(region);
- pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
- vpbl.pbl_pbase);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs,
- stag_index);
- kfree(nesmr);
- ibmr = ERR_PTR(-ENOMEM);
- goto reg_user_mr_err;
- }
- root_vpbl.leaf_vpbl = kcalloc(1024,
- sizeof(*root_vpbl.leaf_vpbl),
- GFP_KERNEL);
- if (!root_vpbl.leaf_vpbl) {
- ib_umem_release(region);
- pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
- root_vpbl.pbl_pbase);
- pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
- vpbl.pbl_pbase);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs,
- stag_index);
- kfree(nesmr);
- ibmr = ERR_PTR(-ENOMEM);
- goto reg_user_mr_err;
- }
- root_vpbl.pbl_vbase[0].pa_low =
- cpu_to_le32((u32)vpbl.pbl_pbase);
- root_vpbl.pbl_vbase[0].pa_high =
- cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
- root_vpbl.leaf_vpbl[0] = vpbl;
- }
- vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
- &vpbl.pbl_pbase);
- nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
- vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
- if (!vpbl.pbl_vbase) {
- ib_umem_release(region);
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
- ibmr = ERR_PTR(-ENOMEM);
- kfree(nesmr);
- goto reg_user_mr_err;
- }
- if (1 <= root_pbl_index) {
- root_vpbl.pbl_vbase[root_pbl_index].pa_low =
- cpu_to_le32((u32)vpbl.pbl_pbase);
- root_vpbl.pbl_vbase[root_pbl_index].pa_high =
- cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
- root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
- }
- root_pbl_index++;
- cur_pbl_index = 0;
- }
- if (single_page) {
- if (page_count != 0) {
- if ((last_dma_addr + 4096) != sg_page_iter_dma_address(&dma_iter))
- single_page = 0;
- last_dma_addr = sg_page_iter_dma_address(&dma_iter);
- } else {
- first_dma_addr = sg_page_iter_dma_address(&dma_iter);
- last_dma_addr = first_dma_addr;
- }
- }
-
- vpbl.pbl_vbase[cur_pbl_index].pa_low =
- cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter)));
- vpbl.pbl_vbase[cur_pbl_index].pa_high =
- cpu_to_le32((u32)((u64)(sg_page_iter_dma_address(&dma_iter))));
- cur_pbl_index++;
- page_count++;
- }
-
-enough_pages:
- nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
- " stag_key=0x%08x\n",
- stag_index, driver_key, stag_key);
- stag = stag_index << 8;
- stag |= driver_key;
- stag += (u32)stag_key;
-
- iova_start = virt;
- /* Make the leaf PBL the root if only one PBL */
- if (root_pbl_index == 1) {
- root_vpbl.pbl_pbase = vpbl.pbl_pbase;
- }
-
- if (single_page) {
- pbl_count = 0;
- } else {
- pbl_count = root_pbl_index;
- first_dma_addr = 0;
- }
- nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X,"
- " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n",
- stag, (unsigned int)iova_start,
- (unsigned int)region_length, stag_index,
- (unsigned long long)region->length, pbl_count);
- ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl,
- first_dma_addr, pbl_count, (u16)cur_pbl_index, acc,
- &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k);
-
- nes_debug(NES_DBG_MR, "ret=%d\n", ret);
-
- if (ret == 0) {
- nesmr->ibmr.rkey = stag;
- nesmr->ibmr.lkey = stag;
- nesmr->mode = IWNES_MEMREG_TYPE_MEM;
- ibmr = &nesmr->ibmr;
- } else {
- ib_umem_release(region);
- kfree(nesmr);
- ibmr = ERR_PTR(-ENOMEM);
- }
-
-reg_user_mr_err:
- /* free the resources */
- if (root_pbl_index == 1) {
- pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
- vpbl.pbl_pbase);
- } else {
- for (page_index=0; page_index<root_pbl_index; page_index++) {
- pci_free_consistent(nesdev->pcidev, 4096,
- root_vpbl.leaf_vpbl[page_index].pbl_vbase,
- root_vpbl.leaf_vpbl[page_index].pbl_pbase);
- }
- kfree(root_vpbl.leaf_vpbl);
- pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
- root_vpbl.pbl_pbase);
- }
-
- nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr);
-
- return ibmr;
- case IWNES_MEMREG_TYPE_QP:
- case IWNES_MEMREG_TYPE_CQ:
- if (!region->length) {
- nes_debug(NES_DBG_MR, "Unable to register zero length region for CQ\n");
- ib_umem_release(region);
- return ERR_PTR(-EINVAL);
- }
- nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL);
- if (!nespbl) {
- ib_umem_release(region);
- return ERR_PTR(-ENOMEM);
- }
- nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
- if (!nesmr) {
- ib_umem_release(region);
- kfree(nespbl);
- return ERR_PTR(-ENOMEM);
- }
- nesmr->region = region;
- pbl_depth = region->length >> 12;
- pbl_depth += (region->length & (4096-1)) ? 1 : 0;
- nespbl->pbl_size = pbl_depth*sizeof(u64);
- if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
- nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory");
- } else {
- nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory");
- }
-
- nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n",
- nespbl->pbl_size, pbl_depth);
- pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size,
- &nespbl->pbl_pbase);
- if (!pbl) {
- ib_umem_release(region);
- kfree(nesmr);
- kfree(nespbl);
- nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n");
- return ERR_PTR(-ENOMEM);
- }
-
- nespbl->pbl_vbase = (u64 *)pbl;
- nespbl->user_base = start;
- nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
- " pbl_vbase=%p user_base=0x%lx\n",
- nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
- (void *) nespbl->pbl_vbase, nespbl->user_base);
-
- nespbl->page = sg_page(region->sg_head.sgl);
- for_each_sg_dma_page(region->sg_head.sgl, &dma_iter, region->nmap, 0) {
- ((__le32 *)pbl)[0] = cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter)));
- ((__le32 *)pbl)[1] = cpu_to_le32(((u64)(sg_page_iter_dma_address(&dma_iter)))>>32);
- nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
- (unsigned long long)*pbl,
- le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
- pbl++;
- }
-
- if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
- list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list);
- } else {
- list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list);
- }
- nesmr->ibmr.rkey = -1;
- nesmr->ibmr.lkey = -1;
- nesmr->mode = req.reg_type;
- return &nesmr->ibmr;
- }
-
- ib_umem_release(region);
- return ERR_PTR(-ENOSYS);
-}
-
-
-/**
- * nes_dereg_mr
- */
-static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
-{
- struct nes_mr *nesmr = to_nesmr(ib_mr);
- struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- struct nes_hw_cqp_wqe *cqp_wqe;
- struct nes_cqp_request *cqp_request;
- unsigned long flags;
- int ret;
- u16 major_code;
- u16 minor_code;
-
-
- if (nesmr->pages)
- pci_free_consistent(nesdev->pcidev,
- nesmr->max_pages * sizeof(u64),
- nesmr->pages,
- nesmr->paddr);
-
- if (nesmr->region) {
- ib_umem_release(nesmr->region);
- }
- if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) {
- kfree(nesmr);
- return 0;
- }
-
- /* Deallocate the region with the adapter */
-
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- cqp_request->waiting = 1;
- cqp_wqe = &cqp_request->cqp_wqe;
-
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO |
- NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey);
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey);
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u,"
- " CQP Major:Minor codes = 0x%04X:0x%04X\n",
- ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code);
-
- major_code = cqp_request->major_code;
- minor_code = cqp_request->minor_code;
-
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (!ret) {
- nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag,"
- " ib_mr=%p, rkey = 0x%08X\n",
- ib_mr, ib_mr->rkey);
- return -ETIME;
- } else if (major_code) {
- nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting"
- " to destroy STag, ib_mr=%p, rkey = 0x%08X\n",
- major_code, minor_code, ib_mr, ib_mr->rkey);
- return -EIO;
- }
-
- if (nesmr->pbls_used != 0) {
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (nesmr->pbl_4k) {
- nesadapter->free_4kpbl += nesmr->pbls_used;
- if (nesadapter->free_4kpbl > nesadapter->max_4kpbl)
- printk(KERN_ERR PFX "free 4KB PBLs(%u) has "
- "exceeded the max(%u)\n",
- nesadapter->free_4kpbl,
- nesadapter->max_4kpbl);
- } else {
- nesadapter->free_256pbl += nesmr->pbls_used;
- if (nesadapter->free_256pbl > nesadapter->max_256pbl)
- printk(KERN_ERR PFX "free 256B PBLs(%u) has "
- "exceeded the max(%u)\n",
- nesadapter->free_256pbl,
- nesadapter->max_256pbl);
- }
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- }
- nes_free_resource(nesadapter, nesadapter->allocated_mrs,
- (ib_mr->rkey & 0x0fffff00) >> 8);
-
- kfree(nesmr);
-
- return 0;
-}
-
-
-/**
- * show_rev
- */
-static ssize_t hw_rev_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- struct nes_ib_device *nesibdev =
- rdma_device_to_drv_device(dev, struct nes_ib_device, ibdev);
- struct nes_vnic *nesvnic = nesibdev->nesvnic;
-
- nes_debug(NES_DBG_INIT, "\n");
- return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev);
-}
-static DEVICE_ATTR_RO(hw_rev);
-
-/**
- * show_hca
- */
-static ssize_t hca_type_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- nes_debug(NES_DBG_INIT, "\n");
- return sprintf(buf, "NES020\n");
-}
-static DEVICE_ATTR_RO(hca_type);
-
-/**
- * show_board
- */
-static ssize_t board_id_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- nes_debug(NES_DBG_INIT, "\n");
- return sprintf(buf, "%.*s\n", 32, "NES020 Board ID");
-}
-static DEVICE_ATTR_RO(board_id);
-
-static struct attribute *nes_dev_attributes[] = {
- &dev_attr_hw_rev.attr,
- &dev_attr_hca_type.attr,
- &dev_attr_board_id.attr,
- NULL
-};
-
-static const struct attribute_group nes_attr_group = {
- .attrs = nes_dev_attributes,
-};
-
-/**
- * nes_query_qp
- */
-static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
- int attr_mask, struct ib_qp_init_attr *init_attr)
-{
- struct nes_qp *nesqp = to_nesqp(ibqp);
-
- nes_debug(NES_DBG_QP, "\n");
-
- attr->qp_access_flags = 0;
- attr->cap.max_send_wr = nesqp->hwqp.sq_size;
- attr->cap.max_recv_wr = nesqp->hwqp.rq_size;
- attr->cap.max_recv_sge = 1;
- if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA)
- attr->cap.max_inline_data = 0;
- else
- attr->cap.max_inline_data = 64;
-
- init_attr->event_handler = nesqp->ibqp.event_handler;
- init_attr->qp_context = nesqp->ibqp.qp_context;
- init_attr->send_cq = nesqp->ibqp.send_cq;
- init_attr->recv_cq = nesqp->ibqp.recv_cq;
- init_attr->srq = nesqp->ibqp.srq;
- init_attr->cap = attr->cap;
-
- return 0;
-}
-
-
-/**
- * nes_hw_modify_qp
- */
-int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp,
- u32 next_iwarp_state, u32 termlen, u32 wait_completion)
-{
- struct nes_hw_cqp_wqe *cqp_wqe;
- /* struct iw_cm_id *cm_id = nesqp->cm_id; */
- /* struct iw_cm_event cm_event; */
- struct nes_cqp_request *cqp_request;
- int ret;
- u16 major_code;
-
- nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
-
- cqp_request = nes_get_cqp_request(nesdev);
- if (cqp_request == NULL) {
- nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n");
- return -ENOMEM;
- }
- if (wait_completion) {
- cqp_request->waiting = 1;
- } else {
- cqp_request->waiting = 0;
- }
- cqp_wqe = &cqp_request->cqp_wqe;
-
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
- NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state);
- nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n",
- next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]));
- nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
- set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase);
-
- /* If sending a terminate message, fill in the length (in words) */
- if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) &&
- !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) {
- termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT;
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen);
- }
-
- atomic_set(&cqp_request->refcount, 2);
- nes_post_cqp_request(nesdev, cqp_request);
-
- /* Wait for CQP */
- if (wait_completion) {
- /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n",
- nesqp->hwqp.qp_id); */
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
- NES_EVENT_TIMEOUT);
- nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, "
- "CQP Major:Minor codes = 0x%04X:0x%04X.\n",
- nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code);
- major_code = cqp_request->major_code;
- if (major_code) {
- nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed"
- "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n",
- nesqp->hwqp.qp_id, cqp_request->major_code,
- cqp_request->minor_code, next_iwarp_state);
- }
-
- nes_put_cqp_request(nesdev, cqp_request);
-
- if (!ret)
- return -ETIME;
- else if (major_code)
- return -EIO;
- else
- return 0;
- } else {
- return 0;
- }
-}
-
-
-/**
- * nes_modify_qp
- */
-int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
- int attr_mask, struct ib_udata *udata)
-{
- struct nes_qp *nesqp = to_nesqp(ibqp);
- struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- /* u32 cqp_head; */
- /* u32 counter; */
- u32 next_iwarp_state = 0;
- int err;
- unsigned long qplockflags;
- int ret;
- u16 original_last_aeq;
- u8 issue_modify_qp = 0;
- u8 dont_wait = 0;
-
- nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u,"
- " iwarp_state=0x%X, refcount=%d\n",
- nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state,
- nesqp->iwarp_state, atomic_read(&nesqp->refcount));
-
- spin_lock_irqsave(&nesqp->lock, qplockflags);
-
- nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X,"
- " QP Access Flags=0x%X, attr_mask = 0x%0x\n",
- nesqp->hwqp.qp_id, nesqp->hw_iwarp_state,
- nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask);
-
- if (attr_mask & IB_QP_STATE) {
- switch (attr->qp_state) {
- case IB_QPS_INIT:
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n",
- nesqp->hwqp.qp_id);
- if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE;
- issue_modify_qp = 1;
- break;
- case IB_QPS_RTR:
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n",
- nesqp->hwqp.qp_id);
- if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE;
- issue_modify_qp = 1;
- break;
- case IB_QPS_RTS:
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n",
- nesqp->hwqp.qp_id);
- if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- if (nesqp->cm_id == NULL) {
- nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n",
- nesqp->hwqp.qp_id );
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS;
- if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS)
- next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID |
- NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID;
- issue_modify_qp = 1;
- nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED;
- nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS;
- nesqp->hte_added = 1;
- break;
- case IB_QPS_SQD:
- issue_modify_qp = 1;
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n",
- nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail);
- if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return 0;
- } else {
- if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {
- nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing"
- " ignored due to current iWARP state\n",
- nesqp->hwqp.qp_id);
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) {
- nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing"
- " already done based on hw state.\n",
- nesqp->hwqp.qp_id);
- issue_modify_qp = 0;
- }
- switch (nesqp->hw_iwarp_state) {
- case NES_AEQE_IWARP_STATE_CLOSING:
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
- break;
- case NES_AEQE_IWARP_STATE_TERMINATE:
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
- break;
- case NES_AEQE_IWARP_STATE_ERROR:
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
- break;
- default:
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
- nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
- break;
- }
- }
- break;
- case IB_QPS_SQE:
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n",
- nesqp->hwqp.qp_id);
- if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
- nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
- issue_modify_qp = 1;
- break;
- case IB_QPS_ERR:
- case IB_QPS_RESET:
- if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- }
- nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n",
- nesqp->hwqp.qp_id);
- if (nesqp->term_flags)
- del_timer(&nesqp->terminate_timer);
-
- next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
- /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
- if (nesqp->hte_added) {
- nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n");
- next_iwarp_state |= NES_CQP_QP_DEL_HTE;
- nesqp->hte_added = 0;
- }
- if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&
- (nesdev->iw_status) &&
- (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {
- next_iwarp_state |= NES_CQP_QP_RESET;
- } else {
- nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
- nesqp->hwqp.qp_id, nesqp->hw_tcp_state);
- dont_wait = 1;
- }
- issue_modify_qp = 1;
- nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
- break;
- default:
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- return -EINVAL;
- break;
- }
-
- nesqp->ibqp_state = attr->qp_state;
- nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK;
- nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n",
- nesqp->iwarp_state);
- }
-
- if (attr_mask & IB_QP_ACCESS_FLAGS) {
- if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |
- NES_QPCONTEXT_MISC_RDMA_READ_EN);
- issue_modify_qp = 1;
- }
- if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN);
- issue_modify_qp = 1;
- }
- if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN);
- issue_modify_qp = 1;
- }
- if (attr->qp_access_flags & IB_ACCESS_MW_BIND) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN);
- issue_modify_qp = 1;
- }
-
- if (nesqp->user_mode) {
- nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |
- NES_QPCONTEXT_MISC_RDMA_READ_EN);
- issue_modify_qp = 1;
- }
- }
-
- original_last_aeq = nesqp->last_aeq;
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
-
- nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp);
-
- ret = 0;
-
-
- if (issue_modify_qp) {
- nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n");
- ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1);
- if (ret)
- nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)"
- " failed for QP%u.\n",
- next_iwarp_state, nesqp->hwqp.qp_id);
-
- }
-
- if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) {
- nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d),"
- " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- if (!ret || original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
- if (dont_wait) {
- if (nesqp->cm_id && nesqp->hw_tcp_state != 0) {
- nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d),"
- " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- /* this one is for the cm_disconnect thread */
- spin_lock_irqsave(&nesqp->lock, qplockflags);
- nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
- nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- nes_cm_disconn(nesqp);
- } else {
- nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
- }
- } else {
- spin_lock_irqsave(&nesqp->lock, qplockflags);
- if (nesqp->cm_id) {
- /* These two are for the timer thread */
- if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
- nesqp->cm_id->add_ref(nesqp->cm_id);
- nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d),"
- " need ae to finish up, original_last_aeq = 0x%04X."
- " last_aeq = 0x%04X, scheduling timer.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0);
- }
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- } else {
- spin_unlock_irqrestore(&nesqp->lock, qplockflags);
- nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d),"
- " need ae to finish up, original_last_aeq = 0x%04X."
- " last_aeq = 0x%04X.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- }
- }
- } else {
- nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
- " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- }
- } else {
- nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
- " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
- original_last_aeq, nesqp->last_aeq);
- }
-
- err = 0;
-
- nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n",
- nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
-
- return err;
-}
-
-static inline void
-fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, const struct ib_send_wr *ib_wr,
- u32 uselkey)
-{
- int sge_index;
- int total_payload_length = 0;
- for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) {
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4),
- ib_wr->sg_list[sge_index].addr);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4),
- ib_wr->sg_list[sge_index].length);
- if (uselkey)
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4),
- (ib_wr->sg_list[sge_index].lkey));
- else
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0);
-
- total_payload_length += ib_wr->sg_list[sge_index].length;
- }
- nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n",
- total_payload_length);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
- total_payload_length);
-}
-
-/**
- * nes_post_send
- */
-static int nes_post_send(struct ib_qp *ibqp, const struct ib_send_wr *ib_wr,
- const struct ib_send_wr **bad_wr)
-{
- u64 u64temp;
- unsigned long flags = 0;
- struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_qp *nesqp = to_nesqp(ibqp);
- struct nes_hw_qp_wqe *wqe;
- int err = 0;
- u32 qsize = nesqp->hwqp.sq_size;
- u32 head;
- u32 wqe_misc = 0;
- u32 wqe_count = 0;
- u32 counter;
-
- if (nesqp->ibqp_state > IB_QPS_RTS) {
- err = -EINVAL;
- goto out;
- }
-
- spin_lock_irqsave(&nesqp->lock, flags);
-
- head = nesqp->hwqp.sq_head;
-
- while (ib_wr) {
- /* Check for QP error */
- if (nesqp->term_flags) {
- err = -EINVAL;
- break;
- }
-
- /* Check for SQ overflow */
- if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {
- err = -ENOMEM;
- break;
- }
-
- wqe = &nesqp->hwqp.sq_vbase[head];
- /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n",
- nesqp->hwqp.qp_id, wqe, head); */
- nes_fill_init_qp_wqe(wqe, nesqp, head);
- u64temp = (u64)(ib_wr->wr_id);
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
- u64temp);
- switch (ib_wr->opcode) {
- case IB_WR_SEND:
- case IB_WR_SEND_WITH_INV:
- if (IB_WR_SEND == ib_wr->opcode) {
- if (ib_wr->send_flags & IB_SEND_SOLICITED)
- wqe_misc = NES_IWARP_SQ_OP_SENDSE;
- else
- wqe_misc = NES_IWARP_SQ_OP_SEND;
- } else {
- if (ib_wr->send_flags & IB_SEND_SOLICITED)
- wqe_misc = NES_IWARP_SQ_OP_SENDSEINV;
- else
- wqe_misc = NES_IWARP_SQ_OP_SENDINV;
-
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
- ib_wr->ex.invalidate_rkey);
- }
-
- if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
- err = -EINVAL;
- break;
- }
-
- if (ib_wr->send_flags & IB_SEND_FENCE)
- wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
-
- if ((ib_wr->send_flags & IB_SEND_INLINE) &&
- ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
- (ib_wr->sg_list[0].length <= 64)) {
- memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
- (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
- ib_wr->sg_list[0].length);
- wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
- } else {
- fill_wqe_sg_send(wqe, ib_wr, 1);
- }
-
- break;
- case IB_WR_RDMA_WRITE:
- wqe_misc = NES_IWARP_SQ_OP_RDMAW;
- if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
- nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
- ib_wr->num_sge, nesdev->nesadapter->max_sge);
- err = -EINVAL;
- break;
- }
-
- if (ib_wr->send_flags & IB_SEND_FENCE)
- wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
-
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
- rdma_wr(ib_wr)->rkey);
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
- rdma_wr(ib_wr)->remote_addr);
-
- if ((ib_wr->send_flags & IB_SEND_INLINE) &&
- ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
- (ib_wr->sg_list[0].length <= 64)) {
- memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
- (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
- ib_wr->sg_list[0].length);
- wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
- } else {
- fill_wqe_sg_send(wqe, ib_wr, 1);
- }
-
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
- wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
- break;
- case IB_WR_RDMA_READ:
- case IB_WR_RDMA_READ_WITH_INV:
- /* iWARP only supports 1 sge for RDMA reads */
- if (ib_wr->num_sge > 1) {
- nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
- ib_wr->num_sge);
- err = -EINVAL;
- break;
- }
- if (ib_wr->opcode == IB_WR_RDMA_READ) {
- wqe_misc = NES_IWARP_SQ_OP_RDMAR;
- } else {
- wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV;
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
- ib_wr->ex.invalidate_rkey);
- }
-
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
- rdma_wr(ib_wr)->remote_addr);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
- rdma_wr(ib_wr)->rkey);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
- ib_wr->sg_list->length);
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
- ib_wr->sg_list->addr);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
- ib_wr->sg_list->lkey);
- break;
- case IB_WR_LOCAL_INV:
- wqe_misc = NES_IWARP_SQ_OP_LOCINV;
- set_wqe_32bit_value(wqe->wqe_words,
- NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX,
- ib_wr->ex.invalidate_rkey);
- break;
- case IB_WR_REG_MR:
- {
- struct nes_mr *mr = to_nesmr(reg_wr(ib_wr)->mr);
- int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size);
- int flags = reg_wr(ib_wr)->access;
-
- if (mr->npages > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) {
- nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n");
- err = -EINVAL;
- break;
- }
- wqe_misc = NES_IWARP_SQ_OP_FAST_REG;
- set_wqe_64bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX,
- mr->ibmr.iova);
- set_wqe_32bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX,
- lower_32_bits(mr->ibmr.length));
- set_wqe_32bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX, 0);
- set_wqe_32bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX,
- reg_wr(ib_wr)->key);
-
- if (page_shift == 12) {
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K;
- } else if (page_shift == 21) {
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M;
- } else {
- nes_debug(NES_DBG_IW_TX, "Invalid page shift,"
- " ib_wr=%u, max=1\n", ib_wr->num_sge);
- err = -EINVAL;
- break;
- }
-
- /* Set access_flags */
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ;
- if (flags & IB_ACCESS_LOCAL_WRITE)
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE;
-
- if (flags & IB_ACCESS_REMOTE_WRITE)
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE;
-
- if (flags & IB_ACCESS_REMOTE_READ)
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ;
-
- if (flags & IB_ACCESS_MW_BIND)
- wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND;
-
- /* Fill in PBL info: */
- set_wqe_64bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX,
- mr->paddr);
-
- set_wqe_32bit_value(wqe->wqe_words,
- NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX,
- mr->npages * 8);
-
- nes_debug(NES_DBG_IW_TX, "SQ_REG_MR: iova_start: %llx, "
- "length: %lld, rkey: %0x, pgl_paddr: %llx, "
- "page_list_len: %u, wqe_misc: %x\n",
- (unsigned long long) mr->ibmr.iova,
- mr->ibmr.length,
- reg_wr(ib_wr)->key,
- (unsigned long long) mr->paddr,
- mr->npages,
- wqe_misc);
- break;
- }
- default:
- /* error */
- err = -EINVAL;
- break;
- }
-
- if (err)
- break;
-
- if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)
- wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
-
- wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc);
-
- ib_wr = ib_wr->next;
- head++;
- wqe_count++;
- if (head >= qsize)
- head = 0;
-
- }
-
- nesqp->hwqp.sq_head = head;
- barrier();
- while (wqe_count) {
- counter = min(wqe_count, ((u32)255));
- wqe_count -= counter;
- nes_write32(nesdev->regs + NES_WQE_ALLOC,
- (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id);
- }
-
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
-out:
- if (err)
- *bad_wr = ib_wr;
- return err;
-}
-
-
-/**
- * nes_post_recv
- */
-static int nes_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr,
- const struct ib_recv_wr **bad_wr)
-{
- u64 u64temp;
- unsigned long flags = 0;
- struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_qp *nesqp = to_nesqp(ibqp);
- struct nes_hw_qp_wqe *wqe;
- int err = 0;
- int sge_index;
- u32 qsize = nesqp->hwqp.rq_size;
- u32 head;
- u32 wqe_count = 0;
- u32 counter;
- u32 total_payload_length;
-
- if (nesqp->ibqp_state > IB_QPS_RTS) {
- err = -EINVAL;
- goto out;
- }
-
- spin_lock_irqsave(&nesqp->lock, flags);
-
- head = nesqp->hwqp.rq_head;
-
- while (ib_wr) {
- /* Check for QP error */
- if (nesqp->term_flags) {
- err = -EINVAL;
- break;
- }
-
- if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
- err = -EINVAL;
- break;
- }
- /* Check for RQ overflow */
- if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) {
- err = -ENOMEM;
- break;
- }
-
- nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge);
- wqe = &nesqp->hwqp.rq_vbase[head];
-
- /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n",
- nesqp->hwqp.qp_id, wqe, head); */
- nes_fill_init_qp_wqe(wqe, nesqp, head);
- u64temp = (u64)(ib_wr->wr_id);
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
- u64temp);
- total_payload_length = 0;
- for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4),
- ib_wr->sg_list[sge_index].addr);
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4),
- ib_wr->sg_list[sge_index].length);
- set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4),
- ib_wr->sg_list[sge_index].lkey);
-
- total_payload_length += ib_wr->sg_list[sge_index].length;
- }
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX,
- total_payload_length);
-
- ib_wr = ib_wr->next;
- head++;
- wqe_count++;
- if (head >= qsize)
- head = 0;
- }
-
- nesqp->hwqp.rq_head = head;
- barrier();
- while (wqe_count) {
- counter = min(wqe_count, ((u32)255));
- wqe_count -= counter;
- nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id);
- }
-
- spin_unlock_irqrestore(&nesqp->lock, flags);
-
-out:
- if (err)
- *bad_wr = ib_wr;
- return err;
-}
-
-/**
- * nes_drain_sq - drain sq
- * @ibqp: pointer to ibqp
- */
-static void nes_drain_sq(struct ib_qp *ibqp)
-{
- struct nes_qp *nesqp = to_nesqp(ibqp);
-
- if (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)
- wait_for_completion(&nesqp->sq_drained);
-}
-
-/**
- * nes_drain_rq - drain rq
- * @ibqp: pointer to ibqp
- */
-static void nes_drain_rq(struct ib_qp *ibqp)
-{
- struct nes_qp *nesqp = to_nesqp(ibqp);
-
- if (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)
- wait_for_completion(&nesqp->rq_drained);
-}
-
-/**
- * nes_poll_cq
- */
-static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
-{
- u64 u64temp;
- u64 wrid;
- unsigned long flags = 0;
- struct nes_vnic *nesvnic = to_nesvnic(ibcq->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_cq *nescq = to_nescq(ibcq);
- struct nes_qp *nesqp;
- struct nes_hw_cqe cqe;
- u32 head;
- u32 wq_tail = 0;
- u32 cq_size;
- u32 cqe_count = 0;
- u32 wqe_index;
- u32 u32temp;
- u32 move_cq_head = 1;
- u32 err_code;
-
- nes_debug(NES_DBG_CQ, "\n");
-
- spin_lock_irqsave(&nescq->lock, flags);
-
- head = nescq->hw_cq.cq_head;
- cq_size = nescq->hw_cq.cq_size;
-
- while (cqe_count < num_entries) {
- if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) &
- NES_CQE_VALID) == 0)
- break;
-
- /*
- * Make sure we read CQ entry contents *after*
- * we've checked the valid bit.
- */
- rmb();
-
- cqe = nescq->hw_cq.cq_vbase[head];
- u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
- wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1);
- u32temp &= ~(NES_SW_CONTEXT_ALIGN-1);
- /* parse CQE, get completion context from WQE (either rq or sq) */
- u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
- ((u64)u32temp);
-
- if (u64temp) {
- nesqp = (struct nes_qp *)(unsigned long)u64temp;
- memset(entry, 0, sizeof *entry);
- if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) {
- entry->status = IB_WC_SUCCESS;
- } else {
- err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
- if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) {
- entry->status = err_code & 0x0000ffff;
-
- /* The rest of the cqe's will be marked as flushed */
- nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
- cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) |
- NES_IWARP_CQE_MINOR_FLUSH);
- } else
- entry->status = IB_WC_WR_FLUSH_ERR;
- }
-
- entry->qp = &nesqp->ibqp;
- entry->src_qp = nesqp->hwqp.qp_id;
-
- if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
- if (nesqp->skip_lsmm) {
- nesqp->skip_lsmm = 0;
- nesqp->hwqp.sq_tail++;
- }
-
- /* Working on a SQ Completion*/
- wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].
- wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
- ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].
- wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])));
- entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
- wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
-
- switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
- wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
- case NES_IWARP_SQ_OP_RDMAW:
- nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n");
- entry->opcode = IB_WC_RDMA_WRITE;
- break;
- case NES_IWARP_SQ_OP_RDMAR:
- nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n");
- entry->opcode = IB_WC_RDMA_READ;
- entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
- wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
- break;
- case NES_IWARP_SQ_OP_SENDINV:
- case NES_IWARP_SQ_OP_SENDSEINV:
- case NES_IWARP_SQ_OP_SEND:
- case NES_IWARP_SQ_OP_SENDSE:
- nes_debug(NES_DBG_CQ, "Operation = Send.\n");
- entry->opcode = IB_WC_SEND;
- break;
- case NES_IWARP_SQ_OP_LOCINV:
- entry->opcode = IB_WC_LOCAL_INV;
- break;
- case NES_IWARP_SQ_OP_FAST_REG:
- entry->opcode = IB_WC_REG_MR;
- break;
- }
-
- nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);
- if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) {
- move_cq_head = 0;
- wq_tail = nesqp->hwqp.sq_tail;
- }
- } else {
- /* Working on a RQ Completion*/
- entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
- wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) |
- ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
- entry->opcode = IB_WC_RECV;
-
- nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1);
- if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) {
- move_cq_head = 0;
- wq_tail = nesqp->hwqp.rq_tail;
- }
- }
-
- if (nesqp->iwarp_state > NES_CQP_QP_IWARP_STATE_RTS) {
- if (nesqp->hwqp.sq_tail == nesqp->hwqp.sq_head)
- complete(&nesqp->sq_drained);
- if (nesqp->hwqp.rq_tail == nesqp->hwqp.rq_head)
- complete(&nesqp->rq_drained);
- }
-
- entry->wr_id = wrid;
- entry++;
- cqe_count++;
- }
-
- if (move_cq_head) {
- nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
- if (++head >= cq_size)
- head = 0;
- nescq->polled_completions++;
-
- if ((nescq->polled_completions > (cq_size / 2)) ||
- (nescq->polled_completions == 255)) {
- nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes"
- " are pending %u of %u.\n",
- nescq->hw_cq.cq_number, nescq->polled_completions, cq_size);
- nes_write32(nesdev->regs+NES_CQE_ALLOC,
- nescq->hw_cq.cq_number | (nescq->polled_completions << 16));
- nescq->polled_completions = 0;
- }
- } else {
- /* Update the wqe index and set status to flush */
- wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
- wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail;
- nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
- cpu_to_le32(wqe_index);
- move_cq_head = 1; /* ready for next pass */
- }
- }
-
- if (nescq->polled_completions) {
- nes_write32(nesdev->regs+NES_CQE_ALLOC,
- nescq->hw_cq.cq_number | (nescq->polled_completions << 16));
- nescq->polled_completions = 0;
- }
-
- nescq->hw_cq.cq_head = head;
- nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n",
- cqe_count, nescq->hw_cq.cq_number);
-
- spin_unlock_irqrestore(&nescq->lock, flags);
-
- return cqe_count;
-}
-
-
-/**
- * nes_req_notify_cq
- */
-static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
- {
- struct nes_vnic *nesvnic = to_nesvnic(ibcq->device);
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_cq *nescq = to_nescq(ibcq);
- u32 cq_arm;
-
- nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n",
- nescq->hw_cq.cq_number);
-
- cq_arm = nescq->hw_cq.cq_number;
- if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP)
- cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT;
- else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
- cq_arm |= NES_CQE_ALLOC_NOTIFY_SE;
- else
- return -EINVAL;
-
- nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm);
- nes_read32(nesdev->regs+NES_CQE_ALLOC);
-
- return 0;
-}
-
-static int nes_port_immutable(struct ib_device *ibdev, u8 port_num,
- struct ib_port_immutable *immutable)
-{
- struct ib_port_attr attr;
- int err;
-
- immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
-
- err = nes_query_port(ibdev, port_num, &attr);
- if (err)
- return err;
-
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
- immutable->gid_tbl_len = attr.gid_tbl_len;
-
- return 0;
-}
-
-static void get_dev_fw_str(struct ib_device *dev, char *str)
-{
- struct nes_ib_device *nesibdev =
- container_of(dev, struct nes_ib_device, ibdev);
- struct nes_vnic *nesvnic = nesibdev->nesvnic;
-
- nes_debug(NES_DBG_INIT, "\n");
- snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u",
- (nesvnic->nesdev->nesadapter->firmware_version >> 16),
- (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff));
-}
-
-static const struct ib_device_ops nes_dev_ops = {
- .alloc_mr = nes_alloc_mr,
- .alloc_mw = nes_alloc_mw,
- .alloc_pd = nes_alloc_pd,
- .alloc_ucontext = nes_alloc_ucontext,
- .create_cq = nes_create_cq,
- .create_qp = nes_create_qp,
- .dealloc_mw = nes_dealloc_mw,
- .dealloc_pd = nes_dealloc_pd,
- .dealloc_ucontext = nes_dealloc_ucontext,
- .dereg_mr = nes_dereg_mr,
- .destroy_cq = nes_destroy_cq,
- .destroy_qp = nes_destroy_qp,
- .drain_rq = nes_drain_rq,
- .drain_sq = nes_drain_sq,
- .get_dev_fw_str = get_dev_fw_str,
- .get_dma_mr = nes_get_dma_mr,
- .get_port_immutable = nes_port_immutable,
- .iw_accept = nes_accept,
- .iw_add_ref = nes_add_ref,
- .iw_connect = nes_connect,
- .iw_create_listen = nes_create_listen,
- .iw_destroy_listen = nes_destroy_listen,
- .iw_get_qp = nes_get_qp,
- .iw_reject = nes_reject,
- .iw_rem_ref = nes_rem_ref,
- .map_mr_sg = nes_map_mr_sg,
- .mmap = nes_mmap,
- .modify_qp = nes_modify_qp,
- .poll_cq = nes_poll_cq,
- .post_recv = nes_post_recv,
- .post_send = nes_post_send,
- .query_device = nes_query_device,
- .query_gid = nes_query_gid,
- .query_pkey = nes_query_pkey,
- .query_port = nes_query_port,
- .query_qp = nes_query_qp,
- .reg_user_mr = nes_reg_user_mr,
- .req_notify_cq = nes_req_notify_cq,
- INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd),
- INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext),
-};
-
-/**
- * nes_init_ofa_device
- */
-struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
-{
- struct nes_ib_device *nesibdev;
- struct nes_vnic *nesvnic = netdev_priv(netdev);
- struct nes_device *nesdev = nesvnic->nesdev;
-
- nesibdev = ib_alloc_device(nes_ib_device, ibdev);
- if (nesibdev == NULL) {
- return NULL;
- }
- nesibdev->ibdev.owner = THIS_MODULE;
-
- nesibdev->ibdev.node_type = RDMA_NODE_RNIC;
- memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid));
- memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6);
-
- nesibdev->ibdev.uverbs_cmd_mask =
- (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
- (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
- (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
- (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_REG_MR) |
- (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
- (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
- (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
- (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
- (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
- (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
- (1ull << IB_USER_VERBS_CMD_BIND_MW) |
- (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) |
- (1ull << IB_USER_VERBS_CMD_POST_RECV) |
- (1ull << IB_USER_VERBS_CMD_POST_SEND);
-
- nesibdev->ibdev.phys_port_cnt = 1;
- nesibdev->ibdev.num_comp_vectors = 1;
- nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev;
-
- ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops);
- memcpy(nesibdev->ibdev.iw_ifname, netdev->name,
- sizeof(nesibdev->ibdev.iw_ifname));
-
- return nesibdev;
-}
-
-
-/**
- * nes_handle_delayed_event
- */
-static void nes_handle_delayed_event(struct timer_list *t)
-{
- struct nes_vnic *nesvnic = from_timer(nesvnic, t, event_timer);
-
- if (nesvnic->delayed_event != nesvnic->last_dispatched_event) {
- struct ib_event event;
-
- event.device = &nesvnic->nesibdev->ibdev;
- if (!event.device)
- goto stop_timer;
- event.event = nesvnic->delayed_event;
- event.element.port_num = nesvnic->logical_port + 1;
- ib_dispatch_event(&event);
- }
-
-stop_timer:
- nesvnic->event_timer.function = NULL;
-}
-
-
-void nes_port_ibevent(struct nes_vnic *nesvnic)
-{
- struct nes_ib_device *nesibdev = nesvnic->nesibdev;
- struct nes_device *nesdev = nesvnic->nesdev;
- struct ib_event event;
- event.device = &nesibdev->ibdev;
- event.element.port_num = nesvnic->logical_port + 1;
- event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
-
- if (!nesvnic->event_timer.function) {
- ib_dispatch_event(&event);
- nesvnic->last_dispatched_event = event.event;
- nesvnic->event_timer.function = nes_handle_delayed_event;
- nesvnic->event_timer.expires = jiffies + NES_EVENT_DELAY;
- add_timer(&nesvnic->event_timer);
- } else {
- mod_timer(&nesvnic->event_timer, jiffies + NES_EVENT_DELAY);
- }
- nesvnic->delayed_event = event.event;
-}
-
-
-/**
- * nes_destroy_ofa_device
- */
-void nes_destroy_ofa_device(struct nes_ib_device *nesibdev)
-{
- if (nesibdev == NULL)
- return;
-
- nes_unregister_ofa_device(nesibdev);
-
- ib_dealloc_device(&nesibdev->ibdev);
-}
-
-
-/**
- * nes_register_ofa_device
- */
-int nes_register_ofa_device(struct nes_ib_device *nesibdev)
-{
- struct nes_vnic *nesvnic = nesibdev->nesvnic;
- struct nes_device *nesdev = nesvnic->nesdev;
- struct nes_adapter *nesadapter = nesdev->nesadapter;
- int ret;
-
- rdma_set_device_sysfs_group(&nesvnic->nesibdev->ibdev, &nes_attr_group);
- nesvnic->nesibdev->ibdev.driver_id = RDMA_DRIVER_NES;
- ret = ib_register_device(&nesvnic->nesibdev->ibdev, "nes%d");
- if (ret) {
- return ret;
- }
-
- /* Get the resources allocated to this device */
- nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count;
- nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count;
- nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count;
- nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count;
-
- nesvnic->of_device_registered = 1;
-
- return 0;
-}
-
-
-/**
- * nes_unregister_ofa_device
- */
-static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev)
-{
- struct nes_vnic *nesvnic = nesibdev->nesvnic;
-
- if (nesvnic->of_device_registered)
- ib_unregister_device(&nesibdev->ibdev);
-
- nesvnic->of_device_registered = 0;
-}
diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h
deleted file mode 100644
index 114a9b59fefd..000000000000
--- a/drivers/infiniband/hw/nes/nes_verbs.h
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#ifndef NES_VERBS_H
-#define NES_VERBS_H
-
-struct nes_device;
-
-#define NES_MAX_USER_DB_REGIONS 4096
-#define NES_MAX_USER_WQ_REGIONS 4096
-
-#define NES_TERM_SENT 0x01
-#define NES_TERM_RCVD 0x02
-#define NES_TERM_DONE 0x04
-
-struct nes_ucontext {
- struct ib_ucontext ibucontext;
- struct nes_device *nesdev;
- unsigned long mmap_wq_offset;
- unsigned long mmap_cq_offset; /* to be removed */
- int index; /* rnic index (minor) */
- unsigned long allocated_doorbells[BITS_TO_LONGS(NES_MAX_USER_DB_REGIONS)];
- u16 mmap_db_index[NES_MAX_USER_DB_REGIONS];
- u16 first_free_db;
- unsigned long allocated_wqs[BITS_TO_LONGS(NES_MAX_USER_WQ_REGIONS)];
- struct nes_qp *mmap_nesqp[NES_MAX_USER_WQ_REGIONS];
- u16 first_free_wq;
- struct list_head cq_reg_mem_list;
- struct list_head qp_reg_mem_list;
- u32 mcrqf;
-};
-
-struct nes_pd {
- struct ib_pd ibpd;
- u16 pd_id;
- atomic_t sqp_count;
- u16 mmap_db_index;
-};
-
-struct nes_mr {
- union {
- struct ib_mr ibmr;
- struct ib_mw ibmw;
- struct ib_fmr ibfmr;
- };
- struct ib_umem *region;
- u16 pbls_used;
- u8 mode;
- u8 pbl_4k;
- __le64 *pages;
- dma_addr_t paddr;
- u32 max_pages;
- u32 npages;
-};
-
-struct nes_hw_pb {
- __le32 pa_low;
- __le32 pa_high;
-};
-
-struct nes_vpbl {
- dma_addr_t pbl_pbase;
- struct nes_hw_pb *pbl_vbase;
-};
-
-struct nes_root_vpbl {
- dma_addr_t pbl_pbase;
- struct nes_hw_pb *pbl_vbase;
- struct nes_vpbl *leaf_vpbl;
-};
-
-struct nes_fmr {
- struct nes_mr nesmr;
- u32 leaf_pbl_cnt;
- struct nes_root_vpbl root_vpbl;
- struct ib_qp *ib_qp;
- int access_rights;
- struct ib_fmr_attr attr;
-};
-
-struct nes_av;
-
-struct nes_cq {
- struct ib_cq ibcq;
- struct nes_hw_cq hw_cq;
- u32 polled_completions;
- u32 cq_mem_size;
- spinlock_t lock;
- u8 virtual_cq;
- u8 pad[3];
- u32 mcrqf;
-};
-
-struct nes_wq {
- spinlock_t lock;
-};
-
-struct disconn_work {
- struct work_struct work;
- struct nes_qp *nesqp;
-};
-
-struct iw_cm_id;
-struct ietf_mpa_frame;
-
-struct nes_qp {
- struct ib_qp ibqp;
- void *allocated_buffer;
- struct iw_cm_id *cm_id;
- struct nes_cq *nesscq;
- struct nes_cq *nesrcq;
- struct nes_pd *nespd;
- void *cm_node; /* handle of the node this QP is associated with */
- void *ietf_frame;
- u8 ietf_frame_size;
- dma_addr_t ietf_frame_pbase;
- struct ib_mr *lsmm_mr;
- struct nes_hw_qp hwqp;
- struct work_struct work;
- enum ib_qp_state ibqp_state;
- u32 iwarp_state;
- u32 hte_index;
- u32 last_aeq;
- u32 qp_mem_size;
- atomic_t refcount;
- atomic_t close_timer_started;
- u32 mmap_sq_db_index;
- u32 mmap_rq_db_index;
- spinlock_t lock;
- spinlock_t pau_lock;
- struct nes_qp_context *nesqp_context;
- dma_addr_t nesqp_context_pbase;
- void *pbl_vbase;
- dma_addr_t pbl_pbase;
- struct page *page;
- struct timer_list terminate_timer;
- enum ib_event_type terminate_eventtype;
- struct sk_buff_head pau_list;
- u32 pau_rcv_nxt;
- u16 active_conn:1;
- u16 skip_lsmm:1;
- u16 user_mode:1;
- u16 hte_added:1;
- u16 flush_issued:1;
- u16 destroyed:1;
- u16 sig_all:1;
- u16 pau_mode:1;
- u16 rsvd:8;
- u16 private_data_len;
- u16 term_sq_flush_code;
- u16 term_rq_flush_code;
- u8 hw_iwarp_state;
- u8 hw_tcp_state;
- u8 term_flags;
- u8 sq_kmapped;
- u8 pau_busy;
- u8 pau_pending;
- u8 pau_state;
- __u64 nesuqp_addr;
- struct completion sq_drained;
- struct completion rq_drained;
-};
-
-struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
- u64 addr, u64 size, int acc, u64 *iova_start);
-
-#endif /* NES_VERBS_H */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 5127e2ea4bdd..b2dd4e0a4be2 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -1888,14 +1888,13 @@ mem_err:
return status;
}
-int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq)
+void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq)
{
- int status = -ENOMEM;
struct ocrdma_destroy_cq *cmd;
cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_CQ, sizeof(*cmd));
if (!cmd)
- return status;
+ return;
ocrdma_init_mch(&cmd->req, OCRDMA_CMD_DELETE_CQ,
OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
@@ -1903,11 +1902,10 @@ int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq)
(cq->id << OCRDMA_DESTROY_CQ_QID_SHIFT) &
OCRDMA_DESTROY_CQ_QID_MASK;
- status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+ ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
ocrdma_unbind_eq(dev, cq->eqn);
dma_free_coherent(&dev->nic_info.pdev->dev, cq->len, cq->va, cq->pa);
kfree(cmd);
- return status;
}
int ocrdma_mbx_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr,
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
index 06ec59326a90..12c23a7652b9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
@@ -122,7 +122,7 @@ int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr,
u32 pd_id, int acc);
int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *,
int entries, int dpp_cq, u16 pd_id);
-int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *);
+void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq);
int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs,
u8 enable_dpp_cq, u16 dpp_cq_id, u16 *dpp_offset,
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index fc6c0962dea9..c15cfc6cef81 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -144,6 +144,10 @@ static const struct attribute_group ocrdma_attr_group = {
};
static const struct ib_device_ops ocrdma_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_OCRDMA,
+ .uverbs_abi_ver = OCRDMA_ABI_VERSION,
+
.alloc_mr = ocrdma_alloc_mr,
.alloc_pd = ocrdma_alloc_pd,
.alloc_ucontext = ocrdma_alloc_ucontext,
@@ -178,6 +182,7 @@ static const struct ib_device_ops ocrdma_dev_ops = {
.resize_cq = ocrdma_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, ocrdma_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, ocrdma_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, ocrdma_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, ocrdma_ucontext, ibucontext),
};
@@ -200,8 +205,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
sizeof(OCRDMA_NODE_DESC));
- dev->ibdev.owner = THIS_MODULE;
- dev->ibdev.uverbs_abi_ver = OCRDMA_ABI_VERSION;
dev->ibdev.uverbs_cmd_mask =
OCRDMA_UVERBS(GET_CONTEXT) |
OCRDMA_UVERBS(QUERY_DEVICE) |
@@ -249,7 +252,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
ib_set_device_ops(&dev->ibdev, &ocrdma_dev_srq_ops);
}
rdma_set_device_sysfs_group(&dev->ibdev, &ocrdma_attr_group);
- dev->ibdev.driver_id = RDMA_DRIVER_OCRDMA;
ret = ib_device_set_netdev(&dev->ibdev, dev->nic_info.netdev, 1);
if (ret)
return ret;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 35ec87015792..bccc11378109 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -925,8 +925,7 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
/* it could be user registered memory. */
- if (mr->umem)
- ib_umem_release(mr->umem);
+ ib_umem_release(mr->umem);
kfree(mr);
/* Don't stop cleanup, in case FW is unresponsive */
@@ -977,12 +976,12 @@ err:
return status;
}
-struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
- struct ocrdma_cq *cq;
+ struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
udata, struct ocrdma_ucontext, ibucontext);
@@ -991,16 +990,13 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
struct ocrdma_create_cq_ureq ureq;
if (attr->flags)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (udata) {
if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
- return ERR_PTR(-EFAULT);
+ return -EFAULT;
} else
ureq.dpp_cq = 0;
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
spin_lock_init(&cq->cq_lock);
spin_lock_init(&cq->comp_handler_lock);
@@ -1011,10 +1007,9 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
pd_id = uctx->cntxt_pd->id;
status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
- if (status) {
- kfree(cq);
- return ERR_PTR(status);
- }
+ if (status)
+ return status;
+
if (udata) {
status = ocrdma_copy_cq_uresp(dev, cq, udata);
if (status)
@@ -1022,12 +1017,11 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
}
cq->phase = OCRDMA_CQE_VALID;
dev->cq_tbl[cq->id] = cq;
- return &cq->ibcq;
+ return 0;
ctx_err:
ocrdma_mbx_destroy_cq(dev, cq);
- kfree(cq);
- return ERR_PTR(status);
+ return status;
}
int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
@@ -1070,7 +1064,7 @@ static void ocrdma_flush_cq(struct ocrdma_cq *cq)
spin_unlock_irqrestore(&cq->cq_lock, flags);
}
-int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
struct ocrdma_eq *eq = NULL;
@@ -1080,14 +1074,13 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
dev->cq_tbl[cq->id] = NULL;
indx = ocrdma_get_eq_table_index(dev, cq->eqn);
- BUG_ON(indx == -EINVAL);
eq = &dev->eq_tbl[indx];
irq = ocrdma_get_irq(dev, eq);
synchronize_irq(irq);
ocrdma_flush_cq(cq);
- (void)ocrdma_mbx_destroy_cq(dev, cq);
+ ocrdma_mbx_destroy_cq(dev, cq);
if (cq->ucontext) {
pdid = cq->ucontext->cntxt_pd->id;
ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
@@ -1096,9 +1089,6 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
ocrdma_get_db_addr(dev, pdid),
dev->nic_info.db_page_size);
}
-
- kfree(cq);
- return 0;
}
static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index d76aae7ed0d3..32488da1b752 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -71,11 +71,10 @@ int ocrdma_mmap(struct ib_ucontext *, struct vm_area_struct *vma);
int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
void ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
-struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *);
-int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
struct ib_qp *ocrdma_create_qp(struct ib_pd *,
struct ib_qp_init_attr *attrs,
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index 083c2c00a8e9..a0a7ba0a5af4 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -183,6 +183,10 @@ static void qedr_roce_register_device(struct qedr_dev *dev)
}
static const struct ib_device_ops qedr_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_QEDR,
+ .uverbs_abi_ver = QEDR_ABI_VERSION,
+
.alloc_mr = qedr_alloc_mr,
.alloc_pd = qedr_alloc_pd,
.alloc_ucontext = qedr_alloc_ucontext,
@@ -220,6 +224,7 @@ static const struct ib_device_ops qedr_dev_ops = {
.resize_cq = qedr_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, qedr_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext),
@@ -231,8 +236,6 @@ static int qedr_register_device(struct qedr_dev *dev)
dev->ibdev.node_guid = dev->attr.node_guid;
memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC));
- dev->ibdev.owner = THIS_MODULE;
- dev->ibdev.uverbs_abi_ver = QEDR_ABI_VERSION;
dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) |
QEDR_UVERBS(QUERY_DEVICE) |
@@ -274,7 +277,6 @@ static int qedr_register_device(struct qedr_dev *dev)
rdma_set_device_sysfs_group(&dev->ibdev, &qedr_attr_group);
ib_set_device_ops(&dev->ibdev, &qedr_dev_ops);
- dev->ibdev.driver_id = RDMA_DRIVER_QEDR;
rc = ib_device_set_netdev(&dev->ibdev, dev->ndev, 1);
if (rc)
return rc;
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 3d7bde19838e..27d90a84ea01 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -159,54 +159,47 @@ int qedr_query_device(struct ib_device *ibdev,
return 0;
}
-#define QEDR_SPEED_SDR (1)
-#define QEDR_SPEED_DDR (2)
-#define QEDR_SPEED_QDR (4)
-#define QEDR_SPEED_FDR10 (8)
-#define QEDR_SPEED_FDR (16)
-#define QEDR_SPEED_EDR (32)
-
static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
u8 *ib_width)
{
switch (speed) {
case 1000:
- *ib_speed = QEDR_SPEED_SDR;
+ *ib_speed = IB_SPEED_SDR;
*ib_width = IB_WIDTH_1X;
break;
case 10000:
- *ib_speed = QEDR_SPEED_QDR;
+ *ib_speed = IB_SPEED_QDR;
*ib_width = IB_WIDTH_1X;
break;
case 20000:
- *ib_speed = QEDR_SPEED_DDR;
+ *ib_speed = IB_SPEED_DDR;
*ib_width = IB_WIDTH_4X;
break;
case 25000:
- *ib_speed = QEDR_SPEED_EDR;
+ *ib_speed = IB_SPEED_EDR;
*ib_width = IB_WIDTH_1X;
break;
case 40000:
- *ib_speed = QEDR_SPEED_QDR;
+ *ib_speed = IB_SPEED_QDR;
*ib_width = IB_WIDTH_4X;
break;
case 50000:
- *ib_speed = QEDR_SPEED_QDR;
- *ib_width = IB_WIDTH_4X;
+ *ib_speed = IB_SPEED_HDR;
+ *ib_width = IB_WIDTH_1X;
break;
case 100000:
- *ib_speed = QEDR_SPEED_EDR;
+ *ib_speed = IB_SPEED_EDR;
*ib_width = IB_WIDTH_4X;
break;
default:
/* Unsupported */
- *ib_speed = QEDR_SPEED_SDR;
+ *ib_speed = IB_SPEED_SDR;
*ib_width = IB_WIDTH_1X;
}
}
@@ -813,20 +806,20 @@ int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
return 0;
}
-struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
udata, struct qedr_ucontext, ibucontext);
struct qed_rdma_destroy_cq_out_params destroy_oparams;
struct qed_rdma_destroy_cq_in_params destroy_iparams;
struct qedr_dev *dev = get_qedr_dev(ibdev);
struct qed_rdma_create_cq_in_params params;
- struct qedr_create_cq_ureq ureq;
+ struct qedr_create_cq_ureq ureq = {};
int vector = attr->comp_vector;
int entries = attr->cqe;
- struct qedr_cq *cq;
+ struct qedr_cq *cq = get_qedr_cq(ibcq);
int chain_entries;
int page_cnt;
u64 pbl_ptr;
@@ -841,18 +834,13 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
DP_ERR(dev,
"create cq: the number of entries %d is too high. Must be equal or below %d.\n",
entries, QEDR_MAX_CQES);
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
chain_entries = qedr_align_cq_entries(entries);
chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
-
if (udata) {
- memset(&ureq, 0, sizeof(ureq));
if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
DP_ERR(dev,
"create cq: problem copying data from user space\n");
@@ -930,7 +918,7 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
"create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
cq->icid, cq, params.cq_size);
- return &cq->ibcq;
+ return 0;
err3:
destroy_iparams.icid = cq->icid;
@@ -945,8 +933,7 @@ err1:
if (udata)
ib_umem_release(cq->q.umem);
err0:
- kfree(cq);
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
@@ -962,14 +949,13 @@ int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
#define QEDR_DESTROY_CQ_ITER_DURATION (10)
-int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct qedr_dev *dev = get_qedr_dev(ibcq->device);
struct qed_rdma_destroy_cq_out_params oparams;
struct qed_rdma_destroy_cq_in_params iparams;
struct qedr_cq *cq = get_qedr_cq(ibcq);
int iter;
- int rc;
DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
@@ -977,13 +963,10 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
/* GSIs CQs are handled by driver, so they don't exist in the FW */
if (cq->cq_type == QEDR_CQ_TYPE_GSI)
- goto done;
+ return;
iparams.icid = cq->icid;
- rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
- if (rc)
- return rc;
-
+ dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
dev->ops->common->chain_free(dev->cdev, &cq->pbl);
if (udata) {
@@ -1014,27 +997,11 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
iter--;
}
- if (oparams.num_cq_notif != cq->cnq_notif)
- goto err;
-
/* Note that we don't need to have explicit code to wait for the
* completion of the event handler because it is invoked from the EQ.
* Since the destroy CQ ramrod has also been received on the EQ we can
* be certain that there's no event handler in process.
*/
-done:
- cq->sig = ~cq->sig;
-
- kfree(cq);
-
- return 0;
-
-err:
- DP_ERR(dev,
- "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
- cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
-
- return -EINVAL;
}
static inline int get_gid_info_from_table(struct ib_qp *ibqp,
@@ -1605,12 +1572,10 @@ qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
{
- if (qp->usq.umem)
- ib_umem_release(qp->usq.umem);
+ ib_umem_release(qp->usq.umem);
qp->usq.umem = NULL;
- if (qp->urq.umem)
- ib_umem_release(qp->urq.umem);
+ ib_umem_release(qp->urq.umem);
qp->urq.umem = NULL;
}
@@ -2713,8 +2678,7 @@ int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
/* it could be user registered memory. */
- if (mr->umem)
- ib_umem_release(mr->umem);
+ ib_umem_release(mr->umem);
kfree(mr);
diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h
index 9328c80375ef..9aaa90283d6e 100644
--- a/drivers/infiniband/hw/qedr/verbs.h
+++ b/drivers/infiniband/hw/qedr/verbs.h
@@ -50,11 +50,10 @@ int qedr_mmap(struct ib_ucontext *, struct vm_area_struct *vma);
int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
-struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *);
-int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs,
struct ib_udata *);
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 2ac4c67f5ba1..8d9a94d6f685 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -921,20 +921,11 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
rvt_add_retry_timer(qp);
while (qp->s_last != qp->s_acked) {
- u32 s_last;
-
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
break;
- s_last = qp->s_last;
- if (++s_last >= qp->s_size)
- s_last = 0;
- qp->s_last = s_last;
- /* see post_send() */
- barrier();
- rvt_put_qp_swqe(qp, wqe);
- rvt_qp_swqe_complete(qp,
+ rvt_qp_complete_swqe(qp,
wqe,
ib_qib_wc_opcode[wqe->wr.opcode],
IB_WC_SUCCESS);
@@ -972,21 +963,12 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
* is finished.
*/
if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
- qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
- u32 s_last;
-
- rvt_put_qp_swqe(qp, wqe);
- s_last = qp->s_last;
- if (++s_last >= qp->s_size)
- s_last = 0;
- qp->s_last = s_last;
- /* see post_send() */
- barrier();
- rvt_qp_swqe_complete(qp,
+ qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0)
+ rvt_qp_complete_swqe(qp,
wqe,
ib_qib_wc_opcode[wqe->wr.opcode],
IB_WC_SUCCESS);
- } else
+ else
this_cpu_inc(*ibp->rvp.rc_delayed_comp);
qp->s_retry = qp->s_retry_cnt;
diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c
index f712fb7fa82f..bfbfbb7e0ff4 100644
--- a/drivers/infiniband/hw/qib/qib_user_pages.c
+++ b/drivers/infiniband/hw/qib/qib_user_pages.c
@@ -40,13 +40,10 @@
static void __qib_release_user_pages(struct page **p, size_t num_pages,
int dirty)
{
- size_t i;
-
- for (i = 0; i < num_pages; i++) {
- if (dirty)
- set_page_dirty_lock(p[i]);
- put_page(p[i]);
- }
+ if (dirty)
+ put_user_pages_dirty_lock(p, num_pages);
+ else
+ put_user_pages(p, num_pages);
}
/**
diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c
index 0c204776263f..05190edc2611 100644
--- a/drivers/infiniband/hw/qib/qib_user_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_user_sdma.c
@@ -317,7 +317,7 @@ static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
* the caller can ignore this page.
*/
if (put) {
- put_page(page);
+ put_user_page(page);
} else {
/* coalesce case */
kunmap(page);
@@ -631,7 +631,7 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev,
kunmap(pkt->addr[i].page);
if (pkt->addr[i].put_page)
- put_page(pkt->addr[i].page);
+ put_user_page(pkt->addr[i].page);
else
__free_page(pkt->addr[i].page);
} else if (pkt->addr[i].kvaddr) {
@@ -706,7 +706,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
/* if error, return all pages not managed by pkt */
free_pages:
while (i < j)
- put_page(pages[i++]);
+ put_user_page(pages[i++]);
done:
return ret;
@@ -904,10 +904,11 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
}
if (frag_size) {
- int pktsize, tidsmsize, n;
+ int tidsmsize, n;
+ size_t pktsize;
n = npages*((2*PAGE_SIZE/frag_size)+1);
- pktsize = sizeof(*pkt) + sizeof(pkt->addr[0])*n;
+ pktsize = struct_size(pkt, addr, n);
/*
* Determine if this is tid-sdma or just sdma.
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 2c4e569ce438..33778d451b82 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1480,6 +1480,9 @@ static void qib_fill_device_attr(struct qib_devdata *dd)
}
static const struct ib_device_ops qib_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_QIB,
+
.init_port = qib_create_port_files,
.modify_device = qib_modify_device,
.process_mad = qib_process_mad,
@@ -1543,7 +1546,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
if (!ib_qib_sys_image_guid)
ib_qib_sys_image_guid = ppd->guid;
- ibdev->owner = THIS_MODULE;
ibdev->node_guid = ppd->guid;
ibdev->phys_port_cnt = dd->num_pports;
ibdev->dev.parent = &dd->pcidev->dev;
@@ -1614,7 +1616,7 @@ int qib_register_ib_device(struct qib_devdata *dd)
rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &qib_attr_group);
ib_set_device_ops(ibdev, &qib_dev_ops);
- ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_QIB);
+ ret = rvt_register_device(&dd->verbs_dev.rdi);
if (ret)
goto err_tx;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib.h b/drivers/infiniband/hw/usnic/usnic_ib.h
index 525bf272671e..84dd682d2334 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib.h
@@ -61,6 +61,10 @@ struct usnic_ib_pd {
struct usnic_uiom_pd *umem_pd;
};
+struct usnic_ib_cq {
+ struct ib_cq ibcq;
+};
+
struct usnic_ib_mr {
struct ib_mr ibmr;
struct usnic_uiom_reg *umem;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c
index d88d9f8a7f9a..6ae5ce007fed 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_main.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c
@@ -329,6 +329,10 @@ static void usnic_get_dev_fw_str(struct ib_device *device, char *str)
}
static const struct ib_device_ops usnic_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_USNIC,
+ .uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION,
+
.alloc_pd = usnic_ib_alloc_pd,
.alloc_ucontext = usnic_ib_alloc_ucontext,
.create_cq = usnic_ib_create_cq,
@@ -350,6 +354,7 @@ static const struct ib_device_ops usnic_dev_ops = {
.query_qp = usnic_ib_query_qp,
.reg_user_mr = usnic_ib_reg_mr,
INIT_RDMA_OBJ_SIZE(ib_pd, usnic_ib_pd, ibpd),
+ INIT_RDMA_OBJ_SIZE(ib_cq, usnic_ib_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, usnic_ib_ucontext, ibucontext),
};
@@ -384,12 +389,10 @@ static void *usnic_ib_device_add(struct pci_dev *dev)
us_ibdev->pdev = dev;
us_ibdev->netdev = pci_get_drvdata(dev);
- us_ibdev->ib_dev.owner = THIS_MODULE;
us_ibdev->ib_dev.node_type = RDMA_NODE_USNIC_UDP;
us_ibdev->ib_dev.phys_port_cnt = USNIC_IB_PORT_CNT;
us_ibdev->ib_dev.num_comp_vectors = USNIC_IB_NUM_COMP_VECTORS;
us_ibdev->ib_dev.dev.parent = &dev->dev;
- us_ibdev->ib_dev.uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION;
us_ibdev->ib_dev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
@@ -412,7 +415,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev)
ib_set_device_ops(&us_ibdev->ib_dev, &usnic_dev_ops);
- us_ibdev->ib_dev.driver_id = RDMA_DRIVER_USNIC;
rdma_set_device_sysfs_group(&us_ibdev->ib_dev, &usnic_attr_group);
ret = ib_device_set_netdev(&us_ibdev->ib_dev, us_ibdev->netdev, 1);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index e9352750e029..eeb07b245ef9 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -587,28 +587,18 @@ out_unlock:
return status;
}
-struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
- struct ib_cq *cq;
-
- usnic_dbg("\n");
if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-EBUSY);
+ return -EINVAL;
- return cq;
+ return 0;
}
-int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
- usnic_dbg("\n");
- kfree(cq);
- return 0;
+ return;
}
struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length,
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
index 028f322f8e9b..2aedf78c13cf 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
@@ -58,10 +58,9 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd,
int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata);
-struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c
index e312f522a66d..0b0237d41613 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.c
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.c
@@ -75,9 +75,10 @@ static void usnic_uiom_put_pages(struct list_head *chunk_list, int dirty)
for_each_sg(chunk->page_list, sg, chunk->nents, i) {
page = sg_page(sg);
pa = sg_phys(sg);
- if (!PageDirty(page) && dirty)
- set_page_dirty_lock(page);
- put_page(page);
+ if (dirty)
+ put_user_pages_dirty_lock(&page, 1);
+ else
+ put_user_page(page);
usnic_dbg("pa: %pa\n", &pa);
}
kfree(chunk);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 3c633ab58052..c142f5e7f25f 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -456,7 +456,7 @@ static inline enum pvrdma_wr_opcode ib_wr_opcode_to_pvrdma(enum ib_wr_opcode op)
return PVRDMA_WR_MASKED_ATOMIC_CMP_AND_SWP;
case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
return PVRDMA_WR_MASKED_ATOMIC_FETCH_AND_ADD;
- case IB_WR_REG_SIG_MR:
+ case IB_WR_REG_MR_INTEGRITY:
return PVRDMA_WR_REG_SIG_MR;
default:
return PVRDMA_WR_ERROR;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
index d7deb19a2800..7800e6930502 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -92,20 +92,19 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq,
/**
* pvrdma_create_cq - create completion queue
- * @ibdev: the device
+ * @ibcq: Allocated CQ
* @attr: completion queue attributes
* @udata: user data
*
- * @return: ib_cq completion queue pointer on success,
- * otherwise returns negative errno.
+ * @return: 0 on success
*/
-struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
struct pvrdma_dev *dev = to_vdev(ibdev);
- struct pvrdma_cq *cq;
+ struct pvrdma_cq *cq = to_vcq(ibcq);
int ret;
int npages;
unsigned long flags;
@@ -113,7 +112,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
union pvrdma_cmd_resp rsp;
struct pvrdma_cmd_create_cq *cmd = &req.create_cq;
struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp;
- struct pvrdma_create_cq_resp cq_resp = {0};
+ struct pvrdma_create_cq_resp cq_resp = {};
struct pvrdma_create_cq ucmd;
struct pvrdma_ucontext *context = rdma_udata_to_drv_context(
udata, struct pvrdma_ucontext, ibucontext);
@@ -122,16 +121,10 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
entries = roundup_pow_of_two(entries);
if (entries < 1 || entries > dev->dsr->caps.max_cqe)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (!atomic_add_unless(&dev->num_cqs, 1, dev->dsr->caps.max_cq))
- return ERR_PTR(-ENOMEM);
-
- cq = kzalloc(sizeof(*cq), GFP_KERNEL);
- if (!cq) {
- atomic_dec(&dev->num_cqs);
- return ERR_PTR(-ENOMEM);
- }
+ return -ENOMEM;
cq->ibcq.cqe = entries;
cq->is_kernel = !udata;
@@ -211,22 +204,19 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
dev_warn(&dev->pdev->dev,
"failed to copy back udata\n");
pvrdma_destroy_cq(&cq->ibcq, udata);
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
}
- return &cq->ibcq;
+ return 0;
err_page_dir:
pvrdma_page_dir_cleanup(dev, &cq->pdir);
err_umem:
- if (!cq->is_kernel)
- ib_umem_release(cq->umem);
+ ib_umem_release(cq->umem);
err_cq:
atomic_dec(&dev->num_cqs);
- kfree(cq);
-
- return ERR_PTR(ret);
+ return ret;
}
static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq)
@@ -235,21 +225,17 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq)
complete(&cq->free);
wait_for_completion(&cq->free);
- if (!cq->is_kernel)
- ib_umem_release(cq->umem);
+ ib_umem_release(cq->umem);
pvrdma_page_dir_cleanup(dev, &cq->pdir);
- kfree(cq);
}
/**
* pvrdma_destroy_cq - destroy completion queue
* @cq: the completion queue to destroy.
* @udata: user data or null for kernel object
- *
- * @return: 0 for success.
*/
-int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
struct pvrdma_cq *vcq = to_vcq(cq);
union pvrdma_cmd_req req;
@@ -275,8 +261,6 @@ int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
pvrdma_free_cq(dev, vcq);
atomic_dec(&dev->num_cqs);
-
- return ret;
}
static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i)
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index 40182297f87f..e580ae9cc55a 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -144,6 +144,10 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
}
static const struct ib_device_ops pvrdma_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_VMW_PVRDMA,
+ .uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION,
+
.add_gid = pvrdma_add_gid,
.alloc_mr = pvrdma_alloc_mr,
.alloc_pd = pvrdma_alloc_pd,
@@ -178,6 +182,7 @@ static const struct ib_device_ops pvrdma_dev_ops = {
.req_notify_cq = pvrdma_req_notify_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, pvrdma_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, pvrdma_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, pvrdma_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_ucontext, pvrdma_ucontext, ibucontext),
};
@@ -198,10 +203,8 @@ static int pvrdma_register_device(struct pvrdma_dev *dev)
dev->ib_dev.node_guid = dev->dsr->caps.node_guid;
dev->sys_image_guid = dev->dsr->caps.sys_image_guid;
dev->flags = 0;
- dev->ib_dev.owner = THIS_MODULE;
dev->ib_dev.num_comp_vectors = 1;
dev->ib_dev.dev.parent = &dev->pdev->dev;
- dev->ib_dev.uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION;
dev->ib_dev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -261,7 +264,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev)
if (!dev->srq_tbl)
goto err_qp_free;
}
- dev->ib_dev.driver_id = RDMA_DRIVER_VMW_PVRDMA;
ret = ib_device_set_netdev(&dev->ib_dev, dev->netdev, 1);
if (ret)
return ret;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
index 65dc47ffb8f3..f3a3d22ee8d7 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
@@ -290,8 +290,7 @@ int pvrdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
"could not deregister mem region, error: %d\n", ret);
pvrdma_page_dir_cleanup(dev, &mr->pdir);
- if (mr->umem)
- ib_umem_release(mr->umem);
+ ib_umem_release(mr->umem);
kfree(mr->pages);
kfree(mr);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index 0eaaead5baec..bca6a58a442e 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -391,12 +391,8 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
err_pdir:
pvrdma_page_dir_cleanup(dev, &qp->pdir);
err_umem:
- if (!qp->is_kernel) {
- if (qp->rumem)
- ib_umem_release(qp->rumem);
- if (qp->sumem)
- ib_umem_release(qp->sumem);
- }
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
err_qp:
kfree(qp);
atomic_dec(&dev->num_qps);
@@ -429,12 +425,8 @@ static void pvrdma_free_qp(struct pvrdma_qp *qp)
complete(&qp->free);
wait_for_completion(&qp->free);
- if (!qp->is_kernel) {
- if (qp->rumem)
- ib_umem_release(qp->rumem);
- if (qp->sumem)
- ib_umem_release(qp->sumem);
- }
+ ib_umem_release(qp->rumem);
+ ib_umem_release(qp->sumem);
pvrdma_page_dir_cleanup(dev, &qp->pdir);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
index 9d7b021e1c59..e4a48f5c0c85 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
@@ -409,10 +409,9 @@ struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
-struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
+int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index a06e6da7a026..b46714a92b7a 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -166,43 +166,37 @@ static void send_complete(struct work_struct *work)
/**
* rvt_create_cq - create a completion queue
- * @ibdev: the device this completion queue is attached to
+ * @ibcq: Allocated CQ
* @attr: creation attributes
* @udata: user data for libibverbs.so
*
* Called by ib_create_cq() in the generic verbs code.
*
- * Return: pointer to the completion queue or negative errno values
- * for failure.
+ * Return: 0 on success
*/
-struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+int rvt_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
+ struct ib_device *ibdev = ibcq->device;
struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
- struct rvt_cq *cq;
+ struct rvt_cq *cq = container_of(ibcq, struct rvt_cq, ibcq);
struct rvt_cq_wc *wc;
- struct ib_cq *ret;
u32 sz;
unsigned int entries = attr->cqe;
int comp_vector = attr->comp_vector;
+ int err;
if (attr->flags)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (entries < 1 || entries > rdi->dparms.props.max_cqe)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (comp_vector < 0)
comp_vector = 0;
comp_vector = comp_vector % rdi->ibdev.num_comp_vectors;
- /* Allocate the completion queue structure. */
- cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, rdi->dparms.node);
- if (!cq)
- return ERR_PTR(-ENOMEM);
-
/*
* Allocate the completion queue entries and head/tail pointers.
* This is allocated separately so that it can be resized and
@@ -218,36 +212,29 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
wc = udata ?
vmalloc_user(sz) :
vzalloc_node(sz, rdi->dparms.node);
- if (!wc) {
- ret = ERR_PTR(-ENOMEM);
- goto bail_cq;
- }
-
+ if (!wc)
+ return -ENOMEM;
/*
* Return the address of the WC as the offset to mmap.
* See rvt_mmap() for details.
*/
if (udata && udata->outlen >= sizeof(__u64)) {
- int err;
-
cq->ip = rvt_create_mmap_info(rdi, sz, udata, wc);
if (!cq->ip) {
- ret = ERR_PTR(-ENOMEM);
+ err = -ENOMEM;
goto bail_wc;
}
err = ib_copy_to_udata(udata, &cq->ip->offset,
sizeof(cq->ip->offset));
- if (err) {
- ret = ERR_PTR(err);
+ if (err)
goto bail_ip;
- }
}
spin_lock_irq(&rdi->n_cqs_lock);
if (rdi->n_cqs_allocated == rdi->dparms.props.max_cq) {
spin_unlock_irq(&rdi->n_cqs_lock);
- ret = ERR_PTR(-ENOMEM);
+ err = -ENOMEM;
goto bail_ip;
}
@@ -279,19 +266,14 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
INIT_WORK(&cq->comptask, send_complete);
cq->queue = wc;
- ret = &cq->ibcq;
-
trace_rvt_create_cq(cq, attr);
- goto done;
+ return 0;
bail_ip:
kfree(cq->ip);
bail_wc:
vfree(wc);
-bail_cq:
- kfree(cq);
-done:
- return ret;
+ return err;
}
/**
@@ -300,10 +282,8 @@ done:
* @udata: user data or NULL for kernel object
*
* Called by ib_destroy_cq() in the generic verbs code.
- *
- * Return: always 0
*/
-int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+void rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
struct rvt_dev_info *rdi = cq->rdi;
@@ -316,9 +296,6 @@ int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
kref_put(&cq->ip->ref, rvt_release_mmap_info);
else
vfree(cq->queue);
- kfree(cq);
-
- return 0;
}
/**
diff --git a/drivers/infiniband/sw/rdmavt/cq.h b/drivers/infiniband/sw/rdmavt/cq.h
index 3ad6faf18ecb..5e26a2eb19a4 100644
--- a/drivers/infiniband/sw/rdmavt/cq.h
+++ b/drivers/infiniband/sw/rdmavt/cq.h
@@ -51,10 +51,9 @@
#include <rdma/rdma_vt.h>
#include <rdma/rdmavt_cq.h>
-struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
-int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int rvt_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags);
int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry);
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index f48240f66b8f..0867a11d074e 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -562,8 +562,7 @@ int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
if (ret)
goto out;
rvt_deinit_mregion(&mr->mr);
- if (mr->umem)
- ib_umem_release(mr->umem);
+ ib_umem_release(mr->umem);
kfree(mr);
out:
return ret;
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c5a50614a6c6..0d804a58f954 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -989,9 +989,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
case IB_QPT_UC:
case IB_QPT_RC:
case IB_QPT_UD:
- sz = sizeof(struct rvt_sge) *
- init_attr->cap.max_send_sge +
- sizeof(struct rvt_swqe);
+ sz = struct_size(swq, sg_list, init_attr->cap.max_send_sge);
swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node);
if (!swq)
return ERR_PTR(-ENOMEM);
@@ -1856,10 +1854,9 @@ static inline int rvt_qp_is_avail(
/* see rvt_qp_wqe_unreserve() */
smp_mb__before_atomic();
- reserved_used = atomic_read(&qp->s_reserved_used);
if (unlikely(reserved_op)) {
/* see rvt_qp_wqe_unreserve() */
- smp_mb__before_atomic();
+ reserved_used = atomic_read(&qp->s_reserved_used);
if (reserved_used >= rdi->dparms.reserved_operations)
return -ENOMEM;
return 0;
@@ -1867,14 +1864,13 @@ static inline int rvt_qp_is_avail(
/* non-reserved operations */
if (likely(qp->s_avail))
return 0;
- slast = READ_ONCE(qp->s_last);
+ /* See rvt_qp_complete_swqe() */
+ slast = smp_load_acquire(&qp->s_last);
if (qp->s_head >= slast)
avail = qp->s_size - (qp->s_head - slast);
else
avail = slast - qp->s_head;
- /* see rvt_qp_wqe_unreserve() */
- smp_mb__before_atomic();
reserved_used = atomic_read(&qp->s_reserved_used);
avail = avail - 1 -
(rdi->dparms.reserved_operations - reserved_used);
@@ -2667,27 +2663,16 @@ void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
enum ib_wc_status status)
{
u32 old_last, last;
- struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
+ struct rvt_dev_info *rdi;
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
return;
+ rdi = ib_to_rvt(qp->ibqp.device);
- last = qp->s_last;
- old_last = last;
- trace_rvt_qp_send_completion(qp, wqe, last);
- if (++last >= qp->s_size)
- last = 0;
- trace_rvt_qp_send_completion(qp, wqe, last);
- qp->s_last = last;
- /* See post_send() */
- barrier();
- rvt_put_qp_swqe(qp, wqe);
-
- rvt_qp_swqe_complete(qp,
- wqe,
- rdi->wc_opcode[wqe->wr.opcode],
- status);
-
+ old_last = qp->s_last;
+ trace_rvt_qp_send_completion(qp, wqe, old_last);
+ last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode],
+ status);
if (qp->s_acked == old_last)
qp->s_acked = last;
if (qp->s_cur == old_last)
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 9546a837a8ac..18da1e1ea979 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -382,6 +382,8 @@ enum {
};
static const struct ib_device_ops rvt_dev_ops = {
+ .uverbs_abi_ver = RVT_UVERBS_ABI_VERSION,
+
.alloc_fmr = rvt_alloc_fmr,
.alloc_mr = rvt_alloc_mr,
.alloc_pd = rvt_alloc_pd,
@@ -427,6 +429,7 @@ static const struct ib_device_ops rvt_dev_ops = {
.unmap_fmr = rvt_unmap_fmr,
INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext),
@@ -530,7 +533,7 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb)
*
* Return: 0 on success otherwise an errno.
*/
-int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
+int rvt_register_device(struct rvt_dev_info *rdi)
{
int ret = 0, i;
@@ -600,7 +603,6 @@ int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
* exactly which functions rdmavt supports, nor do they know the ABI
* version, so we do all of this sort of stuff here.
*/
- rdi->ibdev.uverbs_abi_ver = RVT_UVERBS_ABI_VERSION;
rdi->ibdev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
@@ -636,7 +638,6 @@ int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
if (!rdi->ibdev.num_comp_vectors)
rdi->ibdev.num_comp_vectors = 1;
- rdi->ibdev.driver_id = driver_id;
/* We are now good to announce we exist */
ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev));
if (ret) {
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index f501f72489d8..ea6a819b7167 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -96,8 +96,7 @@ void rxe_mem_cleanup(struct rxe_pool_entry *arg)
struct rxe_mem *mem = container_of(arg, typeof(*mem), pelem);
int i;
- if (mem->umem)
- ib_umem_release(mem->umem);
+ ib_umem_release(mem->umem);
if (mem->map) {
for (i = 0; i < mem->num_map; i++)
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 56cf18af016a..fbcbac52290b 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -72,6 +72,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
[RXE_TYPE_CQ] = {
.name = "rxe-cq",
.size = sizeof(struct rxe_cq),
+ .flags = RXE_POOL_NO_ALLOC,
.cleanup = rxe_cq_cleanup,
},
[RXE_TYPE_MR] = {
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 8c3e2a18cfe4..4ebdfcf4d33e 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -778,55 +778,43 @@ err1:
return err;
}
-static struct ib_cq *rxe_create_cq(struct ib_device *dev,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
+static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
{
int err;
+ struct ib_device *dev = ibcq->device;
struct rxe_dev *rxe = to_rdev(dev);
- struct rxe_cq *cq;
+ struct rxe_cq *cq = to_rcq(ibcq);
struct rxe_create_cq_resp __user *uresp = NULL;
if (udata) {
if (udata->outlen < sizeof(*uresp))
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
uresp = udata->outbuf;
}
if (attr->flags)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
if (err)
- goto err1;
-
- cq = rxe_alloc(&rxe->cq_pool);
- if (!cq) {
- err = -ENOMEM;
- goto err1;
- }
+ return err;
err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
uresp);
if (err)
- goto err2;
-
- return &cq->ibcq;
+ return err;
-err2:
- rxe_drop_ref(cq);
-err1:
- return ERR_PTR(err);
+ return rxe_add_to_pool(&rxe->cq_pool, &cq->pelem);
}
-static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+static void rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct rxe_cq *cq = to_rcq(ibcq);
rxe_cq_disable(cq);
rxe_drop_ref(cq);
- return 0;
}
static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
@@ -1111,6 +1099,10 @@ static int rxe_enable_driver(struct ib_device *ib_dev)
}
static const struct ib_device_ops rxe_dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_RXE,
+ .uverbs_abi_ver = RXE_UVERBS_ABI_VERSION,
+
.alloc_hw_stats = rxe_ib_alloc_hw_stats,
.alloc_mr = rxe_alloc_mr,
.alloc_pd = rxe_alloc_pd,
@@ -1157,6 +1149,7 @@ static const struct ib_device_ops rxe_dev_ops = {
.resize_cq = rxe_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
@@ -1170,7 +1163,6 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
strlcpy(dev->node_desc, "rxe", sizeof(dev->node_desc));
- dev->owner = THIS_MODULE;
dev->node_type = RDMA_NODE_IB_CA;
dev->phys_port_cnt = 1;
dev->num_comp_vectors = num_possible_cpus();
@@ -1182,7 +1174,6 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
dma_coerce_mask_and_coherent(&dev->dev,
dma_get_required_mask(&dev->dev));
- dev->uverbs_abi_ver = RXE_UVERBS_ABI_VERSION;
dev->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT)
| BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)
| BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE)
@@ -1230,7 +1221,6 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
rxe->tfm = tfm;
rdma_set_device_sysfs_group(dev, &rxe_attr_group);
- dev->driver_id = RDMA_DRIVER_RXE;
err = ib_register_device(dev, ibdev_name);
if (err)
pr_warn("%s failed with error %d\n", __func__, err);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index e8be7f44e3be..6c997d39a418 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -85,8 +85,8 @@ struct rxe_cqe {
};
struct rxe_cq {
- struct rxe_pool_entry pelem;
struct ib_cq ibcq;
+ struct rxe_pool_entry pelem;
struct rxe_queue *queue;
spinlock_t cq_lock;
u8 notify;
diff --git a/drivers/infiniband/ulp/ipoib/Kconfig b/drivers/infiniband/ulp/ipoib/Kconfig
index 4760ce465d89..7af68604af77 100644
--- a/drivers/infiniband/ulp/ipoib/Kconfig
+++ b/drivers/infiniband/ulp/ipoib/Kconfig
@@ -7,7 +7,7 @@ config INFINIBAND_IPOIB
transports IP packets over InfiniBand so you can use your IB
device as a fancy NIC.
- See Documentation/infiniband/ipoib.txt for more information
+ See Documentation/infiniband/ipoib.rst for more information
config INFINIBAND_IPOIB_CM
bool "IP-over-InfiniBand Connected Mode support"
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
index 83429925dfc6..63e4f9d15fd9 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
@@ -138,7 +138,6 @@ static void ipoib_get_strings(struct net_device __always_unused *dev,
p += ETH_GSTRING_LEN;
}
break;
- case ETH_SS_TEST:
default:
break;
}
@@ -149,7 +148,6 @@ static int ipoib_get_sset_count(struct net_device __always_unused *dev,
switch (sset) {
case ETH_SS_STATS:
return IPOIB_GLOBAL_STATS_LEN;
- case ETH_SS_TEST:
default:
break;
}
@@ -222,6 +220,7 @@ static const struct ethtool_ops ipoib_ethtool_ops = {
.get_strings = ipoib_get_strings,
.get_ethtool_stats = ipoib_get_ethtool_stats,
.get_sset_count = ipoib_get_sset_count,
+ .get_link = ethtool_op_get_link,
};
void ipoib_set_ethtool_ops(struct net_device *dev)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index ba09068f6200..b69304d28f06 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -260,11 +260,8 @@ void ipoib_transport_dev_cleanup(struct net_device *dev)
priv->qp = NULL;
}
- if (ib_destroy_cq(priv->send_cq))
- ipoib_warn(priv, "ib_cq_destroy (send) failed\n");
-
- if (ib_destroy_cq(priv->recv_cq))
- ipoib_warn(priv, "ib_cq_destroy (recv) failed\n");
+ ib_destroy_cq(priv->send_cq);
+ ib_destroy_cq(priv->recv_cq);
}
void ipoib_event(struct ib_event_handler *handler,
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 9c185a8dabd3..c7a3d75fb308 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -205,7 +205,8 @@ iser_initialize_task_headers(struct iscsi_task *task,
goto out;
}
- tx_desc->wr_idx = 0;
+ tx_desc->inv_wr.next = NULL;
+ tx_desc->reg_wr.wr.next = NULL;
tx_desc->mapped = true;
tx_desc->dma_addr = dma_addr;
tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
@@ -406,13 +407,10 @@ static u8
iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
{
struct iscsi_iser_task *iser_task = task->dd_data;
+ enum iser_data_dir dir = iser_task->dir[ISER_DIR_IN] ?
+ ISER_DIR_IN : ISER_DIR_OUT;
- if (iser_task->dir[ISER_DIR_IN])
- return iser_check_task_pi_status(iser_task, ISER_DIR_IN,
- sector);
- else
- return iser_check_task_pi_status(iser_task, ISER_DIR_OUT,
- sector);
+ return iser_check_task_pi_status(iser_task, dir, sector);
}
/**
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 36d525110fd2..39bf213444cb 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -225,14 +225,6 @@ enum iser_desc_type {
ISCSI_TX_DATAOUT
};
-/* Maximum number of work requests per task:
- * Data memory region local invalidate + fast registration
- * Protection memory region local invalidate + fast registration
- * Signature memory region local invalidate + fast registration
- * PDU send
- */
-#define ISER_MAX_WRS 7
-
/**
* struct iser_tx_desc - iSER TX descriptor
*
@@ -245,11 +237,9 @@ enum iser_desc_type {
* unsolicited data-out or control
* @num_sge: number sges used on this TX task
* @mapped: Is the task header mapped
- * @wr_idx: Current WR index
- * @wrs: Array of WRs per task
- * @data_reg: Data buffer registration details
- * @prot_reg: Protection buffer registration details
- * @sig_attrs: Signature attributes
+ * reg_wr: registration WR
+ * send_wr: send WR
+ * inv_wr: invalidate WR
*/
struct iser_tx_desc {
struct iser_ctrl iser_header;
@@ -260,15 +250,9 @@ struct iser_tx_desc {
int num_sge;
struct ib_cqe cqe;
bool mapped;
- u8 wr_idx;
- union iser_wr {
- struct ib_send_wr send;
- struct ib_reg_wr fast_reg;
- struct ib_sig_handover_wr sig;
- } wrs[ISER_MAX_WRS];
- struct iser_mem_reg data_reg;
- struct iser_mem_reg prot_reg;
- struct ib_sig_attrs sig_attrs;
+ struct ib_reg_wr reg_wr;
+ struct ib_send_wr send_wr;
+ struct ib_send_wr inv_wr;
};
#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
@@ -388,6 +372,7 @@ struct iser_device {
*
* @mr: memory region
* @fmr_pool: pool of fmrs
+ * @sig_mr: signature memory region
* @page_vec: fast reg page list used by fmr pool
* @mr_valid: is mr valid indicator
*/
@@ -396,36 +381,22 @@ struct iser_reg_resources {
struct ib_mr *mr;
struct ib_fmr_pool *fmr_pool;
};
+ struct ib_mr *sig_mr;
struct iser_page_vec *page_vec;
u8 mr_valid:1;
};
/**
- * struct iser_pi_context - Protection information context
- *
- * @rsc: protection buffer registration resources
- * @sig_mr: signature enable memory region
- * @sig_mr_valid: is sig_mr valid indicator
- * @sig_protected: is region protected indicator
- */
-struct iser_pi_context {
- struct iser_reg_resources rsc;
- struct ib_mr *sig_mr;
- u8 sig_mr_valid:1;
- u8 sig_protected:1;
-};
-
-/**
* struct iser_fr_desc - Fast registration descriptor
*
* @list: entry in connection fastreg pool
* @rsc: data buffer registration resources
- * @pi_ctx: protection information context
+ * @sig_protected: is region protected indicator
*/
struct iser_fr_desc {
struct list_head list;
struct iser_reg_resources rsc;
- struct iser_pi_context *pi_ctx;
+ bool sig_protected;
struct list_head all_list;
};
@@ -674,21 +645,6 @@ void
iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
struct iser_fr_desc *desc);
-static inline struct ib_send_wr *
-iser_tx_next_wr(struct iser_tx_desc *tx_desc)
-{
- struct ib_send_wr *cur_wr = &tx_desc->wrs[tx_desc->wr_idx].send;
- struct ib_send_wr *last_wr;
-
- if (tx_desc->wr_idx) {
- last_wr = &tx_desc->wrs[tx_desc->wr_idx - 1].send;
- last_wr->next = cur_wr;
- }
- tx_desc->wr_idx++;
-
- return cur_wr;
-}
-
static inline struct iser_conn *
to_iser_conn(struct ib_conn *ib_conn)
{
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 96af06cfe0af..5cbb4b3a0566 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -592,15 +592,14 @@ void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc)
static inline int
iser_inv_desc(struct iser_fr_desc *desc, u32 rkey)
{
- if (likely(rkey == desc->rsc.mr->rkey)) {
- desc->rsc.mr_valid = 0;
- } else if (likely(desc->pi_ctx && rkey == desc->pi_ctx->sig_mr->rkey)) {
- desc->pi_ctx->sig_mr_valid = 0;
- } else {
+ if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) ||
+ (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) {
iser_err("Bogus remote invalidation for rkey %#x\n", rkey);
return -EINVAL;
}
+ desc->rsc.mr_valid = 0;
+
return 0;
}
@@ -750,6 +749,9 @@ void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
iser_task->prot[ISER_DIR_IN].data_len = 0;
iser_task->prot[ISER_DIR_OUT].data_len = 0;
+ iser_task->prot[ISER_DIR_IN].dma_nents = 0;
+ iser_task->prot[ISER_DIR_OUT].dma_nents = 0;
+
memset(&iser_task->rdma_reg[ISER_DIR_IN], 0,
sizeof(struct iser_mem_reg));
memset(&iser_task->rdma_reg[ISER_DIR_OUT], 0,
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 2ba70729d7b0..2cc89a9b9e9b 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -302,8 +302,7 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
}
static void
-iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs,
- struct ib_sig_domain *domain)
+iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain)
{
domain->sig_type = IB_SIG_TYPE_T10_DIF;
domain->sig.dif.pi_interval = scsi_prot_interval(sc);
@@ -326,21 +325,21 @@ iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)
case SCSI_PROT_WRITE_INSERT:
case SCSI_PROT_READ_STRIP:
sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
- iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
+ iser_set_dif_domain(sc, &sig_attrs->wire);
sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
break;
case SCSI_PROT_READ_INSERT:
case SCSI_PROT_WRITE_STRIP:
sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
- iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
+ iser_set_dif_domain(sc, &sig_attrs->mem);
sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
IB_T10DIF_CSUM : IB_T10DIF_CRC;
break;
case SCSI_PROT_READ_PASS:
case SCSI_PROT_WRITE_PASS:
- iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
+ iser_set_dif_domain(sc, &sig_attrs->wire);
sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
- iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
+ iser_set_dif_domain(sc, &sig_attrs->mem);
sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
IB_T10DIF_CSUM : IB_T10DIF_CRC;
break;
@@ -366,27 +365,29 @@ iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
static inline void
iser_inv_rkey(struct ib_send_wr *inv_wr,
struct ib_mr *mr,
- struct ib_cqe *cqe)
+ struct ib_cqe *cqe,
+ struct ib_send_wr *next_wr)
{
inv_wr->opcode = IB_WR_LOCAL_INV;
inv_wr->wr_cqe = cqe;
inv_wr->ex.invalidate_rkey = mr->rkey;
inv_wr->send_flags = 0;
inv_wr->num_sge = 0;
+ inv_wr->next = next_wr;
}
static int
iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
- struct iser_pi_context *pi_ctx,
- struct iser_mem_reg *data_reg,
- struct iser_mem_reg *prot_reg,
+ struct iser_data_buf *mem,
+ struct iser_data_buf *sig_mem,
+ struct iser_reg_resources *rsc,
struct iser_mem_reg *sig_reg)
{
struct iser_tx_desc *tx_desc = &iser_task->desc;
- struct ib_sig_attrs *sig_attrs = &tx_desc->sig_attrs;
struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
- struct ib_sig_handover_wr *wr;
- struct ib_mr *mr = pi_ctx->sig_mr;
+ struct ib_mr *mr = rsc->sig_mr;
+ struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
+ struct ib_reg_wr *wr = &tx_desc->reg_wr;
int ret;
memset(sig_attrs, 0, sizeof(*sig_attrs));
@@ -396,33 +397,36 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask);
- if (pi_ctx->sig_mr_valid)
- iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
+ if (rsc->mr_valid)
+ iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr);
ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
- wr = container_of(iser_tx_next_wr(tx_desc), struct ib_sig_handover_wr,
- wr);
- wr->wr.opcode = IB_WR_REG_SIG_MR;
+ ret = ib_map_mr_sg_pi(mr, mem->sg, mem->dma_nents, NULL,
+ sig_mem->sg, sig_mem->dma_nents, NULL, SZ_4K);
+ if (unlikely(ret)) {
+ iser_err("failed to map PI sg (%d)\n",
+ mem->dma_nents + sig_mem->dma_nents);
+ goto err;
+ }
+
+ memset(wr, 0, sizeof(*wr));
+ wr->wr.next = &tx_desc->send_wr;
+ wr->wr.opcode = IB_WR_REG_MR_INTEGRITY;
wr->wr.wr_cqe = cqe;
- wr->wr.sg_list = &data_reg->sge;
- wr->wr.num_sge = 1;
+ wr->wr.num_sge = 0;
wr->wr.send_flags = 0;
- wr->sig_attrs = sig_attrs;
- wr->sig_mr = mr;
- if (scsi_prot_sg_count(iser_task->sc))
- wr->prot = &prot_reg->sge;
- else
- wr->prot = NULL;
- wr->access_flags = IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_READ |
- IB_ACCESS_REMOTE_WRITE;
- pi_ctx->sig_mr_valid = 1;
+ wr->mr = mr;
+ wr->key = mr->rkey;
+ wr->access = IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_READ |
+ IB_ACCESS_REMOTE_WRITE;
+ rsc->mr_valid = 1;
sig_reg->sge.lkey = mr->lkey;
sig_reg->rkey = mr->rkey;
- sig_reg->sge.addr = 0;
- sig_reg->sge.length = scsi_transfer_length(iser_task->sc);
+ sig_reg->sge.addr = mr->iova;
+ sig_reg->sge.length = mr->length;
iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n",
sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr,
@@ -439,11 +443,11 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
struct iser_tx_desc *tx_desc = &iser_task->desc;
struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
struct ib_mr *mr = rsc->mr;
- struct ib_reg_wr *wr;
+ struct ib_reg_wr *wr = &tx_desc->reg_wr;
int n;
if (rsc->mr_valid)
- iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
+ iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr);
ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
@@ -454,7 +458,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
return n < 0 ? n : -EINVAL;
}
- wr = container_of(iser_tx_next_wr(tx_desc), struct ib_reg_wr, wr);
+ wr->wr.next = &tx_desc->send_wr;
wr->wr.opcode = IB_WR_REG_MR;
wr->wr.wr_cqe = cqe;
wr->wr.send_flags = 0;
@@ -479,21 +483,6 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
}
static int
-iser_reg_prot_sg(struct iscsi_iser_task *task,
- struct iser_data_buf *mem,
- struct iser_fr_desc *desc,
- bool use_dma_key,
- struct iser_mem_reg *reg)
-{
- struct iser_device *device = task->iser_conn->ib_conn.device;
-
- if (use_dma_key)
- return iser_reg_dma(device, mem, reg);
-
- return device->reg_ops->reg_mem(task, mem, &desc->pi_ctx->rsc, reg);
-}
-
-static int
iser_reg_data_sg(struct iscsi_iser_task *task,
struct iser_data_buf *mem,
struct iser_fr_desc *desc,
@@ -516,7 +505,6 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
struct iser_device *device = ib_conn->device;
struct iser_data_buf *mem = &task->data[dir];
struct iser_mem_reg *reg = &task->rdma_reg[dir];
- struct iser_mem_reg *data_reg;
struct iser_fr_desc *desc = NULL;
bool use_dma_key;
int err;
@@ -529,32 +517,17 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
reg->mem_h = desc;
}
- if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL)
- data_reg = reg;
- else
- data_reg = &task->desc.data_reg;
-
- err = iser_reg_data_sg(task, mem, desc, use_dma_key, data_reg);
- if (unlikely(err))
- goto err_reg;
-
- if (scsi_get_prot_op(task->sc) != SCSI_PROT_NORMAL) {
- struct iser_mem_reg *prot_reg = &task->desc.prot_reg;
-
- if (scsi_prot_sg_count(task->sc)) {
- mem = &task->prot[dir];
- err = iser_reg_prot_sg(task, mem, desc,
- use_dma_key, prot_reg);
- if (unlikely(err))
- goto err_reg;
- }
-
- err = iser_reg_sig_mr(task, desc->pi_ctx, data_reg,
- prot_reg, reg);
+ if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) {
+ err = iser_reg_data_sg(task, mem, desc, use_dma_key, reg);
+ if (unlikely(err))
+ goto err_reg;
+ } else {
+ err = iser_reg_sig_mr(task, mem, &task->prot[dir],
+ &desc->rsc, reg);
if (unlikely(err))
goto err_reg;
- desc->pi_ctx->sig_protected = 1;
+ desc->sig_protected = 1;
}
return 0;
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 4ff3d98fa6a4..a6548de0e218 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -233,116 +233,63 @@ void iser_free_fmr_pool(struct ib_conn *ib_conn)
kfree(desc);
}
-static int
-iser_alloc_reg_res(struct iser_device *device,
- struct ib_pd *pd,
- struct iser_reg_resources *res,
- unsigned int size)
+static struct iser_fr_desc *
+iser_create_fastreg_desc(struct iser_device *device,
+ struct ib_pd *pd,
+ bool pi_enable,
+ unsigned int size)
{
+ struct iser_fr_desc *desc;
struct ib_device *ib_dev = device->ib_device;
enum ib_mr_type mr_type;
int ret;
+ desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+ if (!desc)
+ return ERR_PTR(-ENOMEM);
+
if (ib_dev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
mr_type = IB_MR_TYPE_SG_GAPS;
else
mr_type = IB_MR_TYPE_MEM_REG;
- res->mr = ib_alloc_mr(pd, mr_type, size);
- if (IS_ERR(res->mr)) {
- ret = PTR_ERR(res->mr);
+ desc->rsc.mr = ib_alloc_mr(pd, mr_type, size);
+ if (IS_ERR(desc->rsc.mr)) {
+ ret = PTR_ERR(desc->rsc.mr);
iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret);
- return ret;
+ goto err_alloc_mr;
}
- res->mr_valid = 0;
-
- return 0;
-}
-static void
-iser_free_reg_res(struct iser_reg_resources *rsc)
-{
- ib_dereg_mr(rsc->mr);
-}
-
-static int
-iser_alloc_pi_ctx(struct iser_device *device,
- struct ib_pd *pd,
- struct iser_fr_desc *desc,
- unsigned int size)
-{
- struct iser_pi_context *pi_ctx = NULL;
- int ret;
-
- desc->pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL);
- if (!desc->pi_ctx)
- return -ENOMEM;
-
- pi_ctx = desc->pi_ctx;
-
- ret = iser_alloc_reg_res(device, pd, &pi_ctx->rsc, size);
- if (ret) {
- iser_err("failed to allocate reg_resources\n");
- goto alloc_reg_res_err;
- }
-
- pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2);
- if (IS_ERR(pi_ctx->sig_mr)) {
- ret = PTR_ERR(pi_ctx->sig_mr);
- goto sig_mr_failure;
+ if (pi_enable) {
+ desc->rsc.sig_mr = ib_alloc_mr_integrity(pd, size, size);
+ if (IS_ERR(desc->rsc.sig_mr)) {
+ ret = PTR_ERR(desc->rsc.sig_mr);
+ iser_err("Failed to allocate sig_mr err=%d\n", ret);
+ goto err_alloc_mr_integrity;
+ }
}
- pi_ctx->sig_mr_valid = 0;
- desc->pi_ctx->sig_protected = 0;
-
- return 0;
+ desc->rsc.mr_valid = 0;
-sig_mr_failure:
- iser_free_reg_res(&pi_ctx->rsc);
-alloc_reg_res_err:
- kfree(desc->pi_ctx);
+ return desc;
- return ret;
-}
+err_alloc_mr_integrity:
+ ib_dereg_mr(desc->rsc.mr);
+err_alloc_mr:
+ kfree(desc);
-static void
-iser_free_pi_ctx(struct iser_pi_context *pi_ctx)
-{
- iser_free_reg_res(&pi_ctx->rsc);
- ib_dereg_mr(pi_ctx->sig_mr);
- kfree(pi_ctx);
+ return ERR_PTR(ret);
}
-static struct iser_fr_desc *
-iser_create_fastreg_desc(struct iser_device *device,
- struct ib_pd *pd,
- bool pi_enable,
- unsigned int size)
+static void iser_destroy_fastreg_desc(struct iser_fr_desc *desc)
{
- struct iser_fr_desc *desc;
- int ret;
-
- desc = kzalloc(sizeof(*desc), GFP_KERNEL);
- if (!desc)
- return ERR_PTR(-ENOMEM);
-
- ret = iser_alloc_reg_res(device, pd, &desc->rsc, size);
- if (ret)
- goto reg_res_alloc_failure;
+ struct iser_reg_resources *res = &desc->rsc;
- if (pi_enable) {
- ret = iser_alloc_pi_ctx(device, pd, desc, size);
- if (ret)
- goto pi_ctx_alloc_failure;
+ ib_dereg_mr(res->mr);
+ if (res->sig_mr) {
+ ib_dereg_mr(res->sig_mr);
+ res->sig_mr = NULL;
}
-
- return desc;
-
-pi_ctx_alloc_failure:
- iser_free_reg_res(&desc->rsc);
-reg_res_alloc_failure:
kfree(desc);
-
- return ERR_PTR(ret);
}
/**
@@ -399,10 +346,7 @@ void iser_free_fastreg_pool(struct ib_conn *ib_conn)
list_for_each_entry_safe(desc, tmp, &fr_pool->all_list, all_list) {
list_del(&desc->all_list);
- iser_free_reg_res(&desc->rsc);
- if (desc->pi_ctx)
- iser_free_pi_ctx(desc->pi_ctx);
- kfree(desc);
+ iser_destroy_fastreg_desc(desc);
++i;
}
@@ -455,7 +399,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
init_attr.qp_type = IB_QPT_RC;
if (ib_conn->pi_support) {
init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
- init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
+ init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
iser_conn->max_cmds =
ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
} else {
@@ -707,6 +651,7 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
struct ib_device_attr *attr = &device->ib_device->attrs;
unsigned short sg_tablesize, sup_sg_tablesize;
unsigned short reserved_mr_pages;
+ u32 max_num_sg;
/*
* FRs without SG_GAPS or FMRs can only map up to a (device) page per
@@ -720,12 +665,17 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
else
reserved_mr_pages = 1;
+ if (iser_conn->ib_conn.pi_support)
+ max_num_sg = attr->max_pi_fast_reg_page_list_len;
+ else
+ max_num_sg = attr->max_fast_reg_page_list_len;
+
sg_tablesize = DIV_ROUND_UP(max_sectors * 512, SIZE_4K);
if (attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)
sup_sg_tablesize =
min_t(
uint, ISCSI_ISER_MAX_SG_TABLESIZE,
- attr->max_fast_reg_page_list_len - reserved_mr_pages);
+ max_num_sg - reserved_mr_pages);
else
sup_sg_tablesize = ISCSI_ISER_MAX_SG_TABLESIZE;
@@ -762,7 +712,7 @@ static void iser_addr_handler(struct rdma_cm_id *cma_id)
/* connection T10-PI support */
if (iser_pi_enable) {
if (!(device->ib_device->attrs.device_cap_flags &
- IB_DEVICE_SIGNATURE_HANDOVER)) {
+ IB_DEVICE_INTEGRITY_HANDOVER)) {
iser_warn("T10-PI requested but not supported on %s, "
"continue without T10-PI\n",
dev_name(&ib_conn->device->ib_device->dev));
@@ -1087,7 +1037,8 @@ int iser_post_recvm(struct iser_conn *iser_conn, int count)
int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
bool signal)
{
- struct ib_send_wr *wr = iser_tx_next_wr(tx_desc);
+ struct ib_send_wr *wr = &tx_desc->send_wr;
+ struct ib_send_wr *first_wr;
int ib_ret;
ib_dma_sync_single_for_device(ib_conn->device->ib_device,
@@ -1101,7 +1052,14 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
wr->opcode = IB_WR_SEND;
wr->send_flags = signal ? IB_SEND_SIGNALED : 0;
- ib_ret = ib_post_send(ib_conn->qp, &tx_desc->wrs[0].send, NULL);
+ if (tx_desc->inv_wr.next)
+ first_wr = &tx_desc->inv_wr;
+ else if (tx_desc->reg_wr.wr.next)
+ first_wr = &tx_desc->reg_wr.wr;
+ else
+ first_wr = wr;
+
+ ib_ret = ib_post_send(ib_conn->qp, first_wr, NULL);
if (ib_ret)
iser_err("ib_post_send failed, ret:%d opcode:%d\n",
ib_ret, wr->opcode);
@@ -1118,9 +1076,9 @@ u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
struct ib_mr_status mr_status;
int ret;
- if (desc && desc->pi_ctx->sig_protected) {
- desc->pi_ctx->sig_protected = 0;
- ret = ib_check_mr_status(desc->pi_ctx->sig_mr,
+ if (desc && desc->sig_protected) {
+ desc->sig_protected = 0;
+ ret = ib_check_mr_status(desc->rsc.sig_mr,
IB_MR_CHECK_SIG_STATUS, &mr_status);
if (ret) {
pr_err("ib_check_mr_status failed, ret %d\n", ret);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 0205cf142b2f..a1a035270cab 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -133,7 +133,7 @@ isert_create_qp(struct isert_conn *isert_conn,
attr.sq_sig_type = IB_SIGNAL_REQ_WR;
attr.qp_type = IB_QPT_RC;
if (device->pi_capable)
- attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
+ attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
ret = rdma_create_qp(cma_id, device->pd, &attr);
if (ret) {
@@ -309,7 +309,7 @@ isert_create_device_ib_res(struct isert_device *device)
/* Check signature cap */
device->pi_capable = ib_dev->attrs.device_cap_flags &
- IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
+ IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
return 0;
@@ -1669,7 +1669,7 @@ isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
isert_dbg("Cmd %p\n", isert_cmd);
- ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr);
+ ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr);
isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
if (ret) {
@@ -1715,7 +1715,7 @@ isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
iscsit_stop_dataout_timer(cmd);
if (isert_prot_cmd(isert_conn, se_cmd))
- ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr);
+ ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr);
isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
cmd->write_data_done = 0;
@@ -2059,8 +2059,7 @@ isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
}
static inline void
-isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
- struct ib_sig_domain *domain)
+isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain)
{
domain->sig_type = IB_SIG_TYPE_T10_DIF;
domain->sig.dif.bg_type = IB_T10DIF_CRC;
@@ -2088,17 +2087,17 @@ isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
case TARGET_PROT_DIN_INSERT:
case TARGET_PROT_DOUT_STRIP:
sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
- isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
+ isert_set_dif_domain(se_cmd, &sig_attrs->wire);
break;
case TARGET_PROT_DOUT_INSERT:
case TARGET_PROT_DIN_STRIP:
sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
- isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
+ isert_set_dif_domain(se_cmd, &sig_attrs->mem);
break;
case TARGET_PROT_DIN_PASS:
case TARGET_PROT_DOUT_PASS:
- isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
- isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
+ isert_set_dif_domain(se_cmd, &sig_attrs->wire);
+ isert_set_dif_domain(se_cmd, &sig_attrs->mem);
break;
default:
isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 4305da2c9037..0b09d0cd9b3c 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -3483,13 +3483,14 @@ static const match_table_t srp_opt_tokens = {
* @net: [in] Network namespace.
* @sa: [out] Address family, IP address and port number.
* @addr_port_str: [in] IP address and port number.
+ * @has_port: [out] Whether or not @addr_port_str includes a port number.
*
* Parse the following address formats:
* - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5.
* - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5.
*/
static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
- const char *addr_port_str)
+ const char *addr_port_str, bool *has_port)
{
char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL);
char *port_str;
@@ -3498,9 +3499,12 @@ static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
if (!addr)
return -ENOMEM;
port_str = strrchr(addr, ':');
- if (!port_str)
- return -EINVAL;
- *port_str++ = '\0';
+ if (port_str && strchr(port_str, ']'))
+ port_str = NULL;
+ if (port_str)
+ *port_str++ = '\0';
+ if (has_port)
+ *has_port = port_str != NULL;
ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa);
if (ret && addr[0]) {
addr_end = addr + strlen(addr) - 1;
@@ -3522,6 +3526,7 @@ static int srp_parse_options(struct net *net, const char *buf,
char *p;
substring_t args[MAX_OPT_ARGS];
unsigned long long ull;
+ bool has_port;
int opt_mask = 0;
int token;
int ret = -EINVAL;
@@ -3620,7 +3625,8 @@ static int srp_parse_options(struct net *net, const char *buf,
ret = -ENOMEM;
goto out;
}
- ret = srp_parse_in(net, &target->rdma_cm.src.ss, p);
+ ret = srp_parse_in(net, &target->rdma_cm.src.ss, p,
+ NULL);
if (ret < 0) {
pr_warn("bad source parameter '%s'\n", p);
kfree(p);
@@ -3636,7 +3642,10 @@ static int srp_parse_options(struct net *net, const char *buf,
ret = -ENOMEM;
goto out;
}
- ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p);
+ ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p,
+ &has_port);
+ if (!has_port)
+ ret = -EINVAL;
if (ret < 0) {
pr_warn("bad dest parameter '%s'\n", p);
kfree(p);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 243368dc23db..cf8d2b74a2fc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -13,7 +13,7 @@ obj-$(CONFIG_MLX5_CORE) += mlx5_core.o
#
mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \
- transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
+ transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
lib/devcom.o diag/fs_tracepoint.o diag/fw_tracer.o
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index e94686c42000..8cdd7e66f8df 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -316,7 +316,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
case MLX5_CMD_OP_DEALLOC_MEMIC:
case MLX5_CMD_OP_PAGE_FAULT_RESUME:
- case MLX5_CMD_OP_QUERY_HOST_PARAMS:
+ case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
return MLX5_CMD_STAT_OK;
case MLX5_CMD_OP_QUERY_HCA_CAP:
@@ -632,7 +632,7 @@ const char *mlx5_command_str(int command)
MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
- MLX5_COMMAND_STR_CASE(QUERY_HOST_PARAMS);
+ MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
MLX5_COMMAND_STR_CASE(CREATE_UCTX);
MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
MLX5_COMMAND_STR_CASE(CREATE_UMEM);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
index 0ccd6d40baf7..d2228e37450f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
@@ -83,30 +83,3 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev)
mlx5_peer_pf_cleanup(dev);
}
-
-static int mlx5_query_host_params_context(struct mlx5_core_dev *dev,
- u32 *out, int outlen)
-{
- u32 in[MLX5_ST_SZ_DW(query_host_params_in)] = {};
-
- MLX5_SET(query_host_params_in, in, opcode,
- MLX5_CMD_OP_QUERY_HOST_PARAMS);
-
- return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
-}
-
-int mlx5_query_host_params_num_vfs(struct mlx5_core_dev *dev, int *num_vf)
-{
- u32 out[MLX5_ST_SZ_DW(query_host_params_out)] = {};
- int err;
-
- err = mlx5_query_host_params_context(dev, out, sizeof(out));
- if (err)
- return err;
-
- *num_vf = MLX5_GET(query_host_params_out, out,
- host_params_context.host_num_of_vfs);
- mlx5_core_dbg(dev, "host_num_of_vfs %d\n", *num_vf);
-
- return 0;
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.h b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.h
index 346372df218f..d3d7a00a02ac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.h
@@ -16,7 +16,6 @@ enum {
bool mlx5_read_embedded_cpu(struct mlx5_core_dev *dev);
int mlx5_ec_init(struct mlx5_core_dev *dev);
void mlx5_ec_cleanup(struct mlx5_core_dev *dev);
-int mlx5_query_host_params_num_vfs(struct mlx5_core_dev *dev, int *num_vf);
#else /* CONFIG_MLX5_ESWITCH */
@@ -24,9 +23,6 @@ static inline bool
mlx5_read_embedded_cpu(struct mlx5_core_dev *dev) { return false; }
static inline int mlx5_ec_init(struct mlx5_core_dev *dev) { return 0; }
static inline void mlx5_ec_cleanup(struct mlx5_core_dev *dev) {}
-static inline int
-mlx5_query_host_params_num_vfs(struct mlx5_core_dev *dev, int *num_vf)
-{ return -EOPNOTSUPP; }
#endif /* CONFIG_MLX5_ESWITCH */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 2f406b161bcf..51066f27ef1e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -1771,7 +1771,7 @@ mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
}
rpriv->netdev = netdev;
- rep->rep_if[REP_ETH].priv = rpriv;
+ rep->rep_data[REP_ETH].priv = rpriv;
INIT_LIST_HEAD(&rpriv->vport_sqs_list);
if (rep->vport == MLX5_VPORT_UPLINK) {
@@ -1845,16 +1845,17 @@ static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep)
return rpriv->netdev;
}
+static const struct mlx5_eswitch_rep_ops rep_ops = {
+ .load = mlx5e_vport_rep_load,
+ .unload = mlx5e_vport_rep_unload,
+ .get_proto_dev = mlx5e_vport_rep_get_proto_dev
+};
+
void mlx5e_rep_register_vport_reps(struct mlx5_core_dev *mdev)
{
struct mlx5_eswitch *esw = mdev->priv.eswitch;
- struct mlx5_eswitch_rep_if rep_if = {};
-
- rep_if.load = mlx5e_vport_rep_load;
- rep_if.unload = mlx5e_vport_rep_unload;
- rep_if.get_proto_dev = mlx5e_vport_rep_get_proto_dev;
- mlx5_eswitch_register_vport_reps(esw, &rep_if, REP_ETH);
+ mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH);
}
void mlx5e_rep_unregister_vport_reps(struct mlx5_core_dev *mdev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index 83b573b1abac..e34573fd88c1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -91,7 +91,7 @@ struct mlx5e_rep_priv {
static inline
struct mlx5e_rep_priv *mlx5e_rep_to_rep_priv(struct mlx5_eswitch_rep *rep)
{
- return (struct mlx5e_rep_priv *)rep->rep_if[REP_ETH].priv;
+ return rep->rep_data[REP_ETH].priv;
}
struct mlx5e_neigh {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 23883d1fa22f..58fff2f39b38 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -61,17 +61,21 @@ enum {
MLX5_EQ_DOORBEL_OFFSET = 0x40,
};
-struct mlx5_irq_info {
- cpumask_var_t mask;
- char name[MLX5_MAX_IRQ_NAME];
- void *context; /* dev_id provided to request_irq */
+/* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update
+ * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is
+ * used to set the EQ size, budget must be smaller than the EQ size.
+ */
+enum {
+ MLX5_EQ_POLLING_BUDGET = 128,
};
+static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
+
struct mlx5_eq_table {
struct list_head comp_eqs_list;
- struct mlx5_eq pages_eq;
- struct mlx5_eq cmd_eq;
- struct mlx5_eq async_eq;
+ struct mlx5_eq_async pages_eq;
+ struct mlx5_eq_async cmd_eq;
+ struct mlx5_eq_async async_eq;
struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
@@ -79,11 +83,8 @@ struct mlx5_eq_table {
struct mlx5_nb cq_err_nb;
struct mutex lock; /* sync async eqs creations */
- int num_comp_vectors;
- struct mlx5_irq_info *irq_info;
-#ifdef CONFIG_RFS_ACCEL
- struct cpu_rmap *rmap;
-#endif
+ int num_comp_eqs;
+ struct mlx5_irq_table *irq_table;
};
#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \
@@ -124,16 +125,24 @@ static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn)
return cq;
}
-static irqreturn_t mlx5_eq_comp_int(int irq, void *eq_ptr)
+static int mlx5_eq_comp_int(struct notifier_block *nb,
+ __always_unused unsigned long action,
+ __always_unused void *data)
{
- struct mlx5_eq_comp *eq_comp = eq_ptr;
- struct mlx5_eq *eq = eq_ptr;
+ struct mlx5_eq_comp *eq_comp =
+ container_of(nb, struct mlx5_eq_comp, irq_nb);
+ struct mlx5_eq *eq = &eq_comp->core;
struct mlx5_eqe *eqe;
- int set_ci = 0;
+ int num_eqes = 0;
u32 cqn = -1;
- while ((eqe = next_eqe_sw(eq))) {
+ eqe = next_eqe_sw(eq);
+ if (!eqe)
+ goto out;
+
+ do {
struct mlx5_core_cq *cq;
+
/* Make sure we read EQ entry contents after we've
* checked the ownership bit.
*/
@@ -151,26 +160,16 @@ static irqreturn_t mlx5_eq_comp_int(int irq, void *eq_ptr)
}
++eq->cons_index;
- ++set_ci;
- /* The HCA will think the queue has overflowed if we
- * don't tell it we've been processing events. We
- * create our EQs with MLX5_NUM_SPARE_EQE extra
- * entries, so we must update our consumer index at
- * least that often.
- */
- if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
- eq_update_ci(eq, 0);
- set_ci = 0;
- }
- }
+ } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
+out:
eq_update_ci(eq, 1);
if (cqn != -1)
tasklet_schedule(&eq_comp->tasklet_ctx.task);
- return IRQ_HANDLED;
+ return 0;
}
/* Some architectures don't latch interrupts when they are disabled, so using
@@ -184,25 +183,32 @@ u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq)
disable_irq(eq->core.irqn);
count_eqe = eq->core.cons_index;
- mlx5_eq_comp_int(eq->core.irqn, eq);
+ mlx5_eq_comp_int(&eq->irq_nb, 0, NULL);
count_eqe = eq->core.cons_index - count_eqe;
enable_irq(eq->core.irqn);
return count_eqe;
}
-static irqreturn_t mlx5_eq_async_int(int irq, void *eq_ptr)
+static int mlx5_eq_async_int(struct notifier_block *nb,
+ unsigned long action, void *data)
{
- struct mlx5_eq *eq = eq_ptr;
+ struct mlx5_eq_async *eq_async =
+ container_of(nb, struct mlx5_eq_async, irq_nb);
+ struct mlx5_eq *eq = &eq_async->core;
struct mlx5_eq_table *eqt;
struct mlx5_core_dev *dev;
struct mlx5_eqe *eqe;
- int set_ci = 0;
+ int num_eqes = 0;
dev = eq->dev;
eqt = dev->priv.eq_table;
- while ((eqe = next_eqe_sw(eq))) {
+ eqe = next_eqe_sw(eq);
+ if (!eqe)
+ goto out;
+
+ do {
/*
* Make sure we read EQ entry contents after we've
* checked the ownership bit.
@@ -217,23 +223,13 @@ static irqreturn_t mlx5_eq_async_int(int irq, void *eq_ptr)
atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe);
++eq->cons_index;
- ++set_ci;
- /* The HCA will think the queue has overflowed if we
- * don't tell it we've been processing events. We
- * create our EQs with MLX5_NUM_SPARE_EQE extra
- * entries, so we must update our consumer index at
- * least that often.
- */
- if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
- eq_update_ci(eq, 0);
- set_ci = 0;
- }
- }
+ } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
+out:
eq_update_ci(eq, 1);
- return IRQ_HANDLED;
+ return 0;
}
static void init_eq_buf(struct mlx5_eq *eq)
@@ -248,23 +244,19 @@ static void init_eq_buf(struct mlx5_eq *eq)
}
static int
-create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name,
+create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
struct mlx5_eq_param *param)
{
- struct mlx5_eq_table *eq_table = dev->priv.eq_table;
struct mlx5_cq_table *cq_table = &eq->cq_table;
u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
struct mlx5_priv *priv = &dev->priv;
- u8 vecidx = param->index;
+ u8 vecidx = param->irq_index;
__be64 *pas;
void *eqc;
int inlen;
u32 *in;
int err;
- if (eq_table->irq_info[vecidx].context)
- return -EEXIST;
-
/* Init CQ table */
memset(cq_table, 0, sizeof(*cq_table));
spin_lock_init(&cq_table->lock);
@@ -307,34 +299,19 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name,
if (err)
goto err_in;
- snprintf(eq_table->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
- name, pci_name(dev->pdev));
- eq_table->irq_info[vecidx].context = param->context;
-
eq->vecidx = vecidx;
eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
eq->irqn = pci_irq_vector(dev->pdev, vecidx);
eq->dev = dev;
eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
- err = request_irq(eq->irqn, param->handler, 0,
- eq_table->irq_info[vecidx].name, param->context);
- if (err)
- goto err_eq;
err = mlx5_debug_eq_add(dev, eq);
if (err)
- goto err_irq;
-
- /* EQs are created in ARMED state
- */
- eq_update_ci(eq, 1);
+ goto err_eq;
kvfree(in);
return 0;
-err_irq:
- free_irq(eq->irqn, eq);
-
err_eq:
mlx5_cmd_destroy_eq(dev, eq->eqn);
@@ -346,18 +323,48 @@ err_buf:
return err;
}
-static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
+/**
+ * mlx5_eq_enable - Enable EQ for receiving EQEs
+ * @dev - Device which owns the eq
+ * @eq - EQ to enable
+ * @nb - notifier call block
+ * mlx5_eq_enable - must be called after EQ is created in device.
+ */
+int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+ struct notifier_block *nb)
{
struct mlx5_eq_table *eq_table = dev->priv.eq_table;
- struct mlx5_irq_info *irq_info;
int err;
- irq_info = &eq_table->irq_info[eq->vecidx];
+ err = mlx5_irq_attach_nb(eq_table->irq_table, eq->vecidx, nb);
+ if (!err)
+ eq_update_ci(eq, 1);
- mlx5_debug_eq_remove(dev, eq);
+ return err;
+}
+EXPORT_SYMBOL(mlx5_eq_enable);
- free_irq(eq->irqn, irq_info->context);
- irq_info->context = NULL;
+/**
+ * mlx5_eq_disable - Enable EQ for receiving EQEs
+ * @dev - Device which owns the eq
+ * @eq - EQ to disable
+ * @nb - notifier call block
+ * mlx5_eq_disable - must be called before EQ is destroyed.
+ */
+void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+ struct notifier_block *nb)
+{
+ struct mlx5_eq_table *eq_table = dev->priv.eq_table;
+
+ mlx5_irq_detach_nb(eq_table->irq_table, eq->vecidx, nb);
+}
+EXPORT_SYMBOL(mlx5_eq_disable);
+
+static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
+{
+ int err;
+
+ mlx5_debug_eq_remove(dev, eq);
err = mlx5_cmd_destroy_eq(dev, eq->eqn);
if (err)
@@ -423,6 +430,7 @@ int mlx5_eq_table_init(struct mlx5_core_dev *dev)
for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)
ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);
+ eq_table->irq_table = dev->priv.irq_table;
return 0;
kvfree_eq_table:
@@ -439,19 +447,20 @@ void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev)
/* Async EQs */
-static int create_async_eq(struct mlx5_core_dev *dev, const char *name,
+static int create_async_eq(struct mlx5_core_dev *dev,
struct mlx5_eq *eq, struct mlx5_eq_param *param)
{
struct mlx5_eq_table *eq_table = dev->priv.eq_table;
int err;
mutex_lock(&eq_table->lock);
- if (param->index >= MLX5_EQ_MAX_ASYNC_EQS) {
- err = -ENOSPC;
+ /* Async EQs must share irq index 0 */
+ if (param->irq_index != 0) {
+ err = -EINVAL;
goto unlock;
}
- err = create_map_eq(dev, eq, name, param);
+ err = create_map_eq(dev, eq, param);
unlock:
mutex_unlock(&eq_table->lock);
return err;
@@ -480,7 +489,7 @@ static int cq_err_event_notifier(struct notifier_block *nb,
/* type == MLX5_EVENT_TYPE_CQ_ERROR */
eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
- eq = &eqt->async_eq;
+ eq = &eqt->async_eq.core;
eqe = data;
cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
@@ -533,8 +542,9 @@ static u64 gather_async_events_mask(struct mlx5_core_dev *dev)
if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters))
async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER);
- if (mlx5_core_is_ecpf_esw_manager(dev))
- async_event_mask |= (1ull << MLX5_EVENT_TYPE_HOST_PARAMS_CHANGE);
+ if (mlx5_eswitch_is_funcs_handler(dev))
+ async_event_mask |=
+ (1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
return async_event_mask;
}
@@ -548,55 +558,73 @@ static int create_async_eqs(struct mlx5_core_dev *dev)
MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
mlx5_eq_notifier_register(dev, &table->cq_err_nb);
+ table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
- .index = MLX5_EQ_CMD_IDX,
+ .irq_index = 0,
.mask = 1ull << MLX5_EVENT_TYPE_CMD,
.nent = MLX5_NUM_CMD_EQE,
- .context = &table->cmd_eq,
- .handler = mlx5_eq_async_int,
};
- err = create_async_eq(dev, "mlx5_cmd_eq", &table->cmd_eq, &param);
+ err = create_async_eq(dev, &table->cmd_eq.core, &param);
if (err) {
mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
goto err0;
}
-
+ err = mlx5_eq_enable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb);
+ if (err) {
+ mlx5_core_warn(dev, "failed to enable cmd EQ %d\n", err);
+ goto err1;
+ }
mlx5_cmd_use_events(dev);
+ table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
- .index = MLX5_EQ_ASYNC_IDX,
+ .irq_index = 0,
.mask = gather_async_events_mask(dev),
.nent = MLX5_NUM_ASYNC_EQE,
- .context = &table->async_eq,
- .handler = mlx5_eq_async_int,
};
- err = create_async_eq(dev, "mlx5_async_eq", &table->async_eq, &param);
+ err = create_async_eq(dev, &table->async_eq.core, &param);
if (err) {
mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
- goto err1;
+ goto err2;
+ }
+ err = mlx5_eq_enable(dev, &table->async_eq.core,
+ &table->async_eq.irq_nb);
+ if (err) {
+ mlx5_core_warn(dev, "failed to enable async EQ %d\n", err);
+ goto err3;
}
+ table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
- .index = MLX5_EQ_PAGEREQ_IDX,
+ .irq_index = 0,
.mask = 1 << MLX5_EVENT_TYPE_PAGE_REQUEST,
.nent = /* TODO: sriov max_vf + */ 1,
- .context = &table->pages_eq,
- .handler = mlx5_eq_async_int,
};
- err = create_async_eq(dev, "mlx5_pages_eq", &table->pages_eq, &param);
+ err = create_async_eq(dev, &table->pages_eq.core, &param);
if (err) {
mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
- goto err2;
+ goto err4;
+ }
+ err = mlx5_eq_enable(dev, &table->pages_eq.core,
+ &table->pages_eq.irq_nb);
+ if (err) {
+ mlx5_core_warn(dev, "failed to enable pages EQ %d\n", err);
+ goto err5;
}
return err;
+err5:
+ destroy_async_eq(dev, &table->pages_eq.core);
+err4:
+ mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb);
+err3:
+ destroy_async_eq(dev, &table->async_eq.core);
err2:
- destroy_async_eq(dev, &table->async_eq);
-
-err1:
mlx5_cmd_use_polling(dev);
- destroy_async_eq(dev, &table->cmd_eq);
+ mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb);
+err1:
+ destroy_async_eq(dev, &table->cmd_eq.core);
err0:
mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
return err;
@@ -607,19 +635,22 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev)
struct mlx5_eq_table *table = dev->priv.eq_table;
int err;
- err = destroy_async_eq(dev, &table->pages_eq);
+ mlx5_eq_disable(dev, &table->pages_eq.core, &table->pages_eq.irq_nb);
+ err = destroy_async_eq(dev, &table->pages_eq.core);
if (err)
mlx5_core_err(dev, "failed to destroy pages eq, err(%d)\n",
err);
- err = destroy_async_eq(dev, &table->async_eq);
+ mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb);
+ err = destroy_async_eq(dev, &table->async_eq.core);
if (err)
mlx5_core_err(dev, "failed to destroy async eq, err(%d)\n",
err);
mlx5_cmd_use_polling(dev);
- err = destroy_async_eq(dev, &table->cmd_eq);
+ mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb);
+ err = destroy_async_eq(dev, &table->cmd_eq.core);
if (err)
mlx5_core_err(dev, "failed to destroy command eq, err(%d)\n",
err);
@@ -629,24 +660,24 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev)
struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
{
- return &dev->priv.eq_table->async_eq;
+ return &dev->priv.eq_table->async_eq.core;
}
void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
{
- synchronize_irq(dev->priv.eq_table->async_eq.irqn);
+ synchronize_irq(dev->priv.eq_table->async_eq.core.irqn);
}
void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
{
- synchronize_irq(dev->priv.eq_table->cmd_eq.irqn);
+ synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn);
}
/* Generic EQ API for mlx5_core consumers
* Needed For RDMA ODP EQ for now
*/
struct mlx5_eq *
-mlx5_eq_create_generic(struct mlx5_core_dev *dev, const char *name,
+mlx5_eq_create_generic(struct mlx5_core_dev *dev,
struct mlx5_eq_param *param)
{
struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL);
@@ -655,7 +686,7 @@ mlx5_eq_create_generic(struct mlx5_core_dev *dev, const char *name,
if (!eq)
return ERR_PTR(-ENOMEM);
- err = create_async_eq(dev, name, eq, param);
+ err = create_async_eq(dev, eq, param);
if (err) {
kvfree(eq);
eq = ERR_PTR(err);
@@ -713,84 +744,14 @@ void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
}
EXPORT_SYMBOL(mlx5_eq_update_ci);
-/* Completion EQs */
-
-static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
-{
- struct mlx5_priv *priv = &mdev->priv;
- int vecidx = MLX5_EQ_VEC_COMP_BASE + i;
- int irq = pci_irq_vector(mdev->pdev, vecidx);
- struct mlx5_irq_info *irq_info = &priv->eq_table->irq_info[vecidx];
-
- if (!zalloc_cpumask_var(&irq_info->mask, GFP_KERNEL)) {
- mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
- return -ENOMEM;
- }
-
- cpumask_set_cpu(cpumask_local_spread(i, priv->numa_node),
- irq_info->mask);
-
- if (IS_ENABLED(CONFIG_SMP) &&
- irq_set_affinity_hint(irq, irq_info->mask))
- mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", irq);
-
- return 0;
-}
-
-static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
-{
- int vecidx = MLX5_EQ_VEC_COMP_BASE + i;
- struct mlx5_priv *priv = &mdev->priv;
- int irq = pci_irq_vector(mdev->pdev, vecidx);
- struct mlx5_irq_info *irq_info = &priv->eq_table->irq_info[vecidx];
-
- irq_set_affinity_hint(irq, NULL);
- free_cpumask_var(irq_info->mask);
-}
-
-static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev)
-{
- int err;
- int i;
-
- for (i = 0; i < mdev->priv.eq_table->num_comp_vectors; i++) {
- err = set_comp_irq_affinity_hint(mdev, i);
- if (err)
- goto err_out;
- }
-
- return 0;
-
-err_out:
- for (i--; i >= 0; i--)
- clear_comp_irq_affinity_hint(mdev, i);
-
- return err;
-}
-
-static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev)
-{
- int i;
-
- for (i = 0; i < mdev->priv.eq_table->num_comp_vectors; i++)
- clear_comp_irq_affinity_hint(mdev, i);
-}
-
static void destroy_comp_eqs(struct mlx5_core_dev *dev)
{
struct mlx5_eq_table *table = dev->priv.eq_table;
struct mlx5_eq_comp *eq, *n;
- clear_comp_irqs_affinity_hints(dev);
-
-#ifdef CONFIG_RFS_ACCEL
- if (table->rmap) {
- free_irq_cpu_rmap(table->rmap);
- table->rmap = NULL;
- }
-#endif
list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
list_del(&eq->list);
+ mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
if (destroy_unmap_eq(dev, &eq->core))
mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n",
eq->core.eqn);
@@ -802,23 +763,17 @@ static void destroy_comp_eqs(struct mlx5_core_dev *dev)
static int create_comp_eqs(struct mlx5_core_dev *dev)
{
struct mlx5_eq_table *table = dev->priv.eq_table;
- char name[MLX5_MAX_IRQ_NAME];
struct mlx5_eq_comp *eq;
- int ncomp_vec;
+ int ncomp_eqs;
int nent;
int err;
int i;
INIT_LIST_HEAD(&table->comp_eqs_list);
- ncomp_vec = table->num_comp_vectors;
+ ncomp_eqs = table->num_comp_eqs;
nent = MLX5_COMP_EQ_SIZE;
-#ifdef CONFIG_RFS_ACCEL
- table->rmap = alloc_irq_cpu_rmap(ncomp_vec);
- if (!table->rmap)
- return -ENOMEM;
-#endif
- for (i = 0; i < ncomp_vec; i++) {
- int vecidx = i + MLX5_EQ_VEC_COMP_BASE;
+ for (i = 0; i < ncomp_eqs; i++) {
+ int vecidx = i + MLX5_IRQ_VEC_COMP_BASE;
struct mlx5_eq_param param = {};
eq = kzalloc(sizeof(*eq), GFP_KERNEL);
@@ -833,33 +788,29 @@ static int create_comp_eqs(struct mlx5_core_dev *dev)
tasklet_init(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb,
(unsigned long)&eq->tasklet_ctx);
-#ifdef CONFIG_RFS_ACCEL
- irq_cpu_rmap_add(table->rmap, pci_irq_vector(dev->pdev, vecidx));
-#endif
- snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
+ eq->irq_nb.notifier_call = mlx5_eq_comp_int;
param = (struct mlx5_eq_param) {
- .index = vecidx,
+ .irq_index = vecidx,
.mask = 0,
.nent = nent,
- .context = &eq->core,
- .handler = mlx5_eq_comp_int
};
- err = create_map_eq(dev, &eq->core, name, &param);
+ err = create_map_eq(dev, &eq->core, &param);
+ if (err) {
+ kfree(eq);
+ goto clean;
+ }
+ err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
if (err) {
+ destroy_unmap_eq(dev, &eq->core);
kfree(eq);
goto clean;
}
+
mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
/* add tail, to keep the list ordered, for mlx5_vector2eqn to work */
list_add_tail(&eq->list, &table->comp_eqs_list);
}
- err = set_comp_irq_affinity_hints(dev);
- if (err) {
- mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n");
- goto clean;
- }
-
return 0;
clean:
@@ -890,22 +841,24 @@ EXPORT_SYMBOL(mlx5_vector2eqn);
unsigned int mlx5_comp_vectors_count(struct mlx5_core_dev *dev)
{
- return dev->priv.eq_table->num_comp_vectors;
+ return dev->priv.eq_table->num_comp_eqs;
}
EXPORT_SYMBOL(mlx5_comp_vectors_count);
struct cpumask *
mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
{
- /* TODO: consider irq_get_affinity_mask(irq) */
- return dev->priv.eq_table->irq_info[vector + MLX5_EQ_VEC_COMP_BASE].mask;
+ int vecidx = vector + MLX5_IRQ_VEC_COMP_BASE;
+
+ return mlx5_irq_get_affinity_mask(dev->priv.eq_table->irq_table,
+ vecidx);
}
EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask);
#ifdef CONFIG_RFS_ACCEL
struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev)
{
- return dev->priv.eq_table->rmap;
+ return mlx5_irq_get_rmap(dev->priv.eq_table->irq_table);
}
#endif
@@ -926,82 +879,19 @@ struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
{
struct mlx5_eq_table *table = dev->priv.eq_table;
- int i, max_eqs;
-
- clear_comp_irqs_affinity_hints(dev);
-
-#ifdef CONFIG_RFS_ACCEL
- if (table->rmap) {
- free_irq_cpu_rmap(table->rmap);
- table->rmap = NULL;
- }
-#endif
mutex_lock(&table->lock); /* sync with create/destroy_async_eq */
- max_eqs = table->num_comp_vectors + MLX5_EQ_VEC_COMP_BASE;
- for (i = max_eqs - 1; i >= 0; i--) {
- if (!table->irq_info[i].context)
- continue;
- free_irq(pci_irq_vector(dev->pdev, i), table->irq_info[i].context);
- table->irq_info[i].context = NULL;
- }
+ mlx5_irq_table_destroy(dev);
mutex_unlock(&table->lock);
- pci_free_irq_vectors(dev->pdev);
-}
-
-static int alloc_irq_vectors(struct mlx5_core_dev *dev)
-{
- struct mlx5_priv *priv = &dev->priv;
- struct mlx5_eq_table *table = priv->eq_table;
- int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
- MLX5_CAP_GEN(dev, max_num_eqs) :
- 1 << MLX5_CAP_GEN(dev, log_max_eq);
- int nvec;
- int err;
-
- nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
- MLX5_EQ_VEC_COMP_BASE;
- nvec = min_t(int, nvec, num_eqs);
- if (nvec <= MLX5_EQ_VEC_COMP_BASE)
- return -ENOMEM;
-
- table->irq_info = kcalloc(nvec, sizeof(*table->irq_info), GFP_KERNEL);
- if (!table->irq_info)
- return -ENOMEM;
-
- nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_EQ_VEC_COMP_BASE + 1,
- nvec, PCI_IRQ_MSIX);
- if (nvec < 0) {
- err = nvec;
- goto err_free_irq_info;
- }
-
- table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE;
-
- return 0;
-
-err_free_irq_info:
- kfree(table->irq_info);
- return err;
-}
-
-static void free_irq_vectors(struct mlx5_core_dev *dev)
-{
- struct mlx5_priv *priv = &dev->priv;
-
- pci_free_irq_vectors(dev->pdev);
- kfree(priv->eq_table->irq_info);
}
int mlx5_eq_table_create(struct mlx5_core_dev *dev)
{
+ struct mlx5_eq_table *eq_table = dev->priv.eq_table;
int err;
- err = alloc_irq_vectors(dev);
- if (err) {
- mlx5_core_err(dev, "alloc irq vectors failed\n");
- return err;
- }
+ eq_table->num_comp_eqs =
+ mlx5_irq_get_num_comp(eq_table->irq_table);
err = create_async_eqs(dev);
if (err) {
@@ -1019,7 +909,6 @@ int mlx5_eq_table_create(struct mlx5_core_dev *dev)
err_comp_eqs:
destroy_async_eqs(dev);
err_async_eqs:
- free_irq_vectors(dev);
return err;
}
@@ -1027,7 +916,6 @@ void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
{
destroy_comp_eqs(dev);
destroy_async_eqs(dev);
- free_irq_vectors(dev);
}
int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 6a921e24cd5e..ea4c8dc93ce4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -1686,13 +1686,23 @@ static int eswitch_vport_event(struct notifier_block *nb,
return NOTIFY_OK;
}
+int mlx5_esw_query_functions(struct mlx5_core_dev *dev, u32 *out, int outlen)
+{
+ u32 in[MLX5_ST_SZ_DW(query_esw_functions_in)] = {};
+
+ MLX5_SET(query_esw_functions_in, in, opcode,
+ MLX5_CMD_OP_QUERY_ESW_FUNCTIONS);
+
+ return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
+}
+
/* Public E-Switch API */
#define ESW_ALLOWED(esw) ((esw) && MLX5_ESWITCH_MANAGER((esw)->dev))
int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
{
- int vf_nvports = 0, total_nvports = 0;
struct mlx5_vport *vport;
+ int total_nvports = 0;
int err;
int i, enabled_events;
@@ -1711,15 +1721,10 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
esw_info(esw->dev, "E-Switch enable SRIOV: nvfs(%d) mode (%d)\n", nvfs, mode);
if (mode == SRIOV_OFFLOADS) {
- if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
- err = mlx5_query_host_params_num_vfs(esw->dev, &vf_nvports);
- if (err)
- return err;
+ if (mlx5_core_is_ecpf_esw_manager(esw->dev))
total_nvports = esw->total_vports;
- } else {
- vf_nvports = nvfs;
+ else
total_nvports = nvfs + MLX5_SPECIAL_VPORTS(esw->dev);
- }
}
esw->mode = mode;
@@ -1733,7 +1738,7 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
} else {
mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_ETH);
mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_IB);
- err = esw_offloads_init(esw, vf_nvports, total_nvports);
+ err = esw_offloads_init(esw, nvfs, total_nvports);
}
if (err)
@@ -2452,6 +2457,17 @@ u8 mlx5_eswitch_mode(struct mlx5_eswitch *esw)
}
EXPORT_SYMBOL_GPL(mlx5_eswitch_mode);
+enum devlink_eswitch_encap_mode
+mlx5_eswitch_get_encap_mode(const struct mlx5_core_dev *dev)
+{
+ struct mlx5_eswitch *esw;
+
+ esw = dev->priv.eswitch;
+ return ESW_ALLOWED(esw) ? esw->offloads.encap :
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+}
+EXPORT_SYMBOL(mlx5_eswitch_get_encap_mode);
+
bool mlx5_esw_lag_prereq(struct mlx5_core_dev *dev0, struct mlx5_core_dev *dev1)
{
if ((dev0->priv.eswitch->mode == SRIOV_NONE &&
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index d043d6f9797d..c9589f12c839 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -173,9 +173,10 @@ struct mlx5_esw_offload {
struct mutex peer_mutex;
DECLARE_HASHTABLE(encap_tbl, 8);
DECLARE_HASHTABLE(mod_hdr_tbl, 8);
+ const struct mlx5_eswitch_rep_ops *rep_ops[NUM_REP_TYPES];
u8 inline_mode;
u64 num_flows;
- u8 encap;
+ enum devlink_eswitch_encap_mode encap;
};
/* E-Switch MC FDB table hash node */
@@ -190,7 +191,7 @@ struct mlx5_host_work {
struct mlx5_eswitch *esw;
};
-struct mlx5_host_info {
+struct mlx5_esw_functions {
struct mlx5_nb nb;
u16 num_vfs;
};
@@ -219,7 +220,7 @@ struct mlx5_eswitch {
int mode;
int nvports;
u16 manager_vport;
- struct mlx5_host_info host_info;
+ struct mlx5_esw_functions esw_funcs;
};
void esw_offloads_cleanup(struct mlx5_eswitch *esw);
@@ -356,9 +357,11 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
struct netlink_ext_ack *extack);
int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode);
int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, int nvfs, u8 *mode);
-int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, u8 encap,
+int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode encap,
struct netlink_ext_ack *extack);
-int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap);
+int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode *encap);
void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type);
int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
@@ -386,6 +389,8 @@ bool mlx5_esw_lag_prereq(struct mlx5_core_dev *dev0,
bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0,
struct mlx5_core_dev *dev1);
+int mlx5_esw_query_functions(struct mlx5_core_dev *dev, u32 *out, int outlen);
+
#define MLX5_DEBUG_ESWITCH_MASK BIT(3)
#define esw_info(__dev, format, ...) \
@@ -404,6 +409,18 @@ static inline u16 mlx5_eswitch_manager_vport(struct mlx5_core_dev *dev)
MLX5_VPORT_ECPF : MLX5_VPORT_PF;
}
+static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev)
+{
+ /* Ideally device should have the functions changed supported
+ * capability regardless of it being ECPF or PF wherever such
+ * event should be processed such as on eswitch manager device.
+ * However, some ECPF based device might not have this capability
+ * set. Hence OR for ECPF check to cover such device.
+ */
+ return MLX5_CAP_ESW(dev, esw_functions_changed) ||
+ mlx5_core_is_ecpf_esw_manager(dev);
+}
+
static inline int mlx5_eswitch_uplink_idx(struct mlx5_eswitch *esw)
{
/* Uplink always locate at the last element of the array.*/
@@ -498,6 +515,12 @@ static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {}
static inline int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode) { return 0; }
static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw) {}
static inline bool mlx5_esw_lag_prereq(struct mlx5_core_dev *dev0, struct mlx5_core_dev *dev1) { return true; }
+static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; }
+static inline int
+mlx5_esw_query_functions(struct mlx5_core_dev *dev, u32 *out, int outlen)
+{
+ return -EOPNOTSUPP;
+}
#define FDB_MAX_CHAIN 1
#define FDB_SLOW_PATH_CHAIN (FDB_MAX_CHAIN + 1)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 47b446d30f71..6f5bd602a47e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -41,7 +41,6 @@
#include "en.h"
#include "fs_core.h"
#include "lib/devcom.h"
-#include "ecpf.h"
#include "lib/eq.h"
/* There are two match-all miss flows, one for unicast dst mac and
@@ -333,7 +332,7 @@ static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
for (vf_vport = 1; vf_vport < esw->enabled_vports; vf_vport++) {
rep = &esw->offloads.vport_reps[vf_vport];
- if (atomic_read(&rep->rep_if[REP_ETH].state) != REP_LOADED)
+ if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
continue;
err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
@@ -1278,7 +1277,7 @@ int esw_offloads_init_reps(struct mlx5_eswitch *esw)
ether_addr_copy(rep->hw_id, hw_id);
for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
- atomic_set(&rep->rep_if[rep_type].state,
+ atomic_set(&rep->rep_data[rep_type].state,
REP_UNREGISTERED);
}
@@ -1288,9 +1287,9 @@ int esw_offloads_init_reps(struct mlx5_eswitch *esw)
static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
struct mlx5_eswitch_rep *rep, u8 rep_type)
{
- if (atomic_cmpxchg(&rep->rep_if[rep_type].state,
+ if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
REP_LOADED, REP_REGISTERED) == REP_LOADED)
- rep->rep_if[rep_type].unload(rep);
+ esw->offloads.rep_ops[rep_type]->unload(rep);
}
static void __unload_reps_special_vport(struct mlx5_eswitch *esw, u8 rep_type)
@@ -1351,11 +1350,11 @@ static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
{
int err = 0;
- if (atomic_cmpxchg(&rep->rep_if[rep_type].state,
+ if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
- err = rep->rep_if[rep_type].load(esw->dev, rep);
+ err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
if (err)
- atomic_set(&rep->rep_if[rep_type].state,
+ atomic_set(&rep->rep_data[rep_type].state,
REP_REGISTERED);
}
@@ -1438,34 +1437,13 @@ err_reps:
return err;
}
-static int __load_reps_all_vport(struct mlx5_eswitch *esw, int nvports,
- u8 rep_type)
-{
- int err;
-
- /* Special vports must be loaded first. */
- err = __load_reps_special_vport(esw, rep_type);
- if (err)
- return err;
-
- err = __load_reps_vf_vport(esw, nvports, rep_type);
- if (err)
- goto err_vfs;
-
- return 0;
-
-err_vfs:
- __unload_reps_special_vport(esw, rep_type);
- return err;
-}
-
-static int esw_offloads_load_all_reps(struct mlx5_eswitch *esw, int nvports)
+static int esw_offloads_load_special_vport(struct mlx5_eswitch *esw)
{
u8 rep_type = 0;
int err;
for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
- err = __load_reps_all_vport(esw, nvports, rep_type);
+ err = __load_reps_special_vport(esw, rep_type);
if (err)
goto err_reps;
}
@@ -1474,7 +1452,7 @@ static int esw_offloads_load_all_reps(struct mlx5_eswitch *esw, int nvports)
err_reps:
while (rep_type-- > 0)
- __unload_reps_all_vport(esw, nvports, rep_type);
+ __unload_reps_special_vport(esw, rep_type);
return err;
}
@@ -1784,57 +1762,102 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
esw_prio_tag_acls_cleanup(esw);
}
-static void esw_host_params_event_handler(struct work_struct *work)
+static void esw_functions_changed_event_handler(struct work_struct *work)
{
+ u32 out[MLX5_ST_SZ_DW(query_esw_functions_out)] = {};
struct mlx5_host_work *host_work;
struct mlx5_eswitch *esw;
- int err, num_vf = 0;
+ u16 num_vfs = 0;
+ int err;
host_work = container_of(work, struct mlx5_host_work, work);
esw = host_work->esw;
- err = mlx5_query_host_params_num_vfs(esw->dev, &num_vf);
- if (err || num_vf == esw->host_info.num_vfs)
+ err = mlx5_esw_query_functions(esw->dev, out, sizeof(out));
+ num_vfs = MLX5_GET(query_esw_functions_out, out,
+ host_params_context.host_num_of_vfs);
+ if (err || num_vfs == esw->esw_funcs.num_vfs)
goto out;
/* Number of VFs can only change from "0 to x" or "x to 0". */
- if (esw->host_info.num_vfs > 0) {
- esw_offloads_unload_vf_reps(esw, esw->host_info.num_vfs);
+ if (esw->esw_funcs.num_vfs > 0) {
+ esw_offloads_unload_vf_reps(esw, esw->esw_funcs.num_vfs);
} else {
- err = esw_offloads_load_vf_reps(esw, num_vf);
+ err = esw_offloads_load_vf_reps(esw, num_vfs);
if (err)
goto out;
}
- esw->host_info.num_vfs = num_vf;
+ esw->esw_funcs.num_vfs = num_vfs;
out:
kfree(host_work);
}
-static int esw_host_params_event(struct notifier_block *nb,
- unsigned long type, void *data)
+static void esw_emulate_event_handler(struct work_struct *work)
{
+ struct mlx5_host_work *host_work =
+ container_of(work, struct mlx5_host_work, work);
+ struct mlx5_eswitch *esw = host_work->esw;
+ int err;
+
+ if (esw->esw_funcs.num_vfs) {
+ err = esw_offloads_load_vf_reps(esw, esw->esw_funcs.num_vfs);
+ if (err)
+ esw_warn(esw->dev, "Load vf reps err=%d\n", err);
+ }
+ kfree(host_work);
+}
+
+static int esw_functions_changed_event(struct notifier_block *nb,
+ unsigned long type, void *data)
+{
+ struct mlx5_esw_functions *esw_funcs;
struct mlx5_host_work *host_work;
- struct mlx5_host_info *host_info;
struct mlx5_eswitch *esw;
host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
if (!host_work)
return NOTIFY_DONE;
- host_info = mlx5_nb_cof(nb, struct mlx5_host_info, nb);
- esw = container_of(host_info, struct mlx5_eswitch, host_info);
+ esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
+ esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
host_work->esw = esw;
- INIT_WORK(&host_work->work, esw_host_params_event_handler);
+ if (mlx5_eswitch_is_funcs_handler(esw->dev))
+ INIT_WORK(&host_work->work,
+ esw_functions_changed_event_handler);
+ else
+ INIT_WORK(&host_work->work, esw_emulate_event_handler);
queue_work(esw->work_queue, &host_work->work);
return NOTIFY_OK;
}
+static void esw_functions_changed_event_init(struct mlx5_eswitch *esw,
+ u16 vf_nvports)
+{
+ if (mlx5_eswitch_is_funcs_handler(esw->dev)) {
+ esw->esw_funcs.num_vfs = 0;
+ MLX5_NB_INIT(&esw->esw_funcs.nb, esw_functions_changed_event,
+ ESW_FUNCTIONS_CHANGED);
+ mlx5_eq_notifier_register(esw->dev, &esw->esw_funcs.nb);
+ } else {
+ esw->esw_funcs.num_vfs = vf_nvports;
+ }
+}
+
+static void esw_functions_changed_event_cleanup(struct mlx5_eswitch *esw)
+{
+ if (!mlx5_eswitch_is_funcs_handler(esw->dev))
+ return;
+
+ mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb);
+ flush_workqueue(esw->work_queue);
+}
+
int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports,
int total_nvports)
{
@@ -1844,21 +1867,30 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports,
if (err)
return err;
- err = esw_offloads_load_all_reps(esw, vf_nvports);
+ /* Only load special vports reps. VF reps will be loaded in
+ * context of functions_changed event handler through real
+ * or emulated event.
+ */
+ err = esw_offloads_load_special_vport(esw);
if (err)
goto err_reps;
esw_offloads_devcom_init(esw);
- if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
- MLX5_NB_INIT(&esw->host_info.nb, esw_host_params_event,
- HOST_PARAMS_CHANGE);
- mlx5_eq_notifier_register(esw->dev, &esw->host_info.nb);
- esw->host_info.num_vfs = vf_nvports;
- }
+ esw_functions_changed_event_init(esw, vf_nvports);
mlx5_rdma_enable_roce(esw->dev);
+ /* Call esw_functions_changed event to load VF reps:
+ * 1. HW does not support the event then emulate it
+ * Or
+ * 2. The event was already notified when num_vfs changed
+ * and eswitch was in legacy mode
+ */
+ esw_functions_changed_event(&esw->esw_funcs.nb.nb,
+ MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED,
+ NULL);
+
return 0;
err_reps:
@@ -1887,19 +1919,10 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw,
void esw_offloads_cleanup(struct mlx5_eswitch *esw)
{
- u16 num_vfs;
-
- if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
- mlx5_eq_notifier_unregister(esw->dev, &esw->host_info.nb);
- flush_workqueue(esw->work_queue);
- num_vfs = esw->host_info.num_vfs;
- } else {
- num_vfs = esw->dev->priv.sriov.num_vfs;
- }
-
+ esw_functions_changed_event_cleanup(esw);
mlx5_rdma_disable_roce(esw->dev);
esw_offloads_devcom_cleanup(esw);
- esw_offloads_unload_all_reps(esw, num_vfs);
+ esw_offloads_unload_all_reps(esw, esw->esw_funcs.num_vfs);
esw_offloads_steering_cleanup(esw);
}
@@ -2139,7 +2162,8 @@ out:
return 0;
}
-int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, u8 encap,
+int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode encap,
struct netlink_ext_ack *extack)
{
struct mlx5_core_dev *dev = devlink_priv(devlink);
@@ -2188,7 +2212,8 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, u8 encap,
return err;
}
-int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap)
+int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode *encap)
{
struct mlx5_core_dev *dev = devlink_priv(devlink);
struct mlx5_eswitch *esw = dev->priv.eswitch;
@@ -2203,21 +2228,17 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap)
}
void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
- struct mlx5_eswitch_rep_if *__rep_if,
+ const struct mlx5_eswitch_rep_ops *ops,
u8 rep_type)
{
- struct mlx5_eswitch_rep_if *rep_if;
+ struct mlx5_eswitch_rep_data *rep_data;
struct mlx5_eswitch_rep *rep;
int i;
+ esw->offloads.rep_ops[rep_type] = ops;
mlx5_esw_for_all_reps(esw, i, rep) {
- rep_if = &rep->rep_if[rep_type];
- rep_if->load = __rep_if->load;
- rep_if->unload = __rep_if->unload;
- rep_if->get_proto_dev = __rep_if->get_proto_dev;
- rep_if->priv = __rep_if->priv;
-
- atomic_set(&rep_if->state, REP_REGISTERED);
+ rep_data = &rep->rep_data[rep_type];
+ atomic_set(&rep_data->state, REP_REGISTERED);
}
}
EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
@@ -2232,7 +2253,7 @@ void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
__unload_reps_all_vport(esw, max_vf, rep_type);
mlx5_esw_for_all_reps(esw, i, rep)
- atomic_set(&rep->rep_if[rep_type].state, REP_UNREGISTERED);
+ atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
}
EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
@@ -2241,7 +2262,7 @@ void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
struct mlx5_eswitch_rep *rep;
rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
- return rep->rep_if[rep_type].priv;
+ return rep->rep_data[rep_type].priv;
}
void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
@@ -2252,9 +2273,9 @@ void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
rep = mlx5_eswitch_get_rep(esw, vport);
- if (atomic_read(&rep->rep_if[rep_type].state) == REP_LOADED &&
- rep->rep_if[rep_type].get_proto_dev)
- return rep->rep_if[rep_type].get_proto_dev(rep);
+ if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
+ esw->offloads.rep_ops[rep_type]->get_proto_dev)
+ return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
return NULL;
}
EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c
index a81e8d2168d8..8bcf3426b9c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/events.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c
@@ -108,8 +108,8 @@ static const char *eqe_type_str(u8 type)
return "MLX5_EVENT_TYPE_STALL_EVENT";
case MLX5_EVENT_TYPE_CMD:
return "MLX5_EVENT_TYPE_CMD";
- case MLX5_EVENT_TYPE_HOST_PARAMS_CHANGE:
- return "MLX5_EVENT_TYPE_HOST_PARAMS_CHANGE";
+ case MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED:
+ return "MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED";
case MLX5_EVENT_TYPE_PAGE_REQUEST:
return "MLX5_EVENT_TYPE_PAGE_REQUEST";
case MLX5_EVENT_TYPE_PAGE_FAULT:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index 013b1ca4a791..bb24c3797218 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -147,6 +147,7 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
{
int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+ int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION);
u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0};
struct mlx5_core_dev *dev = ns->dev;
@@ -167,6 +168,8 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
en_decap);
MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en,
en_encap);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table,
+ term);
switch (ft->op_mod) {
case FS_FT_OP_MOD_NORMAL:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
index c0fb6d72b695..24bd991a727e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
@@ -7,7 +7,6 @@
#include <linux/mlx5/eq.h>
#include <linux/mlx5/cq.h>
-#define MLX5_MAX_IRQ_NAME (32)
#define MLX5_EQE_SIZE (sizeof(struct mlx5_eqe))
struct mlx5_eq_tasklet {
@@ -36,8 +35,14 @@ struct mlx5_eq {
struct mlx5_rsc_debug *dbg;
};
+struct mlx5_eq_async {
+ struct mlx5_eq core;
+ struct notifier_block irq_nb;
+};
+
struct mlx5_eq_comp {
- struct mlx5_eq core; /* Must be first */
+ struct mlx5_eq core;
+ struct notifier_block irq_nb;
struct mlx5_eq_tasklet tasklet_ctx;
struct list_head list;
};
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 23d53163ce15..2a766defb74e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -169,18 +169,28 @@ static struct mlx5_profile profile[] = {
#define FW_INIT_TIMEOUT_MILI 2000
#define FW_INIT_WAIT_MS 2
-#define FW_PRE_INIT_TIMEOUT_MILI 10000
+#define FW_PRE_INIT_TIMEOUT_MILI 120000
+#define FW_INIT_WARN_MESSAGE_INTERVAL 20000
-static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili)
+static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
+ u32 warn_time_mili)
{
+ unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili);
unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
int err = 0;
+ BUILD_BUG_ON(FW_PRE_INIT_TIMEOUT_MILI < FW_INIT_WARN_MESSAGE_INTERVAL);
+
while (fw_initializing(dev)) {
if (time_after(jiffies, end)) {
err = -EBUSY;
break;
}
+ if (warn_time_mili && time_after(jiffies, warn)) {
+ mlx5_core_warn(dev, "Waiting for FW initialization, timeout abort in %ds\n",
+ jiffies_to_msecs(end - warn) / 1000);
+ warn = jiffies + msecs_to_jiffies(warn_time_mili);
+ }
msleep(FW_INIT_WAIT_MS);
}
@@ -794,10 +804,16 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
goto err_devcom;
}
+ err = mlx5_irq_table_init(dev);
+ if (err) {
+ mlx5_core_err(dev, "failed to initialize irq table\n");
+ goto err_devcom;
+ }
+
err = mlx5_eq_table_init(dev);
if (err) {
mlx5_core_err(dev, "failed to initialize eq\n");
- goto err_devcom;
+ goto err_irq_cleanup;
}
err = mlx5_events_init(dev);
@@ -834,32 +850,32 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
goto err_rl_cleanup;
}
- err = mlx5_eswitch_init(dev);
+ err = mlx5_sriov_init(dev);
if (err) {
- mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
+ mlx5_core_err(dev, "Failed to init sriov %d\n", err);
goto err_mpfs_cleanup;
}
- err = mlx5_sriov_init(dev);
+ err = mlx5_eswitch_init(dev);
if (err) {
- mlx5_core_err(dev, "Failed to init sriov %d\n", err);
- goto err_eswitch_cleanup;
+ mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
+ goto err_sriov_cleanup;
}
err = mlx5_fpga_init(dev);
if (err) {
mlx5_core_err(dev, "Failed to init fpga device %d\n", err);
- goto err_sriov_cleanup;
+ goto err_eswitch_cleanup;
}
dev->tracer = mlx5_fw_tracer_create(dev);
return 0;
-err_sriov_cleanup:
- mlx5_sriov_cleanup(dev);
err_eswitch_cleanup:
mlx5_eswitch_cleanup(dev->priv.eswitch);
+err_sriov_cleanup:
+ mlx5_sriov_cleanup(dev);
err_mpfs_cleanup:
mlx5_mpfs_cleanup(dev);
err_rl_cleanup:
@@ -873,6 +889,8 @@ err_events_cleanup:
mlx5_events_cleanup(dev);
err_eq_cleanup:
mlx5_eq_table_cleanup(dev);
+err_irq_cleanup:
+ mlx5_irq_table_cleanup(dev);
err_devcom:
mlx5_devcom_unregister_device(dev->priv.devcom);
@@ -883,8 +901,8 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
{
mlx5_fw_tracer_destroy(dev->tracer);
mlx5_fpga_cleanup(dev);
- mlx5_sriov_cleanup(dev);
mlx5_eswitch_cleanup(dev->priv.eswitch);
+ mlx5_sriov_cleanup(dev);
mlx5_mpfs_cleanup(dev);
mlx5_cleanup_rl_table(dev);
mlx5_vxlan_destroy(dev->vxlan);
@@ -895,6 +913,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
mlx5_cq_debugfs_cleanup(dev);
mlx5_events_cleanup(dev);
mlx5_eq_table_cleanup(dev);
+ mlx5_irq_table_cleanup(dev);
mlx5_devcom_unregister_device(dev->priv.devcom);
}
@@ -911,7 +930,7 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
/* wait for firmware to accept initialization segments configurations
*/
- err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI);
+ err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI, FW_INIT_WARN_MESSAGE_INTERVAL);
if (err) {
mlx5_core_err(dev, "Firmware over %d MS in pre-initializing state, aborting\n",
FW_PRE_INIT_TIMEOUT_MILI);
@@ -924,7 +943,7 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
return err;
}
- err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
+ err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI, 0);
if (err) {
mlx5_core_err(dev, "Firmware over %d MS in initializing state, aborting\n",
FW_INIT_TIMEOUT_MILI);
@@ -1028,6 +1047,12 @@ static int mlx5_load(struct mlx5_core_dev *dev)
mlx5_events_start(dev);
mlx5_pagealloc_start(dev);
+ err = mlx5_irq_table_create(dev);
+ if (err) {
+ mlx5_core_err(dev, "Failed to alloc IRQs\n");
+ goto err_irq_table;
+ }
+
err = mlx5_eq_table_create(dev);
if (err) {
mlx5_core_err(dev, "Failed to create EQs\n");
@@ -1099,6 +1124,8 @@ err_fpga_start:
err_fw_tracer:
mlx5_eq_table_destroy(dev);
err_eq_table:
+ mlx5_irq_table_destroy(dev);
+err_irq_table:
mlx5_pagealloc_stop(dev);
mlx5_events_stop(dev);
mlx5_put_uars_page(dev, dev->priv.uar);
@@ -1115,6 +1142,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
mlx5_fpga_device_stop(dev);
mlx5_fw_tracer_cleanup(dev->tracer);
mlx5_eq_table_destroy(dev);
+ mlx5_irq_table_destroy(dev);
mlx5_pagealloc_stop(dev);
mlx5_events_stop(dev);
mlx5_put_uars_page(dev, dev->priv.uar);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 22e69d4813e4..e0f6783a5f6e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -153,6 +153,19 @@ int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam,
void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev);
void mlx5_lag_remove(struct mlx5_core_dev *dev);
+int mlx5_irq_table_init(struct mlx5_core_dev *dev);
+void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev);
+int mlx5_irq_table_create(struct mlx5_core_dev *dev);
+void mlx5_irq_table_destroy(struct mlx5_core_dev *dev);
+int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx,
+ struct notifier_block *nb);
+int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx,
+ struct notifier_block *nb);
+struct cpumask *
+mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx);
+struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *table);
+int mlx5_irq_get_num_comp(struct mlx5_irq_table *table);
+
int mlx5_events_init(struct mlx5_core_dev *dev);
void mlx5_events_cleanup(struct mlx5_core_dev *dev);
void mlx5_events_start(struct mlx5_core_dev *dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
new file mode 100644
index 000000000000..373981a659c7
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2019 Mellanox Technologies. */
+
+#include <linux/interrupt.h>
+#include <linux/notifier.h>
+#include <linux/module.h>
+#include <linux/mlx5/driver.h>
+#include "mlx5_core.h"
+#ifdef CONFIG_RFS_ACCEL
+#include <linux/cpu_rmap.h>
+#endif
+
+#define MLX5_MAX_IRQ_NAME (32)
+
+struct mlx5_irq {
+ struct atomic_notifier_head nh;
+ cpumask_var_t mask;
+ char name[MLX5_MAX_IRQ_NAME];
+};
+
+struct mlx5_irq_table {
+ struct mlx5_irq *irq;
+ int nvec;
+#ifdef CONFIG_RFS_ACCEL
+ struct cpu_rmap *rmap;
+#endif
+};
+
+int mlx5_irq_table_init(struct mlx5_core_dev *dev)
+{
+ struct mlx5_irq_table *irq_table;
+
+ irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL);
+ if (!irq_table)
+ return -ENOMEM;
+
+ dev->priv.irq_table = irq_table;
+ return 0;
+}
+
+void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
+{
+ kvfree(dev->priv.irq_table);
+}
+
+int mlx5_irq_get_num_comp(struct mlx5_irq_table *table)
+{
+ return table->nvec - MLX5_IRQ_VEC_COMP_BASE;
+}
+
+static struct mlx5_irq *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx)
+{
+ struct mlx5_irq_table *irq_table = dev->priv.irq_table;
+
+ return &irq_table->irq[vecidx];
+}
+
+int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx,
+ struct notifier_block *nb)
+{
+ struct mlx5_irq *irq;
+
+ irq = &irq_table->irq[vecidx];
+ return atomic_notifier_chain_register(&irq->nh, nb);
+}
+
+int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx,
+ struct notifier_block *nb)
+{
+ struct mlx5_irq *irq;
+
+ irq = &irq_table->irq[vecidx];
+ return atomic_notifier_chain_unregister(&irq->nh, nb);
+}
+
+static irqreturn_t mlx5_irq_int_handler(int irq, void *nh)
+{
+ atomic_notifier_call_chain(nh, 0, NULL);
+ return IRQ_HANDLED;
+}
+
+static void irq_set_name(char *name, int vecidx)
+{
+ if (vecidx == 0) {
+ snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async");
+ return;
+ }
+
+ snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d",
+ vecidx - MLX5_IRQ_VEC_COMP_BASE);
+ return;
+}
+
+static int request_irqs(struct mlx5_core_dev *dev, int nvec)
+{
+ char name[MLX5_MAX_IRQ_NAME];
+ int err;
+ int i;
+
+ for (i = 0; i < nvec; i++) {
+ struct mlx5_irq *irq = mlx5_irq_get(dev, i);
+ int irqn = pci_irq_vector(dev->pdev, i);
+
+ irq_set_name(name, i);
+ ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
+ snprintf(irq->name, MLX5_MAX_IRQ_NAME,
+ "%s@pci:%s", name, pci_name(dev->pdev));
+ err = request_irq(irqn, mlx5_irq_int_handler, 0, irq->name,
+ &irq->nh);
+ if (err) {
+ mlx5_core_err(dev, "Failed to request irq\n");
+ goto err_request_irq;
+ }
+ }
+ return 0;
+
+err_request_irq:
+ for (; i >= 0; i--) {
+ struct mlx5_irq *irq = mlx5_irq_get(dev, i);
+ int irqn = pci_irq_vector(dev->pdev, i);
+
+ free_irq(irqn, &irq->nh);
+ }
+ return err;
+}
+
+static void irq_clear_rmap(struct mlx5_core_dev *dev)
+{
+#ifdef CONFIG_RFS_ACCEL
+ struct mlx5_irq_table *irq_table = dev->priv.irq_table;
+
+ free_irq_cpu_rmap(irq_table->rmap);
+#endif
+}
+
+static int irq_set_rmap(struct mlx5_core_dev *mdev)
+{
+ int err = 0;
+#ifdef CONFIG_RFS_ACCEL
+ struct mlx5_irq_table *irq_table = mdev->priv.irq_table;
+ int num_affinity_vec;
+ int vecidx;
+
+ num_affinity_vec = mlx5_irq_get_num_comp(irq_table);
+ irq_table->rmap = alloc_irq_cpu_rmap(num_affinity_vec);
+ if (!irq_table->rmap) {
+ err = -ENOMEM;
+ mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
+ goto err_out;
+ }
+
+ vecidx = MLX5_IRQ_VEC_COMP_BASE;
+ for (; vecidx < irq_table->nvec; vecidx++) {
+ err = irq_cpu_rmap_add(irq_table->rmap,
+ pci_irq_vector(mdev->pdev, vecidx));
+ if (err) {
+ mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
+ err);
+ goto err_irq_cpu_rmap_add;
+ }
+ }
+ return 0;
+
+err_irq_cpu_rmap_add:
+ irq_clear_rmap(mdev);
+err_out:
+#endif
+ return err;
+}
+
+/* Completion IRQ vectors */
+
+static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
+{
+ int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
+ struct mlx5_irq *irq;
+ int irqn;
+
+ irq = mlx5_irq_get(mdev, vecidx);
+ irqn = pci_irq_vector(mdev->pdev, vecidx);
+ if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {
+ mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
+ return -ENOMEM;
+ }
+
+ cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node),
+ irq->mask);
+ if (IS_ENABLED(CONFIG_SMP) &&
+ irq_set_affinity_hint(irqn, irq->mask))
+ mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x",
+ irqn);
+
+ return 0;
+}
+
+static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
+{
+ int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
+ struct mlx5_irq *irq;
+ int irqn;
+
+ irq = mlx5_irq_get(mdev, vecidx);
+ irqn = pci_irq_vector(mdev->pdev, vecidx);
+ irq_set_affinity_hint(irqn, NULL);
+ free_cpumask_var(irq->mask);
+}
+
+static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev)
+{
+ int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
+ int err;
+ int i;
+
+ for (i = 0; i < nvec; i++) {
+ err = set_comp_irq_affinity_hint(mdev, i);
+ if (err)
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ for (i--; i >= 0; i--)
+ clear_comp_irq_affinity_hint(mdev, i);
+
+ return err;
+}
+
+static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev)
+{
+ int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
+ int i;
+
+ for (i = 0; i < nvec; i++)
+ clear_comp_irq_affinity_hint(mdev, i);
+}
+
+struct cpumask *
+mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx)
+{
+ return irq_table->irq[vecidx].mask;
+}
+
+#ifdef CONFIG_RFS_ACCEL
+struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table)
+{
+ return irq_table->rmap;
+}
+#endif
+
+static void unrequest_irqs(struct mlx5_core_dev *dev)
+{
+ struct mlx5_irq_table *table = dev->priv.irq_table;
+ int i;
+
+ for (i = 0; i < table->nvec; i++)
+ free_irq(pci_irq_vector(dev->pdev, i),
+ &mlx5_irq_get(dev, i)->nh);
+}
+
+int mlx5_irq_table_create(struct mlx5_core_dev *dev)
+{
+ struct mlx5_priv *priv = &dev->priv;
+ struct mlx5_irq_table *table = priv->irq_table;
+ int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
+ MLX5_CAP_GEN(dev, max_num_eqs) :
+ 1 << MLX5_CAP_GEN(dev, log_max_eq);
+ int nvec;
+ int err;
+
+ nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
+ MLX5_IRQ_VEC_COMP_BASE;
+ nvec = min_t(int, nvec, num_eqs);
+ if (nvec <= MLX5_IRQ_VEC_COMP_BASE)
+ return -ENOMEM;
+
+ table->irq = kcalloc(nvec, sizeof(*table->irq), GFP_KERNEL);
+ if (!table->irq)
+ return -ENOMEM;
+
+ nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1,
+ nvec, PCI_IRQ_MSIX);
+ if (nvec < 0) {
+ err = nvec;
+ goto err_free_irq;
+ }
+
+ table->nvec = nvec;
+
+ err = irq_set_rmap(dev);
+ if (err)
+ goto err_set_rmap;
+
+ err = request_irqs(dev, nvec);
+ if (err)
+ goto err_request_irqs;
+
+ err = set_comp_irq_affinity_hints(dev);
+ if (err) {
+ mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n");
+ goto err_set_affinity;
+ }
+
+ return 0;
+
+err_set_affinity:
+ unrequest_irqs(dev);
+err_request_irqs:
+ irq_clear_rmap(dev);
+err_set_rmap:
+ pci_free_irq_vectors(dev->pdev);
+err_free_irq:
+ kfree(table->irq);
+ return err;
+}
+
+void mlx5_irq_table_destroy(struct mlx5_core_dev *dev)
+{
+ struct mlx5_irq_table *table = dev->priv.irq_table;
+ int i;
+
+ /* free_irq requires that affinity and rmap will be cleared
+ * before calling it. This is why there is asymmetry with set_rmap
+ * which should be called after alloc_irq but before request_irq.
+ */
+ irq_clear_rmap(dev);
+ clear_comp_irqs_affinity_hints(dev);
+ for (i = 0; i < table->nvec; i++)
+ free_irq(pci_irq_vector(dev->pdev, i),
+ &mlx5_irq_get(dev, i)->nh);
+ pci_free_irq_vectors(dev->pdev);
+ kfree(table->irq);
+}
+
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c
index 86f77456f873..401441aefbcb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c
@@ -106,10 +106,10 @@ static int mlx5_rdma_enable_roce_steering(struct mlx5_core_dev *dev)
return 0;
-destroy_flow_table:
- mlx5_destroy_flow_table(ft);
destroy_flow_group:
mlx5_destroy_flow_group(fg);
+destroy_flow_table:
+ mlx5_destroy_flow_table(ft);
free:
kvfree(spec);
kvfree(flow_group_in);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
index a249b3c3843d..2eecb831c499 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
@@ -208,6 +208,27 @@ void mlx5_sriov_detach(struct mlx5_core_dev *dev)
mlx5_device_disable_sriov(dev);
}
+static u16 mlx5_get_max_vfs(struct mlx5_core_dev *dev)
+{
+ u32 out[MLX5_ST_SZ_DW(query_esw_functions_out)] = {};
+ u16 host_total_vfs;
+ int err;
+
+ if (mlx5_core_is_ecpf_esw_manager(dev)) {
+ err = mlx5_esw_query_functions(dev, out, sizeof(out));
+ host_total_vfs = MLX5_GET(query_esw_functions_out, out,
+ host_params_context.host_total_vfs);
+
+ /* Old FW doesn't support getting total_vfs from esw func
+ * but supports getting it from pci_sriov.
+ */
+ if (!err && host_total_vfs)
+ return host_total_vfs;
+ }
+
+ return pci_sriov_get_totalvfs(dev->pdev);
+}
+
int mlx5_sriov_init(struct mlx5_core_dev *dev)
{
struct mlx5_core_sriov *sriov = &dev->priv.sriov;
@@ -218,6 +239,7 @@ int mlx5_sriov_init(struct mlx5_core_dev *dev)
return 0;
total_vfs = pci_sriov_get_totalvfs(pdev);
+ sriov->max_vfs = mlx5_get_max_vfs(dev);
sriov->num_vfs = pci_num_vf(pdev);
sriov->vfs_ctx = kcalloc(total_vfs, sizeof(*sriov->vfs_ctx), GFP_KERNEL);
if (!sriov->vfs_ctx)
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 97f668a39ae1..f3e3a8c861d6 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -482,7 +482,7 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs,
queue->queue_size,
IB_MR_TYPE_MEM_REG,
- nvme_rdma_get_max_fr_pages(ibdev));
+ nvme_rdma_get_max_fr_pages(ibdev), 0);
if (ret) {
dev_err(queue->ctrl->ctrl.device,
"failed to initialize MR pool sized %d for QID %d\n",
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index fc2b6e807f06..5e760067ac41 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -342,7 +342,7 @@ enum mlx5_event {
MLX5_EVENT_TYPE_PAGE_FAULT = 0xc,
MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0xd,
- MLX5_EVENT_TYPE_HOST_PARAMS_CHANGE = 0xe,
+ MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0xe,
MLX5_EVENT_TYPE_DCT_DRAINED = 0x1c,
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 5a27246db883..d8ab633406c2 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -107,6 +107,7 @@ enum {
MLX5_REG_FPGA_CAP = 0x4022,
MLX5_REG_FPGA_CTRL = 0x4023,
MLX5_REG_FPGA_ACCESS_REG = 0x4024,
+ MLX5_REG_CORE_DUMP = 0x402e,
MLX5_REG_PCAP = 0x5001,
MLX5_REG_PMTU = 0x5003,
MLX5_REG_PTYS = 0x5004,
@@ -469,6 +470,7 @@ struct mlx5_core_sriov {
struct mlx5_vf_context *vfs_ctx;
int num_vfs;
int enabled_vfs;
+ u16 max_vfs;
};
struct mlx5_fc_stats {
@@ -490,6 +492,7 @@ struct mlx5_eswitch;
struct mlx5_lag;
struct mlx5_devcom;
struct mlx5_eq_table;
+struct mlx5_irq_table;
struct mlx5_rate_limit {
u32 rate;
@@ -519,6 +522,8 @@ struct mlx5_core_roce {
};
struct mlx5_priv {
+ /* IRQ table valid only for real pci devices PF or VF */
+ struct mlx5_irq_table *irq_table;
struct mlx5_eq_table *eq_table;
/* pages stuff */
@@ -1102,13 +1107,9 @@ static inline bool mlx5_ecpf_vport_exists(struct mlx5_core_dev *dev)
return mlx5_core_is_pf(dev) && MLX5_CAP_ESW(dev, ecpf_vport_exists);
}
-#define MLX5_HOST_PF_MAX_VFS (127u)
static inline u16 mlx5_core_max_vfs(struct mlx5_core_dev *dev)
{
- if (mlx5_core_is_ecpf_esw_manager(dev))
- return MLX5_HOST_PF_MAX_VFS;
- else
- return pci_sriov_get_totalvfs(dev->pdev);
+ return dev->priv.sriov.max_vfs;
}
static inline int mlx5_get_gid_table_len(u16 param)
diff --git a/include/linux/mlx5/eq.h b/include/linux/mlx5/eq.h
index 00045cc4ea11..70e16dcfb4c4 100644
--- a/include/linux/mlx5/eq.h
+++ b/include/linux/mlx5/eq.h
@@ -4,17 +4,7 @@
#ifndef MLX5_CORE_EQ_H
#define MLX5_CORE_EQ_H
-enum {
- MLX5_EQ_PAGEREQ_IDX = 0,
- MLX5_EQ_CMD_IDX = 1,
- MLX5_EQ_ASYNC_IDX = 2,
- /* reserved to be used by mlx5_core ulps (mlx5e/mlx5_ib) */
- MLX5_EQ_PFAULT_IDX = 3,
- MLX5_EQ_MAX_ASYNC_EQS,
- /* completion eqs vector indices start here */
- MLX5_EQ_VEC_COMP_BASE = MLX5_EQ_MAX_ASYNC_EQS,
-};
-
+#define MLX5_IRQ_VEC_COMP_BASE 1
#define MLX5_NUM_CMD_EQE (32)
#define MLX5_NUM_ASYNC_EQE (0x1000)
#define MLX5_NUM_SPARE_EQE (0x80)
@@ -23,18 +13,19 @@ struct mlx5_eq;
struct mlx5_core_dev;
struct mlx5_eq_param {
- u8 index;
+ u8 irq_index;
int nent;
u64 mask;
- void *context;
- irq_handler_t handler;
};
struct mlx5_eq *
-mlx5_eq_create_generic(struct mlx5_core_dev *dev, const char *name,
- struct mlx5_eq_param *param);
+mlx5_eq_create_generic(struct mlx5_core_dev *dev, struct mlx5_eq_param *param);
int
mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
+int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+ struct notifier_block *nb);
+void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+ struct notifier_block *nb);
struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc);
void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm);
diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h
index cf226c190329..ee1335ab1df0 100644
--- a/include/linux/mlx5/eswitch.h
+++ b/include/linux/mlx5/eswitch.h
@@ -7,6 +7,7 @@
#define _MLX5_ESWITCH_
#include <linux/mlx5/driver.h>
+#include <net/devlink.h>
#define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_manager)
@@ -29,17 +30,19 @@ enum {
};
struct mlx5_eswitch_rep;
-struct mlx5_eswitch_rep_if {
- int (*load)(struct mlx5_core_dev *dev,
- struct mlx5_eswitch_rep *rep);
- void (*unload)(struct mlx5_eswitch_rep *rep);
- void *(*get_proto_dev)(struct mlx5_eswitch_rep *rep);
- void *priv;
- atomic_t state;
+struct mlx5_eswitch_rep_ops {
+ int (*load)(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep);
+ void (*unload)(struct mlx5_eswitch_rep *rep);
+ void *(*get_proto_dev)(struct mlx5_eswitch_rep *rep);
+};
+
+struct mlx5_eswitch_rep_data {
+ void *priv;
+ atomic_t state;
};
struct mlx5_eswitch_rep {
- struct mlx5_eswitch_rep_if rep_if[NUM_REP_TYPES];
+ struct mlx5_eswitch_rep_data rep_data[NUM_REP_TYPES];
u16 vport;
u8 hw_id[ETH_ALEN];
u16 vlan;
@@ -47,7 +50,7 @@ struct mlx5_eswitch_rep {
};
void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
- struct mlx5_eswitch_rep_if *rep_if,
+ const struct mlx5_eswitch_rep_ops *ops,
u8 rep_type);
void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type);
void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
@@ -60,4 +63,15 @@ u8 mlx5_eswitch_mode(struct mlx5_eswitch *esw);
struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw,
u16 vport_num, u32 sqn);
+
+#ifdef CONFIG_MLX5_ESWITCH
+enum devlink_eswitch_encap_mode
+mlx5_eswitch_get_encap_mode(const struct mlx5_core_dev *dev);
+#else /* CONFIG_MLX5_ESWITCH */
+static inline enum devlink_eswitch_encap_mode
+mlx5_eswitch_get_encap_mode(const struct mlx5_core_dev *dev)
+{
+ return DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+}
+#endif /* CONFIG_MLX5_ESWITCH */
#endif
diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h
index e690ba0f965c..2ddaa97f2179 100644
--- a/include/linux/mlx5/fs.h
+++ b/include/linux/mlx5/fs.h
@@ -47,6 +47,7 @@ enum {
enum {
MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT = BIT(0),
MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1),
+ MLX5_FLOW_TABLE_TERMINATION = BIT(2),
};
#define LEFTOVERS_RULE_NUM 2
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 5e74305e2e57..e3c154b573a2 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -155,7 +155,7 @@ enum {
MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY = 0x725,
MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY = 0x726,
MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS = 0x727,
- MLX5_CMD_OP_QUERY_HOST_PARAMS = 0x740,
+ MLX5_CMD_OP_QUERY_ESW_FUNCTIONS = 0x740,
MLX5_CMD_OP_QUERY_VPORT_STATE = 0x750,
MLX5_CMD_OP_MODIFY_VPORT_STATE = 0x751,
MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT = 0x752,
@@ -382,7 +382,8 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
u8 reformat_and_modify_action[0x1];
u8 reserved_at_15[0x2];
u8 table_miss_action_domain[0x1];
- u8 reserved_at_18[0x8];
+ u8 termination_table[0x1];
+ u8 reserved_at_19[0x7];
u8 reserved_at_20[0x2];
u8 log_max_ft_size[0x6];
u8 log_max_modify_header_context[0x8];
@@ -664,7 +665,9 @@ struct mlx5_ifc_e_switch_cap_bits {
u8 vport_svlan_insert[0x1];
u8 vport_cvlan_insert_if_not_exist[0x1];
u8 vport_cvlan_insert_overwrite[0x1];
- u8 reserved_at_5[0x16];
+ u8 reserved_at_5[0x14];
+ u8 esw_functions_changed[0x1];
+ u8 reserved_at_1a[0x1];
u8 ecpf_vport_exists[0x1];
u8 counter_eswitch_affinity[0x1];
u8 merged_eswitch[0x1];
@@ -715,7 +718,9 @@ struct mlx5_ifc_qos_cap_bits {
};
struct mlx5_ifc_debug_cap_bits {
- u8 reserved_at_0[0x20];
+ u8 core_dump_general[0x1];
+ u8 core_dump_qp[0x1];
+ u8 reserved_at_2[0x1e];
u8 reserved_at_20[0x2];
u8 stall_detect[0x1];
@@ -2531,6 +2536,7 @@ union mlx5_ifc_hca_cap_union_bits {
struct mlx5_ifc_e_switch_cap_bits e_switch_cap;
struct mlx5_ifc_vector_calc_cap_bits vector_calc_cap;
struct mlx5_ifc_qos_cap_bits qos_cap;
+ struct mlx5_ifc_debug_cap_bits debug_cap;
struct mlx5_ifc_fpga_cap_bits fpga_cap;
u8 reserved_at_0[0x8000];
};
@@ -7236,7 +7242,8 @@ struct mlx5_ifc_create_flow_table_out_bits {
struct mlx5_ifc_flow_table_context_bits {
u8 reformat_en[0x1];
u8 decap_en[0x1];
- u8 reserved_at_2[0x2];
+ u8 reserved_at_2[0x1];
+ u8 termination_table[0x1];
u8 table_miss_action[0x4];
u8 level[0x8];
u8 reserved_at_10[0x8];
@@ -8546,6 +8553,18 @@ struct mlx5_ifc_qcam_reg_bits {
u8 reserved_at_1c0[0x80];
};
+struct mlx5_ifc_core_dump_reg_bits {
+ u8 reserved_at_0[0x18];
+ u8 core_dump_type[0x8];
+
+ u8 reserved_at_20[0x30];
+ u8 vhca_id[0x10];
+
+ u8 reserved_at_60[0x8];
+ u8 qpn[0x18];
+ u8 reserved_at_80[0x180];
+};
+
struct mlx5_ifc_pcap_reg_bits {
u8 reserved_at_0[0x8];
u8 local_port[0x8];
@@ -9692,7 +9711,7 @@ struct mlx5_ifc_host_params_context_bits {
u8 reserved_at_8[0x8];
u8 host_num_of_vfs[0x10];
- u8 reserved_at_20[0x10];
+ u8 host_total_vfs[0x10];
u8 host_pci_bus[0x10];
u8 reserved_at_40[0x10];
@@ -9704,7 +9723,7 @@ struct mlx5_ifc_host_params_context_bits {
u8 reserved_at_80[0x180];
};
-struct mlx5_ifc_query_host_params_in_bits {
+struct mlx5_ifc_query_esw_functions_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
@@ -9714,7 +9733,7 @@ struct mlx5_ifc_query_host_params_in_bits {
u8 reserved_at_40[0x40];
};
-struct mlx5_ifc_query_host_params_out_bits {
+struct mlx5_ifc_query_esw_functions_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index 3ba4edbd17a6..08e43cd9e742 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -37,7 +37,8 @@
#include <linux/mlx5/driver.h>
#define MLX5_INVALID_LKEY 0x100
-#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5)
+/* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */
+#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)
#define MLX5_DIF_SIZE 8
#define MLX5_STRIDE_BLOCK_OP 0x400
#define MLX5_CPY_GRD_MASK 0xc0
diff --git a/include/net/devlink.h b/include/net/devlink.h
index c9fbeb5b701f..811a4ce96950 100644
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -526,8 +526,10 @@ struct devlink_ops {
int (*eswitch_inline_mode_get)(struct devlink *devlink, u8 *p_inline_mode);
int (*eswitch_inline_mode_set)(struct devlink *devlink, u8 inline_mode,
struct netlink_ext_ack *extack);
- int (*eswitch_encap_mode_get)(struct devlink *devlink, u8 *p_encap_mode);
- int (*eswitch_encap_mode_set)(struct devlink *devlink, u8 encap_mode,
+ int (*eswitch_encap_mode_get)(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode *p_encap_mode);
+ int (*eswitch_encap_mode_set)(struct devlink *devlink,
+ enum devlink_eswitch_encap_mode encap_mode,
struct netlink_ext_ack *extack);
int (*info_get)(struct devlink *devlink, struct devlink_info_req *req,
struct netlink_ext_ack *extack);
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 040d853077c6..1052d0d62be7 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -46,7 +46,6 @@ struct ib_umem {
struct mm_struct *owning_mm;
size_t length;
unsigned long address;
- int page_shift;
u32 writable : 1;
u32 is_odp : 1;
struct work_struct work;
@@ -58,24 +57,14 @@ struct ib_umem {
/* Returns the offset of the umem start relative to the first page. */
static inline int ib_umem_offset(struct ib_umem *umem)
{
- return umem->address & (BIT(umem->page_shift) - 1);
-}
-
-/* Returns the first page of an ODP umem. */
-static inline unsigned long ib_umem_start(struct ib_umem *umem)
-{
- return umem->address - ib_umem_offset(umem);
-}
-
-/* Returns the address of the page after the last one of an ODP umem. */
-static inline unsigned long ib_umem_end(struct ib_umem *umem)
-{
- return ALIGN(umem->address + umem->length, BIT(umem->page_shift));
+ return umem->address & ~PAGE_MASK;
}
static inline size_t ib_umem_num_pages(struct ib_umem *umem)
{
- return (ib_umem_end(umem) - ib_umem_start(umem)) >> umem->page_shift;
+ return (ALIGN(umem->address + umem->length, PAGE_SIZE) -
+ ALIGN_DOWN(umem->address, PAGE_SIZE)) >>
+ PAGE_SHIFT;
}
#ifdef CONFIG_INFINIBAND_USER_MEM
diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index eeec4e53c448..479db5c98ff6 100644
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -76,6 +76,7 @@ struct ib_umem_odp {
struct completion notifier_completion;
int dying;
+ unsigned int page_shift;
struct work_struct work;
};
@@ -84,6 +85,25 @@ static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem)
return container_of(umem, struct ib_umem_odp, umem);
}
+/* Returns the first page of an ODP umem. */
+static inline unsigned long ib_umem_start(struct ib_umem_odp *umem_odp)
+{
+ return ALIGN_DOWN(umem_odp->umem.address, 1UL << umem_odp->page_shift);
+}
+
+/* Returns the address of the page after the last one of an ODP umem. */
+static inline unsigned long ib_umem_end(struct ib_umem_odp *umem_odp)
+{
+ return ALIGN(umem_odp->umem.address + umem_odp->umem.length,
+ 1UL << umem_odp->page_shift);
+}
+
+static inline size_t ib_umem_odp_num_pages(struct ib_umem_odp *umem_odp)
+{
+ return (ib_umem_end(umem_odp) - ib_umem_start(umem_odp)) >>
+ umem_odp->page_shift;
+}
+
/*
* The lower 2 bits of the DMA address signal the R/W permissions for
* the entry. To upgrade the permissions, provide the appropriate
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 54873085f2da..26e9c2594913 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -63,6 +63,7 @@
#include <linux/preempt.h>
#include <uapi/rdma/ib_user_verbs.h>
#include <rdma/restrack.h>
+#include <rdma/signature.h>
#include <uapi/rdma/rdma_user_ioctl.h>
#include <uapi/rdma/ib_user_ioctl_verbs.h>
@@ -132,17 +133,6 @@ struct ib_gid_attr {
u8 port_num;
};
-enum rdma_node_type {
- /* IB values map to NodeInfo:NodeType. */
- RDMA_NODE_IB_CA = 1,
- RDMA_NODE_IB_SWITCH,
- RDMA_NODE_IB_ROUTER,
- RDMA_NODE_RNIC,
- RDMA_NODE_USNIC,
- RDMA_NODE_USNIC_UDP,
- RDMA_NODE_UNSPECIFIED,
-};
-
enum {
/* set the local administered indication */
IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2,
@@ -164,7 +154,7 @@ enum rdma_protocol_type {
};
__attribute_const__ enum rdma_transport_type
-rdma_node_get_transport(enum rdma_node_type node_type);
+rdma_node_get_transport(unsigned int node_type);
enum rdma_network_type {
RDMA_NETWORK_IB,
@@ -263,7 +253,7 @@ enum ib_device_cap_flags {
*/
IB_DEVICE_CROSS_CHANNEL = (1 << 27),
IB_DEVICE_MANAGED_FLOW_STEERING = (1 << 29),
- IB_DEVICE_SIGNATURE_HANDOVER = (1 << 30),
+ IB_DEVICE_INTEGRITY_HANDOVER = (1 << 30),
IB_DEVICE_ON_DEMAND_PAGING = (1ULL << 31),
IB_DEVICE_SG_GAPS_REG = (1ULL << 32),
IB_DEVICE_VIRTUAL_FUNCTION = (1ULL << 33),
@@ -275,17 +265,6 @@ enum ib_device_cap_flags {
IB_DEVICE_ALLOW_USER_UNREG = (1ULL << 37),
};
-enum ib_signature_prot_cap {
- IB_PROT_T10DIF_TYPE_1 = 1,
- IB_PROT_T10DIF_TYPE_2 = 1 << 1,
- IB_PROT_T10DIF_TYPE_3 = 1 << 2,
-};
-
-enum ib_signature_guard_cap {
- IB_GUARD_T10DIF_CRC = 1,
- IB_GUARD_T10DIF_CSUM = 1 << 1,
-};
-
enum ib_atomic_cap {
IB_ATOMIC_NONE,
IB_ATOMIC_HCA,
@@ -411,6 +390,7 @@ struct ib_device_attr {
int max_srq_wr;
int max_srq_sge;
unsigned int max_fast_reg_page_list_len;
+ unsigned int max_pi_fast_reg_page_list_len;
u16 max_pkeys;
u8 local_ca_ack_delay;
int sig_prot_cap;
@@ -796,118 +776,26 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
* enum ib_mr_type - memory region type
* @IB_MR_TYPE_MEM_REG: memory region that is used for
* normal registration
- * @IB_MR_TYPE_SIGNATURE: memory region that is used for
- * signature operations (data-integrity
- * capable regions)
* @IB_MR_TYPE_SG_GAPS: memory region that is capable to
* register any arbitrary sg lists (without
* the normal mr constraints - see
* ib_map_mr_sg)
+ * @IB_MR_TYPE_DM: memory region that is used for device
+ * memory registration
+ * @IB_MR_TYPE_USER: memory region that is used for the user-space
+ * application
+ * @IB_MR_TYPE_DMA: memory region that is used for DMA operations
+ * without address translations (VA=PA)
+ * @IB_MR_TYPE_INTEGRITY: memory region that is used for
+ * data integrity operations
*/
enum ib_mr_type {
IB_MR_TYPE_MEM_REG,
- IB_MR_TYPE_SIGNATURE,
IB_MR_TYPE_SG_GAPS,
-};
-
-/**
- * Signature types
- * IB_SIG_TYPE_NONE: Unprotected.
- * IB_SIG_TYPE_T10_DIF: Type T10-DIF
- */
-enum ib_signature_type {
- IB_SIG_TYPE_NONE,
- IB_SIG_TYPE_T10_DIF,
-};
-
-/**
- * Signature T10-DIF block-guard types
- * IB_T10DIF_CRC: Corresponds to T10-PI mandated CRC checksum rules.
- * IB_T10DIF_CSUM: Corresponds to IP checksum rules.
- */
-enum ib_t10_dif_bg_type {
- IB_T10DIF_CRC,
- IB_T10DIF_CSUM
-};
-
-/**
- * struct ib_t10_dif_domain - Parameters specific for T10-DIF
- * domain.
- * @bg_type: T10-DIF block guard type (CRC|CSUM)
- * @pi_interval: protection information interval.
- * @bg: seed of guard computation.
- * @app_tag: application tag of guard block
- * @ref_tag: initial guard block reference tag.
- * @ref_remap: Indicate wethear the reftag increments each block
- * @app_escape: Indicate to skip block check if apptag=0xffff
- * @ref_escape: Indicate to skip block check if reftag=0xffffffff
- * @apptag_check_mask: check bitmask of application tag.
- */
-struct ib_t10_dif_domain {
- enum ib_t10_dif_bg_type bg_type;
- u16 pi_interval;
- u16 bg;
- u16 app_tag;
- u32 ref_tag;
- bool ref_remap;
- bool app_escape;
- bool ref_escape;
- u16 apptag_check_mask;
-};
-
-/**
- * struct ib_sig_domain - Parameters for signature domain
- * @sig_type: specific signauture type
- * @sig: union of all signature domain attributes that may
- * be used to set domain layout.
- */
-struct ib_sig_domain {
- enum ib_signature_type sig_type;
- union {
- struct ib_t10_dif_domain dif;
- } sig;
-};
-
-/**
- * struct ib_sig_attrs - Parameters for signature handover operation
- * @check_mask: bitmask for signature byte check (8 bytes)
- * @mem: memory domain layout desciptor.
- * @wire: wire domain layout desciptor.
- */
-struct ib_sig_attrs {
- u8 check_mask;
- struct ib_sig_domain mem;
- struct ib_sig_domain wire;
-};
-
-enum ib_sig_err_type {
- IB_SIG_BAD_GUARD,
- IB_SIG_BAD_REFTAG,
- IB_SIG_BAD_APPTAG,
-};
-
-/**
- * Signature check masks (8 bytes in total) according to the T10-PI standard:
- * -------- -------- ------------
- * | GUARD | APPTAG | REFTAG |
- * | 2B | 2B | 4B |
- * -------- -------- ------------
- */
-enum {
- IB_SIG_CHECK_GUARD = 0xc0,
- IB_SIG_CHECK_APPTAG = 0x30,
- IB_SIG_CHECK_REFTAG = 0x0f,
-};
-
-/**
- * struct ib_sig_err - signature error descriptor
- */
-struct ib_sig_err {
- enum ib_sig_err_type err_type;
- u32 expected;
- u32 actual;
- u64 sig_err_offset;
- u32 key;
+ IB_MR_TYPE_DM,
+ IB_MR_TYPE_USER,
+ IB_MR_TYPE_DMA,
+ IB_MR_TYPE_INTEGRITY,
};
enum ib_mr_status_check {
@@ -1164,7 +1052,7 @@ enum ib_qp_create_flags {
IB_QP_CREATE_MANAGED_SEND = 1 << 3,
IB_QP_CREATE_MANAGED_RECV = 1 << 4,
IB_QP_CREATE_NETIF_QP = 1 << 5,
- IB_QP_CREATE_SIGNATURE_EN = 1 << 6,
+ IB_QP_CREATE_INTEGRITY_EN = 1 << 6,
/* FREE = 1 << 7, */
IB_QP_CREATE_SCATTER_FCS = 1 << 8,
IB_QP_CREATE_CVLAN_STRIPPING = 1 << 9,
@@ -1343,7 +1231,7 @@ enum ib_wr_opcode {
/* These are kernel only and can not be issued by userspace */
IB_WR_REG_MR = 0x20,
- IB_WR_REG_SIG_MR,
+ IB_WR_REG_MR_INTEGRITY,
/* reserve values for low level drivers' internal use.
* These values will not be used at all in the ib core layer.
@@ -1453,20 +1341,6 @@ static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
return container_of(wr, struct ib_reg_wr, wr);
}
-struct ib_sig_handover_wr {
- struct ib_send_wr wr;
- struct ib_sig_attrs *sig_attrs;
- struct ib_mr *sig_mr;
- int access_flags;
- struct ib_sge *prot;
-};
-
-static inline const struct ib_sig_handover_wr *
-sig_handover_wr(const struct ib_send_wr *wr)
-{
- return container_of(wr, struct ib_sig_handover_wr, wr);
-}
-
struct ib_recv_wr {
struct ib_recv_wr *next;
union {
@@ -1818,6 +1692,7 @@ struct ib_qp {
struct ib_qp_security *qp_sec;
u8 port;
+ bool integrity_en;
/*
* Implementation details of the RDMA core, don't use in drivers:
*/
@@ -1840,6 +1715,7 @@ struct ib_mr {
u64 iova;
u64 length;
unsigned int page_size;
+ enum ib_mr_type type;
bool need_inval;
union {
struct ib_uobject *uobject; /* user */
@@ -1847,7 +1723,7 @@ struct ib_mr {
};
struct ib_dm *dm;
-
+ struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
/*
* Implementation details of the RDMA core, don't use in drivers:
*/
@@ -2329,6 +2205,11 @@ struct iw_cm_conn_param;
* need to define the supported operations, otherwise they will be set to null.
*/
struct ib_device_ops {
+ struct module *owner;
+ enum rdma_driver_id driver_id;
+ u32 uverbs_abi_ver;
+ unsigned int uverbs_no_driver_id_binding:1;
+
int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
const struct ib_send_wr **bad_send_wr);
int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
@@ -2454,11 +2335,10 @@ struct ib_device_ops {
int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
- struct ib_cq *(*create_cq)(struct ib_device *device,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata);
+ int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
- int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
+ void (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
@@ -2470,6 +2350,9 @@ struct ib_device_ops {
int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
+ struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
+ u32 max_num_data_sg,
+ u32 max_num_meta_sg);
int (*advise_mr)(struct ib_pd *pd,
enum ib_uverbs_advise_mr_advice advice, u32 flags,
struct ib_sge *sg_list, u32 num_sge,
@@ -2516,7 +2399,7 @@ struct ib_device_ops {
struct ib_wq *(*create_wq)(struct ib_pd *pd,
struct ib_wq_init_attr *init_attr,
struct ib_udata *udata);
- int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
+ void (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
u32 wq_attr_mask, struct ib_udata *udata);
struct ib_rwq_ind_table *(*create_rwq_ind_table)(
@@ -2538,6 +2421,11 @@ struct ib_device_ops {
int (*read_counters)(struct ib_counters *counters,
struct ib_counters_read_attr *counters_read_attr,
struct uverbs_attr_bundle *attrs);
+ int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset);
+
/**
* alloc_hw_stats - Allocate a struct rdma_hw_stats and fill in the
* driver initialized data. The struct is kfree()'ed by the sysfs
@@ -2597,6 +2485,7 @@ struct ib_device_ops {
int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
DECLARE_RDMA_OBJ_SIZE(ib_ah);
+ DECLARE_RDMA_OBJ_SIZE(ib_cq);
DECLARE_RDMA_OBJ_SIZE(ib_pd);
DECLARE_RDMA_OBJ_SIZE(ib_srq);
DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
@@ -2636,7 +2525,6 @@ struct ib_device {
int num_comp_vectors;
- struct module *owner;
union {
struct device dev;
struct ib_core_device coredev;
@@ -2648,7 +2536,6 @@ struct ib_device {
*/
const struct attribute_group *groups[3];
- int uverbs_abi_ver;
u64 uverbs_cmd_mask;
u64 uverbs_ex_cmd_mask;
@@ -2672,7 +2559,6 @@ struct ib_device {
struct rdma_restrack_root *res;
const struct uapi_definition *driver_def;
- enum rdma_driver_id driver_id;
/*
* Positive refcount indicates that the device is currently
@@ -2694,11 +2580,15 @@ struct ib_device {
u32 iw_driver_flags;
};
+struct ib_client_nl_info;
struct ib_client {
const char *name;
void (*add) (struct ib_device *);
void (*remove)(struct ib_device *, void *client_data);
void (*rename)(struct ib_device *dev, void *client_data);
+ int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
+ struct ib_client_nl_info *res);
+ int (*get_global_nl_info)(struct ib_client_nl_info *res);
/* Returns the net_dev belonging to this ib_client and matching the
* given parameters.
@@ -3859,9 +3749,9 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
*
* NOTE: for user cq use ib_destroy_cq_user with valid udata!
*/
-static inline int ib_destroy_cq(struct ib_cq *cq)
+static inline void ib_destroy_cq(struct ib_cq *cq)
{
- return ib_destroy_cq_user(cq, NULL);
+ ib_destroy_cq_user(cq, NULL);
}
/**
@@ -4148,6 +4038,10 @@ static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);
}
+struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_data_sg,
+ u32 max_num_meta_sg);
+
/**
* ib_update_fast_reg_key - updates the key portion of the fast_reg MR
* R_Key and L_Key.
@@ -4332,6 +4226,10 @@ int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset, unsigned int page_size);
+int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
+ int data_sg_nents, unsigned int *data_sg_offset,
+ struct scatterlist *meta_sg, int meta_sg_nents,
+ unsigned int *meta_sg_offset, unsigned int page_size);
static inline int
ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
diff --git a/include/rdma/mr_pool.h b/include/rdma/mr_pool.h
index 83763ef82354..e77123bcb43b 100644
--- a/include/rdma/mr_pool.h
+++ b/include/rdma/mr_pool.h
@@ -11,7 +11,7 @@ struct ib_mr *ib_mr_pool_get(struct ib_qp *qp, struct list_head *list);
void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr);
int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,
- enum ib_mr_type type, u32 max_num_sg);
+ enum ib_mr_type type, u32 max_num_sg, u32 max_num_meta_sg);
void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list);
#endif /* _RDMA_MR_POOL_H */
diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
index 10732ab31ba2..6631624e4d7c 100644
--- a/include/rdma/rdma_netlink.h
+++ b/include/rdma/rdma_netlink.h
@@ -6,6 +6,12 @@
#include <linux/netlink.h>
#include <uapi/rdma/rdma_netlink.h>
+enum {
+ RDMA_NLDEV_ATTR_EMPTY_STRING = 1,
+ RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
+ RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE = 32,
+};
+
struct rdma_nl_cbs {
int (*doit)(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack);
@@ -110,4 +116,6 @@ void rdma_link_register(struct rdma_link_ops *ops);
void rdma_link_unregister(struct rdma_link_ops *ops);
#define MODULE_ALIAS_RDMA_LINK(type) MODULE_ALIAS("rdma-link-" type)
+#define MODULE_ALIAS_RDMA_CLIENT(type) MODULE_ALIAS("rdma-client-" type)
+
#endif /* _RDMA_NETLINK_H */
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index b9cd06db1a71..997f42678806 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -555,7 +555,7 @@ static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
struct rvt_dev_info *rvt_alloc_device(size_t size, int nports);
void rvt_dealloc_device(struct rvt_dev_info *rdi);
-int rvt_register_device(struct rvt_dev_info *rvd, u32 driver_id);
+int rvt_register_device(struct rvt_dev_info *rvd);
void rvt_unregister_device(struct rvt_dev_info *rvd);
int rvt_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 68e38c20afc0..84d0f36afc2f 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -565,42 +565,6 @@ static inline void rvt_qp_wqe_unreserve(
extern const enum ib_wc_opcode ib_rvt_wc_opcode[];
-/**
- * rvt_qp_swqe_complete() - insert send completion
- * @qp - the qp
- * @wqe - the send wqe
- * @status - completion status
- *
- * Insert a send completion into the completion
- * queue if the qp indicates it should be done.
- *
- * See IBTA 10.7.3.1 for info on completion
- * control.
- */
-static inline void rvt_qp_swqe_complete(
- struct rvt_qp *qp,
- struct rvt_swqe *wqe,
- enum ib_wc_opcode opcode,
- enum ib_wc_status status)
-{
- if (unlikely(wqe->wr.send_flags & RVT_SEND_RESERVE_USED))
- return;
- if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
- (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
- status != IB_WC_SUCCESS) {
- struct ib_wc wc;
-
- memset(&wc, 0, sizeof(wc));
- wc.wr_id = wqe->wr.wr_id;
- wc.status = status;
- wc.opcode = opcode;
- wc.qp = &qp->ibqp;
- wc.byte_len = wqe->length;
- rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc,
- status != IB_WC_SUCCESS);
- }
-}
-
/*
* Compare the lower 24 bits of the msn values.
* Returns an integer <, ==, or > than zero.
@@ -737,6 +701,78 @@ static inline void rvt_put_qp_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
}
+/**
+ * rvt_qp_sqwe_incr - increment ring index
+ * @qp: the qp
+ * @val: the starting value
+ *
+ * Return: the new value wrapping as appropriate
+ */
+static inline u32
+rvt_qp_swqe_incr(struct rvt_qp *qp, u32 val)
+{
+ if (++val >= qp->s_size)
+ val = 0;
+ return val;
+}
+
+/**
+ * rvt_qp_complete_swqe - insert send completion
+ * @qp - the qp
+ * @wqe - the send wqe
+ * @opcode - wc operation (driver dependent)
+ * @status - completion status
+ *
+ * Update the s_last information, and then insert a send
+ * completion into the completion
+ * queue if the qp indicates it should be done.
+ *
+ * See IBTA 10.7.3.1 for info on completion
+ * control.
+ *
+ * Return: new last
+ */
+static inline u32
+rvt_qp_complete_swqe(struct rvt_qp *qp,
+ struct rvt_swqe *wqe,
+ enum ib_wc_opcode opcode,
+ enum ib_wc_status status)
+{
+ bool need_completion;
+ u64 wr_id;
+ u32 byte_len, last;
+ int flags = wqe->wr.send_flags;
+
+ rvt_put_qp_swqe(qp, wqe);
+
+ need_completion =
+ !(flags & RVT_SEND_RESERVE_USED) &&
+ (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
+ (flags & IB_SEND_SIGNALED) ||
+ status != IB_WC_SUCCESS);
+ if (need_completion) {
+ wr_id = wqe->wr.wr_id;
+ byte_len = wqe->length;
+ /* above fields required before writing s_last */
+ }
+ last = rvt_qp_swqe_incr(qp, qp->s_last);
+ /* see rvt_qp_is_avail() */
+ smp_store_release(&qp->s_last, last);
+ if (need_completion) {
+ struct ib_wc w = {
+ .wr_id = wr_id,
+ .status = status,
+ .opcode = opcode,
+ .qp = &qp->ibqp,
+ .byte_len = byte_len,
+ };
+
+ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &w,
+ status != IB_WC_SUCCESS);
+ }
+ return last;
+}
+
extern const int ib_rvt_state_ops[];
struct rvt_dev_info;
diff --git a/include/rdma/rw.h b/include/rdma/rw.h
index 494f79ca3e62..6ad9dc836c10 100644
--- a/include/rdma/rw.h
+++ b/include/rdma/rw.h
@@ -39,15 +39,6 @@ struct rdma_rw_ctx {
struct ib_send_wr inv_wr;
struct ib_mr *mr;
} *reg;
-
- struct {
- struct rdma_rw_reg_ctx data;
- struct rdma_rw_reg_ctx prot;
- struct ib_send_wr sig_inv_wr;
- struct ib_mr *sig_mr;
- struct ib_sge sig_sge;
- struct ib_sig_handover_wr sig_wr;
- } *sig;
};
};
diff --git a/include/rdma/signature.h b/include/rdma/signature.h
new file mode 100644
index 000000000000..f24cc2a1d3c5
--- /dev/null
+++ b/include/rdma/signature.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) */
+/*
+ * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
+ */
+
+#ifndef _RDMA_SIGNATURE_H_
+#define _RDMA_SIGNATURE_H_
+
+enum ib_signature_prot_cap {
+ IB_PROT_T10DIF_TYPE_1 = 1,
+ IB_PROT_T10DIF_TYPE_2 = 1 << 1,
+ IB_PROT_T10DIF_TYPE_3 = 1 << 2,
+};
+
+enum ib_signature_guard_cap {
+ IB_GUARD_T10DIF_CRC = 1,
+ IB_GUARD_T10DIF_CSUM = 1 << 1,
+};
+
+/**
+ * enum ib_signature_type - Signature types
+ * @IB_SIG_TYPE_NONE: Unprotected.
+ * @IB_SIG_TYPE_T10_DIF: Type T10-DIF
+ */
+enum ib_signature_type {
+ IB_SIG_TYPE_NONE,
+ IB_SIG_TYPE_T10_DIF,
+};
+
+/**
+ * enum ib_t10_dif_bg_type - Signature T10-DIF block-guard types
+ * @IB_T10DIF_CRC: Corresponds to T10-PI mandated CRC checksum rules.
+ * @IB_T10DIF_CSUM: Corresponds to IP checksum rules.
+ */
+enum ib_t10_dif_bg_type {
+ IB_T10DIF_CRC,
+ IB_T10DIF_CSUM,
+};
+
+/**
+ * struct ib_t10_dif_domain - Parameters specific for T10-DIF
+ * domain.
+ * @bg_type: T10-DIF block guard type (CRC|CSUM)
+ * @pi_interval: protection information interval.
+ * @bg: seed of guard computation.
+ * @app_tag: application tag of guard block
+ * @ref_tag: initial guard block reference tag.
+ * @ref_remap: Indicate wethear the reftag increments each block
+ * @app_escape: Indicate to skip block check if apptag=0xffff
+ * @ref_escape: Indicate to skip block check if reftag=0xffffffff
+ * @apptag_check_mask: check bitmask of application tag.
+ */
+struct ib_t10_dif_domain {
+ enum ib_t10_dif_bg_type bg_type;
+ u16 pi_interval;
+ u16 bg;
+ u16 app_tag;
+ u32 ref_tag;
+ bool ref_remap;
+ bool app_escape;
+ bool ref_escape;
+ u16 apptag_check_mask;
+};
+
+/**
+ * struct ib_sig_domain - Parameters for signature domain
+ * @sig_type: specific signauture type
+ * @sig: union of all signature domain attributes that may
+ * be used to set domain layout.
+ */
+struct ib_sig_domain {
+ enum ib_signature_type sig_type;
+ union {
+ struct ib_t10_dif_domain dif;
+ } sig;
+};
+
+/**
+ * struct ib_sig_attrs - Parameters for signature handover operation
+ * @check_mask: bitmask for signature byte check (8 bytes)
+ * @mem: memory domain layout descriptor.
+ * @wire: wire domain layout descriptor.
+ * @meta_length: metadata length
+ */
+struct ib_sig_attrs {
+ u8 check_mask;
+ struct ib_sig_domain mem;
+ struct ib_sig_domain wire;
+ int meta_length;
+};
+
+enum ib_sig_err_type {
+ IB_SIG_BAD_GUARD,
+ IB_SIG_BAD_REFTAG,
+ IB_SIG_BAD_APPTAG,
+};
+
+/*
+ * Signature check masks (8 bytes in total) according to the T10-PI standard:
+ * -------- -------- ------------
+ * | GUARD | APPTAG | REFTAG |
+ * | 2B | 2B | 4B |
+ * -------- -------- ------------
+ */
+enum {
+ IB_SIG_CHECK_GUARD = 0xc0,
+ IB_SIG_CHECK_APPTAG = 0x30,
+ IB_SIG_CHECK_REFTAG = 0x0f,
+};
+
+/*
+ * struct ib_sig_err - signature error descriptor
+ */
+struct ib_sig_err {
+ enum ib_sig_err_type err_type;
+ u32 expected;
+ u32 actual;
+ u64 sig_err_offset;
+ u32 key;
+};
+
+#endif /* _RDMA_SIGNATURE_H_ */
diff --git a/include/uapi/rdma/ib_user_cm.h b/include/uapi/rdma/ib_user_cm.h
deleted file mode 100644
index e2709bb8cb18..000000000000
--- a/include/uapi/rdma/ib_user_cm.h
+++ /dev/null
@@ -1,326 +0,0 @@
-/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
-/*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef IB_USER_CM_H
-#define IB_USER_CM_H
-
-#include <linux/types.h>
-#include <rdma/ib_user_sa.h>
-
-#define IB_USER_CM_ABI_VERSION 5
-
-enum {
- IB_USER_CM_CMD_CREATE_ID,
- IB_USER_CM_CMD_DESTROY_ID,
- IB_USER_CM_CMD_ATTR_ID,
-
- IB_USER_CM_CMD_LISTEN,
- IB_USER_CM_CMD_NOTIFY,
-
- IB_USER_CM_CMD_SEND_REQ,
- IB_USER_CM_CMD_SEND_REP,
- IB_USER_CM_CMD_SEND_RTU,
- IB_USER_CM_CMD_SEND_DREQ,
- IB_USER_CM_CMD_SEND_DREP,
- IB_USER_CM_CMD_SEND_REJ,
- IB_USER_CM_CMD_SEND_MRA,
- IB_USER_CM_CMD_SEND_LAP,
- IB_USER_CM_CMD_SEND_APR,
- IB_USER_CM_CMD_SEND_SIDR_REQ,
- IB_USER_CM_CMD_SEND_SIDR_REP,
-
- IB_USER_CM_CMD_EVENT,
- IB_USER_CM_CMD_INIT_QP_ATTR,
-};
-/*
- * command ABI structures.
- */
-struct ib_ucm_cmd_hdr {
- __u32 cmd;
- __u16 in;
- __u16 out;
-};
-
-struct ib_ucm_create_id {
- __aligned_u64 uid;
- __aligned_u64 response;
-};
-
-struct ib_ucm_create_id_resp {
- __u32 id;
-};
-
-struct ib_ucm_destroy_id {
- __aligned_u64 response;
- __u32 id;
- __u32 reserved;
-};
-
-struct ib_ucm_destroy_id_resp {
- __u32 events_reported;
-};
-
-struct ib_ucm_attr_id {
- __aligned_u64 response;
- __u32 id;
- __u32 reserved;
-};
-
-struct ib_ucm_attr_id_resp {
- __be64 service_id;
- __be64 service_mask;
- __be32 local_id;
- __be32 remote_id;
-};
-
-struct ib_ucm_init_qp_attr {
- __aligned_u64 response;
- __u32 id;
- __u32 qp_state;
-};
-
-struct ib_ucm_listen {
- __be64 service_id;
- __be64 service_mask;
- __u32 id;
- __u32 reserved;
-};
-
-struct ib_ucm_notify {
- __u32 id;
- __u32 event;
-};
-
-struct ib_ucm_private_data {
- __aligned_u64 data;
- __u32 id;
- __u8 len;
- __u8 reserved[3];
-};
-
-struct ib_ucm_req {
- __u32 id;
- __u32 qpn;
- __u32 qp_type;
- __u32 psn;
- __be64 sid;
- __aligned_u64 data;
- __aligned_u64 primary_path;
- __aligned_u64 alternate_path;
- __u8 len;
- __u8 peer_to_peer;
- __u8 responder_resources;
- __u8 initiator_depth;
- __u8 remote_cm_response_timeout;
- __u8 flow_control;
- __u8 local_cm_response_timeout;
- __u8 retry_count;
- __u8 rnr_retry_count;
- __u8 max_cm_retries;
- __u8 srq;
- __u8 reserved[5];
-};
-
-struct ib_ucm_rep {
- __aligned_u64 uid;
- __aligned_u64 data;
- __u32 id;
- __u32 qpn;
- __u32 psn;
- __u8 len;
- __u8 responder_resources;
- __u8 initiator_depth;
- __u8 target_ack_delay;
- __u8 failover_accepted;
- __u8 flow_control;
- __u8 rnr_retry_count;
- __u8 srq;
- __u8 reserved[4];
-};
-
-struct ib_ucm_info {
- __u32 id;
- __u32 status;
- __aligned_u64 info;
- __aligned_u64 data;
- __u8 info_len;
- __u8 data_len;
- __u8 reserved[6];
-};
-
-struct ib_ucm_mra {
- __aligned_u64 data;
- __u32 id;
- __u8 len;
- __u8 timeout;
- __u8 reserved[2];
-};
-
-struct ib_ucm_lap {
- __aligned_u64 path;
- __aligned_u64 data;
- __u32 id;
- __u8 len;
- __u8 reserved[3];
-};
-
-struct ib_ucm_sidr_req {
- __u32 id;
- __u32 timeout;
- __be64 sid;
- __aligned_u64 data;
- __aligned_u64 path;
- __u16 reserved_pkey;
- __u8 len;
- __u8 max_cm_retries;
- __u8 reserved[4];
-};
-
-struct ib_ucm_sidr_rep {
- __u32 id;
- __u32 qpn;
- __u32 qkey;
- __u32 status;
- __aligned_u64 info;
- __aligned_u64 data;
- __u8 info_len;
- __u8 data_len;
- __u8 reserved[6];
-};
-/*
- * event notification ABI structures.
- */
-struct ib_ucm_event_get {
- __aligned_u64 response;
- __aligned_u64 data;
- __aligned_u64 info;
- __u8 data_len;
- __u8 info_len;
- __u8 reserved[6];
-};
-
-struct ib_ucm_req_event_resp {
- struct ib_user_path_rec primary_path;
- struct ib_user_path_rec alternate_path;
- __be64 remote_ca_guid;
- __u32 remote_qkey;
- __u32 remote_qpn;
- __u32 qp_type;
- __u32 starting_psn;
- __u8 responder_resources;
- __u8 initiator_depth;
- __u8 local_cm_response_timeout;
- __u8 flow_control;
- __u8 remote_cm_response_timeout;
- __u8 retry_count;
- __u8 rnr_retry_count;
- __u8 srq;
- __u8 port;
- __u8 reserved[7];
-};
-
-struct ib_ucm_rep_event_resp {
- __be64 remote_ca_guid;
- __u32 remote_qkey;
- __u32 remote_qpn;
- __u32 starting_psn;
- __u8 responder_resources;
- __u8 initiator_depth;
- __u8 target_ack_delay;
- __u8 failover_accepted;
- __u8 flow_control;
- __u8 rnr_retry_count;
- __u8 srq;
- __u8 reserved[5];
-};
-
-struct ib_ucm_rej_event_resp {
- __u32 reason;
- /* ari in ib_ucm_event_get info field. */
-};
-
-struct ib_ucm_mra_event_resp {
- __u8 timeout;
- __u8 reserved[3];
-};
-
-struct ib_ucm_lap_event_resp {
- struct ib_user_path_rec path;
-};
-
-struct ib_ucm_apr_event_resp {
- __u32 status;
- /* apr info in ib_ucm_event_get info field. */
-};
-
-struct ib_ucm_sidr_req_event_resp {
- __u16 pkey;
- __u8 port;
- __u8 reserved;
-};
-
-struct ib_ucm_sidr_rep_event_resp {
- __u32 status;
- __u32 qkey;
- __u32 qpn;
- /* info in ib_ucm_event_get info field. */
-};
-
-#define IB_UCM_PRES_DATA 0x01
-#define IB_UCM_PRES_INFO 0x02
-#define IB_UCM_PRES_PRIMARY 0x04
-#define IB_UCM_PRES_ALTERNATE 0x08
-
-struct ib_ucm_event_resp {
- __aligned_u64 uid;
- __u32 id;
- __u32 event;
- __u32 present;
- __u32 reserved;
- union {
- struct ib_ucm_req_event_resp req_resp;
- struct ib_ucm_rep_event_resp rep_resp;
- struct ib_ucm_rej_event_resp rej_resp;
- struct ib_ucm_mra_event_resp mra_resp;
- struct ib_ucm_lap_event_resp lap_resp;
- struct ib_ucm_apr_event_resp apr_resp;
-
- struct ib_ucm_sidr_req_event_resp sidr_req_resp;
- struct ib_ucm_sidr_rep_event_resp sidr_rep_resp;
-
- __u32 send_status;
- } u;
-};
-
-#endif /* IB_USER_CM_H */
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 41db51367efa..650cee8c4bf1 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -147,6 +147,18 @@ enum {
IWPM_NLA_HELLO_MAX
};
+/* For RDMA_NLDEV_ATTR_DEV_NODE_TYPE */
+enum {
+ /* IB values map to NodeInfo:NodeType. */
+ RDMA_NODE_IB_CA = 1,
+ RDMA_NODE_IB_SWITCH,
+ RDMA_NODE_IB_ROUTER,
+ RDMA_NODE_RNIC,
+ RDMA_NODE_USNIC,
+ RDMA_NODE_USNIC_UDP,
+ RDMA_NODE_UNSPECIFIED,
+};
+
/*
* Local service operations:
* RESOLVE - The client requests the local service to resolve a path.
@@ -267,11 +279,9 @@ enum rdma_nldev_command {
RDMA_NLDEV_CMD_RES_PD_GET, /* can dump */
- RDMA_NLDEV_NUM_OPS
-};
+ RDMA_NLDEV_CMD_GET_CHARDEV,
-enum {
- RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
+ RDMA_NLDEV_NUM_OPS
};
enum rdma_nldev_print_type {
@@ -480,6 +490,19 @@ enum rdma_nldev_attr {
RDMA_NLDEV_NET_NS_FD, /* u32 */
/*
+ * Information about a chardev.
+ * CHARDEV_TYPE is the name of the chardev ABI (ie uverbs, umad, etc)
+ * CHARDEV_ABI signals the ABI revision (historical)
+ * CHARDEV_NAME is the kernel name for the /dev/ file (no directory)
+ * CHARDEV is the 64 bit dev_t for the inode
+ */
+ RDMA_NLDEV_ATTR_CHARDEV_TYPE, /* string */
+ RDMA_NLDEV_ATTR_CHARDEV_NAME, /* string */
+ RDMA_NLDEV_ATTR_CHARDEV_ABI, /* u64 */
+ RDMA_NLDEV_ATTR_CHARDEV, /* u64 */
+ RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID, /* u64 */
+
+ /*
* Always the end
*/
RDMA_NLDEV_ATTR_MAX
diff --git a/net/core/devlink.c b/net/core/devlink.c
index 132f4b757963..5f0ca1216c70 100644
--- a/net/core/devlink.c
+++ b/net/core/devlink.c
@@ -1548,7 +1548,8 @@ static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
u32 seq, int flags)
{
const struct devlink_ops *ops = devlink->ops;
- u8 inline_mode, encap_mode;
+ enum devlink_eswitch_encap_mode encap_mode;
+ u8 inline_mode;
void *hdr;
int err = 0;
u16 mode;
@@ -1624,7 +1625,8 @@ static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
{
struct devlink *devlink = info->user_ptr[0];
const struct devlink_ops *ops = devlink->ops;
- u8 inline_mode, encap_mode;
+ enum devlink_eswitch_encap_mode encap_mode;
+ u8 inline_mode;
int err = 0;
u16 mode;
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 66c6eb56072b..5a42ebb892cd 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -611,11 +611,11 @@ send_hdrs_dma_out:
qp_out:
rdma_destroy_qp(ic->i_cm_id);
recv_cq_out:
- if (!ib_destroy_cq(ic->i_recv_cq))
- ic->i_recv_cq = NULL;
+ ib_destroy_cq(ic->i_recv_cq);
+ ic->i_recv_cq = NULL;
send_cq_out:
- if (!ib_destroy_cq(ic->i_send_cq))
- ic->i_send_cq = NULL;
+ ib_destroy_cq(ic->i_send_cq);
+ ic->i_send_cq = NULL;
rds_ibdev_out:
rds_ib_remove_conn(rds_ibdev, conn);
out: