Program Listing for File IBVUtils.h

↰ Return to documentation for file (/home/docs/checkouts/readthedocs.org/user_builds/ska-telescope-ska-pst-recv/checkouts/latest/src/ska/pst/recv/network/IBVUtils.h)

/*
 * Copyright 2022 Square Kilometre Array Observatory
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <memory>
#include <vector>
#include <cstdint>

#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>

#ifndef SKA_PST_RECV_NETWORK_IBVUtils_h
#define SKA_PST_RECV_NETWORK_IBVUtils_h

namespace ska::pst::recv {

  typedef struct ibv_slot
  {
    ibv_recv_wr wr;

    ibv_sge sge;

  } ibv_slot_t;

  class RDMAEventChannel
  {
    public:
      RDMAEventChannel();

      ~RDMAEventChannel();

      rdma_event_channel * get() const { return ec; };

    private:

      rdma_event_channel * ec;
  };

  class RDMACommunicationManager
  {
    public:

      RDMACommunicationManager() = default;

      explicit RDMACommunicationManager(RDMAEventChannel * ec);

      ~RDMACommunicationManager();

      void bind(const std::string &ip_addr);

      rdma_cm_id * get() const { return cm_id; };

    private:

      rdma_cm_id * cm_id{nullptr};
  };

  class IBVCompletionQueue
  {
    public:

      IBVCompletionQueue() = default;

      IBVCompletionQueue(RDMACommunicationManager * cm, int n_slots);

      ~IBVCompletionQueue();

      int poll(int num_entries, ibv_wc *wc);

      ibv_cq * get() const { return cq; };

    private:

      ibv_cq * cq;
  };

  class IBVProtectionDomain
  {
    public:

      IBVProtectionDomain() = default;

      explicit IBVProtectionDomain(RDMACommunicationManager * cm);

      ~IBVProtectionDomain();

      ibv_pd * get() const { return pd; };

    private:

      ibv_pd * pd;
  };

  class IBVQueuePair
  {
    public:

      IBVQueuePair() = default;

      IBVQueuePair(RDMACommunicationManager * cm, IBVProtectionDomain * pd, IBVCompletionQueue * cq, int n_slots);

      ~IBVQueuePair();

      ibv_qp * get() const { return qp; };

      void post_recv(ibv_recv_wr *wr);

      void modify(ibv_qp_state qp_state);

    private:

      void modify(ibv_qp_state qp_state, int port_num);

      void modify(ibv_qp_attr *attr, int attr_mask);

      ibv_qp * qp{nullptr};
  };

  class IBVMemoryRegion
  {
    public:

      IBVMemoryRegion() = default;

      IBVMemoryRegion(IBVProtectionDomain * pd, void *addr, std::size_t length);

      ~IBVMemoryRegion();

      ibv_mr * get() const { return mr; };

    private:

      ibv_mr * mr{nullptr};
  };

  class IBVFlow
  {
    static constexpr uint8_t dst_mac_mask = 0xFF;

    static constexpr uint8_t dst_ip_mask = 0xFF;

    static constexpr uint16_t dst_port_mask = 0xFFFF;

    public:
      IBVFlow() = default;

      IBVFlow(RDMACommunicationManager * cm, IBVQueuePair * qp, const std::string &ip, int port);

      ~IBVFlow();

    private:

      ibv_flow * flow{nullptr};
  };

} // namespace ska::pst::recv

#endif // SKA_PST_RECV_NETWORK_IBVUtils_h