From 1525407bd408662ebd85f60768ccd2c17121a5ae Mon Sep 17 00:00:00 2001 From: Saikrishna Edupuganti Date: Fri, 5 Jun 2020 09:06:08 -0700 Subject: [PATCH] Update multi-net components to latest releases (#329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update multi-net components to latest releases Multus CNI 3.4.2 SR-IOV CNI 2.3 SR-IOV DP 3.2 Tested as per the README. Works fine. Signed-off-by: Saikrishna Edupuganti * Provide dpdk-stable 3 LTS and 1 recent release Testpmd manifests of the last 3 LTS and 1 latest release from stable repo to help test actual DPDK app instead of sleep. Currently 17.11 and 19.11 are the only functioning ones without privileged. ``` NAME READY STATUS RESTARTS AGE dpdk-1711 1/1 Running 0 3m5s dpdk-1811 0/1 Error 0 3m5s dpdk-1911 1/1 Running 0 3m5s dpdk-2002 0/1 Error 0 3m5s ``` ``` EAL: PCI device 0000:07:06.4 on NUMA socket 0 EAL: probe driver: 8086:154c net_i40e_vf EAL: Getting a vfio_dev_fd for 0000:07:06.4 failed EAL: Requested device 0000:07:06.4 cannot be used … testpmd: No probed ethernet devices EAL: Error - exiting with code: 1 Cause: Invalid port 1 ``` Signed-off-by: Saikrishna Edupuganti --- clr-k8s-examples/9-multi-network/Dockerfile | 38 ++++--- clr-k8s-examples/9-multi-network/README.md | 9 +- .../9-multi-network/multus-sriov-ds.yaml | 98 ++++++++++------- .../9-multi-network/sriov-conf.yaml | 12 +-- .../test/sriov/2-pod-dpdk-ver.yaml | 100 ++++++++++++++++++ .../9-multi-network/test/sriov/Dockerfile | 17 +++ 6 files changed, 210 insertions(+), 64 deletions(-) create mode 100644 clr-k8s-examples/9-multi-network/test/sriov/2-pod-dpdk-ver.yaml create mode 100644 clr-k8s-examples/9-multi-network/test/sriov/Dockerfile diff --git a/clr-k8s-examples/9-multi-network/Dockerfile b/clr-k8s-examples/9-multi-network/Dockerfile index 6cc7a3c..fb23800 100644 --- a/clr-k8s-examples/9-multi-network/Dockerfile +++ b/clr-k8s-examples/9-multi-network/Dockerfile @@ -1,20 +1,28 @@ # Build multus plugin -FROM golang:1.10 AS multus -RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni -WORKDIR /go/src/github.com/intel/multus-cni -RUN ./build +FROM busybox AS multus +ARG MULTUS_VER=3.4.2 +RUN wget -O multus.tgz https://github.com/intel/multus-cni/releases/download/v${MULTUS_VER}/multus-cni_${MULTUS_VER}_linux_amd64.tar.gz +RUN tar xvzf multus.tgz --strip-components=1 -C /bin # Build sriov plugin -FROM golang:1.10 AS sriov-cni -RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni -WORKDIR /go/src/github.com/intel-corp/sriov-cni -RUN ./build +FROM golang AS sriov-cni +ARG SRIOV_CNI_VER=2.3 +RUN wget -qO sriov-cni.tgz https://github.com/intel/sriov-cni/archive/v${SRIOV_CNI_VER}.tar.gz +RUN mkdir -p sriov-cni && \ + tar xzf sriov-cni.tgz --strip-components=1 -C sriov-cni && \ + cd sriov-cni && \ + make && \ + cp build/sriov /bin # Build sriov device plugin -FROM golang:1.10 AS sriov-dp -RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin -WORKDIR /go/src/github.com/intel/sriov-network-device-plugin -RUN make +FROM golang AS sriov-dp +ARG SRIOV_DP_VER=3.2 +RUN wget -qO sriov-dp.tgz https://github.com/intel/sriov-network-device-plugin/archive/v${SRIOV_DP_VER}.tar.gz +RUN mkdir -p sriov-dp && \ + tar xzf sriov-dp.tgz --strip-components=1 -C sriov-dp && \ + cd sriov-dp && \ + make && \ + cp build/sriovdp /bin # Build vfioveth plugin FROM busybox as vfioveth @@ -25,9 +33,9 @@ RUN chmod +x /bin/vfioveth /bin/jq # Final image FROM centos/systemd WORKDIR /tmp/cni/bin -COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus . -COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov . +COPY --from=multus /bin/multus-cni . +COPY --from=sriov-cni /bin/sriov . COPY --from=vfioveth /bin/vfioveth . COPY --from=vfioveth /bin/jq . WORKDIR /usr/bin -COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp . +COPY --from=sriov-dp /bin/sriovdp . diff --git a/clr-k8s-examples/9-multi-network/README.md b/clr-k8s-examples/9-multi-network/README.md index be1bc4e..56d381f 100644 --- a/clr-k8s-examples/9-multi-network/README.md +++ b/clr-k8s-examples/9-multi-network/README.md @@ -9,11 +9,10 @@ directories on the host with the necessary binaries and configuration files. ### Customization -The device plugin will register the SR-IOV enabled devices on the host, specified as -`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service) -file is provided, which enables SR-IOV for the above `rootDevices` - -> NOTE: This assumes homogenous nodes in the cluster +The device plugin will register the SR-IOV enabled devices on the host, specified with +`selectors` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service) +file is provided, which enables SR-IOV for the above devices. More config options +are listed [here](https://github.com/intel/sriov-network-device-plugin#configurations). ### Pre-req (SR-IOV only) diff --git a/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml b/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml index 80a1b53..f829ee8 100644 --- a/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml +++ b/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml @@ -2,25 +2,34 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - # name must match the spec fields below, and be in the form: . name: network-attachment-definitions.k8s.cni.cncf.io spec: - # group name to use for REST API: /apis// group: k8s.cni.cncf.io - # version name to use for REST API: /apis// - version: v1 - # either Namespaced or Cluster scope: Namespaced names: - # plural name to be used in the URL: /apis/// plural: network-attachment-definitions - # singular name to be used as an alias on the CLI and for display singular: network-attachment-definition - # kind is normally the CamelCased singular type. Your resource manifests use this. kind: NetworkAttachmentDefinition - # shortNames allow shorter string to match your resource on the CLI shortNames: - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string --- apiVersion: v1 kind: ServiceAccount @@ -40,17 +49,43 @@ type: kubernetes.io/service-account-token kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: multus-pod-networks-lister + name: multus-pod-updater rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get"] -- apiGroups: [""] - resources: ["pods/status"] - verbs: ["update"] -- apiGroups: ["k8s.cni.cncf.io"] - resources: ["*"] - verbs: ["get"] + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus-pod-updater +subjects: +- kind: ServiceAccount + name: multus-sa + namespace: kube-system --- kind: ConfigMap apiVersion: v1 @@ -94,8 +129,9 @@ data: MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)" cat > /host/etc/cni/net.d/00-multus.conf <&1 +RUN cd ${DPDK_DIR} && \ + sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \ + sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \ + make config T=x86_64-native-linuxapp-gcc && \ + make -j $CPUS +ENV PATH="$PATH:$DPDK_DIR/build/app/"