mirror of
https://github.com/clearlinux/clear-linux-documentation.git
synced 2026-08-24 16:57:45 +00:00
Compare commits
58 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 22d818e12b | |||
| 4957e04bda | |||
| 6d549e1cd8 | |||
| e86d9b26ab | |||
| 66664ca9cc | |||
| 9c73b1523a | |||
| 27f3887799 | |||
| 4f60ec1063 | |||
| 9f768ea54f | |||
| 61a60f1899 | |||
| a732aab497 | |||
| 0ce1f36dc7 | |||
| 4964178e84 | |||
| 66972a46a8 | |||
| 6815543aa1 | |||
| 20fe2d74a8 | |||
| 6fa14d0e57 | |||
| 7f7d6b4a05 | |||
| f3e3653efe | |||
| abf2872eb9 | |||
| b98c4cdf75 | |||
| 6fe7e61bd8 | |||
| 160e230e7a | |||
| 70c59abca4 | |||
| 6a166177ea | |||
| c0fed8d533 | |||
| b54da75a80 | |||
| b5a5dfe6e0 | |||
| 1878b3738c | |||
| 3ea85a8d99 | |||
| 67b2d4d6d1 | |||
| e6ec4f30f7 | |||
| 96a3fad54d | |||
| 9e82a646ee | |||
| b44fa5c566 | |||
| f6b1c47834 | |||
| cc16a5efd4 | |||
| fe1aab8b23 | |||
| 66876a7db0 | |||
| baf79c8a22 | |||
| 5ca6316b6d | |||
| b188657398 | |||
| ec0b823a91 | |||
| f1411470c9 | |||
| 6f045e7b1d | |||
| 51eb12dfd5 | |||
| 1b9c891c9b | |||
| 9ce2c83816 | |||
| 7069fd70a1 | |||
| e1e313674a | |||
| d3b0b1cc6c | |||
| 830f35e59c | |||
| 62ce1a65b7 | |||
| 7e52a3e128 | |||
| 4f0b8c2c81 | |||
| 60d975b3fa | |||
| 0093ed4fc0 | |||
| 1b85a8c514 |
@@ -28,8 +28,9 @@ jobs:
|
|||||||
wget https://github.com/clearlinux/clear-linux-documentation-zh-CN/releases/latest/download/clearlinux-docs-zh-CN.tar.gz
|
wget https://github.com/clearlinux/clear-linux-documentation-zh-CN/releases/latest/download/clearlinux-docs-zh-CN.tar.gz
|
||||||
mkdir source/_build/html/zh_CN
|
mkdir source/_build/html/zh_CN
|
||||||
tar xvzf clearlinux-docs-zh-CN.tar.gz -C source/_build/html/zh_CN
|
tar xvzf clearlinux-docs-zh-CN.tar.gz -C source/_build/html/zh_CN
|
||||||
|
touch source/_build/html/.nojekyll
|
||||||
mv source/_build/html $HOME/output
|
mv source/_build/html $HOME/output
|
||||||
- name: Deploy the docs
|
- name: Deploy and publish docs
|
||||||
run: |
|
run: |
|
||||||
cd $HOME/output
|
cd $HOME/output
|
||||||
git init
|
git init
|
||||||
@@ -38,7 +39,3 @@ jobs:
|
|||||||
git add .
|
git add .
|
||||||
git commit -m "latest html output"
|
git commit -m "latest html output"
|
||||||
git push -f https://${GITHUB_ACTOR}:${{secrets.ACCESS_TOKEN}}@github.com/clearlinux/clear-linux-documentation.git HEAD:latestHTML
|
git push -f https://${GITHUB_ACTOR}:${{secrets.ACCESS_TOKEN}}@github.com/clearlinux/clear-linux-documentation.git HEAD:latestHTML
|
||||||
- name: Publish the docs
|
|
||||||
run: |
|
|
||||||
wget ${{secrets.PUBLISH_URL}}
|
|
||||||
cat clearlinux-latest
|
|
||||||
|
|||||||
@@ -5,38 +5,38 @@ SHELL := /bin/bash
|
|||||||
PY_VERSION ?= 3.6
|
PY_VERSION ?= 3.6
|
||||||
|
|
||||||
all:
|
all:
|
||||||
make -C source html
|
$(MAKE) -C source html
|
||||||
|
|
||||||
htmlall:
|
htmlall:
|
||||||
make -C source htmlall
|
$(MAKE) -C source htmlall
|
||||||
|
|
||||||
htmlzh:
|
htmlzh:
|
||||||
make -C source htmlzh
|
$(MAKE) -C source htmlzh
|
||||||
|
|
||||||
htmlde:
|
htmlde:
|
||||||
make -C source htmlde
|
$(MAKE) -C source htmlde
|
||||||
|
|
||||||
html:
|
html:
|
||||||
make -C source html
|
$(MAKE) -C source html
|
||||||
|
|
||||||
linkcheck:
|
linkcheck:
|
||||||
make -C source linkcheck
|
$(MAKE) -C source linkcheck
|
||||||
|
|
||||||
py:
|
py:
|
||||||
make -C source py
|
$(MAKE) -C source py
|
||||||
|
|
||||||
man:
|
man:
|
||||||
make -C source man
|
$(MAKE) -C source man
|
||||||
|
|
||||||
clean-man:
|
clean-man:
|
||||||
make -C source clean-man
|
$(MAKE) -C source clean-man
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "Please use \`make <target>' where <target> is one of"
|
@echo "Please use \`make <target>' where <target> is one of"
|
||||||
@echo " html to make standalone HTML files"
|
@echo " html to make standalone HTML files"
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
make -C source clean
|
$(MAKE) -C source clean
|
||||||
rm -rf venv
|
rm -rf venv
|
||||||
|
|
||||||
venv:
|
venv:
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
## DISCONTINUATION OF PROJECT.
|
||||||
|
|
||||||
|
This project will no longer be maintained by Intel.
|
||||||
|
|
||||||
|
Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project.
|
||||||
|
|
||||||
|
Contact: webadmin@linux.intel.com
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
# SOME DESCRIPTIVE TITLE.
|
|
||||||
# Copyright (C) 2019, many
|
|
||||||
# This file is distributed under the same license as the Clear Linux*
|
|
||||||
# Project Docs package.
|
|
||||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
|
|
||||||
#
|
|
||||||
msgid ""
|
|
||||||
msgstr "Project-Id-Version: Clear Linux* Project Docs latest\n"
|
|
||||||
"Report-Msgid-Bugs-To: \n"
|
|
||||||
"POT-Creation-Date: 2019-08-09 14:33-0700\n"
|
|
||||||
"PO-Revision-Date: 2019-09-04 16:21-0008\n"
|
|
||||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
|
||||||
"Language: zh-Hans\n"
|
|
||||||
"Language-Team: zh-Hans\n"
|
|
||||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
|
||||||
"MIME-Version: 1.0\n"
|
|
||||||
"Content-Type: text/plain; charset=utf-8\n"
|
|
||||||
"Content-Transfer-Encoding: 8bit\n"
|
|
||||||
"Generated-By: Intel® International Developer Studio Version 4.1.273.0\n"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:4
|
|
||||||
msgid "Data Analytics Reference Stack"
|
|
||||||
msgstr "数据分析参考堆栈"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:6
|
|
||||||
msgid ""
|
|
||||||
"This guide explains how to use the :abbr:`DARS (Data Analytics Reference "
|
|
||||||
"Stack)`, and to optionally build your own DARS container image."
|
|
||||||
msgstr "本指南说明了如何使用 :abbr:`DARS (Data Analytics Reference Stack)`,以及如何选择性地构建您自己的 DARS 容器映像。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:9
|
|
||||||
msgid ""
|
|
||||||
"Any system that supports Docker\\* containers can be used with DARS. This"
|
|
||||||
" steps in this guide use |CL-ATTR| as the host system."
|
|
||||||
msgstr "任何支持 Docker\\* 容器的系统都可与 DARS 一起使用。本指南中的这些步骤使用 |CL-ATTR| 作为主机系统。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:17
|
|
||||||
msgid "The Data Analytics Reference Stack release"
|
|
||||||
msgstr "数据分析参考堆栈版本"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:19
|
|
||||||
msgid ""
|
|
||||||
"The Data Analytics Reference Stack (DARS) provides developers and "
|
|
||||||
"enterprises a straightforward, highly optimized software stack for "
|
|
||||||
"storing and processing large amounts of data. More detail is available "
|
|
||||||
"on the `DARS architecture and performance benchmarks`_."
|
|
||||||
msgstr "数据分析参考堆栈 (DARS) 为开发人员和企业提供了一个简单、高度优化的软件堆栈来存储和处理大量数据。更多详细信息请参阅 `DARS architecture and performance benchmarks`_。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:23
|
|
||||||
msgid ""
|
|
||||||
"The Data Analytics Reference Stack provides two pre-built Docker images, "
|
|
||||||
"available on `Docker Hub`_:"
|
|
||||||
msgstr "数据分析参考堆栈提供了两个预构建的 Docker 映像,可在 `Docker Hub`_ 获得:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:26
|
|
||||||
msgid "A |CL|-derived `DARS with OpenBlas`_ stack optimized for `OpenBLAS`_"
|
|
||||||
msgstr "一个从 |CL| 派生且针对 `OpenBLAS`_ 优化的 `DARS with OpenBlas`_ 堆栈"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:27
|
|
||||||
msgid "A |CL|-derived `DARS with Intel® MKL`_ stack optimized for `MKL`_"
|
|
||||||
msgstr "一个从 |CL| 派生且针对 `MKL`_ 优化的 `DARS with MKL`_ 堆栈"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:29
|
|
||||||
msgid ""
|
|
||||||
"We recommend you view the latest component versions for each image in the"
|
|
||||||
" :file:`README` found in the `Data Analytics Reference Stack`_ GitHub\\* "
|
|
||||||
"repository. Because |CL| is a rolling distribution, the package version "
|
|
||||||
"numbers in the |CL|-based containers may not be the latest released by "
|
|
||||||
"|CL|."
|
|
||||||
msgstr "我们建议您在 `DARS repository`_ 中找到 :file:`README`,查看每个映像的最新组件版本。由于 |CL| 是滚动发行的,基于 |CL| 的容器中的软件包版本号可能不是 |CL| 最新发布的版本号。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:36
|
|
||||||
msgid ""
|
|
||||||
"The Data Analytics Reference Stack is a collective work, and each piece "
|
|
||||||
"of software within the work has its own license. Please see the `DARS "
|
|
||||||
"Terms of Use`_ for more details about licensing and usage of the Data "
|
|
||||||
"Analytics Reference Stack."
|
|
||||||
msgstr "数据分析参考堆栈是一项集体成果,成果中的每一个软件都有自己的许可证。有关数据分析参考堆栈的许可和使用的更多详细信息,请参阅 `DARS Terms of Use`_。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:42
|
|
||||||
msgid "Using the Docker images"
|
|
||||||
msgstr "使用 Docker 映像"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:44
|
|
||||||
msgid ""
|
|
||||||
"To immediately start using the latest stable DARS images, pull an image "
|
|
||||||
"directly from `Docker Hub`_. This example uses the `DARS with Intel® "
|
|
||||||
"MKL`_ Docker image."
|
|
||||||
msgstr "要立即开始使用最新的稳定版 DARS 映像,请直接从 `Docker Hub`_ 提取。在本教程中,我们将使用 `Dars with MKL`_ 版本堆栈。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:48
|
|
||||||
msgid "Once you have downloaded the image, you can run it with"
|
|
||||||
msgstr "下载完映像后,您可以使用以下命令运行它:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:54
|
|
||||||
msgid ""
|
|
||||||
"This will launch the image and drop you into a bash shell inside the "
|
|
||||||
"container. You will see output similar to the following:"
|
|
||||||
msgstr "此命令将启动映像,并进入容器内的 bash shell 中。您将看到类似以下内容的输出:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:75
|
|
||||||
msgid ""
|
|
||||||
"The :command:`--ulimit nofile` parameter is currently required in order "
|
|
||||||
"to increase the number of open files opened at certain point by the spark"
|
|
||||||
" engine."
|
|
||||||
msgstr ":command:`--ulimit nofile` 参数是当前必需的参数,以便增加 spark 引擎在某一时点打开的打开文件的数量。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:80
|
|
||||||
msgid "Building DARS images"
|
|
||||||
msgstr "构建 DARS 映像"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:82
|
|
||||||
msgid ""
|
|
||||||
"If you choose to build your own DARS container images, you can customize "
|
|
||||||
"them as needed. Use the provided Dockerfile as a baseline."
|
|
||||||
msgstr "如果选择构建您自己的 DARS 容器映像,您可以根据需要对它们进行自定义。将提供的 Dockerfile 用作基准。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:85
|
|
||||||
msgid ""
|
|
||||||
"To construct images with |CL|, start with a |CL| development platform "
|
|
||||||
"that has the :command:`containers-basic-dev` bundle installed. Learn more"
|
|
||||||
" about bundles and installing them by using :ref:`swupd-guide`."
|
|
||||||
msgstr "要使用 |CL| 构建映像,请从安装了 :command:`containers-basic-dev` 捆绑包的 |CL| 开发平台开始。使用 :ref:`swupd-guide` 了解有关捆绑包和安装捆绑包的更多信息。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:89
|
|
||||||
msgid "Clone the `Data Analytics Reference Stack`_ GitHub\\* repository."
|
|
||||||
msgstr "克隆 `Data Analytics Reference Stack`_ GitHub\\* 存储库。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:95
|
|
||||||
msgid ""
|
|
||||||
"Inside the DARS directory, run :command:`make` to build OpenBLAS and MKL "
|
|
||||||
"images."
|
|
||||||
msgstr "在 DARS 目录中,运行 :command:`make` 来构建 OpenBLAS 和 MKL 映像。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:101
|
|
||||||
msgid ""
|
|
||||||
"Run :command:`make baseline` to build the baseline CentOS image. "
|
|
||||||
"Depending on the system, it may take a while to finish building."
|
|
||||||
msgstr "然后运行 :command:`make baseline` 构建基准 CentOS 映像。根据系统的不同,可能需要一段时间才能完成构建。完成后,使用 :command:`Docker` 检查生成的映像。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:108
|
|
||||||
msgid "Once completed, check the resulting images with :command:`Docker`"
|
|
||||||
msgstr "完成后,使用 :command:`Docker` 检查生成的映像"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dars.rst:114
|
|
||||||
msgid ""
|
|
||||||
"You can use any of the resulting images to launch fully functional "
|
|
||||||
"containers. If you need to customize the containers, you can edit the "
|
|
||||||
"provided :file:`Dockerfile`."
|
|
||||||
msgstr "您可以使用任何一个生成的映像来启动功能齐全的容器。如果需要自定义容器,您可以编辑所提供的 :file:`Dockerfile`。"
|
|
||||||
|
|
||||||
#~ msgid ""
|
|
||||||
#~ "This tutorial shows you how to use"
|
|
||||||
#~ " the Data Analytics Reference Stack "
|
|
||||||
#~ "(DARS), and to optionally build your "
|
|
||||||
#~ "own images with the baseline Dockerfiles"
|
|
||||||
#~ " provided in the `DARS repository`_. "
|
|
||||||
#~ "Our assumption is that |CL-ATTR| "
|
|
||||||
#~ "is the host. However, any system "
|
|
||||||
#~ "that supports Docker\\* containers can "
|
|
||||||
#~ "be used to follow these steps."
|
|
||||||
#~ msgstr ""
|
|
||||||
#~ "本教程介绍如何使用数据分析参考堆栈 (DARS),以及如何使用 `DARS repository`_"
|
|
||||||
#~ " 中提供的基准 Dockerfiles 来选择构建您自己的映像。我们假设 |CL-"
|
|
||||||
#~ "ATTR| 是主机。但是,任何支持 Docker\\* 容器的系统都可以用来执行这些步骤。"
|
|
||||||
|
|
||||||
#~ msgid ""
|
|
||||||
#~ "If you choose to build your own"
|
|
||||||
#~ " DARS container images, you can "
|
|
||||||
#~ "customize them as needed. Use the "
|
|
||||||
#~ "provided Dockerfile as a baseline. To"
|
|
||||||
#~ " construct images with |CL|, start "
|
|
||||||
#~ "with a |CL| development platform that"
|
|
||||||
#~ " has the :command:`containers-basic-dev`"
|
|
||||||
#~ " bundle installed. Learn more about "
|
|
||||||
#~ "bundles and installing them by using "
|
|
||||||
#~ ":ref:`swupd-guide`."
|
|
||||||
#~ msgstr ""
|
|
||||||
#~ "如果选择构建您自己的 DARS 容器映像,您可以根据需要对它们进行自定义。将提供的 Dockerfile"
|
|
||||||
#~ " 用作基准。要使用 |CL| 构建映像,请从安装了 :command"
|
|
||||||
#~ ":`containers-basic-dev` 捆绑包的 |CL| 开发平台开始。使用"
|
|
||||||
#~ " :ref:`swupd-guide` 了解有关捆绑包和安装捆绑包的更多信息。"
|
|
||||||
|
|
||||||
#~ msgid "First, clone the `DARS repository`_ from GitHub."
|
|
||||||
#~ msgstr "首先,从 GitHub 中克隆 `DARS repository`_。"
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,656 +0,0 @@
|
|||||||
# SOME DESCRIPTIVE TITLE.
|
|
||||||
# Copyright (C) 2019, many
|
|
||||||
# This file is distributed under the same license as the Clear Linux*
|
|
||||||
# Project Docs package.
|
|
||||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
|
|
||||||
#
|
|
||||||
msgid ""
|
|
||||||
msgstr "Project-Id-Version: Clear Linux* Project Docs latest\n"
|
|
||||||
"Report-Msgid-Bugs-To: \n"
|
|
||||||
"POT-Creation-Date: 2019-08-09 14:33-0700\n"
|
|
||||||
"PO-Revision-Date: 2019-09-04 16:21-0008\n"
|
|
||||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
|
||||||
"Language: zh-Hans\n"
|
|
||||||
"Language-Team: zh-Hans\n"
|
|
||||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
|
||||||
"MIME-Version: 1.0\n"
|
|
||||||
"Content-Type: text/plain; charset=utf-8\n"
|
|
||||||
"Content-Transfer-Encoding: 8bit\n"
|
|
||||||
"Generated-By: Intel® International Developer Studio Version 4.1.273.0\n"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:4
|
|
||||||
msgid "Deep Learning Reference Stack"
|
|
||||||
msgstr "深度学习参考堆栈"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:6
|
|
||||||
msgid ""
|
|
||||||
"This guide describes how to run benchmarking workloads for TensorFlow\\*,"
|
|
||||||
" PyTorch\\*, and Kubeflow in |CL-ATTR| using the Deep Learning Reference "
|
|
||||||
"Stack."
|
|
||||||
msgstr "本教程介绍如何在 |CL-ATTR| 中使用深度学习参考堆栈运行 TensorFlow\\*、PyTorch\\* 和 Kubeflow 基准工作负载。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:14
|
|
||||||
msgid "Overview"
|
|
||||||
msgstr "概述"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:16
|
|
||||||
msgid ""
|
|
||||||
"We created the Deep Learning Reference Stack to help AI developers "
|
|
||||||
"deliver the best experience on Intel® Architecture. This stack reduces "
|
|
||||||
"complexity common with deep learning software components, provides "
|
|
||||||
"flexibility for customized solutions, and enables you to quickly "
|
|
||||||
"prototype and deploy Deep Learning workloads. Use this guide to run "
|
|
||||||
"benchmarking workloads on your solution."
|
|
||||||
msgstr "我们打造了深度学习参考堆栈来帮助 AI 开发人员在英特尔架构上获得最佳开发体验。此堆栈降低了深度学习软件组件常见的复杂性,为自定义解决方案提供了灵活性,并使您能够快速构建原型并部署深度学习工作负载。使用本教程可在您的解决方案上运行基准工作负载。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:23
|
|
||||||
msgid "The Deep Learning Reference Stack is available in the following versions:"
|
|
||||||
msgstr "深度学习参考堆栈有以下版本:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:25
|
|
||||||
msgid ""
|
|
||||||
"`Intel MKL-DNN-VNNI`_, which is optimized using Intel® Math Kernel "
|
|
||||||
"Library for Deep Neural Networks (Intel® MKL-DNN) primitives and "
|
|
||||||
"introduces support for Intel® AVX-512 Vector Neural Network Instructions "
|
|
||||||
"(VNNI)."
|
|
||||||
msgstr "`Intel MKL-DNN-VNNI`_,它使用面向深度神经网络(英特尔® MKL-DNN)原语的英特尔®数学内核库进行优化,并支持英特尔® AVX-512 矢量神经网络指令 (VNI)。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:28
|
|
||||||
msgid ""
|
|
||||||
"`Intel MKL-DNN`_, which includes the TensorFlow framework optimized using"
|
|
||||||
" Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) "
|
|
||||||
"primitives."
|
|
||||||
msgstr "`Intel MKL-DNN`_,它包括使用面向深度神经网络(英特尔® MKL-DNN)原语的英特尔®数学内核库进行优化的 TensorFlow 框架。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:31
|
|
||||||
msgid "`Eigen`_, which includes `TensorFlow`_ optimized for Intel® architecture."
|
|
||||||
msgstr "`Eigen`_,它包括针对英特尔®架构优化的 `TensorFlow`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:32
|
|
||||||
msgid "`PyTorch with OpenBLAS`_, which includes PyTorch with OpenBlas."
|
|
||||||
msgstr "`PyTorch with OpenBLAS`_,它包括 PyTorch with OpenBlas。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:33
|
|
||||||
msgid ""
|
|
||||||
"`PyTorch with Intel MKL-DNN`_, which includes PyTorch optimized using "
|
|
||||||
"Intel® Math Kernel Library (Intel® MKL) and Intel MKL-DNN."
|
|
||||||
msgstr "`PyTorch with Intel MKL-DNN`_,它包括使用英特尔®数学内核库(英特尔® MKL)和英特尔 MKL-DNN 进行优化的 PyTorch。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:38
|
|
||||||
msgid ""
|
|
||||||
"To take advantage of the Intel® AVX-512 and VNNI functionality with the "
|
|
||||||
"Deep Learning Reference Stack, you must use the following hardware:"
|
|
||||||
msgstr "要利用英特尔® AVX-512 和 VNI 功能以及深度学习参考堆栈,您必须使用以下硬件:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:41
|
|
||||||
msgid "Intel® AVX-512 images require an Intel® Xeon® Scalable Platform"
|
|
||||||
msgstr "英特尔® AVX-512 映像需要使用英特尔®至强®可扩展平台"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:42
|
|
||||||
msgid "VNNI requires a 2nd generation Intel® Xeon® Scalable Platform"
|
|
||||||
msgstr "VNNI 需要使用第二代英特尔®至强®可扩展平台"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:45
|
|
||||||
msgid "Stack features"
|
|
||||||
msgstr "堆栈功能和特性"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:47
|
|
||||||
msgid "`DLRS V3.0`_ release announcement."
|
|
||||||
msgstr "`DLRS V3.0`_ 发布公告。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:48
|
|
||||||
msgid "Deep Learning Reference Stack v2.0 including current `PyTorch benchmark`_."
|
|
||||||
msgstr "深度学习参考堆栈 v2.0,包括最新的 `PyTorch benchmark results`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:50
|
|
||||||
msgid ""
|
|
||||||
"Deep Learning Reference Stack v1.0 including current `TensorFlow "
|
|
||||||
"benchmark`_ results."
|
|
||||||
msgstr "深度学习参考堆栈 v1.0,包括最新的 `TensorFlow benchmark results`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:52
|
|
||||||
msgid ""
|
|
||||||
"`DLRS Release notes`_ on Github\\* for the latest release of Deep "
|
|
||||||
"Learning Reference Stack."
|
|
||||||
msgstr "`DLRS Release notes`_ on Github\\*,了解深度学习参考堆栈的最新版本。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:57
|
|
||||||
msgid ""
|
|
||||||
"The Deep Learning Reference Stack is a collective work, and each piece of"
|
|
||||||
" software within the work has its own license. Please see the `DLRS "
|
|
||||||
"Terms of Use`_ for more details about licensing and usage of the Deep "
|
|
||||||
"Learning Reference Stack."
|
|
||||||
msgstr "深度学习参考堆栈是一项集体成果,成果中的每一个软件都有自己的许可证。有关深度学习参考堆栈的许可和使用的更多详细信息,请参阅 `DLRS Terms of Use`_。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:62
|
|
||||||
msgid "Prerequisites"
|
|
||||||
msgstr "必备条件"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:64
|
|
||||||
msgid ":ref:`Install <bare-metal-install-desktop>` |CL| on your host system"
|
|
||||||
msgstr "在主机系统上 :ref:`Install <bare-metal-install-desktop>` |CL|"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:65
|
|
||||||
msgid ":command:`containers-basic` bundle"
|
|
||||||
msgstr ":command:`containers-basic` 捆绑包"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:66
|
|
||||||
msgid ":command:`cloud-native-basic` bundle"
|
|
||||||
msgstr ":command:`cloud-native-basic` 捆绑包"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:68
|
|
||||||
msgid ""
|
|
||||||
"In |CL|, :command:`containers-basic` includes Docker\\*, which is "
|
|
||||||
"required for TensorFlow and PyTorch benchmarking. Use the "
|
|
||||||
":command:`swupd` utility to check if :command:`containers-basic` and "
|
|
||||||
":command:`cloud-native-basic` are present:"
|
|
||||||
msgstr "在 |CL| 中,:command:`containers-basic` 包括 TensorFlow 和 PyTorch 基准测试所必需的 Docker\\*。使用 :command:`swupd` 实用程序检查 :command:`containers-basic` 和 :command:`cloud-native-basic` 是否存在:"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:77
|
|
||||||
msgid ""
|
|
||||||
"To install the :command:`containers-basic` or :command:`cloud-native-"
|
|
||||||
"basic` bundles, enter:"
|
|
||||||
msgstr "要安装 :command:`containers-basic` 或 :command:`cloud-native-basic` 捆绑包,请输入:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:84
|
|
||||||
msgid ""
|
|
||||||
"Docker is not started upon installation of the :command:`containers-"
|
|
||||||
"basic` bundle. To start Docker, enter:"
|
|
||||||
msgstr "安装 :command:`containers-basic` 捆绑包后 Docker 不会启动。要启动 Docker,请输入:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:91
|
|
||||||
msgid ""
|
|
||||||
"To ensure that Kubernetes is correctly installed and configured, follow "
|
|
||||||
"the instructions in :ref:`kubernetes`."
|
|
||||||
msgstr "要确保正确安装和配置 Kubernetes,请遵循 :ref:`kubernetes` 中的说明。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:95
|
|
||||||
msgid "Version compatibility"
|
|
||||||
msgstr "版本兼容性"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:97
|
|
||||||
msgid "We validated these steps against the following software package versions:"
|
|
||||||
msgstr "我们根据以下软件包版本验证了这些步骤:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:99
|
|
||||||
msgid "|CL| 26240 (Minimum supported version)"
|
|
||||||
msgstr "|CL| 26240(支持的最低版本)"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:100
|
|
||||||
msgid "Docker 18.06.1"
|
|
||||||
msgstr "Docker 18.06.1"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:101
|
|
||||||
msgid "Kubernetes 1.11.3"
|
|
||||||
msgstr "Kubernetes 1.11.3"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:102
|
|
||||||
msgid "Go 1.11.12"
|
|
||||||
msgstr "Go 1.11.12"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:107
|
|
||||||
msgid ""
|
|
||||||
"The Deep Learning Reference Stack was developed to provide the best user "
|
|
||||||
"experience when executed on a |CL| host. However, as the stack runs in a"
|
|
||||||
" container environment, you should be able to complete the following "
|
|
||||||
"sections of this guide on other Linux* distributions, provided they "
|
|
||||||
"comply with the Docker*, Kubernetes* and Go* package versions listed "
|
|
||||||
"above. Look for your distribution documentation on how to update packages"
|
|
||||||
" and manage Docker services."
|
|
||||||
msgstr "深度学习参考堆栈是为了在 |CL| 主机上执行时获得最佳用户体验而开发的。但是,该堆栈在容器环境中运行时,您应该能够在其他 Linux* 发行版上完成本教程的以下部分,只要这些发行版满足上面列出的 Docker*、Kubernetes* 和 Go* 软件包版本。查找关于如何更新软件包和管理 Docker 服务的分发版文档。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:112
|
|
||||||
msgid "TensorFlow single and multi-node benchmarks"
|
|
||||||
msgstr "TensorFlow 单节点和多节点基准测试"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:114
|
|
||||||
msgid ""
|
|
||||||
"This section describes running the `TensorFlow Benchmarks`_ in single "
|
|
||||||
"node. For multi-node testing, replicate these steps for each node. These "
|
|
||||||
"steps provide a template to run other benchmarks, provided that they can "
|
|
||||||
"invoke TensorFlow."
|
|
||||||
msgstr "本部分介绍在单节点中运行 `TensorFlow benchmarks`_。对于多节点测试,请为每个节点重复这些步骤。这些步骤提供了运行其他基准测试的模板,前提是它们可以调用 TensorFlow。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:121
|
|
||||||
msgid ""
|
|
||||||
"Performance test results for the Deep Learning Reference Stack and for "
|
|
||||||
"this guide were obtained using `runc` as the runtime."
|
|
||||||
msgstr "深度学习参考堆栈和本教程的性能测试结果是使用 `runc` 作为运行时获得的。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:124
|
|
||||||
msgid ""
|
|
||||||
"Download either the `Eigen`_ or the `Intel MKL-DNN`_ Docker image from "
|
|
||||||
"`Docker Hub`_."
|
|
||||||
msgstr "从 `Docker Hub`_ 下载 `Eigen`_ 或 `Intel MKL-DNN`_ Docker 映像。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:127 ../../guides/stacks/dlrs/dlrs.rst:169
|
|
||||||
msgid "Run the image with Docker:"
|
|
||||||
msgstr "使用 Docker 运行映像:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:136 ../../guides/stacks/dlrs/dlrs.rst:177
|
|
||||||
msgid ""
|
|
||||||
"Launching the Docker image with the :command:`-i` argument starts "
|
|
||||||
"interactive mode within the container. Enter the following commands in "
|
|
||||||
"the running container."
|
|
||||||
msgstr "使用 :command:`-i` 参数启动 Docker 映像,从而在容器内启动交互模式。在正在运行的容器中输入以下命令。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:140
|
|
||||||
msgid "Clone the benchmark repository in the container:"
|
|
||||||
msgstr "克隆容器中的基准测试存储库:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:146 ../../guides/stacks/dlrs/dlrs.rst:187
|
|
||||||
msgid "Execute the benchmark script:"
|
|
||||||
msgstr "执行基准测试脚本:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:154
|
|
||||||
msgid ""
|
|
||||||
"You can replace the model with one of your choice supported by the "
|
|
||||||
"TensorFlow benchmarks."
|
|
||||||
msgstr "您可以将该模型更换为 TensorFlow 支持的其他模型。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:157
|
|
||||||
msgid ""
|
|
||||||
"If you are using an FP32 based model, it can be converted to an int8 "
|
|
||||||
"model using `Intel® quantization tools`_."
|
|
||||||
msgstr "如果使用基于 FP32 的模型,可以使用 `Intel® quantization tools`_ 将其转换为 int8 模型。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:161
|
|
||||||
msgid "PyTorch single and multi-node benchmarks"
|
|
||||||
msgstr "PyTorch 单节点和多节点基准测试"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:163
|
|
||||||
msgid ""
|
|
||||||
"This section describes running the `PyTorch benchmarks`_ for Caffe2 in "
|
|
||||||
"single node."
|
|
||||||
msgstr "本部分介绍在单节点中运行针对 Caffe2 的 `PyTorch benchmarks`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:166
|
|
||||||
msgid ""
|
|
||||||
"Download either the `PyTorch with OpenBLAS`_ or the `PyTorch with Intel "
|
|
||||||
"MKL-DNN`_ Docker image from `Docker Hub`_."
|
|
||||||
msgstr "从 `Docker Hub`_ 下载 `PyTorch with OpenBLAS`_ 或 `PyTorch with Intel MKL-DNN`_ Docker 映像。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:181
|
|
||||||
msgid "Clone the benchmark repository:"
|
|
||||||
msgstr "克隆基准测试存储库:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:197
|
|
||||||
msgid "Kubeflow multi-node benchmarks"
|
|
||||||
msgstr "Kubeflow 多节点基准测试"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:199
|
|
||||||
msgid ""
|
|
||||||
"The benchmark workload runs in a Kubernetes cluster. The guide uses "
|
|
||||||
"`Kubeflow`_ for the Machine Learning workload deployment on three nodes."
|
|
||||||
msgstr "基准测试工作负载在 Kubernetes 集群中运行。本教程使用 `Kubeflow`_ 在三个节点上部署机器学习工作负载。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:204
|
|
||||||
msgid ""
|
|
||||||
"If you choose the Intel® MKL-DNN or Intel® MKL-DNN-VNNI image, your "
|
|
||||||
"platform must support the Intel® AVX-512 instruction set. Otherwise, an "
|
|
||||||
"*illegal instruction* error may appear, and you won’t be able to complete"
|
|
||||||
" this guide."
|
|
||||||
msgstr "如果选择英特尔® MKL-DNN 或英特尔® MKL-DNN-VNNI 映像,您的平台必须支持英特尔® AVX-512 指令集。否则,可能会出现非法指令错误,导致无法完成本教程。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:210
|
|
||||||
msgid "Kubernetes setup"
|
|
||||||
msgstr "Kubernetes 设置"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:212
|
|
||||||
msgid ""
|
|
||||||
"Follow the instructions in the :ref:`kubernetes` tutorial to get set up "
|
|
||||||
"on |CL|. The Kubernetes community also has instructions for creating a "
|
|
||||||
"cluster, described in `Creating a single control-plane cluster with "
|
|
||||||
"kubeadm`_."
|
|
||||||
msgstr "按照 :ref:`kubernetes` 教程中的说明在 |CL| 上进行设置。Kubernetes 社区也提供了 `Creating a single control-plane cluster with kubeadm`_。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:217
|
|
||||||
msgid "Kubernetes networking"
|
|
||||||
msgstr "Kubernetes 网络连接"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:219
|
|
||||||
msgid ""
|
|
||||||
"We used `flannel`_ as the network provider for these tests. If you prefer"
|
|
||||||
" a different network layer, refer to the Kubernetes network documentation"
|
|
||||||
" described in `Creating a single control-plane cluster with kubeadm`_ for"
|
|
||||||
" setup."
|
|
||||||
msgstr "在这些测试中,我们使用 `flannel`_ 作为网络提供程序。如果青睐不同的网络层,请参阅 Kubernetes `Creating a single control-plane cluster with kubeadm`_ 进行设置。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:224
|
|
||||||
msgid "Kubectl"
|
|
||||||
msgstr "Kubectl"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:226
|
|
||||||
msgid ""
|
|
||||||
"You can use kubectl to run commands against your Kubernetes cluster. "
|
|
||||||
"Refer to the `Overview of kubectl`_ for details on syntax and operations."
|
|
||||||
" Once you have a working cluster on Kubernetes, use the following YAML "
|
|
||||||
"script to start a pod with a simple shell script, and keep the pod open."
|
|
||||||
msgstr "您可以使用 kubectl 对您的 Kubernetes 集群运行命令。有关语法和操作的详细信息,请参阅 `Overview of kubectl`_。建立一个 Kubernetes 工作集群后,请使用下面的 YAML 脚本启动一个含有简单 shell 脚本的 Pod,并保持该 Pod 处于打开状态。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:231
|
|
||||||
msgid "Copy this example.yaml script to your system:"
|
|
||||||
msgstr "将 example.yaml 脚本复制到您的系统中:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:248
|
|
||||||
msgid "Execute the script with kubectl:"
|
|
||||||
msgstr "使用 kubectl 执行该脚本:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:254
|
|
||||||
msgid ""
|
|
||||||
"This script opens a single pod. More robust solutions would create a "
|
|
||||||
"deployment or inject a python script or larger shell script into the "
|
|
||||||
"container."
|
|
||||||
msgstr "该脚本打开一个 Pod。更稳健的解决方案是创建部署,或者将 python 脚本或更大的 shell 脚本注入容器。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:258
|
|
||||||
msgid "Images"
|
|
||||||
msgstr "图像"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:260
|
|
||||||
msgid ""
|
|
||||||
"You must add `launcher.py`_ to the Docker image to include the Deep "
|
|
||||||
"Learning Reference Stack and put the benchmarks repo in the correct "
|
|
||||||
"location. Note that this guide uses Kubeflow v0.4.0, and cannot guarantee"
|
|
||||||
" results if you use a different version."
|
|
||||||
msgstr "您必须将 `launcher.py`_ 添加到 Docker 映像中,以包含深度学习参考堆栈,并将基准测试存储库放在正确的位置。请注意,本教程使用 Kubeflow v0.4.0。如果使用不同的版本,则不能保证结果。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:264
|
|
||||||
msgid "From the Docker image, run the following:"
|
|
||||||
msgstr "从 Docker 映像中,运行以下命令:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:273
|
|
||||||
msgid "Your entry point becomes: :file:`/opt/launcher.py`."
|
|
||||||
msgstr "您的入口点变成 :file:`/opt/launcher.py`。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:275
|
|
||||||
msgid "This builds an image that can be consumed directly by TFJob from Kubeflow."
|
|
||||||
msgstr "这会构建一个可供 TFJob 从 Kubeflow 直接使用的映像。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:278
|
|
||||||
msgid "ksonnet\\*"
|
|
||||||
msgstr "ksonnet\\*"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:280
|
|
||||||
msgid ""
|
|
||||||
"Kubeflow uses ksonnet\\* to manage deployments, so you must install it "
|
|
||||||
"before setting up Kubeflow."
|
|
||||||
msgstr "Kubeflow 使用 ksonnet\\* 来管理部署,因此您必须在设置 Kubeflow 之前安装它。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:283
|
|
||||||
msgid ""
|
|
||||||
"ksonnet was added to the :command:`cloud-native-basic` bundle in |CL| "
|
|
||||||
"version 27550. If you are using an older |CL| version (not recommended), "
|
|
||||||
"you must manually install ksonnet as described below."
|
|
||||||
msgstr "ksonnet 已添加到 |CL| 版本 27550 中的 :command:`cloud-native-basic` 捆绑包中。如果使用的是较旧的 |CL| 版本(不推荐),您必须如下所述手动安装 ksonnet。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:287
|
|
||||||
msgid "On |CL|, follow these steps:"
|
|
||||||
msgstr "在 |CL| 上,请按照下列步骤操作:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:298
|
|
||||||
msgid ""
|
|
||||||
"After the ksonnet installation is complete, ensure that binary `ks` is "
|
|
||||||
"accessible across the environment."
|
|
||||||
msgstr "ksonnet 安装完成后,确保可在整个环境中访问 `ks` 二进制文件。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:302
|
|
||||||
msgid "Kubeflow"
|
|
||||||
msgstr "Kubeflow"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:304
|
|
||||||
msgid ""
|
|
||||||
"Once you have Kubernetes running on your nodes, set up `Kubeflow`_ by "
|
|
||||||
"following these instructions from the `Getting Started with Kubeflow`_ "
|
|
||||||
"guide."
|
|
||||||
msgstr "Kubernetes 在节点上运行后,请按照 `Getting Started with Kubeflow`_ 中的说明设置 `Kubeflow`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:322
|
|
||||||
msgid "Next, deploy the primary package for our purposes: tf-job-operator."
|
|
||||||
msgstr "接下来,为我们的目的部署主要软件包:tf-job-operator。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:332
|
|
||||||
msgid ""
|
|
||||||
"This creates the CustomResourceDefinition (CRD) endpoint to launch a "
|
|
||||||
"TFJob."
|
|
||||||
msgstr "这将创建 CustomResourceDefinition (CRD) 端点来启动 TFJob。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:335
|
|
||||||
msgid "Run a TFJob"
|
|
||||||
msgstr "运行 TFJob"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:337
|
|
||||||
msgid "Get the ksonnet registries for deploying TFJobs from `dlrs-tfjob`_."
|
|
||||||
msgstr "从 `dlrs-tfjob`_ 获取用于部署 TFJobs 的 ksonnet 注册表。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:339
|
|
||||||
msgid "Install the TFJob components as follows:"
|
|
||||||
msgstr "按照以下步骤安装 TFJob 组件:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:347
|
|
||||||
msgid "Export the image name to use for the deployment:"
|
|
||||||
msgstr "导出用于部署的映像名称:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:355
|
|
||||||
msgid "Replace <docker_name> with the image name you specified in previous steps."
|
|
||||||
msgstr "将 <docker_name> 替换为前述步骤中指定的映像名称。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:357
|
|
||||||
msgid ""
|
|
||||||
"Generate Kubernetes manifests for the workloads and apply them using "
|
|
||||||
"these commands:"
|
|
||||||
msgstr "为工作负载生成 Kubernetes 清单,并使用以下命令应用这些清单:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:367
|
|
||||||
msgid "This replicates and deploys three test setups in your Kubernetes cluster."
|
|
||||||
msgstr "这会在 Kubernetes 集群中复制和部署三个测试设置。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:370
|
|
||||||
msgid "Results of running this guide"
|
|
||||||
msgstr "运行本教程的结果"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:372
|
|
||||||
msgid ""
|
|
||||||
"You must parse the logs of the Kubernetes pod to retrieve performance "
|
|
||||||
"data. The pods will still exist post-completion and will be in "
|
|
||||||
"‘Completed’ state. You can get the logs from any of the pods to inspect "
|
|
||||||
"the benchmark results. More information about Kubernetes logging is "
|
|
||||||
"available in the Kubernetes `Logging Architecture`_ documentation."
|
|
||||||
msgstr "您必须解析 Kubernetes Pod 的日志来检索性能数据。完成后,Pod 仍会存在,并将处于“已完成”状态。您可以从任何一个 Pod 中获取日志来检查基准测试结果。有关 Kubernetes 日志记录的更多信息,请参见 Kubernetes `Logging Architecture`_ 文档。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:379
|
|
||||||
msgid "Use Jupyter Notebook"
|
|
||||||
msgstr "使用 Jupyter Notebook"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:381
|
|
||||||
msgid ""
|
|
||||||
"This example uses the `PyTorch with OpenBLAS`_ container image. After it "
|
|
||||||
"is downloaded, run the Docker image with :command:`-p` to specify the "
|
|
||||||
"shared port between the container and the host. This example uses port "
|
|
||||||
"8888."
|
|
||||||
msgstr "本示例使用 `PyTorch with OpenBLAS`_ 容器映像。下载后,使用 :command:`-p` 运行 Docker 映像,以指定容器和主机之间的共享端口。本示例使用端口 8888。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:389
|
|
||||||
msgid ""
|
|
||||||
"After you start the container, launch the Jupyter Notebook. This command "
|
|
||||||
"is executed inside the container image."
|
|
||||||
msgstr "启动容器后,启动 Jupyter Notebook。该命令在容器映像内执行。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:396
|
|
||||||
msgid ""
|
|
||||||
"After the notebook has loaded, you will see output similar to the "
|
|
||||||
"following:"
|
|
||||||
msgstr "加载笔记本后,您将看到类似以下内容的输出:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:404
|
|
||||||
msgid ""
|
|
||||||
"From your host system, or any system that can access the host's IP "
|
|
||||||
"address, start a web browser with the following. If you are not running "
|
|
||||||
"the browser on the host system, replace :command:`127.0.0.1` with the IP "
|
|
||||||
"address of the host."
|
|
||||||
msgstr "从您的主机系统或任何可以访问主机 IP 地址的系统,使用以下命令启动 Web 浏览器。如果没有在主机系统上运行浏览器,请将 :command:`127.0.0.1` 更换为主机的 IP 地址。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:412
|
|
||||||
msgid "Your browser displays the following:"
|
|
||||||
msgstr "您的浏览器会显示以下内容:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:418
|
|
||||||
msgid "Figure 1: :guilabel:`Jupyter Notebook`"
|
|
||||||
msgstr "图 1: :guilabel:`Jupyter Notebook`"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:421
|
|
||||||
msgid ""
|
|
||||||
"To create a new notebook, click :guilabel:`New` and select "
|
|
||||||
":guilabel:`Python 3`."
|
|
||||||
msgstr "要创建新笔记本,请点击 :guilabel:`New`,然后选择 :guilabel:`Python 3`。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:427
|
|
||||||
msgid "Figure 2: Create a new notebook"
|
|
||||||
msgstr "图 2:创建一个新笔记本"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:429
|
|
||||||
msgid "A new, blank notebook is displayed, with a cell ready for input."
|
|
||||||
msgstr "此时将显示一个新的空白笔记本,其中有一个单元格可供输入内容。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:436
|
|
||||||
msgid ""
|
|
||||||
"To verify that PyTorch is working, copy the following snippet into the "
|
|
||||||
"blank cell, and run the cell."
|
|
||||||
msgstr "要验证 PyTorch 是否正在工作,请将以下片段复制到空白单元格中,并运行该单元格。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:450
|
|
||||||
msgid "When you run the cell, your output will look something like this:"
|
|
||||||
msgstr "运行单元格时,您的输出将如下所示:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:456
|
|
||||||
msgid ""
|
|
||||||
"You can continue working in this notebook, or you can download existing "
|
|
||||||
"notebooks to take advantage of the Deep Learning Reference Stack's "
|
|
||||||
"optimized deep learning frameworks. Refer to `Jupyter Notebook`_ for "
|
|
||||||
"details."
|
|
||||||
msgstr "您可以继续在此笔记本中工作,也可以下载现有笔记本来利用深度学习参考堆栈的优化深度学习框架。详情请参阅 `Jupyter Notebook`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:461
|
|
||||||
msgid "Uninstallation"
|
|
||||||
msgstr "卸载"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:463
|
|
||||||
msgid ""
|
|
||||||
"To uninstall the Deep Learning Reference Stack, you can choose to stop "
|
|
||||||
"the container so that it is not using system resources, or you can stop "
|
|
||||||
"the container and delete it to free storage space."
|
|
||||||
msgstr "要卸载深度学习参考堆栈,您可以选择停止容器以使其不使用系统资源,或者可以停止容器并将其删除以释放存储空间。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:467
|
|
||||||
msgid "To stop the container, execute the following from your host system:"
|
|
||||||
msgstr "要停止容器,请从主机系统执行以下操作:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:469
|
|
||||||
msgid "Find the container's ID"
|
|
||||||
msgstr "找到容器的 ID"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:475
|
|
||||||
msgid "This will result in output similar to the following:"
|
|
||||||
msgstr "这将产生类似于以下内容的输出:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:482
|
|
||||||
msgid ""
|
|
||||||
"You can then use the ID or container name to stop the container. This "
|
|
||||||
"example uses the name \"oss\":"
|
|
||||||
msgstr "然后,您可以使用 ID 或容器名称来停止容器。本示例使用名称 \"oss\":"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:490
|
|
||||||
msgid "Verify that the container is not running"
|
|
||||||
msgstr "验证容器未在运行"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:497
|
|
||||||
msgid "To delete the container from your system you need to know the Image ID:"
|
|
||||||
msgstr "要从系统中删除容器,您需要知道映像 ID:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:503
|
|
||||||
msgid "This command results in output similar to the following:"
|
|
||||||
msgstr "该命令会产生类似于以下内容的输出:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:511
|
|
||||||
msgid "To remove an image use the image ID:"
|
|
||||||
msgstr "要移除映像,请使用映像 ID:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:527
|
|
||||||
msgid ""
|
|
||||||
"Note that you can execute the :command:`docker rmi` command using only "
|
|
||||||
"the first few characters of the image ID, provided they are unique on the"
|
|
||||||
" system."
|
|
||||||
msgstr "请注意,您可以只使用映像 ID 的前几个字符来执行 :command:`docker rmi` 命令,前提是它们在系统上是唯一的。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:529
|
|
||||||
msgid "Once you have removed the image, you can verify it has been deleted with:"
|
|
||||||
msgstr "移除映像后,您可以通过以下方式验证它是否已被移除:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:537
|
|
||||||
msgid "Related topics"
|
|
||||||
msgstr "相关主题"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:539
|
|
||||||
msgid "`DLRS V3.0`_ release announcement"
|
|
||||||
msgstr "`DLRS V3.0`_ 发布公告"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:540
|
|
||||||
msgid "`TensorFlow Benchmarks`_"
|
|
||||||
msgstr "`TensorFlow Benchmarks`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:541
|
|
||||||
msgid "`PyTorch benchmarks`_"
|
|
||||||
msgstr "`PyTorch benchmarks`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:542
|
|
||||||
msgid "`Kubeflow`_"
|
|
||||||
msgstr "`Kubeflow`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:543
|
|
||||||
msgid ":ref:`kubernetes` tutorial"
|
|
||||||
msgstr ":ref:`kubernetes` 教程"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/dlrs/dlrs.rst:544
|
|
||||||
msgid "`Jupyter Notebook`_"
|
|
||||||
msgstr "`Jupyter Notebook`_"
|
|
||||||
|
|
||||||
#~ msgid "Deep Learning Reference Stack `V3.0 release announcement`_."
|
|
||||||
#~ msgstr "深度学习参考堆栈 `V3.0 release announcement`_。"
|
|
||||||
|
|
||||||
#~ msgid ""
|
|
||||||
#~ "You must parse the logs of the "
|
|
||||||
#~ "Kubernetes pod to retrieve performance "
|
|
||||||
#~ "data. The pods will still exist "
|
|
||||||
#~ "post-completion and will be in "
|
|
||||||
#~ "‘Completed’ state. You can get the "
|
|
||||||
#~ "logs from any of the pods to "
|
|
||||||
#~ "inspect the benchmark results. More "
|
|
||||||
#~ "information about `Kubernetes logging`_ is "
|
|
||||||
#~ "available from the Kubernetes community."
|
|
||||||
#~ msgstr ""
|
|
||||||
#~ "您必须解析 Kubernetes Pod 的日志来检索性能数据。完成后,Pod "
|
|
||||||
#~ "仍会存在,并将处于“已完成”状态。您可以从任何一个 Pod 中获取日志来检查基准测试结果。有关 "
|
|
||||||
#~ "`Kubernetes logging`_ 的更多信息可从 Kubernetes 社区获取。"
|
|
||||||
|
|
||||||
#~ msgid "Deep Learning Reference Stack `V3.0 release announcement`_"
|
|
||||||
#~ msgstr "深度学习参考堆栈 `V3.0 release announcement`_"
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,711 +0,0 @@
|
|||||||
# SOME DESCRIPTIVE TITLE.
|
|
||||||
# Copyright (C) 2019, many
|
|
||||||
# This file is distributed under the same license as the Clear Linux*
|
|
||||||
# Project Docs package.
|
|
||||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
|
|
||||||
#
|
|
||||||
msgid ""
|
|
||||||
msgstr "Project-Id-Version: Clear Linux* Project Docs latest\n"
|
|
||||||
"Report-Msgid-Bugs-To: \n"
|
|
||||||
"POT-Creation-Date: 2019-08-09 14:33-0700\n"
|
|
||||||
"PO-Revision-Date: 2019-09-04 16:21-0008\n"
|
|
||||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
|
||||||
"Language: zh-Hans\n"
|
|
||||||
"Language-Team: zh-Hans\n"
|
|
||||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
|
||||||
"MIME-Version: 1.0\n"
|
|
||||||
"Content-Type: text/plain; charset=utf-8\n"
|
|
||||||
"Content-Transfer-Encoding: 8bit\n"
|
|
||||||
"Generated-By: Intel® International Developer Studio Version 4.1.273.0\n"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:4
|
|
||||||
msgid "Enable AWS Greengrass\\* and OpenVINO™ toolkit"
|
|
||||||
msgstr "启用 AWS Greengrass\\* 和 OpenVINO™ 工具包"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:6
|
|
||||||
msgid ""
|
|
||||||
"This guide explains how to enable AWS Greengrass\\* and OpenVINO™ "
|
|
||||||
"toolkit. Specifically, the guide demonstrates how to:"
|
|
||||||
msgstr "本指南说明了如何启用 AWS Greengrass\\* 和 OpenVINO™ 工具包。具体而言,该指南演示了如何:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:9
|
|
||||||
msgid "Set up the Intel® edge device with |CL-ATTR|"
|
|
||||||
msgstr "使用 |CL-ATTR| 设置英特尔®边缘设备"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:10
|
|
||||||
msgid ""
|
|
||||||
"Install the OpenVINO™ toolkit and Amazon Web Services\\* (AWS\\*) "
|
|
||||||
"Greengrass\\* software stacks"
|
|
||||||
msgstr "安装 OpenVINO™ 工具包和 Amazon Web Services\\* (AWS\\*) Greengrass\\* 软件堆栈"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:12
|
|
||||||
msgid ""
|
|
||||||
"Use AWS Greengrass\\* and AWS Lambda\\* to deploy the FaaS samples from "
|
|
||||||
"the cloud"
|
|
||||||
msgstr "使用 AWS Greengrass\\* 和 AWS Lambda\\* 从云中部署 FaaS 示例"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:20
|
|
||||||
msgid "Overview"
|
|
||||||
msgstr "概述"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:22
|
|
||||||
msgid ""
|
|
||||||
"Hardware accelerated Function-as-a-Service (FaaS) enables cloud "
|
|
||||||
"developers to deploy inference functionalities [1] on Intel® IoT edge "
|
|
||||||
"devices with accelerators (CPU, Integrated GPU, Intel® FPGA, and Intel® "
|
|
||||||
"Movidius™ technology). These functions provide a great developer "
|
|
||||||
"experience and seamless migration of visual analytics from cloud to edge "
|
|
||||||
"in a secure manner using a containerized environment. Hardware-"
|
|
||||||
"accelerated FaaS provides the best-in-class performance by accessing "
|
|
||||||
"optimized deep learning libraries on Intel® IoT edge devices with "
|
|
||||||
"accelerators."
|
|
||||||
msgstr "硬件加速的功能即服务 (FaaS) 有助于云开发人员在搭载加速器的英特尔® IoT 边缘设备(CPU、集成 GPU、英特尔® FPGA 和英特尔® Movidius™ 技术)上部署推理功能 [1]。这些功能使用容器化环境,为开发人员提供了出色的体验,有助于开发人员将可视化分析从云安全地迁移到边缘。硬件加速的 FaaS 支持在搭载加速器的英特尔® IoT 边缘设备上访问经过优化的深度学习库,实现业界最佳性能。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:32
|
|
||||||
msgid "Supported platforms"
|
|
||||||
msgstr "支持的平台"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:34
|
|
||||||
msgid "Operating System: |CL| latest release"
|
|
||||||
msgstr "操作系统:|CL| 最新版本"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:35
|
|
||||||
msgid "Hardware: Intel® core platforms (that support inference on CPU only)"
|
|
||||||
msgstr "硬件:英特尔®酷睿™平台(本教程仅支持 CPU 推理。)"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:38
|
|
||||||
msgid "Sample description"
|
|
||||||
msgstr "示例说明"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:40
|
|
||||||
msgid ""
|
|
||||||
"The AWS Greengrass samples are located at `Edge-Analytics-FaaS`_. This "
|
|
||||||
"guide uses the 1.0 version of the source code."
|
|
||||||
msgstr "AWS Greengrass 示例位于 `Edge-Analytics-FaaS`_ 中。本教程使用 1.0 版本的源代码。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:43
|
|
||||||
msgid "|CL| provides the following AWS Greengrass samples:"
|
|
||||||
msgstr "|CL| 提供以下 AWS Greengrass 示例:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:45
|
|
||||||
msgid "`greengrass_classification_sample.py`_"
|
|
||||||
msgstr "`greengrass_classification_sample.py`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:47
|
|
||||||
msgid ""
|
|
||||||
"This AWS Greengrass sample classifies a video stream using classification"
|
|
||||||
" networks such as AlexNet and GoogLeNet and publishes top-10 results on "
|
|
||||||
"AWS\\* IoT Cloud every second."
|
|
||||||
msgstr "此 AWS Greengrass 示例使用 AlexNet 和 GoogLeNet 等分类网络对视频流进行分类,并每秒在 AWS\\* IoT 云上发布前十名结果。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:51
|
|
||||||
msgid "`greengrass_object_detection_sample_ssd.py`_"
|
|
||||||
msgstr "`greengrass_object_detection_sample_ssd.py`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:53
|
|
||||||
msgid ""
|
|
||||||
"This AWS Greengrass sample detects objects in a video stream and "
|
|
||||||
"classifies them using single-shot multi-box detection (SSD) networks such"
|
|
||||||
" as SSD Squeezenet, SSD Mobilenet, and SSD300. This sample publishes "
|
|
||||||
"detection outputs such as class label, class confidence, and bounding box"
|
|
||||||
" coordinates on AWS IoT Cloud every second."
|
|
||||||
msgstr "此 AWS Greengrass 示例会检测视频流中的对象,并使用单步多框检测 (SSD) 网络(例如 SSD Squeezenet、SSD Mobilenet 和 SSD300)对它们进行分类。此示例每秒在 AWS IoT 云上发布检测输出,如类标签、类置信度和边界框坐标。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:61
|
|
||||||
msgid "Install the OS on the edge device"
|
|
||||||
msgstr "在边缘设备上安装操作系统"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:63
|
|
||||||
msgid ""
|
|
||||||
"Start with a clean installation of |CL| on a new system, using the :ref"
|
|
||||||
":`bare-metal-install-desktop`, found in :ref:`get-started`."
|
|
||||||
msgstr "使用 :ref:`get-started` 中的 :ref:`bare-metal-install-desktop`,在新系统上安装干净的 |CL|。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:67
|
|
||||||
msgid "Create user accounts"
|
|
||||||
msgstr "创建用户帐户"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:69
|
|
||||||
msgid ""
|
|
||||||
"After |CL| is installed, create two user accounts. Create an "
|
|
||||||
"administrative user in |CL| and create a user account for the Greengrass "
|
|
||||||
"services to use ( see Greengrass user below)."
|
|
||||||
msgstr "安装 |CL| 后,创建两个用户帐户。在 |CL| 中创建一个管理用户,并为要使用的 Greengrass 服务创建一个用户帐户(请参阅下面的 Greengrass 用户)。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:73
|
|
||||||
msgid ""
|
|
||||||
"Create a new user and set a password for that user. Enter the following "
|
|
||||||
"commands as ``root``:"
|
|
||||||
msgstr "创建新用户并为该用户设置密码。以 ``root`` 用户身份输入以下命令:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:81
|
|
||||||
msgid ""
|
|
||||||
"Next, enable the :command:`sudo` command for your new <userid>. Add "
|
|
||||||
"<userid> to the `wheel` group:"
|
|
||||||
msgstr "接下来,为新的 <userid> 启用 :command:`sudo` 命令。将 <userid> 添加到 `wheel` 组:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:88
|
|
||||||
msgid "Create a :file:`/etc/fstab` file."
|
|
||||||
msgstr "创建一个 :file:`/etc/fstab` 文件。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:96
|
|
||||||
msgid ""
|
|
||||||
"By default, |CL| does not create an :file:`/etc/fstab` file. You must "
|
|
||||||
"create this file before the Greengrass service runs."
|
|
||||||
msgstr "默认情况下,|CL| 不会创建 :file:`/etc/fstab` 文件。您必须在 Greengrass 服务运行之前创建此文件。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:100
|
|
||||||
msgid "Add required bundles"
|
|
||||||
msgstr "添加所需的捆绑包"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:102
|
|
||||||
msgid ""
|
|
||||||
"Use the :command:`swupd` software updater utility to add the prerequisite"
|
|
||||||
" bundles for the OpenVINO software stack:"
|
|
||||||
msgstr "使用 :command:`swupd` 软件更新程序实用程序添加 OpenVINO 软件堆栈必备的软件包:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:111
|
|
||||||
msgid "Learn more about how to :ref:`swupd-guide`."
|
|
||||||
msgstr "详细了解如何 :ref:`swupd-guide`。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:113
|
|
||||||
msgid ""
|
|
||||||
"The :command:`computer-vision-basic` bundle installs the OpenVINO™ "
|
|
||||||
"toolkit, and the sample models optimized for Intel® edge platforms."
|
|
||||||
msgstr ":command:`computer-vision-basic` 捆绑包会安装 OpenVINO™ 工具包以及针对英特尔®边缘平台优化的示例模型。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:117
|
|
||||||
msgid "Convert deep learning models"
|
|
||||||
msgstr "转换深度学习模型"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:120
|
|
||||||
msgid "Locate sample models"
|
|
||||||
msgstr "找到示例模型"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:122
|
|
||||||
msgid ""
|
|
||||||
"There are two types of provided models that can be used in conjunction "
|
|
||||||
"with AWS Greengrass for this guide: classification or object detection."
|
|
||||||
msgstr "本教程中提供了两种可以与 AWS Greengrass 配合使用的模型:分类和对象检测。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:125
|
|
||||||
msgid ""
|
|
||||||
"To complete this guide using an image classification model, download the "
|
|
||||||
"BVLC AlexNet model files `bvlc_alexnet.caffemodel`_ and "
|
|
||||||
"`deploy.prototxt`_ to the default model_location at "
|
|
||||||
":file:`/usr/share/openvino/models`. Any custom pre-trained classification"
|
|
||||||
" models can be used with the classification sample."
|
|
||||||
msgstr "要使用图像分类模型完成本教程,请将 BVLC AlexNet 模型文件 `bvlc_alexnet.caffemodel`_ 和 `deploy.prototxt`_ 下载到 :file:`/usr/share/openvino/models` 处的默认 model_location。预先训练的任何自定义分类模型都可与分类示例配合使用。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:131
|
|
||||||
msgid ""
|
|
||||||
"For object detection, the sample models optimized for Intel® edge "
|
|
||||||
"platforms are included with the computer-vision-basic bundle installation"
|
|
||||||
" at :file:`/usr/share/openvino/models`. These models are provided as an "
|
|
||||||
"example; you may also use a custom SSD model with the Greengrass object "
|
|
||||||
"detection sample."
|
|
||||||
msgstr "对于对象检测,安装 computer-vision-basic 捆绑包时会在 :file:`/usr/share/openvino/models` 处附带针对英特尔®边缘平台优化的示例模型。这些模型作为示例提供;但是,您也可以将自定义 SSD 模型与 Greengrass 对象检测示例结合使用。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:137
|
|
||||||
msgid "Run model optimizer"
|
|
||||||
msgstr "运行模型优化器"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:139
|
|
||||||
msgid ""
|
|
||||||
"Follow the instructions in the `Model Optimizer Developer Guide`_ for "
|
|
||||||
"converting deep learning models to Intermediate Representation using "
|
|
||||||
"Model Optimizer. To optimize either of the sample models described above,"
|
|
||||||
" run one of the following commands."
|
|
||||||
msgstr "遵循 `Model Optimizer Developer Guide`_ 中的说明,使用 Model Optimizer 将深度学习模型转换为 Intermediate Representation。要优化上述任一示例模型,请运行以下命令之一。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:143
|
|
||||||
msgid "For classification using BVLC AlexNet model:"
|
|
||||||
msgstr "对于使用 BVLC AlexNet 模型的分类:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:152
|
|
||||||
msgid "For object detection using SqueezeNetSSD-5Class model:"
|
|
||||||
msgstr "对于使用 SqueezeNetSSD-5Class 模型的对象检测:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:161
|
|
||||||
msgid "In these examples:"
|
|
||||||
msgstr "在这些示例中:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:163
|
|
||||||
msgid "`<model_location>` is :file:`/usr/share/openvino/models`."
|
|
||||||
msgstr "`<model_location>` 是 :file:`/usr/share/openvino/models`。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:165
|
|
||||||
msgid "`<data_type>` is FP32 or FP16, depending on target device."
|
|
||||||
msgstr "`<data_type>` 是 FP32 或 FP16,具体取决于目标设备。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:167
|
|
||||||
msgid ""
|
|
||||||
"`<output_dir>` is the directory where the Intermediate Representation "
|
|
||||||
"(IR) is stored. IR contains .xml format corresponding to the network "
|
|
||||||
"structure and .bin format corresponding to weights. This .xml file should"
|
|
||||||
" be passed to :command:`<PARAM_MODEL_XML>`."
|
|
||||||
msgstr "`<output_dir>` 是存储中间表示 (IR) 的目录。IR 包含与网络结构对应的 .xml 格式以及与权重对应的 .bin 格式。此 .xml 文件应传递给 :command:`<PARAM_MODEL_XML>`。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:172
|
|
||||||
msgid ""
|
|
||||||
"In the BVLC AlexNet model, the prototxt defines the input shape with "
|
|
||||||
"batch size 10 by default. In order to use any other batch size, the "
|
|
||||||
"entire input shape must be provided as an argument to the model "
|
|
||||||
"optimizer. For example, to use batch size 1, you must provide: "
|
|
||||||
"`--input_shape [1,3,227,227]`"
|
|
||||||
msgstr "在 BVLC AlexNet 模型中,默认情况下,prototxt 会定义批处理大小为 10 的输入形状。要使用任何其他批处理大小,必须将整个输入形状作为参数提供给模型优化器。例如,要使用批处理大小 1,您必须提供 `--input_shape [1,3,227,227]`"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:180
|
|
||||||
msgid "Configure AWS Greengrass group"
|
|
||||||
msgstr "配置 AWS Greengrass 组"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:182
|
|
||||||
msgid ""
|
|
||||||
"For each Intel® edge platform, you must create a new AWS Greengrass group"
|
|
||||||
" and install AWS Greengrass core software to establish the connection "
|
|
||||||
"between cloud and edge."
|
|
||||||
msgstr "对于每个英特尔®边缘平台,您必须创建一个新的 AWS Greengrass 组,并安装 AWS Greengrass 核心软件,以在云和边缘之间建立连接。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:186
|
|
||||||
msgid ""
|
|
||||||
"To create an AWS Greengrass group, follow the instructions in `Configure "
|
|
||||||
"AWS IoT Greengrass on AWS IoT`_."
|
|
||||||
msgstr "要创建 AWS Greengrass 组,请按照 `Configure AWS IoT Greengrass on AWS IoT`_ 中的说明执行操作。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:189
|
|
||||||
msgid ""
|
|
||||||
"To install and configure AWS Greengrass core on edge platform, follow the"
|
|
||||||
" instructions in `Start AWS Greengrass on the Core Device`_. In step "
|
|
||||||
"8(b), download the x86_64 Ubuntu\\* configuration of the AWS Greengrass "
|
|
||||||
"core software."
|
|
||||||
msgstr "要在边缘平台上安装和配置 AWS Greengrass 核心,请按照 `Start AWS Greengrass on the Core Device`_ 中的说明执行操作。在步骤 8(b) 中,下载 AWS Greengrass 核心软件的 x86_64 Ubuntu\\* 配置。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:196
|
|
||||||
msgid ""
|
|
||||||
"You do not need to run the :file:`cgroupfs-mount.sh` script in step #6 of"
|
|
||||||
" Module 1 of the `AWS Greengrass Developer Guide`_ because this is "
|
|
||||||
"enabled already in |CL|."
|
|
||||||
msgstr "您不需要在 `AWS Greengrass developer guide`_ 模块 1 的步骤 6 中运行 :file:`cgroupfs-mount.sh` 脚本,因为它已经在 |CL| 中启用。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:200
|
|
||||||
msgid ""
|
|
||||||
"Be sure to download both the security resources and the AWS Greengrass "
|
|
||||||
"core software."
|
|
||||||
msgstr "请务必下载安全资源和 AWS Greengrass 核心软件。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:205
|
|
||||||
msgid "Security certificates are linked to your AWS account."
|
|
||||||
msgstr "安全证书会链接到您的 AWS 帐户。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:209
|
|
||||||
msgid "Create and package Lambda function"
|
|
||||||
msgstr "创建并打包 Lambda 函数"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:211
|
|
||||||
msgid ""
|
|
||||||
"Complete steps 1-4 of the AWS Greengrass guide at `Create and Package a "
|
|
||||||
"Lambda Function`_."
|
|
||||||
msgstr "在 `Create and Package a Lambda Function`_ 中完成 AWS Greengrass 教程的步骤 1-4。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:216
|
|
||||||
msgid ""
|
|
||||||
"This creates the tarball needed to create the AWS Greengrass environment "
|
|
||||||
"on the edge device."
|
|
||||||
msgstr "这会创建必要的 tarball,以便在边缘设备上创建 AWS Greengrass 环境。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:220
|
|
||||||
msgid ""
|
|
||||||
"In step 5, replace :file:`greengrassHelloWorld.py` with the "
|
|
||||||
"classification or object detection Greengrass sample from `Edge-"
|
|
||||||
"Analytics-Faas`_:"
|
|
||||||
msgstr "在步骤 5 中,将 :file:`greengrassHelloWorld.py` 替换为 `Edge-Analytics-Faas`_ 中的分类或对象检测 Greengrass 示例:"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:223
|
|
||||||
msgid "Classification: `greengrass_classification_sample.py`_"
|
|
||||||
msgstr "分类:`greengrass_classification_sample.py`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:225
|
|
||||||
msgid "Object Detection: `greengrass_object_detection_sample_ssd.py`_"
|
|
||||||
msgstr "对象检测:`greengrass_object_detection_sample_ssd.py`_"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:227
|
|
||||||
msgid ""
|
|
||||||
"Zip the selected Greengrass sample with the extracted Greengrass SDK "
|
|
||||||
"folders from the previous step into "
|
|
||||||
":file:`greengrass_sample_python_lambda.zip`."
|
|
||||||
msgstr "将所选的 Greengrass 示例以及从上一步提取的 Greengrass SDK 文件夹压缩到 :file:`greengrass_sample_python_lambda.zip`。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:230
|
|
||||||
msgid "The zip should contain:"
|
|
||||||
msgstr "压缩包应包含:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:232
|
|
||||||
msgid "greengrasssdk"
|
|
||||||
msgstr "greengrasssdk"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:234
|
|
||||||
msgid "greengrass classification or object detection sample"
|
|
||||||
msgstr "greengrass 分类或对象检测示例"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:236
|
|
||||||
msgid "For example:"
|
|
||||||
msgstr "例如:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:243
|
|
||||||
msgid ""
|
|
||||||
"Return to the AWS documentation section called `Create and Package a "
|
|
||||||
"Lambda Function`_ and complete the procedure."
|
|
||||||
msgstr "返回名为 `Create and Package a Lambda Function`_ 的 AWS 文档部分,并完成步骤。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:248
|
|
||||||
msgid ""
|
|
||||||
"In step 9(a) of the AWS documentation, while uploading the zip file, make"
|
|
||||||
" sure to name the handler to one of the following, depending on the AWS "
|
|
||||||
"Greengrass sample you are using:"
|
|
||||||
msgstr "在 AWS 文档的步骤 9(a) 中,上传 Zip 文件,并确保根据使用的 AWS Greengrass 示例将处理程序命名为以下名称之一:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:252
|
|
||||||
msgid "greengrass_object_detection_sample_ssd.function_handler"
|
|
||||||
msgstr "greengrass_object_detection_sample_ssd.function_handler"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:253
|
|
||||||
msgid "greengrass_classification_sample.function_handler"
|
|
||||||
msgstr "greengrass_classification_sample.function_handler"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:257
|
|
||||||
msgid "Configure Lambda function"
|
|
||||||
msgstr "配置 Lambda 函数"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:259
|
|
||||||
msgid ""
|
|
||||||
"After creating the Greengrass group and the Lambda function, start "
|
|
||||||
"configuring the Lambda function for AWS Greengrass."
|
|
||||||
msgstr "创建 Greengrass 组和 Lambda 函数后,开始为 AWS Greengrass 配置 Lambda 函数。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:262
|
|
||||||
msgid ""
|
|
||||||
"Follow steps 1-8 in `Configure the Lambda Function for AWS IoT "
|
|
||||||
"Greengrass`_ in the AWS documentation."
|
|
||||||
msgstr "按照 AWS 文档中 `Configure the Lambda Function for AWS IoT Greengrass`_ 中的步骤 1-8 执行操作。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:265
|
|
||||||
msgid ""
|
|
||||||
"In addition to the details mentioned in step 8, change the Memory limit "
|
|
||||||
"to 2048 MB to accommodate large input video streams."
|
|
||||||
msgstr "除了步骤 8 中提到的细节之外,将内存限制更改为 2048 MB,以容纳较大的输入视频流。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:268
|
|
||||||
msgid ""
|
|
||||||
"Add the following environment variables as key-value pairs when editing "
|
|
||||||
"the Lambda configuration and click on update:"
|
|
||||||
msgstr "编辑 Lambda 配置时,添加以下环境变量作为键值对,然后点击更新:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:271
|
|
||||||
msgid "**Table 1. Environment variables: Lambda configuration**"
|
|
||||||
msgstr "**表 1.环境变量:Lambda 配置**"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:275
|
|
||||||
msgid "Key"
|
|
||||||
msgstr "键"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:276
|
|
||||||
msgid "Value"
|
|
||||||
msgstr "值"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:277
|
|
||||||
msgid "PARAM_MODEL_XML"
|
|
||||||
msgstr "PARAM_MODEL_XML"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:278
|
|
||||||
msgid ""
|
|
||||||
"<MODEL_DIR>/<IR.xml>, where <MODEL_DIR> is user specified and contains "
|
|
||||||
"IR.xml, the Intermediate Representation file from Intel® Model Optimizer."
|
|
||||||
" For this guide, <MODEL_DIR> should be set to "
|
|
||||||
"'/usr/share/openvino/models' or one of its subdirectories."
|
|
||||||
msgstr "<MODEL_DIR>/<IR.xml>,其中 <MODEL_DIR> 是用户指定的,包含来自英特尔®模型优化器的中间表示文件 IR.xml。在本教程中,<MODEL_DIR> 应设置为 '/usr/share/openvino/models' 或其某个子目录。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:282
|
|
||||||
msgid "PARAM_INPUT_SOURCE"
|
|
||||||
msgstr "PARAM_INPUT_SOURCE"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:283
|
|
||||||
msgid "<DATA_DIR>/input.webm to be specified by user. Holds both input and"
|
|
||||||
msgstr "<DATA_DIR>由用户指定的 /input.webm。保存输入和"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:284
|
|
||||||
msgid "output data. For webcam, set PARAM_INPUT_SOURCE to ‘/dev/video0’"
|
|
||||||
msgstr "输出数据。对于网络摄像头,请将 PARAM_INPUT_SOURCE 设置为 ‘/dev/video0’"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:285
|
|
||||||
msgid "PARAM_DEVICE"
|
|
||||||
msgstr "PARAM_DEVICE"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:286
|
|
||||||
msgid "\"CPU\""
|
|
||||||
msgstr "\"CPU\""
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:287
|
|
||||||
msgid "PARAM_CPU_EXTENSION_PATH"
|
|
||||||
msgstr "PARAM_CPU_EXTENSION_PATH"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:288
|
|
||||||
msgid "/usr/lib64/libcpu_extension.so"
|
|
||||||
msgstr "/usr/lib64/libcpu_extension.so"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:289
|
|
||||||
msgid "PARAM_OUTPUT_DIRECTORY"
|
|
||||||
msgstr "PARAM_OUTPUT_DIRECTORY"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:290
|
|
||||||
msgid "<DATA_DIR> to be specified by user. Holds both input and output data"
|
|
||||||
msgstr "<DATA_DIR> 由用户指定。保存输入和输出数据"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:292
|
|
||||||
msgid "PARAM_NUM_TOP_RESULTS"
|
|
||||||
msgstr "PARAM_NUM_TOP_RESULTS"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:293
|
|
||||||
msgid ""
|
|
||||||
"User specified for classification sample. (e.g. 1 for top-1 result, 5 for"
|
|
||||||
" top-5 results)"
|
|
||||||
msgstr "为分类示例指定的用户。(例如,1 为 前 1 名结果,5 为前 5 名结果)"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:296
|
|
||||||
msgid ""
|
|
||||||
"Add subscription to subscribe, or publish messages from AWS Greengrass "
|
|
||||||
"Lambda function by completing the procedure in `Configure the Lambda "
|
|
||||||
"Function for AWS IoT Greengrass`_."
|
|
||||||
msgstr "完成 `Configure the Lambda Function for AWS IoT Greengrass`_ 中的步骤,添加订阅以进行订阅或发布来自 AWS Greengrass Lambda 函数的消息。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:301
|
|
||||||
msgid ""
|
|
||||||
"The optional topic filter field is the topic mentioned inside the Lambda "
|
|
||||||
"function. In this guide, sample topics include the following: "
|
|
||||||
":command:`openvino/ssd` or :command:`openvino/classification`"
|
|
||||||
msgstr "可选主题过滤器字段是 Lambda 函数中提到的主题。在本教程中,示例主题包括以下 :command:`openvino/ssd` 或 :command:`openvino/classification`"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:305
|
|
||||||
msgid "Add local resources"
|
|
||||||
msgstr "添加本地资源"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:307
|
|
||||||
msgid ""
|
|
||||||
"Refer to the AWS documentation `Access Local Resources with Lambda "
|
|
||||||
"Functions and Connectors`_ for details about local resources and access "
|
|
||||||
"privileges."
|
|
||||||
msgstr "有关 `Access Local Resources with Lambda Functions and Connectors`_ 的详细信息,请参阅 AWS 文档。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:310
|
|
||||||
msgid "The following table describes the local resources needed for the CPU:"
|
|
||||||
msgstr "下表列出了 CPU 所需的本地资源:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:312
|
|
||||||
msgid "**Local resources**"
|
|
||||||
msgstr "**本地资源**"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:316
|
|
||||||
msgid "Name"
|
|
||||||
msgstr "名称"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:317
|
|
||||||
msgid "Resource type"
|
|
||||||
msgstr "资源类型"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:318
|
|
||||||
msgid "Local path"
|
|
||||||
msgstr "本地路径"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:319
|
|
||||||
msgid "Access"
|
|
||||||
msgstr "访问"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:321
|
|
||||||
msgid "ModelDir"
|
|
||||||
msgstr "ModelDir"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:322
|
|
||||||
#: ../../guides/stacks/greengrass.rst:332
|
|
||||||
msgid "Volume"
|
|
||||||
msgstr "卷"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:323
|
|
||||||
msgid "<MODEL_DIR> to be specified by user"
|
|
||||||
msgstr "<MODEL_DIR> 由用户指定"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:324
|
|
||||||
#: ../../guides/stacks/greengrass.rst:329
|
|
||||||
msgid "Read-Only"
|
|
||||||
msgstr "只读"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:326
|
|
||||||
msgid "Webcam"
|
|
||||||
msgstr "网络摄像头"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:327
|
|
||||||
msgid "Device"
|
|
||||||
msgstr "设备"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:328
|
|
||||||
msgid "/dev/video0"
|
|
||||||
msgstr "/dev/video0"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:331
|
|
||||||
msgid "DataDir"
|
|
||||||
msgstr "DataDir"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:333
|
|
||||||
msgid "<DATA_DIR> to be specified by user. Holds both input and output data."
|
|
||||||
msgstr "<DATA_DIR> 由用户指定。保存输入和输出数据。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:335
|
|
||||||
msgid "Read and Write"
|
|
||||||
msgstr "读取和写入"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:338
|
|
||||||
msgid "Deploy Lambda function"
|
|
||||||
msgstr "部署 Lambda 函数"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:340
|
|
||||||
msgid ""
|
|
||||||
"Refer to the AWS documentation `Deploy Cloud Configurations to an AWS IoT"
|
|
||||||
" Greengrass Core Device`_ for instructions on how to deploy the lambda "
|
|
||||||
"function to AWS Greengrass core device. Select *Deployments* on the group"
|
|
||||||
" page and follow the instructions."
|
|
||||||
msgstr "有关如何 `Deploy Cloud Configurations to an AWS IoT Greengrass Core Device`_ 的说明,请参阅 AWS 文档。在组页面上选择 *Deployments*,并按照说明执行操作。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:344
|
|
||||||
msgid "Output consumption"
|
|
||||||
msgstr "输出的使用"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:346
|
|
||||||
msgid ""
|
|
||||||
"There are four options available for output consumption. These options "
|
|
||||||
"are used to report, stream, upload, or store inference output at an "
|
|
||||||
"interval defined by the variable :command:`reporting_interval` in the AWS"
|
|
||||||
" Greengrass samples."
|
|
||||||
msgstr "使用输出时有四种可用选项。这些选项用于按 AWS Greengrass 示例中 :command:`reporting_interval` 变量定义的间隔,报告、流式传输、上传或存储推理输出。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:350
|
|
||||||
msgid "IoT cloud output:"
|
|
||||||
msgstr "IoT 云输出:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:352
|
|
||||||
msgid ""
|
|
||||||
"This option is enabled by default in the AWS Greengrass samples using the"
|
|
||||||
" :command:`enable_iot_cloud_output` variable. You can use it to verify "
|
|
||||||
"the lambda running on the edge device. It enables publishing messages to "
|
|
||||||
"IoT cloud using the subscription topic specified in the lambda. (For "
|
|
||||||
"example, topics may include :command:`openvino/classification` for "
|
|
||||||
"classification and :command:`openvino/ssd` for object detection samples.)"
|
|
||||||
" For classification, top-1 result with class label are published to IoT "
|
|
||||||
"cloud. For SSD object detection, detection results such as bounding box "
|
|
||||||
"coordinates of objects, class label, and class confidence are published."
|
|
||||||
msgstr "在 AWS Greengrass 示例中,默认情况下使用 :command:`enable_iot_cloud_output` 变量启用此选项。您可以使用它来验证在边缘设备上运行的 lambda。它支持使用 lambda 中指定的订阅主题向 IoT 云发布消息。(例如,主题可能包括用于分类示例的 :command:`openvino/classification` 以及用于对象检测示例的 :command:`openvino/ssd`。) 对于分类,具有类标签的前 1 名结果会发布到 IoT 云。对于 SSD 对象检测,则发布对象的边界框坐标、类标签和类置信度等检测结果。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:362
|
|
||||||
msgid ""
|
|
||||||
"Refer to the AWS documentation `Verify the Lambda Function Is Running on "
|
|
||||||
"the Device`_ for instructions on how to view the output on IoT cloud."
|
|
||||||
msgstr "有关如何在 IoT 云上查看输出的说明,请参考 AWS 文档 `Verify the Lambda Function Is Running on the Device`_。"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:366
|
|
||||||
msgid "Kinesis streaming:"
|
|
||||||
msgstr "Kinesis 流式传输:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:368
|
|
||||||
msgid ""
|
|
||||||
"This option enables inference output to be streamed from the edge device "
|
|
||||||
"to cloud using Kinesis [3] streams when :command:`enable_kinesis_output` "
|
|
||||||
"is set to True. The edge devices act as data producers and continually "
|
|
||||||
"push processed data to the cloud. You must set up and specify Kinesis "
|
|
||||||
"stream name, Kinesis shard, and AWS region in the AWS Greengrass samples."
|
|
||||||
msgstr ":command:`enable_kinesis_output` 设置为 True 时,此选项支持使用 Kinesis [3] 流将推理输出从边缘设备流式传输到云。边缘设备充当数据生产者,并将处理后的数据不断推送到云中。您必须在 AWS Greengrass 示例中设置和指定 Kinesis 流名称、Kinesis shard 和 AWS 区域。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:375
|
|
||||||
msgid "Cloud storage using AWS S3 bucket:"
|
|
||||||
msgstr "使用 AWS S3 存储桶的云存储:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:377
|
|
||||||
msgid ""
|
|
||||||
"When the :command:`enable_s3_jpeg_output` variable is set to True, it "
|
|
||||||
"enables uploading and storing processed frames (in jpeg format) in an AWS"
|
|
||||||
" S3 bucket. You must set up and specify the S3 bucket name in the AWS "
|
|
||||||
"Greengrass samples to store the JPEG images. The images are named using "
|
|
||||||
"the timestamp and uploaded to S3."
|
|
||||||
msgstr "将 :command:`enable_s3_jpeg_output` 变量设置为 True 时,它允许在 AWS S3 存储桶中上传和存储已处理的帧(jpeg 格式)。您必须在 AWS Greengrass 示例中设置和指定用来存储 JPEG 图像的 S3 存储桶名称。这些映像使用时间戳命名,并上传到 S3。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:383
|
|
||||||
msgid "Local storage:"
|
|
||||||
msgstr "本地存储:"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:385
|
|
||||||
msgid ""
|
|
||||||
"When the :command:`enable_s3_jpeg_output` variable is set to True, it "
|
|
||||||
"enables storing processed frames (in jpeg format) on the edge device. The"
|
|
||||||
" images are named using the timestamp and stored in a directory specified"
|
|
||||||
" by :command:`PARAM_OUTPUT_DIRECTORY`."
|
|
||||||
msgstr "将 :command:`enable_s3_jpeg_output` 变量设置为 True 时,它允许在边缘设备上存储已处理的帧(jpeg 格式)。这些映像使用时间戳命名,并存储在由 :command:`PARAM_OUTPUT_DIRECTORY` 指定的目录中。"
|
|
||||||
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:391
|
|
||||||
msgid "References"
|
|
||||||
msgstr "参考"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:393
|
|
||||||
msgid "AWS Greengrass: https://aws.amazon.com/greengrass/"
|
|
||||||
msgstr "AWS Greengrass:https://aws.amazon.com/greengrass/"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:394
|
|
||||||
msgid "AWS Lambda: https://aws.amazon.com/lambda/"
|
|
||||||
msgstr "AWS Lambda:https://aws.amazon.com/lambda/"
|
|
||||||
|
|
||||||
#: ../../guides/stacks/greengrass.rst:395
|
|
||||||
msgid "AWS Kinesis: https://aws.amazon.com/kinesis/"
|
|
||||||
msgstr "AWS Kinesis:https://aws.amazon.com/kinesis/"
|
|
||||||
|
|
||||||
#~ msgid "This tutorial demonstrates how to:"
|
|
||||||
#~ msgstr "本教程演示了如何:"
|
|
||||||
|
|
||||||
#~ msgid "Refer to the following topics:"
|
|
||||||
#~ msgstr "请参阅以下主题:"
|
|
||||||
|
|
||||||
#~ msgid ""
|
|
||||||
#~ "Follow these instructions for `converting "
|
|
||||||
#~ "deep learning models to Intermediate "
|
|
||||||
#~ "Representation using Model Optimizer`_. To "
|
|
||||||
#~ "optimize either of the sample models "
|
|
||||||
#~ "described above, run one of the "
|
|
||||||
#~ "following commands."
|
|
||||||
#~ msgstr ""
|
|
||||||
#~ "按照 `converting deep learning models to"
|
|
||||||
#~ " Intermediate Representation using Model "
|
|
||||||
#~ "Optimizer`_ 中的说明执行操作。要优化上述任一示例模型,请运行以下命令之一。"
|
|
||||||
|
|
||||||
#~ msgid "Follow the instructions here to `view the output on IoT cloud`_."
|
|
||||||
#~ msgstr "按照这里的说明`view the output on IoT cloud`_。"
|
|
||||||
|
|
||||||
|
|
||||||
@@ -64,7 +64,7 @@ msgstr ":ref:`guides`"
|
|||||||
#: ../../index.rst:31
|
#: ../../index.rst:31
|
||||||
msgid ""
|
msgid ""
|
||||||
"Guides cover a range of topics from |CL| features and tooling, to system "
|
"Guides cover a range of topics from |CL| features and tooling, to system "
|
||||||
"maintenance, network, and stacks."
|
"maintenance, and network."
|
||||||
msgstr "指南页面涵盖了从 |CL| 功能和工具到系统维护、网络和堆栈的一系列主题。"
|
msgstr "指南页面涵盖了从 |CL| 功能和工具到系统维护、网络和堆栈的一系列主题。"
|
||||||
|
|
||||||
#: ../../index.rst:34
|
#: ../../index.rst:34
|
||||||
|
|||||||
+8
-8
@@ -1,10 +1,10 @@
|
|||||||
breathe==4.9.1
|
breathe
|
||||||
sphinx==2.2.0
|
sphinx
|
||||||
docutils==0.14
|
docutils
|
||||||
sphinx_rtd_theme
|
sphinx_rtd_theme
|
||||||
sphinx-intl==2.0.0
|
sphinx-intl
|
||||||
sphinx-sitemap==1.0.2
|
sphinx-sitemap
|
||||||
Jinja2==2.10.1
|
Jinja2
|
||||||
GitPython==3.0.8
|
GitPython
|
||||||
sphinx-tabs
|
sphinx-tabs>1
|
||||||
sphinx-copybutton
|
sphinx-copybutton
|
||||||
|
|||||||
@@ -226,8 +226,8 @@ ZFS is not available with |CL| because of copyright and licensing
|
|||||||
complexities. BTRFS is an alternative filesystem that is available in |CL|
|
complexities. BTRFS is an alternative filesystem that is available in |CL|
|
||||||
natively.
|
natively.
|
||||||
|
|
||||||
A user on GitHub notes that the `ZFS kernel module can be compiled, built, and
|
A community contributed tutorial has been shared on how to :ref:`manually
|
||||||
installed manually <https://github.com/clearlinux/distribution/issues/631>`_.
|
install ZFS <zfs>`.
|
||||||
|
|
||||||
|
|
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ There are multiple ways to help improve our documentation:
|
|||||||
repository.
|
repository.
|
||||||
* `Log an issue`_: Enter an issue in the documentation repository for
|
* `Log an issue`_: Enter an issue in the documentation repository for
|
||||||
minor issues such as typos.
|
minor issues such as typos.
|
||||||
* `Make a suggestion`_: Send your documentation suggestion to the mailing list.
|
* `Make a suggestion`_: Send your documentation suggestion to the dev email inbox.
|
||||||
* Test documentation: Step through our guides and tutorials to verify the
|
* Test documentation: Step through our guides and tutorials to verify the
|
||||||
instructions. `Log an issue`_ or `submit a pull request`_ with your findings.
|
instructions. `Log an issue`_ or `submit a pull request`_ with your findings.
|
||||||
|
|
||||||
@@ -64,11 +64,11 @@ We use the following references for grammar, style, and formatting:
|
|||||||
|
|
||||||
|
|
||||||
.. _`code of conduct`: https://clearlinux.org/community/code-of-conduct
|
.. _`code of conduct`: https://clearlinux.org/community/code-of-conduct
|
||||||
.. _Make a suggestion: https://lists.clearlinux.org/postorius/lists/dev.lists.clearlinux.org/
|
.. _Make a suggestion: mailto:dev@clearlinux.discoursemail.com
|
||||||
.. _GitHub flow: https://guides.github.com/introduction/flow/
|
.. _GitHub flow: https://guides.github.com/introduction/flow/
|
||||||
.. _Log an issue: https://github.com/clearlinux/clear-linux-documentation/issues
|
.. _Log an issue: https://github.com/clearlinux/clear-linux-documentation/issues
|
||||||
.. _Contribute via GitHub: https://github.com/clearlinux/clear-linux-documentation
|
.. _Contribute via GitHub: https://github.com/clearlinux/clear-linux-documentation
|
||||||
.. _submit a pull request: https://github.com/clearlinux/clear-linux-documentation
|
.. _submit a pull request: https://github.com/clearlinux/clear-linux-documentation
|
||||||
.. _documentation repository: https://github.com/clearlinux/clear-linux-documentation
|
.. _documentation repository: https://github.com/clearlinux/clear-linux-documentation
|
||||||
.. _Microsoft Writing Style Guide: https://docs.microsoft.com/en-us/style-guide/welcome/
|
.. _Microsoft Writing Style Guide: https://docs.microsoft.com/en-us/style-guide/welcome/
|
||||||
.. _Merriam-Webster Dictionary: https://www.merriam-webster.com/
|
.. _Merriam-Webster Dictionary: https://www.merriam-webster.com/
|
||||||
|
|||||||
+9
-18
@@ -69,7 +69,7 @@ master_doc = 'index'
|
|||||||
# General information about the project.
|
# General information about the project.
|
||||||
#project = u'Clear Linux* project'
|
#project = u'Clear Linux* project'
|
||||||
project = u'Clear Linux* Project Docs'
|
project = u'Clear Linux* Project Docs'
|
||||||
copyright = u'2020 Intel Corporation. All Rights Reserved.'
|
copyright = u'2022 Intel Corporation. All Rights Reserved.'
|
||||||
author = u'many'
|
author = u'many'
|
||||||
|
|
||||||
# The version info for the project you're documenting, acts as replacement for
|
# The version info for the project you're documenting, acts as replacement for
|
||||||
@@ -86,7 +86,7 @@ author = u'many'
|
|||||||
#
|
#
|
||||||
# This is also used if you do content translation via gettext catalogs.
|
# This is also used if you do content translation via gettext catalogs.
|
||||||
# Usually you set "language" from the command line for these cases.
|
# Usually you set "language" from the command line for these cases.
|
||||||
language = None
|
language = 'en'
|
||||||
|
|
||||||
# There are two options for replacing |today|: either, you set today to some
|
# There are two options for replacing |today|: either, you set today to some
|
||||||
# non-false value, then it is used:
|
# non-false value, then it is used:
|
||||||
@@ -133,23 +133,14 @@ rst_epilog = """
|
|||||||
|
|
||||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||||
# a list of builtin themes.
|
# a list of builtin themes.
|
||||||
#html_theme = 'sphinx_rtd_theme'
|
html_theme = 'bizstyle'
|
||||||
html_theme = 'otc_tcs_sphinx_theme'
|
|
||||||
|
|
||||||
version = current_version = "latest"
|
version = current_version = "latest"
|
||||||
|
|
||||||
# Theme options are theme-specific and customize the look and feel of a theme
|
# Theme options are theme-specific and customize the look and feel of a theme
|
||||||
# further. For a list of options available for each theme, see the
|
# further. For a list of options available for each theme, see the
|
||||||
# documentation.
|
# documentation.
|
||||||
html_theme_options = {
|
html_theme_options = {}
|
||||||
'canonical_url': 'docs.01.org/clearlinux/',
|
|
||||||
'style_nav_header_background': '#007ab2',
|
|
||||||
'navigation_depth': 4,
|
|
||||||
'display_version': False,
|
|
||||||
'collapse_navigation': False,
|
|
||||||
'prev_next_buttons_location': 'None',
|
|
||||||
'sticky_navigation': True
|
|
||||||
}
|
|
||||||
|
|
||||||
html_context = {
|
html_context = {
|
||||||
"display_github": True, # Integrate GitHub
|
"display_github": True, # Integrate GitHub
|
||||||
@@ -158,12 +149,12 @@ html_context = {
|
|||||||
"github_version": "master", # Version
|
"github_version": "master", # Version
|
||||||
"conf_py_path": "/source/", # Path in the checkout to the docs root
|
"conf_py_path": "/source/", # Path in the checkout to the docs root
|
||||||
"current_version": current_version,
|
"current_version": current_version,
|
||||||
"languages": ( ("English", "/clearlinux/latest"),
|
"languages": ( ("English", "/clear-linux-documentation"),
|
||||||
("简体中文 (Simplified Chinese)", "/clearlinux/latest/zh_CN")
|
("简体中文 (Simplified Chinese)", "/clear-linux-documentation/zh_CN")
|
||||||
#("Chinese", "/clearlinux/latest/zh_CN")
|
#("Chinese", "/clearlinux/latest/zh_CN")
|
||||||
),
|
),
|
||||||
"versions": ( ("latest", "/clearlinux/latest"),
|
"versions": ( ("latest", "/clear-linux-documentation"),
|
||||||
("Future versions", "/clearlinux/latest"))
|
("future versions","/clear-linux-documentation"))
|
||||||
#("L19.01", "/clearlinux/L19.01"))
|
#("L19.01", "/clearlinux/L19.01"))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -347,5 +338,5 @@ texinfo_documents = [
|
|||||||
locale_dirs = ['../locale/'] # path for lang-specific po files.
|
locale_dirs = ['../locale/'] # path for lang-specific po files.
|
||||||
gettext_compact = False # optional.
|
gettext_compact = False # optional.
|
||||||
|
|
||||||
html_baseurl = 'https://docs.01.org/clearlinux/'
|
html_baseurl = 'https://clearlinux.github.io/clear-linux-documentation'
|
||||||
|
|
||||||
|
|||||||
@@ -285,12 +285,12 @@ YAML syntax.
|
|||||||
- login: clrlinux
|
- login: clrlinux
|
||||||
username: Clear Linux
|
username: Clear Linux
|
||||||
# Password is "clear123"
|
# Password is "clear123"
|
||||||
password: \$6\$SJJMfnInWQg.CvMA\$m2F8dJGj71zvi9mSNMktHMsPH3qhBm8pgXDNdaBe2yFfgi479JXvEqWkvQ6OxIUgGNQ5YXFIF0tCn.hEXB90G/
|
password: $6$SJJMfnInWQg.CvMA$m2F8dJGj71zvi9mSNMktHMsPH3qhBm8pgXDNdaBe2yFfgi479JXvEqWkvQ6OxIUgGNQ5YXFIF0tCn.hEXB90G/
|
||||||
admin: true
|
admin: true
|
||||||
- login: root
|
- login: root
|
||||||
username: Root Root
|
username: Root Root
|
||||||
# Password is "clear123"
|
# Password is "clear123"
|
||||||
password: \$6\$SJJMfnInWQg.CvMA\$m2F8dJGj71zvi9mSNMktHMsPH3qhBm8pgXDNdaBe2yFfgi479JXvEqWkvQ6OxIUgGNQ5YXFIF0tCn.hEXB90G/
|
password: $6$SJJMfnInWQg.CvMA$m2F8dJGj71zvi9mSNMktHMsPH3qhBm8pgXDNdaBe2yFfgi479JXvEqWkvQ6OxIUgGNQ5YXFIF0tCn.hEXB90G/
|
||||||
admin: true
|
admin: true
|
||||||
|
|
||||||
pre-install: [
|
pre-install: [
|
||||||
|
|||||||
@@ -41,13 +41,13 @@ Download and launch the virtual machine image
|
|||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
curl -O https://cdn.download.clearlinux.org/image/$(curl https://cdn.download.clearlinux.org/image/latest-images | grep '[0-9]'-kvm'\.')
|
curl -o clear.img.xz https://cdn.download.clearlinux.org/image/$(curl https://cdn.download.clearlinux.org/image/latest-images.json | grep -o 'clear-[0-9]*-kvm.img.xz' | head -1)
|
||||||
|
|
||||||
#. Uncompress the downloaded image:
|
#. Uncompress the downloaded image:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
unxz -v clear-<version>-kvm.img.xz
|
xz -dv clear.img.xz
|
||||||
|
|
||||||
#. Download the 3 OVMF files (`OVMF.fd`, `OVMF_CODE.fd`, `OVMF_VARS.fd`) that
|
#. Download the 3 OVMF files (`OVMF.fd`, `OVMF_CODE.fd`, `OVMF_VARS.fd`) that
|
||||||
provides UEFI support for virtual machines.
|
provides UEFI support for virtual machines.
|
||||||
@@ -75,17 +75,11 @@ Download and launch the virtual machine image
|
|||||||
|
|
||||||
curl -O https://cdn.download.clearlinux.org/image/start_qemu.sh
|
curl -O https://cdn.download.clearlinux.org/image/start_qemu.sh
|
||||||
|
|
||||||
#. Make the script executable:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
chmod +x start_qemu.sh
|
|
||||||
|
|
||||||
#. Start the |CL| KVM virtual machine:
|
#. Start the |CL| KVM virtual machine:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
sudo ./start_qemu.sh clear-<version>-kvm.img
|
sudo bash ./start_qemu.sh clear.img
|
||||||
|
|
||||||
#. Log in as ``root`` user and set a new password.
|
#. Log in as ``root`` user and set a new password.
|
||||||
|
|
||||||
@@ -178,7 +172,7 @@ To add :abbr:`GDM (GNOME Display Manager)` to the |CL| VM, follow these steps:
|
|||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
sudo ./start_qemu.sh clear-<version>-kvm.img
|
sudo ./start_qemu.sh clear.img
|
||||||
|
|
||||||
#. Determine the IP address of the host on which you will launch the VM.
|
#. Determine the IP address of the host on which you will launch the VM.
|
||||||
Substitute <ip-addr-of-kvm-host> in the next step with this information.
|
Substitute <ip-addr-of-kvm-host> in the next step with this information.
|
||||||
@@ -186,6 +180,7 @@ To add :abbr:`GDM (GNOME Display Manager)` to the |CL| VM, follow these steps:
|
|||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
ip a
|
ip a
|
||||||
|
|
||||||
|
|
||||||
#. From the local host or remote system, open a new terminal emulator window
|
#. From the local host or remote system, open a new terminal emulator window
|
||||||
and connect into the |CL| VM using the Spice viewer:
|
and connect into the |CL| VM using the Spice viewer:
|
||||||
|
|||||||
@@ -0,0 +1,321 @@
|
|||||||
|
.. _kubernetes-migration:
|
||||||
|
|
||||||
|
Kubernetes\* migration
|
||||||
|
######################
|
||||||
|
|
||||||
|
This guide describes how to migrate `Kubernetes container orchestration system`_ on |CL-ATTR| from 1.17.x to 1.19.x.
|
||||||
|
|
||||||
|
.. contents::
|
||||||
|
:local:
|
||||||
|
:depth: 1
|
||||||
|
|
||||||
|
Background
|
||||||
|
**********
|
||||||
|
|
||||||
|
The version of Kubernetes\* was bumped from 1.17.7 to 1.19.4 in |CL-ATTR|
|
||||||
|
release 34090. This guide and the |CL| bundle `k8s-migration` were created
|
||||||
|
to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
|
||||||
|
|
||||||
|
The new |CL| bundle `k8s-migration` was added in |CL-ATTR| release 34270.
|
||||||
|
|
||||||
|
Prerequisites
|
||||||
|
*************
|
||||||
|
|
||||||
|
* Make sure you check any updates to kubernetes upgrade doc for caveats related to the version that is running in the cluster.
|
||||||
|
* Make sure ALL the nodes are in Ready state. Without that, the cluster cannot be upgraded.
|
||||||
|
Either fix the broken nodes or remove them from the cluster.
|
||||||
|
|
||||||
|
.. contents::
|
||||||
|
:local:
|
||||||
|
:depth: 1
|
||||||
|
|
||||||
|
Upgrade 1.17.x ---> 1.18.15
|
||||||
|
***************************
|
||||||
|
|
||||||
|
#. Upgrade Control Node to 1.18.15 first
|
||||||
|
|
||||||
|
First step would be to upgrade one of the main control node and
|
||||||
|
update kubernetes components on them. You will need to have a newer
|
||||||
|
version of :command:`kubeadm` for the upgrade to work. Please consult
|
||||||
|
`kubeadm upgrade guide`_
|
||||||
|
for any caveats from your current version to the new one.
|
||||||
|
|
||||||
|
Update |CL| to the latest release to update the kubernetes version.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E swupd update
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
Note: PLEASE DO NOT REBOOT YOUR SYSTEM AT THIS TIME. |CL| is awesome and
|
||||||
|
your stuff will work just fine.
|
||||||
|
|
||||||
|
#. Add the new Kubernetes migration bundle which contains the 1.18.15 binaries.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E swupd bundle-add k8s-migration
|
||||||
|
|
||||||
|
#. Find the upgrade version of kubeadm that can used. This should be 1.18.15.
|
||||||
|
|
||||||
|
This command will show the command and possible jumps that can be made from the current kubernetes version.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E /usr/k8s-migration/bin/kubeadm upgrade plan
|
||||||
|
|
||||||
|
Sample output:
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
[upgrade/config] Making sure the configuration is correct:
|
||||||
|
[upgrade/config] Reading configuration from the cluster...
|
||||||
|
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
|
||||||
|
[preflight] Running pre-flight checks.
|
||||||
|
[upgrade] Running cluster health checks
|
||||||
|
[upgrade] Fetching available versions to upgrade to
|
||||||
|
[upgrade/versions] Cluster version: v1.17.17
|
||||||
|
[upgrade/versions] kubeadm version: v1.18.15
|
||||||
|
I0209 21:12:49.868786 832739 version.go:252] remote version is much newer: v1.20.2; falling back to: stable-1.18
|
||||||
|
[upgrade/versions] Latest stable version: v1.18.15
|
||||||
|
[upgrade/versions] Latest stable version: v1.18.15
|
||||||
|
[upgrade/versions] Latest version in the v1.17 series: v1.17.17
|
||||||
|
[upgrade/versions] Latest version in the v1.17 series: v1.17.17
|
||||||
|
|
||||||
|
Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
|
||||||
|
COMPONENT CURRENT AVAILABLE
|
||||||
|
Kubelet 3 x v1.17.7 v1.18.15
|
||||||
|
|
||||||
|
Upgrade to the latest stable version:
|
||||||
|
|
||||||
|
COMPONENT CURRENT AVAILABLE
|
||||||
|
API Server v1.17.17 v1.18.15
|
||||||
|
Controller Manager v1.17.17 v1.18.15
|
||||||
|
Scheduler v1.17.17 v1.18.15
|
||||||
|
Kube Proxy v1.17.17 v1.18.15
|
||||||
|
CoreDNS 1.6.5 1.6.7
|
||||||
|
Etcd 3.4.3 3.4.3-0
|
||||||
|
|
||||||
|
You can now apply the upgrade by executing the following command:
|
||||||
|
|
||||||
|
kubeadm upgrade apply v1.18.15
|
||||||
|
|
||||||
|
#. Upgrade the node to the intermediate 1.18.15 version of Kubernetes.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E /usr/k8s-migration/bin/kubeadm upgrade apply v1.18.15
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
Note: Do **not** reboot the system yet.
|
||||||
|
|
||||||
|
#. Upgrade Additional Control Nodes to 1.18.15
|
||||||
|
|
||||||
|
In multi-node control plane, verify all the control plane nodes are updated prior to upgrading the worker nodes/SUTs.
|
||||||
|
|
||||||
|
#. Upgrade Other Nodes to 1.18.15
|
||||||
|
|
||||||
|
For each of the other nodes:
|
||||||
|
|
||||||
|
a. Update |CL| to the latest release to update the kubernetes version.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E swupd update
|
||||||
|
|
||||||
|
#. Add the new Kubernetes migration bundle which contains the 1.18.15 binaries.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E swupd bundle-add k8s-migration
|
||||||
|
|
||||||
|
#. On the **Admin node**, drain the Client node *FIRST*
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
/usr/k8s-migration/bin/kubectl drain <CLIENT_NODE_NAME> --ignore-daemonsets --delete-local-data
|
||||||
|
|
||||||
|
#. Back on the **Client node**, upgrade Kubernetes on the Client
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E /usr/k8s-migration/bin/kubeadm upgrade node
|
||||||
|
|
||||||
|
#. On the **Admin node**, re-enable the Client
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
/usr/k8s-migration/bin/kubectl uncordon <CLIENT_NODE_NAME>
|
||||||
|
|
||||||
|
|
||||||
|
#. Back on the **Client node**, restart Kubernetes on the Client
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E systemctl restart kubelet
|
||||||
|
|
||||||
|
#. Restart Kubernetes on the Admin node(s) to finish the 1.18.x upgrade
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E systemctl restart kubelet
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
Note: Wait for all nodes to be Ready and showing the 1.19.x version.
|
||||||
|
This version will now show as it is the released version the
|
||||||
|
service files will see and use, but the Nodes are *not* upgraded yet.
|
||||||
|
|
||||||
|
Upgrade 1.18.15 ---> 1.19.x
|
||||||
|
***************************
|
||||||
|
|
||||||
|
#. Upgrade Control Node to 1.19.x
|
||||||
|
|
||||||
|
Now that systems are upgraded to the intermediate release of 1.18.15
|
||||||
|
each of the nodes can be upgraded to the latest 1.19.x release.
|
||||||
|
|
||||||
|
#. Find the upgrade version of kubeadm that can used. This should be 1.19.x.
|
||||||
|
|
||||||
|
This command will show the command and possible jumps that can be made from the current kubernetes version.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E kubeadm upgrade plan
|
||||||
|
|
||||||
|
Sample output:
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
[upgrade/config] Making sure the configuration is correct:
|
||||||
|
[upgrade/config] Reading configuration from the cluster...
|
||||||
|
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
|
||||||
|
[preflight] Running pre-flight checks.
|
||||||
|
[upgrade] Running cluster health checks
|
||||||
|
[upgrade] Fetching available versions to upgrade to
|
||||||
|
[upgrade/versions] Cluster version: v1.18.15
|
||||||
|
[upgrade/versions] kubeadm version: v1.19.7
|
||||||
|
I0209 23:08:23.810900 925910 version.go:252] remote version is much newer: v1.20.2; falling back to: stable-1.19
|
||||||
|
[upgrade/versions] Latest stable version: v1.19.7
|
||||||
|
[upgrade/versions] Latest stable version: v1.19.7
|
||||||
|
[upgrade/versions] Latest version in the v1.18 series: v1.18.15
|
||||||
|
[upgrade/versions] Latest version in the v1.18 series: v1.18.15
|
||||||
|
|
||||||
|
Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
|
||||||
|
COMPONENT CURRENT AVAILABLE
|
||||||
|
kubelet 3 x v1.17.7 v1.19.7
|
||||||
|
|
||||||
|
Upgrade to the latest stable version:
|
||||||
|
|
||||||
|
COMPONENT CURRENT AVAILABLE
|
||||||
|
kube-apiserver v1.18.15 v1.19.7
|
||||||
|
kube-controller-manager v1.18.15 v1.19.7
|
||||||
|
kube-scheduler v1.18.15 v1.19.7
|
||||||
|
kube-proxy v1.18.15 v1.19.7
|
||||||
|
CoreDNS 1.6.7 1.7.0
|
||||||
|
etcd 3.4.3-0 3.4.13-0
|
||||||
|
|
||||||
|
You can now apply the upgrade by executing the following command:
|
||||||
|
|
||||||
|
kubeadm upgrade apply v1.19.7
|
||||||
|
|
||||||
|
The table below shows the current state of component configs as understood by this version of kubeadm.
|
||||||
|
Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or
|
||||||
|
resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually
|
||||||
|
upgrade to is denoted in the "PREFERRED VERSION" column.
|
||||||
|
|
||||||
|
API GROUP CURRENT VERSION PREFERRED VERSION MANUAL UPGRADE REQUIRED
|
||||||
|
kubeproxy.config.k8s.io v1alpha1 v1alpha1 no
|
||||||
|
kubelet.config.k8s.io v1beta1 v1beta1 no
|
||||||
|
|
||||||
|
#. Upgrade the node to the latest 1.19.x version of Kubernetes.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E /usr/bin/kubeadm upgrade apply v1.19.7
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
Note: Do **not** reboot the system yet.
|
||||||
|
|
||||||
|
#. Upgrade Additional Control Nodes to 1.19.x
|
||||||
|
|
||||||
|
In multi-node control plane, verify all the control plane nodes are updated prior to upgrading the worker nodes/SUTs.
|
||||||
|
|
||||||
|
#. Upgrade Other Nodes to 1.19.x
|
||||||
|
|
||||||
|
For each of the other nodes:
|
||||||
|
|
||||||
|
a. On the **Admin node**, drain the Client *FIRST*
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
kubectl drain <CLIENT_NODE_NAME> --ignore-daemonsets
|
||||||
|
|
||||||
|
#. Back on the **Client node**, upgrade Kubernetes on the Client
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E kubeadm upgrade node
|
||||||
|
|
||||||
|
#. On the **Admin node**, re-enable the Client
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
kubectl uncordon <CLIENT_NODE_NAME>
|
||||||
|
|
||||||
|
#. Back on the **Client node**, if you wish reboot the Client, it is now safe to do so.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo reboot
|
||||||
|
|
||||||
|
#. Reboot the Control Node (optional)
|
||||||
|
|
||||||
|
*If you wish reboot the nodes, it is now safe to do so.*
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo reboot
|
||||||
|
|
||||||
|
**Congratulations!**
|
||||||
|
|
||||||
|
You've successfully installed and set up Kubernetes in |CL| using CRI-O and kata-runtime. You are now ready to follow on-screen instructions to deploy a pod network to the cluster and join worker nodes with the displayed token and IP information.
|
||||||
|
|
||||||
|
Clean up: Remove the migration bundle for each node
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo -E swupd bundle-remove k8s-migration
|
||||||
|
|
||||||
|
Related topics
|
||||||
|
**************
|
||||||
|
|
||||||
|
Read the Kubernetes documentation to learn more about:
|
||||||
|
|
||||||
|
* `Kubernetes tutorial <tutorials/kubernetes>`_
|
||||||
|
|
||||||
|
* `Kubernetes best practices <tutorials/kubernetes-bp>`_
|
||||||
|
|
||||||
|
* Deploying Kubernetes with a `cloud-native-setup`_
|
||||||
|
|
||||||
|
* `Understanding basic Kubernetes architecture`_
|
||||||
|
|
||||||
|
* `Deploying an application to your cluster`_
|
||||||
|
|
||||||
|
* Installing a `pod network add-on`_
|
||||||
|
|
||||||
|
* `Joining your nodes`_
|
||||||
|
|
||||||
|
|
||||||
|
.. _kubeadm upgrade guide: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/
|
||||||
|
|
||||||
|
.. _Kubernetes container orchestration system: https://kubernetes.io/
|
||||||
|
|
||||||
|
.. _Understanding basic Kubernetes architecture: https://kubernetes.io/docs/user-journeys/users/application-developer/foundational/#section-3
|
||||||
|
|
||||||
|
.. _Deploying an application to your cluster: https://kubernetes.io/docs/user-journeys/users/application-developer/foundational/#section-2
|
||||||
|
|
||||||
|
.. _pod network add-on: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network
|
||||||
|
|
||||||
|
.. _Joining your nodes: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#join-nodes
|
||||||
|
|
||||||
|
.. _cloud-native-setup: https://github.com/clearlinux/cloud-native-setup/tree/master/clr-k8s-examples
|
||||||
@@ -91,7 +91,7 @@ optimize software builds for runtime performance. Some significant flags that
|
|||||||
Options used to tune generated code with optimized instructions for specific
|
Options used to tune generated code with optimized instructions for specific
|
||||||
CPU types instead of creating generic code for maximum compatibility.
|
CPU types instead of creating generic code for maximum compatibility.
|
||||||
|
|
||||||
|CL| defines it's minimum hardware requirements to be second-generation
|
|CL| defines its minimum hardware requirements to be second-generation
|
||||||
Intel® microarchitecture code name Westmere (released in 2010) or later.
|
Intel® microarchitecture code name Westmere (released in 2010) or later.
|
||||||
This enables compiler optimizations that are available only on newer
|
This enables compiler optimizations that are available only on newer
|
||||||
architectures. Whenever possible, |CL| tunes code for the Haswell generation
|
architectures. Whenever possible, |CL| tunes code for the Haswell generation
|
||||||
@@ -238,4 +238,4 @@ Related topics
|
|||||||
* `Boosting Python* from profile-guided to platform-specific optimizations <https://clearlinux.org/news-blogs/boosting-python-profile-guided-platform-specific-optimizations>`_
|
* `Boosting Python* from profile-guided to platform-specific optimizations <https://clearlinux.org/news-blogs/boosting-python-profile-guided-platform-specific-optimizations>`_
|
||||||
* `Transparent use of library packages optimized for Intel® architecture <https://clearlinux.org/news-blogs/transparent-use-library-packages-optimized-intel-architecture>`_
|
* `Transparent use of library packages optimized for Intel® architecture <https://clearlinux.org/news-blogs/transparent-use-library-packages-optimized-intel-architecture>`_
|
||||||
|
|
||||||
*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
|
*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
|
||||||
|
|||||||
@@ -47,11 +47,3 @@ Kernel
|
|||||||
|
|
||||||
kernel/*
|
kernel/*
|
||||||
|
|
||||||
Stacks
|
|
||||||
=======
|
|
||||||
|
|
||||||
.. toctree::
|
|
||||||
:maxdepth: 1
|
|
||||||
:glob:
|
|
||||||
|
|
||||||
stacks/*
|
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
.. _change-kernel-boot:
|
||||||
|
|
||||||
|
Change Kernel Boot
|
||||||
|
########################
|
||||||
|
|
||||||
|
This tutorial explains the process of change kernel boot entry |CL-ATTR|.
|
||||||
|
|
||||||
|
.. contents::
|
||||||
|
:local:
|
||||||
|
:depth: 1
|
||||||
|
|
||||||
|
Description
|
||||||
|
***********
|
||||||
|
|
||||||
|
For this tutorial, you will modify your kernel list to boot with the kernel you want to use. This process is valid when you cannot compile third-party kernel modules and need to come back to old, or if you compile your custom kernel.
|
||||||
|
|
||||||
|
|
||||||
|
Get the current boot status
|
||||||
|
***************************
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
bootctl status
|
||||||
|
|
||||||
|
This is an example output:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
System:
|
||||||
|
Firmware: UEFI 2.70 (HP 265.256)
|
||||||
|
Firmware Arch: x64
|
||||||
|
Secure Boot: disabled
|
||||||
|
TPM2 Support: yes
|
||||||
|
Measured UKI: no
|
||||||
|
Boot into FW: supported
|
||||||
|
|
||||||
|
Current Boot Loader:
|
||||||
|
Product: systemd-boot 255
|
||||||
|
Features: ✓ Boot counting
|
||||||
|
✓ Menu timeout control
|
||||||
|
✓ One-shot menu timeout control
|
||||||
|
✓ Default entry control
|
||||||
|
✓ One-shot entry control
|
||||||
|
✓ Support for XBOOTLDR partition
|
||||||
|
✓ Support for passing random seed to OS
|
||||||
|
✓ Load drop-in drivers
|
||||||
|
✓ Support Type #1 sort-key field
|
||||||
|
✓ Support @saved pseudo-entry
|
||||||
|
✓ Support Type #1 devicetree field
|
||||||
|
✓ Enroll SecureBoot keys
|
||||||
|
✓ Retain SHIM protocols
|
||||||
|
✓ Menu can be disabled
|
||||||
|
✓ Boot loader sets ESP information
|
||||||
|
ESP: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/org.clearlinux/loaderx64.efi
|
||||||
|
|
||||||
|
Random Seed:
|
||||||
|
System Token: set
|
||||||
|
Exists: yes
|
||||||
|
|
||||||
|
Available Boot Loaders on ESP:
|
||||||
|
ESP: /boot (/dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5)
|
||||||
|
File: ├─/EFI/systemd/systemd-bootx64.efi (systemd-boot 255)
|
||||||
|
└─/EFI/BOOT/BOOTX64.EFI (systemd-boot 255)
|
||||||
|
|
||||||
|
Boot Loaders Listed in EFI Variables:
|
||||||
|
Title: Linux bootloader
|
||||||
|
ID: 0x0007
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/org.clearlinux/bootloaderx64.efi
|
||||||
|
|
||||||
|
Title: Linux Boot Manager
|
||||||
|
ID: 0x0001
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/systemd/systemd-bootx64.efi
|
||||||
|
|
||||||
|
Title: Windows Boot Manager
|
||||||
|
ID: 0x0000
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/48d8a9eb-d84d-4a62-8302-edff383290e5
|
||||||
|
File: └─/EFI/Microsoft/Boot/bootmgfw.efi
|
||||||
|
|
||||||
|
Boot Loader Entries:
|
||||||
|
$BOOT: /boot (/dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5)
|
||||||
|
token: clear-linux-os
|
||||||
|
|
||||||
|
Default Boot Loader Entry:
|
||||||
|
type: Boot Loader Specification Type #1 (.conf)
|
||||||
|
title: Clear Linux OS (Clear-linux-native-6.8.10-1434.conf)
|
||||||
|
id: Clear-linux-native-6.8.10-1434.conf
|
||||||
|
source: /boot//loader/entries/Clear-linux-native-6.8.10-1434.conf
|
||||||
|
linux: /boot//EFI/org.clearlinux/kernel-org.clearlinux.native.6.8.10-1434
|
||||||
|
initrd: /boot//EFI/org.clearlinux/freestanding-00-early-ucode.cpio
|
||||||
|
/boot//EFI/org.clearlinux/initrd-org.clearlinux.native.6.8.10-1434
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-clr-init.cpio.gz
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-i915-firmware.cpio
|
||||||
|
options: root=UUID=67e7ac9a-f7a1-4d5e-bbd6-012f5fa81cb5 rd.luks.uuid=abe6aaf2-3425-4eb1-b7f5-3f36746426fa quiet console=tty0 console=ttyS0,115200n8 cryptomgr.notests init=/usr/bin/initra-desktop initcall>
|
||||||
|
lines 20-71/71 (END)
|
||||||
|
✓ Support @saved pseudo-entry
|
||||||
|
✓ Support Type #1 devicetree field
|
||||||
|
✓ Enroll SecureBoot keys
|
||||||
|
✓ Retain SHIM protocols
|
||||||
|
✓ Menu can be disabled
|
||||||
|
✓ Boot loader sets ESP information
|
||||||
|
ESP: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/org.clearlinux/loaderx64.efi
|
||||||
|
|
||||||
|
Random Seed:
|
||||||
|
System Token: set
|
||||||
|
Exists: yes
|
||||||
|
|
||||||
|
Available Boot Loaders on ESP:
|
||||||
|
ESP: /boot (/dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5)
|
||||||
|
File: ├─/EFI/systemd/systemd-bootx64.efi (systemd-boot 255)
|
||||||
|
└─/EFI/BOOT/BOOTX64.EFI (systemd-boot 255)
|
||||||
|
|
||||||
|
Boot Loaders Listed in EFI Variables:
|
||||||
|
Title: Linux bootloader
|
||||||
|
ID: 0x0007
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/org.clearlinux/bootloaderx64.efi
|
||||||
|
|
||||||
|
Title: Linux Boot Manager
|
||||||
|
ID: 0x0001
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5
|
||||||
|
File: └─/EFI/systemd/systemd-bootx64.efi
|
||||||
|
|
||||||
|
Title: Windows Boot Manager
|
||||||
|
ID: 0x0000
|
||||||
|
Status: active, boot-order
|
||||||
|
Partition: /dev/disk/by-partuuid/48d8a9eb-d84d-4a62-8302-edff383290e5
|
||||||
|
File: └─/EFI/Microsoft/Boot/bootmgfw.efi
|
||||||
|
|
||||||
|
Boot Loader Entries:
|
||||||
|
$BOOT: /boot (/dev/disk/by-partuuid/ea2e4278-5c0c-4498-bc99-dfc48a71ceb5)
|
||||||
|
token: clear-linux-os
|
||||||
|
|
||||||
|
Default Boot Loader Entry:
|
||||||
|
type: Boot Loader Specification Type #1 (.conf)
|
||||||
|
title: Clear Linux OS (Clear-linux-native-6.8.10-1434.conf)
|
||||||
|
id: Clear-linux-native-6.8.10-1434.conf
|
||||||
|
source: /boot//loader/entries/Clear-linux-native-6.8.10-1434.conf
|
||||||
|
linux: /boot//EFI/org.clearlinux/kernel-org.clearlinux.native.6.8.10-1434
|
||||||
|
initrd: /boot//EFI/org.clearlinux/freestanding-00-early-ucode.cpio
|
||||||
|
/boot//EFI/org.clearlinux/initrd-org.clearlinux.native.6.8.10-1434
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-clr-init.cpio.gz
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-i915-firmware.cpio
|
||||||
|
options: root=UUID=67e7ac9a-f7a1-4d5e-bbd6-012f5fa81cb5 rd.luks.uuid=abe6aaf2-3425-4eb1-b7f5-3f36746426fa quiet console=tty0 console=ttyS0,115200n8 cryptomgr.notests init=/usr/bin/initra-desktop initcall_debug intel_iommu=igfx_off kvm-intel.nested=1 no_timer_check noreplace-smp page_alloc.shuffle=1 rcupdate.rcu_expedited=1 rootfstype=ext4,btrfs,xfs,f2fs tsc=reliable rw module.sig_unenforce rootflags=x-systemd.device-timeout=0
|
||||||
|
|
||||||
|
Get the kernel list installed
|
||||||
|
*****************************
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
bootctl list
|
||||||
|
|
||||||
|
And example output:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
type: Boot Loader Specification Type #1 (.conf)
|
||||||
|
title: Clear Linux OS (Clear-linux-preempt_rt-6.1.38-105.conf)
|
||||||
|
id: Clear-linux-preempt_rt-6.1.38-105.conf
|
||||||
|
source: /boot//loader/entries/Clear-linux-preempt_rt-6.1.38-105.conf
|
||||||
|
linux: /boot//EFI/org.clearlinux/kernel-org.clearlinux.preempt_rt.6.1.38-105
|
||||||
|
initrd: /boot//EFI/org.clearlinux/freestanding-00-early-ucode.cpio
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-clr-init.cpio.gz
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-i915-firmware.cpio
|
||||||
|
options: root=UUID=67e7ac9a-f7a1-4d5e-bbd6-012f5fa81cb5 rd.luks.uuid=abe6aaf2-3425-4eb1-b7f5-3f36746426fa quiet console=tty0 console=ttyS0,115200n8 cryptomgr.notests init=/usr/bin/initra-desktop initcall_debug intel_iommu=igfx_off kvm-intel.nested=1 no_timer_check noreplace-smp page_alloc.shuffle=1 rcupdate.rcu_expedited=1 rootfstype=ext4,btrfs,xfs t>
|
||||||
|
|
||||||
|
type: Boot Loader Specification Type #1 (.conf)
|
||||||
|
title: Clear Linux OS (Clear-linux-native-6.9.1-1436.conf)
|
||||||
|
id: Clear-linux-native-6.9.1-1436.conf
|
||||||
|
source: /boot//loader/entries/Clear-linux-native-6.9.1-1436.conf
|
||||||
|
linux: /boot//EFI/org.clearlinux/kernel-org.clearlinux.native.6.9.1-1436
|
||||||
|
initrd: /boot//EFI/org.clearlinux/freestanding-00-early-ucode.cpio
|
||||||
|
/boot//EFI/org.clearlinux/initrd-org.clearlinux.native.6.9.1-1436
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-clr-init.cpio.gz
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-i915-firmware.cpio
|
||||||
|
options: root=UUID=67e7ac9a-f7a1-4d5e-bbd6-012f5fa81cb5 rd.luks.uuid=abe6aaf2-3425-4eb1-b7f5-3f36746426fa quiet console=tty0 console=ttyS0,115200n8 cryptomgr.notests init=/usr/bin/initra-desktop initcall_debug intel_iommu=igfx_off kvm-intel.nested=1 no_timer_check noreplace-smp page_alloc.shuffle=1 rcupdate.rcu_expedited=1 rootfstype=ext4,btrfs,xfs,f>
|
||||||
|
|
||||||
|
type: Boot Loader Specification Type #1 (.conf)
|
||||||
|
title: Clear Linux OS (Clear-linux-native-6.8.10-1434.conf) (default) (selected)
|
||||||
|
id: Clear-linux-native-6.8.10-1434.conf
|
||||||
|
source: /boot//loader/entries/Clear-linux-native-6.8.10-1434.conf
|
||||||
|
linux: /boot//EFI/org.clearlinux/kernel-org.clearlinux.native.6.8.10-1434
|
||||||
|
initrd: /boot//EFI/org.clearlinux/freestanding-00-early-ucode.cpio
|
||||||
|
/boot//EFI/org.clearlinux/initrd-org.clearlinux.native.6.8.10-1434
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-clr-init.cpio.gz
|
||||||
|
/boot//EFI/org.clearlinux/freestanding-i915-firmware.cpio
|
||||||
|
options: root=UUID=67e7ac9a-f7a1-4d5e-bbd6-012f5fa81cb5 rd.luks.uuid=abe6aaf2-3425-4eb1-b7f5-3f36746426fa quiet console=tty0 console=ttyS0,115200n8 cryptomgr.notests init=/usr/bin/initra-desktop initcall_debug intel_iommu=igfx_off kvm-intel.nested=1 no_timer_check noreplace-smp page_alloc.shuffle=1 rcupdate.rcu_expedited=1 rootfstype=ext4,btrfs,xfs,f>
|
||||||
|
|
||||||
|
type: Automatic
|
||||||
|
title: Reboot Into Firmware Interface
|
||||||
|
id: auto-reboot-to-firmware-setup
|
||||||
|
source: /sys/firmware/efi/efivars/LoaderEntries-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f
|
||||||
|
|
||||||
|
Set default kernel to boot
|
||||||
|
**************************
|
||||||
|
|
||||||
|
You can check the id from the latest command:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
bootctl list |grep id: |cut -f 2 -d ":"
|
||||||
|
id: Clear-linux-preempt_rt-6.1.38-105.conf
|
||||||
|
id: Clear-linux-native-6.9.1-1436.conf
|
||||||
|
id: Clear-linux-native-6.8.10-1434.conf
|
||||||
|
id: auto-reboot-to-firmware-setup
|
||||||
|
|
||||||
|
|
||||||
|
Set the kernel
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo bootctl set-default ID
|
||||||
|
|
||||||
|
For example to set 6.9.1 entry:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo bootctl set-default Clear-linux-native-6.9.1-1436.conf
|
||||||
|
|
||||||
|
Just reboot
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo systemctl reboot
|
||||||
|
|
||||||
|
You will boot with the kernel set before.
|
||||||
@@ -64,8 +64,10 @@ The :command:`kernel-native-dkms` bundle also:
|
|||||||
kernel. This is especially important for systems where a successful boot
|
kernel. This is especially important for systems where a successful boot
|
||||||
relies on a kernel module.
|
relies on a kernel module.
|
||||||
|
|
||||||
|
.. _kernel-modules-dkms-install-begin-alt:
|
||||||
|
|
||||||
Install the :command:`kernel-native-dkms` or :command:`kernel-lts-dkms`
|
Install the :command:`kernel-native-dkms` or :command:`kernel-lts-dkms`
|
||||||
bundle:
|
bundle.
|
||||||
|
|
||||||
#. Determine which kernel variant is running on |CL|. Only the *native*
|
#. Determine which kernel variant is running on |CL|. Only the *native*
|
||||||
and *lts* kernels are enabled to build and load out-of-tree kernel modules
|
and *lts* kernels are enabled to build and load out-of-tree kernel modules
|
||||||
@@ -93,7 +95,8 @@ bundle:
|
|||||||
sudo swupd bundle-add kernel-lts-dkms
|
sudo swupd bundle-add kernel-lts-dkms
|
||||||
|
|
||||||
|
|
||||||
#. Update the |CL| bootloader and reboot.
|
#. Update the |CL| bootloader and reboot, and
|
||||||
|
ensure that you can start the new kernel.
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ to add hugepages to the system and how to change the default hugepage size.
|
|||||||
|
|
||||||
The active option is enclosed in brackets. In this case, always is active,
|
The active option is enclosed in brackets. In this case, always is active,
|
||||||
which means hugepages are enabled for every process. The `madvise`
|
which means hugepages are enabled for every process. The `madvise`
|
||||||
option means that hugepages are enabled for processes that explicitely
|
option means that hugepages are enabled for processes that explicitly
|
||||||
call `madvise`_.
|
call `madvise`_.
|
||||||
|
|
||||||
#. To check the size of hugepages, run the below command.
|
#. To check the size of hugepages, run the below command.
|
||||||
@@ -71,4 +71,4 @@ to add hugepages to the system and how to change the default hugepage size.
|
|||||||
sudo clr-boot-manager update
|
sudo clr-boot-manager update
|
||||||
sudo reboot
|
sudo reboot
|
||||||
|
|
||||||
.. _madvise: https://linux.die.net/man/2/madvise
|
.. _madvise: https://linux.die.net/man/2/madvise
|
||||||
|
|||||||
@@ -1,703 +0,0 @@
|
|||||||
.. _dars:
|
|
||||||
|
|
||||||
Data Analytics Reference Stack
|
|
||||||
##############################
|
|
||||||
|
|
||||||
This guide explains how to use the :abbr:`DARS (Data Analytics Reference Stack)`, and to optionally build your own DARS container image.
|
|
||||||
|
|
||||||
Any system that supports Docker\* containers can be used with DARS. The steps
|
|
||||||
in this guide use |CL-ATTR| as the host system.
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
Overview
|
|
||||||
********
|
|
||||||
|
|
||||||
The Data Analytics Reference Stack (DARS) provides developers and enterprises a straightforward, highly optimized software stack for storing and processing large amounts of data. More detail is available on the `DARS architecture and performance benchmarks`_.
|
|
||||||
|
|
||||||
Stack Features
|
|
||||||
==============
|
|
||||||
|
|
||||||
The Data Analytics Reference Stack provides two pre-built Docker images,
|
|
||||||
available on `Docker Hub`_:
|
|
||||||
|
|
||||||
* A |CL|-derived `DARS with OpenBlas`_ stack optimized for `OpenBLAS`_
|
|
||||||
* A |CL|-derived `DARS with Intel® MKL`_ stack optimized for
|
|
||||||
`Intel® Math Kernel Library`_ (Intel® MKL)
|
|
||||||
|
|
||||||
We recommend you view the latest component versions for each image in the
|
|
||||||
:file:`releasenote` found in the `Data Analytics Reference Stack`_ GitHub\*
|
|
||||||
repository. Because |CL| is a rolling distribution, the package version numbers
|
|
||||||
in the |CL|-based containers may not be the latest released by |CL|.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The Data Analytics Reference Stack is a collective work, and each piece
|
|
||||||
of software within the work has its own license. Please see the
|
|
||||||
`DARS Terms of Use`_ for more details about licensing and usage of the Data
|
|
||||||
Analytics Reference Stack.
|
|
||||||
|
|
||||||
Using the Docker images
|
|
||||||
***********************
|
|
||||||
|
|
||||||
Launching the Image
|
|
||||||
===================
|
|
||||||
|
|
||||||
#. To use the latest stable DARS images, pull an image
|
|
||||||
directly from `Docker Hub`_. This example uses the
|
|
||||||
`DARS with Intel® MKL`_ Docker image.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker pull clearlinux/stacks-dars-mkl
|
|
||||||
|
|
||||||
|
|
||||||
#. Once you have downloaded the image, you can run it with this command, which will launch the image and drop you into a bash shell inside the container.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run -it --ulimit nofile=1000000:1000000 --name mkl --network host --rm -i -t <name-of-image>
|
|
||||||
|
|
||||||
|
|
||||||
Command Flags
|
|
||||||
|
|
||||||
:command:`--ulimit nofile=` is required in order to increase the allowed number of open files for the Apache Spark\* engine.
|
|
||||||
|
|
||||||
:command:`--name` can be any name of your choice. This guide is using `mkl`
|
|
||||||
|
|
||||||
:command:`--network host` enables the host machine's IP address to be used to access the container.
|
|
||||||
|
|
||||||
If you need to verify the name of the DARS image for the <name-of-image> flag, you can use the :command:`docker image ls` command to see which images reside on your system.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker image ls
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
|
||||||
clearlinux/stacks-dars-mkl test-img 49a70a22231f 23 hours ago 2.66GB
|
|
||||||
ubuntu latest 2ca708c1c9cc 7 days ago 64.2MB
|
|
||||||
katadocker/kata-deploy latest bd6dc92f8060 7 days ago 673MB
|
|
||||||
clearlinux/stacks-dars-mkl latest 2c9555536d5f 4 weeks ago 2.62GB
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
All of the DARS components are compiled on Open JDK11\*. The container will have preinstalled JDK11 at :file:`/usr/lib/jvm/java-1.11.0-openjdk/` and it has been set as the default Java version. While the DARS containers also contain Open JDK8, it is not covered in this guide.
|
|
||||||
|
|
||||||
|
|
||||||
Building DARS images
|
|
||||||
====================
|
|
||||||
|
|
||||||
If you choose to build your own DARS container images, you can customize them as needed. Use the :file:`Dockerfile` included in the Github\* repository as your baseline.
|
|
||||||
|
|
||||||
To construct images with |CL|, start with a |CL| development platform that has the :command:`containers-basic-dev` bundle installed. Learn more about bundles and installing them by using :ref:`swupd-guide`.
|
|
||||||
|
|
||||||
#. The `Data Analytics Reference Stack`_ is part of the Intel stacks GitHub\* repository. Clone the :file:`stacks` repository.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/intel/stacks.git
|
|
||||||
|
|
||||||
#. Inside the :file:`stacks/dars/clearlinux/mkl` directory, use docker with the :file:`Dockerfile` to build the MKL image.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
cd ./stacks/dars/clearlinux/mkl
|
|
||||||
docker build --no-cache -t clearlinux/stacks-dars-mkl .
|
|
||||||
|
|
||||||
|
|
||||||
#. Once completed, check the resulting images with :command:`Docker`
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker images | grep dars
|
|
||||||
|
|
||||||
#. You can use any of the resulting images to launch fully functional containers. If you need to customize the containers, you can edit the provided :file:`Dockerfile`.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The environment variables for Apache Hadoop* and Apache Spark have been configured in the Dockerfile for the DARS container. For Apache Hadoop\* use :file:`/etc/hadoop` as `HADOOP_CONF_DIR` folder. For Apache Spark use :file:`/etc/spark` as `SPARK_CONF_DIR` folder.
|
|
||||||
|
|
||||||
|
|
||||||
Using Apache Spark\* in DARS
|
|
||||||
****************************
|
|
||||||
|
|
||||||
After launching the container, you can start Apache Spark with either the Scala or PySpark environment. For these examples we will use PySpark, which is the Python\* API for Apache Spark.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
pyspark
|
|
||||||
|
|
||||||
|
|
||||||
Launching is as simple as this. Depending on your system configuration and capabilities, you may need to define proxy or memory allocation settings on the command line or in a config file for optimal performance. Refer to the `Apache Spark documentation`_ for more detail.
|
|
||||||
|
|
||||||
After executing :command:`pyspark`, you will see output similar to this.
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
root@fd5155b89857 /root # pyspark
|
|
||||||
Welcome to
|
|
||||||
____ __
|
|
||||||
/ __/__ ___ _____/ /__
|
|
||||||
_\ \/ _ \/ _ `/ __/ '_/
|
|
||||||
/__ / .__/\_,_/_/ /_/\_\ version 2.4.0
|
|
||||||
/_/
|
|
||||||
|
|
||||||
Using Python version 3.7.4 (default, Jul 13 2019 06:59:17)
|
|
||||||
SparkSession available as 'spark'.
|
|
||||||
>>>
|
|
||||||
|
|
||||||
|
|
||||||
Execute code directly in PySpark
|
|
||||||
================================
|
|
||||||
|
|
||||||
A simple example for verifying that pyspark is working correctly is to run a small python function from a `PySpark getting started guide`_ to estimate the value of Pi. Run these lines in the PySpark shell.
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
import random
|
|
||||||
NUM_SAMPLES = 100000000
|
|
||||||
def inside(p):
|
|
||||||
x, y = random.random(), random.random()
|
|
||||||
return x*x + y*y < 1
|
|
||||||
|
|
||||||
count = sc.parallelize(range(0, NUM_SAMPLES)).filter(inside).count()
|
|
||||||
pi = 4 * count / NUM_SAMPLES
|
|
||||||
print(“Pi is roughly”, pi)
|
|
||||||
|
|
||||||
|
|
||||||
Run Python programs with spark-submit
|
|
||||||
=====================================
|
|
||||||
|
|
||||||
You can also run python scripts in Apache Spark from the command line. We'll use the Apache Spark example found in the :file:`/usr/share/apache-spark/examples/src/main/python/pi.py` file. Note that we have turned off the INFO and WARN messages in Apache Spark for this example.
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
#spark-submit /usr/share/apache-spark/examples/src/main/python/pi.py
|
|
||||||
Config directory: /usr/share/defaults/spark/
|
|
||||||
Pi is roughly 3.134700
|
|
||||||
|
|
||||||
DARS Usecase example
|
|
||||||
====================
|
|
||||||
|
|
||||||
The DARS container is used in conjunction with the Deep Learning Reference Stack container to implement a real world use case. Refer to the `Github Issue Classification`_ Usecase found in the `stacks-usecase`_ repository for a walkthrough. This usecase is implemented using the Scala environment, rather than PySpark.
|
|
||||||
|
|
||||||
|
|
||||||
Using Apache Hadoop in DARS
|
|
||||||
***************************
|
|
||||||
|
|
||||||
Apache Hadoop is an open source framework allowing for distributed processing of large data sets across clusters of computers using simple programming models. This framework is designed to scale up from a few servers to thousands of machines, each offering local computation and storage.
|
|
||||||
|
|
||||||
Single Node Hadoop Cluster Setup
|
|
||||||
================================
|
|
||||||
|
|
||||||
In this mode, all the daemons involved (e.g., the DataNode, NameNode, TaskTracker, JobTracker) run as Java processes on the same machine. This setup is useful for developing and testing Apache Hadoop applications.
|
|
||||||
|
|
||||||
The components of an Apache Hadoop Cluster are described below:
|
|
||||||
|
|
||||||
* NameNode manages HDFS storage. HDFS exposes a filesystem namespace and allows user data to be stored in files. Internally a file is split into one or more blocks and these blocks are stored in a set of DataNodes.
|
|
||||||
* DataNode is also known as Slave node. It is responsible for storing and managing the data in that node and responds to the NameNode for all filesystem operations.
|
|
||||||
* JobTracker is a master which creates and runs the job through tasktrackers. It also tracks resource availability and task lifecycle management.
|
|
||||||
* TaskTracker manages the processing resources on each worker node and send status updates to the JobTracker periodically.
|
|
||||||
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
=============
|
|
||||||
|
|
||||||
#. To setup a single node cluster, run a DARS container with the following flags:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --ulimit nofile=1000000:1000000 -ti --rm --network host clearlinux/stacks-dars-mkl cp -r -n /usr/share/defaults/hadoop/* /etc/hadoop
|
|
||||||
|
|
||||||
#. In the running container, set configuration in the :file:`/etc/hadoop/mapred-site.xml` file
|
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>mapreduce.framework.name</name>
|
|
||||||
<value>yarn</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>yarn.app.mapreduce.am.env</name>
|
|
||||||
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>mapreduce.map.env</name>
|
|
||||||
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>mapreduce.reduce.env</name>
|
|
||||||
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
||||||
#. Set up the :file:`/etc/hadoop/yarn-site.xml` as follows
|
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>yarn.nodemanager.aux-services</name>
|
|
||||||
<value>mapreduce_shuffle</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
|
|
||||||
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
||||||
Start the Apache Hadoop daemons
|
|
||||||
===============================
|
|
||||||
|
|
||||||
#. Format the NameNode server using this command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
hdfs namenode -format
|
|
||||||
|
|
||||||
#. Start the Apache Hadoop services
|
|
||||||
|
|
||||||
HDFS Namenode service :
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
hdfs --daemon start namenode
|
|
||||||
|
|
||||||
|
|
||||||
HDFS Datanode service :
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
hdfs --daemon start datanode
|
|
||||||
|
|
||||||
|
|
||||||
Yarn ResourceManager :
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
yarn --daemon start resourcemanager
|
|
||||||
|
|
||||||
|
|
||||||
Yarn NodeManager :
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
yarn --daemon start nodemanager
|
|
||||||
|
|
||||||
|
|
||||||
jobhistory service :
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
mapred --daemon start historyserver
|
|
||||||
|
|
||||||
#. Verify the nodes are alive with this command:
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
yarn node -list 2
|
|
||||||
|
|
||||||
Your output will look similar to:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
Total Nodes:1
|
|
||||||
Node-Id Node-State Node-Http-Address Number-of-Running-Containers
|
|
||||||
<hostname>:43489 RUNNING <hostname>:8042 0
|
|
||||||
|
|
||||||
|
|
||||||
Example application
|
|
||||||
===================
|
|
||||||
|
|
||||||
Apache Hadoop comes packages with a set of example applications. In this example we will show how to use the cluster to calculate Pi. The JAR file containing the compiled class can be found on your running DARS container at :file:`/usr/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.2.0.jar`
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
hadoop jar /usr/share/hadoop/mapreduce/hadoop-mapreduce-examples-$(hadoop version | grep Hadoop | cut -d ' ' -f2).jar pi 16 100
|
|
||||||
|
|
||||||
|
|
||||||
Deploy DARS on Kubernetes\*
|
|
||||||
***************************
|
|
||||||
|
|
||||||
Many containerized workloads are deployed in clusters managed by orchestration software like Kubernetes.
|
|
||||||
|
|
||||||
Prerequisites
|
|
||||||
=============
|
|
||||||
|
|
||||||
* A running Kubernetes cluster at version >= 1.6 with access configured to it using kubectl.
|
|
||||||
* You must have appropriate permissions to list, create, edit and delete pods in your cluster.
|
|
||||||
* The service account credentials used by the driver pods must be allowed to create pods, services and configmaps.
|
|
||||||
* You must have Kubernetes DNS configured in your cluster.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
To ensure that Kubernetes is correctly installed and configured for |CL|, follow the instructions in :ref:`kubernetes`.
|
|
||||||
|
|
||||||
|
|
||||||
#. For this example we will create the following Dockerfile
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
cat > $(pwd)/Dockerfile << 'EOF'
|
|
||||||
ARG DERIVED_IMAGE
|
|
||||||
FROM ${DERIVED_IMAGE}
|
|
||||||
|
|
||||||
RUN mkdir -p /etc/passwd /etc/pam.d /opt/spark/conf /opt/spark/work-dir
|
|
||||||
|
|
||||||
RUN set -ex && \
|
|
||||||
rm /bin/sh && \
|
|
||||||
ln -sv /bin/bash /bin/sh && \
|
|
||||||
touch /etc/pam.d/su \
|
|
||||||
echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
|
|
||||||
chgrp root /etc/passwd && chmod ug+rw /etc/passwd
|
|
||||||
|
|
||||||
RUN ln -s /usr/share/apache-spark/jars/ /opt/spark/ && \
|
|
||||||
ln -s /usr/share/apache-spark/bin/ /opt/spark/ && \
|
|
||||||
ln -s /usr/share/apache-spark/sbin/ /opt/spark/ && \
|
|
||||||
ln -s /usr/share/apache-spark/examples/ /opt/spark/ && \
|
|
||||||
ln -s /usr/share/apache-spark/kubernetes/tests/ /opt/spark/ && \
|
|
||||||
ln -s /usr/share/apache-spark/data/ /opt/spark/ && \
|
|
||||||
ln -s /etc/spark/* /opt/spark/conf/
|
|
||||||
|
|
||||||
COPY entrypoint.sh /opt/
|
|
||||||
ENV JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk
|
|
||||||
ENV PATH="${JAVA_HOME}/bin:${PATH}"
|
|
||||||
ENV SPARK_HOME /opt/spark
|
|
||||||
WORKDIR /opt/spark/work-dir
|
|
||||||
ENTRYPOINT [ "/opt/entrypoint.sh" ]
|
|
||||||
EOF
|
|
||||||
|
|
||||||
|
|
||||||
#. Create the :file:`entrypoint.sh` file. The Dockerfile requires an entrypoint script, to allow spark-submit to interact with the container.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
cat > $(pwd)/entrypoint.sh << 'EOF'
|
|
||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
# echo commands to the terminal output
|
|
||||||
set -ex
|
|
||||||
|
|
||||||
# Check whether there is a passwd entry for the container UID
|
|
||||||
myuid=$(id -u)
|
|
||||||
mygid=$(id -g)
|
|
||||||
# turn off -e for getent because it will return error code in anonymous uid case
|
|
||||||
set +e
|
|
||||||
uidentry=$(getent passwd $myuid)
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# If there is no passwd entry for the container UID, attempt to create one
|
|
||||||
if [ -z "$uidentry" ] ; then
|
|
||||||
if [ -w /etc/passwd ] ; then
|
|
||||||
echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd
|
|
||||||
else
|
|
||||||
echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
SPARK_K8S_CMD="$1"
|
|
||||||
case "$SPARK_K8S_CMD" in
|
|
||||||
driver | driver-py | driver-r | executor)
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
"")
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..."
|
|
||||||
exec /sbin/tini -s -- "$@"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
|
|
||||||
env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
|
|
||||||
readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
|
|
||||||
|
|
||||||
if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
|
|
||||||
SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$PYSPARK_FILES" ]; then
|
|
||||||
PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PYSPARK_ARGS=""
|
|
||||||
if [ -n "$PYSPARK_APP_ARGS" ]; then
|
|
||||||
PYSPARK_ARGS="$PYSPARK_APP_ARGS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
R_ARGS=""
|
|
||||||
if [ -n "$R_APP_ARGS" ]; then
|
|
||||||
R_ARGS="$R_APP_ARGS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then
|
|
||||||
pyv="$(python -V 2>&1)"
|
|
||||||
export PYTHON_VERSION="${pyv:7}"
|
|
||||||
export PYSPARK_PYTHON="python"
|
|
||||||
export PYSPARK_DRIVER_PYTHON="python"
|
|
||||||
elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
|
|
||||||
pyv3="$(python3 -V 2>&1)"
|
|
||||||
export PYTHON_VERSION="${pyv3:7}"
|
|
||||||
export PYSPARK_PYTHON="python3"
|
|
||||||
export PYSPARK_DRIVER_PYTHON="python3"
|
|
||||||
fi
|
|
||||||
|
|
||||||
case "$SPARK_K8S_CMD" in
|
|
||||||
driver)
|
|
||||||
CMD=(
|
|
||||||
"$SPARK_HOME/bin/spark-submit"
|
|
||||||
--conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
|
|
||||||
--deploy-mode client
|
|
||||||
"$@"
|
|
||||||
)
|
|
||||||
;;
|
|
||||||
driver-py)
|
|
||||||
CMD=(
|
|
||||||
"$SPARK_HOME/bin/spark-submit"
|
|
||||||
--conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
|
|
||||||
--deploy-mode client
|
|
||||||
"$@" $PYSPARK_PRIMARY $PYSPARK_ARGS
|
|
||||||
)
|
|
||||||
;;
|
|
||||||
driver-r)
|
|
||||||
CMD=(
|
|
||||||
"$SPARK_HOME/bin/spark-submit"
|
|
||||||
--conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
|
|
||||||
--deploy-mode client
|
|
||||||
"$@" $R_PRIMARY $R_ARGS
|
|
||||||
)
|
|
||||||
;;
|
|
||||||
executor)
|
|
||||||
CMD=(
|
|
||||||
${JAVA_HOME}/bin/java
|
|
||||||
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
|
|
||||||
-Xms$SPARK_EXECUTOR_MEMORY
|
|
||||||
-Xmx$SPARK_EXECUTOR_MEMORY
|
|
||||||
-cp "$SPARK_CLASSPATH"
|
|
||||||
org.apache.spark.executor.CoarseGrainedExecutorBackend
|
|
||||||
--driver-url $SPARK_DRIVER_URL
|
|
||||||
--executor-id $SPARK_EXECUTOR_ID
|
|
||||||
--cores $SPARK_EXECUTOR_CORES
|
|
||||||
--app-id $SPARK_APPLICATION_ID
|
|
||||||
--hostname $SPARK_EXECUTOR_POD_IP
|
|
||||||
)
|
|
||||||
;;
|
|
||||||
|
|
||||||
*)
|
|
||||||
echo "Unknown command: $SPARK_K8S_CMD" 1>&2
|
|
||||||
exit 1
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Execute the container CMD
|
|
||||||
exec "${CMD[@]}"
|
|
||||||
EOF
|
|
||||||
|
|
||||||
|
|
||||||
#. Make :file:`entrypoint.sh` executable
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo chmod +x $(pwd)/entrypoint.sh
|
|
||||||
|
|
||||||
#. Build the Docker image, for this example we will use dars_k8s_spark for the name of the image.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build . --build-arg DERIVED_IMAGE=clearlinux/stacks-dars-mkl -t dars_k8s_spark
|
|
||||||
|
|
||||||
|
|
||||||
#. Verify your built image. Execute the following command looking for the given name dars_k8s_spark
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker images | grep "dars_k8s_spark"
|
|
||||||
|
|
||||||
You should see something like:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
dars_k8s_spark latest 1fa3278a3421 1 minutes ago 6.56GB
|
|
||||||
|
|
||||||
#. Use a variable to store the image's given name:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
DARS_K8S_IMAGE=dars_k8s_spark
|
|
||||||
|
|
||||||
|
|
||||||
Configure RBAC
|
|
||||||
==============
|
|
||||||
|
|
||||||
Create the Spark service account and cluster role binding to allow Spark on Kubernetes to create Executors as required. For this example use the default namespace.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
kubectl create serviceaccount spark-serviceaccount --namespace default
|
|
||||||
kubectl create clusterrolebinding spark-rolebinding --clusterrole=edit --serviceaccount=default:spark-serviceaccount --namespace=default
|
|
||||||
|
|
||||||
|
|
||||||
Prepare to Submit the Spark Job
|
|
||||||
===============================
|
|
||||||
|
|
||||||
#. Determine the Kubernetes master address:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
kubectl cluster-info
|
|
||||||
|
|
||||||
You should see something like:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
Kubernetes master is running at https://192.168.39.127:8443
|
|
||||||
|
|
||||||
#. Use a variable to store the master address:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
MASTER_ADDRESS='https://192.168.39.127:8443'
|
|
||||||
|
|
||||||
#. Submit the Spark Job on Minikube using the MASTER_ADDRESS and DARS_K8S variables. The driver pod will be called spark-pi-driver.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
spark-submit \
|
|
||||||
--master k8s://${MASTER_ADDRESS} \
|
|
||||||
--deploy-mode cluster \
|
|
||||||
--name spark-pi \
|
|
||||||
--class org.apache.spark.examples.SparkPi \
|
|
||||||
--conf spark.executor.instances=2 \
|
|
||||||
--conf spark.kubernetes.container.image=${DARS_K8S_IMAGE} \
|
|
||||||
--conf spark.kubernetes.driver.pod.name=spark-pi-driver \
|
|
||||||
--conf spark.kubernetes.namespace=default \
|
|
||||||
--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-serviceaccount \
|
|
||||||
local:///usr/share/apache-spark/examples/jars/spark-examples_2.12-2.4.0.jar
|
|
||||||
|
|
||||||
|
|
||||||
#. Check the Job. Read the logs and look for the Pi result:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
kubectl logs spark-pi-driver | grep "Pi is roughly"
|
|
||||||
|
|
||||||
You should see something like:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
Pi is roughly 3.1418957094785473
|
|
||||||
|
|
||||||
More information about spark-submit configuration is available in the `running-on-kubernetes`_ documentation.
|
|
||||||
|
|
||||||
|
|
||||||
Troubleshooting
|
|
||||||
***************
|
|
||||||
|
|
||||||
Dropped or refused connection
|
|
||||||
=============================
|
|
||||||
|
|
||||||
If Pyspark / Spark-shell warns of a dropped connection exception or Connection refused, check if the `HADOOP_CONF_DIR` environment variable is set. These APIs assume they will use Hadoop Distributed File System.
|
|
||||||
You can unset `HADOOP_CONF_DIR` and use Spark RDDs, or start Hadoop services and then create your directories and files as required using hdfs.
|
|
||||||
|
|
||||||
It is also possible to change the file system to local without unsetting `HADOOP_CONF_DIR` using one of these commands.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
pyspark --conf "spark.hadoop.fs.defaultFS=file:///"
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
spark-shell --conf "spark.hadoop.fs.defaultFS=file:///"
|
|
||||||
|
|
||||||
Using Spark with proxy settings
|
|
||||||
===============================
|
|
||||||
|
|
||||||
There are two ways to work with proxies:
|
|
||||||
|
|
||||||
#. Add the following line to :file:`$SPARK_CONF_DIR/spark-defaults.conf` for both `spark.executor.extraJavaOptions` and `spark.driver.extraJavaOptions` variables:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
-Dhttp.proxyHost=<URL> -Dhttp.proxyPort=<PORT> -Dhttps.proxyHost=<URL> -Dhttps.proxyPort=<PORT>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#. Give the proxies URL and Port as a configuration parameter
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
pyspark --conf "spark.hadoop.fs.defaultFS=file:///" --conf "spark.driver.extraJavaOptions=-Dhttp.proxyHost=example.proxy -Dhttp.proxyPort=111 -Dhttps.proxyHost=example.proxy -Dhttps.proxyPort=112"
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
spark-shell --conf "spark.hadoop.fs.defaultFS=file:///" --conf "spark.driver.extraJavaOptions=-Dhttp.proxyHost=example.proxy -Dhttp.proxyPort=111 -Dhttps.proxyHost=example.proxy -Dhttps.proxyPort=112"
|
|
||||||
|
|
||||||
|
|
||||||
Known issues
|
|
||||||
============
|
|
||||||
|
|
||||||
#. There is an exception message `Unrecognized Hadoop major version number: 3.2.0 at org.apache.hadoop.hive.shims.ShimLoader.getMajorVersion.`
|
|
||||||
|
|
||||||
This exception can be disregarded because DARS does not use hadoop.hive.shims. Hive binaries installed from Apache on |CL| with JDK11 does not work at the time of this writing.
|
|
||||||
|
|
||||||
#. There is an exception message `Exception in thread "Thread-3" java.lang.ExceptionInInitializerError at org.apache.hadoop.hive.conf.HiveConf` This is related to the same issue with |CL| and JDK11 noted above, and does not affect DARS for the same reason.
|
|
||||||
|
|
||||||
*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
|
|
||||||
|
|
||||||
.. _Data Analytics Reference Stack: https://github.com/intel/stacks/tree/master/dars/clearlinux
|
|
||||||
|
|
||||||
.. _Docker Hub: https://hub.docker.com/
|
|
||||||
|
|
||||||
.. _OpenBLAS: http://www.openblas.net/
|
|
||||||
|
|
||||||
.. _Intel® Math Kernel Library: https://software.intel.com/en-us/mkl
|
|
||||||
|
|
||||||
.. _CentOS: https://www.centos.org/
|
|
||||||
|
|
||||||
.. _DARS with OpenBLAS: https://hub.docker.com/r/clearlinux/stacks-dars-openblas/
|
|
||||||
|
|
||||||
.. _DARS with Intel® MKL: https://hub.docker.com/r/clearlinux/stacks-dars-mkl/
|
|
||||||
|
|
||||||
.. _DARS architecture and performance benchmarks: https://clearlinux.org/stacks/data-analytics-stack-v1
|
|
||||||
|
|
||||||
.. _DARS Terms of Use: https://clearlinux.org/stacks/data-analytics/terms-of-use
|
|
||||||
|
|
||||||
.. _PySpark getting started guide: https://towardsdatascience.com/how-to-get-started-with-pyspark-1adc142456ec
|
|
||||||
|
|
||||||
.. _Apache Spark documentation: https://spark.apache.org/docs/latest/
|
|
||||||
|
|
||||||
.. _stacks-usecase: https://github.com/intel/stacks-usecase
|
|
||||||
|
|
||||||
.. _Github Issue Classification: https://github.com/intel/stacks-usecase/tree/master/github-issue-classification
|
|
||||||
|
|
||||||
.. _running-on-kubernetes: https://spark.apache.org/docs/latest/running-on-kubernetes.html#configuration
|
|
||||||
@@ -1,714 +0,0 @@
|
|||||||
.. _dbrs:
|
|
||||||
|
|
||||||
Database Reference Stack
|
|
||||||
########################
|
|
||||||
|
|
||||||
This guide describes the hardware and installation requirements for using the
|
|
||||||
:abbr:`DBRS (Database Reference Stack)`, along with getting started configuration examples, using |CL-ATTR| as the host system.
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
Overview
|
|
||||||
********
|
|
||||||
|
|
||||||
The Database Reference Stack is integrated, highly-performant, open source,
|
|
||||||
and optimized for 2nd generation Intel® Xeon® Scalable processors and Intel®
|
|
||||||
Optane™ persistent memory. This open source community release is part of
|
|
||||||
an effort to ensure developers have easy access to the features and
|
|
||||||
functionality of Intel Platforms.
|
|
||||||
|
|
||||||
Stack Features
|
|
||||||
==============
|
|
||||||
|
|
||||||
Current supported database applications are Apache Cassandra* and Redis*, which
|
|
||||||
have been enabled for `Intel Optane PMM`_.
|
|
||||||
|
|
||||||
DBRS with Apache Cassandra can be deployed as a standalone container or inside a
|
|
||||||
Kubernetes* cluster.
|
|
||||||
|
|
||||||
The Redis stack application is enabled for a multinode Kubernetes
|
|
||||||
environment, using AEP PMem DIMM in fsdax mode for storage.
|
|
||||||
|
|
||||||
Releases
|
|
||||||
********
|
|
||||||
|
|
||||||
Refer to the `Database Reference Stack website`_ for information and download links for the different versions and offerings of the stack.
|
|
||||||
|
|
||||||
The release announcement for each release provides more detail about the stack features, as well as benchmark results.
|
|
||||||
|
|
||||||
* `DBRS V2.0`_ release announcement.
|
|
||||||
* `DBRS V1.0`_ release announcement.
|
|
||||||
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The Database Reference Stack is a collective work, and each piece
|
|
||||||
of software within the work has its own license. Please see the
|
|
||||||
`DBRS Terms of Use`_ for more details about licensing and usage of the Database Reference Stack.
|
|
||||||
|
|
||||||
|
|
||||||
Hardware Requirements
|
|
||||||
*********************
|
|
||||||
|
|
||||||
* Intel Xeon Scalable platform with Intel® C620 series chipset
|
|
||||||
* 2nd Gen Intel Xeon Scalable processor CPU (Intel® Optane™ PMem-enabled stepping). Provides cache & memory control. Intel Optane PMem works only on systems powered by 2nd Generation Intel® Xeon® Platinum or Intel® Xeon® Gold processors.
|
|
||||||
* BIOS with Reference Code
|
|
||||||
* Intel Optane PMem
|
|
||||||
|
|
||||||
Hardware configuration used in stacks development
|
|
||||||
=================================================
|
|
||||||
|
|
||||||
* Intel® Server System R2208WFTZSR
|
|
||||||
* BIOS with Reference Code
|
|
||||||
* BIOS ID: SE5C620.86B.0D.01.0438.032620191658
|
|
||||||
* BMC Firmware: 1.94.6b42b91d
|
|
||||||
* Intel Optane PMem Firmware: 1.2.0.5310
|
|
||||||
* 2x Intel Xeon Platinum 8268 Processor
|
|
||||||
* Intel® SSD Data Center Family S5600 Series 960GB 2.5in SATA Drive
|
|
||||||
* 64 GB RAM - Distributed in 4x 16 GB DDR4 DIMM's
|
|
||||||
* 2x Intel Optane PMem 256GB Module
|
|
||||||
* 1-1-1 Layout 8 Intel Optane : 1 RAM ratio
|
|
||||||
|
|
||||||
|
|
||||||
.. list-table:: **Table 1. IMC**
|
|
||||||
:widths: 16,16,16,16,16,16
|
|
||||||
:header-rows: 1
|
|
||||||
|
|
||||||
* - Channel 2
|
|
||||||
- Channel 2
|
|
||||||
- Channel 1
|
|
||||||
- Channel 1
|
|
||||||
- Channel 0
|
|
||||||
- Channel 0
|
|
||||||
|
|
||||||
* - Slot 1
|
|
||||||
- Slot 0
|
|
||||||
- Slot 1
|
|
||||||
- Slot 0
|
|
||||||
- Slot 1
|
|
||||||
- Slot 0
|
|
||||||
|
|
||||||
* -
|
|
||||||
- 256 GB DCPMM
|
|
||||||
-
|
|
||||||
- 16 GB DRAM
|
|
||||||
-
|
|
||||||
- 16 GB DRAM
|
|
||||||
|
|
||||||
Firmware configuration
|
|
||||||
**********************
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
When updating DCPMM Firmware, all DCPMM parts must be in the same mode (you cannot mix 1LM and 2LM parts).
|
|
||||||
|
|
||||||
The latest firmware download for the Intel® Server Board S2600WF Family is available at the `Intel Download Center`_
|
|
||||||
|
|
||||||
Firmware Update Steps
|
|
||||||
=====================
|
|
||||||
|
|
||||||
#. Unzip the contents of the update package and copy all files to the root directory of a removable media (USB flash drive).
|
|
||||||
#. Insert the USB flash drive to any available USB port on the system to be updated.
|
|
||||||
#. Boot to EFI shell.
|
|
||||||
#. Input "fsx(x:0,1,...):" to enter into your usb disk
|
|
||||||
#. Run "startup.nsh"
|
|
||||||
#. After update BMC firmware, system BIOS, ME firmware,FD, FRUSDR, system will reboot automatically.
|
|
||||||
|
|
||||||
|
|
||||||
If Intel Optane PMem is installed, run startup.nsh a second time after the first reboot to upgrade Intel Optane PMem Firmware:
|
|
||||||
|
|
||||||
* Boot to EFI shell.
|
|
||||||
* Input "fsx(x:0,1,...):" to enter into your usb disk
|
|
||||||
* Run "startup.nsh" again to update the corresponding AEP FW.
|
|
||||||
|
|
||||||
.. _dbrs-hardware-configuration:
|
|
||||||
|
|
||||||
Hardware Configuration
|
|
||||||
**********************
|
|
||||||
|
|
||||||
|
|
||||||
Online Resources
|
|
||||||
================
|
|
||||||
|
|
||||||
Before going through the configuration steps, we strongly recommend visiting the following resources and wikis to have a broader understanding of what is being done
|
|
||||||
|
|
||||||
* `Quick Start Guide`_ Configure Intel Optane PMem Modules on Linux
|
|
||||||
* `Managing NVDIMMs`_
|
|
||||||
* `Configure, Manage, and Profile`_ Intel Optane PMem Modules
|
|
||||||
|
|
||||||
Optane™ DIMM Configuration
|
|
||||||
==========================
|
|
||||||
|
|
||||||
The PMem DIMMs can be configured in devdax or fsdax mode. The use case to enable database stack on a kubernetes environment currently only support fsdax mode.
|
|
||||||
|
|
||||||
Configuration Steps
|
|
||||||
===================
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
Run the following steps with root privileges (sudo) as shown in the examples
|
|
||||||
|
|
||||||
|
|
||||||
#. To configure Optane™ DIMMs for App direct mode run this command
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ipmctl create -goal PersistentMemoryType=AppDirect
|
|
||||||
|
|
||||||
#. Verify the Optane™ Configuration by showing the defined region, then reboot the system for your changes to take effect
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ipmctl show -region
|
|
||||||
|
|
||||||
|
|
||||||
#. Next, list the defined namespaces for the pmem devices in the system. If they are not defined, create them as shown in the following step.
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ndctl list -N
|
|
||||||
|
|
||||||
|
|
||||||
#. Create namespaces based on the regions and set mode as fsdax -- use the names of the regions listed in previous step as the –-region parameter (default is region0 and region1; one for each CPU socket)
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ndctl create-namespace --region=region0 --mode=fsdax
|
|
||||||
sudo ndctl create-namespace --region=region1 --mode=fsdax
|
|
||||||
|
|
||||||
|
|
||||||
#. Create the filesystem and mount it. We are using /mnt/dax{#} as a convention in this guide to mount our devices
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo mkfs.ext4 /dev/pmem0
|
|
||||||
sudo mount -o dax /dev/pmem0 /mnt/dax0
|
|
||||||
sudo mkfs.ext4 /dev/pmem1
|
|
||||||
sudo mount -o dax /dev/pmem1 /mnt/dax1
|
|
||||||
|
|
||||||
|
|
||||||
Running DBRS with Apache Cassandra*
|
|
||||||
***********************************
|
|
||||||
|
|
||||||
DBRS with Apache Cassandra can be deployed as a standalone container or inside
|
|
||||||
Kubernetes\*. Instructions for both cases is included here. Note that you can
|
|
||||||
use the released `Docker image with Apache Cassandra`_ (Docker\* examples below).
|
|
||||||
These instructions provide a baseline for creating your own container image.
|
|
||||||
If you are using the released image, skip this section.
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
At the initial release of DBRS, Apache Cassandra is considered to be Engineering Preview release quality and may not be suitable for production release. Please take this into consideration when planning your project.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Build the DBRS with Apache Cassandra container
|
|
||||||
==============================================
|
|
||||||
|
|
||||||
To build the container with Apache Cassandra, you must build cassandra-pmem, and then build the container using the :command:`docker build` command. We are using |CL| as our container host as well as the OS in the container.
|
|
||||||
|
|
||||||
Build cassandra-pmem
|
|
||||||
====================
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
At the initial release of DBRS, the pmem-csi driver is considered to be Engineering Preview release quality and may not be suitable for production release. Please take this into consideration when planning your project.
|
|
||||||
|
|
||||||
|
|
||||||
In the `DBRS github repository`_, there is a file called `build-cassandra-pmem.sh`_, which handles all the requirements for compiling cassandra-pmem for Dockerfile usage. The dependencies for this build can be installed with :command:`swupd`.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo swupd bundle-add c-basic java-basic devpkg-pmdk pmdk
|
|
||||||
|
|
||||||
|
|
||||||
Once installed, we run the script
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
./build-cassandra-pmem.sh
|
|
||||||
|
|
||||||
|
|
||||||
At the completion of the build you will have a file called :file:`cassandra-pmem-build.tar.gz`. Place this file in the same directory with the Dockerfile to build the Docker image.
|
|
||||||
|
|
||||||
Build the Docker container
|
|
||||||
==========================
|
|
||||||
|
|
||||||
To build the Docker image, run the Dockerfile in the same directory with the :file:`cassandra-pmem-build.tar.gz`
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build --force-rm --no-cache -f Dockerfile -t $build_image_name .
|
|
||||||
|
|
||||||
|
|
||||||
Once it completes, the Docker image is ready to be used.
|
|
||||||
|
|
||||||
Deploy Apache Cassandra PMEM as a standalone container
|
|
||||||
======================================================
|
|
||||||
|
|
||||||
Requirements
|
|
||||||
------------
|
|
||||||
|
|
||||||
To deploy Apache Cassandra PMEM, you must meet the following requirements
|
|
||||||
|
|
||||||
* PMEM memory must be configured in `devdax` or `fsdax` mode. The container image is able to handle both modes and depending on the PMEM mode, the mount points inside the container must be different.
|
|
||||||
* In order to make available `devdax` pmem devices inside the container you must use the `--device` directive. Internally the container always uses :command:`/dev/dax0.0`, so the mapping should be: :command:`--device=/dev/<host-device>:/dev/dax0.0`
|
|
||||||
* In a similar fashion for `fsdax` we need the device to be mapped to :command:`/mnt/pmem` inside the container: :command:`--mount type=bind,source=<source-mount-point>,target=/mnt/pmem`
|
|
||||||
|
|
||||||
|
|
||||||
Preparing PMEM for container use
|
|
||||||
--------------------------------
|
|
||||||
|
|
||||||
The cassandra-pmem image is capable of using both `fsdax` and `devdax`, the necessary steps to configure the PMEM to work with cassandra are documented here.
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: devdax
|
|
||||||
|
|
||||||
We need to verify the device we want to use is in `devdax` mode
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ndctl create-namespace -fe namespace0.0 --mode=devdax
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
{
|
|
||||||
"dev":"namespace0.0",
|
|
||||||
"mode":"devdax",
|
|
||||||
"map":"dev",
|
|
||||||
"size":"3.94 GiB (4.23 GB)",
|
|
||||||
"uuid":"cb738cc7-711d-4578-bebf-1f7ba02ca169",
|
|
||||||
"daxregion":{
|
|
||||||
"id":0,
|
|
||||||
"size":"3.94 GiB (4.23 GB)",
|
|
||||||
"align":2097152,
|
|
||||||
"devices":[
|
|
||||||
{
|
|
||||||
"chardev":"dax0.0",
|
|
||||||
"size":"3.94 GiB (4.23 GB)"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"align":2097152
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
If needed, we can reconfigure it using :command:`ndctl create-namespace -fe <namespace-name> --mode=devdax`.
|
|
||||||
|
|
||||||
Before using a `devdax` device we need to clear the device:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo pmempool rm -vaf /dev/dax0.0
|
|
||||||
|
|
||||||
|
|
||||||
The `jvm.options` configuration for Apache Cassandra should look like the following:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
-Dpmem_path=/dev/dax0.0
|
|
||||||
-Dpool_size=0
|
|
||||||
|
|
||||||
Where
|
|
||||||
* pmem_path is the `devdax` device.
|
|
||||||
* pool_size=0 indicates to use the entire `devdax` device.
|
|
||||||
|
|
||||||
When using the `Docker image with Apache Cassandra`_, the file `jvm.options` is automatically populated.
|
|
||||||
|
|
||||||
.. group-tab:: fsdax
|
|
||||||
|
|
||||||
Verify that the PMEM is in `fsdax` mode
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo ndctl list -u
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
{
|
|
||||||
"dev":"namespace0.0",
|
|
||||||
"mode":"fsdax",
|
|
||||||
"map":"mem",
|
|
||||||
"size":"4.00 GiB (4.29 GB)",
|
|
||||||
"sector_size":512,
|
|
||||||
"blockdev":"pmem0"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
If for some reason the device is not in `fsdax` mode you can reconfigure the namespace as follows:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo `ndctl create-namespace -fe <namespace-name> --mode=fsdax`
|
|
||||||
|
|
||||||
|
|
||||||
Once the PMEM namespace is configured, you will see a device named :file:`/dev/pmem{0-9}`. We will create a filesystem on that device. The filesystem could be `ext4` or `xfs`, for this example we are going to use `ext4`.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo mkfs.ext4 /dev/pmem0
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
mke2fs 1.45.2 (27-May-2019)
|
|
||||||
Creating filesystem with 1031680 4k blocks and 258048 inodes
|
|
||||||
Filesystem UUID: 303c03f5-ac4e-4462-8bf9-bc6b0fae53fe
|
|
||||||
Superblock backups stored on blocks:
|
|
||||||
32768, 98304, 163840, 229376, 294912, 819200, 884736
|
|
||||||
|
|
||||||
Allocating group tables: done
|
|
||||||
Writing inode tables: done
|
|
||||||
Creating journal (16384 blocks): done
|
|
||||||
Writing superblocks and filesystem accounting information: done
|
|
||||||
|
|
||||||
|
|
||||||
Once the filesystem is created, we mount it with the dax option
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo mount /dev/pmem0 /mnt/pmem -o dax
|
|
||||||
|
|
||||||
|
|
||||||
When using `fsdax` mode cassandra-pmem creates a pool file on the pmem mountpoint, so the `jvm.options` configuration should look like the output below:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
-Dpmem_path=/mnt/pmem/cassandra_pool
|
|
||||||
-Dpool_size=3221225472
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Where
|
|
||||||
* `pmem_path` is the path to the pool file, which should include the path itself and the file name
|
|
||||||
* `pool_size` is the size of the pool file in bytes. If you are using the `Docker image with Apache Cassandra`_ you can pass this value as an environment variable to the container runtime in Gb and the calculation is done automatically.
|
|
||||||
|
|
||||||
Is important to note that when creating the filesystem in the pmem device certain amount of space of the device is used by the filesystem metadata so the pool_size should be smaller than the total pmem namespace size.
|
|
||||||
|
|
||||||
When using the `Docker image with Apache Cassandra`_, the file `jvm.options` is automatically populated with the environment variables `CASSANDRA_PMEM_POOL_NAME` and `CASSANDRA_FSDAX_POOL_SIZE_GB`.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Run the DBRS Container
|
|
||||||
======================
|
|
||||||
|
|
||||||
Replace `<image-id>` in the following commands with the name of the image you are using.
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: devdax
|
|
||||||
|
|
||||||
In `devdax` mode:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --device=/<devdax-device>:/dev/dax0.0 --ulimit nofile=262144:262144 -p 9042:9042 -p 7000:7000 -it --name cassandra-test <image-id>
|
|
||||||
|
|
||||||
.. group-tab:: fsdax
|
|
||||||
|
|
||||||
In `fsdax` mode:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/<fsdax-mountpoint>,target=/mnt/pmem --ulimit nofile=262144:262144 -p 9042:9042 -p 7000:7000 -it -e 'CASSANDRA_FSDAX_POOL_SIZE_GB=<fsdax-pool-size-in-gb>' --name cassandra-test <image-id>
|
|
||||||
|
|
||||||
|
|
||||||
Container Configuration
|
|
||||||
=======================
|
|
||||||
|
|
||||||
Using environment variables
|
|
||||||
---------------------------
|
|
||||||
|
|
||||||
The container listens on the primary container IP address, but if required, some parameters can be provided as environment variables using `--env`.
|
|
||||||
|
|
||||||
* `CASSANDRA_CLUSTER_NAME` Cassandra cluster name, by default `Cassandra Cluster`
|
|
||||||
* `CASSANDRA_LISTEN_ADDRESS` Cassandra listen address
|
|
||||||
* `CASSANDRA_RPC_ADDRESS` Cassandra RPC address
|
|
||||||
* `CASSANDRA_SEED_ADDRESSES` A comma separated list of hosts in the cluster, if not provided, cassandra is going to run as a single node.
|
|
||||||
* `CASSANDRA_SNITCH` The snitch type for the cluster, by default it is `SimpleSnitch`, for more complex snitches you can mount your own `cassandra-rackdc.properties` file.
|
|
||||||
* `LOCAL_JMX` If set to `no` the JMX service will listen on all IP addresses, the default is `yes` and listens just on localhost 127.0.0.1
|
|
||||||
* `JVM_OPTS` When set you can pass additional arguments to the JVM for cassandra execution, for example for specifying memory heap sizes `JVM_OPTS=-Xms16G -Xmx16G -Xmn12G`
|
|
||||||
|
|
||||||
When using PMEM in `fsdax` mode, there are some parameters to control the allocation of memory:
|
|
||||||
|
|
||||||
|
|
||||||
* `CASSANDRA_FSDAX_POOL_SIZE_GB` The size of the fsdax pool in GB, if it is not specified the pool size is `1`
|
|
||||||
* `CASSANDRA_PMEM_POOL_NAME` The filename of the pool created in PMEM, by default `cassandra_pool`
|
|
||||||
|
|
||||||
Using custom files
|
|
||||||
------------------
|
|
||||||
|
|
||||||
For more complex deployments it is also possible to provide custom `cassandra.yaml` and `jvm.options` files as shown below:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/<fsdax-mountpoint>,target=/mnt/pmem -it --ulimit nofile=262144:262144 --mount type=bind,source=/<path-to-file>/cassandra.yaml,target=/workspace/cassandra/conf/cassandra.yaml --mount type=bind,source=/path-to-file>/jvm.options,target=/workspace/cassandra/conf/jvm.options --name cassandra-custom-files
|
|
||||||
|
|
||||||
|
|
||||||
Clustering
|
|
||||||
==========
|
|
||||||
|
|
||||||
For a simple two node cluster using PMEM in `fsdax` mode on both containers:
|
|
||||||
|
|
||||||
Node 1
|
|
||||||
------
|
|
||||||
|
|
||||||
* IP: 172.17.0.2
|
|
||||||
* PMEM mountpoint: /mnt/pmem1
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/mnt/pmem1,target=/mnt/pmem --ulimit nofile=262144:262144 -it -e 'CASSANDRA_FSDAX_POOL_SIZE_GB=2' -e 'CASSANDRA_SEED_ADDRESSES=172.17.0.2:7000,172.17.0.3:7000' --name cassandra-node1 <image-id>
|
|
||||||
|
|
||||||
|
|
||||||
Node 2
|
|
||||||
------
|
|
||||||
|
|
||||||
* IP: 172.17.0.3
|
|
||||||
* PMEM mountpoint: /mnt/pmem2
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/mnt/pmem2,target=/mnt/pmem --ulimit nofile=262144:262144 -it -e 'CASSANDRA_FSDAX_POOL_SIZE_GB=2' -e 'CASSANDRA_SEED_ADDRESSES=172.17.0.2:7000,172.17.0.3:7000' --name cassandra-node2 <image-id>
|
|
||||||
|
|
||||||
|
|
||||||
Once both nodes are running, eventually the gossip is settled and we can use `nodetool` on either container to check cluster status.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker exec -it <container-id> bash /workspace/cassandra/bin/nodetool status
|
|
||||||
|
|
||||||
|
|
||||||
The output should look similar to this:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
|
|
||||||
Datacenter: datacenter1
|
|
||||||
=======================
|
|
||||||
Status=Up/Down
|
|
||||||
|/ State=Normal/Leaving/Joining/Moving
|
|
||||||
-- Address Load Tokens Owns (effective) Host ID Rack
|
|
||||||
UN 172.17.0.3 0 bytes 256 100.0% 22387159-8192-41cf-8b6c-8bf0e1049eb7 rack1
|
|
||||||
UN 172.17.0.2 0 bytes 256 100.0% 219b56ba-c07c-400b-a018-a5dc20edeb09 rack1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Persistence
|
|
||||||
===========
|
|
||||||
|
|
||||||
By default you can access the data written to Apache Cassandra as long as the container exists. In order to persist the data past that, you can mount volumes or bind mounts on :file:`/workspace/cassandra/data` and :file:`/workspace/cassandra/logs` and in this way the data can still be accessed once the container is deleted.
|
|
||||||
|
|
||||||
Deploy An Apache Cassandra-PMEM cluster on Kubernetes*
|
|
||||||
******************************************************
|
|
||||||
|
|
||||||
Many containerized workloads are deployed in clusters and orchestration software like Kubernetes can be useful. We will use the `cassandra-pmem-helm`_ Helm* chart in this example.
|
|
||||||
|
|
||||||
Requirements
|
|
||||||
============
|
|
||||||
|
|
||||||
* Kubectl* must be configured to access the Kubernetes Cluster
|
|
||||||
|
|
||||||
* A Kubernetes cluster with `pmem-csi`_ enabled
|
|
||||||
|
|
||||||
* The Kubernetes cluster must have `helm`_ and tiller installed
|
|
||||||
|
|
||||||
* PMEM hardware
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
When selecting the `fsdax` pool file size, it is important to consider that when requesting a volume, certain amount of space is used by the filesystem metadata on that volume and the available space turns out to be less than total amount specified. Taking this into consideration the size of the fsdax pool file should be ~2G less than the total volume size requested.
|
|
||||||
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
=============
|
|
||||||
|
|
||||||
In order to configure the Apache Cassandra PMEM cluster some variables and values are provided. These values are set in :file:`test/cassandra-pmem-helm/values.yaml`, and can be modified according to your specific needs. A summary of those parameters is shown below:
|
|
||||||
|
|
||||||
|
|
||||||
* clusterName: The cluster Name set across all deployed nodes
|
|
||||||
* replicaCount: The number of nodes in the cluster to be deployed
|
|
||||||
* image.repository: The address of the container registry where the cassandra-pmem image should be pulled
|
|
||||||
* image.tag: The tag of the image to be pulled during deployment
|
|
||||||
* image.name: The name of the image to be pulled during deployment
|
|
||||||
* pmem.containerPmemAllocation: The size of the persistent volume claim to be used as heap, it uses the storage class `pmem-csi-sc-ext4` from pmem-csi The size of the fsdax pool to be created inside the persistent volume claim, in practice it should be `1G` less than pmem.containerPmemAllocation
|
|
||||||
* pmem.fsdaxPoolSizeInGB: The size of the fsdax pool to be created inside the persistent volume claim, in practice it should be 1G less than pmem.containerPmemAllocation
|
|
||||||
* enablePersistence: If set to `true`, K8s persistent volumes are deployed to store data and logs
|
|
||||||
* persistentVolumes.logsVolumeSize: The size of the persistent volume used for storing logs on each node, the default is `4G`
|
|
||||||
* persistentVolumes.dataVolumeSize: The size of the persistent volume used for storing data on each node, the default is `4G`
|
|
||||||
* persistentVolumes.logsStorageClass: Storage class used by the logs pvc, by default it uses `pmem-csi-sc-ext4`
|
|
||||||
* persistentVolumes.dataStorageClass: Storage class used by the data pvc, by default it uses `pmem-csi-sc-ext4`
|
|
||||||
* provideCustomConfig: If set to `true`, it mounts all the files located on `<helm-chart-dir>/files/conf` on `/workspace/cassandra/conf` inside each container in order to provide a way to customize the deployment beyond the options provided here
|
|
||||||
* exposeJmxPort: When set to `true` it exposes the JMX port as part of the Kubernetes headless service. It should be used together with `enableAdditionalFilesConfigMap` in order to provide authentication files needed for JMX when the remote connections are allowed. When set to `false` only local access through 127.0.0.1 is granted and no additional authentication is needed.
|
|
||||||
* enableClientToolsPod: If set to `true`, an additional pod independent from the cluster is deployed, this pod contains various Cassandra client tools and mounts test profiles located under `<helm-chart-dir>/files/testProfiles` to `/testProfiles` inside the pod. This pod is useful to test and launch benchmarks
|
|
||||||
* enableAdditionalFilesConfigMap: When set to true, it takes the files located in `<helm-chart-dir>/files/additionalFiles` and mount them in `/etc/cassandra` inside the pods, some additional files for cassandra can be stored here, such as JMX auth files
|
|
||||||
* jvmOpts.enabled: If set to `true` the environment variable `JVM_OPTS` is overridden with the value provided on jvmOpts.value
|
|
||||||
* jvmOpts.value: Sets the value of the environment variable `JVM_OPTS`, in this way some java runtime configurations can be provided such as RAM heap usage
|
|
||||||
* resources.enabled: if set to `true`, the resource constraints are set on each pod using the values under resources.requests and resources.limits
|
|
||||||
* resources.requests.memory: Initial resource allocation for each pod in the cluster
|
|
||||||
* resources.request.cpu: Initial resource allocation for each pod in the cluster
|
|
||||||
* resources.limits.memory: Limits for memory allocation for each pod in the cluster
|
|
||||||
* resources.limits.cpu: Limits for cpu allocation for each pod in the cluster
|
|
||||||
|
|
||||||
Installation
|
|
||||||
============
|
|
||||||
|
|
||||||
Once all the configurations are set, to install the chart inside a given Kubernetes cluster you must run:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
helm install ./cassandra-pmem-helm
|
|
||||||
|
|
||||||
|
|
||||||
Eventually all the given nodes will be shown as running using :command:`kubectl get pods`.
|
|
||||||
|
|
||||||
|
|
||||||
Running DBRS with Redis
|
|
||||||
***********************
|
|
||||||
|
|
||||||
The Redis stack application is enabled for a multinode Kubernetes environment using Intel Optane DCPMM PMem DIMMs in fsdax mode for storage.
|
|
||||||
|
|
||||||
The source code used for this application can be found in the `Github repository`_
|
|
||||||
|
|
||||||
The following examples will use the `Docker image with Redis`_. You can also build your own image with Docker by using the :file:`Dockerfile` and running with this command
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build --force-rm --no-cache -f Dockerfile -t ${DOCKER_IMAGE} .
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Single node
|
|
||||||
===========
|
|
||||||
|
|
||||||
Prior to starting the container, you will need to have the Intel Optane DCPMM module in fsdax with a file system and mounted in `/mnt/dax0` as shown above.
|
|
||||||
|
|
||||||
Use the following to start the container, replacing ${DOCKER_IMAGE} with the name of the image you are using.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/mnt/dax0,target=/mnt/pmem0 -i -d --name pmem-redis ${DOCKER_IMAGE} --nvm-maxcapacity 200 --nvm-dir /mnt/pmem0 --nvm-threshold 64 --protected-mode no
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Redis Operator in a Kubernetes cluster
|
|
||||||
======================================
|
|
||||||
|
|
||||||
After setting up :ref:`kubernetes` in |CL|, you will need to enable it to support DCPMM using the pmem-cls driver. To install the driver follow the instructions in the `pmem-csi`_ repository.
|
|
||||||
|
|
||||||
We are using source code from the `Redis operator`_ .
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
If you already have a redis-operator, you will need to delete it before installing a new one.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
After installing the operator you are ready to deploy redisfailover instances using a yaml file, like this `example for persistent memory`_. You can download it and change the source of the image to reflect your environment. We have named our yaml `redis-failover.yml`
|
|
||||||
|
|
||||||
To start a redisfailover instance in Kubernetes run the following
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
kubectl create -f redis-failover.yml
|
|
||||||
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
There is a `known issue`_ in which the sentinels do not have enough memory to create the InitContainer. The current workaround is to build the image increasing the limits for the InitContainer memory to 32Mb
|
|
||||||
|
|
||||||
Running DBRS with Memcached
|
|
||||||
***************************
|
|
||||||
|
|
||||||
With DBRS V2.0 you can use the DBRS stack with `Memcached`_, a free and open source, high performance, distributed meory object caching system. This stack is ready to use DCPMM in fsdax for storage. The source for this application can be found in the `Memcached`_ repository.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The DBRS v2.0 release does not support Redis or Cassandra.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Build the DBRS Memcached image
|
|
||||||
==============================
|
|
||||||
|
|
||||||
To build the Memcached enabled image, use the Dockerfile with this command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build --force-rm --no-cache -f Dockerfile -t ${DOCKER_IMAGE} .
|
|
||||||
|
|
||||||
|
|
||||||
Run DBRS with Memcached as a standalone container
|
|
||||||
=================================================
|
|
||||||
|
|
||||||
Prior to launching the container, you will need to configure the DCPMM in fsdax mode with a file system, and have it mounted in :file:`/mnt/dax0`. Instructions for configuration can be found in :ref:`dbrs-hardware-configuration`.
|
|
||||||
|
|
||||||
To launch the container run this command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run --mount type=bind,source=/mnt/dax0,target=/mnt/pmem0 -i -d --name pmem-memchached ${DOCKER_IMAGE} -e /mnt/pmem0/memcached.file -m 64 -c 1024 -p 11211
|
|
||||||
|
|
||||||
where:
|
|
||||||
|
|
||||||
:command:`-m` is the maximum memory limit to use in megabytes
|
|
||||||
:command:`-e` is the mmap path for external memory (DCPMM storage). For this container the DCPMM sould be mounted inside the container on :file:`/mnt/pmem0`
|
|
||||||
:command:`-c` is the number of concurrent connections
|
|
||||||
:command:`-p` is the TCP connection port.
|
|
||||||
|
|
||||||
For more information please refer to this `blog post`_ from `Memcached`_
|
|
||||||
|
|
||||||
|
|
||||||
*Intel, Xeon, Intel Optane, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
|
|
||||||
|
|
||||||
|
|
||||||
.. _Intel Download Center: https://downloadcenter.intel.com/download/28695/Intel-Server-Board-S2600WF-Family-BIOS-and-Firmware-Update-Package-for-UEFI
|
|
||||||
|
|
||||||
.. _Quick Start Guide: https://software.intel.com/en-us/articles/quick-start-guide-configure-intel-optane-dc-persistent-memory-on-linux
|
|
||||||
|
|
||||||
.. _Managing NVDIMMs: https://docs.pmem.io/ndctl-user-guide/managing-nvdimms
|
|
||||||
|
|
||||||
.. _Configure, Manage, and Profile: https://software.intel.com/en-us/articles/configure-manage-and-profile-intel-optane-dc-persistent-memory-modules
|
|
||||||
|
|
||||||
.. _DBRS github repository: https://github.com/clearlinux/dockerfiles/tree/master/stacks/dbrs
|
|
||||||
|
|
||||||
.. _build-cassandra-pmem.sh: https://github.com/clearlinux/dockerfiles/tree/master/stacks/dbrs/cassandra/scripts/
|
|
||||||
|
|
||||||
.. _cassandra-pmem-helm: https://github.com/clearlinux/dockerfiles/tree/master/stacks/dbrs/cassandra/cassandra-pmem-helm
|
|
||||||
|
|
||||||
.. _helm: https://helm.sh/
|
|
||||||
|
|
||||||
.. _Github repository: https://github.com/pmem/pmem-redis
|
|
||||||
|
|
||||||
.. _Redis operator: https://github.com/spotahome/redis-operator
|
|
||||||
|
|
||||||
.. _example for persistent memory: https://github.com/spotahome/redis-operator/blob/master/example/redisfailover/pmem.yaml
|
|
||||||
|
|
||||||
.. _known issue: https://github.com/spotahome/redis-operator/issues/176
|
|
||||||
|
|
||||||
.. _Docker image with Apache Cassandra: https://hub.docker.com/r/clearlinux/stacks-dbrs-cassandra
|
|
||||||
|
|
||||||
.. _Docker image with Redis: https://hub.docker.com/r/clearlinux/stacks-dbrs-redis
|
|
||||||
|
|
||||||
.. _Intel Optane PMM: https://www.intel.com/content/www/us/en/architecture-and-technology/optane-technology/optane-for-data-centers.html
|
|
||||||
|
|
||||||
.. _pmem-csi: https://github.com/intel/pmem-csi/blob/release-0.6/README.md
|
|
||||||
|
|
||||||
.. _DBRS Terms of Use: https://clearlinux.org/stacks/database/terms-of-use
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
.. _Database Reference Stack website: https://clearlinux.org/stacks/database-reference
|
|
||||||
|
|
||||||
.. _DBRS V1.0: https://clearlinux.org/news-blogs/database-reference-stack-dbrs-v10-now-available
|
|
||||||
|
|
||||||
.. _DBRS V2.0: https://clearlinux.org/blogs-news/database-reference-stack-dbrs-v2-now-available
|
|
||||||
|
|
||||||
.. _Memcached: https://memcached.org
|
|
||||||
|
|
||||||
.. _blog post: https://memcached.org/blog/persistent-memory/
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,430 +0,0 @@
|
|||||||
.. _greengrass:
|
|
||||||
|
|
||||||
Enable AWS Greengrass\* and OpenVINO™ toolkit
|
|
||||||
#############################################
|
|
||||||
|
|
||||||
This guide explains how to enable AWS Greengrass\* and OpenVINO™ toolkit.
|
|
||||||
Specifically, the guide demonstrates how to:
|
|
||||||
|
|
||||||
* Set up the Intel edge device with |CL-ATTR|
|
|
||||||
* Install the OpenVINO™ toolkit and Amazon Web Services\* (AWS\*)
|
|
||||||
Greengrass\* software stacks
|
|
||||||
* Use AWS Greengrass\* and AWS Lambda\* to deploy the FaaS samples from
|
|
||||||
the cloud
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
Overview
|
|
||||||
********
|
|
||||||
|
|
||||||
Hardware accelerated Function-as-a-Service (FaaS) enables cloud developers to
|
|
||||||
deploy inference functionalities [1] on Intel IoT edge devices with
|
|
||||||
accelerators (CPU, Integrated GPU, Intel® FPGA, and Intel® Movidius™
|
|
||||||
technology). These functions provide a great developer experience and
|
|
||||||
seamless migration of visual analytics from cloud to edge in a secure manner
|
|
||||||
using a containerized environment. Hardware-accelerated FaaS provides the
|
|
||||||
best-in-class performance by accessing optimized deep learning libraries on
|
|
||||||
Intel IoT edge devices with accelerators.
|
|
||||||
|
|
||||||
Supported platforms
|
|
||||||
*******************
|
|
||||||
|
|
||||||
* Operating System: |CL| latest release
|
|
||||||
* Hardware: Intel® Core™ processors (that support inference on CPU only)
|
|
||||||
|
|
||||||
Sample description
|
|
||||||
==================
|
|
||||||
|
|
||||||
The AWS Greengrass samples are located at `Edge-Analytics-FaaS`_. This
|
|
||||||
guide uses the 1.0 version of the source code.
|
|
||||||
|
|
||||||
|CL| provides the following AWS Greengrass samples:
|
|
||||||
|
|
||||||
* `greengrass_classification_sample.py`_
|
|
||||||
|
|
||||||
This AWS Greengrass sample classifies a video stream using classification
|
|
||||||
networks such as AlexNet and GoogLeNet and publishes top-10 results on AWS\*
|
|
||||||
IoT Cloud every second.
|
|
||||||
|
|
||||||
* `greengrass_object_detection_sample_ssd.py`_
|
|
||||||
|
|
||||||
This AWS Greengrass sample detects objects in a video stream and
|
|
||||||
classifies them using single-shot multi-box detection (SSD) networks such
|
|
||||||
as SSD Squeezenet, SSD Mobilenet, and SSD300. This sample publishes
|
|
||||||
detection outputs such as class label, class confidence, and bounding box
|
|
||||||
coordinates on AWS IoT Cloud every second.
|
|
||||||
|
|
||||||
|
|
||||||
Install the OS on the edge device
|
|
||||||
*********************************
|
|
||||||
|
|
||||||
Start with a clean installation of |CL| on a new system, using the
|
|
||||||
:ref:`bare-metal-install-desktop`, found in :ref:`get-started`.
|
|
||||||
|
|
||||||
Create user accounts
|
|
||||||
====================
|
|
||||||
|
|
||||||
After |CL| is installed, create two user accounts. Create an administrative
|
|
||||||
user in |CL| and create a user account for the Greengrass services to use (
|
|
||||||
see Greengrass user below).
|
|
||||||
|
|
||||||
#. Create a new user and set a password for that user. Enter the following
|
|
||||||
commands as ``root``:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
useradd <userid>
|
|
||||||
passwd <userid>
|
|
||||||
|
|
||||||
#. Next, enable the :command:`sudo` command for your new <userid>. Add
|
|
||||||
<userid> to the `wheel` group:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
usermod -G wheel -a <userid>
|
|
||||||
|
|
||||||
#. Create a :file:`/etc/fstab` file.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
touch /etc/fstab
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
By default, |CL| does not create an :file:`/etc/fstab` file.
|
|
||||||
You must create this file before the Greengrass service runs.
|
|
||||||
|
|
||||||
Add required bundles
|
|
||||||
====================
|
|
||||||
|
|
||||||
Use the :command:`swupd` software updater utility to add the prerequisite bundles
|
|
||||||
for the OpenVINO software stack:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
swupd bundle-add os-clr-on-clr desktop-autostart computer-vision-basic
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
Learn more about how to :ref:`swupd-guide`.
|
|
||||||
|
|
||||||
The :command:`computer-vision-basic` bundle installs the OpenVINO™ toolkit,
|
|
||||||
and the sample models optimized for Intel edge platforms.
|
|
||||||
|
|
||||||
.. _convert-dl-models:
|
|
||||||
|
|
||||||
Convert deep learning models
|
|
||||||
============================
|
|
||||||
|
|
||||||
Locate sample models
|
|
||||||
--------------------
|
|
||||||
|
|
||||||
There are two types of provided models that can be used in conjunction with
|
|
||||||
AWS Greengrass for this guide: classification or object detection.
|
|
||||||
|
|
||||||
To complete this guide using an image classification model,
|
|
||||||
download the BVLC AlexNet model files `bvlc_alexnet.caffemodel`_ and
|
|
||||||
`deploy.prototxt`_ to the default model_location at
|
|
||||||
:file:`/usr/share/openvino/models`. Any custom pre-trained classification models
|
|
||||||
can be used with the classification sample.
|
|
||||||
|
|
||||||
For object detection, the sample models optimized for Intel edge platforms
|
|
||||||
are included with the computer-vision-basic bundle installation at
|
|
||||||
:file:`/usr/share/openvino/models`. These models are provided as an example;
|
|
||||||
you may also use a custom SSD model with the Greengrass object detection sample.
|
|
||||||
|
|
||||||
Run model optimizer
|
|
||||||
-------------------
|
|
||||||
|
|
||||||
Follow the instructions in the `Model Optimizer Developer Guide`_ for converting
|
|
||||||
deep learning models to Intermediate Representation using Model Optimizer. To
|
|
||||||
optimize either of the sample models described above, run one of the following commands.
|
|
||||||
|
|
||||||
For classification using BVLC AlexNet model:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
python3 mo.py --framework caffe --input_model
|
|
||||||
<model_location>/bvlc_alexnet.caffemodel --input_proto
|
|
||||||
<model_location>/deploy.prototxt --data_type <data_type> --output_dir
|
|
||||||
<output_dir> --input_shape [1,3,227,227]
|
|
||||||
|
|
||||||
For object detection using SqueezeNetSSD-5Class model:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
python3 mo.py --framework caffe --input_model
|
|
||||||
<model_location>/'SqueezeNet 5-Class detection'/SqueezeNetSSD-5Class.caffemodel
|
|
||||||
--input_proto <model_location>/'SqueezeNet 5-Class detection'/SqueezeNetSSD-5Class.prototxt
|
|
||||||
--data_type <data_type> --output_dir <output_dir>
|
|
||||||
|
|
||||||
In these examples:
|
|
||||||
|
|
||||||
* `<model_location>` is :file:`/usr/share/openvino/models`.
|
|
||||||
|
|
||||||
* `<data_type>` is FP32 or FP16, depending on target device.
|
|
||||||
|
|
||||||
* `<output_dir>` is the directory where the Intermediate Representation
|
|
||||||
(IR) is stored. IR contains .xml format corresponding to the network
|
|
||||||
structure and .bin format corresponding to weights. This .xml file should be
|
|
||||||
passed to :command:`<PARAM_MODEL_XML>`.
|
|
||||||
|
|
||||||
* In the BVLC AlexNet model, the prototxt defines the input shape with
|
|
||||||
batch size 10 by default. In order to use any other batch size, the
|
|
||||||
entire input shape must be provided as an argument to the model
|
|
||||||
optimizer. For example, to use batch size 1, you must provide:
|
|
||||||
`--input_shape [1,3,227,227]`
|
|
||||||
|
|
||||||
|
|
||||||
Configure AWS Greengrass group
|
|
||||||
******************************
|
|
||||||
|
|
||||||
For each Intel edge platform, you must create a new AWS Greengrass group
|
|
||||||
and install AWS Greengrass core software to establish the connection between
|
|
||||||
cloud and edge.
|
|
||||||
|
|
||||||
#. To create an AWS Greengrass group, follow the instructions in
|
|
||||||
`Configure AWS IoT Greengrass on AWS IoT`_.
|
|
||||||
|
|
||||||
#. To install and configure AWS Greengrass core on edge platform, follow
|
|
||||||
the instructions in `Start AWS Greengrass on the Core Device`_. In
|
|
||||||
step 8(b), download the x86_64 Ubuntu\* configuration of the AWS Greengrass
|
|
||||||
core software.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
You do not need to run the :file:`cgroupfs-mount.sh` script in step #6
|
|
||||||
of Module 1 of the `AWS Greengrass Developer Guide`_ because this is
|
|
||||||
enabled already in |CL|.
|
|
||||||
|
|
||||||
#. Be sure to download both the security resources and the AWS Greengrass
|
|
||||||
core software.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
Security certificates are linked to your AWS account.
|
|
||||||
|
|
||||||
|
|
||||||
Create and package Lambda function
|
|
||||||
**********************************
|
|
||||||
|
|
||||||
#. Complete steps 1-4 of the AWS Greengrass guide at
|
|
||||||
`Create and Package a Lambda Function`_.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
This creates the tarball needed to create the AWS Greengrass
|
|
||||||
environment on the edge device.
|
|
||||||
|
|
||||||
|
|
||||||
#. In step 5, replace :file:`greengrassHelloWorld.py` with the classification or
|
|
||||||
object detection Greengrass sample from `Edge-Analytics-Faas`_:
|
|
||||||
|
|
||||||
* Classification: `greengrass_classification_sample.py`_
|
|
||||||
|
|
||||||
* Object Detection: `greengrass_object_detection_sample_ssd.py`_
|
|
||||||
|
|
||||||
#. Zip the selected Greengrass sample with the extracted Greengrass SDK folders
|
|
||||||
from the previous step into :file:`greengrass_sample_python_lambda.zip`.
|
|
||||||
|
|
||||||
The zip should contain:
|
|
||||||
|
|
||||||
* greengrasssdk
|
|
||||||
|
|
||||||
* greengrass classification or object detection sample
|
|
||||||
|
|
||||||
For example:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
zip -r greengrass_lambda.zip greengrasssdk
|
|
||||||
greengrass_object_detection_sample_ssd.py
|
|
||||||
|
|
||||||
#. Return to the AWS documentation section called
|
|
||||||
`Create and Package a Lambda Function`_ and complete the procedure.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
In step 9(a) of the AWS documentation, while uploading the zip file,
|
|
||||||
make sure to name the handler to one of the following, depending on the
|
|
||||||
AWS Greengrass sample you are using:
|
|
||||||
|
|
||||||
* greengrass_object_detection_sample_ssd.function_handler
|
|
||||||
* greengrass_classification_sample.function_handler
|
|
||||||
|
|
||||||
|
|
||||||
Configure Lambda function
|
|
||||||
*************************
|
|
||||||
|
|
||||||
After creating the Greengrass group and the Lambda function, start
|
|
||||||
configuring the Lambda function for AWS Greengrass.
|
|
||||||
|
|
||||||
#. Follow steps 1-8 in `Configure the Lambda Function for AWS IoT Greengrass`_
|
|
||||||
in the AWS documentation.
|
|
||||||
|
|
||||||
#. In addition to the details mentioned in step 8, change the Memory limit
|
|
||||||
to 2048 MB to accommodate large input video streams.
|
|
||||||
|
|
||||||
#. Add the following environment variables as key-value pairs when editing
|
|
||||||
the Lambda configuration and click on update:
|
|
||||||
|
|
||||||
.. list-table:: **Table 1. Environment variables: Lambda configuration**
|
|
||||||
:widths: 20 80
|
|
||||||
:header-rows: 1
|
|
||||||
|
|
||||||
* - Key
|
|
||||||
- Value
|
|
||||||
* - PARAM_MODEL_XML
|
|
||||||
- <MODEL_DIR>/<IR.xml>, where <MODEL_DIR> is user specified and
|
|
||||||
contains IR.xml, the Intermediate Representation file from the
|
|
||||||
OpenVINO™ Model Optimizer.
|
|
||||||
For this guide, <MODEL_DIR> should be set to '/usr/share/openvino/models'
|
|
||||||
or one of its subdirectories.
|
|
||||||
* - PARAM_INPUT_SOURCE
|
|
||||||
- <DATA_DIR>/input.webm to be specified by user. Holds both input and
|
|
||||||
output data. For webcam, set PARAM_INPUT_SOURCE to ‘/dev/video0’
|
|
||||||
* - PARAM_DEVICE
|
|
||||||
- "CPU"
|
|
||||||
* - PARAM_CPU_EXTENSION_PATH
|
|
||||||
- /usr/lib64/libcpu_extension.so
|
|
||||||
* - PARAM_OUTPUT_DIRECTORY
|
|
||||||
- <DATA_DIR> to be specified by user. Holds both input and output
|
|
||||||
data
|
|
||||||
* - PARAM_NUM_TOP_RESULTS
|
|
||||||
- User specified for classification sample.
|
|
||||||
(e.g. 1 for top-1 result, 5 for top-5 results)
|
|
||||||
|
|
||||||
#. Add subscription to subscribe, or publish messages from AWS Greengrass
|
|
||||||
Lambda function by completing the procedure in `Configure the Lambda Function for AWS IoT Greengrass`_.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The optional topic filter field is the topic mentioned inside the Lambda function. In this guide, sample topics include the following:
|
|
||||||
:command:`openvino/ssd` or :command:`openvino/classification`
|
|
||||||
|
|
||||||
Add local resources
|
|
||||||
===================
|
|
||||||
|
|
||||||
Refer to the AWS documentation `Access Local Resources with Lambda Functions and Connectors`_
|
|
||||||
for details about local resources and access privileges.
|
|
||||||
|
|
||||||
The following table describes the local resources needed for the CPU:
|
|
||||||
|
|
||||||
.. list-table:: **Local resources**
|
|
||||||
:widths: 20, 20, 20, 20
|
|
||||||
:header-rows: 1
|
|
||||||
|
|
||||||
* - Name
|
|
||||||
- Resource type
|
|
||||||
- Local path
|
|
||||||
- Access
|
|
||||||
|
|
||||||
* - ModelDir
|
|
||||||
- Volume
|
|
||||||
- <MODEL_DIR> to be specified by user
|
|
||||||
- Read-Only
|
|
||||||
|
|
||||||
* - Webcam
|
|
||||||
- Device
|
|
||||||
- /dev/video0
|
|
||||||
- Read-Only
|
|
||||||
|
|
||||||
* - DataDir
|
|
||||||
- Volume
|
|
||||||
- <DATA_DIR> to be specified by user. Holds both input and output
|
|
||||||
data.
|
|
||||||
- Read and Write
|
|
||||||
|
|
||||||
Deploy Lambda function
|
|
||||||
**********************
|
|
||||||
|
|
||||||
Refer to the AWS documentation `Deploy Cloud Configurations to an AWS IoT Greengrass Core Device`_ for instructions on how to deploy the lambda function to AWS
|
|
||||||
Greengrass core device. Select *Deployments* on the group page and follow the instructions.
|
|
||||||
|
|
||||||
Output consumption
|
|
||||||
==================
|
|
||||||
|
|
||||||
There are four options available for output consumption. These options are
|
|
||||||
used to report, stream, upload, or store inference output at an interval
|
|
||||||
defined by the variable :command:`reporting_interval` in the AWS Greengrass samples.
|
|
||||||
|
|
||||||
#. IoT cloud output:
|
|
||||||
|
|
||||||
This option is enabled by default in the AWS Greengrass samples using the
|
|
||||||
:command:`enable_iot_cloud_output` variable. You can use it to verify the lambda
|
|
||||||
running on the edge device. It enables publishing messages to IoT cloud
|
|
||||||
using the subscription topic specified in the lambda. (For example, topics
|
|
||||||
may include :command:`openvino/classification` for classification and :command:`openvino/ssd`
|
|
||||||
for object detection samples.) For classification, top-1 result with class
|
|
||||||
label are published to IoT cloud. For SSD object detection, detection
|
|
||||||
results such as bounding box coordinates of objects, class label, and
|
|
||||||
class confidence are published.
|
|
||||||
|
|
||||||
Refer to the AWS documentation
|
|
||||||
`Verify the Lambda Function Is Running on the Device`_ for instructions on
|
|
||||||
how to view the output on IoT cloud.
|
|
||||||
|
|
||||||
#. Kinesis streaming:
|
|
||||||
|
|
||||||
This option enables inference output to be streamed from the edge device
|
|
||||||
to cloud using Kinesis [3] streams when :command:`enable_kinesis_output` is set
|
|
||||||
to True. The edge devices act as data producers and continually push
|
|
||||||
processed data to the cloud. You must set up and specify
|
|
||||||
Kinesis stream name, Kinesis shard, and AWS region in the AWS Greengrass
|
|
||||||
samples.
|
|
||||||
|
|
||||||
#. Cloud storage using AWS S3 bucket:
|
|
||||||
|
|
||||||
When the :command:`enable_s3_jpeg_output` variable is set to True, it enables
|
|
||||||
uploading and storing processed frames (in jpeg format) in an AWS S3
|
|
||||||
bucket. You must set up and specify the S3 bucket name in the AWS
|
|
||||||
Greengrass samples to store the JPEG images. The images are named using the
|
|
||||||
timestamp and uploaded to S3.
|
|
||||||
|
|
||||||
#. Local storage:
|
|
||||||
|
|
||||||
When the :command:`enable_s3_jpeg_output` variable is set to True, it enables
|
|
||||||
storing processed frames (in jpeg format) on the edge device. The images
|
|
||||||
are named using the timestamp and stored in a directory specified by
|
|
||||||
:command:`PARAM_OUTPUT_DIRECTORY`.
|
|
||||||
|
|
||||||
References
|
|
||||||
**********
|
|
||||||
|
|
||||||
#. AWS Greengrass: https://aws.amazon.com/greengrass/
|
|
||||||
#. AWS Lambda: https://aws.amazon.com/lambda/
|
|
||||||
#. AWS Kinesis: https://aws.amazon.com/kinesis/
|
|
||||||
|
|
||||||
|
|
||||||
*Intel, OpenVINO, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
|
|
||||||
|
|
||||||
|
|
||||||
.. _Edge-Analytics-FaaS: https://github.com/intel/Edge-Analytics-FaaS/tree/v1.0/AWS%20Greengrass
|
|
||||||
|
|
||||||
.. _bvlc_alexnet.caffemodel: http://dl.caffe.berkeleyvision.org/bvlc_alexnet.caffemodel
|
|
||||||
|
|
||||||
.. _deploy.prototxt: https://github.com/BVLC/caffe/blob/master/models/bvlc_alexnet/deploy.prototxt
|
|
||||||
|
|
||||||
.. _greengrass_classification_sample.py: https://github.com/intel/Edge-Analytics-FaaS/blob/v1.0/AWS%20Greengrass/greengrass_classification_sample.py
|
|
||||||
|
|
||||||
.. _greengrass_object_detection_sample_ssd.py: https://github.com/intel/Edge-Analytics-FaaS/blob/v1.0/AWS%20Greengrass/greengrass_object_detection_sample_ssd.py
|
|
||||||
|
|
||||||
.. _Model Optimizer Developer Guide: https://software.intel.com/en-us/articles/OpenVINO-ModelOptimizer
|
|
||||||
|
|
||||||
.. _AWS Greengrass Developer Guide: https://docs.aws.amazon.com/greengrass/latest/developerguide/what-is-gg.html
|
|
||||||
|
|
||||||
.. _Configure AWS IoT Greengrass on AWS IoT: https://docs.aws.amazon.com/greengrass/latest/developerguide/gg-config.html
|
|
||||||
|
|
||||||
.. _Start AWS Greengrass on the Core Device: https://docs.aws.amazon.com/greengrass/latest/developerguide/gg-device-start.html
|
|
||||||
|
|
||||||
.. _Configure the Lambda Function for AWS IoT Greengrass: https://docs.aws.amazon.com/greengrass/latest/developerguide/config-lambda.html
|
|
||||||
|
|
||||||
.. _Access Local Resources with Lambda Functions and Connectors: https://docs.aws.amazon.com/greengrass/latest/developerguide/access-local-resources.html
|
|
||||||
|
|
||||||
.. _Deploy Cloud Configurations to an AWS IoT Greengrass Core Device: https://docs.aws.amazon.com/greengrass/latest/developerguide/configs-core.html
|
|
||||||
|
|
||||||
.. _Verify the Lambda Function Is Running on the Device: https://docs.aws.amazon.com/greengrass/latest/developerguide/lambda-check.html
|
|
||||||
|
|
||||||
.. _Create and Package a Lambda Function: https://docs.aws.amazon.com/greengrass/latest/developerguide/create-lambda.html
|
|
||||||
@@ -1,242 +0,0 @@
|
|||||||
.. _hpcrs:
|
|
||||||
|
|
||||||
High Performance Computing Reference Stack
|
|
||||||
##########################################
|
|
||||||
|
|
||||||
This guide gives examples for converting Docker* containers, such as those provided by the :ref:`dlrs` into Singularity* containers suited for HPC, and then walking through a multi-node benchmarking example with TensorFlow*.
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
|
|
||||||
Overview
|
|
||||||
********
|
|
||||||
|
|
||||||
The High Performance Computing Reference Stack (HPCRS) meets the needs of deploying HPC and AI workloads on the same system. This software solution reduces the complexities associated with integrating software components for High Performance Computing (HPC) Platforms. `Singularity`_ is an open source container platform to package entire scientific workflows, software and libraries, and even data.
|
|
||||||
|
|
||||||
|
|
||||||
Installing Singularity
|
|
||||||
**********************
|
|
||||||
The installation instructions are for Linux* systems, and have been enabled for installation on |CL-ATTR|.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The steps for installation can also be found on the `Singularity quick-start`_ https://sylabs.io/guides/3.0/user-guide/quick_start.html#quick-installation
|
|
||||||
|
|
||||||
#. Install Go*.
|
|
||||||
|
|
||||||
This guide requires version 1.13 of Go, for compatibility with Singularity v3.0.0. Please use these steps to ensure the correct version of Go is installed:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
$ export VERSION=1.13 OS=linux ARCH=amd64 && \
|
|
||||||
wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz && \
|
|
||||||
sudo tar -C /usr/local -xzvf go$VERSION.$OS-$ARCH.tar.gz && \
|
|
||||||
rm go$VERSION.$OS-$ARCH.tar.gz
|
|
||||||
|
|
||||||
#. Setup the environment for Go.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
echo 'export GOPATH=${HOME}/go' >> ~/.bashrc && \
|
|
||||||
echo 'export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin' >> ~/.bashrc && \
|
|
||||||
source ~/.bashrc
|
|
||||||
|
|
||||||
#. Install :command:`dep` for dependency resolution with Singularity v3.0.0.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
go get -u github.com/golang/dep/cmd/dep
|
|
||||||
|
|
||||||
#. Download Singularity.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
go get -d github.com/sylabs/singularity
|
|
||||||
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
Go will complain that there are no Go files, but it will still download the Singularity source code to the appropriate directory within the $GOPATH.
|
|
||||||
|
|
||||||
#. Checkout version 3.0.0 of Singularity.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
export VERSION=v3.0.3 # or another tag or branch if you like && \
|
|
||||||
cd $GOPATH/src/github.com/sylabs/singularity && \
|
|
||||||
git fetch && \
|
|
||||||
git checkout $VERSION # omit this command to install the latest bleeding edge code from master
|
|
||||||
|
|
||||||
#. Build Singularity.
|
|
||||||
|
|
||||||
Singularity uses a custom build system called makeit. mconfig is called to generate a Makefile and then make is used to compile and install.
|
|
||||||
The devpkg-openssl, devpkg-util-linux package may be required and can be installed using the :command:`sudo swupd bundle-add <pkg-name>`.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
./mconfig && \
|
|
||||||
make -C ./builddir && \
|
|
||||||
sudo make -C ./builddir install
|
|
||||||
|
|
||||||
|
|
||||||
#. Configure bash completion (optional).
|
|
||||||
|
|
||||||
To enjoy bash completion with Singularity commands and options, source the bash completion file. Add this command to your ~/.bashrc file so that bash completion continues to work in new shells
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
. /usr/local/etc/bash_completion.d/singularity
|
|
||||||
|
|
||||||
|
|
||||||
Converting Docker images to Singularity Images
|
|
||||||
**********************************************
|
|
||||||
|
|
||||||
#. Download d2s.
|
|
||||||
|
|
||||||
:command:`d2s` os an open source tool to convert Docker images to
|
|
||||||
Singularity images. You can use the script in the location where it is
|
|
||||||
downloaded, or install it using the included :file:`setup.py` file with
|
|
||||||
the :command:`python setup.py install`
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/intel/stacks.git
|
|
||||||
cd stacks/hpcrs/d2s
|
|
||||||
|
|
||||||
#. List local Docker images.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
python d2s.py --list_docker_images
|
|
||||||
|
|
||||||
Your output can appear like this:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
==============================
|
|
||||||
Docker images present locally
|
|
||||||
==============================
|
|
||||||
ID NAME
|
|
||||||
0: clearlinux/stacks-dlrs-mkl
|
|
||||||
1: clearlinux/stacks-dlrs_2-mkl
|
|
||||||
==============================
|
|
||||||
|
|
||||||
#. Convert to Singularity images.
|
|
||||||
|
|
||||||
To convert the Docker images to Singularity images, use the :command:`d2s`
|
|
||||||
script with the ID numbers of the images you wish to convert.
|
|
||||||
We strongly recommend using one of the :file:`clearlinux/stacks-dlrs-mkl`
|
|
||||||
or :file:`sysstacks/stacks-dlrs-mkl` based images for this guide. Other
|
|
||||||
images may be incompatible with expected configuration or filesystem
|
|
||||||
options.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
python d2s.py --convert_docker_images <ID_1> <ID_2>
|
|
||||||
|
|
||||||
#. Use the Singularity image.
|
|
||||||
|
|
||||||
To use the container shell to run workloads, launch the image and you
|
|
||||||
will be dropped into the shell. The Singularity image name will be the
|
|
||||||
same as the name of the Docker image, with slashes converted to
|
|
||||||
underscores.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
singularity shell <singularity image>
|
|
||||||
|
|
||||||
Using the example output above, after conversion you could launch the clearlinux/stacks-dlrs-mkl Singularity image with
|
|
||||||
:command:`singularity shell clearlinux_stacks-dlrs-mkl`
|
|
||||||
|
|
||||||
Execute a multi-node benchmark on an HPC cluster
|
|
||||||
************************************************
|
|
||||||
The following example was executed on an Intel(r) Xeon(r) Processor-based
|
|
||||||
HPC infrastructure. The following steps may need to be adjusted for
|
|
||||||
different environments. See this `Intel Whitepaper`_ for more information.
|
|
||||||
|
|
||||||
Running a ResNet50 workload multi-node
|
|
||||||
--------------------------------------
|
|
||||||
|
|
||||||
#. Download the TensorFlow benchmark.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
git clone http://github.com/tensorflow/benchmarks -b cnn_tf_v1.13_compatible
|
|
||||||
|
|
||||||
#. Copy the Singularity image and the benchmark files to the HPC cluster
|
|
||||||
environment.
|
|
||||||
|
|
||||||
#. Install OpenMPI* if needed.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
If the HPC host does not have OpenMPI installed, install a custom
|
|
||||||
local version in the user's home directory. This version must be the
|
|
||||||
same as the version installed in the DLRS container. Follow the steps
|
|
||||||
for `building OpenMPI`_ from their documentation.
|
|
||||||
|
|
||||||
#. Adjust PATH variables.
|
|
||||||
|
|
||||||
Include the OpenMPI install locations in the PATH and LD_LIBRARY_PATH
|
|
||||||
environment variables.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
export PATH="$PATH:<openmpi install path>/bin"
|
|
||||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:<openmpi install path>/lib/"
|
|
||||||
|
|
||||||
#. Execute the TF benchmark script on single or multiple nodes using OpenMPI
|
|
||||||
through the :command:`mpirun` command. Replace variables in {} braces to
|
|
||||||
reflect your environment.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
mpirun --np ${NUM_COPIES} \
|
|
||||||
-bind-to none \
|
|
||||||
-map-by slot \
|
|
||||||
--display-map \
|
|
||||||
-host ${HOSTNAMES} \
|
|
||||||
--report-bindings \
|
|
||||||
--oversubscribe \
|
|
||||||
-x LD_LIBRARY_PATH \
|
|
||||||
-x PATH \
|
|
||||||
-x HOROVOD_FUSION_THRESHOLD \
|
|
||||||
-x OMP_NUM_THREADS=${OMP_NUM_THREADS} \
|
|
||||||
singularity exec ${PATH_TO_SING_IMAGE} \
|
|
||||||
python ${PATH_TO_TF_BENCH}/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py \
|
|
||||||
--batch_size=128 \
|
|
||||||
--model=resnet50 \
|
|
||||||
--num_intra_threads=${NUM_INTRA_THREADS} \
|
|
||||||
--num_inter_threads=${NUM_INTER_THREADS} \
|
|
||||||
--data_format=NHWC \
|
|
||||||
--device=cpu \
|
|
||||||
--variable_update=horovod \
|
|
||||||
--horovod_device=cpu
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
Refer to the `DLRS script`_ for recommended values for setting environment variables in the :command:`mpirun` command.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
You may see an error regarding a missing library while executing the DLRS container.
|
|
||||||
“tensorflow.python.framework.errors_impl.NotFoundError: libnuma.so.1: cannot open shared object file: No such file or directory”
|
|
||||||
|
|
||||||
A workaround for this error is to bind the path to the library from the host.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
--bind /usr/lib64/libnuma.so.1:/usr/lib64/libnuma.so.1
|
|
||||||
|
|
||||||
.. _Singularity: https://sylabs.io/
|
|
||||||
|
|
||||||
.. _Singularity quick-start: https://sylabs.io/guides/3.0/user-guide/quick_start.html#quick-installation
|
|
||||||
|
|
||||||
.. _Intel Whitepaper: https://www.intel.com/content/www/us/en/artificial-intelligence/solutions/best-known-methods-for-scaling-deep-learning-with-tensorflow-on-xeon-processor-based-clusters.html
|
|
||||||
|
|
||||||
.. _building OpenMPI: https://www.open-mpi.org/faq/?category=building#easy-build
|
|
||||||
|
|
||||||
.. _DLRS script: https://github.com/intel/stacks/blob/master/dlrs/clearlinux/tensorflow_2/mkl/scripts/set_env.sh
|
|
||||||
@@ -1,564 +0,0 @@
|
|||||||
.. _mers:
|
|
||||||
|
|
||||||
Media Reference Stack
|
|
||||||
#####################
|
|
||||||
|
|
||||||
The Media Reference Stack (MeRS) is a highly optimized software stack for
|
|
||||||
Intel® Architecture Processors (the CPU) and Intel® Processor Graphics (the
|
|
||||||
GPU) to enable media prioritized workloads, such as transcoding and analytics.
|
|
||||||
|
|
||||||
This guide explains how to use the pre-built |MERS| container image, build
|
|
||||||
your own |MERS| container image, and use the reference stack.
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
Overview
|
|
||||||
********
|
|
||||||
|
|
||||||
Developers face challenges due to the complexity of software integration for
|
|
||||||
media tasks that require investing time and engineering effort.
|
|
||||||
For example:
|
|
||||||
|
|
||||||
* Finding the balance between quality and performance.
|
|
||||||
* Understanding available standard-compliant encoders.
|
|
||||||
* Optimizing across the hardware-software stack for efficiency.
|
|
||||||
|
|
||||||
|MERS| abstracts away the complexity of integrating multiple software
|
|
||||||
components and specifically tunes them for Intel platforms. |MERS| enables
|
|
||||||
media and visual cloud developers to deliver experiences using a simple
|
|
||||||
containerized solution.
|
|
||||||
|
|
||||||
|
|
||||||
Releases
|
|
||||||
********
|
|
||||||
|
|
||||||
Refer to the `System Stacks for Linux* OS repository
|
|
||||||
<https://github.com/intel/stacks>`_ for information and download links for the
|
|
||||||
different versions and offerings of the stack.
|
|
||||||
|
|
||||||
* MeRS V0.2.0 release announcement including media processing on GPU and
|
|
||||||
analytics on CPU.
|
|
||||||
|
|
||||||
* MeRS V0.1.0 including media processing and analytics CPU.
|
|
||||||
|
|
||||||
* `MeRS Release notes on Github*
|
|
||||||
<https://github.com/intel/stacks/blob/master/mers/NEWS.md>`_ for the
|
|
||||||
latest release of Deep Learning Reference Stack
|
|
||||||
|
|
||||||
|
|
||||||
Prerequisites
|
|
||||||
=============
|
|
||||||
|
|
||||||
|MERS| can run on any host system that supports Docker\*. This guide uses
|
|
||||||
|CL-ATTR| as the host system.
|
|
||||||
|
|
||||||
- To install |CL| on a host system, see how to
|
|
||||||
:ref:`install Clear Linux* OS from the live desktop
|
|
||||||
<bare-metal-install-desktop>`.
|
|
||||||
|
|
||||||
- To install Docker* on a |CL| host system, see
|
|
||||||
the :ref:`instructions for installing Docker* <docker>`.
|
|
||||||
|
|
||||||
.. important::
|
|
||||||
|
|
||||||
For optimal media analytics performance, a processor with Vector Neural
|
|
||||||
Network Instructions (VNNI) should be used. VNNI is an extension of Intel®
|
|
||||||
Advanced Vector Extensions 512 (Intel® AVX-512) and is available starting
|
|
||||||
with the 2nd generation of Intel® Xeon® Scalable processors, providing AI
|
|
||||||
inference acceleration.
|
|
||||||
|
|
||||||
Stack features
|
|
||||||
==============
|
|
||||||
|
|
||||||
The |MERS| provides a `pre-built Docker image available on DockerHub
|
|
||||||
<https://hub.docker.com/r/sysstacks/mers-clearlinux>`_, which includes
|
|
||||||
instructions on building the image from source. |MERS| is open-sourced to
|
|
||||||
make sure developers have easy access to the source code and are able to
|
|
||||||
customize it. |MERS| is built using the latest *clearlinux/os-core* Docker
|
|
||||||
image and aims to support the latest |CL| version.
|
|
||||||
|
|
||||||
|MERS| provides the following libraries and drivers:
|
|
||||||
|
|
||||||
.. list-table::
|
|
||||||
:widths: 15 85
|
|
||||||
|
|
||||||
* - SVT-HEVC
|
|
||||||
- Scalable Video Technology for HEVC encoding, also known as H.265
|
|
||||||
* - SVT-AV1
|
|
||||||
- Scalable Video Technology for AV1 encoding
|
|
||||||
* - x264
|
|
||||||
- x264 for H.264/MPEG-4 AVC encoding
|
|
||||||
* - dav1d
|
|
||||||
- `dav1d <https://code.videolan.org/videolan/dav1d>`_ for AV1 decoding
|
|
||||||
* - libVA
|
|
||||||
- `VAAPI (Video Acceleration API) open-source library (LibVA),
|
|
||||||
<https://github.com/intel/libva>`_ which provides access to graphics
|
|
||||||
hardware acceleration capabilities.
|
|
||||||
* - media-driver
|
|
||||||
- `Intel® Media Driver for VAAPI <https://github.com/intel/media-driver/>`_
|
|
||||||
for supporting hardware acceleration on Intel® Gen graphics hardware
|
|
||||||
platforms.
|
|
||||||
* - gmmlib
|
|
||||||
- `Intel® Graphics Memory Management Library
|
|
||||||
<https://github.com/intel/gmmlib>`_ provides device specific and buffer
|
|
||||||
management for the Intel® Graphics Compute Runtime for oneAPI Level Zero
|
|
||||||
and OpenCL™ Driver and the Intel Media Driver for VAAPI.
|
|
||||||
|
|
||||||
Components of the |MERS| include:
|
|
||||||
|
|
||||||
* |CL| as a base for performance and security.
|
|
||||||
|
|
||||||
* `OpenVINO™ toolkit
|
|
||||||
<https://01.org/openvinotoolkit>`_ for inference.
|
|
||||||
|
|
||||||
* `FFmpeg* <https://www.ffmpeg.org>`_ with plugins for:
|
|
||||||
|
|
||||||
- `Scalable Video Technology (SVT)
|
|
||||||
<https://01.org/svt>`_
|
|
||||||
|
|
||||||
* `GStreamer* <https://gstreamer.freedesktop.org/>`_ with plugins for:
|
|
||||||
|
|
||||||
- `Scalable Video
|
|
||||||
Technology (SVT) <https://01.org/svt>`_
|
|
||||||
- `OpenVINO™ toolkit
|
|
||||||
<https://01.org/openvinotoolkit>`_
|
|
||||||
- `VAAPI <https://github.com/GStreamer/gstreamer-vaapi>`_
|
|
||||||
|
|
||||||
* `Intel® Media SDK <https://github.com/Intel-Media-SDK/MediaSDK>`_
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The |MERS| is validated on 11th generation Intel Processor Graphics and
|
|
||||||
newer. Older generations should work but are not tested against.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The pre-built |MERS| container image configures FFmpeg without certain
|
|
||||||
elements (specific encoder, decoder, muxer, etc.) that you may require. If
|
|
||||||
you require changes to FFmpeg we suggest starting at
|
|
||||||
:ref:`building-the-mers-container-image`.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The Media Reference Stack is a collective work, and each piece of software
|
|
||||||
within the work has its own license. Please see the `MeRS Terms of Use
|
|
||||||
<https://clearlinux.org/stacks/media/terms-of-use>`_ for more details about
|
|
||||||
licensing and usage of the Media Reference Stack.
|
|
||||||
|
|
||||||
|
|
||||||
Get the pre-built |MERS| container image
|
|
||||||
****************************************
|
|
||||||
|
|
||||||
Pre-built |MERS| Docker images are available on DockerHub* at
|
|
||||||
https://hub.docker.com/r/sysstacks/mers-clearlinux
|
|
||||||
|
|
||||||
|
|
||||||
To use the |MERS|:
|
|
||||||
|
|
||||||
#. Pull the image directly from `Docker Hub
|
|
||||||
<https://hub.docker.com/r/sysstacks/mers-clearlinux>`_.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker pull sysstacks/mers-clearlinux
|
|
||||||
|
|
||||||
.. note ::
|
|
||||||
|
|
||||||
The |MERS| docker image is large in size and will take some time to
|
|
||||||
download depending on your Internet connection.
|
|
||||||
|
|
||||||
If you are on a network with outbound proxies, be sure to configure
|
|
||||||
Docker to allow access. See the `Docker service proxy
|
|
||||||
<https://docs.docker.com/config/daemon/systemd/#httphttps-proxy>`_ and
|
|
||||||
`Docker client proxy
|
|
||||||
<https://docs.docker.com/network/proxy/#configure-the-docker-client>`_
|
|
||||||
documentation for more details.
|
|
||||||
|
|
||||||
#. Once you have downloaded the image, run it using the following command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker run -it sysstacks/mers-clearlinux
|
|
||||||
|
|
||||||
This will launch the image and drop you into a bash shell inside the
|
|
||||||
container. GStreamer and FFmpeg programs are installed in the container
|
|
||||||
image and accessible in the default $PATH. Use these programs as you would
|
|
||||||
outside of |MERS|.
|
|
||||||
|
|
||||||
Paths to media files and video devices, such as cameras, can be shared from
|
|
||||||
the host to the container with the :command:`--volume` switch `using Docker
|
|
||||||
volumes <https://docs.docker.com/storage/volumes/>`_.
|
|
||||||
|
|
||||||
.. _building-the-mers-container-image:
|
|
||||||
|
|
||||||
Build the |MERS| container image from source
|
|
||||||
********************************************
|
|
||||||
|
|
||||||
If you choose to build your own MeRS container image, you can optionally add
|
|
||||||
customizations as needed. The :file:`Dockerfile` for the MeRS is available on
|
|
||||||
`GitHub <https://github.com/intel/stacks/tree/master/mers>`_ and can be used
|
|
||||||
as a reference when creating your own container image.
|
|
||||||
|
|
||||||
#. The |MERS| image is part of the dockerfiles repository inside the |CL|
|
|
||||||
organization on GitHub. Clone the :file:`stacks` repository.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/intel/stacks.git
|
|
||||||
|
|
||||||
#. Navigate to the :file:`stacks/mers/clearlinux` directory which contains
|
|
||||||
the Dockerfile for the |MERS|.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
cd ./stacks/mers/clearlinux
|
|
||||||
|
|
||||||
#. Use the :command:`docker build` command with the :file:`Dockerfile` to
|
|
||||||
build the MeRS container image.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build --no-cache -t sysstacks/mers-clearlinux .
|
|
||||||
|
|
||||||
Use the |MERS| container image
|
|
||||||
******************************
|
|
||||||
|
|
||||||
This section shows examples of how the |MERS| container image can be used to
|
|
||||||
process media files.
|
|
||||||
|
|
||||||
The models and video source can be substituted from your use-case. Some
|
|
||||||
publicly licensed sample videos are available at `sample-videos repository
|
|
||||||
<https://github.com/intel-iot-devkit/sample-videos>`_ for testing.
|
|
||||||
|
|
||||||
|
|
||||||
Media Transcoding
|
|
||||||
=================
|
|
||||||
|
|
||||||
The examples below show transcoding using the GPU or CPU for processing.
|
|
||||||
|
|
||||||
#. On the host system, setup a workspace for data and models:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
mkdir ~/ffmpeg
|
|
||||||
mkdir ~/ffmpeg/input
|
|
||||||
mkdir ~/ffmpeg/output
|
|
||||||
|
|
||||||
#. Copy a video file to :file:`~/ffmpeg/input`.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
cp </path/to/video> ~/ffmpeg/input
|
|
||||||
|
|
||||||
#. Run the *sysstacks/mers-clearlinux* Docker image, allowing shared access to
|
|
||||||
the workspace on the host:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
docker run -it \
|
|
||||||
--volume ~/ffmpeg:/home/mers-user:ro \
|
|
||||||
--device=/dev/dri \
|
|
||||||
--env QSV_DEVICE=/dev/dri/renderD128 \
|
|
||||||
sysstacks/mers-clearlinux:latest
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
The :command:`--device` parameter and the **GSV_DEVICE** environment
|
|
||||||
variable allow shared access to the GPU on the host system. The values
|
|
||||||
needed may be different depending on host's graphics configuration.
|
|
||||||
|
|
||||||
After running the :command:`docker run` command, you enter a bash shell
|
|
||||||
inside the container.
|
|
||||||
|
|
||||||
#. From the container shell, you can run FFmpeg and
|
|
||||||
GStreamer commands against the videos in :file:`/home/mers-user/input` as
|
|
||||||
you would normally outside of |MERS|.
|
|
||||||
|
|
||||||
Some sample commands are provided for reference.
|
|
||||||
|
|
||||||
For more information on using the *FFmpeg* commands, refer to the `FFmpeg
|
|
||||||
documentation <https://ffmpeg.org/documentation.html>`_.
|
|
||||||
|
|
||||||
For more information on using the *GStreamer* commands, refer to the
|
|
||||||
`GStreamer documentation
|
|
||||||
<https://gstreamer.freedesktop.org/documentation>`_.
|
|
||||||
|
|
||||||
|
|
||||||
Example: Transcoding using GPU
|
|
||||||
-------------------------------
|
|
||||||
|
|
||||||
The examples below show transcoding using the GPU for processing.
|
|
||||||
|
|
||||||
|
|
||||||
Using a FFmpeg to transcode raw content to SVT-HEVC and mp4:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
ffmpeg -y -vaapi_device /dev/dri/renderD128 -f rawvideo -video_size 320x240 -r 30 -i </home/mers-user/input/test.yuv> -vf 'format=nv12, hwupload' -c:v h264_vaapi -y </home/mers-user/output/test.mp4>
|
|
||||||
|
|
||||||
Using a GStreamer to transcode H264 to H265:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
gst-launch-1.0 filesrc location=</home/mers-user/input/test.264> ! h264parse ! vaapih264dec ! vaapih265enc rate-control=cbr bitrate=5000 ! video/x-h265,profile=main ! h265parse ! filesink location=</home/mers-user/output/test.265>
|
|
||||||
|
|
||||||
|MERS| builds FFmpeg with `HWAccel
|
|
||||||
<https://trac.ffmpeg.org/wiki/HWAccelIntro>`_ enabled which supports VAAPI.
|
|
||||||
Refer to the `FFmpeg wiki on VAAPI
|
|
||||||
<https://trac.ffmpeg.org/wiki/Hardware/VAAPI>`_ and `GStreamer with Media-SDK
|
|
||||||
wiki
|
|
||||||
<https://github.com/Intel-Media-SDK/MediaSDK/wiki/Build-and-use-GStreamer-with-MediaSDK#usage-examples>`_
|
|
||||||
for more usage examples and compatibility information.
|
|
||||||
|
|
||||||
|
|
||||||
Example: Transcoding using CPU
|
|
||||||
------------------------------
|
|
||||||
|
|
||||||
The example below shows transcoding of raw yuv420 content to SVT-HEVC and mp4,
|
|
||||||
using the CPU for processing.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
ffmpeg -f rawvideo -vcodec rawvideo -s 320x240 -r 30 -pix_fmt yuv420p -i </home/mers-user/input/test.yuv> -c:v libsvt_hevc -y </home/mers-user/output/test.mp4>
|
|
||||||
|
|
||||||
Additional generic examples of FFmpeg commands can be found in the
|
|
||||||
`OpenVisualCloud repository
|
|
||||||
<https://github.com/OpenVisualCloud/Dockerfiles/blob/master/doc/ffmpeg.md>`_
|
|
||||||
and used for reference with |MERS|.
|
|
||||||
|
|
||||||
|
|
||||||
Media Analytics
|
|
||||||
===============
|
|
||||||
|
|
||||||
This example shows how to perform analytics and inferences with GStreamer
|
|
||||||
using the CPU for processing.
|
|
||||||
|
|
||||||
The steps here are referenced from the `gst-video-analytics Getting Started
|
|
||||||
Guide <https://github.com/opencv/gst-video-analytics/wiki>`_ except simply
|
|
||||||
substituting the *gst-video-analytics* docker image for the
|
|
||||||
*sysstacks/mers-clearlinux* image.
|
|
||||||
|
|
||||||
The example below shows how to use the |MERS| container image to perform video
|
|
||||||
with object detection and attributes recognition of a video using GStreamer
|
|
||||||
using pre-trained models and sample video files.
|
|
||||||
|
|
||||||
#. On the host system, setup a workspace for data and models:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
mkdir ~/gva
|
|
||||||
mkdir ~/gva/data
|
|
||||||
mkdir ~/gva/data/models
|
|
||||||
mkdir ~/gva/data/models/intel
|
|
||||||
mkdir ~/gva/data/models/common
|
|
||||||
mkdir ~/gva/data/video
|
|
||||||
|
|
||||||
#. Clone the opencv/gst-video-analytics repository into the workspace:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/opencv/gst-video-analytics ~/gva/gst-video-analytics
|
|
||||||
cd ~/gva/gst-video-analytics
|
|
||||||
git submodule init
|
|
||||||
git submodule update
|
|
||||||
|
|
||||||
#. Clone the Open Model Zoo repository into the workspace:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/opencv/open_model_zoo.git ~/gva/open_model_zoo
|
|
||||||
|
|
||||||
#. Use the Model Downloader tool of Open Model Zoo to download ready to use
|
|
||||||
pre-trained models in IR format.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
If you are on a network with outbound proxies, you will need to
|
|
||||||
configure set environment variables with the proxy server.
|
|
||||||
Refer to the documentation on :ref:`proxy` for detailed steps.
|
|
||||||
|
|
||||||
On |CL| systems you will need the *python-extras* bundle.
|
|
||||||
Use :command:`sudo swupd bundle-add python-extras` for the downloader script to work.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
cd ~/gva/open_model_zoo/tools/downloader
|
|
||||||
python3 downloader.py --list ~/gva/gst-video-analytics/samples/model_downloader_configs/intel_models_for_samples.LST -o ~/gva/data/models/intel
|
|
||||||
|
|
||||||
|
|
||||||
#. Copy a video file in h264 or mp4 format to :file:`~/gva/data/video`. Any
|
|
||||||
video with cars, pedestrians, human bodies, and/or human faces can be used.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/intel-iot-devkit/sample-videos.git ~/gva/data/video
|
|
||||||
|
|
||||||
This example simply clones all the video files from the `sample-videos
|
|
||||||
repsoitory <https://github.com/intel-iot-devkit/sample-videos>`_.
|
|
||||||
|
|
||||||
#. From a desktop terminal, allow local access to the X host display.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
xhost local:root
|
|
||||||
|
|
||||||
export DATA_PATH=~/gva/data
|
|
||||||
export GVA_PATH=~/gva/gst-video-analytics
|
|
||||||
export MODELS_PATH=~/gva/data/models
|
|
||||||
export INTEL_MODELS_PATH=~/gva/data/models/intel
|
|
||||||
export VIDEO_EXAMPLES_PATH=~/gva/data/video
|
|
||||||
|
|
||||||
#. Run the *sysstacks/mers-clearlinux* docker image, allowing shared access
|
|
||||||
to the X server and workspace on the host:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
docker run -it --runtime=runc --net=host \
|
|
||||||
-v ~/.Xauthority:/root/.Xauthority \
|
|
||||||
-v /tmp/.X11-unix:/tmp/.X11-unix \
|
|
||||||
-e DISPLAY=$DISPLAY \
|
|
||||||
-e HTTP_PROXY=$HTTP_PROXY \
|
|
||||||
-e HTTPS_PROXY=$HTTPS_PROXY \
|
|
||||||
-e http_proxy=$http_proxy \
|
|
||||||
-e https_proxy=$https_proxy \
|
|
||||||
-v $GVA_PATH:/home/mers-user/gst-video-analytics \
|
|
||||||
-v $INTEL_MODELS_PATH:/home/mers-user/intel_models \
|
|
||||||
-v $MODELS_PATH:/home/mers-user/models \
|
|
||||||
-v $VIDEO_EXAMPLES_PATH:/home/mers-user/video-examples \
|
|
||||||
-e MODELS_PATH=/home/mers-user/intel_models:/home/mers-user/models \
|
|
||||||
-e VIDEO_EXAMPLES_DIR=/home/mers-user/video-examples \
|
|
||||||
sysstacks/mers-clearlinux:latest
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
In the :command:`docker run` command above:
|
|
||||||
|
|
||||||
- :command:`--runtime=runc` specifies the container runtime to be
|
|
||||||
*runc* for this container. It is needed for correct interaction with X
|
|
||||||
server.
|
|
||||||
|
|
||||||
- :command:`--net=host` provides host network access to the container.
|
|
||||||
It is needed for correct interaction with X server.
|
|
||||||
|
|
||||||
- Files :file:`~/.Xauthority` and :file:`/tmp/.X11-unix` mapped to the
|
|
||||||
container are needed to ensure smooth authentication with X server.
|
|
||||||
|
|
||||||
- :command:`-v` instances are needed to map host system directories
|
|
||||||
inside the Docker container.
|
|
||||||
|
|
||||||
- :command:`-e` instances set the Docker container environment
|
|
||||||
variables. Some examples need these variables set correctly in order
|
|
||||||
to operate correctly. Proxy variables are needed if host is behind a
|
|
||||||
firewall.
|
|
||||||
|
|
||||||
|
|
||||||
After running the :command:`docker run` command, it will drop you into a
|
|
||||||
bash shell inside the container.
|
|
||||||
|
|
||||||
#. From the container shell, run a sample analytics program in
|
|
||||||
:file:`~/gva/gst-video-analytics/samples` against your video source.
|
|
||||||
|
|
||||||
Below are sample analytics that can be run against the sample videos.
|
|
||||||
Choose one to run:
|
|
||||||
|
|
||||||
- Samples with *face detection and classification*:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
./gst-video-analytics/samples/shell/face_detection_and_classification.sh $VIDEO_EXAMPLES_DIR/face-demographics-walking-and-pause.mp4
|
|
||||||
./gst-video-analytics/samples/shell/face_detection_and_classification.sh $VIDEO_EXAMPLES_DIR/face-demographics-walking.mp4
|
|
||||||
./gst-video-analytics/samples/shell/face_detection_and_classification.sh $VIDEO_EXAMPLES_DIR/head-pose-face-detection-female-and-male.mp4
|
|
||||||
./gst-video-analytics/samples/shell/face_detection_and_classification.sh $VIDEO_EXAMPLES_DIR/head-pose-face-detection-male.mp4
|
|
||||||
./gst-video-analytics/samples/shell/face_detection_and_classification.sh $VIDEO_EXAMPLES_DIR/head-pose-face-detection-female.mp4
|
|
||||||
|
|
||||||
When running, a video with object detection and attributes recognition
|
|
||||||
(bounding boxes around faces with recognized attributes) should be
|
|
||||||
played.
|
|
||||||
|
|
||||||
.. figure:: /_figures/stacks/mers-fig-1.png
|
|
||||||
:scale: 60%
|
|
||||||
:align: center
|
|
||||||
:alt: Face detection with the Clear Linux* OS Media Reference Stack
|
|
||||||
|
|
||||||
Figure 1: Screenshot of |MERS| running face detection with GSTreamer
|
|
||||||
and OpenVINO.
|
|
||||||
|
|
||||||
- Sample with *vehicle detection*:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
./gst-video-analytics/samples/shell/vehicle_detection_2sources_cpu.sh $VIDEO_EXAMPLES_DIR/car-detection.mp4
|
|
||||||
|
|
||||||
When running, a video with object detection and attributes recognition
|
|
||||||
(bounding boxes around vehicles with recognized attributes) should be
|
|
||||||
played.
|
|
||||||
|
|
||||||
.. figure:: /_figures/stacks/mers-fig-2.png
|
|
||||||
:scale: 60%
|
|
||||||
:align: center
|
|
||||||
:alt: Vehicle detection with the Clear Linux* OS Media Reference Stack
|
|
||||||
|
|
||||||
Figure 2: Screenshot of |MERS| running vehicle detection with
|
|
||||||
GSTreamer and OpenVINO.
|
|
||||||
|
|
||||||
- Sample with *FPS measurement*:
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
./gst-video-analytics/samples/shell/console_measure_fps_cpu.sh $VIDEO_EXAMPLES_DIR/bolt-detection.mp4
|
|
||||||
|
|
||||||
|
|
||||||
Add AOM support
|
|
||||||
***************
|
|
||||||
|
|
||||||
The current version of |MERS| does not include the `Alliance for Open Media
|
|
||||||
<https://aomedia.org/>`_ Video Codec (AOM). AOM can be built from source on an
|
|
||||||
individual basis.
|
|
||||||
|
|
||||||
To add AOM support to the |MERS| image:
|
|
||||||
|
|
||||||
|
|
||||||
#. The following programs are needed to add AOM support to |MERS|: **docker,
|
|
||||||
git, patch**. On |CL| these can be installed with the commands below. For
|
|
||||||
other operating systems, install the appropriate packages.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
sudo swupd bundle-add containers-basic dev-utils
|
|
||||||
|
|
||||||
|
|
||||||
#. Clone the Intel Stacks repository from GitHub.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/intel/stacks.git
|
|
||||||
|
|
||||||
#. Navigate to the directory for the |MERS| image.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
cd stacks/mers/clearlinux/
|
|
||||||
|
|
||||||
#. Apply the patch to the :file:`Dockerfile`.
|
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
patch -p1 < aom-patches/stacks-mers-v2-include-aom.diff
|
|
||||||
|
|
||||||
#. Use the :command:`docker build` command to build a local copy of the
|
|
||||||
MeRS container image tagged as *aom*.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
docker build --no-cache -t sysstacks/mers-clearlinux:aom .
|
|
||||||
|
|
||||||
Once the build has completed successfully, the local image can be used
|
|
||||||
following the same steps in this tutorial by substituting the image name with
|
|
||||||
*sysstacks/mers-clearlinux:aom*.
|
|
||||||
|
|
||||||
|
|
||||||
*Intel, Xeon, OpenVINO, and the Intel logo are trademarks of Intel
|
|
||||||
Corporation or its subsidiaries. OpenCL and the OpenCL logo are trademarks of
|
|
||||||
Apple Inc. used by permission by Khronos.*
|
|
||||||
+2
-8
@@ -15,11 +15,6 @@
|
|||||||
**autospec** is a tool to assist in the automated creation and
|
**autospec** is a tool to assist in the automated creation and
|
||||||
maintenance of RPM packaging in Clear Linux OS.
|
maintenance of RPM packaging in Clear Linux OS.
|
||||||
|
|
||||||
:ref:`dlrs`
|
|
||||||
This tutorial shows you how to run benchmarking workloads in Clear
|
|
||||||
Linux OS using TensorFlow\* or PyTorch\* with the Deep Learning
|
|
||||||
Reference Stack.
|
|
||||||
|
|
||||||
:ref:`docker`
|
:ref:`docker`
|
||||||
Clear Linux OS supports multiple containerization platforms,
|
Clear Linux OS supports multiple containerization platforms,
|
||||||
including a Docker solution.
|
including a Docker solution.
|
||||||
@@ -55,9 +50,8 @@
|
|||||||
|
|
||||||
**Community**
|
**Community**
|
||||||
|
|
||||||
| `Ask the Clear Linux experts <https://clearlinux.org/community/mailing-list>`_
|
|
||||||
| `Clear Linux Forum <https://community.clearlinux.org/>`_
|
| `Clear Linux Forum <https://community.clearlinux.org/>`_
|
||||||
| `Freenode IRC: #clearlinux <https://webchat.freenode.net/#clearlinux>`_
|
| `irc.libera.chat IRC: #clearlinux <https://kiwiirc.com/client/irc.libera.chat/?nick=web_guest|?#clearlinux>`_
|
||||||
|
|
||||||
|
|
||||||
.. container:: video
|
.. container:: video
|
||||||
@@ -75,4 +69,4 @@
|
|||||||
tutorials/index
|
tutorials/index
|
||||||
reference/index
|
reference/index
|
||||||
FAQ/index
|
FAQ/index
|
||||||
collaboration/collaboration
|
collaboration/collaboration
|
||||||
|
|||||||
@@ -39,7 +39,9 @@ System requirements
|
|||||||
*******************
|
*******************
|
||||||
|
|
||||||
|CL| requires an x86 64-bit processor which supports Intel® Streaming SIMD
|
|CL| requires an x86 64-bit processor which supports Intel® Streaming SIMD
|
||||||
Extensions 4.2 (Intel® SSE 4.2), and it requires a system that supports UEFI.
|
Extensions 4.2 (Intel® SSE 4.2).
|
||||||
|
|
||||||
|
For information on the boot loader, see the `clr-boot-manager readme`_ .
|
||||||
|
|
||||||
The |CL| installer performs a system compatibility check upon booting. To
|
The |CL| installer performs a system compatibility check upon booting. To
|
||||||
manually verify system compatibility with |CL|, run the :ref:`compatibility
|
manually verify system compatibility with |CL|, run the :ref:`compatibility
|
||||||
@@ -88,4 +90,6 @@ Network Active Internet connection
|
|||||||
|
|
||||||
|
|
||||||
*Intel, Intel Core, Xeon, Intel Atom, and the Intel logo are trademarks of
|
*Intel, Intel Core, Xeon, Intel Atom, and the Intel logo are trademarks of
|
||||||
Intel Corporation or its subsidiaries.*
|
Intel Corporation or its subsidiaries.*
|
||||||
|
|
||||||
|
.. _clr-boot-manager readme: https://github.com/clearlinux/clr-boot-manager
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
.. _spark:
|
|
||||||
|
|
||||||
Apache\* Spark
|
|
||||||
##############
|
|
||||||
|
|
||||||
This tutorial describes how to install, configure, and run Apache Spark on
|
|
||||||
|CL-ATTR| on a single machine running the master daemon and a worker daemon.
|
|
||||||
|
|
||||||
.. contents::
|
|
||||||
:local:
|
|
||||||
:depth: 1
|
|
||||||
|
|
||||||
Description
|
|
||||||
***********
|
|
||||||
|
|
||||||
Apache Spark is a fast, general-purpose cluster computing system with
|
|
||||||
the following features:
|
|
||||||
|
|
||||||
* Provides high-level APIs in Java\*, Scala\*, Python\*, and R\*.
|
|
||||||
* Includes an optimized engine that supports general execution graphs.
|
|
||||||
* Supports high-level tools including Spark SQL, MLlib, GraphX, and Spark
|
|
||||||
Streaming.
|
|
||||||
|
|
||||||
|
|
||||||
Prerequisites
|
|
||||||
*************
|
|
||||||
|
|
||||||
* |CL| installed on your host system.
|
|
||||||
|
|
||||||
For detailed instructions on installing |CL| on a bare metal system, visit
|
|
||||||
the :ref:`bare metal installation guide <bare-metal-install-desktop>`.
|
|
||||||
|
|
||||||
* Before installing any new packages, update |CL| with the following command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo swupd update
|
|
||||||
|
|
||||||
Install Apache Spark
|
|
||||||
********************
|
|
||||||
|
|
||||||
Apache Spark is included in the :command:`big-data-basic` bundle. To install the
|
|
||||||
framework, run the following command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo swupd bundle-add big-data-basic
|
|
||||||
|
|
||||||
Configure Apache Spark
|
|
||||||
**********************
|
|
||||||
|
|
||||||
#. Create the configuration directory:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo mkdir /etc/spark
|
|
||||||
|
|
||||||
#. Copy the default templates from :file:`/usr/share/defaults/spark` to
|
|
||||||
:file:`/etc/spark`:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo cp /usr/share/defaults/spark/* /etc/spark
|
|
||||||
|
|
||||||
.. note:: Since |CL| is a stateless system, you should never modify the
|
|
||||||
files under the :file:`/usr/share/defaults` directory. The software
|
|
||||||
updater overwrites those files.
|
|
||||||
|
|
||||||
#. Copy the template files shown below to create custom configuration files:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo cp /etc/spark/spark-defaults.conf.template /etc/spark/spark-defaults.conf
|
|
||||||
sudo cp /etc/spark/spark-env.sh.template /etc/spark/spark-env.sh
|
|
||||||
sudo cp /etc/spark/log4j.properties.template /etc/spark/log4j.properties
|
|
||||||
|
|
||||||
#. Edit the :file:`/etc/spark/spark-env.sh` file and add the
|
|
||||||
:envvar:`SPARK_MASTER_HOST` variable. Replace the example address below
|
|
||||||
with your localhost IP address. View your IP address using the
|
|
||||||
:command:`hostname -I` command.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
SPARK_MASTER_HOST="10.300.200.100"
|
|
||||||
|
|
||||||
.. note:: This optional step enables the master's web user interface to
|
|
||||||
view information needed later in this tutorial.
|
|
||||||
|
|
||||||
#. Edit the :file:`/etc/spark/spark-defaults.conf` file and update the
|
|
||||||
:envvar:`spark.master` variable with the `SPARK_MASTER_HOST` address and port
|
|
||||||
`7077`.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
spark.master spark://10.300.200.100:7077
|
|
||||||
|
|
||||||
Start the master server and a worker daemon
|
|
||||||
*******************************************
|
|
||||||
|
|
||||||
#. Start the master server:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo /usr/share/apache-spark/sbin/./start-master.sh
|
|
||||||
|
|
||||||
#. Start one worker daemon and connect it to the master using the
|
|
||||||
:envvar:`spark.master` variable defined earlier:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo /usr/share/apache-spark/sbin/./start-slave.sh spark://10.300.200.100:7077
|
|
||||||
|
|
||||||
#. Open an internet browser and view the worker daemon information using
|
|
||||||
the master's IP address and port `8080`:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
http://10.300.200.100:8080
|
|
||||||
|
|
||||||
Run the Spark wordcount example
|
|
||||||
*******************************
|
|
||||||
|
|
||||||
#. Run the wordcount example using a file on your local host and output the
|
|
||||||
results to a new file with the following command:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo spark-submit /usr/share/apache-spark/examples/src/main/python/wordcount.py ~/Documents/example_file > ~/Documents/results
|
|
||||||
|
|
||||||
#. Open an internet browser and view the application information using
|
|
||||||
the master's IP address and port `8080`:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
http://10.300.200.100:8080
|
|
||||||
|
|
||||||
#. View the results of the wordcount application in the :file:`~/Documents/results` file.
|
|
||||||
|
|
||||||
**Congratulations!**
|
|
||||||
|
|
||||||
You have successfully installed and set up a standalone Apache Spark cluster,
|
|
||||||
and ran a simple wordcount example.
|
|
||||||
@@ -31,7 +31,6 @@ sorted by difficulty level! Learn about :ref:`how we evaluate tutorials <tutoria
|
|||||||
Moderate
|
Moderate
|
||||||
|
|
||||||
- :ref:`hadoop`
|
- :ref:`hadoop`
|
||||||
- :ref:`spark`
|
|
||||||
- :ref:`broadcom`
|
- :ref:`broadcom`
|
||||||
- :ref:`fmv`
|
- :ref:`fmv`
|
||||||
- :ref:`hpc`
|
- :ref:`hpc`
|
||||||
|
|||||||
@@ -115,6 +115,14 @@ running quickly.
|
|||||||
|
|
||||||
sudo systemctl stop docker
|
sudo systemctl stop docker
|
||||||
sudo systemctl stop containerd
|
sudo systemctl stop containerd
|
||||||
|
|
||||||
|
|
||||||
|
#. Install git as it's a dependency of the :file:`create_stack.sh`.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sudo swupd bundle-add git
|
||||||
|
|
||||||
|
|
||||||
#. Run the :file:`create_stack.sh` script to initialize the Kubernetes node
|
#. Run the :file:`create_stack.sh` script to initialize the Kubernetes node
|
||||||
and setup a container network plugin.
|
and setup a container network plugin.
|
||||||
@@ -124,7 +132,7 @@ running quickly.
|
|||||||
sudo /usr/share/clr-k8s-examples/create_stack.sh minimal
|
sudo /usr/share/clr-k8s-examples/create_stack.sh minimal
|
||||||
|
|
||||||
#. Follow the output on the screen and continue onto the section on `using
|
#. Follow the output on the screen and continue onto the section on `using
|
||||||
your cluster <Use your cluster>`_.
|
your cluster <#use-your-cluster>`_.
|
||||||
|
|
||||||
|
|
||||||
Uninstalling
|
Uninstalling
|
||||||
@@ -184,7 +192,7 @@ below are necessary to ensure those preflight checks pass successfully.
|
|||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
sudo systemctl mask $(sed -n -e 's#^/dev/\([0-9a-z]*\).*#dev-\1.swap#p' /proc/swaps) 2>/dev/null
|
sudo systemctl mask $(sed -n -e 's#^/var/\([0-9a-z]*\).*#var-\1.swap#p' /proc/swaps) 2>/dev/null
|
||||||
sudo swapoff -a
|
sudo swapoff -a
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ and is available in the database-basic |CL| bundle.
|
|||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
sudo swupd bundle-add database-basic
|
sudo swupd bundle-add mariadb
|
||||||
|
|
||||||
#. To start MariaDB after it is installed and set it to start automatically on
|
#. To start MariaDB after it is installed and set it to start automatically on
|
||||||
boot, enter the following commands:
|
boot, enter the following commands:
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ and sustainable on |CL|.
|
|||||||
|
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
|
|
||||||
c. Add the service as a depndency to the |CL| updates trigger causing the
|
c. Add the service as a dependency to the |CL| updates trigger causing the
|
||||||
service to run after every update.
|
service to run after every update.
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|||||||
+58
-103
@@ -32,105 +32,69 @@ Prerequisites
|
|||||||
Install the DKMS kernel
|
Install the DKMS kernel
|
||||||
=======================
|
=======================
|
||||||
|
|
||||||
If you do not currently use a DKMS kernel, install it by using one of the options below.
|
.. include:: ../guides/kernel/kernel-modules-dkms.rst
|
||||||
|
:start-after: kernel-modules-dkms-install-begin-alt:
|
||||||
#. Check whether you have an LTS or native kernel:
|
:end-before: kernel-modules-dkms-install-end:
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
uname -r
|
|
||||||
|
|
||||||
a. If 'native' appears in the kernel name, then install a native
|
|
||||||
kernel with DKMS support:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo swupd bundle-add kernel-native-dkms
|
|
||||||
|
|
||||||
#. If 'lts' appears in the kernel name, then install the latest LTS
|
|
||||||
kernel with DKMS support:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo swupd bundle-add kernel-lts-dkms
|
|
||||||
|
|
||||||
#. Reboot and make sure you can start the new kernel.
|
|
||||||
|
|
||||||
Bundles
|
Bundles
|
||||||
=======
|
=======
|
||||||
|
|
||||||
Before you install OpenZFS, install the bundles that contain the build dependencies.
|
Before installing OpenZFS, install the bundles that contain the build dependencies.
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: console
|
||||||
|
|
||||||
sudo swupd bundle-add os-core-dev devpkg-openssl devpkg-util-linux
|
sudo swupd bundle-add wget devpkg-openssl devpkg-util-linux
|
||||||
|
|
||||||
Install
|
Install
|
||||||
*******
|
*******
|
||||||
|
|
||||||
Clone OpenZFS code
|
Download OpenZFS release
|
||||||
==================
|
========================
|
||||||
|
|
||||||
|
In this section, we download release 2.0.0 directly from the `OpenZFS repository` (the latest available as of the latest revision of this page).
|
||||||
|
|
||||||
In this section, you download the source code directly from the `ZFS on Linux repository`_.
|
Download release 2.0.0
|
||||||
|
|
||||||
.. note::
|
.. code-block:: console
|
||||||
|
|
||||||
OpenZFS will not run from this location. We copy the source code into a DKMS directory in the following steps.
|
cd /usr/src
|
||||||
|
sudo wget https://github.com/openzfs/zfs/releases/download/zfs-2.0.0/zfs-2.0.0.tar.gz
|
||||||
#. Create a directory. In this example, we use :file:`/opt/src/zfs`,
|
sudo tar -xvf zfs-2.0.0.tar.gz
|
||||||
but you can choose any workspace you like.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo mkdir -p /tmp/zfs/
|
|
||||||
|
|
||||||
#. Clone the repository.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
git clone https://github.com/openzfs/zfs.git /tmp/zfs
|
|
||||||
sudo cp -Rv /tmp/zfs /opt/src/zfs
|
|
||||||
|
|
||||||
Remember where you clone the git repository because you will need it
|
|
||||||
when you upgrade ZFS. Do not delete the source location when you have completed this tutorial. You will need it later.
|
|
||||||
|
|
||||||
Compile the module
|
Compile the module
|
||||||
==================
|
==================
|
||||||
|
|
||||||
We will build the module using DKMS. This will enable us to keep the module up to date as new kernels are released in the future.
|
We will build the module using DKMS. This will enable us to keep the module up to date as new kernels are released in the future.
|
||||||
|
|
||||||
|
The ZFS distribution provides a script to build a suitable dkms.conf file.
|
||||||
|
|
||||||
#. Check out the tagged version that you plan to use. As of the time of this
|
|
||||||
writing, the latest release tag is `0.8.4`:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
cd /opt/src/zfs
|
|
||||||
git checkout zfs-0.8.4
|
|
||||||
|
|
||||||
#. Copy the source code into `/usr/src/zfs-0.8.4`. This exposes the source
|
|
||||||
code to DKMS. We will build the code from the new location:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
sudo cp -Rv /opt/src/zfs /usr/src/zfs-0.8.4
|
|
||||||
|
|
||||||
#. The ZFS distribution provides a script to build a suitable dkms.conf file.
|
|
||||||
Build dkms.conf and install it into the DKMS tree.
|
Build dkms.conf and install it into the DKMS tree.
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: console
|
||||||
|
|
||||||
cd /usr/src/zfs-0.8.4
|
cd /usr/src/zfs-2.0.0
|
||||||
sudo scripts/dkms.mkconf -n zfs -v 0.8.4 -f dkms.conf
|
sudo scripts/dkms.mkconf -n zfs -v 2.0.0 -f dkms.conf
|
||||||
sudo dkms add -m zfs -v 0.8.4
|
sudo dkms add -m zfs -v 2.0.0
|
||||||
sudo dkms build -m zfs -v 0.8.4
|
sudo dkms build -m zfs -v 2.0.0
|
||||||
sudo dkms install -m zfs -v 0.8.4
|
sudo dkms install -m zfs -v 2.0.0
|
||||||
|
|
||||||
|
Observe that this install the zfs kernel modules to:
|
||||||
|
|
||||||
#. Observe that this install the zfs kernel modules to:
|
|
||||||
:file:`/usr/lib/modules/<kernel-name>/extra/zfs`
|
:file:`/usr/lib/modules/<kernel-name>/extra/zfs`
|
||||||
|
|
||||||
|
Compile userspace tools
|
||||||
|
=======================
|
||||||
|
|
||||||
In addition, this installs the zfs userspace tools to:
|
Here we compile and install the zfs userspace tools (e.g., zpool, zfs, etc.).
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
cd /usr/src/zfs-2.0.0
|
||||||
|
sudo ./configure
|
||||||
|
sudo make
|
||||||
|
sudo make install
|
||||||
|
|
||||||
|
The binaries are installed at the following directory. While not required, it's recommended to add :file:`/usr/local/sbin` to your path variable.
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
@@ -163,21 +127,14 @@ We will build the module using DKMS. This will enable us to keep the module up t
|
|||||||
|--ztest
|
|--ztest
|
||||||
|--zstreamdump
|
|--zstreamdump
|
||||||
|--+ src/
|
|--+ src/
|
||||||
|--+ zfs-0.8.4/
|
|--+ zfs-2.0.0/
|
||||||
|--+ spl-0.8.4/
|
|--+ spl-2.0.0/
|
||||||
|
|
||||||
|
|
||||||
#. Load the new kernel module:
|
|
||||||
|
|
||||||
.. code-block: bash
|
|
||||||
|
|
||||||
sudo modprobe zfs
|
|
||||||
|
|
||||||
|
|
||||||
Set up systemd
|
Set up systemd
|
||||||
==============
|
==============
|
||||||
|
|
||||||
You should now have these unit files available.
|
We now have these unit files available.
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
@@ -189,19 +146,16 @@ You should now have these unit files available.
|
|||||||
zfs-share.service
|
zfs-share.service
|
||||||
zfs-volume-wait.service
|
zfs-volume-wait.service
|
||||||
|
|
||||||
OpenZFS requires you to explicitly install and enable the services you want.
|
|
||||||
If you want to use other ZFS service units, you could create symlinks for them, similar to the example below.
|
OpenZFS requires that we explicitly install and enable the services desired.
|
||||||
|
|
||||||
To use ZFS automatic zpool import and filesystem mount services, link the systemd.unit files into :file:`/etc` and enable them.
|
To use ZFS automatic zpool import and filesystem mount services, enable them.
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: console
|
||||||
|
|
||||||
sudo ln -s ./etc/systemd/system/zfs-import-cache.service /etc/systemd/system/
|
|
||||||
sudo ln -s ./etc/systemd/system/zfs-mount.service /etc/systemd/system/
|
|
||||||
|
|
||||||
sudo systemctl enable zfs-import-cache
|
sudo systemctl enable zfs-import-cache
|
||||||
sudo systemctl enable zfs-import.target
|
sudo systemctl enable zfs-import.target
|
||||||
|
sudo systemctl enable zfs-import-scan
|
||||||
sudo systemctl enable zfs-mount
|
sudo systemctl enable zfs-mount
|
||||||
sudo systemctl enable zfs.target
|
sudo systemctl enable zfs.target
|
||||||
|
|
||||||
@@ -211,55 +165,56 @@ Load the kernel module at boot
|
|||||||
|
|
||||||
OpenZFS kernel modules must be loaded before any OpenZFS filesystems are mounted. For convenience, load the kernel modules at boot.
|
OpenZFS kernel modules must be loaded before any OpenZFS filesystems are mounted. For convenience, load the kernel modules at boot.
|
||||||
|
|
||||||
#. Systemd uses the `/etc/modules-load.d/` directory to load out-of-tree
|
Systemd uses the `/etc/modules-load.d/` directory to load out-of-tree kernel modules. Make sure that the directory exists:
|
||||||
kernel modules. Make sure that the directory exists:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
sudo mkdir -p /etc/modules-load.d
|
sudo mkdir -p /etc/modules-load.d
|
||||||
|
|
||||||
#. Create the configuration file:
|
Create the configuration file:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
echo "zfs" | sudo tee /etc/modules-load.d/01-zfs.conf
|
echo "zfs" | sudo tee /etc/modules-load.d/01-zfs.conf
|
||||||
|
|
||||||
#. Reboot your system. zfs.ko should be loaded automatically.
|
Reboot your system. zfs.ko should be loaded automatically (the module should appear in the outout of command :file:`lsmod`).
|
||||||
|
|
||||||
.. CAUTION::
|
.. CAUTION::
|
||||||
|
|
||||||
When the |CL| kernel is upgraded, DKMS will attempt to rebuild your OpenZFS module for the new kernel.
|
When the |CL| kernel is upgraded, DKMS will attempt to rebuild the OpenZFS module for the new kernel.
|
||||||
|
|
||||||
- DKMS may not have rebuilt the module
|
- DKMS may not have rebuilt the module
|
||||||
- DKMS may not have auto-installed the module
|
- DKMS may not have auto-installed the module
|
||||||
- The new kernel might introduce breaking changes that prevent zfs
|
- The new kernel might introduce breaking changes that prevent zfs
|
||||||
from compiling
|
from compiling
|
||||||
|
|
||||||
To fix this situation, you may have to recompile zfs.ko with the new kernel code. OpenZFS *might* not compile at all with the new kernel.
|
To fix this situation, recompile zfs.ko with the new kernel code. OpenZFS *might* not compile at all with the new kernel.
|
||||||
|
|
||||||
.. CAUTION::
|
.. CAUTION::
|
||||||
|
|
||||||
**Be sure you don't put anything on an OpenZFS pool that you would need
|
**Be sure not to put anything on an OpenZFS pool that will be needed to rebuild kernel modules.** Ensure compatibility of OpenZFS with new Linux kernels when released.
|
||||||
in order to rebuild kernel modules.** You must ensure the compatibility of OpenZFS with new Linux kernels when they are released.
|
|
||||||
|
|
||||||
Troubleshooting
|
Troubleshooting
|
||||||
===============
|
===============
|
||||||
|
|
||||||
If you suspect an issue with DKMS rebuilding your module, you can check two places for information. The dkms-new-kernel service will show status that may help in troubleshooting:
|
If you suspect an issue with DKMS rebuilding your module, you can check two places for information. The dkms-new-kernel service will show status that may help in troubleshooting:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: console
|
||||||
|
|
||||||
systemctl status dkms-new-kernel.service
|
systemctl status dkms-new-kernel.service
|
||||||
|
|
||||||
Also, the systemd journal may have important information:
|
The systemd journal may also have important information:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: console
|
||||||
|
|
||||||
journalctl -xe
|
journalctl -xe
|
||||||
|
|
||||||
|
.. CAUTION::
|
||||||
|
As of OpenZFS 2.0.0, the included file :file:`script/dkms.mkconf` contains a minor incompability -- it calls the command :file:`lsb_release`, which is not available on Clear Linux by default. It is trivial to edit :file:`dkms.mkconf` and remove the singular reference to :file:`lsb_release` without any ill effects, and then execute the :file:`dkms` commands above. However, keeping the file as provided is perfectly fine, but **will result in :file:`dkms` warnings**.
|
||||||
|
|
||||||
Next steps
|
Next steps
|
||||||
**********
|
**********
|
||||||
You are ready to create zpools and datasets! For more information on using ZFS, see:
|
You're now ready to create zpools and datasets! For more information on using ZFS, see:
|
||||||
|
|
||||||
* `FreeBSD Handbook chapter on ZFS`_
|
* `FreeBSD Handbook chapter on ZFS`_
|
||||||
* `ZFS-on-Linux issue tracker`_
|
* `ZFS-on-Linux issue tracker`_
|
||||||
@@ -267,4 +222,4 @@ You are ready to create zpools and datasets! For more information on using ZFS,
|
|||||||
.. _FreeBSD Handbook chapter on ZFS: https://www.freebsd.org/doc/handbook/zfs.html
|
.. _FreeBSD Handbook chapter on ZFS: https://www.freebsd.org/doc/handbook/zfs.html
|
||||||
.. _ZFS-on-Linux issue tracker: https://github.com/openzfs/zfs/issues/10068
|
.. _ZFS-on-Linux issue tracker: https://github.com/openzfs/zfs/issues/10068
|
||||||
.. _ZFS on Linux repository: https://github.com/openzfs/zfs
|
.. _ZFS on Linux repository: https://github.com/openzfs/zfs
|
||||||
.. _OpenZFS* file system and volume manager: https://github.com/openzfs/zfs
|
.. _OpenZFS* file system and volume manager: https://github.com/openzfs/zfs
|
||||||
|
|||||||
Reference in New Issue
Block a user