Update index.rst

Update to the new IRC channel

.github/workflows/check.yml

@@ -13,11 +13,12 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
         

.github/workflows/linkcheck.yml

@@ -0,0 +1,22 @@
+name: Linkcheck
+
+on:
+  schedule:
+    - cron: '0 0 * * Mon,Wed,Fri'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install dependencies
+      run: |
+        export PATH="$HOME/.local/bin:$PATH"
+        sudo apt-get install -y python3-setuptools
+        pip3 install --user -r requirements.txt
+    - name: CheckLinks
+      run: |
+        export PATH="$HOME/.local/bin:$PATH"
+        make linkcheck

.github/workflows/main.yml

@@ -15,12 +15,13 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
     - name: Pre-deploy
       run: |

.github/workflows/publish.yml

@@ -15,12 +15,13 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
     - name: Pre-deploy
       run: |

.gitignore

@@ -17,3 +17,13 @@ bundles.html.txt
 
 # ignore the venv, used for running make py
 venv
+
+#ignore for reDocs GUI
+.tox
+MANIFEST
+
+# ignore artifacts of man page generation
+source/_scripts/_python/manpages/*.rst
+source/reference/manpages
+source/_scripts/_python/manpages/*/
+source/reference/man-pages.rst

Makefile

@@ -25,6 +25,12 @@ linkcheck:
 py:
 	make -C source py
 
+man:
+	make -C source man
+
+clean-man:
+	make -C source clean-man
+
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"

README.rst

@@ -63,8 +63,13 @@ running ``make html``:
 Open one of the HTML pages found in ``source/_build/html`` in a web browser
 to view the rendered documentation.
 
-If you want to build the documentation exactly as seen on the website, use
+This build will generate several warnings as there are two other optional make commands required to build the full documentation.
-``make py`` followed by ``make htmlall``. This builds some
+
+1. ``make py`` to generate the bundle reference material.
+2. ``make man`` to generate man page reference material.
+
+To build the documentation exactly as seen on the website, use
+``make man``, ``make py``, and ``make htmlall``. This builds both
 external dependencies and all supported languages.
 
 Use virtualenv 
@@ -163,6 +168,9 @@ build before building again by running ``make clean``:
 This will completely remove the previous build output, including artifacts 
 from the `make venv` target when done outside an active venv.
 
+Before running ``make man``, please run ``make clean-man`` to clear out any
+previous attempts.
+
 Convenience script
 ==================
 

make.bat

@@ -5,7 +5,9 @@ REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
-set BUILDDIR=build
+
+set SCRIPTDIR=source\_scripts\_python
+set BUILDDIR=source\_build
 set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source
 set I18NSPHINXOPTS=%SPHINXOPTS% source
 if NOT "%PAPER%" == "" (
@@ -30,6 +32,7 @@ if "%1" == "help" (
 	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
 	echo.  text       to make text files
 	echo.  man        to make manual pages
+	echo.  clean-man  to clean up after man page generation
 	echo.  texinfo    to make Texinfo files
 	echo.  gettext    to make PO message catalogs
 	echo.  changes    to make an overview over all changed/added/deprecated items
@@ -77,6 +80,17 @@ if "%1" == "html" (
 	if errorlevel 1 exit /b 1
 	echo.
 	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	copy source\_scripts\js\copybutton.js %BUILDDIR%\html\_static
+	goto end
+)
+
+if "%1" == "py" (
+	cd %SCRIPTDIR%
+	python.exe bundle_lister.py
+	copy bundles.html.txt ..\..\reference\bundles
+	for /d %%i in (cloned_repo\*) do rmdir /q /s %%i
+	del /q /s bundles.html.txt
+	echo "Python bundle script finished successfully!"
 	goto end
 )
 
@@ -186,10 +200,14 @@ if "%1" == "text" (
 )
 
 if "%1" == "man" (
-	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+    cd source/_scripts/_python/manpages
-	if errorlevel 1 exit /b 1
+    man.bat man
-	echo.
+	goto end
-	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+)
+
+if "%1" == "clean-man" (
+	cd source/_scripts/_python/manpages
+	man.bat clean-man
 	goto end
 )
 

requirements.txt

@@ -1,9 +1,10 @@
 breathe==4.9.1
-sphinx==1.8
+sphinx==2.2.0
 docutils==0.14
 sphinx_rtd_theme
 sphinx-intl==2.0.0
 sphinx-sitemap==1.0.2
-Jinja2==2.10.1
+Jinja2==2.11.3
 GitPython==3.0.8
 sphinx-tabs
+sphinx-copybutton

setup.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='Clear Linux Documentation',
+      version='',
+      description='Sphinx build of Clear Linux documentation',
+      author='Many',
+      author_email='kevin.putnam@intel.com',
+      url='https://github.com/clearlinux/clear-linux-documentation/',
+      )

source/FAQ/index.rst

@@ -198,6 +198,8 @@ Visual Studio Code
 
 |
 
+.. _licensing_restrict:
+
 Is FFmpeg available?
 ====================
 
@@ -224,8 +226,8 @@ ZFS is not available with |CL| because of copyright and licensing
 complexities. BTRFS is an alternative filesystem that is available in |CL|
 natively.
 
-A user on GitHub notes that the `ZFS kernel module can be compiled, built, and
+A community contributed tutorial has been shared on how to :ref:`manually
-installed manually <https://github.com/clearlinux/distribution/issues/631>`_.
+install ZFS <zfs>`.
 
 |
 

source/Makefile

@@ -50,6 +50,8 @@ help:
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  coverage   to run coverage check of the documentation (if enabled)"
 	@echo "  py         to trigger an update of bundle content"
+	@echo "  man        to pull and create manpage rst files in reference section"
+	@echo "  clean-man  to clean up manpage generation"
 
 clean:
 	rm -rf $(BUILDDIR)/*
@@ -71,6 +73,7 @@ htmlzh:
 
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	cp _scripts/js/copybutton.js $(BUILDDIR)/html/_static
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
@@ -210,3 +213,9 @@ pseudoxml:
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+man:
+	$(MAKE) -C $(SCRIPTDIR)/manpages man
+
+clean-man:
+	$(MAKE) -C $(SCRIPTDIR)/manpages clean-man

source/_scripts/_python/manpages/Makefile

@@ -0,0 +1,25 @@
+man:
+	git clone https://github.com/clearlinux/clr-man-pages.git
+	git clone https://github.com/clearlinux/clr-power-tweaks.git
+	git clone https://github.com/clearlinux/clrtrust.git
+	git clone https://github.com/clearlinux/mixer-tools.git
+	git clone https://github.com/clearlinux/swupd-client.git
+	git clone https://github.com/clearlinux/telemetrics-client.git
+	git clone https://github.com/clearlinux/tallow.git
+	git clone https://github.com/clearlinux/micro-config-drive.git
+	python3 manpages.py
+	- mkdir ../../../reference/manpages
+	cp *.rst ../../../reference/manpages
+
+clean-man:
+	- rm -rf clr-man-pages
+	- rm -rf clr-power-tweaks
+	- rm -rf clrtrust
+	- rm -rf mixer-tools
+	- rm -rf swupd-client
+	- rm -rf telemetrics-client
+	- rm -rf tallow
+	- rm -rf micro-config-drive
+	- rm *.rst
+	- rm ../../../reference/manpages/*.rst
+	- rm ../../../reference/man-pages.rst

source/_scripts/_python/manpages/man.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+REM Command file for building man pages
+
+if "%1" == "man" (
+    git clone https://github.com/clearlinux/clr-man-pages.git
+    git clone https://github.com/clearlinux/clr-power-tweaks.git
+    git clone https://github.com/clearlinux/clrtrust.git
+    git clone https://github.com/clearlinux/mixer-tools.git
+    git clone https://github.com/clearlinux/swupd-client.git
+    git clone https://github.com/clearlinux/telemetrics-client.git
+    git clone https://github.com/clearlinux/tallow.git
+    git clone https://github.com/clearlinux/micro-config-drive.git
+    python.exe manpages.py
+    mkdir ..\..\..\reference\manpages
+    copy *.rst ..\..\..\reference\manpages
+    goto end
+)
+
+if "%1" == "clean-man" (
+    rmdir /q /s clr-man-pages
+    rmdir /q /s clr-power-tweaks
+    rmdir /q /s clrtrust
+    rmdir /q /s mixer-tools
+    rmdir /q /s swupd-client
+    rmdir /q /s telemetrics-client
+    rmdir /q /s tallow
+    rmdir /q /s micro-config-drive
+    del *.rst
+    del ..\..\..\reference\manpages\*.rst
+    del ..\..\..\reference\man-pages.rst
+    goto end
+)
+
+:end

source/_scripts/_python/manpages/manpages.py

@@ -0,0 +1,196 @@
+#
+# manpages.py
+#
+# maintainer: intelkevinputnam
+#
+# usage: python3 manpages.py
+#
+# dependencies: 1. clone of Clear Linux documentation https://github.com/clearlinux/clear-linux-documentation
+#               2. Makefile or make.bat to create directories and clone repositories (``make man`` to generate. ``make clean-man`` to clean up.)
+#
+# This script does 4 things:
+#
+# 1. Find and move reStructuredText versions of man pages (looks for manName.sectionNumber.fileExtension: mixer.1.rst) to reference/manpages directory of Clear Linux docs.
+# 2. Massage markdown and reStructuredText man pages into a normalized format.
+# 3. Add cross reference links wherever man pages reference each other.
+# 4. Create manpages.rst in reference directory (already included in toctree of reference/index.rst)
+#
+#
+
+import sys
+import re
+import subprocess
+from os import listdir
+from os.path import join, isfile
+from shutil import copyfile
+
+paths = ["clr-man-pages","clr-power-tweaks/man","clrtrust/man","micro-config-drive/docs/","mixer-tools/docs","swupd-client/docs","tallow/man","telemetrics-client/docs/man"]
+pathToRefSection = "../../../reference/"
+manPageRegex = '.[0-9]+.rst'
+mdManPageRegex = '.[0-9]+.md'
+mdBoldItalicRegex = '\\*\\*\\`[a-z-._]+\\`\\*\\*'
+manFiles = []
+manGroups = {}
+manNamePerms = {}
+seeAlsoRefs = {}
+gitHubGroup = "https://github.com/clearlinux/"
+TOC = "\n.. contents::\n    :local:\n"
+manPageRest = """.. _man-pages:
+
+Man pages
+#########
+
+These pages are generated from `Clear Linux\\* tool
+repositories <https://github.com/clearlinux>`__. Updated |today|.
+
+
+"""
+manTOC = """.. toctree::
+   :maxdepth: 1
+
+"""
+
+def getPages(paths):
+    for path in paths:
+        manGroups[path] = []
+        files = listdir(path)
+        for file in files:
+            if file.endswith(".rst"):
+                regex_found = re.search(manPageRegex,file)
+                if regex_found:
+                    filePath = join(path,file)
+                    copyfile(filePath,file)
+                    manFiles.append(file)
+                    manGroups[path].append(file)
+                    addManNamePermutations(file)
+            elif file.endswith(".md"):
+                regex_found = re.search(mdManPageRegex,file)
+                if regex_found:
+                    filePath = join(path,file)
+                    rstFileName = processMDFile(file,filePath)
+                    manFiles.append(rstFileName)
+                    manGroups[path].append(rstFileName)
+                    addManNamePermutations(rstFileName)
+
+def processMDFile(fileName,filePath):
+    manName = fileName.rstrip('.md')
+    manSection = manName.split('.')[-1]
+    manName = manName.rstrip('.' + manSection)
+    lines = []
+    headerInsert = ""
+    with open(filePath,'r') as m:
+        lines = m.readlines()
+        if "SYNOPSIS" in lines[0]:
+                header='='*len(manName)
+                lines.insert(0,"# " + manName + "\n\n:Manual section: "+manSection+"\n\n")
+        index = 0
+        first = True
+        for line in lines:
+            newLine = line.replace("**`","`") #Fix some markdown formatting weirdness that doesn't translate to reST
+            newLine = newLine.replace("`**","`")
+            if newLine.startswith("#") and not newLine.startswith("##"): #Fix if all headers are first level
+                if first:
+                    first = False
+                else:
+                    newLine = newLine.replace("#","##")
+            if "===" in newLine: #fix rst style header that is actually description in some docs
+                newLine = ""
+                headerInsert = manName + "\n" + '='*len(manName) + "\n\n"
+            lines[index] = newLine
+            index += 1
+        lines.insert(0,headerInsert)
+
+    with open(filePath,'w') as md:
+        md.writelines(lines)
+        rstFilePath = filePath.replace(".md",".rst")
+        rstFileName = fileName.replace(".md",".rst")
+
+    command = "pandoc " +  filePath + " -o " + rstFilePath
+    subprocess.run(command, shell=True)
+    if isfile(rstFilePath):
+        copyfile(rstFilePath,rstFileName)
+    return rstFileName
+
+def addManNamePermutations(fileName): #Based on all the syntactical permutations of references to man pages in the documents.
+    (manName,subsection) = getNameAndSubsection(fileName)
+    manNames = []
+    manNames.append("``" + manName + "``\\(" + subsection +")") # ``mixer.init``\(1)
+    manNames.append("``" + manName + "``\\ (" + subsection +")") # ``mixer.init``\ (1)
+    manNames.append("**``" + manName + "(" + subsection + ")``**")  # **`mixer.init(1)`**
+    manNames.append("``" + manName + "(" + subsection + ")``") # ``mixer.init(1)``
+    manNames.append("**" + manName + "(" + subsection + ")**") # **mixer.init(1)**
+    manNames.append("`" + manName + "(" + subsection + ")`") # `mixer.init(1)`
+    manNamePerms[(manName,subsection)] = manNames
+
+def getNameAndSubsection(manFileName):
+    manName = manFileName.rstrip('.rst')
+    manSection = manName.split('.')[-1]
+    manName = manName.rstrip('.'+manSection)
+    return (manName,manSection)
+
+def linkToMan(manName,manSection):
+    return "`" + manName + "(" + manSection + ") <" + manName + "." + manSection + ".html>`__"
+ 
+def buildManName(name,section):
+    return name + "(" + section + ")"
+
+def updateManPages():
+    #makeSeeAlsoReplacements()
+    for file in manFiles:
+        manFile = ""
+        with open(file,'r',encoding="utf8") as f:
+            manFile = f.read()
+            #manFile = addTOC(manFile,file) # Not convinced adding TOC adds value.
+            #
+            # Add linked cross referencing for all manpages discovered.
+            # 1. the manNamePerms dictionary is created once all of the man page source files are discovered
+            # 2. Each document is checked for each permutation. 
+            # 3. When a permuation is found it is replaced with a normalized version
+            # 4. Once normalized it is turned into a reStructuredText link (def linkToMan)
+            # 
+            for nameAndSection, listOfPerms in manNamePerms.items():
+                for perm in listOfPerms:
+                    manFile = manFile.replace(perm,buildManName(nameAndSection[0],nameAndSection[1]))
+            for manName, doNotUse in manNamePerms.items():
+                manFile = manFile.replace(buildManName(manName[0],manName[1]),linkToMan(manName[0],manName[1]))
+        with open(file,'w',encoding="utf8") as w:
+            w.write(manFile)
+
+def addTOC(manContent,file):
+# Find the first instance of the manpage Name
+# Skip the underline of the header
+# Add a new line
+# Add contents directive with local modifier
+# Add a new line
+    manSectionMeta = ":Manual section:"
+    manContentLines = manContent.split('\n')
+    index = 0
+    for line in manContentLines:
+        if manSectionMeta in line:
+            manContentLines.insert(index + 2,TOC)
+            break
+        index += 1
+    output = ""
+    for line in manContentLines:
+        output = output + line + "\n"
+    return output
+
+def createManpagesRST():
+    filePath = join(pathToRefSection,"man-pages.rst")
+    with open(filePath,'w') as f:
+        manGrouping = manPageRest
+        for path, fileList in manGroups.items():
+            repoName = path.split("/")[0]
+            repoLink = gitHubGroup + repoName
+            repoReST = "`" + repoName + " <" + repoLink + ">`__"
+            manGrouping += repoReST + "\n"
+            manGrouping += "="*len(repoReST) + "\n\n"
+            manGrouping += manTOC
+            for file in fileList:
+                manGrouping += "   manpages/" + file + "\n"
+            manGrouping += "\n"
+        f.write(manGrouping)
+
+getPages(paths)
+updateManPages()
+createManpagesRST()

source/_scripts/js/copybutton.js

@@ -0,0 +1,147 @@
+// introduces special behavior for ShellSession
+
+// Localization support
+const messages = {
+  'en': {
+    'copy': 'Copy',
+    'copy_to_clipboard': 'Copy to clipboard',
+    'copy_success': 'Copied!',
+    'copy_failure': 'Failed to copy',
+  },
+  'es' : {
+    'copy': 'Copiar',
+    'copy_to_clipboard': 'Copiar al portapapeles',
+    'copy_success': '¡Copiado!',
+    'copy_failure': 'Error al copiar',
+  },
+  'de' : {
+    'copy': 'Kopieren',
+    'copy_to_clipboard': 'In die Zwischenablage kopieren',
+    'copy_success': 'Kopiert!',
+    'copy_failure': 'Fehler beim Kopieren',
+  }
+}
+
+let locale = 'en'
+if( document.documentElement.lang !== undefined
+    && messages[document.documentElement.lang] !== undefined ) {
+  locale = document.documentElement.lang
+}
+
+/**
+ * Set up copy/paste for code blocks
+ */
+
+const runWhenDOMLoaded = cb => {
+  if (document.readyState != 'loading') {
+    cb()
+  } else if (document.addEventListener) {
+    document.addEventListener('DOMContentLoaded', cb)
+  } else {
+    document.attachEvent('onreadystatechange', function() {
+      if (document.readyState == 'complete') cb()
+    })
+  }
+}
+
+const codeCellId = index => `codecell${index}`
+
+// Clears selected text since ClipboardJS will select the text when copying
+const clearSelection = () => {
+  if (window.getSelection) {
+    window.getSelection().removeAllRanges()
+  } else if (document.selection) {
+    document.selection.empty()
+  }
+}
+
+// Changes tooltip text for two seconds, then changes it back
+const temporarilyChangeTooltip = (el, newText) => {
+  const oldText = el.getAttribute('data-tooltip')
+  el.setAttribute('data-tooltip', newText)
+  setTimeout(() => el.setAttribute('data-tooltip', oldText), 2000)
+}
+
+// Callback when a copy button is clicked. Will be passed the node that was clicked
+// should then grab the text and replace pieces of text that shouldn't be used in output
+var copyTargetText = (trigger) => {
+  var target = document.querySelector(trigger.attributes['data-clipboard-target'].value);
+  var textContent = target.innerText.split('\n');
+  var copybuttonPromptText = '$ '; // Inserted from config
+  var onlyCopyPromptLines = true; // Inserted from config
+  var removePrompts = true; // Inserted from config
+
+  grandParent = target.parentElement.parentElement;
+  blockType = grandParent.classList;
+  if (blockType[0].includes("ShellSession")) {
+    onlyCopyPromptLines = false;
+  }
+
+  // Text content line filtering based on prompts (if a prompt text is given)
+  if (copybuttonPromptText.length > 0) {
+    // If only copying prompt lines, remove all lines that don't start w/ prompt
+    if (onlyCopyPromptLines) {
+      linesWithPrompt = textContent.filter((line) => {
+        return line.startsWith(copybuttonPromptText) || (line.length == 0); // Keep newlines
+      });
+      // Check to make sure we have at least one non-empty line
+      var nonEmptyLines = linesWithPrompt.filter((line) => {return line.length > 0});
+      // If we detected lines w/ prompt, then overwrite textContent w/ those lines
+      if ((linesWithPrompt.length > 0) && (nonEmptyLines.length > 0)) {
+        textContent = linesWithPrompt;
+      }
+    }
+    // Remove the starting prompt from any remaining lines
+    if (removePrompts) {
+      textContent.forEach((line, index) => {
+        if (line.startsWith(copybuttonPromptText)) {
+          textContent[index] = line.slice(copybuttonPromptText.length);
+        }
+      });
+    }
+  }
+  textContent = textContent.join('\n');
+  // Remove a trailing newline to avoid auto-running when pasting
+  if (textContent.endsWith("\n")) {
+     textContent = textContent.slice(0, -1)
+  }
+  return textContent
+}
+
+const addCopyButtonToCodeCells = () => {
+  // If ClipboardJS hasn't loaded, wait a bit and try again. This
+  // happens because we load ClipboardJS asynchronously.
+  if (window.ClipboardJS === undefined) {
+    setTimeout(addCopyButtonToCodeCells, 250)
+    return
+  }
+
+  // Add copybuttons to all of our code cells
+  const codeCells = document.querySelectorAll('div.highlight pre')
+  codeCells.forEach((codeCell, index) => {
+    const id = codeCellId(index)
+    codeCell.setAttribute('id', id)
+    const pre_bg = getComputedStyle(codeCell).backgroundColor;
+
+    const clipboardButton = id =>
+    `<a class="copybtn o-tooltip--left" style="background-color: ${pre_bg}" data-tooltip="${messages[locale]['copy']}" data-clipboard-target="#${id}">
+      <img src="${DOCUMENTATION_OPTIONS.URL_ROOT}_static/copy-button.svg" alt="${messages[locale]['copy_to_clipboard']}">
+    </a>`
+    codeCell.insertAdjacentHTML('afterend', clipboardButton(id))
+  })
+
+  // Initialize with a callback so we can modify the text before copy
+  const clipboard = new ClipboardJS('.copybtn', {text: copyTargetText})
+
+  // Update UI with error/success messages
+  clipboard.on('success', event => {
+    clearSelection()
+    temporarilyChangeTooltip(event.trigger, messages[locale]['copy_success'])
+  })
+
+  clipboard.on('error', event => {
+    temporarilyChangeTooltip(event.trigger, messages[locale]['copy_failure'])
+  })
+}
+
+runWhenDOMLoaded(addCopyButtonToCodeCells)

source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css

@@ -306,9 +306,9 @@ button:hover a.headerlink:after {
 /*End support for custom Clear Linux header*/
 
 /*Adds a bit of spacing after the last paragraph in a bulleted list*/
-.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child {
+/*.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child {
     margin-bottom: 10px;
-}
+}*/
 
 div.admonition ul {
   margin-top: 20px;
@@ -368,6 +368,15 @@ div.highlight-python .highlight:before{
   white-space: pre;
 }
 
+div.highlight-ShellSession .highlight:before{
+  background: #909090;
+  color: white;
+  content: " Shell ";
+  font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace;
+  font-size: 14px;
+  white-space: pre;
+}
+
 div.highlight-console .highlight:before{
   background: #909090;
   color: white;
@@ -494,6 +503,11 @@ div.linenodiv:before { /*add extra new line to make sure code and line numbers a
   background: white;
 }
 
+.column.narrow {
+  width: 300px;
+  height: 450px;
+}
+
 .column.featurecard {
   background: #CCE9F5;
   width: 300px;
@@ -511,6 +525,12 @@ div.linenodiv:before { /*add extra new line to make sure code and line numbers a
   clear: both;
 }
 
+.colh3 {
+  font-size: 125%;
+  font-weight: 700;
+  font-family: "Roboto Slab","ff-tisa-web-pro","Georgia",Arial,sans-serif;
+}
+
 .colh2 {
   font-size: 150%;
   font-weight: 700;

source/about.rst

@@ -3,57 +3,131 @@
 About
 #####
 
-The |CL| delivers a secure, hardware optimized OS. Its easy updates ensure that
+|CL-ATTR| does things differently. Our software architecture provides a
-software dependencies remain mutually compatible.
+unique and innovative platform for Linux* developers focused on
-
+performance and security for compute, server, and the cloud.
-|CL| does this via custom infrastructure components and process innovations.
 
 .. contents::
    :local:
    :depth: 1
 
+What is |CL|?
+*************
 
-For detailed information on these topics, refer to the :ref:`cl-guides` guides.
+|CL| is an open source, rolling-release Linux distribution, optimized for
+performance and security from the cloud to the Edge. Designed from the ground up,
+|CL| provides an industry blueprint on how to incorporate Intel® architecture
+features for a modern, modular Linux OS. |CL| is not based on any other Linux
+distro.
 
+What |CL| isn't?
+****************
+
+|CL| is not intended to be a general-purpose Linux distribution, suitable 
+for novice end-users. While we ship common applications, our purpose isn’t 
+to make an OS for routine desktop tasks and provide immunity from all 
+security threats in all situations. Our unique focus means that what we consider *essential* use cases, *optional* use cases, or even *unsupported* use cases, differs from other Linux distros. See our :ref:`target audience <target-audience>` below. 
+
+Is |CL| completely Open Source?
+*******************************
+
+|CL| aims to be completely open source. Our project `source code`_ and 
+`packages source code`_ are available on GitHub\*. When considering projects 
+for inclusion, we check that they are in active development and are well 
+maintained. We have a very strict requirement for not accepting proprietary 
+packages and non-open source components.  For example, many Linux distros 
+may not be able to include certain media codecs due to 
+:ref:`licensing restrictions <licensing_restrict>`, but manual installation and `third party alternatives`_ are available. 
+
+.. _target-audience:
+
+Who is the target audience?
+***************************
+
+|CL| mainly targets professionals in IT, DevOps, Cloud/Container deployments, and :abbr:`AI (Artificial Intelligence)`. 
+
+Rather than making a standard Linux distribution, the |CL| team decided to
+build a unique Linux distro. Developing a distro in house allows us to experiment and iterate faster, which means we continually optimize performance and deliver security patches, :ref:`several times per week <release-cadence>`. Yet our experiments are only valuable if our software architecture gives you the freedom to innovate, too. To improve manageability, |CL| employs a :ref:`stateless` design, separating user and system management.  
+
+We leverage the pool of knowledge and skills at Intel to drive improvements to |CL|.
+
+Intel has worked with the Linux community and other distros for many years.
+Understanding what it takes to integrate features in our own Linux distro
+helps us collaborate with other distro owners and submit enhancements to
+upstream. We demonstrate the value of our distro by offering users the same
+tools we use. For example, :ref:`mixer`, a tool unique to |CL|, allows users
+to build custom derivatives and act as their own :abbr:`OSV (Operating System
+Vendor)`. 
+
+For more details on |CL| features, visit our :ref:`cl-guides` guides.
+
+How does |CL| address security?
+*******************************
+
+Several :ref:`security features <security>` are designed to work 
+out-of-the-box, yet they're not intended to be intrusive. We focus on 
+*essential* use cases and ignore *unwanted* or *unsupported* use cases. 
+For example, while |CL| does not enable antivirus by default, we provide a 
+bundle for it (``clamav``). We leave antivirus configuration to our users.  
+In addition, firewalls are less important if the OS doesn’t expose services 
+to the outside by default. In |CL|, we enforce this strategy by disabling 
+network services by default - e.g. ``mariadb`` listens on a UNIX socket; 
+``nginx`` won’t listen at all; and other services similarly are restricted 
+from being accessed over the network. This strategy alone makes firewall 
+software much less urgent--there simply isn’t anything that a firewall could 
+easily block.
+
+What’s the thinking around Server vs. Desktop?
+**********************************************
+
+|CL| focuses on performance for server and cloud use-cases first because 
+many design decisions associated with them are applicable to other 
+use-cases, such as IoT and the desktop client. While our initial focus was 
+on the command line, we realized that many people valued the ease-of-use of 
+a desktop environment.  Whereas in the past we tried to accommodate those 
+interested in a desktop version, we were forced to confront clear limits as
+to how we could meet this need. |CL| minimizes the customizations and patches in support of the desktop and provides a generic GNOME implementation. Other window managers or desktops are available; however, testing in |CL| is focused on GNOME. 
+
+What makes |CL| different?
+**************************
+
+.. _release-cadence:
 
 Release Cadence
-***************
+===============
 
 |CL| updates are based on a rolling release that can occur daily, up to a few
-times per week. Each release has a unique version number that
+times per week. Each release has a unique version number that identifies
-identifies every component in the OS from kernel, to driver, to tool, to GUI
+every component in the OS from kernel, to driver, to tool, to GUI
-application. Most components are included in entities called *bundles*.
+application. Most components are included in entities called :ref:`bundles<bundles>`.
 
 Updates
-*******
+=======
 
-By default, |CL| automatically checks for updates, ensuring that the latest
+By default, |CL| automatically checks for updates, ensuring the latest
 performance and security fixes are installed as soon as they are available.
-:ref:`swupd-guide` is the custom tool designed to manage updates and bundles.
+|CL| stays in lockstep with upstream for current security upgrades and is 
+designed to rapidly deliver security mitigations to customers.
+:ref:`swupd-guide` is designed to manage updates and bundles.
 
-|CL| is :ref:`stateless` to make sure that system components can be updated
+Ease of Use 
-without impacting user settings.
+===========
-
-
-Ease of Use
-***********
 
 |CL| makes it easier to manage a number of difficult problems.
 
 * :ref:`autoproxy` makes it possible for |CL| tools to operate in some proxy
   environments without needing to be configured.
 
-* Being :ref:`stateless` means that configuration settings are easier to manage
+* :ref:`stateless` means that configuration settings are easier to manage
   and remain untouched when system software is updated.
 
-* :ref:`swupd-guide` simplifies managing software and maintaining compatibility.
+* :ref:`swupd-guide` simplifies managing software and maintaining
+  compatibility.
 
 Custom Derivatives
-******************
+==================
 
-The same tools used to build the |CL| are available *in* the OS. These tools can
+The same tools used to build the |CL| are available *in* the OS. These tools can be used to create a custom distribution that continues to benefit from upstream rolling releases.
-be used to create a custom distribution that continues to benefit from upstream
-rolling releases.
 
 .. figure:: /_figures/about/clear-lifecycle.png
    :scale: 75%
@@ -66,10 +140,7 @@ Create
 ======
 
 To create a custom distribution you need to understand how to use the
-:ref:`autospec` and :ref:`mixer` tools.
+:ref:`autospec` and :ref:`mixer` tools. Additional training materials are available in the `how-to-clear`_ GitHub project to help you get started with |CL| tools.
-
-Additional training materials are available in the `how-to-clear`_ GitHub\*
-project to help you get started with |CL| tools.
 
 Deploy
 ======
@@ -82,4 +153,159 @@ Administrate
 |CL| provides a :ref:`telem-guide` solution for collecting useful information
 about a deployment, as well as :ref:`debug` capabilities.
 
+Why create new components rather than modifying existing projects?
+******************************************************************
+
+One question that's often asked: “Why did you develop your own solution 
+instead of using <XYZ>?” (e.g. `swupd post`_).  We do evaluate existing 
+projects for inclusion in |CL|, yet there are cases where our unique 
+architecture and components would require too much customization to use 
+off-the-shelf projects.  In other situations, we may feel that using a new 
+language to develop the component would give us a performance advantage, 
+ease code development and maintenance, and grow the skills of our engineers 
+on new and upcoming programming languages.  And yes, sometimes there are 
+personal biases for and against some projects by the architects and 
+engineers.  We tend to move fast, and sometimes it’s easier to live with 
+suboptimal choices until we have the time or incentive to re-architect them 
+properly. 
+
+Which Components are used in Clear Linux?
+*****************************************
+
+.. list-table::
+   :widths: 33,33,33
+   :header-rows: 1
+
+   * - Component
+     - Enabled in OS/Bundle
+     - Optional
+
+   * - OS Installer
+     - `Clear Linux installer`_
+     - 
+
+   * - Bootloader
+     - `systemd-boot`_ (UEFI) / `syslinux`_ (Legacy)
+     - 
+
+   * - Boot Manager
+     - `Clear Linux Boot Manager`_
+     - 
+
+   * - Configuration initialization and management
+     - *NA*
+     - `micro-config-drive`_ (minimal cloud-init), Ansible
+
+   * - Software component installer, manager, updater
+     - `swupd`_
+     - 
+
+   * - Software bundle generator - 
+     - `mixer`_ and `Clear Linux Distro Factory`_
+     - 
+
+   * - Software package builder
+     - `autospec`_
+     - 
+
+   * - Software debugging
+     - *NA*
+     - `clr-debug-info`_ 
+
+   * - Unified TLS Trust Store Management
+     - `clrtrust`_
+     - 
+
+   * - System and software telemetry
+     - *NA*
+     - `Telemetrics`_ (disabled by default)
+
+   * - File system
+     - `EXT4`_ (default for rootfs), `VFAT`_, `EXT2 and EXT3`_, `F2FS`_
+     - 
+
+   * - Disk encryption
+     - *NA*
+     - `LUKS`_ 
+
+   * - System /Service manager  
+     - `systemd`_
+     - 
+
+   * - Display manager 
+     - `GNOME`_
+     - ``KDE``, ``Xfce``, ``lightdm``, ``sddm``  (see `Clear Linux store`_)
+
+   * - Display services (Desktop installed)
+     - `X.Org`_
+     - `Wayland`_ compositor
+
+   * - Network services
+     - `NetworkManager`_ by default, `systemd-networkd`_ See Note below.
+     - 
+
+   * - SSH Port scanning blocker
+     - `Tallow`_
+     - 
+
+   * - Firewall
+     - *NA*
+     - iptables and `firewalld`_
+
+   * - Antivirus
+     - *NA*
+     - `ClamAV*`_
+
+   * - Web browser
+     - `Lynx`_ or `links`_ for text environments, `Firefox*`_ for GUI
+     - 
+
+   * - Additional Software
+     - `Supplied Bundles`_
+     - Flatpak, 3rd-party software bundles
+
+.. note::
+   
+   The |CL| OS images targeted for cloud deployments continue to use
+   ``systemd-networkd`` to manage network connections.  In earlier |CL|, 
+   ``systemd-networkd`` was used to manage Ethernet interfaces and NetworkManager was used for wireless interfaces.
+
+
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
+.. _third party alternatives: https://community.clearlinux.org/t/about-the-3rd-party-sw-category/4072
 .. _how-to-clear: https://github.com/clearlinux/how-to-clear
+.. _Clear Linux store: https://clearlinux.org/software
+.. _source code: https://github.com/clearlinux
+.. _swupd post: https://community.clearlinux.org/t/why-does-clearlinux-use-swupd-and-not-apt-deb-rpm/
+.. _swupd: https://github.com/clearlinux/swupd-client
+.. _Clear Linux installer: https://github.com/clearlinux/clr-installer/
+.. _systemd-boot: https://www.freedesktop.org/software/systemd/man/systemd-boot.html
+.. _syslinux: https://wiki.syslinux.org/wiki/index.php?title=The_Syslinux_Project
+.. _Clear Linux Boot Manager: https://github.com/clearlinux/clr-boot-manager
+.. _mixer: https://github.com/clearlinux/mixer-tools
+.. _Clear Linux Distro Factory: https://github.com/clearlinux/clr-distro-factory
+.. _autospec: https://github.com/clearlinux/common
+.. _clr-debug-info: https://github.com/clearlinux/clr-debug-info
+.. _clrtrust: https://github.com/clearlinux/clrtrust
+.. _EXT4: https://ext4.wiki.kernel.org/index.php/Main_Page
+.. _VFAT: https://www.kernel.org/doc/html/latest/filesystems/vfat.html
+.. _EXT2 and EXT3: https://ext4.wiki.kernel.org/index.php/Main_Page
+.. _F2FS: https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
+.. _LUKS: https://gitlab.com/cryptsetup/cryptsetup/
+.. _systemd: https://www.freedesktop.org/wiki/Software/systemd/
+.. _GNOME: https://www.gnome.org/
+.. _X.Org: https://www.x.org/
+.. _Wayland: https://wayland.freedesktop.org/
+.. _NetworkManager: https://wiki.gnome.org/Projects/NetworkManager
+.. _systemd-networkd: https://www.freedesktop.org/software/systemd/man/systemd.network.html
+.. _Tallow: https://github.com/clearlinux/tallow
+.. _firewalld: https://docs.01.org/clearlinux/latest/guides/network/firewall.html#firewalld
+.. _ClamAV*: https://www.clamav.net/
+.. _Lynx: https://lynx.invisible-island.net/
+.. _links: http://links.twibright.com/
+.. _Firefox*: https://www.mozilla.org/en-US/firefox/
+.. _Supplied Bundles: https://clearlinux.org/software
+.. _micro-config-drive: https://github.com/clearlinux/micro-config-drive
+.. _Telemetrics: https://github.com/clearlinux/telemetrics-backend
+.. _packages source code: https://github.com/clearlinux-pkgs/

source/collaboration/collaboration.rst

@@ -41,7 +41,7 @@ Contribution guidelines
 ***********************
 
 The |CL| documentation is written using reStructuredText. Use our guidelines
-and best practices to write consistent, readable documentation.
+and best practices to write consistent, readable documentation. If you're writing a tutorial, review our skill levels to better target a user group.
 
 .. toctree::
    :maxdepth: 1

source/conf.py

@@ -15,6 +15,9 @@
 import sys
 import os
 import shlex
+#support for modified code block
+from pygments.lexers.shell import BashSessionLexer
+from sphinx.highlighting import lexers
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -26,6 +29,17 @@ import shlex
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
 
+#############
+#
+# Add a special lexer to add a class to console lexer
+#
+#############
+
+class copyAllConsole (BashSessionLexer):
+    name = 'ShellSession'
+
+lexers['ShellSession'] = copyAllConsole(startinLine=True)
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -34,7 +48,8 @@ import shlex
 #]
 
 extensions = [
-    'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx_sitemap', 'sphinx_tabs.tabs'
+    'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx_sitemap', 
+    'sphinx_tabs.tabs', 'sphinx_copybutton'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -54,7 +69,7 @@ master_doc = 'index'
 # General information about the project.
 #project = u'Clear Linux* project'
 project = u'Clear Linux* Project Docs'
-copyright = u'2020.'
+copyright = u'2021 Intel Corporation. All Rights Reserved.'
 author = u'many'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -81,7 +96,7 @@ language = None
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build','_themes']
+exclude_patterns = ['_build','_themes','.tox','_scripts']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -174,7 +189,8 @@ html_favicon = '_images/favicon.ico'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-#html_static_path = ['_static']
+# html_static_path = ['_scripts']
+copybutton_prompt_text = "$ "
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

source/get-started/bare-metal-install-desktop.rst

@@ -4,7 +4,9 @@ Install |CL-ATTR| from the live desktop
 #######################################
 
 This page explains how to boot the |CL-ATTR| live desktop image, from which
-you can install |CL| or explore without modifying the host system.
+you can install |CL| or explore without modifying the host system. 
+Alternatively, use a :ref:`YAML configuration file <install-configfile>` 
+to install |CL|. 
 
 .. contents::
    :local:
@@ -61,28 +63,44 @@ these steps.
 
 .. _preliminary-steps-install-desktop-end:
 
-#. Select :guilabel:`Clear Linux OS` in the boot menu, shown in Figure 1.
+Choose boot menu option
+=======================
 
-   .. figure:: /_figures/bare-metal-install-desktop/bare-metal-install-desktop-01.png
+#. Choose one of the options shown in Figure 1.
-      :scale: 100%
-      :alt: Clear Linux OS in boot menu
 
-      Figure 1: Clear Linux OS in boot menu
+   a. Follow `Verify integrity of installer media (optional)`_.
+
+   #. Select :guilabel:`Clear Linux OS` in the boot menu.
+
+      .. figure:: /_figures/bare-metal-install-desktop/bare-metal-install-desktop-01.png
+         :scale: 100%
+         :alt: Clear Linux OS in boot menu
+
+         Figure 1: Clear Linux OS in boot menu
+
+      .. note::
+
+         If no action is taken, the live image starts by default.
 
 .. _install-on-target-end:
 
-Software (optional)
+Verify integrity of installer media (optional)
-===================
+==============================================
 
-Explore |CL| bundles and other software. Double-click the
+Use :guilabel:`Verify ISO Integrity` to verify the checksum of 
-:guilabel:`Software` icon from the Activities menu, shown in Figure 2.
+the image burned to the installer media. The checksum ensures that the ISO 
-Ensure a network connection exists before launching the Software application.
+is uncorrupted (see Figure 1). For every ISO generated, the 
+:guilabel:`clr-installer` implants checksums, which are verified during 
+early boot stage as part of :command:`initrd`. 
+ 
+#. Select :guilabel:`Verify ISO Integrity`. The media will be validated.
 
-.. note::
+#. If the check passes, it will boot into the live image. Continue in 
+   the next section.
 
-   While running the |CL| live desktop image, the Software application is
+#. If the check fails, a failure message appears. 
-   intended only for exploration. Do not attempt to install applications during
+   
-   exploration.
+   * Restart the process at `Preliminary Steps`_. 
 
 .. _install-clr-desktop-start:
 
@@ -344,8 +362,10 @@ boot partition
 
 #. Select :guilabel:`Add`.
 
-swap partition
+swap partition (optional)
---------------
+-------------------------
+
+A swapfile is generated by default during installation. However, if you prefer to create a swap partition, follow the steps below.
 
 #. With :guilabel:`unallocated` highlighted, select from the menu
    :menuselection:`Partition --> New`.
@@ -643,19 +663,24 @@ Create partitions per requirements in Table 1.
      - /boot
      - 150MB
 
-   * - ``linux-swap``
-     - swap
-     -
-     - 256MB
-
    * - ``ext[234], XFS, or f2fs``
      - root
      - /
      - *Size depends upon use case/desired bundles.*
 
+.. note:: 
+   
+   A 64MiB swapfile is generated by default. The default size may be set
+   manually with the ``--swap-file-size`` command-line option.
+
 Troubleshooting
 ***************
 
 :ref:`erase-lvm-troubleshooting-tip`
 
+Related topics
+**************
+
+* :ref:`install-configfile`
+
 .. _Downloads: https://clearlinux.org/downloads

source/get-started/bare-metal-install-server.rst

@@ -4,7 +4,7 @@ Install |CL-ATTR| from the live server
 ######################################
 
 This page explains how to install |CL-ATTR| on bare metal from a bootable USB
-drive using a live server image.
+drive using a live server image. Alternatively, use a :ref:`YAML configuration file <install-configfile>` to install |CL|. 
 
 .. contents::
    :local:
@@ -20,8 +20,8 @@ installation:
 * :ref:`system-requirements`
 * :ref:`compatibility-check`
 
-Download the latest |CL| live server image
+Preliminary steps
-******************************************
+*****************
 
 #. Visit our `Downloads`_ page.
 
@@ -62,7 +62,14 @@ Follow these steps to install |CL| on the target system:
 
 #. Reboot the target system.
 
-#. This action launches the |CL| installer boot menu, shown in Figure 1.
+Choose boot menu option
+=======================
+
+#. Choose one of the options shown in Figure 1.
+
+   a. Follow `Verify integrity of installer media (optional)`_.
+
+   #. Select :guilabel:`Clear Linux OS` in the boot menu.
 
    .. figure:: /_figures/bare-metal-install-server/bare-metal-install-server-01.png
       :scale: 100%
@@ -70,7 +77,27 @@ Follow these steps to install |CL| on the target system:
 
       Figure 1: Clear Linux OS Installer boot menu
 
-#. With :guilabel:`Clear Linux OS` highlighted, select :kbd:`Enter`.
+   .. note::
+
+      If no action is taken, the live image starts by default.
+
+Verify integrity of installer media (optional)
+==============================================
+
+Use :guilabel:`Verify ISO Integrity` to verify the checksum of 
+the image burned to the installer media. The checksum ensures that the ISO 
+is uncorrupted (see Figure 1). For every ISO generated, the 
+:guilabel:`clr-installer` implants checksums, which are verified during 
+early boot stage as part of :command:`initrd`. 
+ 
+#. Select :guilabel:`Verify ISO Integrity`. The media will be validated. 
+
+#. If the check passes, it will boot into the live image. Continue in 
+   the next section.
+
+#. If the check fails, a failure message appears. 
+   
+   * Restart the process at `Preliminary Steps`_. 
 
 .. _install-clr-server-start:
 
@@ -318,8 +345,11 @@ boot partition
 
 Now follow the same process to configure the remaining partitions.
 
-swap partition
+swap partition (optional)
---------------
+-------------------------
+
+A swapfile is generated by default during installation. However, if you
+prefer to create a swap partition, follow the steps below. 
 
 #. Use the :kbd:`Up/Down` arrow to select free space.
 
@@ -915,16 +945,16 @@ Create partitions per requirements in Table 1.
      - /boot
      - 150MB
 
-   * - ``linux-swap``
-     - swap
-     -
-     - 256MB
-
    * - ``ext[234], `XFS`, or f2fs``
      - root
      - /
      - *Size depends upon use case/desired bundles.*
 
+.. note:: 
+   
+   A 64MiB swapfile is generated by default. The default size may be set
+   manually with the ``--swap-file-size`` command-line option.
+
 Troubleshooting
 ***************
 
@@ -1001,4 +1031,12 @@ commands:
    sudo dmsetup remove_all --force
    sudo partprobe
 
+Related topics
+**************
+
+* :ref:`install-configfile`
+
+
 .. _Downloads: https://clearlinux.org/downloads
+
+

source/get-started/cloud-install/import-clr-aws.rst

@@ -63,7 +63,7 @@ Create an S3 bucket
    See Figure 1. 
 
    .. figure:: ../../_figures/aws/import-clr-aws-01.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS Services - S3 Management Console
 
       Figure 1: AWS Services - S3 Management Console
@@ -71,7 +71,7 @@ Create an S3 bucket
 #. Click :guilabel:`+ Create bucket`.
 
    .. figure:: ../../_figures/aws/import-clr-aws-02.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Create bucket
 
       Figure 2: AWS S3 - Create bucket
@@ -80,22 +80,22 @@ Create an S3 bucket
    See Figure 3.
 
    .. figure:: ../../_figures/aws/import-clr-aws-03.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Create bucket - Set bucket name and region
 
       Figure 3: AWS S3 - Create bucket - Set bucket name and region
 
-#. Leave the :guilabel:`Configure options" and :guilabel:`Set permissions`
+#. Leave the :guilabel:`Configure options` and :guilabel:`Set permissions`
    settings as is or configure as desired.  See Figure 4 and 5.
 
    .. figure:: ../../_figures/aws/import-clr-aws-04.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Create bucket - Configure options
 
       Figure 4: AWS S3 - Create bucket - Configure options
 
    .. figure:: ../../_figures/aws/import-clr-aws-05.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Create bucket - Set permissions
 
       Figure 5: AWS S3 - Create bucket - Set permissions
@@ -103,7 +103,7 @@ Create an S3 bucket
 #. At the :guilabel:`Review` screen, click :guilabel:`Create bucket`.
 
    .. figure:: ../../_figures/aws/import-clr-aws-06.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Create bucket - Review
 
       Figure 6: AWS S3 - Create bucket - Review
@@ -111,7 +111,7 @@ Create an S3 bucket
    The created bucket should appear.  See Figure 7.
 
    .. figure:: ../../_figures/aws/import-clr-aws-07.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Created bucket
 
       Figure 7: AWS S3 - Created bucket
@@ -123,7 +123,7 @@ Upload the |CL| image into the bucket
    See Figure 8.
 
    .. figure:: ../../_figures/aws/import-clr-aws-08.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Select bucket
 
       Figure 8: AWS S3 - Select bucket
@@ -132,7 +132,7 @@ Upload the |CL| image into the bucket
    See Figure 9.
 
    .. figure:: ../../_figures/aws/import-clr-aws-09.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Upload
 
       Figure 9: AWS S3 - Upload
@@ -141,7 +141,7 @@ Upload the |CL| image into the bucket
    See Figure 10.
 
    .. figure:: ../../_figures/aws/import-clr-aws-10.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Add files
 
       Figure 10: AWS S3 - Add files
@@ -150,19 +150,19 @@ Upload the |CL| image into the bucket
    See Figure 11, Figure 12, and Figure 13.
 
    .. figure:: ../../_figures/aws/import-clr-aws-11.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Add files
 
       Figure 11: AWS S3 - Add files
 
    .. figure:: ../../_figures/aws/import-clr-aws-12.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Set permissions
 
       Figure 12: AWS S3 - Set permissions
 
    .. figure:: ../../_figures/aws/import-clr-aws-13.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Set properties
 
       Figure 13: AWS S3 - Set properties
@@ -171,7 +171,7 @@ Upload the |CL| image into the bucket
    See Figure 14.
 
    .. figure:: ../../_figures/aws/import-clr-aws-14.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS S3 - Upload
 
       Figure 14: AWS S3 - Upload
@@ -183,8 +183,8 @@ Add a user to IAM with AWS_CLI privilege
    and select :guilabel:`IAM`.
    See Figure 15. 
 
-   .. figure:: ../../_figures/aws/import-clr-aws-08.png
+   .. figure:: ../../_figures/aws/import-clr-aws-15.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS Services - IAM
 
       Figure 15: AWS Services - IAM
@@ -194,7 +194,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 16.
 
    .. figure:: ../../_figures/aws/import-clr-aws-16.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Access management
 
       Figure 16: AWS AIM - Access management
@@ -203,7 +203,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 17.
    
    .. figure:: ../../_figures/aws/import-clr-aws-17.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Add user
 
       Figure 17: AWS AIM - Add user
@@ -212,7 +212,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 18.
 
    .. figure:: ../../_figures/aws/import-clr-aws-18.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Enter user name and select access type
 
       Figure 18: AWS AIM - Enter user name and select access type
@@ -227,7 +227,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 19.
 
    .. figure:: ../../_figures/aws/import-clr-aws-19.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Set user permissions
 
       Figure 19: AWS AIM - Set user permissions
@@ -244,7 +244,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 20.
 
    .. figure:: ../../_figures/aws/import-clr-aws-20.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Create user
 
       Figure 20: AWS AIM - Create user
@@ -255,7 +255,7 @@ Add a user to IAM with AWS_CLI privilege
    See Figure 21.
 
    .. figure:: ../../_figures/aws/import-clr-aws-21.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS AIM - Access key ID and secret access key
 
       Figure 21: AWS AIM - Access key ID and secret access key
@@ -401,29 +401,29 @@ There are 2 methods to create an AMI from the snapshot.
      :guilabel:`EC2`.
      See Figure 22. 
 
-   .. figure:: ../../_figures/aws/import-clr-aws-22.png
+     .. figure:: ../../_figures/aws/import-clr-aws-22.png
-      :scale: 100%
+        :scale: 70%
-      :alt: AWS Services - EC2
+        :alt: AWS Services - EC2
 
-      Figure 22: AWS Services - EC2
+        Figure 22: AWS Services - EC2
     
   #. Click :guilabel:`Snapshots`.
      See Figure 23. 
 
-   .. figure:: ../../_figures/aws/import-clr-aws-23.png
+     .. figure:: ../../_figures/aws/import-clr-aws-23.png
-      :scale: 100%
+        :scale: 70%
-      :alt: AWS Services - Snapshots
+        :alt: AWS Services - Snapshots
 
-      Figure 23: AWS Services - Snapshots
+        Figure 23: AWS Services - Snapshots
 
   #. Locate the snaphot using the `Snapshot ID`.
      See Figure 24.
 
-   .. figure:: ../../_figures/aws/import-clr-aws-24.png
+     .. figure:: ../../_figures/aws/import-clr-aws-24.png
-      :scale: 100%
+        :scale: 70%
-      :alt: AWS Services - Snapshots
+        :alt: AWS Services - Snapshots
 
-      Figure 24: AWS Services - Snapshots
+        Figure 24: AWS Services - Snapshots
     
   #. Right-click it and select :guilabel:`Create Image`.
 
@@ -438,7 +438,7 @@ There are 2 methods to create an AMI from the snapshot.
      See Figure 25.
 
      .. figure:: ../../_figures/aws/import-clr-aws-25.png
-        :scale: 100%
+        :scale: 70%
         :alt: AWS Services - Snapshots
 
         Figure 25: AWS Services - Snapshots
@@ -453,7 +453,7 @@ Launch an instance
    See Figure 26. 
 
    .. figure:: ../../_figures/aws/import-clr-aws-26.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS Services - EC2
 
       Figure 26: AWS Services - EC2
@@ -463,7 +463,7 @@ Launch an instance
    See Figure 27. 
 
    .. figure:: ../../_figures/aws/import-clr-aws-27.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS Services - Launch instance
 
       Figure 27: AWS Services - Launch instance
@@ -472,7 +472,7 @@ Launch an instance
    See Figure 28. 
 
    .. figure:: ../../_figures/aws/import-clr-aws-28.png
-      :scale: 100%
+      :scale: 70%
       :alt: AWS Services - Select AMI
 
       Figure 28: AWS Services - Select AMI

source/get-started/compatibility-check.rst

@@ -1,57 +1,49 @@
 .. _compatibility-check:
 
-Check processor and EFI firmware compatibility
+Check Processor Compatibility
-##############################################
+#############################
 
-Before installing |CL-ATTR|, check your host system's processor and EFI firmware
+Before installing |CL-ATTR|, check your host system's processor compatibility using one of 
-compatibility. To check compatibility, choose one of the following paths:
+the following options:
-
-* From a system with a Linux\* OS installed, follow the instructions to :ref:`check-compatibility-steps`.
-
-* From a non-Linux OS, first :ref:`bare-metal-install-desktop` and then follow
-  the instructions to :ref:`check-compatibility-steps`.
 
 .. note::
    This does not check other system components (for example: storage and
    graphics) for compatibility with |CL|.
 
-.. _check-compatibility-steps:
+Option 1: Use the :command:`clear-linux-check-config.sh` script on an existing Linux system
-
+*******************************************************************************************
-Check compatibility
-*******************
 
 #. Download the `clear-linux-check-config.sh`_ file.
 
    If a browser is not available, use:
 
-   .. code-block:: console
+   .. code-block:: bash
 
       curl -O https://cdn.download.clearlinux.org/current/clear-linux-check-config.sh
 
 #. Make the script executable.
 
-   .. code-block:: console
+   .. code-block:: bash
 
       chmod +x clear-linux-check-config.sh
 
 #. Run the script.
 
-   #. Check to see if the host's processor and EFI firmware is capable of
+   #. Check to see if the host's processor is capable of running |CL|.
-      running |CL|.
 
-      .. code-block:: console
+      .. code-block:: bash
 
          ./clear-linux-check-config.sh host
 
    #. Check to see if the host is capable of running |CL| in a container.
 
-      .. code-block:: console
+      .. code-block:: bash
 
          ./clear-linux-check-config.sh container
 
    The script prints a list of test results similar to the output below.
    All items should return a `SUCCESS` status. This example indicates the
-   host's processor and EFI firmware support running |CL|.
+   host's processor supports running |CL|.
 
    .. code-block:: console
 
@@ -61,8 +53,52 @@ Check compatibility
       SUCCESS: Supplemental Streaming SIMD Extensions 3 (ssse3)
       SUCCESS: Streaming SIMD Extension v4.1 (sse4_1)
       SUCCESS: Streaming SIMD Extensions v4.2 (sse4_2)
-      SUCCESS: Advanced Encryption Standard instruction set (aes)
       SUCCESS: Carry-less Multiplication extensions (pclmulqdq)
-      SUCCESS: EFI Firmware
 
-.. _clear-linux-check-config.sh: https://cdn.download.clearlinux.org/current/clear-linux-check-config.sh
+Option 2: Use a |CL| live image on a non-Linux system
+=====================================================
+
+#. `Download`_ either the `Desktop` or `Server` version of the live image ISO.
+
+#. Follow the instruction to :ref:`bootable-usb`.
+
+#. Boot up the |CL| live image on the USB.
+
+#. Check compatibility as follows:
+
+   * *Desktop version:*
+     
+     a. Open a terminal.
+
+     #. Check compatibility.
+
+        .. code-block:: bash
+
+	   sudo clr-installer --system-check
+
+   * *Server version:*
+
+     a. Log in as `root` and set a password.
+
+     #. Check compatibility.
+
+        .. code-block:: bash
+
+	   clr-installer --system-check
+
+   Expected output for a compatible host processor:
+
+   .. code-block:: console
+
+      Checking for required CPU feature: lm [success]
+      Checking for required CPU feature: sse4_2 [success]
+      Checking for required CPU feature: sse4_1 [success]
+      Checking for required CPU feature: pclmulqdq [success]
+      Checking for required CPU feature: ssse3 [success]
+      Success: System is compatible
+
+.. _clear-linux-check-config.sh: 
+   https://cdn.download.clearlinux.org/current/clear-linux-check-config.sh
+
+.. _Download:
+   https://clearlinux.org/downloads   

source/get-started/virtual-machine-install/hyper-v.rst

@@ -15,8 +15,8 @@ Overview
 ********
 
 Hyper-V is a type 1 bare-metal hypervisor that runs directly on system
-hardware. It is available for `Windows\* server`_ and client operating systems,
+hardware. It is available for `Windows\* server`_ and client operating 
-including `Windows 10`_.
+systems, including `Windows 10`_.
 
 |CL| provides a virtual disk image for Hyper-V, which also includes
 a :ref:`Hyper-V specific kernel <compatible-kernels>` and drivers.
@@ -43,14 +43,16 @@ Prerequisites
 Download the |CL| disk image for Hyper-V
 ****************************************
 
-#. Download the :file:`clear-[VERSION]-hyperv.vhdx.gz` for Microsoft* Hyper-V
+#. Download the :file:`clear-[VERSION]-azure-hyperv.vhd.gz` for Microsoft* 
-   from the `downloads`_ website.
+   Hyper-V from the `downloads`_ website.
 
 #. Verify and extract the image using these instructions:
    :ref:`download-verify-decompress`.
 
-   After extraction, the file should be named :file:`clear-[VERSION]-hyperv.vhdx`.
+#. Extract the compressed file using software such as the 
+   7-Zip\* tool or the WinZip\* tool. 
 
+   After extraction, the file should be named :file:`clear-[VERSION]-azure-hyperv.vhd`.
 
 Create and configure new VM
 ****************************
@@ -63,6 +65,10 @@ Create and configure new VM
 
       Figure 1: Hyper-V Manager from the Start menu
 
+   .. note::
+
+      You may need to manually enable Hyper-V on a Windows\* machine. Review 
+      ``Windows Features``.
 
 #. Create a *New Virtual Machine* by clicking the :guilabel:`Action` menu,
    then selecting :guilabel:`New` and :guilabel:`Virtual Machine...`.
@@ -77,23 +83,22 @@ Create and configure new VM
    specifying the options below:
 
    - **Name**: Choose name (for example, ClearLinuxOS-VM)
-   - **Generation**: Generation 2
+   - **Specify Generation**: Generation 1
    - **Startup memory**: 2048 MB or more
+   - **Configure Networking**: Change :guilabel:`Connection` to `Default Switch`
    - **Connect Virtual Hard Disk**: Select :guilabel:`Use an existing virtual
-     hard disk` and browse to find the :file:`clear-[VERSION]-hyperv.vhdx`
+     hard disk` and browse to find the 
-     file.
+     :file:`clear-[VERSION]-azure-hyperv.vhd` file.
 
    After finishing the wizard, the VM will be created but not powered on.
 
 #. Configure the VM by right-clicking it in the Hyper-V Manager and selecting
-   :guilabel:`Settings...`.
+   :guilabel:`Settings...`. Figure 3 shows the Settings page after configuration selections.
-   Figure 3 shows the Settings page after configuration selections.
 
-   - Under :guilabel:`Firmware`, select the Virtual disk and click
+   **Optional**
-     :guilabel:`Move Up...` until it is at the top of the list.
 
-   - Under :guilabel:`Security`, uncheck the :guilabel:`Enable Secure Boot`
+   - If you wish to `Encrypt state and virtual machine traffic, under 
-     checkbox.
+     :guilabel:`Security`, select :guilabel:`Add Key Storage Drive`.
 
    - Under :guilabel:`Processor`, consider increasing the number of virtual
      processors assigned to the |CL| VM to improve performance.
@@ -112,12 +117,12 @@ Create and configure new VM
 Start the VM
 ************
 
-#. Start the |CL| VM by right-clicking the VM in Hyper-V Manager and selecting
+#. Start the |CL| VM by right-clicking the VM in Hyper-V Manager and 
-   :guilabel:`Start`.
+   selecting :guilabel:`Start`.
 
 #. Connect to the VM console by right-clicking the VM in Hyper-V Manager and
-   selecting :guilabel:`Connect...`. A new *Virtual Machine Connection* window
+   selecting :guilabel:`Connect...`. A new *Virtual Machine Connection* 
-   is displayed.
+   window is displayed.
 
 #. After |CL| is booted, log in to the console with user *root*. You are
    prompted to set a new password immediately.
@@ -133,7 +138,7 @@ Related topics
 
 * :ref:`increase-virtual-disk-size`
 
-
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
 
 .. _`Windows\* Server`: https://docs.microsoft.com/en-us/windows-server/virtualization/hyper-v/hyper-v-on-windows-server
 .. _`Windows 10`: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/index

source/get-started/virtual-machine-install/kvm.rst

@@ -218,6 +218,8 @@ To add :abbr:`GDM (GNOME Display Manager)` to the |CL| VM, follow these steps:
    corner).
 
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _Intel® Virtualization Technology: https://www.intel.com/content/www/us/en/virtualization/virtualization-technology/intel-virtualization-technology.html
 .. _Intel® Virtualization Technology for Directed I/O: https://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices
 .. _start_qemu.sh: https://cdn.download.clearlinux.org/image/start_qemu.sh

source/get-started/virtual-machine-install/virt-manager.rst

@@ -214,6 +214,8 @@ Congratulations! You have successfully installed |CL| in your new VM and can
 begin using it immediately. The `virt-manager` tool is maintained on GitHub\*
 at `virt-manager-github`_.
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _virt-mgr: https://www.virt-manager.org
 
 .. _Downloads: https://clearlinux.org/downloads

source/get-started/virtual-machine-install/virtualbox-cl-installer.rst

@@ -389,6 +389,9 @@ Troubleshooting
 
       bcdedit /set {current} hypervisorlaunchtype Auto
 
+
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _VirtualBox Installation Instructions: https://www.virtualbox.org/manual/ch02.html
 
 .. _VirtualBox: https://www.virtualbox.org

source/get-started/virtual-machine-install/vmw-player.rst

@@ -3,7 +3,7 @@
 |CL-ATTR| on VMware\* Workstation Player
 ########################################
 
-This guide explains how to set up the VMware Workstation Player 15.5.1
+This guide explains how to set up the VMware\* Workstation Player 15.5.1
 hypervisor and instantiate a VM instance of |CL| by installing it using 
 an ISO or using a pre-built image.
 
@@ -33,8 +33,8 @@ it, see :ref:`vmware-esxi-install-cl`.
 Install the VMware Workstation Player hypervisor
 ************************************************
 
-#. Enable :abbr:`Intel® VT (Intel® Virtualization Technology)` and
+#. Enable Intel® Virtualization Technology (Intel® VT) and
-   :abbr:`Intel® VT-d (Intel® Virtualization Technology for Directed I/O)` in
+   Intel® Virtualization Technology for Directed I/O (Intel® VT-d) in
    your system's BIOS.
 
 #. `VMware Workstation Player`_ is available for Windows and Linux.
@@ -407,6 +407,8 @@ For other guides on using the VMWare Player and ESXi, see:
 
 * :ref:`vmware-esxi-install-cl`
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _VMware ESXi: https://www.vmware.com/products/esxi-and-esx.html
 
 .. _VMware Workstation Player:

source/guides/clear/autospec.rst

@@ -594,6 +594,7 @@ Related topics
 **************
 
 * :ref:`Mixer tool <mixer>`
+* :ref:`Proxy Configuration <proxy>`
 
 .. _contributing to an existing software package: https://github.com/clearlinux/distribution/blob/master/contributing.md#contributing-to-an-existing-software-package
 
@@ -607,4 +608,4 @@ Related topics
 .. _RPM Packaging Guide: https://rpm-packaging-guide.github.io/
 
 
-.. TODO:  Add link to how to submit a new package: https://github.com/clearlinux/distribution/blob/master/contributing.md#contributing-a-new-software-package
+.. TODO:  Add link to how to submit a new package: https://github.com/clearlinux/distribution/blob/master/contributing.md#contributing-a-new-software-package

source/guides/clear/compatible-kernels.rst

@@ -12,7 +12,7 @@ Bare metal only
 
 Kernel native
     The *kernel-native* bundle focuses on the bare metal platforms. It is
-    optimized for fast booting and performs best on the Intel® architectures
+    optimized for fast booting and performs best on the Intel® Architecture Processors
     described on the :ref:`supported hardware list<system-requirements>`. The
     optimization patches are found in our `Linux`_ GitHub\* repo.
 
@@ -24,8 +24,8 @@ Also compatible with VMs
 Kernel LTS
     The *kernel-lts* bundle focuses on the bare metal platforms but uses the
     latest :abbr:`LTS (Long Term Support)` Linux kernel. It is optimized for
-    fast booting and performs best on the Intel® architectures described on the
+    fast booting and performs best on the Intel® Architecture Processors described
-    :ref:`supported hardware list<system-requirements>`. Additionally, this
+    on the :ref:`supported hardware list<system-requirements>`. Additionally, this
     kernel includes the VirtualBox\* kernel modules, see our
     :ref:`instructions on using Virtualbox<virtualbox-cl-installer>` for more
     information. The optimization patches are found in our `Linux-LTS`_ GitHub
@@ -37,8 +37,8 @@ VM only
 Kernel KVM
     The *kernel-kvm* bundle focuses on the Linux
     :abbr:`KVM (Kernel-based Virtual Machine)`. It is optimized for fast
-    booting and performs best on Virtual Machines running on the Intel®
+    booting and performs best on Virtual Machines running on the Intel® Architecture
-    architectures described on the
+    Processors described on the
     :ref:`supported hardware list<system-requirements>`. Use this kernel when
     running |CL| as the guest OS on top of *qemu/kvm*. Use this kernel with
     **cloud orchestrators** using *qemu/kvm* internally as their **hypervisor**
@@ -49,7 +49,7 @@ Kernel KVM
 Kernel Hyper-V\*
     The *kernel-hyperv* bundle focuses on running Linux on Microsoft\*
     Hyper-V. It is optimized for fast booting and performs best on Virtual
-    Machines running on the Intel® architectures described on the
+    Machines running on the Intel® Architecture Processors described on the
     :ref:`supported hardware list<system-requirements>`.
     Use this kernel when running |CL| as the guest OS of **Cloud Instances** in
     projects such as Microsoft `Azure`_\*. This kernel can be used in a
@@ -57,18 +57,7 @@ Kernel Hyper-V\*
     for more information. The optimization patches are found in our
     `Linux-HyperV`_ GitHub repo.
 
-Kernel Hyper-V LTS
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
-    The *kernel-hyperv-lts* bundle focuses on running Linux on Microsoft
-    Hyper-V but uses the latest :abbr:`LTS (Long Term Support)` Linux kernel.
-    It is optimized for fast booting and performs best on Virtual
-    Machines running on the Intel® architectures described on the
-    :ref:`supported hardware list<system-requirements>`.
-    Use this kernel when running |CL| as the guest OS of **Cloud Instances** in
-    projects such as Microsoft `Azure`_. This kernel can be used in a
-    standalone |CL| VM, see our :ref:`instructions on using Hyper-V<hyper-v>`
-    for more information. The optimization patches are found in our
-    `Linux-HyperV-LTS`_ GitHub repo.
-
 
 .. _Linux: https://github.com/clearlinux-pkgs/linux
 .. _Linux-LTS: https://github.com/clearlinux-pkgs/linux-lts

source/guides/clear/k8s-migration.rst

@@ -0,0 +1,321 @@
+.. _kubernetes-migration:
+
+Kubernetes\* migration
+######################
+
+This guide describes how to migrate `Kubernetes container orchestration system`_ on |CL-ATTR| from 1.17.x to 1.19.x.
+
+.. contents::
+   :local:
+   :depth: 1
+
+Background
+**********
+
+The version of Kubernetes\* was bumped from 1.17.7 to 1.19.4 in |CL-ATTR|
+release 34090. This guide and the |CL| bundle `k8s-migration` were created
+to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
+
+The new |CL| bundle `k8s-migration` was added in |CL-ATTR| release 34270.
+
+Prerequisites
+*************
+
+* Make sure you check any updates to kubernetes upgrade doc for caveats related to the version that is running in the cluster.
+* Make sure ALL the nodes are in Ready state. Without that, the cluster cannot be upgraded.
+  Either fix the broken nodes or remove them from the cluster.
+
+.. contents::
+   :local:
+   :depth: 1
+
+Upgrade 1.17.x ---> 1.18.15
+***************************
+
+#. Upgrade Control Node to 1.18.15 first
+
+   First step would be to upgrade one of the main control node and
+   update kubernetes components on them. You will need to have a newer
+   version of :command:`kubeadm` for the upgrade to work. Please consult
+   `kubeadm upgrade guide`_
+   for any caveats from your current version to the new one.
+
+   Update |CL| to the latest release to update the kubernetes version.
+
+   .. code-block:: bash
+
+      sudo -E swupd update
+
+   .. note::
+      Note: PLEASE DO NOT REBOOT YOUR SYSTEM AT THIS TIME. |CL| is awesome and
+      your stuff will work just fine.
+
+#. Add the new Kubernetes migration bundle which contains the 1.18.15 binaries.
+
+   .. code-block:: bash
+
+      sudo -E swupd bundle-add k8s-migration
+
+#. Find the upgrade version of kubeadm that can used. This should be 1.18.15.
+
+   This command will show the command and possible jumps that can be made from the current kubernetes version.
+
+   .. code-block:: bash
+
+      sudo -E /usr/k8s-migration/bin/kubeadm upgrade plan
+
+   Sample output:
+
+   .. code-block:: console
+
+      [upgrade/config] Making sure the configuration is correct:
+      [upgrade/config] Reading configuration from the cluster...
+      [upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
+      [preflight] Running pre-flight checks.
+      [upgrade] Running cluster health checks
+      [upgrade] Fetching available versions to upgrade to
+      [upgrade/versions] Cluster version: v1.17.17
+      [upgrade/versions] kubeadm version: v1.18.15
+      I0209 21:12:49.868786  832739 version.go:252] remote version is much newer: v1.20.2; falling back to: stable-1.18
+      [upgrade/versions] Latest stable version: v1.18.15
+      [upgrade/versions] Latest stable version: v1.18.15
+      [upgrade/versions] Latest version in the v1.17 series: v1.17.17
+      [upgrade/versions] Latest version in the v1.17 series: v1.17.17
+
+      Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
+      COMPONENT   CURRENT       AVAILABLE
+      Kubelet     3 x v1.17.7   v1.18.15
+
+      Upgrade to the latest stable version:
+
+      COMPONENT            CURRENT    AVAILABLE
+      API Server           v1.17.17   v1.18.15
+      Controller Manager   v1.17.17   v1.18.15
+      Scheduler            v1.17.17   v1.18.15
+      Kube Proxy           v1.17.17   v1.18.15
+      CoreDNS              1.6.5      1.6.7
+      Etcd                 3.4.3      3.4.3-0
+
+      You can now apply the upgrade by executing the following command:
+
+        kubeadm upgrade apply v1.18.15
+
+#. Upgrade the node to the intermediate 1.18.15 version of Kubernetes.
+
+   .. code-block:: bash
+
+      sudo -E /usr/k8s-migration/bin/kubeadm upgrade apply v1.18.15
+
+   .. note::
+      Note: Do **not** reboot the system yet.
+
+#. Upgrade Additional Control Nodes to 1.18.15
+
+   In multi-node control plane, verify all the control plane nodes are updated prior to upgrading the worker nodes/SUTs.
+
+#. Upgrade Other Nodes to 1.18.15
+
+   For each of the other nodes:
+
+   a. Update |CL| to the latest release to update the kubernetes version.
+
+      .. code-block:: bash
+
+         sudo -E swupd update
+
+   #. Add the new Kubernetes migration bundle which contains the 1.18.15 binaries.
+
+      .. code-block:: bash
+
+         sudo -E swupd bundle-add k8s-migration
+
+   #. On the **Admin node**, drain the Client node *FIRST*
+
+      .. code-block:: bash
+
+         /usr/k8s-migration/bin/kubectl drain <CLIENT_NODE_NAME> --ignore-daemonsets --delete-local-data
+
+   #. Back on the **Client node**, upgrade Kubernetes on the Client
+
+      .. code-block:: bash
+
+         sudo -E /usr/k8s-migration/bin/kubeadm upgrade node
+
+   #. On the **Admin node**, re-enable the Client
+
+      .. code-block:: bash
+
+         /usr/k8s-migration/bin/kubectl uncordon <CLIENT_NODE_NAME>
+
+
+   #. Back on the **Client node**, restart Kubernetes on the Client
+
+      .. code-block:: bash
+
+         sudo -E systemctl restart kubelet
+
+#. Restart Kubernetes on the Admin node(s) to finish the 1.18.x upgrade
+
+   .. code-block:: bash
+
+      sudo -E systemctl restart kubelet
+
+   .. note::
+      Note: Wait for all nodes to be Ready and showing the 1.19.x version.
+      This version will now show as it is the released version the
+      service files will see and use, but the Nodes are *not* upgraded yet.
+
+Upgrade 1.18.15 ---> 1.19.x
+***************************
+
+#. Upgrade Control Node to 1.19.x
+
+   Now that systems are upgraded to the intermediate release of 1.18.15
+   each of the nodes can be upgraded to the latest 1.19.x release.
+
+#. Find the upgrade version of kubeadm that can used. This should be 1.19.x.
+
+   This command will show the command and possible jumps that can be made from the current kubernetes version.
+
+   .. code-block:: bash
+
+      sudo -E kubeadm upgrade plan
+
+   Sample output:
+
+   .. code-block:: console
+
+      [upgrade/config] Making sure the configuration is correct:
+      [upgrade/config] Reading configuration from the cluster...
+      [upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
+      [preflight] Running pre-flight checks.
+      [upgrade] Running cluster health checks
+      [upgrade] Fetching available versions to upgrade to
+      [upgrade/versions] Cluster version: v1.18.15
+      [upgrade/versions] kubeadm version: v1.19.7
+      I0209 23:08:23.810900  925910 version.go:252] remote version is much newer: v1.20.2; falling back to: stable-1.19
+      [upgrade/versions] Latest stable version: v1.19.7
+      [upgrade/versions] Latest stable version: v1.19.7
+      [upgrade/versions] Latest version in the v1.18 series: v1.18.15
+      [upgrade/versions] Latest version in the v1.18 series: v1.18.15
+
+      Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
+      COMPONENT   CURRENT       AVAILABLE
+      kubelet     3 x v1.17.7   v1.19.7
+
+      Upgrade to the latest stable version:
+
+      COMPONENT                 CURRENT    AVAILABLE
+      kube-apiserver            v1.18.15   v1.19.7
+      kube-controller-manager   v1.18.15   v1.19.7
+      kube-scheduler            v1.18.15   v1.19.7
+      kube-proxy                v1.18.15   v1.19.7
+      CoreDNS                   1.6.7      1.7.0
+      etcd                      3.4.3-0    3.4.13-0
+
+      You can now apply the upgrade by executing the following command:
+
+        kubeadm upgrade apply v1.19.7
+
+      The table below shows the current state of component configs as understood by this version of kubeadm.
+      Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or
+      resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually
+      upgrade to is denoted in the "PREFERRED VERSION" column.
+
+      API GROUP                 CURRENT VERSION   PREFERRED VERSION   MANUAL UPGRADE REQUIRED
+      kubeproxy.config.k8s.io   v1alpha1          v1alpha1            no
+      kubelet.config.k8s.io     v1beta1           v1beta1             no
+
+#. Upgrade the node to the latest 1.19.x version of Kubernetes.
+
+   .. code-block:: bash
+
+      sudo -E /usr/bin/kubeadm upgrade apply v1.19.7
+
+   .. note::
+
+      Note: Do **not** reboot the system yet.
+
+#. Upgrade Additional Control Nodes to 1.19.x
+
+   In multi-node control plane, verify all the control plane nodes are updated prior to upgrading the worker nodes/SUTs.
+
+#. Upgrade Other Nodes to 1.19.x
+
+   For each of the other nodes:
+
+   a. On the **Admin node**, drain the Client *FIRST*
+
+      .. code-block:: bash
+
+         kubectl drain <CLIENT_NODE_NAME> --ignore-daemonsets
+
+   #. Back on the **Client node**, upgrade Kubernetes on the Client
+
+      .. code-block:: bash
+
+         sudo -E kubeadm upgrade node
+
+   #. On the **Admin node**, re-enable the Client
+
+      .. code-block:: bash
+
+         kubectl uncordon <CLIENT_NODE_NAME>
+
+   #. Back on the **Client node**, if you wish reboot the Client, it is now safe to do so.
+
+      .. code-block:: bash
+
+         sudo reboot
+
+#. Reboot the Control Node (optional)
+
+  *If you wish reboot the nodes, it is now safe to do so.*
+
+   .. code-block:: bash
+
+      sudo reboot
+
+**Congratulations!**
+
+You've successfully installed and set up Kubernetes in |CL| using CRI-O and kata-runtime. You are now ready to follow on-screen instructions to deploy a pod network to the cluster and join worker nodes with the displayed token and IP information.
+
+Clean up: Remove the migration bundle for each node
+
+.. code-block:: bash
+
+   sudo -E swupd bundle-remove k8s-migration
+
+Related topics
+**************
+
+Read the Kubernetes documentation to learn more about:
+
+*  `Kubernetes tutorial <tutorials/kubernetes>`_
+
+*  `Kubernetes best practices <tutorials/kubernetes-bp>`_
+
+*  Deploying Kubernetes with a `cloud-native-setup`_
+
+* `Understanding basic Kubernetes architecture`_
+
+* `Deploying an application to your cluster`_
+
+* Installing a `pod network add-on`_
+
+* `Joining your nodes`_
+
+
+.. _kubeadm upgrade guide: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/
+
+.. _Kubernetes container orchestration system: https://kubernetes.io/
+
+.. _Understanding basic Kubernetes architecture: https://kubernetes.io/docs/user-journeys/users/application-developer/foundational/#section-3
+
+.. _Deploying an application to your cluster: https://kubernetes.io/docs/user-journeys/users/application-developer/foundational/#section-2
+
+.. _pod network add-on: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network
+
+.. _Joining your nodes: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#join-nodes
+
+.. _cloud-native-setup: https://github.com/clearlinux/cloud-native-setup/tree/master/clr-k8s-examples

source/guides/clear/mixer.rst

@@ -52,6 +52,9 @@ Prerequisites
   Add the mixer tool by installing the :command:`mixer` bundle. Refer to
   :ref:`swupd-guide` for more information on installing bundles.
 
+* If you're working behind a corporate proxy, configure proxy settings using
+  the :ref:`General proxy settings for many applications <proxy>` steps.
+
 * Location to host the update content and images
 
   In order for :command:`swupd` to make use of your mix, the update content for your mix must be hosted on a web server. Your mix will be configured with an update location URL, which :command:`swupd` will use to pull down updates.

source/guides/clear/performance.rst

@@ -0,0 +1,241 @@
+.. _performance:
+
+Performance
+###########
+
+|CL-ATTR| is built with optimizations across the whole stack for improved
+performance. |CL| achieves its performance through a variety of design decisions
+and software building techniques.
+
+.. contents:: :local:
+   :depth: 1
+
+Overview
+********
+
+The |CL| philosophy is to do everything with performance in mind. The |CL| team
+applies this philosophy in the project's codebase and operating culture.
+
+Below are some examples of the |CL| philosophy:
+
+**Consider performance holistically.**
+  Performance optimizations are considered across hardware and software. |CL|
+  shows the performance potential of a holistic approach on Linux, using Intel®
+  architecture with optimizations across the full stack.
+
+**Optimize for runtime performance.**
+  In general, |CL| will trade the one-time cost of longer build time and larger
+  storage footprint for the repeated benefit of improved runtime performance.
+  |CL| users benefit from the optimized software but aren't affected by the
+  increased build time because the |CL| team builds the software before
+  distributing it to |CL| clients.
+
+**Optimize performance for server and cloud use cases first.**
+  Design decisions that optimize performance for server and cloud also benefit
+  other use cases, such as IoT devices and desktop clients.
+
+|CL| has become well-known for the performance it can deliver.
+`Phoronix publishes
+Linux performance comparisons <https://www.phoronix.com/scan.php?page=news_topic&q=Clear+Linux>`_
+that include |CL|.
+
+Software build toolchain
+************************
+
+|CL| uses many techniques in its software build toolchain to improve software
+performance, such as aggressive compiler flags and CPU-specific optimizations.
+If maintained manually, these techniques can become complex to support due to
+the volume of packages and the potential for technical drift of package
+performance configurations. The |CL| team built the :ref:`autospec` tool to
+manage this complexity and to apply the techniques used in the software build
+toolchain across the entire project. autospec is available as part of the OS for
+developers to use when they build their own projects on |CL|.
+
+Latest versions of compilers and low-level libraries
+====================================================
+
+|CL| is a rolling release distribution and follows upstream software
+repositories, including compilers and libraries, for updates. |CL| includes
+upstream source-level optimizations as soon as they're available.
+
+A benchmark approach to compiler performance
+============================================
+
+|CL| chooses the compiler used to build each software package on a case-by-case
+basis to maximize performance. Typically, |CL| uses the open source `GNU Compiler
+Collection <https://gcc.gnu.org/>`_ (GCC) with the standard low-level
+libraries `Glibc <https://www.gnu.org/software/libc/>`_ and
+`libstdc++ <https://gcc.gnu.org/onlinedocs/libstdc++/>`_ for C and C++
+programming languages. If there is a performance advantage, |CL| will build
+packages with `Clang / LLVM <https://clang.llvm.org/>`_.
+
+|CL| uses patched compilers and low-level libraries for exact control of the
+software build. Patches include changes that default to more aggressive
+optimizations or optimizations that haven't yet been merged upstream.
+
+View the full list of patches in the autospec repositories on GitHub:
+
+* https://github.com/clearlinux-pkgs/gcc
+* https://github.com/clearlinux-pkgs/glibc
+* https://github.com/clearlinux-pkgs/llvm
+
+Aggressive compiler flags
+=========================
+
+|CL| uses aggressive
+`compiler flags <https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html>`_ to
+optimize software builds for runtime performance. Some significant flags that
+|CL| often implements are:
+
+`mtune and march <https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html>`_
+	Options used to tune generated code with optimized instructions for specific
+	CPU types instead of creating generic code for maximum compatibility.
+
+	|CL| defines its minimum hardware requirements to be second-generation
+	Intel® microarchitecture code name Westmere (released in 2010) or later.
+	This enables compiler optimizations that are available only on newer
+	architectures. Whenever possible, |CL| tunes code for the Haswell generation
+	processors or newer.
+
+	|CL| sets	:command:`march=westmere` and :command:`mtune=haswell`.
+
+	.. note::
+		|CL| doesn't require Advanced Encryption Standard (AES), so it should
+		run on some Intel CPUs from the first generation of Intel® microarchitecture code name Nehalem (released in 2008). Refer to the
+		`recommended minimum system requirements <https://docs.01.org/clearlinux/latest/reference/system-requirements.html>`_ for specific requirements.
+
+`O3 <https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html>`_
+	The largest preset of compiler options optimizations for performance. O3
+	favors runtime performance.
+
+	View the "Optimize Options" section of the GCC man page for additional
+	information: :command:`man gcc`
+
+`LTO <https://gcc.gnu.org/onlinedocs/gccint/LTO.html>`_
+	Link-time optimization that performs an optimization between compiled object
+	files and creation of executable binaries by adding extra information to the
+	compiled object to help the linker.
+
+`PGO <https://en.wikipedia.org/wiki/Profile-guided_optimization>`_
+	Profile guided optimization or field guided optimization performs
+	optimization based on information sampled during the execution of the program.
+
+
+Compiler flags are set at different levels in the |CL| build environment:
+
+User flags
+	The set of default flags used by |CL| when a user compiles software
+	from source. The flags are exported as system-wide environment variables from
+	the
+	`/usr/share/defaults/etc/profile <https://github.com/clearlinux-pkgs/filesystem/blob/master/profile.x86_64>`_ file to the user’s shell by default. These are the
+	standard variables read by the compiler, named :command:`*FLAGS`, depending
+	on the compiler.
+
+	.. note::
+		Source code may come with software build systems that
+		override these values. This will cause a difference in expected flags.
+		The |CL| autospec tooling will attempt to ignore these overrides, but
+		the build system may still need patching. A manual build will not ignore
+		the build system override values if they exist.
+
+Global flags
+	Compiler flags applied at a global level for all packages. The |CL| RPM
+	configuration (`clr-rpm-config <https://github.com/clearlinux/clr-rpm-config>`_)
+	contains global compiler flags. Search the :file:`macros` file for
+	:command:`global_cflags` and search the :file:`rpmrc` file for
+	:command:`optflags`. Global compiler flags may be overridden.
+
+	.. note::
+		|CL| doesn't use RPMs to install software. |CL|
+		distributes software in the form of :ref:`bundles-guide`. The RPM format
+		is only used during the |CL| build process as a way to resolve
+		dependencies.
+
+Per-package flags
+	Compiler flags applied at a per-package level. The package's autospec
+	repository contains the package-specific compiler flags. Search the
+	:file:`.spec` file for the
+	section starting with :command:`export CFLAGS`.
+
+Multiple builds of libraries with CPU-specific optimizations
+============================================================
+
+To fully use the capabilities in different generations of CPU hardware, |CL|
+will perform multiple builds of libraries with CPU-specific optimizations. For
+example, |CL| builds libraries with Intel® Advanced Vector Extensions 2 (Intel®
+AVX2) and Intel® Advanced Vector Extensions 512 (Intel® AVX-512). |CL| can then
+dynamically link to the library with the newest optimization based on the
+processor in the running system. Runtime libraries used by ordinary applications
+benefit from these CPU specific optimizations.
+
+The autospec repository for Python* shows an example of this optimization:
+https://github.com/clearlinux-pkgs/python3
+
+Kernel
+******
+
+A modern kernel with variants optimized for different platforms
+===============================================================
+
+|CL| is a rolling release distribution that uses the newest upstream Linux
+kernel. The Linux kernel has frequent updates which can include performance
+enhancements. It's a policy of the |CL| team to try to upstream any performance
+enhancements in the Linux kernel for all to use.
+
+|CL| `builds different kernel variants <https://docs.01.org/clearlinux/latest/guides/clear/compatible-kernels.html>`_ for compatibility with specific platforms.
+For example, kernels meant to run on virtual machines skip support for much of
+the physical hardware that doesn’t show up in VM environments and will slow down
+boot.
+
+View the kernel configuration and patches to the default native kernel in the
+autospec repository: https://github.com/clearlinux-pkgs/linux/
+
+Utility to enforce kernel runtime parameters
+============================================
+
+The Linux kernel exposes parameters for tuning the behavior of drivers and
+devices such as certain buffers and resource management strategies. |CL| uses a
+small utility, `clr-power-tweaks <https://github.com/clearlinux-pkgs/clr-power-tweaks>`_,
+to set and enforce kernel parameter values weighted towards performance upon
+boot. View the set performance values by running :command:`sudo clr_power --debug`.
+
+Operating system
+****************
+
+Operating system and software build-time optimizations set the stage for high
+performance. Decisions made after the installation of |CL| are equally as
+important.
+
+
+CPU performance governor
+========================
+
+|CL| uses the performance CPU governor which calls for the CPU to operate at
+maximum clock frequency. In other words, P-state P0. The idea behind prioritizing
+maximum CPU performance is that the faster a program finishes execution, the
+faster the CPU can return to a low energy idle state. See the `CPU Power and
+Performance documentation <https://docs.01.org/clearlinux/latest/guides/maintenance/cpu-performance.html>`_
+for further details.
+
+Restructured boot sequence
+==========================
+
+To optimize boot speed, |CL| uses a restructured order for boot processes that
+minimizes the time services wait on slow operations and the time boot processes
+wait on each other.
+
+Systemd-bootchart is a tool for graphing the boot sequence and writes logs to a
+file under :file:`/run/log`. The tool and corresponding log file make diagnosing slow
+boot problems easier. All |CL| systems have `systemd-bootchart <https://github.com/systemd/systemd-bootchart>`_ enabled by default for every boot. systemd-bootchart configuration is
+non-blocking to not materially slow down boot performance.
+
+Related topics
+**************
+
+* :ref:`cpu-performance`
+* `A Linux* OS for Linux Developers <https://clearlinux.org/blogs-news/linux-os-linux-developers>`_
+* `The Performance Race <https://clearlinux.org/news-blogs/performance-race>`_
+* `Boosting Python* from profile-guided to platform-specific optimizations <https://clearlinux.org/news-blogs/boosting-python-profile-guided-platform-specific-optimizations>`_
+* `Transparent use of library packages optimized for Intel® architecture <https://clearlinux.org/news-blogs/transparent-use-library-packages-optimized-intel-architecture>`_
+
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*

source/guides/clear/security.rst

@@ -138,6 +138,8 @@ some examples:
 * `Tallow`_, a lightweight service which monitors and blocks suspicious SSH
   login patterns, is installed with the :command:`openssh-server` bundle.
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _`Security for software update in Clear Linux* OS`: https://clearlinux.org/blogs/security-software-update-clear-linux-os-intel-architecture
 .. _`Recent GNU* C library improvements`: https://clearlinux.org/blogs/recent-gnu-c-library-improvements
 .. _`rolling release`: https://en.wikipedia.org/wiki/Rolling_release

source/guides/clear/swupd-3rd-party.rst

@@ -23,6 +23,8 @@ client system.
    :local:
    :depth: 1
 
+Also, see our `general guidelines`_ on sharing 3rd-party bundles.  
+
 Prerequisite
 *************
 
@@ -198,7 +200,7 @@ All installed 3rd-party bundles reside in :file:`/opt/3rd-party/bundles/<repo-na
 
       tree /opt/3rd-party
 
-   Example out:
+   Example output:
    
    .. code-block:: console
 
@@ -245,6 +247,11 @@ On the client side:
 
 #. Run :command:`sudo swupd 3rd-party update` to update to the latest version of your mix.
 
+   .. note::
+      
+      If `swupd autoupdate` is enabled, 3rd-party repositories will update 
+      automatically as well during regular swupd update.  
+
 #. Now, you can see and add the new bundles.
 
 Some limitations of 3rd-party bundles
@@ -270,4 +277,5 @@ Related topics
    https://clearlinux.org/software
 .. _bundle definition:
    https://docs.01.org/clearlinux/latest/guides/clear/mixer.html#id16
-
+.. _general guidelines:
+   https://community.clearlinux.org/t/about-the-3rd-party-sw-category/4072

source/guides/kernel/kernel-boot-msg.rst

@@ -0,0 +1,69 @@
+.. _kernel-boot-msg:
+
+Capture Kernel Boot Messages in the Journal
+###########################################
+
+By default |CL| does not capture kernel boot messages in the journal logs, 
+where they're reported as "Missed" messages.  This design decision was made 
+to provide a faster boot performance.  On the other hand, if you wish to 
+see the messages, follow this guide.
+
+Here's an example a journal log with "Missed" messages:
+
+.. code-block:: console
+   :linenos:
+   :emphasize-lines: 4
+
+   -- Reboot --
+   Apr 10 19:55:43 kernel systemd-journald[300]: Journal started
+   Apr 10 19:55:43 kernel systemd-journald[300]: Runtime Journal (/run/log/journal/d01862ca79d1064ea379cd715cfdd53a) is 5.8M, max 47.0M, 41.1M free.
+   Apr 10 19:55:43 kernel systemd-journald[300]: Missed 2233 kernel messages
+   Apr 10 19:55:43 kernel systemd[1]: Started Journal Service.
+
+.. contents::
+   :local:
+   :depth: 1
+ 
+Prerequisites
+*************
+
+* `systemd-journald` version 245 and higher 
+
+Enable journaling of kernel boot messages
+*****************************************
+
+#. Open a terminal window.
+
+#. Create a base journald configuration file.
+
+   .. code-block:: bash
+
+      sudo mkdir -p /etc/systemd/journald.conf.d
+      sudo cp /usr/lib/systemd/journald.conf.d/clear.conf /etc/systemd/journald.conf.d/
+
+#. Append :command:`BootKMsg=true` to it.
+   
+   .. code-block:: bash
+
+      echo "BootKMsg=true" | sudo tee -a /etc/systemd/journald.conf.d/clear.conf
+
+#. Reboot.  
+
+.. tip::
+
+   If you need to increase the kernel buffer length (for example, 1M), do this: 
+
+   .. code-block:: bash
+   
+      sudo mkdir -p /etc/kernel/cmdline.d/
+      echo "log_buf_len=1M" | sudo tee /etc/kernel/cmdline.d/log_buf_len.conf
+      sudo clr-boot-manager update
+
+Alternative
+***********
+
+An alternative is to use :command:`dmesg`.  
+
+.. code-block:: bash
+
+   sudo dmesg 

source/guides/kernel/kernel-modules-dkms.rst

@@ -64,14 +64,16 @@ The :command:`kernel-native-dkms` bundle also:
    kernel. This is especially important for systems where a successful boot
    relies on a kernel module.
 
+.. _kernel-modules-dkms-install-begin-alt:
+
 Install the :command:`kernel-native-dkms` or :command:`kernel-lts-dkms`
-bundle:
+bundle.
 
 #. Determine which kernel variant is running on |CL|. Only the *native*
    and *lts* kernels are enabled to build and load out-of-tree kernel modules
    with DKMS.
 
-   .. code-block:: bash
+   .. code-block:: console
 
       $ uname -r
       5.XX.YY-ZZZZ.native
@@ -93,7 +95,8 @@ bundle:
       sudo swupd bundle-add kernel-lts-dkms
 
 
-#. Update the |CL| bootloader and reboot.
+#. Update the |CL| bootloader and reboot, and 
+   ensure that you can start the new kernel.
 
    .. code-block:: bash
 
@@ -225,9 +228,9 @@ The instructions below show a generic example:
 #. Create or modify the :file:`dkms.conf` file inside of the extracted source
    code directory.
 
-   .. code-block:: bash
+   .. code-block:: ShellSession
 
-      $EDITOR dkms.conf
+      $ EDITOR dkms.conf
 
       MAKE="make -C src/ KERNELDIR=/lib/modules/${kernelver}/build"
       CLEAN="make -C src/ clean"

source/guides/maintenance/configure-hugepages.rst

@@ -23,7 +23,7 @@ to add hugepages to the system and how to change the default hugepage size.
       
       The active option is enclosed in brackets. In this case, always is active, 
       which means hugepages are enabled for every process. The `madvise` 
-      option means that hugepages are enabled for processes that explicitely 
+      option means that hugepages are enabled for processes that explicitly 
       call `madvise`_. 
 
 #. To check the size of hugepages, run the below command.
@@ -71,4 +71,4 @@ to add hugepages to the system and how to change the default hugepage size.
       sudo clr-boot-manager update
       sudo reboot
 
-   .. _madvise: https://linux.die.net/man/2/madvise
+   .. _madvise: https://linux.die.net/man/2/madvise

source/guides/maintenance/cpu-performance.rst

@@ -203,11 +203,10 @@ Better thermal control and performance can be achieved by providing platform
 specific configuration to :command:`thermald`.
 
 `Linux DPTF Extract Utility`_ is a companion tool to :command:`thermald`,
-This tool uses Intel® 
+This tool uses Intel® Dynamic Platform and Thermal Framework (Intel® DPTF)
-:abbr:`DPTF (Dynamic Platform and Thermal Framework)` technology and
+technology and can convert to the :file:`thermal_conf.xml` configuration format
-can convert to the :file:`thermal_conf.xml` configuration format used by
+used by :command:`thermald`. Closed-source projects, like this one, cannot be
-:command:`thermald`. Closed-source projects, like this one, cannot be packaged
+packaged as a bundle in |CL|, so you must install it manually:
-as a bundle in |CL|, so you must install it manually:
 
 #. Make sure your machine's BIOS has DPTF feature and is enabled. It will usually be in the :guilabel:`Advanced` or :guilabel:`Advanced>Power` section of the BIOS. 
 
@@ -248,15 +247,13 @@ The following output means the configuration has already been applied:
 
    thermald[*]: [WARN]Using generated /etc/thermald/thermal-conf.xml.auto
 
-.. admonition:: Disclaimer
+*Intel® Turbo Boost Technology requires a PC with a processor with Intel Turbo
+Boost Technology capability. Intel Turbo Boost Technology performance varies
+depending on hardware, software and overall system configuration. Check with
+your PC manufacturer on whether your system delivers Intel Turbo Boost Technology.
+For more information, see http://www.intel.com/technology/turboboost*
 
-   Intel® Turbo Boost Technology requires a PC with a processor with Intel
+*Intel, Intel SpeedStep, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
-   Turbo Boost Technology capability. Intel Turbo Boost Technology performance
-   varies depending on hardware, software and overall system configuration.
-   Check with your PC manufacturer on whether your system delivers Intel Turbo
-   Boost Technology. For more information, see http://www.intel.com/technology/turboboost
-
-   Intel SpeedStep is a trademark of Intel Corporation or its subsidiaries.
 
 
 .. _`Intel P-state driver`: https://www.kernel.org/doc/Documentation/cpu-freq/intel-pstate.txt

source/guides/maintenance/developer-workstation.rst

@@ -78,9 +78,6 @@ tools you need to start. Consider these profiles as a starting point.
          * - Work with deep learning and edge-optimized models.
            - `computer-vision-models <https://clearlinux.org/software/bundle/computer-vision-models/>`_
 
-         * - Basic OpenVINO™ toolkit.
-           - `computer-vision-openvino <https://clearlinux.org/software/bundle/computer-vision-openvino/>`_
-
          * - API helper for cloud access.
            - `cloud-api <https://clearlinux.org/software/bundle/cloud-api/>`_
 

source/guides/maintenance/fix-broken-install.rst

@@ -37,7 +37,8 @@ Boot a live desktop image to fix target system
 Mount root partition, verify, and fix
 *************************************
 
-#. Ensure the system is connected to the network.
+#. Ensure the system is connected to the Internet in order to access the 
+   the |CL| update server.
 
 #. Open a terminal window.
 
@@ -51,9 +52,8 @@ Mount root partition, verify, and fix
    Example output:
 
    .. code-block:: console
-      :emphasize-lines: 10
+      :emphasize-lines: 9 
 
-      clrlinux@clr-live~ $ lsblk -po NAME,SIZE,LABEL,PARTTYPE,PARTLABEL
       NAME          SIZE LABEL       PARTTYPE                             PARTLABEL
       /dev/loop0  643.6M                                                  
       /dev/sda     14.3G CLR_ISO                                          
@@ -73,7 +73,7 @@ Mount root partition, verify, and fix
       sudo mount /dev/sdb3 /mnt
 
 #. Verify that you mounted the correct root partition by verifying the content
-   of ``/usr/lib/os-release`` looks similar to the example below.  
+   of ``/mnt/usr/lib/os-release`` looks similar to the example below.  
 
    .. code-block:: bash
 
@@ -83,7 +83,6 @@ Mount root partition, verify, and fix
 
    .. code-block:: console
 
-      clrlinux@clr-live~ $ cat /mnt/usr/lib/os-release 
       NAME="Clear Linux OS"
       VERSION=1
       ID=clear-linux-os
@@ -96,15 +95,15 @@ Mount root partition, verify, and fix
       BUG_REPORT_URL="mailto:dev@lists.clearlinux.org"
       PRIVACY_POLICY_URL="http://www.intel.com/privacy"
 
-#. Next, run :command:`swupd` to fix any issues on the target system.
+#. Next, run :command:`swupd repair` to fix any issues on the target system.
 
    .. code-block:: bash
 
-      sudo swupd repair --picky --path=/mnt
+      sudo swupd repair --picky --path=/mnt --statedir=/mnt/var/lib/swupd
 
    :ref:`Learn more about how swupd works <swupd-guide>`.
 
-#. After the process is complete, unmount the root partition:
+#. After the process is complete, unmount the root partition.
 
    .. code-block:: bash
 

source/guides/network/proxy.rst

@@ -0,0 +1,116 @@
+.. _proxy:
+
+Proxy Configuration
+###################
+
+When working behind a corporate proxy server, one typically has to configure
+proxy settings for applications to reach the Internet.  |CL-ATTR| has 
+implemented an :ref:`autoproxy` feature to try and eliminate manual 
+configurations as much as possible.  However, there are still some applications 
+that cannot take full advantage of the :ref:`autoproxy` feature due to their 
+own ways of configuring proxy settings.  This guide shows you how to configure 
+proxy settings for some of the known applications manually.  
+
+.. contents::
+    :local:
+    :depth: 1
+
+Prerequisites
+*************
+
+* You have installed |CL| on your host system.
+
+  For detailed instructions on installing |CL| on a bare metal system, visit
+  the :ref:`bare metal installation guide <bare-metal-install-desktop>`.
+
+General proxy settings for many applications
+============================================
+
+#. First, apply these general proxy settings which should work for many 
+   applications.  If they do not work for a specific application, such as the 
+   ones listed below, apply application-specific proxy settings as needed.
+
+   Proxy settings:
+
+   .. code-block:: none
+
+      export http_proxy=http://<YOUR.HTTP-PROXY.URL:PORT>
+      export https_proxy=http://<YOUR.HTTPS-PROXY.URL:PORT>
+      export ftp_proxy=http://<YOUR.FTP-PROXY.URL:PORT>
+      export socks_proxy=http://<YOUR.SOCKS-PROXY.URL:PORT>
+      export no_proxy="<YOUR-DOMAIN>,localhost"
+      export HTTP_PROXY=$http_proxy
+      export HTTPS_PROXY=$https_proxy
+      export FTP_PROXY=$ftp_proxy
+      export SOCKS_PROXY=$socks_proxy
+      export NO_PROXY=$no_proxy
+
+   * *User-specific*, put them in :file:`$HOME/.bashrc`.
+
+   * *For all users*, put them in :file:`/etc/profile.d/proxy.conf`. 
+
+#. For the proxies to take effect, either :command:`source` the file manually
+   or log out and log back in.
+
+Docker\*
+========
+
+Please refer the official Docker links on how to configure proxies:
+
+* `Docker client`_
+* `Docker daemon`_
+
+git over SSH
+============
+
+Add the following to your :file:`~/.ssh/config` file:
+
+.. code-block:: none
+
+   host github.com
+        port 22
+        user git
+        ProxyCommand connect-proxy -S <YOUR.SSH-PROXY.URL:PORT> %h %p
+
+.. note::
+
+   Though :command:`netcat` is included with |CL|, it is not the BSD version,
+   which is the one usually used to enable git over SSH.
+
+autospec/mock
+=============
+
+:ref:`autospec` uses mock to do builds.  Configure mock's proxy settings with
+these steps:  
+
+#. Override the general mock configuration file with a custom one, otherwise
+   your settings will get overwritten each time autospec is updated.
+
+   .. code-block:: bash
+
+      sudo mkdir -p /etc/mock
+      sudo cp ~/clearlinux/projects/common/conf/clear.cfg /etc/mock/clear-custom.cfg
+
+#. :command:`sudoedit` :file:`/etc/mock/clear-custom.cfg` and add the highlighted
+   lines.
+
+   .. code-block:: none
+      :emphasize-lines: 3-5
+
+      ...
+      config_opts['use_bootstrap_container'] = False
+      config_opts['http_proxy'] = '<YOUR.HTTP.PROXY.URL>:<PORT>'
+      config_opts['https_proxy'] = '<YOUR.HTTPS.PROXY.URL>:<PORT>'
+      config_opts['no_proxy'] = '<YOUR.DOMAIN>,192.168.0.0/16,localhost,127.0.0.0/8'
+
+Kubernetes
+==========
+
+See :ref:`Setting proxy servers for Kubernetes <kubernetes>`.
+
+.. _Docker client:
+   https://docs.docker.com/network/proxy/#configure-the-docker-client
+
+.. _Docker daemon:
+   https://docs.docker.com/config/daemon/systemd/#httphttps-proxy
+

source/guides/network/vnc.rst

@@ -639,7 +639,7 @@ For Method 2:
 
    .. code-block:: bash
 
-      sudo systemctl daemon-load
+      sudo systemctl daemon-reload
       sudo systemctl restart vncserver@:5.service
 
 For Method 3:

source/guides/stacks/dars.rst

@@ -24,7 +24,8 @@ The Data Analytics Reference Stack provides two pre-built Docker images,
 available on `Docker Hub`_:
 
 * A |CL|-derived `DARS with OpenBlas`_ stack optimized for `OpenBLAS`_
-* A |CL|-derived  `DARS with Intel® MKL`_ stack optimized for `MKL`_ (Intel® Math Kernel Library)
+* A |CL|-derived  `DARS with Intel® MKL`_ stack optimized for
+  `Intel® Math Kernel Library`_ (Intel® MKL)
 
 We recommend you view the latest component versions for each image in the
 :file:`releasenote` found in the `Data Analytics Reference Stack`_ GitHub\*
@@ -98,7 +99,7 @@ If you choose to build your own DARS container images, you can customize them as
 
 To construct images with |CL|, start with a |CL| development platform that has the :command:`containers-basic-dev` bundle installed. Learn more about bundles and installing them by using :ref:`swupd-guide`.
 
-#. The `Data Analytics Reference Stack`_ is part of the Intel® stacks GitHub\* repository. Clone the :file:`stacks` repository.
+#. The `Data Analytics Reference Stack`_ is part of the Intel stacks GitHub\* repository. Clone the :file:`stacks` repository.
 
    .. code-block:: bash
 
@@ -671,6 +672,7 @@ This exception can be disregarded because DARS does not use hadoop.hive.shims. H
 
 #. There is an exception message `Exception in thread "Thread-3" java.lang.ExceptionInInitializerError at org.apache.hadoop.hive.conf.HiveConf` This is related to the same issue with |CL| and JDK11 noted above, and does not affect DARS for the same reason.
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
 
 .. _Data Analytics Reference Stack: https://github.com/intel/stacks/tree/master/dars/clearlinux
 
@@ -678,7 +680,7 @@ This exception can be disregarded because DARS does not use hadoop.hive.shims. H
 
 .. _OpenBLAS: http://www.openblas.net/
 
-.. _MKL: https://software.intel.com/en-us/mkl
+.. _Intel® Math Kernel Library: https://software.intel.com/en-us/mkl
 
 .. _CentOS: https://www.centos.org/
 

source/guides/stacks/dbrs.rst

@@ -14,7 +14,7 @@ Overview
 ********
 
 The Database Reference Stack is integrated, highly-performant, open source,
-and optimized for 2nd generation Intel® Xeon® Scalable Processors and Intel®
+and optimized for 2nd generation Intel® Xeon® Scalable processors and Intel®
 Optane™ persistent memory. This open source community release is part of
 an effort to ensure developers have easy access to the features and
 functionality of Intel Platforms.
@@ -52,10 +52,10 @@ The release announcement for each release provides more detail about the stack f
 Hardware Requirements
 *********************
 
-* Intel® Xeon Scalable Platform with Intel® C620 chipset series
+* Intel Xeon Scalable platform with Intel® C620 series chipset
-* 2nd Gen Intel® Xeon Scalable processor CPU (Intel® Optane™ PMM-enabled stepping) Provides cache & memory control.  Intel® Optane™  PMem works only on systems powered by 2nd Generation Intel® Xeon® Platinum or Gold processors.
+* 2nd Gen Intel Xeon Scalable processor CPU (Intel® Optane™ PMem-enabled stepping). Provides cache & memory control.  Intel Optane PMem works only on systems powered by 2nd Generation Intel® Xeon® Platinum or Intel® Xeon® Gold processors.
 * BIOS with Reference Code
-* Intel®Optane™ PMem
+* Intel Optane PMem
 
 Hardware configuration used in stacks development
 =================================================
@@ -64,12 +64,12 @@ Hardware configuration used in stacks development
 * BIOS with Reference Code
   * BIOS ID: SE5C620.86B.0D.01.0438.032620191658
   * BMC Firmware: 1.94.6b42b91d
-  * Intel® Optane™ PMemFirmware: 1.2.0.5310
+  * Intel Optane PMem Firmware: 1.2.0.5310
-* 2x Intel® Xeon Platinum 8268 Processor
+* 2x Intel Xeon Platinum 8268 Processor
-* Intel® SSD DC S5600 Series 960GB 2.5in SATA Drive
+* Intel® SSD Data Center Family S5600 Series 960GB 2.5in SATA Drive
 * 64 GB RAM - Distributed in 4x 16 GB DDR4 DIMM's
-* 2x Intel® Optane™ PMem 256GB Module
+* 2x Intel Optane PMem 256GB Module
-* 1-1-1 Layout 8 Optane™ : 1 RAM ratio
+* 1-1-1 Layout 8 Intel Optane : 1 RAM ratio
 
 
 .. list-table:: **Table 1. IMC**
@@ -104,7 +104,7 @@ Firmware configuration
 
    When updating DCPMM Firmware, all DCPMM parts must be in the same mode (you cannot mix 1LM and 2LM parts).
 
-The latest firmware download for the Intel® Server System S2600WF Family is available at the `Intel Download Center`_
+The latest firmware download for the Intel® Server Board S2600WF Family is available at the `Intel Download Center`_
 
 Firmware Update Steps
 =====================
@@ -117,7 +117,7 @@ Firmware Update Steps
 #. After update BMC firmware, system BIOS, ME firmware,FD, FRUSDR, system will reboot automatically.
 
 
-If Intel® Optane™ PMem is installed, run startup.nsh a second time after the first reboot to upgrade Intel® Optane™ PMem Firmware:
+If Intel Optane PMem is installed, run startup.nsh a second time after the first reboot to upgrade Intel Optane PMem Firmware:
 
 * Boot to EFI shell.
 * Input "fsx(x:0,1,...):" to enter into your usb disk
@@ -134,9 +134,9 @@ Online Resources
 
 Before going through the configuration steps, we strongly recommend visiting the following resources and wikis to have a broader understanding of what is being done
 
-* `Quick Start Guide`_ Configure Intel® Optane™ PMem Modules on Linux
+* `Quick Start Guide`_ Configure Intel Optane PMem Modules on Linux
 * `Managing NVDIMMs`_
-* `Configure, Manage, and Profile`_ Intel® Optane™ PMem Modules
+* `Configure, Manage, and Profile`_ Intel Optane PMem Modules
 
 Optane™ DIMM Configuration
 ==========================
@@ -569,7 +569,7 @@ Eventually all the given nodes will be shown as running using :command:`kubectl
 Running DBRS with Redis
 ***********************
 
-The Redis stack application is enabled for a multinode Kubernetes environment using Intel® Optane™ DCPMM PMem DIMMs in fsdax mode for storage.
+The Redis stack application is enabled for a multinode Kubernetes environment using Intel Optane DCPMM PMem DIMMs in fsdax mode for storage.
 
 The source code used for this application can be found in the `Github repository`_
 
@@ -584,7 +584,7 @@ The following examples will use the `Docker image with Redis`_.  You can also bu
 Single node
 ===========
 
-Prior to starting the container, you will need to have the Intel® Optane™ DCPMM module in fsdax with a file system and mounted in `/mnt/dax0` as shown above.
+Prior to starting the container, you will need to have the Intel Optane DCPMM module in fsdax with a file system and mounted in `/mnt/dax0` as shown above.
 
 Use the following to start the container, replacing ${DOCKER_IMAGE} with the name of the image you are using.
 
@@ -664,6 +664,8 @@ where:
 For more information please refer to this `blog post`_ from `Memcached`_
 
 
+*Intel, Xeon, Intel Optane, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 
 .. _Intel Download Center: https://downloadcenter.intel.com/download/28695/Intel-Server-Board-S2600WF-Family-BIOS-and-Firmware-Update-Package-for-UEFI
 

source/guides/stacks/dlrs-inference.rst

@@ -19,7 +19,7 @@ Overview
 
 The solution covered here requires the following software components:
 
-* `Deep Learning Reference Stack`_ which is a |CL-ATTR| based Docker\* container providing deep learning frameworks and is optimized for Intel Xeon Scalable platforms.
+* `Deep Learning Reference Stack`_ which is a |CL-ATTR| based Docker\* container providing deep learning frameworks and is optimized for Intel® Xeon® Scalable processors.
 * `Kubeflow`_ is the machine learning toolkit for Kubernetes that helps with deployment of Seldon Core and Istio components.
 * `Seldon Core`_ is a software platform for deploying machine learning models. We use the DLRS container to serve the OpenVino\* framework for inference with the Seldon Core.
 * The OpenVino Model Server is included in DLRS and provides the OpenVino framework for inference. From OpenVino, the `OpenVino Toolkit`_ provides improved neural network performance on a variety of Intel processors. For this guide, we converted pre-trained Caffe models into the `Intermediate Representation(IR)`_ of ResNet50 with the OpenVino toolkit.
@@ -85,7 +85,7 @@ Although this guide assumes a |CL| host system, it has also been validated with
 Recommended Hardware
 ====================
 
-We validated this guide on an `Intel Cascade Lake`_ server and this is recommended to get optimal performance and take advantage of the built in  Intel® Deep Learning Boost functionality.
+We validated this guide on a server with a `2nd Generation Intel Xeon Scalable processor`_, formerly Cascade Lake, and this is recommended to get optimal performance and take advantage of the built in Intel® Deep Learning Boost (Intel® DL Boost) functionality.
 
 Required Software
 =================
@@ -917,7 +917,7 @@ To find out how to assign the cores and memory properly run :command:`numactl -H
      1:  21  10
 
 
-In this case, the tests are run on Intel(R) Xeon(R) Platinum 6260L with 2 sockets(nodes) and 24 cores (CPUs) on each socket.
+In this case, the tests are run on Intel® Xeon® Platinum 6260L processor with 2 sockets(nodes) and 24 cores (CPUs) on each socket.
 Running the inference serving the application with :command:`numactl --membind=0 --cpubind=0-3` forces the system to use 0,1,2,3 cores and memory located on the same socket (0). To use all available cores there is a need to create more service deployments assigned to the remaining cores.
 
 The `ai-inferencing` repository contains an example deployment script with 2 cores per instance assignment.
@@ -1029,7 +1029,7 @@ The test performed on a 2 node cluster with 48 cores per node showed that there
       KMP_BLOCKTIME=1
 
 
-
+*Intel, Xeon, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
 
 
 .. _Deep Learning Reference Stack: https://clearlinux.org/stacks/deep-learning
@@ -1040,7 +1040,7 @@ The test performed on a 2 node cluster with 48 cores per node showed that there
 .. _Istio: https://istio.io/
 .. _Source-to-Image: https://github.com/openshift/source-to-image
 .. _Min.io: https://min.io/
-.. _Intel Cascade Lake: https://www.intel.com/content/www/us/en/design/products-and-solutions/processors-and-chipsets/cascade-lake/2nd-gen-intel-xeon-scalable-processors.html
+.. _2nd Generation Intel Xeon Scalable processor: https://www.intel.com/content/www/us/en/design/products-and-solutions/processors-and-chipsets/cascade-lake/2nd-gen-intel-xeon-scalable-processors.html
 .. _Docker 18.09: https://kubernetes.io/docs/setup/production-environment/container-runtimes/
 .. _Kubernetes 1.15.3: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
 .. _gsutil: https://cloud.google.com/storage/docs/gsutil_install#linux

source/guides/stacks/dlrs.rst

@@ -14,29 +14,32 @@ Overview
 ********
 
 We created the Deep Learning Reference Stack to help AI developers deliver
-the best experience on Intel® Architecture. This stack reduces complexity
+the best experience on Intel® architecture. This stack reduces complexity
 common with deep learning software components, provides flexibility for
 customized solutions, and enables you to quickly prototype and deploy Deep
 Learning workloads. Use this guide to run benchmarking workloads on your
 solution.
 
-The latest release of the Deep Learning Reference Stack (`DLRS V5.0`_ ) supports the following features:
+The latest release of the Deep Learning Reference Stack (`DLRS V6.0`_ ) supports the following features:
 
-* TensorFlow* 1.15 and TensorFlow* 2.0, an end-to-end open source platform for machine learning (ML).
+* TensorFlow* 1.15 and TensorFlow* 2.2.0(rc1), an end-to-end open source platform for machine learning (ML).
-* PyTorch* 1.3, an open source machine learning framework that accelerates the path from research prototyping to production deployment.
+* PyTorch* 1.4, an open source machine learning framework that accelerates the path from research prototyping to production deployment.
 * PyTorch Lightning* which is a lightweight wrapper for PyTorch designed to help researchers set up all the boilerplate state-of-the-art training.
-* Transformers* , a state-of-the-art Natural Language Processing (NLP) for TensorFlow 2.0 and PyTorch.
+* Transformers* which is a state-of-the-art Natural Language Processing (NLP) library for TensorFlow 2.0 and PyTorch
-* Intel® OpenVINO™ model server version 2019_R3, delivering improved neural network performance on Intel processors, helping unlock cost-effective, real-time vision applications.
+* Flair*, a PyTorch NLP framework
-* Intel Deep Learning Boost (DL Boost) with AVX-512 Vector Neural Network Instruction (Intel AVX-512 VNNI) designed to accelerate deep neural network-based algorithms.
+* OpenVINO™ model server version 2020.1, delivering improved neural network performance on Intel processors, helping unlock cost-effective, real-time vision applications.
+* Intel® Deep Learning Boost (Intel® DL Boost) with Intel® Advanced Vector
+  Extensions 512 (Intel® AVX-512) Vector Neural Network Instruction , designed to
+  accelerate deep neural network-based algorithms.
 * Deep Learning Compilers (TVM* 0.6), an end-to-end compiler stack.
 
+
 .. important::
 
-   To take advantage of the Intel® AVX-512 and VNNI functionality (including the MKL-DNN releases)  with the Deep
+   To take advantage of the Intel AVX-512 and VNNI functionality (including the Intel® oneAPI Deep Neural Network Library (oneDNN), found at  `oneDNN`_,  with the Deep Learning Reference Stack, you must use the following hardware:
-   Learning Reference Stack, you must use the following hardware:
 
-   * Intel® AVX-512 images require an Intel® Xeon® Scalable Platform
+   * Intel AVX-512 images require an Intel® Xeon® Scalable processor
-   * VNNI requires a 2nd generation Intel® Xeon® Scalable Platform
+   * VNNI requires a 2nd generation Intel Xeon Scalable processor
 
 
 Releases
@@ -44,7 +47,7 @@ Releases
 
 Refer to the `Deep Learning Reference Stack website`_ for information and download links for the different versions and offerings of the stack.
 
-
+* `DLRS V6.0`_ release announcement.
 * `DLRS V5.0`_ release announcement.
 * `DLRS V4.0`_ release announcement, including benchmark results.
 * `DLRS V3.0`_ release announcement, including benchmark results.
@@ -64,8 +67,8 @@ Version compatibility
 
 We validated the steps in this guide against the following software package versions, unless otherwise stated:
 
-* |CL| 26240 (Minimum supported version)
+* |CL| 31290 (Minimum supported version)
-* Docker 18.06.1
+* Docker 19.03
 * Kubernetes 1.11.3
 * Go 1.11.12
 
@@ -133,7 +136,7 @@ a simple shell script, and keep the pod open.
       spec:
         containers:
         - name: ex-pod-container
-          image: clearlinux/stacks-dlrs-mkl:latest
+          image: sysstacks/dlrs-tensorflow-clearlinux:latest
           command: ['/bin/bash', '-c', '--']
           args: [ "while true; do sleep 30; done" ]
 
@@ -160,14 +163,14 @@ TensorFlow.
    Performance test results for the Deep Learning Reference Stack and for this
    guide were obtained using `runc` as the runtime.
 
-#. Download either the `Eigen`_ or the `Intel MKL-DNN`_ Docker image
+#. Download either the `TensorFlow Eigen`_ or the `TensorFlow oneDNN`_ Docker image
    from `Docker Hub`_.
 
 #. Run the image with Docker:
 
    .. code-block:: bash
 
-      docker run --name <image name>  --rm -ti <clearlinux/stacks-dlrs-TYPE> bash
+      docker run --name <image name>  --rm -ti <sysstacks/dlrs-tensorflow-clearlinux> bash
 
    .. note::
 
@@ -193,7 +196,7 @@ TensorFlow.
    TensorFlow benchmarks.
 
    If you are using an FP32 based model, it can be converted to an int8 model
-   using `Intel® quantization tools`_.
+   using `Intel® AI Quantization Tools for TensorFlow`_.
 
 PyTorch single and multi-node benchmarks
 ****************************************
@@ -202,7 +205,7 @@ This section describes running the `PyTorch benchmarks`_ for Caffe2 in
 single node.
 
 #. Download either the `PyTorch with OpenBLAS`_ or the `PyTorch with Intel
-   MKL-DNN`_ Docker image from `Docker Hub`_.
+   oneDNN`_ Docker image from `Docker Hub`_.
 
 #. Run the image with Docker:
 
@@ -236,8 +239,8 @@ TensorFlow Training (TFJob) with Kubeflow and DLRS
 
 .. warning::
 
-   If you choose the Intel® MKL-DNN or Intel® MKL-DNN-VNNI image, your platform
+   If you choose the Intel oneDNN image, your platform
-   must support the Intel® AVX-512 instruction set. Otherwise, an
+   must support the Intel AVX-512 instruction set. Otherwise, an
    *illegal instruction* error may appear, and you won’t be able to complete this guide.
 
 A `TFJob`_  is Kubeflow's custom resource used to run TensorFlow training jobs on Kubernetes. This example shows how to use a TFJob within the DLRS container.
@@ -517,14 +520,69 @@ To use the new image we will follow the `Kubeflow OpenMPI instructions`_. You wi
       # Deploy to your cluster.
       ks apply default
 
+Using Transformers* for Natural Language Processing
+***************************************************
+
+The DLRS v5.0 release includes `Transformers`_, a state-of-the-art Natural Language Processing (NLP) library for TensorFlow 2.0 and PyTorch. The library is configured to work within the container environment.
+
+In this section we use a Jupyter Notebook from inside the container to walk through one of the notebooks shown in the `Transformers`_ repository.
+
+To run the notebook, you will need to run the Deep Learning Reference Stack, mount it to disk and connect a Jupyter Notebook port.
 
 
-Using the Intel® OpenVINO Model Optimizer
+#. Run the DLRS image with Docker:
-*****************************************
 
-The Intel OpenVINO toolkit has two primary tools for deep learning, the inference engine and the model optimizer. The inference engine is integrated into the Deep Learning Reference Stack. It is better to use the model optimizer after training the model, and before inference begins. This example will explain how to use the model optimizer by going through a test case with a pre-trained TensorFlow model.
+   .. code-block:: bash
 
-This example uses resources found in the following OpenVino Toolkit documentation.
+      docker run -it -v ${PWD}:/workspace -p 8888:8888 clearlinux/stacks-pytorch-mkl:latest
+
+
+#. From within the container, navigate to the workspace, and clone the
+   transformers repository in the container:
+
+   .. code-block:: bash
+
+      cd workspace
+      git clone https://gist.github.com/16d38f2c9c688963c166c000330a3c11.git
+
+
+
+#. Start a Jupyter Notebook that is linked to the exterior port.
+   Be sure to copy the token from the output of starting  Jupyter Notebook.
+
+   .. code-block:: bash
+
+      pip install jupyter --upgrade
+      jupyter notebook --ip 0.0.0.0 --no-browser --allow-root
+
+#. To access the Jupyter Notebook, open a browser.
+
+#. Return to the Terminal where you launched Jupyter Notebook.
+   Copy one of the URLs that appears after "Or copy and paste on of these URLs."
+
+#. Paste the URL (with embedded token) into the browser window.
+
+
+The notebook will also be available at the URL of the system serving the notebook.  For example if you are running on 192.168.1.10, you will be able to access the notebook from other systems on that subnet by navigating to \http://192.168.1.10:8888
+
+From the browser, you will see the following notebooks.
+
+.. figure:: ../../_figures/stacks/dlrs-transformers-1.png
+   :scale: 80%
+   :alt: Transformers Jupyter Notebooks
+
+   Figure 1: Transformers Jupyter Notebooks
+
+
+This example along with the other notebooks show how to get up and running with Transformers.  More detail on using Transformers* is available through the `Transformers`_ github repository.
+
+
+Using the OpenVINO™ Model Optimizer
+***********************************
+
+The OpenVINO™ toolkit has two primary tools for deep learning, the inference engine and the model optimizer. The inference engine is integrated into the Deep Learning Reference Stack. It is better to use the model optimizer after training the model, and before inference begins. This example will explain how to use the model optimizer by going through a test case with a pre-trained TensorFlow model.
+
+This example uses resources found in the following OpenVINO™ toolkit documentation.
 
 `Converting a TensorFlow Model`_
 
@@ -539,7 +597,7 @@ In this example, you will:
 
 #. Download a TensorFlow model
 
-   We will be using an OpenVINO supported topology with the Model Optimizer. We will use a TensorFlow Inception V2 frozen model.
+   We will be using an OpenVINO™ toolkit supported topology with the Model Optimizer. We will use a TensorFlow Inception V2 frozen model.
 
    Navigate to the `OpenVINO TensorFlow Model page`_. Then scroll down to the second section titled "Supported Frozen Topologies from TensorFlow Object Detection Models Zoo" and download "SSD Inception V2 COCO."
 
@@ -596,10 +654,12 @@ In this example, you will:
       --reverse_input_channels
 
 
-   You should now see three files in your working directory, :file:`frozen_inference_graph.bin`, :file:`frozen_inference_graph.mapping`, and :file:`frozen_inference_graph.xml`. These are your new models in the Intermediate Representation (IR) format and they are ready for use in the OpenVINO Inference Engine.
+   You should now see three files in your working directory, :file:`frozen_inference_graph.bin`, :file:`frozen_inference_graph.mapping`, and :file:`frozen_inference_graph.xml`. These are your new models in the Intermediate Representation (IR) format and they are ready for use in the OpenVINO™ Inference Engine.
 
-Using the OpenVino Inference Engine
+
-***********************************
+
+Using the OpenVINO™ toolkit Inference Engine
+********************************************
 
 This example walks through the basic instructions for using the inference engine.
 
@@ -607,11 +667,11 @@ This example walks through the basic instructions for using the inference engine
 
    The process is similar to how we start `Jupter notebooks` on our containers
 
-   Run this command to spin up a OpenVino model fetched from GCP
+   Run this command to spin up a OpenVINO™ toolkit model fetched from GCP
 
    .. code-block:: bash
 
-      docker run -p 8000:8000 stacks-dlrs-mkl:latest bash -c ". /workspace/scripts/serve.sh && ie_serving model --model_name resnet --model_path gs://intelai_public_models/resnet_50_i8 --port 8000"
+      docker run -p 8000:8000 stacks-dlrs-mkl:latest bash -c ". /workspace/scripts/serve.sh && ie_serving model --model_name resnet --model_path gs://public-artifacts/intelai_public_models/resnet_50_i8 --port 8000"
 
 
    Once the server is setup, use a :command:`grpc` client to communicate with served model:
@@ -670,10 +730,10 @@ This example walks through the basic instructions for using the inference engine
 
 
 
-Using Seldon and OpenVINO* model server with the Deep Learning Reference Stack
+Using Seldon and OpenVINO™ model server with the Deep Learning Reference Stack
-******************************************************************************
+*************************************************************************************
 
-`Seldon Core`_  is an open source platform for deploying machine learning models on a Kubernetes cluster. In this section we will walk through using a Seldon server with OpenVINO to serve a model.
+`Seldon Core`_  is an open source platform for deploying machine learning models on a Kubernetes cluster. In this section we will walk through using a Seldon server with OpenVINO™ model server.
 
 Pre-requisites
 ==============
@@ -731,7 +791,7 @@ We will create a small pod to get the model into a volume.
 Add the pre-trained model to the image
 --------------------------------------
 
-A custom DLRS image is provided to serve OpenVINO through Seldon. Add a curl command to download your publicly hosted model and save it in :file:`/opt/ml` in the container filesystem. For example, if you have a model on GCP, use this command:
+A custom DLRS image is provided to serve OpenVINO™ model server through Seldon. Add a curl command to download your publicly hosted model and save it in :file:`/opt/ml` in the container filesystem. For example, if you have a model on GCP, use this command:
 
    .. code-block:: bash
 
@@ -742,7 +802,7 @@ A custom DLRS image is provided to serve OpenVINO through Seldon. Add a curl com
 Prepare the DLRS image
 ======================
 
-A base image with Seldon and the OpenVINO inference engine should be created using the :file:`Dockerfile_openvino_base` dockerfile.
+A base image with Seldon and the OpenVINO™ inference engine should be created using the :file:`Dockerfile_openvino_base` dockerfile.
 
    .. code-block:: bash
 
@@ -910,7 +970,7 @@ To stop the container, execute the following from your host system:
    .. code-block:: console
 
       CONTAINER ID        IMAGE                        COMMAND               CREATED             STATUS              PORTS               NAMES
-      e131dc71d339        clearlinux/stacks-dlrs-oss   "/bin/sh -c 'bash'"   23 seconds ago      Up 21 seconds                           oss
+      e131dc71d339        sysstacks/dlrs-tensorflow-clearlinux   "/bin/sh -c 'bash'"   23 seconds ago      Up 21 seconds                           oss
 
 #. You can then use the ID or container name to stop the container. This example
    uses the name "oss":
@@ -938,8 +998,8 @@ To stop the container, execute the following from your host system:
    .. code-block:: console
 
       REPOSITORY                   TAG                 IMAGE ID            CREATED             SIZE
-      clearlinux/stacks-dlrs-oss   latest              82757ec1648a        4 weeks ago         3.43GB
+      sysstacks/dlrs-tensorflow-clearlinux   latest              82757ec1648a        4 weeks ago         3.43GB
-      clearlinux/stacks-dlrs-mkl   latest              61c178102228        4 weeks ago         2.76GB
+      sysstacks/dlrs-tensorflow-clearlinux   latest              61c178102228        4 weeks ago         2.76GB
 
 #. To remove an image use the image ID:
 
@@ -950,8 +1010,8 @@ To stop the container, execute the following from your host system:
    .. code-block:: console
 
       # docker rmi 827
-      Untagged: clearlinux/stacks-dlrs-oss:latest
+      Untagged: sysstacks/dlrs-tensorflow-clearlinux:latest
-      Untagged: clearlinux/stacks-dlrs-oss@sha256:381f4b604537b2cb7fb5b583a8a847a50c4ed776f8e677e2354932eb82f18898
+      Untagged: sysstacks/dlrs-tensorflow-clearlinux@sha256:381f4b604537b2cb7fb5b583a8a847a50c4ed776f8e677e2354932eb82f18898
       Deleted: sha256:82757ec1648a906c504e50e43df74ad5fc333deee043dbfe6559c86908fac15e
       Deleted: sha256:e47ecc039d48409b1c62e5ba874921d7f640243a4c3115bb41b3e1009ecb48e4
       Deleted: sha256:50c212235d3c33a3c035e586ff14359d03895c7bc701bb5dfd62dbe0e91fb486
@@ -965,11 +1025,11 @@ To stop the container, execute the following from your host system:
 
        docker images
 
-Compiling AIXPRT with OpenMP on DLRS
+Compiling AIXPRT for DLRS
-************************************
+*************************
 
-To compile AIXPRT for DLRS, you will have to get the community edition of AIXPRT and update the `compile_AIXPRT_source.sh` file.AIXPRT utilizes
+To compile AIXPRT for DLRS, you will have to get the community edition of AIXPRT and update the `compile_AIXPRT_source.sh` file. AIXPRT utilizes
-build configuration files, so to build AIXPRT on the image, copy, the build files from the base image, this can be done by adding these commands
+build configuration files, so to build AIXPRT in the DLRS image, copy the build files from the base image by adding these commands
 to the end of the stacks-dlrs-mkl dockerfile:
 
    .. code-block:: console
@@ -994,13 +1054,15 @@ The updates to the AIXPRT community edition have been captured in the diff file
 Related topics
 **************
 
-* `DLRS V3.0`_ release announcement
 * `TensorFlow Benchmarks`_
 * `PyTorch benchmarks`_
 * `Kubeflow`_
 * :ref:`kubernetes` tutorial
 * `Jupyter Notebook`_
 
+
+*Intel, OpenVINO, Xeon, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _TensorFlow: https://www.tensorflow.org/
 
 .. _Kubeflow: https://www.kubeflow.org/
@@ -1017,15 +1079,15 @@ Related topics
 
 .. _Getting Started with Kubeflow: https://github.intel.com/verticals/usecases/blob/56717f4642ecd958dc93bbc361c551dfc578d3ed/kubeflow/README.md#getting-started-with-kubeflow
 
-.. _Eigen: https://hub.docker.com/r/clearlinux/stacks-dlrs-oss/
+.. _TensorFlow Eigen: https://hub.docker.com/r/sysstacks/dlrs-tensorflow-clearlinux:v0.6.0-oss
 
-.. _Intel MKL-DNN: https://hub.docker.com/r/clearlinux/stacks-dlrs-mkl/
+.. _TensorFlow oneDNN: https://hub.docker.com/r/sysstacks/dlrs-tensorflow2-clearlinux:v0.6.0
 
-.. _PyTorch with OpenBLAS: https://hub.docker.com/r/clearlinux/stacks-pytorch-oss
+.. _PyTorch with OpenBLAS: https://hub.docker.com/r/sysstacks/dlrs-pytorch-clearlinux:v0.6.0-oss
 
-.. _PyTorch with Intel MKL-DNN: https://hub.docker.com/r/clearlinux/stacks-pytorch-mkl
+.. _PyTorch with Intel oneDNN: https://hub.docker.com/r/sysstacks/dlrs-pytorch-clearlinux:v0.6.0
 
-.. _Intel MKL-DNN-VNNI: https://hub.docker.com/r/clearlinux/stacks-dlrs-mkl-vnni
+.. _Intel oneDNN: https://hub.docker.com/r/sysstacks/dlrs-tensorflow-clearlinux
 
 .. _DLRS V3.0:  https://clearlinux.org/stacks/deep-learning-reference-stack-v3
 
@@ -1033,7 +1095,9 @@ Related topics
 
 .. _DLRS V5.0: https://clearlinux.org/blogs-news/deep-learning-reference-stack-v50-now-available
 
-.. _dlrs-tfjob: https://github.com/clearlinux/dockerfiles/tree/master/stacks/dlrs/kubeflow/dlrs-tfjob
+.. _DLRS V6.0: https://clearlinux.org/blogs-news/deep-learning-reference-stack-v6-now-available
+
+.. _dlrs-tfjob: github.com/intel/stacks
 
 .. _Logging Architecture: https://kubernetes.io/docs/concepts/cluster-administration/logging/
 
@@ -1070,7 +1134,7 @@ Related topics
 .. _Distributed TensorFlow: https://www.tensorflow.org/deploy/distributed
 .. _TFJobs:  https://www.kubeflow.org/docs/components/tftraining/
 
-.. _Intel® quantization tools:  https://github.com/IntelAI/tools/blob/master/tensorflow_quantization/README.md#quantization-tools
+.. _Intel® AI Quantization Tools for TensorFlow:  https://github.com/IntelAI/tools/blob/master/tensorflow_quantization/README.md#quantization-tools
 
 .. _OpenCV open model zoo: https://github.com/opencv/open_model_zoo
 
@@ -1103,3 +1167,7 @@ Related topics
 .. _Kubeflow OpenMPI instructions: https://github.com/kubeflow/mpi-operator/blob/master/README.md
 
 .. _Intel stacks GitHub repository: https://github.com/intel/stacks.git
+
+.. _Transformers: https://github.com/huggingface/transformers
+
+.. _oneDNN: https://github.com/oneapi-src/oneDNN

source/guides/stacks/greengrass.rst

@@ -6,7 +6,7 @@ Enable AWS Greengrass\* and OpenVINO™ toolkit
 This guide explains how to enable AWS Greengrass\* and OpenVINO™ toolkit.
 Specifically, the guide demonstrates how to:
 
-* Set up the Intel® edge device with |CL-ATTR|
+* Set up the Intel edge device with |CL-ATTR|
 * Install the OpenVINO™ toolkit and Amazon Web Services\* (AWS\*)
   Greengrass\* software stacks
 * Use AWS Greengrass\* and AWS Lambda\* to deploy the FaaS samples from
@@ -20,19 +20,19 @@ Overview
 ********
 
 Hardware accelerated Function-as-a-Service (FaaS) enables cloud developers to
-deploy inference functionalities [1] on Intel® IoT edge devices with
+deploy inference functionalities [1] on Intel IoT edge devices with
 accelerators (CPU, Integrated GPU, Intel® FPGA, and Intel® Movidius™
 technology). These functions provide a great developer experience and
 seamless migration of visual analytics from cloud to edge in a secure manner
 using a containerized environment. Hardware-accelerated FaaS provides the
 best-in-class performance by accessing optimized deep learning libraries on
-Intel® IoT edge devices with accelerators.
+Intel IoT edge devices with accelerators.
 
 Supported platforms
 *******************
 
 *	Operating System: |CL| latest release
-*	Hardware:	Intel® core platforms (that support inference on CPU only)
+*	Hardware:	Intel® Core™ processors (that support inference on CPU only)
 
 Sample description
 ==================
@@ -111,7 +111,7 @@ for the OpenVINO software stack:
    Learn more about how to :ref:`swupd-guide`.
 
 The :command:`computer-vision-basic` bundle installs the OpenVINO™ toolkit,
-and the sample models optimized for Intel® edge platforms.
+and the sample models optimized for Intel edge platforms.
 
 .. _convert-dl-models:
 
@@ -130,7 +130,7 @@ download the BVLC AlexNet model files `bvlc_alexnet.caffemodel`_ and
 :file:`/usr/share/openvino/models`. Any custom pre-trained classification models
 can be used with the classification sample.
 
-For object detection, the sample models optimized for Intel® edge platforms
+For object detection, the sample models optimized for Intel edge platforms
 are included with the computer-vision-basic bundle installation at
 :file:`/usr/share/openvino/models`. These models are provided as an example;
 you may also use a custom SSD model with the Greengrass object detection sample.
@@ -181,7 +181,7 @@ In these examples:
 Configure AWS Greengrass group
 ******************************
 
-For each Intel® edge platform, you must create a new AWS Greengrass group
+For each Intel edge platform, you must create a new AWS Greengrass group
 and install AWS Greengrass core software to establish the connection between
 cloud and edge.
 
@@ -278,7 +278,8 @@ configuring the Lambda function for AWS Greengrass.
         - Value
       * - PARAM_MODEL_XML
         - <MODEL_DIR>/<IR.xml>, where <MODEL_DIR> is user specified and
-          contains IR.xml, the Intermediate Representation file from Intel® Model Optimizer.
+          contains IR.xml, the Intermediate Representation file from the
+          OpenVINO™ Model Optimizer.
           For this guide, <MODEL_DIR> should be set to '/usr/share/openvino/models'
           or one of its subdirectories.
       * - PARAM_INPUT_SOURCE
@@ -396,6 +397,10 @@ References
 #. AWS Lambda: https://aws.amazon.com/lambda/
 #. AWS Kinesis: https://aws.amazon.com/kinesis/
 
+
+*Intel, OpenVINO, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
+
 .. _Edge-Analytics-FaaS: https://github.com/intel/Edge-Analytics-FaaS/tree/v1.0/AWS%20Greengrass
 
 .. _bvlc_alexnet.caffemodel: http://dl.caffe.berkeleyvision.org/bvlc_alexnet.caffemodel

source/guides/stacks/hpcrs.rst

@@ -0,0 +1,242 @@
+.. _hpcrs:
+
+High Performance Computing Reference Stack
+##########################################
+
+This guide gives examples for converting Docker* containers, such as those provided by the :ref:`dlrs` into Singularity* containers suited for HPC, and then walking through a multi-node benchmarking example with TensorFlow*.
+
+.. contents::
+   :local:
+   :depth: 1
+
+
+Overview
+********
+
+The High Performance Computing Reference Stack (HPCRS) meets the needs of deploying HPC and AI workloads on the same system. This software solution reduces the complexities associated with integrating software components for High Performance Computing (HPC) Platforms.  `Singularity`_ is an open source container platform to package entire scientific workflows, software and libraries, and even data.
+
+
+Installing Singularity
+**********************
+The installation instructions are for Linux* systems, and have been enabled for installation on |CL-ATTR|.
+
+.. note::
+
+   The steps for installation can also be found on the `Singularity quick-start`_ https://sylabs.io/guides/3.0/user-guide/quick_start.html#quick-installation
+
+#. Install Go*.
+
+   This guide requires version 1.13 of Go, for compatibility with Singularity v3.0.0. Please use these steps to ensure the correct version of Go is installed:
+
+   .. code-block:: bash
+
+      $ export VERSION=1.13 OS=linux ARCH=amd64 && \
+      wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz && \
+      sudo tar -C /usr/local -xzvf go$VERSION.$OS-$ARCH.tar.gz && \
+      rm go$VERSION.$OS-$ARCH.tar.gz
+
+#. Setup the environment for Go.
+
+   .. code-block:: bash
+
+      echo 'export GOPATH=${HOME}/go' >> ~/.bashrc && \
+      echo 'export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin' >> ~/.bashrc && \
+      source ~/.bashrc
+
+#. Install :command:`dep` for dependency resolution with Singularity v3.0.0.
+
+   .. code-block:: bash
+
+      go get -u github.com/golang/dep/cmd/dep
+
+#. Download Singularity.
+
+   .. code-block:: bash
+
+      go get -d github.com/sylabs/singularity
+
+
+   .. note::
+
+      Go will complain that there are no Go files, but it will still download the Singularity source code to the appropriate directory within the $GOPATH.
+
+#. Checkout version 3.0.0 of Singularity.
+
+   .. code-block:: bash
+
+      export VERSION=v3.0.3 # or another tag or branch if you like && \
+      cd $GOPATH/src/github.com/sylabs/singularity && \
+      git fetch && \
+      git checkout $VERSION # omit this command to install the latest bleeding edge code from master
+
+#. Build Singularity.
+
+   Singularity uses a custom build system called makeit. mconfig is called to generate a Makefile and then make is used to compile and install.
+   The devpkg-openssl, devpkg-util-linux  package may be required and can be installed using the :command:`sudo swupd bundle-add <pkg-name>`.
+
+   .. code-block:: bash
+
+      ./mconfig && \
+      make -C ./builddir && \
+      sudo make -C ./builddir install
+
+
+#. Configure bash completion (optional).
+
+   To enjoy bash completion with Singularity commands and options, source the bash completion file. Add this command to your ~/.bashrc file so that bash completion continues to work in new shells
+
+   .. code-block:: bash
+
+      . /usr/local/etc/bash_completion.d/singularity
+
+
+Converting Docker images to Singularity Images
+**********************************************
+
+#. Download d2s.
+
+   :command:`d2s` os an open source tool to convert Docker images to
+   Singularity images. You can use the script in the location where it is 
+   downloaded, or install it using the included :file:`setup.py` file with 
+   the :command:`python setup.py install`
+
+
+   .. code-block:: bash
+
+      git clone https://github.com/intel/stacks.git
+      cd stacks/hpcrs/d2s
+
+#. List local Docker images.
+
+   .. code-block:: bash
+
+      python d2s.py --list_docker_images
+
+   Your output can appear like this:
+
+   .. code-block:: console
+
+      ==============================
+      Docker images present locally
+      ==============================
+      ID         NAME
+      0: clearlinux/stacks-dlrs-mkl
+      1: clearlinux/stacks-dlrs_2-mkl
+      ==============================
+
+#. Convert to Singularity images.
+
+   To convert the Docker images to Singularity images, use the :command:`d2s`
+   script with the ID numbers of the images you wish to convert.
+   We strongly recommend using one of the :file:`clearlinux/stacks-dlrs-mkl` 
+   or :file:`sysstacks/stacks-dlrs-mkl` based images for this guide.  Other 
+   images may be incompatible with expected configuration or filesystem 
+   options.
+
+   .. code-block:: bash
+
+      python d2s.py --convert_docker_images <ID_1> <ID_2>
+
+#. Use the Singularity image.
+
+   To use the container shell to run workloads, launch the image and you
+   will be dropped into the shell. The Singularity image name will be the 
+   same as the name of the Docker image, with slashes converted to 
+   underscores.
+
+   .. code-block:: bash
+
+      singularity shell <singularity image>
+
+   Using the example output above, after conversion you could launch the clearlinux/stacks-dlrs-mkl Singularity image with
+   :command:`singularity shell clearlinux_stacks-dlrs-mkl`
+
+Execute a multi-node benchmark on an HPC cluster
+************************************************
+The following example was executed on an Intel(r) Xeon(r) Processor-based 
+HPC infrastructure. The following steps may need to be adjusted for 
+different environments. See this `Intel Whitepaper`_ for more information.
+
+Running a ResNet50 workload multi-node
+--------------------------------------
+
+#. Download the TensorFlow benchmark.
+
+   .. code-block:: bash
+
+      git clone http://github.com/tensorflow/benchmarks -b cnn_tf_v1.13_compatible
+
+#. Copy the Singularity image and the benchmark files to the HPC cluster 
+   environment.
+
+#. Install OpenMPI* if needed.
+
+   .. note::
+
+      If the HPC host does not have OpenMPI installed, install a custom
+      local version in the user's home directory. This version must be the 
+      same as the version installed in the DLRS container. Follow the steps 
+      for `building OpenMPI`_ from their documentation.
+
+#. Adjust PATH variables.
+
+   Include the OpenMPI install locations in the PATH and LD_LIBRARY_PATH 
+   environment variables.
+
+   .. code-block:: bash
+
+      export PATH="$PATH:<openmpi install path>/bin"
+      export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:<openmpi install path>/lib/"
+
+#. Execute the TF benchmark script on single or multiple nodes using OpenMPI 
+   through the :command:`mpirun` command. Replace variables in {} braces to 
+   reflect your environment.
+
+   .. code-block:: bash
+
+      mpirun --np ${NUM_COPIES}  \
+      -bind-to none \
+      -map-by slot \
+      --display-map \
+      -host ${HOSTNAMES} \
+      --report-bindings \
+      --oversubscribe \
+      -x LD_LIBRARY_PATH \
+      -x PATH \
+      -x HOROVOD_FUSION_THRESHOLD \
+      -x OMP_NUM_THREADS=${OMP_NUM_THREADS} \
+      singularity exec ${PATH_TO_SING_IMAGE} \
+      python ${PATH_TO_TF_BENCH}/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py \
+      --batch_size=128 \
+      --model=resnet50 \
+      --num_intra_threads=${NUM_INTRA_THREADS} \
+      --num_inter_threads=${NUM_INTER_THREADS} \
+      --data_format=NHWC \
+      --device=cpu \
+      --variable_update=horovod \
+      --horovod_device=cpu
+
+   .. note::
+
+      Refer to the `DLRS script`_ for recommended values for setting environment variables in the :command:`mpirun` command.
+
+   .. note::
+
+      You may see an error regarding a missing library while executing the DLRS container.
+      “tensorflow.python.framework.errors_impl.NotFoundError: libnuma.so.1: cannot open shared object file: No such file or directory”
+
+      A workaround for this error is to bind the path to the library from the host.
+
+      .. code-block:: bash
+
+         --bind /usr/lib64/libnuma.so.1:/usr/lib64/libnuma.so.1
+
+.. _Singularity: https://sylabs.io/
+
+.. _Singularity quick-start: https://sylabs.io/guides/3.0/user-guide/quick_start.html#quick-installation
+
+.. _Intel Whitepaper: https://www.intel.com/content/www/us/en/artificial-intelligence/solutions/best-known-methods-for-scaling-deep-learning-with-tensorflow-on-xeon-processor-based-clusters.html
+
+.. _building OpenMPI: https://www.open-mpi.org/faq/?category=building#easy-build
+
+.. _DLRS script: https://github.com/intel/stacks/blob/master/dlrs/clearlinux/tensorflow_2/mkl/scripts/set_env.sh

source/guides/stacks/mers.rst

@@ -4,7 +4,8 @@ Media Reference Stack
 #####################
 
 The Media Reference Stack (MeRS) is a highly optimized software stack for
-Intel® architecture to enable media prioritized workloads, such as transcoding and analytics.
+Intel® Architecture Processors (the CPU) and Intel® Processor Graphics (the
+GPU) to enable media prioritized workloads, such as transcoding and analytics.
 
 This guide explains how to use the pre-built |MERS| container image, build
 your own |MERS| container image, and use the reference stack.
@@ -16,24 +17,42 @@ your own |MERS| container image, and use the reference stack.
 Overview
 ********
 
-Finding the balance between quality and performance, understanding all of the
+Developers face challenges due to the complexity of software integration for
-complex standard-compliant encoders, and optimizing across the
+media tasks that require investing time and engineering effort.
-hardware-software stack for efficiency are all engineering and time
+For example:
-investments for developers.
+
+   * Finding the balance between quality and performance.
+   * Understanding available standard-compliant encoders.
+   * Optimizing across the hardware-software stack for efficiency.
+
+|MERS| abstracts away the complexity of integrating multiple software
+components and specifically tunes them for Intel platforms. |MERS| enables
+media and visual cloud developers to deliver experiences using a simple
+containerized solution.
+
+
+Releases
+********
+
+Refer to the `System Stacks for Linux* OS repository
+<https://github.com/intel/stacks>`_ for information and download links for the
+different versions and offerings of the stack.
+
+* MeRS V0.2.0 release announcement including media processing on GPU and
+  analytics on CPU. 
+
+* MeRS V0.1.0 including media processing and analytics CPU.
+
+* `MeRS Release notes on Github*
+  <https://github.com/intel/stacks/blob/master/mers/NEWS.md>`_ for the
+  latest release of Deep Learning Reference Stack
 
-The Media Reference Stack (MeRS) offers a highly optimized software stack for
-Intel Architecture to enable media prioritized workloads, such as transcoding
-and analytics. |MERS| abstracts away the complexity of integrating multiple
-software components and specifically tunes them for Intel platforms. |MERS|
-allows media and visual cloud developers to deliver experiences using a simple
-containerized solution. 
 
 Prerequisites
 =============
 
-|MERS| can run on any host system that supports Docker\*.
+|MERS| can run on any host system that supports Docker\*. This guide uses
-
+|CL-ATTR| as the host system.
-The steps in this guide use |CL-ATTR| as the host system.
 
 - To install |CL| on a host system, see how to 
   :ref:`install Clear Linux* OS from the live desktop
@@ -44,26 +63,26 @@ The steps in this guide use |CL-ATTR| as the host system.
 
 .. important:: 
 
-   For optimal performance, a processor with Vector Neural Network
+   For optimal media analytics performance, a processor with Vector Neural
-   Instructions (VNNI) should be used. VNNI is an extension of Intel® 
+   Network Instructions (VNNI) should be used. VNNI is an extension of Intel®
-   Advanced Vector Extensions 512 (Intel® AVX-512) and is available starting 
+   Advanced Vector Extensions 512 (Intel® AVX-512) and is available starting
-   with the 2nd generation of Intel® Xeon® Scalable Platform, providing AI 
+   with the 2nd generation of Intel® Xeon® Scalable processors, providing AI
    inference acceleration.
 
-Stack Features
+Stack features
 ==============
 
 The |MERS| provides a `pre-built Docker image available on DockerHub
-<https://hub.docker.com/r/clearlinux/stacks-mers>`_, which includes
+<https://hub.docker.com/r/sysstacks/mers-clearlinux>`_, which includes
-instructions on build the image from source. |MERS| is open-sourced to ensure
+instructions on building the image from source. |MERS| is open-sourced to
-developers have easy access to the source code and are able to customize it.
+make sure developers have easy access to the source code and are able to
-|MERS| is built using the *clearlinux:latest* Docker image and aims to support
+customize it. |MERS| is built using the latest *clearlinux/os-core* Docker
-the latest |CL| version.
+image and aims to support the latest |CL| version.
 
-|MERS| provides the following libraries:
+|MERS| provides the following libraries and drivers:
 
 .. list-table::
-   :widths: auto
+   :widths: 15 85
 
    * - SVT-HEVC
      - Scalable Video Technology for HEVC encoding, also known as H.265
@@ -71,8 +90,21 @@ the latest |CL| version.
      - Scalable Video Technology for AV1 encoding
    * - x264
      - x264 for H.264/MPEG-4 AVC encoding
-   * - MKL-DNN
+   * - dav1d
-     - `Intel® Math Kernel Library for Deep Neural Networks <https://01.org/mkl-dnn>`_
+     - `dav1d <https://code.videolan.org/videolan/dav1d>`_ for AV1 decoding
+   * - libVA
+     - `VAAPI (Video Acceleration API) open-source library (LibVA),
+       <https://github.com/intel/libva>`_ which provides access to graphics
+       hardware acceleration capabilities.
+   * - media-driver
+     - `Intel® Media Driver for VAAPI <https://github.com/intel/media-driver/>`_
+       for supporting hardware acceleration on Intel® Gen graphics hardware
+       platforms.
+   * - gmmlib
+     - `Intel® Graphics Memory Management Library
+       <https://github.com/intel/gmmlib>`_ provides device specific and buffer
+       management for the Intel® Graphics Compute Runtime for oneAPI Level Zero 
+       and OpenCL™ Driver and the Intel Media Driver for VAAPI.
 
 Components of the |MERS| include:
 
@@ -81,19 +113,31 @@ Components of the |MERS| include:
 * `OpenVINO™ toolkit
   <https://01.org/openvinotoolkit>`_ for inference.
 
-* `FFmpeg* <https://www.ffmpeg.org>`_ with `Scalable Video Technology (SVT)
+* `FFmpeg* <https://www.ffmpeg.org>`_ with plugins for:
-  <https://01.org/svt>`_ plugins for encoding, decoding, and transcoding.
 
-* `GStreamer* <https://gstreamer.freedesktop.org/>`_  with `Scalable Video
+   - `Scalable Video Technology (SVT)
-  Technology (SVT) <https://01.org/svt>`_ and `OpenVINO™ toolkit
+     <https://01.org/svt>`_
-  <https://01.org/openvinotoolkit>`_ plugins for analytics.
 
+* `GStreamer* <https://gstreamer.freedesktop.org/>`_  with plugins for:
+
+   - `Scalable Video
+     Technology (SVT) <https://01.org/svt>`_
+   - `OpenVINO™ toolkit
+     <https://01.org/openvinotoolkit>`_
+   - `VAAPI <https://github.com/GStreamer/gstreamer-vaapi>`_
+
+* `Intel® Media SDK <https://github.com/Intel-Media-SDK/MediaSDK>`_ 
 
 .. note::
 
-   The pre-built |MERS| container image configures :command:`FFmpeg` without
+   The |MERS| is validated on 11th generation Intel Processor Graphics and
-   certain elements (specific encoder, decoder, muxer, etc.) that you may
+   newer. Older generations should work but are not tested against.
-   require. If you require changes to :command:`FFmpeg` we suggest starting at
+
+.. note::
+
+   The pre-built |MERS| container image configures FFmpeg without certain
+   elements (specific encoder, decoder, muxer, etc.) that you may require. If
+   you require changes to FFmpeg we suggest starting at
    :ref:`building-the-mers-container-image`.
 
 .. note::
@@ -104,21 +148,21 @@ Components of the |MERS| include:
    licensing and usage of the Media Reference Stack.
 
 
-Getting the pre-built |MERS| container image
+Get the pre-built |MERS| container image
-********************************************
+****************************************
 
-Pre-built |MERS| Docker images are available on DockerHub at
+Pre-built |MERS| Docker images are available on DockerHub* at
-https://hub.docker.com/r/clearlinux/stacks-mers
+https://hub.docker.com/r/sysstacks/mers-clearlinux
 
 
 To use the |MERS|:
 
 #. Pull the image directly from `Docker Hub
-   <https://hub.docker.com/r/clearlinux/stacks-mers>`_. 
+   <https://hub.docker.com/r/sysstacks/mers-clearlinux>`_. 
 
    .. code-block:: bash
 
-      docker pull clearlinux/stacks-mers
+      docker pull sysstacks/mers-clearlinux
 
    .. note ::
 
@@ -126,22 +170,22 @@ To use the |MERS|:
       download depending on your Internet connection.
 
       If you are on a network with outbound proxies, be sure to configure
-      Docker allow access. See the `Docker service proxy
+      Docker to allow access. See the `Docker service proxy
       <https://docs.docker.com/config/daemon/systemd/#httphttps-proxy>`_ and
       `Docker client proxy
       <https://docs.docker.com/network/proxy/#configure-the-docker-client>`_
       documentation for more details.
       
-#. Once you have downloaded the image, run it with:
+#. Once you have downloaded the image, run it using the following command:
 
    .. code-block:: bash
 
-      docker run -it clearlinux/stacks-mers
+      docker run -it sysstacks/mers-clearlinux
 
    This will launch the image and drop you into a bash shell inside the
-   container. :command:`GStreamer` and :command:`FFmpeg` programs are
+   container. GStreamer and FFmpeg programs are installed in the container
-   installed in the container image and accessible in the default $PATH. These
+   image and accessible in the default $PATH. Use these programs as you would
-   programs can be used as you would normally outside of |MERS|.
+   outside of |MERS|.
 
    Paths to media files and video devices, such as cameras, can be shared from
    the host to the container with the :command:`--volume` switch `using Docker
@@ -149,13 +193,13 @@ To use the |MERS|:
 
 .. _building-the-mers-container-image:
 
-Building the |MERS| container image from source
+Build the |MERS| container image from source
-***********************************************
+********************************************
 
 If you choose to build your own MeRS container image, you can optionally add
 customizations as needed. The :file:`Dockerfile` for the MeRS is available on
-`GitHub <https://github.com/intel/stacks/tree/master/mers>`_ and
+`GitHub <https://github.com/intel/stacks/tree/master/mers>`_ and can be used
-can be used for reference.
+as a reference when creating your own container image.
 
 #. The |MERS| image is part of the dockerfiles repository inside the |CL|
    organization on GitHub. Clone the :file:`stacks` repository.
@@ -171,28 +215,28 @@ can be used for reference.
 
       cd ./stacks/mers/clearlinux
        
-#. Use the :command:`docker build` command with the :file:`Dockerfile` to the
+#. Use the :command:`docker build` command with the :file:`Dockerfile` to
-   MeRS container image.
+   build the MeRS container image.
 
    .. code-block:: bash
 
-      docker build --no-cache -t clearlinux/stacks-mers .
+      docker build --no-cache -t sysstacks/mers-clearlinux .
 
-Using the |MERS| container image
+Use the |MERS| container image
-********************************
+******************************
 
-Below are some examples of how the |MERS| container image can be used to
+This section shows examples of how the |MERS| container image can be used to
 process media files.
 
 The models and video source can be substituted from your use-case. Some
-publicly licensed sample videos are available at `sample-videos repsoitory
+publicly licensed sample videos are available at `sample-videos repository
 <https://github.com/intel-iot-devkit/sample-videos>`_ for testing.
 
 
-Example 1: Transcoding
+Media Transcoding
-======================
+=================
 
-This example shows how to perform transcoding with :command:`FFmpeg`.
+The examples below show transcoding using the GPU or CPU for processing.
 
 #. On the host system, setup a workspace for data and models:
 
@@ -208,45 +252,93 @@ This example shows how to perform transcoding with :command:`FFmpeg`.
 
       cp </path/to/video> ~/ffmpeg/input
 
-#. Run the *clearlinux/stack-mers* docker image, allowing shared access to the
+#. Run the *sysstacks/mers-clearlinux* Docker image, allowing shared access to
-   workspace on the host:
+   the workspace on the host:
-
 
    .. code:: bash
 
       docker run -it \
-      -v ~/ffmpeg:/home/mers-user:ro \
+      --volume ~/ffmpeg:/home/mers-user:ro \
-      clearlinux/stacks-mers:latest
+      --device=/dev/dri \
+      --env QSV_DEVICE=/dev/dri/renderD128 \
+      sysstacks/mers-clearlinux:latest
+
+   .. note::
+
+      The :command:`--device` parameter and the **GSV_DEVICE** environment
+      variable allow shared access to the GPU on the host system. The values
+      needed may be different depending on host's graphics configuration.      
 
    After running the :command:`docker run` command, you enter a bash shell
    inside the container. 
 
-#. From the container shell, you can run :command:`FFmpeg` against the videos
+#. From the container shell, you can run FFmpeg and
-   in :file:`/home/mers-user/input` as you would normally outside of |MERS|.
+   GStreamer commands against the videos in :file:`/home/mers-user/input` as
+   you would normally outside of |MERS|.
 
-   For example, to transcode raw yuv420 content to SVT-HEVC and mp4:
+   Some sample commands are provided for reference. 
 
-   .. code:: bash
+   For more information on using the *FFmpeg* commands, refer to the `FFmpeg
-
-      ffmpeg -f rawvideo -vcodec rawvideo -s 320x240 -r 30 -pix_fmt yuv420p -i </home/mers-user/input/test.yuv> -c:v libsvt_hevc -y </home/mers-user/output/test.mp4>
-      
-   Some more generic examples of :command:`FFmpeg` commands can be found in
-   the `OpenVisualCloud repository
-   <https://github.com/OpenVisualCloud/Dockerfiles/blob/master/doc/ffmpeg.md>`_ and used for reference with |MERS|.
-
-   For more information on using :command:`FFmpeg`, refer to the `FFmpeg
    documentation <https://ffmpeg.org/documentation.html>`_.
 
-Example 2: Analytics
+   For more information on using the *GStreamer* commands, refer to the
-====================
+   `GStreamer documentation
+   <https://gstreamer.freedesktop.org/documentation>`_.
 
-This example shows how to perform analytics and inferences with
+
-:command:`GStreamer`.
+Example: Transcoding using GPU
+-------------------------------
+
+The examples below show transcoding using the GPU for processing.
+
+
+Using a FFmpeg to transcode raw content to SVT-HEVC and mp4:
+
+.. code:: bash
+
+   ffmpeg -y -vaapi_device /dev/dri/renderD128 -f rawvideo -video_size 320x240 -r 30 -i </home/mers-user/input/test.yuv> -vf 'format=nv12, hwupload' -c:v h264_vaapi -y </home/mers-user/output/test.mp4>
+
+Using a GStreamer to transcode H264 to H265:
+
+.. code:: bash
+
+   gst-launch-1.0 filesrc location=</home/mers-user/input/test.264> ! h264parse ! vaapih264dec ! vaapih265enc rate-control=cbr bitrate=5000 ! video/x-h265,profile=main ! h265parse ! filesink location=</home/mers-user/output/test.265>
+
+|MERS| builds FFmpeg with `HWAccel
+<https://trac.ffmpeg.org/wiki/HWAccelIntro>`_ enabled which supports VAAPI.
+Refer to the `FFmpeg wiki on VAAPI
+<https://trac.ffmpeg.org/wiki/Hardware/VAAPI>`_ and `GStreamer with Media-SDK
+wiki
+<https://github.com/Intel-Media-SDK/MediaSDK/wiki/Build-and-use-GStreamer-with-MediaSDK#usage-examples>`_
+for more usage examples and compatibility information.
+
+
+Example: Transcoding using CPU
+------------------------------
+
+The example below shows transcoding of raw yuv420 content to SVT-HEVC and mp4,
+using the CPU for processing.
+
+.. code:: bash
+
+   ffmpeg -f rawvideo -vcodec rawvideo -s 320x240 -r 30 -pix_fmt yuv420p -i </home/mers-user/input/test.yuv> -c:v libsvt_hevc -y </home/mers-user/output/test.mp4>
+
+Additional generic examples of FFmpeg commands can be found in the
+`OpenVisualCloud repository
+<https://github.com/OpenVisualCloud/Dockerfiles/blob/master/doc/ffmpeg.md>`_
+and used for reference with |MERS|.
+
+
+Media Analytics
+===============
+
+This example shows how to perform analytics and inferences with GStreamer
+using the CPU for processing.
 
 The steps here are referenced from the `gst-video-analytics Getting Started
 Guide <https://github.com/opencv/gst-video-analytics/wiki>`_ except simply
 substituting the *gst-video-analytics* docker image for the
-*clearlinux/stacks-mers* image.
+*sysstacks/mers-clearlinux* image.
 
 The example below shows how to use the |MERS| container image to perform video
 with object detection and attributes recognition of a video using GStreamer
@@ -318,8 +410,8 @@ using pre-trained models and sample video files.
       export INTEL_MODELS_PATH=~/gva/data/models/intel
       export VIDEO_EXAMPLES_PATH=~/gva/data/video
 
-#. Run the *clearlinux/stack-mers* docker image, allowing shared access to 
+#. Run the *sysstacks/mers-clearlinux* docker image, allowing shared access 
-   the X server and workspace on the host:
+   to the X server and workspace on the host:
 
    .. code:: bash
 
@@ -337,7 +429,7 @@ using pre-trained models and sample video files.
       -v $VIDEO_EXAMPLES_PATH:/home/mers-user/video-examples \
       -e MODELS_PATH=/home/mers-user/intel_models:/home/mers-user/models \      
       -e VIDEO_EXAMPLES_DIR=/home/mers-user/video-examples \
-      clearlinux/stacks-mers:latest
+      sysstacks/mers-clearlinux:latest
 
    .. note:: 
 
@@ -347,18 +439,19 @@ using pre-trained models and sample video files.
         *runc* for this container. It is needed for correct interaction with X
         server.
 
-      - :command:`--net=host` provides host network access to container. It is
+      - :command:`--net=host` provides host network access to the container.
-        needed for correct interaction with X server.
+        It is needed for correct interaction with X server.
       
       - Files :file:`~/.Xauthority` and :file:`/tmp/.X11-unix` mapped to the
         container are needed to ensure smooth authentication with X server.
       
       - :command:`-v` instances are needed to map host system directories
-        inside Docker container.
+        inside the Docker container.
       
-      - :command:`-e` instances set Docker container environment variables.
+      - :command:`-e` instances set the Docker container environment
-        Samples need them some of them set correctly to operate. Proxy variables
+        variables. Some examples need these variables set correctly in order
-        are needed if host is behind firewall.
+        to operate correctly. Proxy variables are needed if host is behind a
+        firewall.
       
 
    After running the :command:`docker run` command, it will drop you into a
@@ -417,4 +510,55 @@ using pre-trained models and sample video files.
        ./gst-video-analytics/samples/shell/console_measure_fps_cpu.sh $VIDEO_EXAMPLES_DIR/bolt-detection.mp4
 
 
-**OpenVINO is a trademark of Intel Corporation or its subsidiaries.**
+Add AOM support
+***************
+
+The current version of |MERS| does not include the `Alliance for Open Media
+<https://aomedia.org/>`_ Video Codec (AOM). AOM can be built from source on an
+individual basis.
+
+To add AOM support to the |MERS| image:
+
+
+#. The following programs are needed to add AOM support to |MERS|: **docker,
+   git, patch**. On |CL| these can be  installed with the commands below. For
+   other operating systems, install the appropriate packages. 
+
+   .. code:: bash
+
+      sudo swupd bundle-add containers-basic dev-utils
+
+
+#. Clone the Intel Stacks repository from GitHub.
+
+   .. code:: bash
+
+      git clone https://github.com/intel/stacks.git 
+
+#. Navigate to the directory for the |MERS| image.
+
+   .. code:: bash 
+
+      cd stacks/mers/clearlinux/
+
+#. Apply the patch to the :file:`Dockerfile`.
+
+   .. code:: bash
+
+      patch -p1 < aom-patches/stacks-mers-v2-include-aom.diff
+
+#. Use the :command:`docker build` command to build a local copy of the
+   MeRS container image tagged as *aom*.
+
+   .. code-block:: bash
+
+      docker build --no-cache -t sysstacks/mers-clearlinux:aom .
+
+Once the build has completed successfully, the local image can be used
+following the same steps in this tutorial by substituting the image name with
+*sysstacks/mers-clearlinux:aom*.
+
+
+*Intel, Xeon, OpenVINO, and the Intel logo are trademarks of Intel
+Corporation or its subsidiaries. OpenCL and the OpenCL logo are trademarks of
+Apple Inc. used by permission by Khronos.*

source/index.rst

@@ -57,7 +57,7 @@
 
       | `Ask the Clear Linux experts <https://clearlinux.org/community/mailing-list>`_
       | `Clear Linux Forum <https://community.clearlinux.org/>`_
-      | `Freenode IRC: #clearlinux <https://webchat.freenode.net/#clearlinux>`_
+      | `irc.libera.chat IRC: #clearlinux <https://kiwiirc.com/client/irc.libera.chat/?nick=web_guest|?#clearlinux>`_
 
 
 .. container:: video
@@ -75,4 +75,4 @@
    tutorials/index
    reference/index
    FAQ/index
-   collaboration/collaboration
+   collaboration/collaboration

source/reference/compatible-hardware.rst

@@ -11,103 +11,104 @@ grow.
     :widths: 20, 20
     :header-rows: 1
 
-    * - Processor SKU
+    * - Processor
       - Platform
 
-    * - Intel® Core™ i5-6260U
+    * - Intel® Core™ i5-6260U processor
       -
 
-    * - Intel® Core™ i5-6560U
+    * - Intel® Core™ i5-6560U processor
       - Dell XPS\* 13 9350
 
-    * - Intel® Celeron® J3455
+    * - Intel® Celeron® J3455 processor
       - NUC6CAYS
 
-    * - Intel® Core™ i5-4250U
+    * - Intel® Core™ i5-4250U processor
       -
 
-    * - Intel® Core™ i7-5557U
+    * - Intel® Core™ i7-5557U processor
       -
 
-    * - Intel® Core™ i9-7900X
+    * - Intel® Core™ i9-7900X X-series processor
       - Gigabyte\* X299
 
-    * - Intel® Core™ i3-4130
+    * - Intel® Core™ i3-4130 processor
       - Lenovo Thinkserver\* TS140
 
-    * - Intel® Core™ i7-7567U
+    * - Intel® Core™ i7-7567U processor
       - NUC7i7BNH
 
-    * - Intel® Core™ i7-8809G
+    * - Intel® Core™ i7-8809G processor
       - NUC8i7HVK
 
-    * - Intel® Core™ i5-7260U
+    * - Intel® Core™ i5-7260U processor
       - NUC7i5BNH
 
-    * - Intel® Core™ i7-8650U
+    * - Intel® Core™ i7-8650U processor
       - NUC7i7DNKE
 
-    * - Intel® Core™ i5-7300U
+    * - Intel® Core™ i5-7300U processor
       - NUC7i5DNHE
 
-    * - Intel® Xeon® Gold 6138
+    * - Intel® Xeon® Gold 6138 processor
       -
 
-    * - Intel® Xeon® E5-2699A v4
+    * - Intel® Xeon® E5-2699A v4 processor
       - Dell PowerEdge\* R630
 
-    * - Intel® Xeon® E5-2620 v3
+    * - Intel® Xeon® E5-2620 v3 processor
       -
 
-    * - Intel® Core™ i5-6600
+    * - Intel® Core™ i5-6600 processor
       - Gigabyte\* Z170X-UD5
 
-    * - Intel® Core™ i5-4250U
+    * - Intel® Core™ i5-4250U processor
       - D54250WYK
 
-    * - Intel® Xeon® E5-2699 v3
+    * - Intel® Xeon® E5-2699 v3 processor
       - S2600WT2
 
-    * - Intel® Atom™ J3455
+    * - Intel Atom™ J3455 processor
       - NUC6CAYB
 
-    * - Intel® Xeon® Bronze 3104
+    * - Intel® Xeon® Bronze 3104 processor
       - 0W23H8
 
-    * - Intel® Atom™ C2750
+    * - Intel Atom™ C2750 processor
       - SuperMicro\* A1SAi
 
-    * - Intel® Atom™ E3825
+    * - Intel Atom™ E3825 processor
       - CircuitCo MinnowBoard MAX\*
 
-    * - Intel® Core™ i7-8700
+    * - Intel® Core™ i7-8700 processor
       - Gigabyte\* H370 WIFI
 
-    * - Intel® Core™ i7-3667U
+    * - Intel® Core™ i7-3667U processor
       - Lenovo ThinkPad\* X1 Carbon laptop
 
-    * - Intel® Core™ i5-4210U
+    * - Intel® Core™ i5-4210U processor
       - Dell XPS\* 13 laptop
 
-    * - Intel® Celeron® J3455
+    * - Intel® Celeron® J3455 processor
       - NUC6CAYB
 
-    * - Intel® Core™ i7-4790
+    * - Intel® Core™ i7-4790 processor
       - Gigabyte\* desktop
 
-    * - Intel® Core™ i5-6260U
+    * - Intel® Core™ i5-6260U processor
       - NUC6I6SYH
 
-    * - Intel® Core™ i7-5557U
+    * - Intel® Core™ i7-5557U processor
       - NUC5I7RYH
 
-    * - Intel® Core™ i7-4700MQ
+    * - Intel® Core™ i7-4700MQ processor
       - Lenovo ThinkPad\* T540p
 
-    * - Intel® Core™ i7-5557U
+    * - Intel® Core™ i7-5557U processor
       - NUC5I7RYB
 
-    * - Intel® Core™ i5-6260U
+    * - Intel® Core™ i5-6260U processor
       - NUC6I5SYH
 
-\* Other names and brands may be claimed as the property of others.
+*Intel, Celeron, Xeon, Intel Atom, and Intel Core are trademarks of Intel
+Corporation or its subsidiaries.*

source/reference/image-types.rst

@@ -51,17 +51,6 @@ Table 2 lists the currently available images that are platform specific.
    * - aws.img
      - Image suitable for use with Amazon\* AWS\*.
 
-   * - azure.vhd
-     - Virtual Hard Disk for use on Microsoft\* Azure\* cloud platform.
-
-   * - azure-docker.vhd
-     - Virtual Hard Disk for use on Microsoft Azure cloud platform with
-       Docker\* pre-installed.
-
-   * - azure-machine-learning.vhd
-     - Virtual Hard Disk for use on Microsoft Azure cloud platform with the
-       `machine-learning-basic` bundle installed.
-
    * - cloudguest.img
      - Image with generic cloud guest virtual machine (VM) requirements
        installed.
@@ -69,8 +58,8 @@ Table 2 lists the currently available images that are platform specific.
    * - gce.tar
      - Image with the Google Compute Engine (GCE) specific kernel.
 
-   * - hyperv.vhdx
+   * - azure-hyperv.vhd
-     - Virtual Hard Disk for use with Microsoft Hyper-V\* hypervisor. Includes
+     - Image for Microsoft* Azure and Hyper-V generation 1 VMs. Includes
        :ref:`optimized kernel <vm-kernels>` for Hyper-V.
 
    * - kvm.img

source/reference/index.rst

@@ -13,3 +13,5 @@ features.
    bundles/bundles
    system-requirements
    image-types
+   man-pages
+   tutorial-ratings

source/reference/system-requirements.rst

@@ -4,11 +4,19 @@ Recommended minimum system requirements
 #######################################
 
 |CL-ATTR| can run on most modern hardware and is capable of running with
-modest hardware resources. Out of the box, |CL| can run on a single CPU core, 1 GB RAM, and minimum of disk space of: 
+modest hardware resources. Out of the box, |CL| can run on a single CPU core,
+1 GB RAM, and minimum of disk space of:
 
 * 4 GB for the *live server*
 * 20 GB for the *live desktop*
 
+.. caution::
+
+   Advanced users who wish to install on a disk using less than the recommended
+   space requirements may use the flag ``--skip-validation-size``. Use of this
+   flag may cause the installation to fail due to inadequate disk space.
+
+
 For use cases requiring minimal resources, |CL| :ref:`about <about>` can
 be used to create a highly customized installation that can even run on a
 system with a 128MB of memory and 600MB of disk space, for example.
@@ -31,7 +39,9 @@ System requirements
 *******************
 
 |CL| requires an x86 64-bit processor which supports Intel® Streaming SIMD
-Extensions 4.2 (Intel® SSE 4.2), and it requires a system that supports UEFI.
+Extensions 4.2 (Intel® SSE 4.2). 
+
+For information on the boot loader, see the `clr-boot-manager readme`_ .
 
 The |CL| installer performs a system compatibility check upon booting. To
 manually verify system compatibility with |CL|, run the :ref:`compatibility
@@ -44,19 +54,19 @@ these features:
 
 * Instruction Set Extensions:
 
-  - Supplemental Streaming SIMD Extension 3 (Intel® SSSE3)
+  - Supplemental Streaming SIMD Extension 3 (SSSE3)
   - Intel® Streaming SIMD Extensions 4.1 (Intel® SSE 4.1)
   - Intel® Streaming SIMD Extensions 4.2 (Intel® SSE 4.2)
   - Carry-less Multiplication (PCLMUL)
 
 The following processor families have been verified to run |CL|:
 
-* Intel® Core™ Processor Family (2nd generation or greater)
+* Intel® Core™ processor family (2nd generation or greater)
 * Intel® Xeon® E3-xxxx processor
 * Intel® Xeon® E5-xxxx processor
 * Intel® Xeon® E7-xxxx processor
-* Intel® Atom® processor C Series
+* Intel Atom® processor C Series
-* Intel® Atom® processor E Series
+* Intel Atom® processor E Series
 
 
 Recommended configurations
@@ -78,3 +88,8 @@ Graphics     Device with openGL support (e.g. Intel HD/UHD Graphics)
 Network      Active Internet connection
 =========    ===============================
 
+
+*Intel, Intel Core, Xeon, Intel Atom, and the Intel logo are trademarks of
+Intel Corporation or its subsidiaries.*
+
+.. _clr-boot-manager readme: https://github.com/clearlinux/clr-boot-manager

source/reference/tutorial-ratings.rst

@@ -0,0 +1,52 @@
+.. _tutorial-ratings:
+
+Tutorial difficulty ratings
+###########################
+
+Tutorial difficulty ratings provide a simple way to start using and developing with |CL-ATTR|. If you're new to the distro, we suggest starting with ``Easy`` tutorials and  working towards the more ``Difficult``. Ratings not only expose learning paths but also provide a starting point from which to advance or improve use cases, so be sure to :ref:`share your insights <collaboration>`. Three main metrics help us to determine how to rate a tutorial: 
+
+.. contents::
+   :local:
+   :depth: 1
+
+The sum total of these metrics, the rating shown in Figure 1, represents the
+ability to successfully complete a tutorial based on skill level, balanced against the risk of failure.   
+
+.. figure:: /_figures/reference/tutorial-ratings-01.svg
+   :scale: 100%
+   :alt: Tutorial difficulty ratings
+
+   Figure 1: Tutorial difficulty ratings
+
+Time and complexity
+*******************
+
+Are there about 8 or more *complex* steps? *Complex steps* are those that: 
+
+* Require more than one action 
+* Require external reading/review
+* Include explanation or context
+* Give alternative(s) 
+
+This metric factors in the cognitive load and its impact on a user. 
+
+User experience level
+*********************
+
+Our tutorials primarily target two types of Linux users.
+
+**Experienced** A Linux\* user who is familiar with common topics like userspace, networking, sudo privileges, services, and more. 
+
+**Advanced** A Linux user who is beyond Experienced and is familiar with most sysadmin and programming topics. 
+
+This metric establishes a starting point for skills, based on user
+experience. 
+
+Impact of failure
+******************
+
+The impact of failure calculates the risk of failing to complete a tutorial as a result of entering incorrect data or configuration, or failing to follow the steps in the given order. We estimate the potential state of a system, given these failure scenarios and their severity. This metric also factors in the ability to troubleshoot and recover when faced with errors. Therefore, the final impact incorporates the previous two metrics while it helps to predict an appropriate difficulty rating. 
+
+* Will impact of errors be inconsequential? ``Easy``
+* Will impact of errors cause inconvenience (but system still works)? ``Moderate``
+* Will impact of errors cause system failure (difficult to recover)? ``Difficult``

source/tutorials/apache-spark.rst

@@ -1,142 +0,0 @@
-  .. _spark:
-
-Apache\* Spark\*
-################
-
-This tutorial describes how to install, configure, and run Apache Spark on
-|CL-ATTR| on a single machine running the master daemon and a worker daemon.
-
-.. contents::
-   :local:
-   :depth: 1
-
-Description
-***********
-
-Apache Spark is a fast, general-purpose cluster computing system with
-the following features:
-
-*  Provides high-level APIs in Java\*, Scala\*, Python\*, and R\*.
-*  Includes an optimized engine that supports general execution graphs.
-*  Supports high-level tools including Spark SQL, MLlib, GraphX, and Spark
-   Streaming.
-
-
-Prerequisites
-*************
-
-* |CL| installed on your host system.
-
-  For detailed instructions on installing |CL| on a bare metal system, visit
-  the :ref:`bare metal installation guide <bare-metal-install-desktop>`.
-
-* Before installing any new packages, update |CL| with the following command:
-
-  .. code-block:: bash
-
-     sudo swupd update
-
-Install Apache Spark
-********************
-
-Apache Spark is included in the :command:`big-data-basic` bundle. To install the
-framework, run the following command:
-
-.. code-block:: bash
-
-   sudo swupd bundle-add big-data-basic
-
-Configure Apache Spark
-**********************
-
-#. Create the configuration directory:
-
-   .. code-block:: bash
-
-      sudo mkdir /etc/spark
-
-#. Copy the default templates from :file:`/usr/share/defaults/spark` to
-   :file:`/etc/spark`:
-
-   .. code-block:: bash
-
-      sudo cp /usr/share/defaults/spark/* /etc/spark
-
-   .. note:: Since |CL| is a stateless system, you should never modify the
-      files under the :file:`/usr/share/defaults` directory. The software
-      updater overwrites those files.
-
-#. Copy the template files shown below to create custom configuration files:
-
-   .. code-block:: bash
-
-      sudo cp /etc/spark/spark-defaults.conf.template /etc/spark/spark-defaults.conf
-      sudo cp /etc/spark/spark-env.sh.template /etc/spark/spark-env.sh
-      sudo cp /etc/spark/log4j.properties.template /etc/spark/log4j.properties
-
-#. Edit the :file:`/etc/spark/spark-env.sh` file and add the
-   :envvar:`SPARK_MASTER_HOST` variable. Replace the example address below
-   with your localhost IP address. View your IP address using the
-   :command:`hostname -I` command.
-
-   .. code-block:: bash
-
-      SPARK_MASTER_HOST="10.300.200.100"
-
-   .. note:: This optional step enables the master's web user interface to
-      view information needed later in this tutorial.
-
-#. Edit the :file:`/etc/spark/spark-defaults.conf` file and update the
-   :envvar:`spark.master` variable with the `SPARK_MASTER_HOST` address and port
-   `7077`.
-
-   .. code-block:: bash
-
-      spark.master    spark://10.300.200.100:7077
-
-Start the master server and a worker daemon
-*******************************************
-
-#. Start the master server:
-
-   .. code-block:: bash
-
-      sudo /usr/share/apache-spark/sbin/./start-master.sh
-
-#. Start one worker daemon and connect it to the master using the
-   :envvar:`spark.master` variable defined earlier:
-
-   .. code-block:: bash
-
-      sudo /usr/share/apache-spark/sbin/./start-slave.sh spark://10.300.200.100:7077
-
-#. Open an internet browser and view the worker daemon information using
-   the master's IP address and port `8080`:
-
-   .. code-block:: bash
-
-      http://10.300.200.100:8080
-
-Run the Spark wordcount example
-*******************************
-
-#. Run the wordcount example using a file on your local host and output the
-   results to a new file with the following command:
-
-   .. code-block:: bash
-
-      sudo spark-submit /usr/share/apache-spark/examples/src/main/python/wordcount.py ~/Documents/example_file > ~/Documents/results
-
-#. Open an internet browser and view the application information using
-   the master's IP address and port `8080`:
-
-   .. code-block:: bash
-
-      http://10.300.200.100:8080
-
-#. View the results of the wordcount application in the :file:`~/Documents/results` file.
-
-**Congratulations!**
-
-You have successfully installed and set up a standalone Apache Spark cluster,
-and ran a simple wordcount example.

source/tutorials/broadcom.rst

@@ -4,7 +4,7 @@ Broadcom\* Drivers
 ##################
 
 Broadcom manufactures wireless network interfaces, including devices that
-support WiFi and Bluetooth. 
+support WiFi and Bluetooth® technology.
 
 Broadcom wireless devices on Linux\* have a lot of different combinations of
 possible required software depending on the exact model of your device. These
@@ -96,7 +96,7 @@ and has to be built as an out-of-tree kernel module.
    It is recommended to use the :ref:`LTS kernel <compatible-kernels>` if you
    have to use this driver.
 
-#. See if your device is supported and download the **Linux® STA 64-bit
+#. See if your device is supported and download the **Linux\* STA 64-bit
    driver** from
    `Broadcom's download website
    <https://www.broadcom.com/support/download-search?pg=&pf=Wireless+LAN+Infrastructure>`_
@@ -236,3 +236,4 @@ Troubleshooting
   different path than expected. Check the output of :command:`sudo dmesg |
   grep -i firmware` for firmware loading issues.
 
+*The Bluetooth® word mark and logos are registered trademarks owned by Bluetooth SIG, Inc. and any use of such marks by Intel Corporation is under license.*

source/tutorials/docker.rst

@@ -143,8 +143,8 @@ typically located at :file:`/etc/docker/daemon.json`. |CL| features a
 
       sudo systemctl restart docker
 
-Pulling and running an image from Docker Hub
+Pulling and running an image from Docker Hub\*
-********************************************
+**********************************************
 
 `Docker Hub`_ is a publicly available container image repository which
 comes pre-configured with Docker. In the example below we will pull and run

source/tutorials/flatpak.rst

@@ -3,7 +3,7 @@
 Flatpak\*
 #########
 
-This tutorial shows how to install a `Flatpak`_ app on |CL| using Gnome\* Software
+This tutorial shows how to install a `Flatpak`_ app on |CL| using GNOME\* Software
 and the command line.
 
 .. contents::
@@ -37,21 +37,21 @@ Prerequisites
 
      sudo swupd bundle-add desktop-autostart
 
-Install a Flatpak app with Gnome Software
+Install a Flatpak app with GNOME Software
 *****************************************
 
-|CL| desktop comes with Gnome Software installed. Flatpak apps can be
+|CL| desktop comes with GNOME Software installed. Flatpak apps can be
-installed from within Gnome Software.
+installed from within GNOME Software.
 
-#. Launch Gnome Software from your desktop.
+#. Launch GNOME Software from your desktop.
 
 #. Search for the Flatpak app that you want to install, as shown in Figure 1.
 
    .. figure:: /_figures/flatpak/flatpak-01.png
       :scale: 50%
-      :alt: Searching for Filezilla app in Gnome Software
+      :alt: Searching for Filezilla app in GNOME Software
 
-      Figure 1: Searching for Filezilla app in Gnome Software
+      Figure 1: Searching for Filezilla\* app in GNOME Software
 
 #. When you find the app you want to install, click it to view application
    details.
@@ -61,12 +61,12 @@ installed from within Gnome Software.
 
    .. figure:: /_figures/flatpak/flatpak-02.png
       :scale: 50%
-      :alt: Filezilla Flatpak detail page in Gnome Software
+      :alt: Filezilla Flatpak detail page in GNOME Software
 
-      Figure 2: Filezilla Flatpak detail page in Gnome Software
+      Figure 2: Filezilla Flatpak detail page in GNOME Software
 
 #. After installation is complete, the new application will be in your
-   Gnome applications list, as shown in Figure 3.
+   GNOME applications list, as shown in Figure 3.
 
    .. figure:: /_figures/flatpak/flatpak-03.png
       :scale: 50%
@@ -81,7 +81,7 @@ Install a Flatpak with the command line
 
 Both Flathub and the Clear Linux Store provide the command line instructions
 for installing a Flatpak. Figure 4 shows the command line instructions to
-install Filezilla from the Clear Linux Store:
+install Filezilla\* from the Clear Linux Store:
 
 .. figure:: /_figures/flatpak/flatpak-04.png
    :scale: 50%

source/tutorials/fmv.rst

@@ -277,8 +277,10 @@ application code.
 
 You have successfully installed an FMV development environment on |CL|.
 Furthermore, you used cutting edge compiler technology to improve the
-performance of your application based on Intel® architecture technology and
+performance of your application based on Intel® architecture and
 profiling of the specific execution of your application.
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _GCC:  https://gcc.gnu.org
 .. _make-fmv-patch: https://github.com/clearlinux/make-fmv-patch

source/tutorials/hpc.rst

@@ -0,0 +1,736 @@
+.. _hpc:
+
+HPC Cluster 
+###########
+
+This tutorial demonstrates how to set a simple :abbr:`HPC (High
+Performance Computing)` cluster using `Slurm`_, `MUNGE`_, and 
+`pdsh`_.  For this tutorial, this cluster consists of a controller node 
+and four worker nodes, as shown in Figure 1.  For the sake of simplicity,
+each node resides on a separate host and their hostnames are hpc-controller,
+hpc-worker1, hpc-worker2, hpc-worker3, and hpc-worker4.  
+
+.. rst-class:: dropshadow
+
+.. figure:: ../_figures/hpc/hpc-01.png
+   :alt: Simple HPC cluster
+
+   Figure 1: Simple HPC cluster
+
+The configuration is intentionally kept simple, notably avoiding setting 
+up cgroups and accounting. These and many more additional configuration 
+options can be added later.
+
+.. note::
+
+   This tutorial assumes you start with a new installation of |CL| for all 
+   nodes.
+
+Prerequisites
+*************
+
+* Knowledge and experience with HPC 
+* Familiarity with Slurm, MUNGE, and pdsh
+* All nodes have synchronized clocks (typically by NTP)
+
+.. contents::
+    :local:
+    :depth: 1
+
+Set up controller node
+**********************
+
+In this step, install the cluster tools, configure and enable the MUNGE service,
+and enable the Slurm controller service.
+
+#. Install |CL| on the controller node, add a user with adminstrator 
+   privilege, and set its hostname to `hpc-controller`.  
+
+#. Boot it up and log in.
+
+#. Update |CL| to the latest version.
+
+   .. code-block:: bash
+
+      sudo swupd update
+
+#. Set the date and time to synchronize with an NTP server.
+
+   .. code-block:: bash
+
+      sudo timedatectl set-ntp true
+
+#. Install the `cluster-tools` bundle.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add cluster-tools
+
+#. Create a MUNGE key and start the MUNGE service.
+   
+   a. Create the MUNGE key.
+      
+      .. code-block:: bash
+         
+         sudo mkdir /etc/munge
+         dd if=/dev/urandom bs=1 count=1024 | sudo tee -a /etc/munge/munge.key
+
+   #. Set the ownership to `munge` and set the correct access permissions.
+     
+      .. code-block:: bash
+         
+         sudo chown munge: /etc/munge/munge.key
+         sudo chmod 400 /etc/munge/munge.key      
+
+   #. Start the MUNGE service and set it to start automatically on boot.
+
+      .. code-block:: bash
+               
+         sudo systemctl enable munge --now
+
+#. Test MUNGE.
+
+   a. Create a MUNGE credential.
+
+      .. code-block:: bash
+
+         munge -n
+
+      Example output:
+
+      .. code-block:: console
+
+         MUNGE:AwQFAAC8QZHhL/+Fqhalhi+ZJBD5LavtMa8RMles1aPq7yuIZq3LtMmrB7KQZcQjG0qkFmoIIvixaCACFe1stLmF4VIg4Bg/7tilxteXHS940cuZ/TxpIuqC6fUH8zLgUZUPwJ4=:
+
+
+   #. Validate a MUNGE credential.  
+      
+      .. code-block:: bash
+
+         munge -n | unmunge | grep STATUS
+
+      Example output:
+        
+      .. code-block:: console
+
+         STATUS:           Success (0)
+
+#. Start the Slurm controller service and enable it to start automatically 
+   on boot.
+
+   .. code-block:: bash
+
+      sudo systemctl enable slurmctld --now
+
+Set up worker nodes
+*******************
+
+For each worker node, perform these steps:
+
+#. Install |CL| on the worker node, add a user with adminstrator privilege,
+   and set its hostname to `hpc-worker` plus its number, i.e. hpc-worker1, 
+   hpc-worker2, etc.  
+
+   Ensure the username is the same as the one on the controller node.  This
+   is needed to simplify password-less-SSH-access setup, which is needed for
+   pdsh, in the next section.  
+
+#. Boot it up and log in.
+
+#. Update |CL| to the latest version.
+
+   .. code-block:: bash
+
+      sudo swupd update
+
+#. Set the date and time to synchronize with an NTP server.
+
+   .. code-block:: bash
+
+      sudo timedatectl set-ntp true
+
+#. Install the `cluster-tools` bundle.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add cluster-tools
+
+Set up password-less SSH access and pdsh on all nodes
+*****************************************************
+
+To efficiently manage a cluster, it is useful to have a tool
+that allows issuing the same command to multiple nodes at once.  
+And that tool is :abbr:`pdsh (parallel distributed shell)`, which is included
+with the `cluster-tools` bundle. pdsh is built with Slurm support, so it can 
+access hosts as defined in the Slurm partitions.  pdsh relies on password-less 
+SSH access in order for it to work properly. There are two ways to set up 
+pasword-less SSH authentication: key-based or host-based.  In this case, 
+the latter approach will be used.  The controller authenticates a user and 
+all worker nodes will trust that authentication and not ask the user to 
+enter a password again.
+
+#. Configure the controller node.
+
+   a. Log into the controller node.
+
+   #. Configure the SSH service for host-based authentication.
+
+      .. code-block:: bash
+
+         sudo tee -a /etc/ssh/ssh_config << EOF
+	 HostbasedAuthentication yes
+         EnableSSHKeysign yes
+	 EOF
+
+   #. Restart the SSH service.
+
+      .. code-block:: bash
+
+         sudo systemctl restart sshd
+
+#. Configure each worker node.
+
+   a. Configure SSH service for host-based authentication.
+
+      .. code-block:: bash
+
+	 sudo tee -a /etc/ssh/sshd_config << EOF 
+         HostbasedAuthentication yes
+         IgnoreRhosts no
+         UseDNS yes
+	 EOF
+
+   #. Create the :file:`/etc/hosts.equiv` file and add the controller's
+      :abbr:`FQDN (fully qualified domain name)`.  This tells the worker 
+      node to accept connection from the controller.  
+
+      For example:
+
+      .. code-block:: console
+
+         hpc-controller.my-domain.com
+
+   #. Set its permission to root access only.
+
+      .. code-block:: bash
+
+         sudo chmod 600 /etc/hosts.equiv
+
+   #. Add the controller's FQDN to :file:`/root/.shosts`.  This allows
+      host-based authentication for the root account so that 
+      actions requiring sudo privileges can be performed.
+
+      .. code-block:: bash
+
+         sudo cp -v /etc/hosts.equiv /root/.shosts
+
+   #. Using the controller's FQDN in :file:`/etc/hosts.equiv`, scan for its 
+      RSA public key and copy it to :file:`/etc/ssh/ssh_known_hosts`.
+      Verify the scanned RSA public key matches the controller's 
+      :file:`/etc/ssh/ssh_rsa_key.pub` file.
+
+      .. code-block:: bash
+
+        sudo ssh-keyscan -t rsa -f /etc/hosts.equiv > ~/ssh_known_hosts
+        sudo cp -v ~/ssh_known_hosts /etc/ssh
+	rm ~/ssh_known_hosts
+
+   #. Restart the SSH service.
+
+      .. code-block:: bash
+
+         sudo systemctl restart sshd
+
+#. On the controller node, SSH into each worker node without having to enter 
+   a password. At the first-time connection to each host, you'll be asked to 
+   add the unknown host to the :file:`$HOME/.ssh/known_hosts` file.  Accept
+   the request.  This is will make future SSH connections to each host be 
+   non-interactive.
+
+   .. code-block:: bash
+
+      ssh <worker-node>
+
+   .. note::
+
+      Setting up host-based authentication on 
+      :abbr:`CSP (Cloud Service Provider)` environments such as Microsoft Azure
+      and Amazon AWS may require some tweaking on the worker nodes' SSH 
+      configurations due to the CSP's virtual network setup. In general, 
+      cloud VMs have a public and private DNS name.  When SSH'ing from the 
+      controller to a worker node, the SSH client may send the controller's 
+      private DNS name, usually something with "internal" in the name,
+      as the `chost` instead of its public FQDN as expected in worker node's 
+      :file:`/etc/hosts.equiv`, :file:`/root/.shosts`, and 
+      :file:`/etc/ssh/ssh_known_hosts` files.  If the above configurations 
+      do not work, meaning you're asked to enter a password when
+      SSH'ing from the controller to a worker node, on a cloud VM, here are 
+      some suggestions for debugging the problem:
+
+      #. On the controller, try to identify the chost data sent by the SSH 
+         client using :command:`ssh -vvv <worker-node>`.  Look for `chost`
+         in the debug log.  If the chost value is different than the controller's
+	 FQDN listed in worker node's :file:`/etc/hosts.equiv`, 
+	 :file:`/root/.shosts`, and :file:`/etc/ssh/ssh_known_hosts` files, 
+	 then that is likely the cause of the problem.  In some cases, chost 
+	 data may not be shown.  If so, it's safe to assume that the SSH client 
+	 is using the controller's private DNS name as the chost.  Proceed to 
+	 steps 2 and 3 below to fix the problem.
+
+      #. Get the controller's private DNS name either by the above step or by
+         getting it from your system administrator.
+
+      #. On the worker node, make these changes:
+
+         #. Change the controller's FQDN in :file:`/etc/hosts.equiv`,
+            :file:`/root/.shosts`, and :file:`/etc/ssh/ssh_known_hosts`
+	    to its private DNS name.
+
+         #. Restart the SSH service on the worker node.
+
+         #. Retest the connection from the controller node to the worker node. 
+            If that still doesn't work, try the SSH directive 
+	    `HostbasedUsesNameFromPacketOnly yes` which tell the SSH service
+	    to accept the supplied host name as is and not try to resolve it.
+	    Also, set the directive `UseDNS` to `no` to disable host name lookup.  
+#. Verify you can issue a simple command over SSH without typing a password.
+
+   a. Issue the :command:`hostname` command.
+
+      .. code-block:: bash
+
+         ssh <worker-node> hostname
+
+   #. Issue the :command:`hostname` command with :command:`sudo`.
+
+      .. code-block:: bash
+
+         ssh <worker-node> sudo hostname
+
+   In both cases, you should get a response with the worker node's hostname.
+   If the `sudo` version requires additional permission, grant the user
+   `NOPASSWD` privilege.  For example:
+
+   #. Edit the sudoer file.
+
+      .. code-block:: bash
+
+         sudo visudo
+
+   #. Add the following:
+
+      .. code-block:: console
+
+         <user> ALL=(ALL) NOPASSWD: ALL
+
+Create :file:`slurm.conf` configuration file
+********************************************
+
+On the controller, create a new :file:`slurm.conf` configuration file 
+that contains general settings, each node's hardware resource information, 
+grouping of nodes into different partitions, and scheduling settings for 
+each partition.  This file will be copied to all worker nodes in the cluster.  
+
+#. Create a base :file:`slurm.conf` configuration file.
+
+   .. code-block:: bash
+
+      sudo mkdir -p /etc/slurm
+      sudo cp -v /usr/share/defaults/slurm/slurm.conf /etc/slurm
+
+#. Add the controller information.
+
+   a. :command:`sudoedit` the :file:`slurm.conf` file. Set the `ControlMachine`
+      value to the controller's resolvable hostname.
+
+      For example:
+
+      .. code-block:: console
+
+	 ControlMachine=hpc-controller
+
+      .. note:: 
+
+         Assuming the controller's FQDN is resolvable, specifying the 
+	 controller's IP address with the `ControlAddr` key is optional.  
+	 However, it maybe helpful to add it.  
+
+   #. Save and exit.
+
+#. Add the worker nodes information.
+   
+   a. Create a file containing a list of the worker nodes.
+
+      .. code-block:: bash
+
+         cat > worker-nodes-list << EOF
+	 hpc-worker1
+	 hpc-worker2
+	 hpc-worker3
+	 hpc-worker4
+	 EOF
+
+   #. Using pdsh, get the hardware configuration of each node.
+
+      .. code-block:: bash
+
+         pdsh -w ^worker-nodes-list slurmd -C
+
+      Example output:
+
+      .. code-block:: console
+
+         hpc-worker4: NodeName=hpc-worker4 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=1915
+         hpc-worker4: UpTime=0-01:23:28
+         hpc-worker3: NodeName=hpc-worker3 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=1663
+         hpc-worker3: UpTime=0-01:33:41
+         hpc-worker2: NodeName=hpc-worker2 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=721
+         hpc-worker2: UpTime=0-01:34:56
+         hpc-worker1: NodeName=hpc-worker1 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=721
+         hpc-worker1: UpTime=0-01:39:21
+
+   #. :command:`sudoedit` the :file:`slurm.conf` file.  Append each worker node 
+      information, but without the `UpTime`, under the `COMPUTE NODES` section.
+
+      .. tip::
+
+         It is strongly recommended to set the `RealMemory` value for each 
+	 worker node slightly below, say 90%, what was reported by 
+	 :command:`slurmd -C` 
+	 in case some memory gets use by some processes, which would 
+	 cause Slurm to make a node not available due to its memory 
+	 resource falling below the stated value in the configuration file.    
+
+      Here's an example with four worker nodes:
+
+      .. code-block:: console
+
+         # 
+         # COMPUTE NODES (mode detailed values reported by "slurmd -C" on each node)
+         NodeName=hpc-worker1 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=648
+         NodeName=hpc-worker2 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=648
+         NodeName=hpc-worker3 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=1497
+         NodeName=hpc-worker4 CPUs=1 Boards=1 SocketsPerBoard=1 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=1723
+
+   #. Create partitions.
+
+      A Slurm partition is basically the grouping of worker nodes.  
+      Give each partition a name and decide which worker node(s) belong to 
+      it.
+
+      For example:
+   
+      .. code-block:: console
+      
+         PartitionName=workers Nodes=hpc-worker1, hpc-worker2, hpc-worker3, hpc-worker4 Default=YES MaxTime=INFINITE State=UP
+         PartitionName=debug Nodes=hpc-worker1, hpc-worker3 MaxTime=INFINITE State=UP
+
+   #. Save and exit.
+
+   #. Set the ownership of the :file:`slurm.conf` file to `slurm`.
+
+      .. code-block:: bash
+         
+	 sudo chown slurm: /etc/slurm/slurm.conf
+
+#. On the controller node, restart the Slurm controller service.
+
+   .. code-block:: bash
+
+      sudo systemctl restart slurmctld
+
+#. Verify the Slurm controller service restarted without any issues before 
+   proceeding.
+
+   .. code-block:: bash
+
+      sudo systemctl status slurmctld
+
+   
+
+Copy MUNGE key and :file:`slurm.conf` to all worker nodes
+*********************************************************
+
+On the controller node, using pdsh, in conjunction with the list of 
+defined nodes in the :file:`slurm.conf`, copy it and the MUNGE key to 
+all worker nodes.  
+
+#. On the controller node, copy the MUNGE key to all worker nodes and start the 
+   MUNGE service.
+          
+   a. Create the :file:`/etc/munge/` directory on each node.
+
+      .. code-block:: bash
+         
+	 sudo pdsh -P workers mkdir /etc/munge
+   
+   #. Copy the MUNGE key over.
+
+      .. code-block:: bash
+         
+	 sudo pdcp -P workers /etc/munge/munge.key /etc/munge
+
+   #. Set the ownership of the :file:`munge.key` file to `munge`.
+
+      .. code-block:: bash
+
+         sudo pdsh -P workers chown munge: /etc/munge/munge.key
+
+    #. Start the MUNGE service and set it to start automatically on boot.
+
+      .. code-block:: bash
+
+         sudo pdsh -P workers systemctl enable munge --now
+
+    #. Verify the MUNGE service is running.
+
+       .. code-block:: bash
+
+          sudo pdsh -P workers "systemctl status munge | grep Active"
+      
+       Example output:
+
+       .. code-block:: console
+
+          hpc-worker3:      Active: active (running) since Wed 2020-04-15 19:47:58 UTC; 55s ago
+          hpc-worker4:      Active: active (running) since Wed 2020-04-15 19:47:58 UTC; 55s ago
+          hpc-worker2:      Active: active (running) since Wed 2020-04-15 19:47:59 UTC; 54s ago
+          hpc-worker1:      Active: active (running) since Wed 2020-04-15 19:47:59 UTC; 54s ago
+
+#. On the controller node, copy the :file:`slurm.conf` file to all 
+   worker nodes and start the slurmd service on them.
+
+   a. Create the :file:`/etc/slurm/` directory on each worker node.
+
+      .. code-block:: bash
+         
+	 sudo pdsh -P workers mkdir /etc/slurm
+
+   #. Copy the :file:`slurm.conf` file over.
+
+      .. code-block:: bash
+         
+	 sudo pdcp -P workers /etc/slurm/slurm.conf /etc/slurm
+
+   #. Set the ownership of the :file:`slurm.conf` file to `slurm`.
+
+      .. code-block:: bash
+
+         sudo pdsh -P workers chown slurm: /etc/slurm/slurm.conf
+
+   #. Start the Slurm service and set it automatically start on boot.
+
+      .. code-block:: bash
+         
+	 sudo pdsh -P workers systemctl enable slurmd --now
+
+   #. Verify the slurmd service is running.
+
+      .. code-block:: bash
+
+         sudo pdsh -P workers systemctl status slurmd | grep Active
+      
+      Example output:
+
+      .. code-block:: console
+
+         hpc-worker3:      Active: active (running) since Wed 2020-04-15 19:39:22 UTC; 1min 17s ago
+         hpc-worker4:      Active: active (running) since Wed 2020-04-15 19:39:22 UTC; 1min 17s ago
+         hpc-worker2:      Active: active (running) since Wed 2020-04-15 19:39:23 UTC; 1min 17s ago
+         hpc-worker1:      Active: active (running) since Wed 2020-04-15 19:39:23 UTC; 1min 17s ago
+
+Verify controller can run jobs on all nodes
+*******************************************
+
+#. Check the state of the worker nodes.  
+
+   .. code-block:: bash
+
+      sinfo
+
+   Example output:
+
+   .. code-block:: console
+
+      PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
+      workers*     up   infinite      4   idle hpc-worker[1-4]
+      debug        up   infinite      2   idle hpc-worker[1,3]
+
+   .. tip::
+
+      If the nodes are in a "down" state, put them in the "idle" state.
+
+      For example:
+
+      .. code-block:: bash
+
+         sudo scontrol update nodename=hpc-worker[1-4] state=idle reason=""
+
+      Additional `Slurm troubleshooting tips`_.  
+
+#. And finally, verify Slurm can run jobs on all 4 worker nodes by issuing
+   a simple :command:`hostname` command.
+
+   .. code-block:: bash
+
+      srun -N4 -p workers hostname
+   
+   Example output:
+     
+   .. code-block:: console
+
+      hpc-worker4
+      hpc-worker3
+      hpc-worker1
+      hpc-worker2
+
+Create and run example scripts
+******************************
+
+Example 1: Return the hostname of each worker and output to :file:`show-hostnames.out`
+======================================================================================
+
+#. On the controller node, create the Slurm :file:`show-hostnames.sh` script.
+
+   .. code-block:: bash
+
+      cat > show-hostnames.sh << EOF
+      #!/bin/bash
+      #
+      #SBATCH --job-name=show-hostnames
+      #SBATCH --output=show-hostnames.out
+      #
+      #SBATCH --ntasks=4
+      #SBATCH --time=10:00
+      #SBATCH --mem-per-cpu=100
+      #SBATCH --ntasks-per-node=1
+
+      srun hostname
+      EOF
+
+#. Execute the script.
+
+   .. code-block:: bash
+
+      sbatch show-hostnames.sh
+   
+   The result will appear on the first node of the partition used. As no 
+   partition was explicitly specified, it would be the default partition.
+
+#. View the result.
+
+   .. code-block:: bash
+
+      pdsh -w hpc-worker1 "cat show-hostnames.out"
+
+   Example output:
+
+   .. code-block:: console
+
+      hpc-worker1: hpc-worker3
+      hpc-worker1: hpc-worker4
+      hpc-worker1: hpc-worker1
+      hpc-worker1: hpc-worker2
+
+Example 2: An MPI "Hello, World!" program
+=========================================
+
+#. On the controller node, create the :file:`mpi-helloworld.c` program.
+
+   .. code-block:: bash
+
+      cat > mpi-helloworld.c << EOF 
+      #include <stdio.h>
+      #include <unistd.h>
+      #include <mpi.h>
+
+      int main(int argc, char** argv)
+      {
+        // Init the MPI environment
+        MPI_Init(NULL, NULL);
+
+        // Get the number of processes
+        int world_size;
+        MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+        // Get the rank of the process
+        int world_rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+        // Get the name of the processor
+        char processor_name[MPI_MAX_PROCESSOR_NAME];
+        int name_len;
+        MPI_Get_processor_name(processor_name, &name_len);
+
+        // Print a hello world message
+        printf("Hello, World! from from processor %s, rank %d out of %d processors\n", processor_name, world_rank, world_size);
+
+        // Finalize the MPI environment
+        MPI_Finalize();
+     }
+     EOF
+
+#. Add the `c-basic` and `devpkg-openmpi` bundles, which are needed to compile
+   it.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add c-basic devpkg-openmpi
+
+#. Compile it.
+   
+   .. code-block:: bash
+
+      mpicc -o mpi-helloworld mpi-helloworld.c
+
+#. Copy the binary to all worker nodes.
+
+   .. code-block:: bash
+
+      pdcp -P workers ./mpi-helloworld $HOME
+
+#. Create a Slurm batch script to run it.
+
+   .. code-block:: bash
+
+      cat > mpi-helloworld.sh << EOF
+      #!/bin/sh
+      #SBATCH -o mpi-helloworld.out
+      #SBATCH --nodes=4
+      #SBATCH --ntasks-per-node=1
+
+      srun ./mpi-helloworld
+      EOF
+
+#. Run the batch script.
+
+   .. code-block:: bash
+
+      sbatch mpi-helloworld.sh
+
+#. View the results on first worker node in the partition.
+
+   .. code-block:: bash
+
+      pdsh -w hpc-worker1 "cat mpi-helloworld.out"
+   
+   Example output:
+
+   .. code-block:: console
+
+      Hello, World! from from processor hpc-worker3, rank 2 out of 4 processors
+      Hello, World! from from processor hpc-worker4, rank 3 out of 4 processors
+      Hello, World! from from processor hpc-worker1, rank 0 out of 4 processors
+      Hello, World! from from processor hpc-worker2, rank 1 out of 4 processors
+
+.. _Slurm: 
+   https://en.wikipedia.org/wiki/Slurm_Workload_Manager
+
+.. _MUNGE:
+   https://dun.github.io/munge/
+
+.. _pdsh:
+   https://linux.die.net/man/1/pdsh
+
+.. _Slurm troubleshooting tips:
+   https://slurm.schedmd.com/troubleshoot.html

source/tutorials/index.rst

@@ -3,9 +3,60 @@
 Tutorials
 #########
 
-Explore our tutorials to discover what you can do with |CL|!
+Explore our tutorials to discover what you can do with |CL|, conveniently
+sorted by difficulty level! Learn about :ref:`how we evaluate tutorials <tutorial-ratings>`.
+
+.. container:: multicolumns
+
+   .. container:: column narrow
+
+      .. rst-class:: colh3
+
+      Easy
+
+      -  :ref:`docker`
+      -  :ref:`flatpak-tutorial`
+      -  :ref:`kata`
+      -  :ref:`lamp-server-install`
+      -  :ref:`proxy`
+      -  :ref:`redis`
+      -  :ref:`smb-desktop`
+      -  :ref:`smb-server`
+      -  :ref:`yubikey-u2f`
+
+   .. container:: column narrow
+
+      .. rst-class:: colh3
+
+      Moderate
+
+      -  :ref:`hadoop`
+      -  :ref:`broadcom`
+      -  :ref:`fmv`
+      -  :ref:`hpc`
+      -  :ref:`kubernetes-bp`
+      -  :ref:`mirror-upstream-server`
+      -  :ref:`nvidia-cuda`
+      -  :ref:`php`
+      -  :ref:`vmware-workstation`
+      -  :ref:`wp-install`
+
+   .. container:: column narrow
+
+      .. rst-class:: colh3
+
+      Difficult
+
+      -  :ref:`kubernetes`
+      -  :ref:`nvidia`
+      -  :ref:`openfaas`
+      -  :ref:`multi-boot`
+      -  :ref:`machine-learning`
+      -  :ref:`zfs`
+
 
 .. toctree::
+   :hidden:
    :maxdepth: 1
    :glob:
 
@@ -21,4 +72,4 @@ Older tutorials that may still be relevant to some users.
    :maxdepth: 1
    :glob:
 
-   archive/*
+   archive/*

source/tutorials/lamp-server.rst

@@ -1,10 +1,10 @@
-.. _web-server-install:
+.. _lamp-server-install:
 
-Set up a LAMP web server on |CL-ATTR|
+LAMP Web Server
-#####################################
+###############
 
 This tutorial provides instructions on how to set up a
-:abbr:`LAMP (Linux, Apache\*, MySQL, PHP)` web server on |CL-ATTR| and how
+:abbr:`LAMP (Linux\*, Apache\*, MySQL\*, PHP)` web server on |CL-ATTR| and how
 to use phpMyAdmin\* to manage an associated database. Note that this
 tutorial installs MariaDB\*, which is a drop-in replacement for MySQL\*.
 
@@ -46,13 +46,12 @@ Apache software bundle on |CL|.
 
       sudo swupd bundle-add httpd
 
-
+#. Start the Apache service and set it start automatically on boot, 
-#. To start the Apache service, enter the following commands:
+   enter the following commands:
 
    .. code-block:: bash
 
-      sudo systemctl enable httpd.service
+      sudo systemctl enable --now httpd.service
-      sudo systemctl start httpd.service
 
 #. To verify that the Apache server application is running, open a web
    browser and navigate to: \http://localhost.
@@ -60,7 +59,9 @@ Apache software bundle on |CL|.
    If the service is running, a confirmation message will appear, similar to the
    message shown in figure 1.
 
-   .. figure:: /_figures/wordpress/web-server-install-1.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-1.png
       :alt: This web server is operational from host.
       :scale: 50%
 
@@ -150,12 +151,11 @@ default values, and define a custom DocumentRoot for your web server.
 
    "It works from its new location!"
 
-#. Stop and then restart ``httpd.service``.
+#. Restart ``httpd.service``.
 
    .. code-block:: bash
 
-      sudo systemctl stop httpd.service
+      sudo systemctl restart httpd.service
-      sudo systemctl start httpd.service
 
 #. Go to \http://localhost to view the new screen. You should see your updated
    default message from step 5.
@@ -168,12 +168,11 @@ default values, and define a custom DocumentRoot for your web server.
 
       sudo nano /etc/httpd/conf.d/httpd.conf
 
-#. Stop and then restart ``httpd.service``.
+#. Restart ``httpd.service``.
 
    .. code-block:: bash
 
-      sudo systemctl stop httpd.service
+      sudo systemctl restart httpd.service
-      sudo systemctl start httpd.service
 
 #. Go to \http://localhost and verify that you can see the default screen
    again.
@@ -198,12 +197,12 @@ functionality to your web server, install PHP on your system.
 
       sudo swupd bundle-add php-basic
 
-#. To enable PHP, enter the following commands:
+#. To enable PHP and set it to start automatically on boot, enter the 
+   following commands:
 
    .. code-block:: bash
 
-      sudo systemctl enable php-fpm.service
+      sudo systemctl enable --now php-fpm.service
-      sudo systemctl start php-fpm.service
       sudo systemctl restart httpd.service
 
    After restarting the Apache service, test your PHP installation.
@@ -225,7 +224,9 @@ functionality to your web server, install PHP on your system.
 
 #. Verify that the PHP information screen appears, similar to figure 2:
 
-   .. figure:: /_figures/wordpress/web-server-install-2.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-2.png
       :alt: PHP information screen
       :width: 600
 
@@ -247,12 +248,12 @@ and is available in the database-basic |CL| bundle.
 
       sudo swupd bundle-add database-basic
 
-#. To start MariaDB after it is installed, enter the following commands:
+#. To start MariaDB after it is installed and set it to start automatically on
+   boot, enter the following commands:
 
    .. code-block:: bash
 
-      sudo systemctl enable mariadb
+      sudo systemctl enable --now mariadb
-      sudo systemctl start mariadb
 
 #. To check the status of MariaDB, enter the following command:
 
@@ -280,7 +281,7 @@ hardening.
 
       Our suggested responses follow each question.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       Enter current password for root (enter for none):
 
@@ -288,7 +289,7 @@ hardening.
    user. For a newly installed MariaDB without a set root password, the
    password is blank. Thus, press enter to continue.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       OK, successfully used password, moving on...
 
@@ -299,19 +300,19 @@ hardening.
    Set the root password to prevent unauthorized MariaDB root user logins.
    To set a root password, type 'y'.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       New password:
 
    Type the desired password for the root user.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       Re-enter new password:
 
    Re-type the desired password for the root user.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       Password updated successfully!
       Reloading privilege tables..
@@ -324,7 +325,7 @@ hardening.
    is intended only for testing and for a smoother installation. To remove
    the anonymous user and make your database more secure, type 'y'.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       ... Success!
       Disallow root login remotely? [Y/n]
@@ -333,7 +334,7 @@ hardening.
    ensures that someone cannot guess the root password from the network. To
    block any remote root login, type 'y'.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       ... Success!
       Remove test database and access to it? [Y/n]
@@ -342,7 +343,7 @@ hardening.
    This database is also intended only for testing and should be removed. To
    remove the test database, type 'y'.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       - Dropping test database...
       ... Success!
@@ -353,7 +354,7 @@ hardening.
    Reloading the privilege tables ensures all changes made so far take
    effect immediately. To reload the privilege tables, type 'y'.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       ... Success!
 
@@ -415,7 +416,9 @@ steps below for setting up a database called "WordPress".
    :ref:`mysql_secure_installation command <set-password>`. Enter your
    credentials and select :guilabel:`Go` to log in:
 
-   .. figure:: /_figures/wordpress/web-server-install-3.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-3.png
       :alt: phpMyAdmin login page
       :width:     600
 
@@ -424,7 +427,9 @@ steps below for setting up a database called "WordPress".
 #. Verify a successful login by confirming that the main phpMyAdmin page
    displays, as shown in figure 4:
 
-   .. figure:: /_figures/wordpress/web-server-install-4.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-4.png
       :alt: phpMyAdmin dashboard
       :width:     600
 
@@ -441,7 +446,9 @@ steps below for setting up a database called "WordPress".
 
 #. Click :guilabel:`Create`.
 
-   .. figure:: /_figures/wordpress/web-server-install-5.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-5.png
       :alt: Databases tab
       :width:     600
 
@@ -452,7 +459,9 @@ steps below for setting up a database called "WordPress".
 
 #. Select the :guilabel:`Privileges` tab. Figure 6 shows its contents.
 
-   .. figure:: /_figures/wordpress/web-server-install-6.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-6.png
       :alt: Privileges tab
       :width:     600
 
@@ -462,7 +471,9 @@ steps below for setting up a database called "WordPress".
    :guilabel:`Privileges` tab. The `Add user account` page appears, as shown
    in figure 7.
 
-   .. figure:: /_figures/wordpress/web-server-install-7.png
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/web-server-install-7.png
       :alt: User accounts tab
       :width:     600
 
@@ -484,7 +495,9 @@ steps below for setting up a database called "WordPress".
 
 If successful, you should see the screen shown in figure 8:
 
-.. figure:: /_figures/wordpress/web-server-install-8.png
+.. rst-class:: dropshadow
+
+.. figure:: ../_figures/wordpress/web-server-install-8.png
    :alt: User added successfully
    :width:     600
 

source/tutorials/mirror-upstream-server.rst

@@ -0,0 +1,244 @@
+.. _mirror-upstream-server:
+  
+Mirror Upstream |CL| Update Server
+##################################
+
+For organizations that want to use the |CL| upstream updates, but want the 
+benefits of a local mirror, this tutorial shows how to set up one and
+configure your |CL| clients to use it.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Prerequisites
+*************
+
+* The recommended disk space for the mirror server should have at least 100GB 
+  of disk space as each complete update content is approximately 45GB.  
+
+Install up |CL| server to host updates
+**************************************
+
+#. Follow the :ref:`bare-metal-install-server` guide to install |CL| server.
+   Add a user with `Administrator` privilege.
+
+#. After installation is complete, boot it up.
+
+#. Add the `wget` bundle.  This will be used to clone the upstream |CL| server.  
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add wget
+
+Clone the |CL| update content
+*****************************
+
+|CL| periodically releases a "minversion", which is a complete update.  
+Then, subsequent releases are small updates until the next minversion.
+
+Download a minversion to start your mirror.  
+
+#. Determine a proper minversion by looking at a `Manifest.MoM`_ file 
+   for a particular release of |CL|.  
+
+   For example, if you look at 
+   https://cdn.download.clearlinux.org/update/33010/Manifest.MoM, 
+   you will see that the minversion is 32900.  So clone this version as the 
+   starting point.
+
+   .. code-block:: console
+      :emphasize-lines: 4
+
+      MANIFEST        30
+      version:        33010
+      previous:       33000
+      minversion:     32900
+      filecount:      1131
+      timestamp:      1588358889
+      contentsize:    0
+
+#. Make a directory to store the cache.
+
+   .. code-block:: bash
+
+      mkdir ~/mirror-download-clearlinux-org && cd $_
+
+#. Recursively download the :file:`update/0` folder.
+
+   .. code-block:: bash
+
+      wget --no-verbose \
+      --no-parent --recursive \
+      --no-host-directories -erobots=off \
+      --reject "index.html" https://cdn.download.clearlinux.org/update/0/
+
+#. Recursively download the :file:`update/version` folder.
+
+   .. code-block:: bash
+
+      wget --no-verbose \
+      --no-parent --recursive \
+      --no-host-directories -erobots=off \
+      --reject "index.html" https://cdn.download.clearlinux.org/update/version/
+
+#. Now, recursively download the determined minversion, which for this example 
+   is 32900.
+
+   .. code-block:: bash
+
+      wget --no-verbose \
+      --no-parent --recursive \
+      --no-host-directories -erobots=off \
+      --reject "index.html" https://cdn.download.clearlinux.org/update/32900/
+
+   .. note::
+
+      A minversion is pretty big, which is approximately 45GB.  Depending on your
+      proximity to the upstream server and your connection speed to the Internet,
+      it may take up to a couple of days or more to complete the download.  So
+      be patient.  
+
+#. Download later versions, up to the latest, if you like.
+
+Setup a web server to host the mirrored content
+***********************************************
+
+By design, the |CL| swupd client communicates with the update server using
+HTTPS for security reasons.  However, it can use HTTP by adding the 
+:command:`--allow-insecure-http` flag, if needed.  Setting an HTTPS is a lot 
+more involved.  For this tutorial, we'll just use an HTTP server for 
+demonstration purpose. 
+
+#. Install the `nginx` bundle.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add nginx
+
+#. Configure the web server.
+
+   a. Create a symbolic link to the mirrored update content directory.
+
+      .. code-block:: bash
+      
+         sudo mkdir -p /var/www && cd $_
+         sudo ln -sf $HOME/mirror-download-clearlinux-org mirror-download-clearlinux-org
+
+   #. Set up nginx configuration files.
+
+      .. code-block:: bash
+
+         sudo mkdir -p /etc/nginx/conf.d
+         sudo cp /usr/share/nginx/conf/nginx.conf.example /etc/nginx/nginx.conf
+
+   #. Grant $USER permission to run the web server.
+      
+      .. code-block:: bash
+
+         sudo tee -a /etc/nginx/nginx.conf << EOF
+         user $USER;
+         EOF
+
+   #. Configure the web server.
+      
+      .. code-block:: bash
+
+         sudo tee -a /etc/nginx/conf.d/mirror-download-clearlinux-org.conf << EOF
+         server {
+           listen 80;
+	   listen [::]:80;
+	   server_name localhost;
+	   location / {
+	     root /var/www/mirror-download-clearlinux-org;
+             autoindex on;
+           }
+         }
+         EOF
+
+   #. Set nginx to start automatically on boot and then start it.
+
+      .. code-block:: bash
+
+         sudo systemctl enable nginx --now
+
+Test your mirror
+****************
+
+Now, try out your mirror by installing |CL| and adding bundles from it.  
+
+#. Download either the live desktop or live server installer ISO of the 
+   `same version` as the mirrored version, which is 32900 for this tutorial.  
+   Go to `https://cdn.download.clearlinux.org/releases/<release-version>/clear`.
+
+#. Burn the ISO to a thumb drive.  See :ref:`bootable-usb`.
+
+#. Boot it up and start the installer.  Depending on which version of 
+   |CL| you want to install, follow one of these guides:
+
+   * *Desktop* version: :ref:`bare-metal-install-desktop`
+   * *Server* version: :ref:`bare-metal-install-server`
+
+   In the :guilabel:`Advanced options` tab of the installer, select 
+   :guilabel:`Swupd Mirror`.  See Figure 1.
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/mirror-upstream-server/mirror-upstream-server-01.png
+      :scale: 100%
+      :alt: Advanced options > Swupd Mirror
+
+      `Figure 1: Advanced options > Swupd Mirror`
+
+   In the :guilabel:`Mirror URL` field, set it to the IP address of your mirror.    It should be something like this: http://<IP address of mirror server>/update.
+   And check the option :guilabel:`Allow installation over insecure connections (http://)`.  See Figure 2.
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/mirror-upstream-server/mirror-upstream-server-02.png
+      :scale: 100%
+      :alt: Advanced options > Mirror URL setting
+
+      `Figure 2: Advanced options > Mirror URL setting`
+
+#. After installation completes, boot up, and log in.
+
+#. Verify that the swupd client is pointing to your mirror.
+
+   .. code-block:: bash
+
+      sudo swupd info
+
+   Example output:
+
+   .. code-block:: console
+
+      Warning: This is an insecure connection
+      The --allow-insecure-http flag was used, be aware that this poses a threat the system
+
+      Distribution:      Clear Linux OS
+      Installed version: 32900
+      Version URL:       https://192.168.1.100/update
+      Content URL:       https://192.168.1.100/update
+
+#. Try listing available bundles on your mirror.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-list -a
+
+#. Add a bundle.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add <bundle-name>
+
+Keep your mirror in sync with upstream
+**************************************
+
+Be sure to keep your mirror in sync with upstream so that your clients have the 
+latest and greatest software and security updates.  You can do that continuing
+to clone the newer upstream releases.  
+
+.. _Manifest.MoM:
+   https://docs.01.org/clearlinux/latest/reference/manpages/swupd.1.html 

source/tutorials/nvidia.rst

@@ -41,21 +41,21 @@ Known issues
 Systems with multiple graphics devices, including integrated graphics (iGPU),
 are known to be problematic.
 
-.. note:: NVIDIA Optimus
+.. note:: NVIDIA Optimus\*
    
    Some systems come with a hybrid graphics configuration for a balanced power
    and performance profile. This configuration is commonly found on
-   laptops. `NVIDIA Optimus* technology
+   laptops. `NVIDIA Optimus technology
    <https://www.geforce.com/hardware/technology/optimus>`_, is designed to
    allow switching seamlessly between a NVIDIA device and another graphics
    devices sharing the same display.
    
-   Getting NVIDIA Optimus* on Linux working well with both graphics devices
+   Getting NVIDIA Optimus on Linux working well with both graphics devices
    adds an additional level of complexity with platform specific steps and may
-   require additional software. Installation for systems with NVIDIA Optimus*
+   require additional software. Installation for systems with NVIDIA Optimus
    with both graphics devices operating is not covered by the scope of this
    documentation. As a simple workaround, some systems can disable one of the
-   graphics devices or NVIDIA Optimus* in the system firmware.
+   graphics devices or NVIDIA Optimus in the system firmware.
 
 .. note::
    The :ref:`Long Term Support (LTS) kernel <compatible-kernels>` variant is
@@ -76,7 +76,7 @@ and sustainable on |CL|.
 
 #. Remove the kernel command-line parameter *intel_iommu=igfx_off* or disable
    input–output memory management unit (IOMMU), also known as Intel®
-   Virtualization Technology for Directed I/O (VT-d), in your system EFI/BIOS.
+   Virtualization Technology (Intel® VT) for Directed I/O (Intel® VT-d), in your system EFI/BIOS.
    See `this GitHub report
    <https://github.com/clearlinux/distribution/issues/1274>`_ and the NVIDIA
    documentation on `DMA issues
@@ -121,7 +121,7 @@ and sustainable on |CL|.
     
          sudo systemctl daemon-reload
        
-   c. Add the service as a depndency to the |CL| updates trigger causing the
+   c. Add the service as a dependency to the |CL| updates trigger causing the
       service to run after every update.
   
       .. code-block:: bash  
@@ -428,8 +428,8 @@ installing the NVIDIA drivers until an Xorg configuration has been defined for
 your monitors.
 
 
-"Oh no! Something has gone wrong" GNOME crash
+"Oh no! Something has gone wrong" GNOME\* crash
-=============================================
+===============================================
 
 
 .. figure:: /_figures/nvidia/nvidia-gnome-crash.png
@@ -446,6 +446,26 @@ Try disabling other graphics devices, including integrated graphics, in your
 system's EFI/BIOS. 
 
 
+Slow boot times
+===============
+
+
+There have been reports of slow boot times with NVIDIA drivers installed.
+Normally, when GDM detects NVIDIA proprietary drivers, it will disable Wayland
+and enable X11. Should GDM fail to disbale Wayland, it may results in slow boot
+times, according to `this GitHub reprot
+<https://github.com/clearlinux/distribution/issues/1780>`_.
+
+To manually disable Wayland:
+
+.. code-block:: bash
+
+      sudo tee /etc/gdm/custom.conf > /dev/null <<'EOF'
+      [daemon]
+      WaylandEnable=false
+      EOF
+
+
 Brightness control
 ==================
 
@@ -502,6 +522,8 @@ Additional resources
 
 * `NVIDIA Accelerated Linux Graphics Driver Installation Guides <https://download.nvidia.com/XFree86/Linux-x86_64/>`_
 
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
 .. _`nouveau project`:  https://nouveau.freedesktop.org/wiki/
 
 .. _`NVIDIA Driver Downloads website`: https://www.nvidia.com/download/index.aspx

source/tutorials/openfaas.rst

@@ -196,7 +196,7 @@ In this example, we'll imagine a FaaS solution where: a user provides a URL to
 a pictures, which invokes a function to do image classification and outputs
 the result.
 
-We will use the `OpenVINO Deep Learning Deployment Toolkit (DLDT)
+We will use the `OpenVINO™ toolkit - Deep Learning Deployment Toolkit (DLDT)
 <https://software.intel.com/en-us/openvino-toolkit/>`_ to do the image
 inference. As inference development is not the focus of this example, we will
 just use the built-in sample "`classification_sample_async
@@ -208,7 +208,7 @@ We'll use the *python3-clearlinux* template as a base and customize it by:
 
 * Adding additional |CL| bundles (*bundles.txt*)
 * Adding additional required python packages (*requirements.txt*)
-* Adding a script to download and convert dldt models (*helper_script.sh*)
+* Adding a script to download and convert DLDT models (*helper_script.sh*)
 * Finally, we'll develop the python function to be run (*handler.py*)
 
 More ways to customize the |CL| based OpenFaaS templates can be found in the
@@ -385,3 +385,4 @@ More ways to customize the |CL| based OpenFaaS templates can be found in the
 
       Figure 3: OpenFaaS web interface invoke function
       
+*Intel, OpenVINO, and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*

source/tutorials/openvino.rst

@@ -1,290 +0,0 @@
- .. _openvino:
-
-OpenVINO™ for Deep Learning
-###########################
-
-This tutorial shows how to install OpenVINO™ on |CL-ATTR|, run an
-OpenVINO sample application for image classification, and run a benchmark_app
-for estimating inference performance---using Squeezenet 1.1.
-
-.. contents::
-   :local:
-   :depth: 1
-
-Prerequisites
-*************
-
-* |CL| installed on the host OS
-
-Install OpenVINO
-****************
-
-OpenVINO in |CL| offers pre-built OpenVINO sample applications with which
-developers can try inferencing immediately.
-
-#. In |CL| OpenVINO is included in the :command:`computer-vision-basic`
-   bundle. To install OpenVINO, enter:
-
-   .. code-block:: bash
-
-      sudo swupd bundle-add computer-vision-basic
-
-#. OpenVINO Inference Engine libraries are located in :file:`/usr/lib64/`
-   To view one added package, enter:
-
-   .. code-block:: bash
-
-      ls /usr/lib64/libinference_engine.so
-
-   If bundle installation is successful, the output shows:
-
-   .. code-block:: console
-
-      /usr/lib64/libinference_engine.so
-
-#. To view the OpenVINO Model Optimizer, enter:
-
-   .. code-block:: console
-
-      ls /usr/share/openvino/model-optimizer
-
-#. To view the OpenVINO sample application Executables, enter:
-
-   .. code-block:: bash
-
-      ls /usr/bin/benchmark_app \
-      /usr/bin/classification_sample_async \
-      /usr/bin/hello_classification \
-      /usr/bin/hello_nv12_input_classification \
-      /usr/bin/hello_query_device \
-      /usr/bin/hello_reshape_ssd \
-      /usr/bin/object_detection_sample_ssd \
-      /usr/bin/speech_sample \
-      /usr/bin/style_transfer_sample \
-
-   .. note::
-
-      If bundle installation is successful, the above files should appear.
-
-#. To view the pre-built OpenVINO sample application source code, enter:
-
-   .. code-block:: bash
-
-      ls /usr/share/doc/inference_engine/samples
-
-In the next section, you learn how to use an OpenVINO sample application.
-
-Run OpenVINO sample application
-*******************************
-
-After installing OpenVINO on |CL|, you need a model against which to test.
-In this example, we use the public squeezenet 1.1 model for image
-classification. Test results vary based on the system used.
-
-Use model to test
-=================
-
-#. If you don’t have any model, you can download an
-   **intel_model** or a public model using OpenVINO Model Downloader.
-
-   - Check the list of public models you can download from
-     :file:`/usr/share/open_model_zoo/models/public`
-
-   - Check the list of Intel® models you can download from
-     :file:`/usr/share/open_model_zoo/intel_models`
-
-#. View the location of OpenVINO Model Downloader:
-
-   .. code-block:: console
-
-      cd /usr/share/open_model_zoo/tools/downloader
-
-#. In general, download models with the following command:
-
-   .. code-block:: bash
-
-      python3 downloader.py --name <model_name> -o <downloading_path>
-
-   .. note::
-
-      * Where :file:`<model_name>` is the one you chose from previous step
-
-      * Where :file:`<downloading_path>` is your project directory
-
-#. For this example, enter:
-
-   .. code-block:: bash
-
-      python3 downloader.py --name squeezenet1.1 -o $HOME/.
-
-#. After running this command, the model appears as downloading at your
-   :file:`$HOME/classification/squeezenet/1.1/caffe` as follows:
-
-   .. code-block:: console
-
-      ###############|| Downloading topologies ||###############
-
-      ========= Downloading /$HOME/classification/squeezenet/1.1/caffe/squeezenet1.1.caffemodel
-      ... 100%, 4834 KB, 2839 KB/s, 1 seconds passed
-
-      ...
-
-Convert model to IR format
-==========================
-
-#. As necessary, follow the instruction on :ref:`convert-dl-models`
-   to convert deep learning models.
-
-#. Navigate to the model:
-
-   .. code-block:: bash
-
-	  cd $HOME/classification/squeezenet/1.1/caffe
-
-#. Enter the command:
-
-   .. code-block:: bash
-
-      python3 /usr/share/openvino/model-optimizer/mo.py --input_model squeezenet1.1.caffemodel
-
-
-   The output will show these files being generated:
-
-   .. code-block:: console
-
-      squeezenet1.1.xml
-
-      squeezenet1.1.bin
-
-#. Finally, enter :command:`ls` to view the newly added model and files.
-
-Run image classification
-========================
-
-This sample application demonstrates how to run the Image Classification in asynchronous mode on supported devices. In this example, we use the image of a specific type of automobile to test the inference engine. Squeezenet 1.1 is designed to perform image classification and has been trained on the `ImageNet`_ database.
-
-#. We provide an image of an automobile, shown in Figure 1. For ease of use,
-   save this image into the :file:`classification` model directory.
-
-   .. figure:: ../_figures/openvino/automobile.png
-      :height: 375 px
-      :width: 500 px
-      :scale: 100 %
-      :alt: Photo by Goh Rhy Yan on Unsplash
-
-      Figure 1: Photo by Goh Rhy Yan on Unsplash
-
-#. To execute the sample application enter the command:
-
-   .. code-block:: bash
-
-      classification_sample_async -i <path_to_image> -m <path_to_model_ir> -d <device>
-
-   .. note::
-
-      * Where :file:`<path_to_image>` is the image that you selected
-
-      * Where :file:`<path_to_model_ir>` is the path to the IR model file
-
-      * Where :file:`<device>` is your choice of CPU, GPU, etc.
-
-#. In this case, we replace the :file:`<path_to_image>` with the previously
-   saved image for CPU inferencing.
-
-   .. code-block:: bash
-
-      classification_sample_async -i ./automobile.png -m squeezenet1.1.xml
-
-   .. note::
-
-      If you do not specify the :file:`device`, the CPU is used by default.
-
-#. The results show the highest probability is 67% for a sports car.
-
-   .. code-block:: bash
-
-      classid probability
-      ------- -----------
-      817     0.6717085
-      511     0.1611409
-
-   +-----------------------+-----------------------------------+
-   |:command:`classid` 817 | :command:`sports car, sport car`  |
-   +-----------------------+-----------------------------------+
-   |:command:`classid` 511 |:command:`convertible`             |
-   +-----------------------+-----------------------------------+
-
-   .. note:
-
-      Label definitions are provided by `ImageNet`_.
-
-#. Next, add :command:`-d GPU` to the end of the above command for GPU
-   inferencing.
-
-   .. code-block:: bash
-
-      classification_sample_async -i ./automobile.png -m squeezenet1.1.xml -d GPU
-
-Run benchmark_app
-*****************
-
-This sample application demonstrates how to use benchmark application to
-estimate deep learning inference **performance** on supported devices.
-We use the same image of an automobile, Figure 1, from the previous section.
-
-#. To execute this sample application, enter:
-
-   .. code-block:: bash
-
-      benchmark_app -i <path_to_image> -m <path_to_model> -d <device>
-
-   .. note::
-
-      * Where :file:`<path_to_image>` is the image that you selected
-
-      * Where :file:`<path_to_model_ir>` is the path to the IR model file
-
-      * Where :file:`<device>` is local your choice of CPU, GPU, etc.
-
-#. Change directory:
-
-   .. code-block:: bash
-
-      cd $HOME/classification/squeezenet/1.1/caffe
-
-#. Enter the following command for CPU inferencing.
-
-   .. code-block:: bash
-
-      benchmark_app -i ./automobile.png -m squeezenet1.1.xml
-
-#. For the CPU, the results show a :guilabel:`Throughput` of 243.202 FPS.
-
-   .. code-block:: console
-      :linenos:
-      :emphasize-lines: 4
-
-      Count:      1464 iterations
-      Duration:   60196.8 ms
-      Latency:    164.104 ms
-      Throughput: 243.202 FPS
-
-#. Next, add :command:`-d GPU` to the end of the same command for GPU
-   inferencing.
-
-   .. code-block:: bash
-
-      benchmark_app -i ./automobile.png -m squeezenet1.1.xml -d GPU
-
-#. For the GPU, the results show a :guilabel:`Throughput` of 372.677 FPS.
-
-   .. code-block:: console
-      :linenos:
-      :emphasize-lines: 4
-
-      Count:      2240 iterations
-      Duration:   60105.7 ms
-      Latency:    107.554 ms
-      Throughput: 372.677 FPS
-
-.. _ImageNet: http://image-net.org/

source/tutorials/php.rst

@@ -61,8 +61,8 @@ Prerequisites
 .. note::
 
    PHP does not require a web server for operation. If you need a web
-   server, refer to :ref:`web-server-install` for instructions on setting
+   server, refer to :ref:`lamp-server-install` for instructions on setting
-   up a :abbr:`LAMP (Linux, Apache\*, MySQL, PHP)` server, or use
+   up a :abbr:`LAMP (Linux, Apache\*, MySQL\*, PHP)` server, or use
    :command:`swupd` to install :file:`nginx` or similar.
 
 

source/tutorials/proxy.rst

@@ -1,118 +0,0 @@
-.. _proxy:
-
-Proxy configuration
-###################
-
-This tutorial shows you how to configure your system for use behind an
-outbound proxy to access the Internet.
-
-|CL-ATTR| applications already benefit from the :ref:`autoproxy`
-feature.
-
-.. contents::
-    :local:
-    :depth: 1
-
-Prerequisites
-*************
-
-* You have installed |CL| on your host system.
-
-  For detailed instructions on installing |CL| on a bare metal system, visit
-  the :ref:`bare metal installation guide <bare-metal-install-desktop>`.
-
-Shells and programs in a desktop session
-****************************************
-
-Terminal
-========
-
-Add the following to your ~/.bashrc:
-
-.. code-block:: bash
-
-   export http_proxy=http://your.http-proxy.url:port
-   export https_proxy=http://your.https-proxy.url:port
-   export ftp_proxy=http://your.ftp-proxy.url:port
-   export socks_proxy=http://your.socks-proxy.url:port
-   export no_proxy=".your-company-domain.com,localhost"
-   export HTTP_PROXY=$http_proxy
-   export HTTPS_PROXY=$https_proxy
-   export FTP_PROXY=$ftp_proxy
-   export SOCKS_PROXY=$socks_proxy
-   export NO_PROXY=$no_proxy
-
-wget
-====
-
-Run the following command to enable downloading from websites from the terminal:
-
-.. code-block:: bash
-
-   echo >> ~/.wgetrc <<EOF
-   http_proxy = your.http-proxy.url:port
-   https_proxy = your.https-proxy.url:port
-   ftp_proxy = your.http-proxy.url:port
-   no_proxy = your-company-domain.com, localhost
-   EOF
-
-System service (Docker\*)
-=========================
-
-For Docker (and other services that use systemd), follow these steps to
-configure and check proxy settings.
-
-.. note::
-
-   Use :command:`sudo`, as you will need elevated permissions.
-
-#. Create the :file:`/etc/systemd/system/docker.service.d` directory to host
-   configuration information for the Docker service.
-
-#. Create the :file:`/etc/systemd/system/docker.service.d/http-proxy.conf`
-   and add the following lines:
-
-   .. code-block:: bash
-
-      [Service]
-      Environment="HTTP_PROXY=http://your.http-proxy.url:port/"
-      Environment="HTTPS_PROXY=http://your.https-proxy.url:port/"
-
-#. Load the changes and restart the service.
-
-   .. code-block:: bash
-
-      sudo systemctl daemon-reload
-      sudo systemctl restart docker
-
-#. Verify that changes have loaded.
-
-   .. code-block:: bash
-
-      systemctl show --property=Environment docker
-
-   .. code-block:: console
-
-      Environment=HTTP_PROXY=http://your.http-proxy.url:port/ HTTPS_PROXY=http://your.https-proxy.url:port/
-
-.. note::
-
-   This process enables the ability to successfully run ``docker pull``.
-   Containers themselves must be configured independently.
-
-git over ssh
-************
-
-Add the following to your :file:`~/.ssh/config`:
-
-.. code-block:: bash
-
-   host github.com
-        port 22
-        user git
-        ProxyCommand connect-proxy -S your.ssh-proxy.url:port %h %p
-
-.. note::
-
-   Though netcat is included with |CL|, it is not the BSD version,
-   which is the one usually used to enable git over ssh.

source/tutorials/redis.rst

@@ -5,7 +5,7 @@ Redis\*
 
 In this tutorial, you'll install :abbr:`Redis (REmote DIctionary Server)`
 and launch a `Redis-server` on |CL-ATTR|. We invite you to pull our
-`Clear Linux Redis instance`_ on dockerhub\* for application or
+`Clear Linux Redis instance`_ on Docker Hub\* for application or
 infrastructure development.
 
 .. contents::
@@ -15,57 +15,49 @@ infrastructure development.
 Description
 ***********
 
-Redis is an in-memory key:value store designed for quick lookups, accessible over a network. While the `redis data structure store`_ can serve
+Redis is an in-memory key:value store designed for quick lookups, accessible 
+over a network. While the `redis data structure store`_ can serve
 as a NoSQL database for a web application, it's also easy to integrate into an
 existing stack. For example, you could use the Redis caching layer for
 real-time responses on a leaderboard in a gaming app. Redis offers many client
 libraries with language-specific bindings for Python\*, Perl\*, Ruby, and more.
 
-Prerequisites
-*************
-
-* Install the :command:`redis-native` bundle in |CL|
-* Install the :command:`containers-basic` bundle in |CL| (only required in
-  Example 2)
-
 Install the Redis bundle
 ************************
 
-In |CL|, find Redis in the :command:`redis-native` bundle.
+#. Log in as a user with administrative privilege.
 
-#. Open a terminal and login as an administrative user.
+#. Open a terminal.
 
-#. Add :command:`redis-native`.
+#. Update your |CL| to the latest version.
+
+   .. code-block:: bash
+
+      sudo swupd update
+
+#. Install the `redis-native` bundle.
 
    .. code-block:: bash
 
       sudo swupd bundle-add redis-native
 
-   .. note::
-
-      If the bundle already exists, no action is required.
-
 Start the Redis-server
 **********************
 
 A :command:`systemd` service unit is available to control the Redis-server.
 By default, Redis runs on port 6379.
 
-#. Start the service.
+#. Start the service and set it to start automatically on boot.
 
    .. code-block:: bash
 
-      systemctl start redis
+      sudo systemctl enable --now redis
-
-   .. note::
-
-      To stop Redis, run :command:`systemctl stop redis`.
 
 #. Confirm the service is running.
 
    .. code-block:: bash
 
-      systemctl status redis
+      sudo systemctl status redis
 
 #. Verify that the Redis-server sends a reply.
 
@@ -73,16 +65,21 @@ By default, Redis runs on port 6379.
 
       redis-cli ping
 
-   .. note::
+   Expected output: 
 
-      Expected output: `PONG`.
+   .. code-block:: console
 
-#. Optional: If you wish to apply the advanced configuration, copy the
+      PONG
-   `redis.conf` into /etc/ directory.
+
+.. note::
+
+   If you wish to customize settings for Redis, copy the
+   default :file:`/usr/share/defaults/etc/redis.conf` file into the 
+   /etc/ directory, make changes as needed, and restart the service.
 
    .. code-block:: bash
 
-      sudo cp /usr/share/defaults/etc/redis.conf /etc/
+      sudo cp -v /usr/share/defaults/etc/redis.conf /etc/
 
 The Redis-server is now ready to use on |CL|. Try some of the examples shown
 below.
@@ -97,7 +94,7 @@ programming languages. These exercises are inspired by `try redis io`_.
 
 After your Redis-server is running, try some basic commands.
 
-#. Enter the `redis-cli`. It provides syntax suggestions as you type.
+#. Start `redis-cli`. It provides syntax suggestions as you type.
 
    .. code-block:: bash
 
@@ -106,66 +103,67 @@ After your Redis-server is running, try some basic commands.
 #. :command:`SET` a key to hold a string value. In the set, create connections
    and increment.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       SET server:name "clearlinux"
 
-   .. code-block:: bash
+   .. code-block:: none
 
       MGET server:name
 
    .. note::
+
       If the key does not exist or hold a key value, `nil` is returned.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       SET connections 100
 
-   .. code-block:: bash
+   .. code-block:: none
 
       INCR connections
 
-   .. code-block:: bash
+   .. code-block:: none
 
       INCR connections
 
-   .. code-block:: bash
+   .. code-block:: none
 
       DEL connections
 
 #. Create a `friends` list and insert new values at the end of the list.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       RPUSH friends "Deb"
 
-   .. code-block:: bash
+   .. code-block:: none
 
       RPUSH friends "David"
 
-   .. code-block:: bash
+   .. code-block:: none
 
       RPUSH friends "Mary"
 
 #. Modify the `friends` list, using a common slice method with a 0-index.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       LRANGE friends 0 1
 
-   .. code-block:: bash
+   .. code-block:: none
 
       LLEN friends
 
-   .. code-block:: bash
+   .. code-block:: none
 
       LPOP friends
 
-   .. code-block:: bash
+   .. code-block:: none
 
       RPOP friends
 
-   .. code-block:: bash
+   .. code-block:: none
 
       LLEN friends
 
@@ -174,19 +172,19 @@ After your Redis-server is running, try some basic commands.
 
    Enter many user key:values with `HMSET`. Then try `HGET` and `HGETALL`.
 
-   .. code-block:: bash
+   .. code-block:: none
 
       HMSET user:1000 name "Robert Noyce" password "SuperEngi9eer" email "robert.noyce@intel.com"
 
-   .. code-block:: bash
+   .. code-block:: none
 
       HGET user:1000 name
 
-   .. code-block:: bash
+   .. code-block:: none
 
       HGET user:1000 email
 
-   .. code-block:: bash
+   .. code-block:: none
 
       HGETALL user:1000
 

source/tutorials/smb-server.rst

@@ -0,0 +1,150 @@
+.. _smb-server:
+
+Samba Server
+############
+
+This tutorial describes how to enable simple file sharing on a system
+running |CL-ATTR| and how to access the share from clients on other operating 
+systems. 
+
+.. contents::
+   :local:
+   :depth: 1
+
+Prerequisites
+*************
+
+* |CL| is installed on your host system.
+
+  For detailed instructions, follow one of these guides:
+
+  * *Desktop* version: :ref:`bare-metal-install-desktop`
+  * *Server* version: :ref:`bare-metal-install-server`
+
+* Update your |CL| system to the latest version. 
+
+  .. code-block:: bash
+
+     sudo swupd update
+
+Set up file sharing
+*******************
+
+#. Add the :command:`samba` bundle, which includes the Samba binaries.
+
+   .. code-block:: bash
+
+      sudo swupd bundle-add samba
+
+#. Create the :file:`/etc/samba` directory.
+
+   .. code-block:: bash
+
+      sudo mkdir /etc/samba
+
+#. Create a configuration file named :file:`/etc/samba/smb.conf`. In this
+   simple example, :envvar:`[SharedDrive]` enables a folder share located in 
+   :file:`/home/clear/mysharedrive` granting users `samba-user-1` and 
+   `samba-user-2` access. 
+
+   If :envvar:`valid users` is not specified, then anyone with a user account
+   on the machine and with their Samba password already set can access the
+   folder. However, the account is only able to access files and folders for
+   which they have appropriate permissions.
+
+   For more advanced sharing, refer to the `Samba guide`_.
+
+   .. code-block:: console
+
+      [Global]
+      map to guest = Bad User
+
+      [SharedDrive]
+      path=/home/clear/myshareddrive
+      read only = no
+      guest ok = no
+      browsable = yes
+      valid users = samba-user-1 samba-user-2
+
+#. Start the Samba service and set it to start automatically on boot.
+
+   .. code-block:: bash
+
+      sudo systemctl enable --now smb
+
+#. Verify the service started properly.
+
+   .. code-block:: bash
+
+      sudo systemctl status smb
+
+#. Use :command:`smbpasswd` to add the initial password for the user
+   account to access the share. Be aware that Samba maintains its own list of
+   passwords for user accounts. The Samba password list can be different from
+   the password used to log in.
+
+   For example:
+
+   .. code-block:: bash
+
+      sudo smbpasswd -a samba-user-1
+
+Access the shared drive
+***********************
+
+Depending on your operating system, connect to the shared drive using one 
+of the methods belows:
+
+* On |CL|:
+
+  a. Add the `samba` bundle.
+
+     .. code-block:: bash
+
+        sudo swupd bundle-add samba
+  
+  #. List available shares.
+
+     .. code-block:: bash
+
+        smbclient -L //<ip-address-of-smb-server>
+
+  #. Connect to a shared drive.
+     
+     .. code-block:: bash
+
+        smbclient //<ip-address-of-smb-server>/<shared-drive> -U <user>
+
+* On Windows:
+
+  a. Open `File Explorer`.
+
+  #. Enter :command:`\\<ip-address-of-samba-server>\<shared-drive>` in the
+     URL field. See Figure 1.
+
+     .. rst-class:: dropshadow
+
+     .. figure:: ../_figures/samba/smb-server-01.png
+        :scale: 100%
+        :alt: Windows > File Explorer > Connect to Samba share
+
+        `Figure 1: Windows > File Explorer > Connect to Samba share`
+
+* On macOS:
+
+  a. Open the `Finder`.
+
+  #. Press :kbd:`Command` + :kbd:`K` to open the dialog box for 
+     connecting to a server.
+
+  #. Enter :command:`smb://<ip-address-of-samba-server>/<shared-drive>`.
+     in the URL field. See Figure 2.
+
+     .. figure:: ../_figures/samba/smb-server-02.png
+        :scale: 100%
+        :alt: macOS > Finder > Connect to Samba share
+
+        `Figure 2: macOS > Finder > Connect to Samba share`
+     
+.. _Samba guide: 
+   https://www.samba.org/samba/docs/using_samba/ch00.html

source/tutorials/smb.rst

@@ -1,139 +0,0 @@
-.. _clear-samba-share-to-Windows:
-
-Samba\* as a host
-#################
-
-This tutorial describes how to enable simple file sharing from a system
-running |CL-ATTR| to a Windows\* computer using Samba. For more advanced
-sharing, refer to the `Samba guide`_.
-
-.. contents::
-   :local:
-   :depth: 1
-
-Prerequisites
-*************
-
-*	|CL| is installed on your host system.
-
-	For detailed instructions, follow the steps in
-	:ref:`bare-metal-install-desktop`.
-
-*	Before you install any new packages, update |CL| with the following
-	command:
-
-	.. code-block:: bash
-
-	   sudo swupd update
-
-
-Set up file sharing
-*******************
-
-#.	Log in and get root privileges.
-
-	.. code-block:: bash
-
-		sudo -s
-
-#.	Add the :command:`storage-utils` bundle, which includes the Samba binaries.
-
-	.. code-block:: bash
-
-		swupd bundle-add storage-utils
-
-	.. note::
-
-		The :command:`os-clr-on-clr` bundle also includes the Samba binaries.
-
-#.	Create a configuration file named :file:`/etc/samba/smb.conf`. In this
-	example, :envvar:`[Downloads]` enables a folder share with a specific user.
-	:envvar:`[Documents]` enables a folder share with any user. The example
-	assumes that the user account :envvar:`clearlinuxuser` already exists.
-
-	If :envvar:`valid users` is not specified, then anyone with a user account
-	on the machine and with their Samba password already set can access the
-	folder. However, the account is only able to access files and folders for
-	which they have appropriate permissions.
-
-	Use either :command:`chown` to change the owner of the file or use
-	:command:`chmod` to change the permissions to allow other users to access
-	the file.
-
-	.. code-block:: console
-
-		[Global]
-		map to guest = bad user
-
-		[Downloads]
-		path=/home/clearlinuxuser/Downloads
-		read only = no
-		guest ok = no
-		valid users = clearlinuxuser
-
-		[Documents]
-		path=/home/clearlinuxuser/Documents
-		read only = no
-		browsable = yes
-		guest ok = yes
-
-#.	Enable the Samba daemon to start every time.
-
-	.. code-block:: bash
-
-		systemctl enable smb
-		systemctl start smb
-
-#.	Use :command:`smbpasswd` to add the initial password for the user
-	account to access the share. Be aware that Samba maintains its own list of
-	passwords for user accounts. The Samba password list can be different from
-	the password used to log in.
-
-	.. code-block:: bash
-
-		smbpasswd -a clearlinuxuser
-
-The setup is now complete and a Windows machine on the same network can access
-the shares. Windows uses the format
-:file:`\\\\[server IP or hostname]\\folder` to access shares. Access the shares
-directly with Windows Explorer or by mapping a network drive.
-
-Use the IP address of the |CL| machine for an easy access method. If the
-|CL| machine is behind an Active Directory domain controller or a DNS server,
-use the hostname of the |CL| machine. For other ways to access shares using a
-hostname instead of an IP address, see `Chapter 7 of the Samba guide`_.
-
-
-Map |CL| drive in Windows
-*************************
-
-#.	Open Windows Explorer and click on the left sidebar on :guilabel:`This PC`
-	to change the options available at the top.
-
-#.	Click the :guilabel:`Map Network Drive` icon and enter the path in the
-	format: :file:`\\\\[server IP or hostname]\\[shared folder]`
-
-#.	Check the box :guilabel:`Connect using different credentials`. Enter
-	the Samba user `clearlinuxuser` and the password created with
-	:command:`smbpasswd`. See Figure 1 for details.
-
-	.. figure:: /_figures/samba/smb-1.png
-		:scale: 70%
-		:alt: Map a network drive in Windows Explorer
-
-		Figure 1: Map a network drive in Windows Explorer.
-
-When complete, Windows Explorer displays the share drive as shown in Figure 2.
-
-.. figure:: /_figures/samba/smb-2.png
-	:scale: 70%
-	:alt: View a share drive in Windows Explorer
-
-	Figure 2: View a share drive in Windows Explorer.
-
-
-
-
-
-.. _Samba guide: https://www.samba.org/samba/docs/using_samba/ch00.html
-.. _Chapter 7 of the Samba guide: https://www.samba.org/samba/docs/using_samba/ch07.html

source/tutorials/wordpress.rst

@@ -1,27 +1,214 @@
-.. _wordpress:
+.. _wp-install:
 
-WordPress\*
+WordPress\* Server
-###########
+##################
 
-This tutorial describes how to setup a web server to host a WordPress\* site on a
+This tutorial shows you how to install the WordPress\* components on your |CL|
-|CL-ATTR| system. The tutorial is split into two sections:
+:abbr:`LAMP (Linux, Apache\*, MySQL\*, PHP)` server. At the end of
+:ref:`lamp-server-install`, you created a WordPress-ready database using
+phpMyAdmin\* and MariaDB\*. Now that the LAMP server is up and running, you
+can add the WordPress components needed to host a WordPress website on your system.
 
-#. :ref:`web-server-install`
+Before you begin
+****************
 
-#. :ref:`wp-install`
+This tutorial assumes that you have successfully completed
+:ref:`bare-metal-install-desktop` and you have :ref:`lamp-server-install`.
 
-If you already have a |CL| based :abbr:`LAMP (Linux, Apache, MySQL, PHP)` web server,
+Create a WordPress server
-you can skip ahead to the second section of this tutorial.
+*************************
+
+WordPress can be installed in a variety of ways. These instructions are
+written for users who have followed our instructions for installing phpMyAdmin
+when they :ref:`set up a LAMP web server <lamp-server-install>`. Note that
+all steps in this tutorial have been tested using a NUC6i5SYH Intel® NUC.
+Visit the `NUC6i5SYH product page`_ for detailed information.
+
+Numerous online articles are available to help you name your website and
+acquire the necessary certificates. Those tasks are beyond the scope of this tutorial.
+
+You can take several actions to harden your website from attacks. The security
+of your website and the data it contains are complex and ever-evolving tasks.
+Prioritize security if you plan to expose your website to the outside world.
+This tutorial does not address security measures that you can take to harden
+your site but we strongly encourage you to take action.
 
 .. note::
 
-   This tutorial is for development and testing purposes only. Additional steps
+   Throughout this tutorial, we reference your website name as <your_website>.
-   are required to secure production systems. Those steps are beyond the scope
-   of this tutorial.
 
-.. toctree::
-   :maxdepth: 1
-   :hidden:
 
-   wordpress/web-server-install.rst
+Download WordPress and manage directories
-   wordpress/wp-install.rst
+=========================================
+
+For this tutorial, you will create a WordPress blog that can be accessed at:
+\http://<your_website>/blog.
+
+To accomplish this setup, you must add WordPress components to the :file:`/var/www/html/blog`
+directory.
+
+Follow these steps:
+
+
+#. Navigate to the top level of the website’s root directory:
+
+   .. code-block:: bash
+
+      cd /var/www/html
+
+#. Download the latest version of WordPress:
+
+   .. code-block:: bash
+
+      sudo curl -O https://wordpress.org/latest.tar.gz
+
+#. Extract all files and directories from the downloaded file:
+
+   .. code-block:: bash
+
+      sudo tar –xzvf latest.tar.gz
+
+#. Rename the top-level WordPress directory to “blog”:
+
+   .. code-block:: bash
+
+      sudo mv wordpress blog
+
+#. Remove the downloaded tar file:
+
+   .. code-block:: bash
+
+      sudo rm latest.tar.gz
+
+Set up WordPress with web-based GUI
+===================================
+
+Recall that you created a database and user when you installed phpMyAdmin when you
+set up a |CL| based :ref:`web server <lamp-server-install>`. Next, you must
+connect WordPress to the database and install WordPress.
+
+To continue with the setup, go to: \http://<your_website>/blog/wp-admin/install.php.
+The WordPress language option screen appears, as shown in figure 1.
+
+#. Select :guilabel:`English` and click :guilabel:`Continue`.
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/wp-install-1.png
+      :alt: WordPress language selection
+      :width:     600
+
+      `Figure 1: WordPress language selection screen.`
+
+
+   The WordPress installation continues until the Welcome screen appears, as shown in
+   figure 2:
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/wp-install-2.png
+      :alt: WordPress welcome screen
+      :width:     600
+
+      `Figure 2: WordPress Welcome screen.`
+
+#. Click :guilabel:`Let’s go!`.
+
+#. Enter database connection specifics in the screen that appears, as shown in figure 3
+   below.
+
+   * Database name:       WordPress
+   * Database username:   wordpressuser
+   * Database password:   wp-example  (asterisks will not appear in the text box)
+   * Database host:  localhost
+   * Table prefix:   wp\_
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/wp-install-3.png
+      :alt: Database connection details
+      :width:     600
+
+      `Figure 3: Information necessary for WordPress to connect to the database.`
+
+#. Click :guilabel:`Submit` to complete the setup.
+
+   Figure 4 shows the confirmation screen that verifies a successful setup. WordPress
+   is connected to the MariaDB database.
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/wp-install-4.png
+      :alt: Successful database connection.
+      :width:     600
+
+      `Figure 4: Successful WordPress connection.`
+
+#. Click :guilabel:`Run the install`.
+   The installer runs until WordPress is fully installed on your system.
+
+Complete successful login
+=========================
+
+Once the installation is complete, you can name your blog and create a WordPress username
+and password. See figure 5.
+
+.. rst-class:: dropshadow
+
+.. figure:: ../_figures/wordpress/wp-install-5.png
+   :alt: WordPress user creation
+   :width:     600
+
+   `Figure 5: WordPress site information screen.`
+
+
+#. Enter all required information.
+#. Click :guilabel:`Install WordPress`.
+#. Verify that the initial login screen appears once the installation is complete. See figure 6:
+
+   .. rst-class:: dropshadow
+
+   .. figure:: ../_figures/wordpress/wp-install-6.png
+      :alt: WordPress login
+      :width:     600
+
+      `Figure 6: The WordPress login screen.`
+
+#. Enter your WordPress username and password.
+#. Check :guilabel:`Remember me` to save your credentials.
+#. Click :guilabel:`Log in`.
+
+Figure 7 shows the WordPress dashboard after a successful login:
+
+.. rst-class:: dropshadow
+
+.. figure:: ../_figures/wordpress/wp-install-7.png
+   :alt: WordPress Dashboard
+   :width:     600
+
+   `Figure 7: The WordPress dashboard.`
+
+You are ready to go!
+
+To check out your blog as it is seen by the outside world, enter:
+\http://<your_website>/blog on your browser. Figure 8 shows the result:
+
+.. figure:: ../_figures/wordpress/wp-install-8.png
+   :alt: WordPress blog
+   :width:     600
+
+   `Figure 8: Your WordPress blog.`
+
+**Congratulations, your WordPress blog is up and running!**
+
+You have successfully installed WordPress on a host system.
+
+Add new entries to your blog and share them with the world using |CL|!
+
+*Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries.*
+
+.. _`step-by-step guide`:
+   https://codex.wordpress.org/Installing_WordPress#Famous_5-Minute_Install
+
+.. _`NUC6i5SYH product page`:
+   http://www.intel.com/content/www/us/en/nuc/nuc-kit-nuc6i5syh.html

source/tutorials/wordpress/wp-install.rst

@@ -1,198 +0,0 @@
-.. _wp-install:
-
-Set up WordPress\* on a LAMP web server
-#######################################
-
-This tutorial shows you how to install the WordPress\* components on your |CL|
-:abbr:`LAMP (Linux, Apache, MySQL, PHP)` server. At the end of
-:ref:`web-server-install`, you created a WordPress-ready database using
-phpMyAdmin\* and MariaDB\*. Now that the LAMP server is up and running, you
-can add the WordPress components needed to host a WordPress website on your system.
-
-Before you begin
-****************
-
-This tutorial assumes that you have successfully completed
-:ref:`bare-metal-install-desktop` and you have :ref:`web-server-install`.
-
-Create a WordPress server
-*************************
-
-WordPress can be installed in a variety of ways. These instructions are
-written for users who have followed our instructions for installing phpMyAdmin
-when they :ref:`set up a LAMP web server <web-server-install>`. Note that
-all steps in this tutorial have been tested using a NUC6i5SYH Intel® NUC.
-Visit the `NUC6i5SYH product page`_ for detailed information.
-
-Numerous online articles are available to help you name your website and
-acquire the necessary certificates. Those tasks are beyond the scope of this tutorial.
-
-You can take several actions to harden your website from attacks. The security
-of your website and the data it contains are complex and ever-evolving tasks.
-Prioritize security if you plan to expose your website to the outside world.
-This tutorial does not address security measures that you can take to harden
-your site but we strongly encourage you to take action.
-
-.. note::
-
-   Throughout this tutorial, we reference your website name as <your_website>.
-
-
-Download WordPress and manage directories
-=========================================
-
-For this tutorial, you will create a WordPress blog that can be accessed at:
-\http://<your_website>/blog.
-
-To accomplish this setup, you must add WordPress components to the :file:`/var/www/html/blog`
-directory.
-
-Follow these steps:
-
-
-#. Navigate to the top level of the website’s root directory:
-
-   .. code-block:: bash
-
-      cd /var/www/html
-
-#. Download the latest version of WordPress:
-
-   .. code-block:: bash
-
-      sudo curl -O https://wordpress.org/latest.tar.gz
-
-#. Extract all files and directories from the downloaded file:
-
-   .. code-block:: bash
-
-      sudo tar –xzvf latest.tar.gz
-
-#. Rename the top-level WordPress directory to “blog”:
-
-   .. code-block:: bash
-
-      sudo mv wordpress blog
-
-#. Remove the downloaded tar file:
-
-   .. code-block:: bash
-
-      sudo rm latest.tar.gz
-
-Set up WordPress with web-based GUI
-===================================
-
-Recall that you created a database and user when you installed phpMyAdmin when you
-set up a |CL| based :ref:`web server <web-server-install>`. Next, you must
-connect WordPress to the database and install WordPress.
-
-To continue with the setup, go to: \http://<your_website>/blog/wp-admin/install.php.
-The WordPress language option screen appears, as shown in figure 1.
-
-#. Select :guilabel:`English` and click :guilabel:`Continue`.
-
-   .. figure:: /_figures/wordpress/wp-install-1.png
-      :alt: WordPress language selection
-      :width:     600
-
-      `Figure 1: WordPress language selection screen.`
-
-
-   The WordPress installation continues until the Welcome screen appears, as shown in
-   figure 2:
-
-   .. figure:: /_figures/wordpress/wp-install-2.png
-      :alt: WordPress welcome screen
-      :width:     600
-
-      `Figure 2: WordPress Welcome screen.`
-
-#. Click :guilabel:`Let’s go!`.
-
-#. Enter database connection specifics in the screen that appears, as shown in figure 3
-   below.
-
-   * Database name:       WordPress
-   * Database username:   wordpressuser
-   * Database password:   wp-example  (asterisks will not appear in the text box)
-   * Database host:  localhost
-   * Table prefix:   wp\_
-
-   .. figure:: /_figures/wordpress/wp-install-3.png
-      :alt: Database connection details
-      :width:     600
-
-      `Figure 3: Information necessary for WordPress to connect to the database.`
-
-#. Click :guilabel:`Submit` to complete the setup.
-
-   Figure 4 shows the confirmation screen that verifies a successful setup. WordPress
-   is connected to the MariaDB database.
-
-   .. figure:: /_figures/wordpress/wp-install-4.png
-      :alt: Successful database connection.
-      :width:     600
-
-      `Figure 4: Successful WordPress connection.`
-
-#. Click :guilabel:`Run the install`.
-   The installer runs until WordPress is fully installed on your system.
-
-Complete successful login
-=========================
-
-Once the installation is complete, you can name your blog and create a WordPress username
-and password. See figure 5.
-
-.. figure:: /_figures/wordpress/wp-install-5.png
-   :alt: WordPress user creation
-   :width:     600
-
-   `Figure 5: WordPress site information screen.`
-
-
-#. Enter all required information.
-#. Click :guilabel:`Install WordPress`.
-#. Verify that the initial login screen appears once the installation is complete. See figure 6:
-
-   .. figure:: /_figures/wordpress/wp-install-6.png
-      :alt: WordPress login
-      :width:     600
-
-      `Figure 6: The WordPress login screen.`
-
-#. Enter your WordPress username and password.
-#. Check :guilabel:`Remember me` to save your credentials.
-#. Click :guilabel:`Log in`.
-
-Figure 7 shows the WordPress dashboard after a successful login:
-
-.. figure:: /_figures/wordpress/wp-install-7.png
-   :alt: WordPress Dashboard
-   :width:     600
-
-   `Figure 7: The WordPress dashboard.`
-
-You are ready to go!
-
-To check out your blog as it is seen by the outside world, enter:
-\http://<your_website>/blog on your browser. Figure 8 shows the result:
-
-.. figure:: /_figures/wordpress/wp-install-8.png
-   :alt: WordPress blog
-   :width:     600
-
-   `Figure 8: Your WordPress blog.`
-
-**Congratulations, your WordPress blog is up and running!**
-
-You have successfully installed WordPress on a host system.
-
-Add new entries to your blog and share them with the world using |CL|!
-
-.. _`step-by-step guide`:
-   https://codex.wordpress.org/Installing_WordPress#Famous_5-Minute_Install
-
-.. _`NUC6i5SYH product page`:
-   http://www.intel.com/content/www/us/en/nuc/nuc-kit-nuc6i5syh.html

source/tutorials/zfs.rst

@@ -0,0 +1,250 @@
+.. _zfs:
+
+OpenZFS*
+########
+
+This tutorial shows how to set up `OpenZFS* file system and volume manager`_ on |CL-ATTR|, using a non-root device for zpools. 
+
+.. contents::
+   :local:
+   :depth: 1
+
+Background
+**********
+
+The OpenZFS storage platform provides volume management, snapshot capabilities, and redundancy detection. |CL| **does not** ship with a binary ZFS kernel module (zfs.ko). |CL| users who wish to incorporate the zfs.ko kernel module must build and maintain this work themselves.
+
+.. CAUTION::
+
+   Use of the OpenZFS kernel module in connection with |CL| is neither recommended nor officially endorsed by the Clear Linux* Project.  Users who follow this tutorial and build zfs.ko kernel module are encouraged to seek independent legal counsel regarding any plan to redistribute a software package containing zfs.ko and |CL|. 
+
+Known Issues
+************
+
+Using a long-term-support (LTS) kernel when running OpenZFS reduces the risk of incompatibilities with kernel updates. When new kernels or new versions of OpenZFS are released, users bear the responsibility to test those releases and ensure compatibility before deploying any updates. 
+
+Prerequisites
+*************
+
+* Learn to :ref:`kernel-modules-dkms`
+* Learn to use :ref:`swupd-guide`
+
+Install the DKMS kernel
+=======================
+
+.. include:: ../guides/kernel/kernel-modules-dkms.rst
+   :start-after: kernel-modules-dkms-install-begin-alt:
+   :end-before: kernel-modules-dkms-install-end:
+
+Bundles
+=======
+
+Before you install OpenZFS, install the bundles that contain the build dependencies.
+
+.. code-block:: bash
+
+    sudo swupd bundle-add os-core-dev devpkg-openssl devpkg-util-linux
+
+Install
+*******
+
+Clone OpenZFS code
+==================
+
+
+In this section, you download the source code directly from the `ZFS on Linux repository`_.
+
+.. note::
+   
+   OpenZFS will not run from this location. We copy the source code into a DKMS directory in the following steps. 
+
+#. Create a directory. In this example, we use :file:`/opt/src/zfs`, 
+   but you can choose any workspace you like. 
+
+   .. code-block:: bash
+
+      sudo mkdir -p /tmp/zfs/ 
+
+#. Clone the repository.
+
+   .. code-block:: bash
+
+      git clone https://github.com/openzfs/zfs.git /tmp/zfs
+      sudo cp -Rv /tmp/zfs /opt/src/zfs
+
+Remember where you clone the git repository because you will need it
+when you upgrade ZFS. Do not delete the source location when you have completed this tutorial. You will need it later.
+
+Compile the module
+==================
+
+We will build the module using DKMS. This will enable us to keep the module up to date as new kernels are released in the future.
+
+
+#. Check out the tagged version that you plan to use. As of the time of this
+   writing, the latest release tag is `0.8.4`:
+
+   .. code-block:: bash
+
+      cd /opt/src/zfs
+      git checkout zfs-0.8.4
+
+#. Copy the source code into `/usr/src/zfs-0.8.4`. This exposes the source
+   code to DKMS. We will build the code from the new location:
+
+   .. code-block:: bash
+
+      sudo cp -Rv /opt/src/zfs /usr/src/zfs-0.8.4
+
+#. The ZFS distribution provides a script to build a suitable dkms.conf file.
+   Build dkms.conf and install it into the DKMS tree.
+
+   .. code-block:: bash
+
+      cd /usr/src/zfs-0.8.4
+      sudo scripts/dkms.mkconf -n zfs -v 0.8.4 -f dkms.conf
+      sudo dkms add -m zfs -v 0.8.4
+      sudo dkms build -m zfs -v 0.8.4
+      sudo dkms install -m zfs -v 0.8.4
+
+#. Observe that this install the zfs kernel modules to: 
+   :file:`/usr/lib/modules/<kernel-name>/extra/zfs`
+
+   In addition, this installs the zfs userspace tools to:
+
+   .. code-block:: console
+
+      + /usr/local/
+      |--+ bin/
+         |--zvol_wait
+         |--zgenhostid
+         |--raidz_test
+      |--+ etc/
+         |--+ zfs/
+            |--* zed.d/
+            |--+ zpool.d/
+            |--zfs-functions
+      |--+ include/libzfs/ [contents omitted]
+      |--+ lib/
+      |--+ libexec/
+         |--+ zfs/
+            |-- zpool.d/
+            |-- zed.d
+      |--+ share/zfs/ [contents omitted]
+      |--+ sbin/
+         |--fsck.zfs
+         |--zpool
+         |--zdb
+         |--zed
+         |--zfs
+         |--zhack
+         |--zinject
+         |--zpool
+         |--ztest
+         |--zstreamdump
+      |--+ src/
+         |--+ zfs-0.8.4/
+         |--+ spl-0.8.4/
+
+
+#. Load the new kernel module:
+
+   .. code-block: bash
+
+      sudo modprobe zfs
+
+
+Set up systemd
+==============
+
+You should now have these unit files available. 
+
+.. code-block:: console
+
+   zfs-env-bootfs.service
+   zfs-zed.service
+   zfs-import-cache.service
+   zfs-import-scan.service
+   zfs-mount.service
+   zfs-share.service
+   zfs-volume-wait.service
+  
+OpenZFS requires you to explicitly install and enable the services you want. 
+If you want to use other ZFS service units, you could create symlinks for them, similar to the example below.
+
+To use ZFS automatic zpool import and filesystem mount services, link the systemd.unit files into :file:`/etc` and enable them.
+
+.. code-block:: bash
+
+   sudo ln -s ./etc/systemd/system/zfs-import-cache.service /etc/systemd/system/
+   sudo ln -s ./etc/systemd/system/zfs-mount.service /etc/systemd/system/
+
+   sudo systemctl enable zfs-import-cache
+   sudo systemctl enable zfs-import.target
+
+   sudo systemctl enable zfs-mount
+   sudo systemctl enable zfs.target
+
+
+Load the kernel module at boot
+==============================
+
+OpenZFS kernel modules must be loaded before any OpenZFS filesystems are mounted. For convenience, load the kernel modules at boot.
+
+#. Systemd uses the `/etc/modules-load.d/` directory to load out-of-tree
+   kernel modules. Make sure that the directory exists:
+
+   .. code-block:: bash
+
+      sudo mkdir -p /etc/modules-load.d
+
+#. Create the configuration file:
+
+   .. code-block:: bash
+
+      echo "zfs" | sudo tee /etc/modules-load.d/01-zfs.conf
+
+#. Reboot your system. zfs.ko should be loaded automatically.
+
+.. CAUTION::
+
+   When the |CL| kernel is upgraded, DKMS will attempt to rebuild your OpenZFS module for the new kernel. 
+
+   - DKMS may not have rebuilt the module
+   - DKMS may not have auto-installed the module
+   - The new kernel might introduce breaking changes that prevent zfs
+     from compiling
+
+To fix this situation, you may have to recompile zfs.ko with the new kernel code. OpenZFS *might* not compile at all with the new kernel.
+
+.. CAUTION::
+   
+   **Be sure you don't put anything on an OpenZFS pool that you would need
+   in order to rebuild kernel modules.** You must ensure the compatibility of OpenZFS with new Linux kernels when they are released.
+
+Troubleshooting
+===============
+
+If you suspect an issue with DKMS rebuilding your module, you can check two places for information. The dkms-new-kernel service will show status that may help in troubleshooting:
+
+.. code-block:: bash
+
+   systemctl status dkms-new-kernel.service
+
+Also, the systemd journal may have important information:
+
+.. code-block:: bash
+
+   journalctl -xe
+
+Next steps
+**********
+You are ready to create zpools and datasets! For more information on using ZFS, see:
+
+* `FreeBSD Handbook chapter on ZFS`_
+* `ZFS-on-Linux issue tracker`_
+
+.. _FreeBSD Handbook chapter on ZFS: https://www.freebsd.org/doc/handbook/zfs.html
+.. _ZFS-on-Linux issue tracker: https://github.com/openzfs/zfs/issues/10068
+.. _ZFS on Linux repository: https://github.com/openzfs/zfs
+.. _OpenZFS* file system and volume manager: https://github.com/openzfs/zfs

tox.ini

@@ -0,0 +1,13 @@
+[tox]
+envlist = py3-{mylinux,mywindows}
+
+[testenv]
+platform = mylinux: linux
+	mywindows: win32
+whitelist_externals = make.bat
+	/usr/bin/make
+deps = -rrequirements.txt
+commands = 
+	mylinux: make {posargs}
+	mywindows: make.bat {posargs}
+