Adds manpages to reference section of Clear Linux docs (#1110)

* Build manpages with "make man".

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Adds "make man" to github workflows.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Remove shell script dependency.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Change git URLs to https.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Moved git clone to Makefile.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Fixed typo.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Adding pandoc install.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Moved to Makefile.

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* First commit toward programmatic creation of manpages.rst

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Multiple updates:

1. man-pages.rst now generated by script.
2. Updated Makefile and make.bat with "man" and "clean-man" recipes.
3. Fixed typo in conf.py.
4. Many improvements to manpages.py

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

* Updated readme for man pages and added man-pages.rst

Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>

.github/workflows/check.yml

@@ -13,11 +13,12 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
         

.github/workflows/main.yml

@@ -15,12 +15,13 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
     - name: Pre-deploy
       run: |

.github/workflows/publish.yml

@@ -15,12 +15,13 @@ jobs:
     - name: Install dependencies
       run: |
         export PATH="$HOME/.local/bin:$PATH"
-        sudo apt-get install -y python3-setuptools
+        sudo apt-get install -y python3-setuptools pandoc
         pip3 install --user -r requirements.txt
     - name: Build the docs
       run: |
         export PATH="$HOME/.local/bin:$PATH"
         make py
+        make man
         make htmlall
     - name: Pre-deploy
       run: |

.gitignore

@@ -17,3 +17,9 @@ bundles.html.txt
 
 # ignore the venv, used for running make py
 venv
+
+# ignore artifacts of man page generation
+source/_scripts/_python/manpages/*.rst
+source/reference/manpages
+source/_scripts/_python/manpages/*/
+source/reference/man-pages.rst

Makefile

@@ -25,6 +25,12 @@ linkcheck:
 py:
 	make -C source py
 
+man:
+	make -C source man
+
+clean-man:
+	make -C source clean-man
+
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"

README.rst

@@ -63,8 +63,13 @@ running ``make html``:
 Open one of the HTML pages found in ``source/_build/html`` in a web browser
 to view the rendered documentation.
 
-If you want to build the documentation exactly as seen on the website, use
-``make py`` followed by ``make htmlall``. This builds some
+This build will generate several warnings as there are two other optional make commands required to build the full documentation.
+
+1. ``make py`` to generate the bundle reference material.
+2. ``make man`` to generate man page reference material.
+
+To build the documentation exactly as seen on the website, use
+``make man``, ``make py``, and ``make htmlall``. This builds both
 external dependencies and all supported languages.
 
 Use virtualenv 
@@ -163,6 +168,9 @@ build before building again by running ``make clean``:
 This will completely remove the previous build output, including artifacts 
 from the `make venv` target when done outside an active venv.
 
+Before running ``make man``, please run ``make clean-man`` to clear out any
+previous attempts.
+
 Convenience script
 ==================
 

make.bat

@@ -5,7 +5,7 @@ REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
-set BUILDDIR=build
+set BUILDDIR=source\_build
 set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source
 set I18NSPHINXOPTS=%SPHINXOPTS% source
 if NOT "%PAPER%" == "" (
@@ -30,6 +30,7 @@ if "%1" == "help" (
 	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
 	echo.  text       to make text files
 	echo.  man        to make manual pages
+	echo.  clean-man  to clean up after man page generation
 	echo.  texinfo    to make Texinfo files
 	echo.  gettext    to make PO message catalogs
 	echo.  changes    to make an overview over all changed/added/deprecated items
@@ -186,10 +187,14 @@ if "%1" == "text" (
 )
 
 if "%1" == "man" (
-	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+    cd source/_scripts/_python/manpages
+    man.bat man
+	goto end
+)
+
+if "%1" == "clean-man" (
+	cd source/_scripts/_python/manpages
+	man.bat clean-man
 	goto end
 )
 

source/Makefile

@@ -50,6 +50,8 @@ help:
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  coverage   to run coverage check of the documentation (if enabled)"
 	@echo "  py         to trigger an update of bundle content"
+	@echo "  man        to pull and create manpage rst files in reference section"
+	@echo "  clean-man  to clean up manpage generation"
 
 clean:
 	rm -rf $(BUILDDIR)/*
@@ -210,3 +212,9 @@ pseudoxml:
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+man:
+	$(MAKE) -C $(SCRIPTDIR)/manpages man
+
+clean-man:
+	$(MAKE) -C $(SCRIPTDIR)/manpages clean-man

source/_scripts/_python/manpages/Makefile

@@ -0,0 +1,25 @@
+man:
+	git clone https://github.com/clearlinux/clr-man-pages.git
+	git clone https://github.com/clearlinux/clr-power-tweaks.git
+	git clone https://github.com/clearlinux/clrtrust.git
+	git clone https://github.com/clearlinux/mixer-tools.git
+	git clone https://github.com/clearlinux/swupd-client.git
+	git clone https://github.com/clearlinux/telemetrics-client.git
+	git clone https://github.com/clearlinux/tallow.git
+	git clone https://github.com/clearlinux/micro-config-drive.git
+	python3 manpages.py
+	- mkdir ../../../reference/manpages
+	cp *.rst ../../../reference/manpages
+
+clean-man:
+	- rm -rf clr-man-pages
+	- rm -rf clr-power-tweaks
+	- rm -rf clrtrust
+	- rm -rf mixer-tools
+	- rm -rf swupd-client
+	- rm -rf telemetrics-client
+	- rm -rf tallow
+	- rm -rf micro-config-drive
+	- rm *.rst
+	- rm ../../../reference/manpages/*.rst
+	- rm ../../../reference/man-pages.rst

source/_scripts/_python/manpages/man.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+REM Command file for building man pages
+
+if "%1" == "man" (
+    git clone https://github.com/clearlinux/clr-man-pages.git
+    git clone https://github.com/clearlinux/clr-power-tweaks.git
+    git clone https://github.com/clearlinux/clrtrust.git
+    git clone https://github.com/clearlinux/mixer-tools.git
+    git clone https://github.com/clearlinux/swupd-client.git
+    git clone https://github.com/clearlinux/telemetrics-client.git
+    git clone https://github.com/clearlinux/tallow.git
+    git clone https://github.com/clearlinux/micro-config-drive.git
+    python.exe manpages.py
+    mkdir ..\..\..\reference\manpages
+    copy *.rst ..\..\..\reference\manpages
+    goto end
+)
+
+if "%1" == "clean-man" (
+    rmdir /q /s clr-man-pages
+    rmdir /q /s clr-power-tweaks
+    rmdir /q /s clrtrust
+    rmdir /q /s mixer-tools
+    rmdir /q /s swupd-client
+    rmdir /q /s telemetrics-client
+    rmdir /q /s tallow
+    rmdir /q /s micro-config-drive
+    del *.rst
+    del ..\..\..\reference\manpages\*.rst
+    del ..\..\..\reference\man-pages.rst
+    goto end
+)
+
+:end

source/_scripts/_python/manpages/manpages.py

@@ -0,0 +1,196 @@
+#
+# manpages.py
+#
+# maintainer: intelkevinputnam
+#
+# usage: python3 manpages.py
+#
+# dependencies: 1. clone of Clear Linux documentation https://github.com/clearlinux/clear-linux-documentation
+#               2. Makefile or make.bat to create directories and clone repositories (``make man`` to generate. ``make clean-man`` to clean up.)
+#
+# This script does 4 things:
+#
+# 1. Find and move reStructuredText versions of man pages (looks for manName.sectionNumber.fileExtension: mixer.1.rst) to reference/manpages directory of Clear Linux docs.
+# 2. Massage markdown and reStructuredText man pages into a normalized format.
+# 3. Add cross reference links wherever man pages reference each other.
+# 4. Create manpages.rst in reference directory (already included in toctree of reference/index.rst)
+#
+#
+
+import sys
+import re
+import subprocess
+from os import listdir
+from os.path import join, isfile
+from shutil import copyfile
+
+paths = ["clr-man-pages","clr-power-tweaks/man","clrtrust/man","micro-config-drive/docs/","mixer-tools/docs","swupd-client/docs","tallow/man","telemetrics-client/docs/man"]
+pathToRefSection = "../../../reference/"
+manPageRegex = '.[0-9]+.rst'
+mdManPageRegex = '.[0-9]+.md'
+mdBoldItalicRegex = '\\*\\*\\`[a-z-._]+\\`\\*\\*'
+manFiles = []
+manGroups = {}
+manNamePerms = {}
+seeAlsoRefs = {}
+gitHubGroup = "https://github.com/clearlinux/"
+TOC = "\n.. contents::\n    :local:\n"
+manPageRest = """.. _man-pages:
+
+Man pages
+#########
+
+These pages are generated from `Clear Linux\\* tool
+repositories <https://github.com/clearlinux>`__. Updated |today|.
+
+
+"""
+manTOC = """.. toctree::
+   :maxdepth: 1
+
+"""
+
+def getPages(paths):
+    for path in paths:
+        manGroups[path] = []
+        files = listdir(path)
+        for file in files:
+            if file.endswith(".rst"):
+                regex_found = re.search(manPageRegex,file)
+                if regex_found:
+                    filePath = join(path,file)
+                    copyfile(filePath,file)
+                    manFiles.append(file)
+                    manGroups[path].append(file)
+                    addManNamePermutations(file)
+            elif file.endswith(".md"):
+                regex_found = re.search(mdManPageRegex,file)
+                if regex_found:
+                    filePath = join(path,file)
+                    rstFileName = processMDFile(file,filePath)
+                    manFiles.append(rstFileName)
+                    manGroups[path].append(rstFileName)
+                    addManNamePermutations(rstFileName)
+
+def processMDFile(fileName,filePath):
+    manName = fileName.rstrip('.md')
+    manSection = manName.split('.')[-1]
+    manName = manName.rstrip('.' + manSection)
+    lines = []
+    headerInsert = ""
+    with open(filePath,'r') as m:
+        lines = m.readlines()
+        if "SYNOPSIS" in lines[0]:
+                header='='*len(manName)
+                lines.insert(0,"# " + manName + "\n\n:Manual section: "+manSection+"\n\n")
+        index = 0
+        first = True
+        for line in lines:
+            newLine = line.replace("**`","`") #Fix some markdown formatting weirdness that doesn't translate to reST
+            newLine = newLine.replace("`**","`")
+            if newLine.startswith("#") and not newLine.startswith("##"): #Fix if all headers are first level
+                if first:
+                    first = False
+                else:
+                    newLine = newLine.replace("#","##")
+            if "===" in newLine: #fix rst style header that is actually description in some docs
+                newLine = ""
+                headerInsert = manName + "\n" + '='*len(manName) + "\n\n"
+            lines[index] = newLine
+            index += 1
+        lines.insert(0,headerInsert)
+
+    with open(filePath,'w') as md:
+        md.writelines(lines)
+        rstFilePath = filePath.replace(".md",".rst")
+        rstFileName = fileName.replace(".md",".rst")
+
+    command = "pandoc " +  filePath + " -o " + rstFilePath
+    subprocess.run(command, shell=True)
+    if isfile(rstFilePath):
+        copyfile(rstFilePath,rstFileName)
+    return rstFileName
+
+def addManNamePermutations(fileName): #Based on all the syntactical permutations of references to man pages in the documents.
+    (manName,subsection) = getNameAndSubsection(fileName)
+    manNames = []
+    manNames.append("``" + manName + "``\\(" + subsection +")") # ``mixer.init``\(1)
+    manNames.append("``" + manName + "``\\ (" + subsection +")") # ``mixer.init``\ (1)
+    manNames.append("**``" + manName + "(" + subsection + ")``**")  # **`mixer.init(1)`**
+    manNames.append("``" + manName + "(" + subsection + ")``") # ``mixer.init(1)``
+    manNames.append("**" + manName + "(" + subsection + ")**") # **mixer.init(1)**
+    manNames.append("`" + manName + "(" + subsection + ")`") # `mixer.init(1)`
+    manNamePerms[(manName,subsection)] = manNames
+
+def getNameAndSubsection(manFileName):
+    manName = manFileName.rstrip('.rst')
+    manSection = manName.split('.')[-1]
+    manName = manName.rstrip('.'+manSection)
+    return (manName,manSection)
+
+def linkToMan(manName,manSection):
+    return "`" + manName + "(" + manSection + ") <" + manName + "." + manSection + ".html>`__"
+ 
+def buildManName(name,section):
+    return name + "(" + section + ")"
+
+def updateManPages():
+    #makeSeeAlsoReplacements()
+    for file in manFiles:
+        manFile = ""
+        with open(file,'r') as f:
+            manFile = f.read()
+            #manFile = addTOC(manFile,file) # Not convinced adding TOC adds value.
+            #
+            # Add linked cross referencing for all manpages discovered.
+            # 1. the manNamePerms dictionary is created once all of the man page source files are discovered
+            # 2. Each document is checked for each permutation. 
+            # 3. When a permuation is found it is replaced with a normalized version
+            # 4. Once normalized it is turned into a reStructuredText link (def linkToMan)
+            # 
+            for nameAndSection, listOfPerms in manNamePerms.items():
+                for perm in listOfPerms:
+                    manFile = manFile.replace(perm,buildManName(nameAndSection[0],nameAndSection[1]))
+            for manName, doNotUse in manNamePerms.items():
+                manFile = manFile.replace(buildManName(manName[0],manName[1]),linkToMan(manName[0],manName[1]))
+        with open(file,'w') as w:
+            w.write(manFile)
+
+def addTOC(manContent,file):
+# Find the first instance of the manpage Name
+# Skip the underline of the header
+# Add a new line
+# Add contents directive with local modifier
+# Add a new line
+    manSectionMeta = ":Manual section:"
+    manContentLines = manContent.split('\n')
+    index = 0
+    for line in manContentLines:
+        if manSectionMeta in line:
+            manContentLines.insert(index + 2,TOC)
+            break
+        index += 1
+    output = ""
+    for line in manContentLines:
+        output = output + line + "\n"
+    return output
+
+def createManpagesRST():
+    filePath = join(pathToRefSection,"man-pages.rst")
+    with open(filePath,'w') as f:
+        manGrouping = manPageRest
+        for path, fileList in manGroups.items():
+            repoName = path.split("/")[0]
+            repoLink = gitHubGroup + repoName
+            repoReST = "`" + repoName + " <" + repoLink + ">`__"
+            manGrouping += repoReST + "\n"
+            manGrouping += "="*len(repoReST) + "\n\n"
+            manGrouping += manTOC
+            for file in fileList:
+                manGrouping += "   manpages/" + file + "\n"
+            manGrouping += "\n"
+        f.write(manGrouping)
+
+getPages(paths)
+updateManPages()
+createManpagesRST()

source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css

@@ -306,9 +306,9 @@ button:hover a.headerlink:after {
 /*End support for custom Clear Linux header*/
 
 /*Adds a bit of spacing after the last paragraph in a bulleted list*/
-.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child {
+/*.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child {
     margin-bottom: 10px;
-}
+}*/
 
 div.admonition ul {
   margin-top: 20px;

source/conf.py

@@ -81,7 +81,7 @@ language = None
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build','_themes']
+exclude_patterns = ['_build','_themes','_scripts']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.

source/reference/index.rst

@@ -13,3 +13,4 @@ features.
    bundles/bundles
    system-requirements
    image-types
+   man-pages