Refresh SPDX license list to latest (version 3.2)

Make the SPDX puller code more robust by reading from JSON sources
rather than parsing HTML. Also, keep all deprecated license identifiers
for now, since they are still present in the spdx list.

Four identifiers were removed since the last refresh, but we are keeping
them for backwards compatibility. The removed license IDs now reside in
the licenses-extra file. At

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

Makefile

@@ -17,5 +17,9 @@ update:
 	curl -f -o packages ${BASE_URL}/source/package-sources
 	cut -f1 packages | LC_ALL=C sort > packages.new && mv packages.new packages
 
+.PHONY: spdx
 spdx:
-	curl -s spdx.org/licenses/ | sed '0,/<tbody>/d;/<\/tbody>/q;s/<tr>/\f/g;s/$$/,/g;s/<[^>]*>//g' | awk 'BEGIN{RS="\f";FS=","} {print $$4}' | sed '/^$$/d;s/^[ ]*//' > spdx
+	JSON=$$(mktemp); \
+	trap "rm $$JSON" EXIT; \
+	curl -f -S -s https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json > $$JSON || exit 1; \
+	jq -r '.licenses[] | .licenseId' < $$JSON | LC_COLLATE=C sort > licenses-spdx

Makefile.common

@@ -45,7 +45,7 @@ upstream:
 #help or that it is in the exceptions list.
 spdxcheck: $(SPECFILE)
 	@for LICENSE in `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{LICENSE}\n' $(SPECFILE)`; do \
-		if ! grep -q "^$$LICENSE$$" $(TOPLVL)/projects/common/licenses-spdx && ! grep -q "$$LICENSE$$" $(TOPLVL)/projects/common/licenses-exceptions; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
+		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"
 
 #help cvecheck: Checks

licenses-extra

@@ -0,0 +1,7 @@
+# This file contains SPDX license identifiers that are not defined in the
+# latest version of SPDX but still needed (or possibly needed) for Clear Linux
+# packages.
+GFDL-1.3+
+MIT-Opengroup
+WXwindows
+w3c

licenses-spdx

@@ -1,3 +1,4 @@
+0BSD
 AAL
 ADSL
 AFL-1.1
@@ -6,7 +7,11 @@ AFL-2.0
 AFL-2.1
 AFL-3.0
 AGPL-1.0
+AGPL-1.0-only
+AGPL-1.0-or-later
 AGPL-3.0
+AGPL-3.0-only
+AGPL-3.0-or-later
 AMDPLPA
 AML
 AMPAS
@@ -29,16 +34,22 @@ Artistic-1.0
 Artistic-1.0-Perl
 Artistic-1.0-cl8
 Artistic-2.0
+BSD-1-Clause
 BSD-2-Clause
 BSD-2-Clause-FreeBSD
 BSD-2-Clause-NetBSD
+BSD-2-Clause-Patent
 BSD-3-Clause
 BSD-3-Clause-Attribution
 BSD-3-Clause-Clear
 BSD-3-Clause-LBNL
+BSD-3-Clause-No-Nuclear-License
+BSD-3-Clause-No-Nuclear-License-2014
+BSD-3-Clause-No-Nuclear-Warranty
 BSD-4-Clause
 BSD-4-Clause-UC
 BSD-Protection
+BSD-Source-Code
 BSL-1.0
 Bahyph
 Barr
@@ -80,11 +91,15 @@ CC-BY-SA-4.0
 CC0-1.0
 CDDL-1.0
 CDDL-1.1
+CDLA-Permissive-1.0
+CDLA-Sharing-1.0
 CECILL-1.0
 CECILL-1.1
 CECILL-2.0
+CECILL-2.1
 CECILL-B
 CECILL-C
+CNRI-Jython
 CNRI-Python
 CNRI-Python-GPL-Compatible
 CPAL-1.0
@@ -95,6 +110,7 @@ Caldera
 ClArtistic
 Condor-1.1
 Crossword
+CrystalStacker
 Cube
 D-FSL-1.0
 DOC
@@ -105,12 +121,15 @@ ECL-2.0
 EFL-1.0
 EFL-2.0
 EPL-1.0
+EPL-2.0
 EUDatagrid
 EUPL-1.0
 EUPL-1.1
+EUPL-1.2
 Entessa
 ErlPL-1.1
 Eurosym
+FSFAP
 FSFUL
 FSFULLR
 FTL
@@ -118,14 +137,23 @@ Fair
 Frameworx-1.0
 FreeImage
 GFDL-1.1
+GFDL-1.1-only
+GFDL-1.1-or-later
 GFDL-1.2
+GFDL-1.2-only
+GFDL-1.2-or-later
 GFDL-1.3
-GFDL-1.3+
+GFDL-1.3-only
+GFDL-1.3-or-later
 GL2PS
 GPL-1.0
 GPL-1.0+
+GPL-1.0-only
+GPL-1.0-or-later
 GPL-2.0
 GPL-2.0+
+GPL-2.0-only
+GPL-2.0-or-later
 GPL-2.0-with-GCC-exception
 GPL-2.0-with-autoconf-exception
 GPL-2.0-with-bison-exception
@@ -133,6 +161,8 @@ GPL-2.0-with-classpath-exception
 GPL-2.0-with-font-exception
 GPL-3.0
 GPL-3.0+
+GPL-3.0-only
+GPL-3.0-or-later
 GPL-3.0-with-GCC-exception
 GPL-3.0-with-autoconf-exception
 Giftware
@@ -148,18 +178,27 @@ IPL-1.0
 ISC
 ImageMagick
 Imlib2
+Info-ZIP
 Intel
 Intel-ACPI
+Interbase-1.0
 JSON
 JasPer-2.0
 LAL-1.2
 LAL-1.3
 LGPL-2.0
 LGPL-2.0+
+LGPL-2.0-only
+LGPL-2.0-or-later
 LGPL-2.1
 LGPL-2.1+
+LGPL-2.1-only
+LGPL-2.1-or-later
 LGPL-3.0
 LGPL-3.0+
+LGPL-3.0-only
+LGPL-3.0-or-later
+LGPLLR
 LPL-1.0
 LPL-1.02
 LPPL-1.0
@@ -169,10 +208,14 @@ LPPL-1.3a
 LPPL-1.3c
 Latex2e
 Leptonica
+LiLiQ-P-1.1
+LiLiQ-R-1.1
+LiLiQ-Rplus-1.1
 Libpng
+Linux-OpenIB
 MIT
+MIT-0
 MIT-CMU
-MIT-Opengroup
 MIT-advertising
 MIT-enna
 MIT-feh
@@ -193,6 +236,7 @@ NASA-1.3
 NBPL-1.0
 NCSA
 NGPL
+NLOD-1.0
 NLPL
 NOSL
 NPL-1.0
@@ -201,12 +245,15 @@ NPOSL-3.0
 NRL
 NTP
 Naumen
+Net-SNMP
 NetCDF
 Newsletr
 Nokia
 Noweb
 Nunit
+OCCT-PL
 OCLC-2.0
+ODC-By-1.0
 ODbL-1.0
 OFL-1.0
 OFL-1.1
@@ -229,6 +276,7 @@ OLDAP-2.7
 OLDAP-2.8
 OML
 OPL-1.0
+OSET-PL-2.1
 OSL-1.0
 OSL-1.1
 OSL-2.0
@@ -247,6 +295,7 @@ RHeCos-1.1
 RPL-1.1
 RPL-1.5
 RPSL-1.0
+RSA-MD
 RSCPL
 Rdisc
 Ruby
@@ -258,18 +307,29 @@ SGI-B-2.0
 SISSL
 SISSL-1.2
 SMLNJ
+SMPPL
 SNIA
 SPL-1.0
 SWL
 Saxpath
+Sendmail
 SimPL-2.0
 Sleepycat
+Spencer-86
+Spencer-94
+Spencer-99
 StandardML-NJ
 SugarCRM-1.1.3
 TCL
+TCP-wrappers
 TMate
 TORQUE-1.1
 TOSL
+TU-Berlin-1.0
+TU-Berlin-2.0
+UPL-1.0
+Unicode-DFS-2015
+Unicode-DFS-2016
 Unicode-TOU
 Unlicense
 VOSTROM
@@ -277,8 +337,8 @@ VSL-1.0
 Vim
 W3C
 W3C-19980720
+W3C-20150513
 WTFPL
-WXwindows
 Watcom-1.0
 Wsuipa
 X11
@@ -298,6 +358,7 @@ Zimbra-1.4
 Zlib
 bzip2-1.0.5
 bzip2-1.0.6
+curl
 diffmark
 dvipdfm
 eCos-2.0
@@ -309,7 +370,7 @@ libtiff
 mpich2
 psfrag
 psutils
-w3c
+wxWindows
 xinetd
 xpp
 zlib-acknowledgement