add test inspector as separate project and add command line option to enable it on demand

test/common/make-testinsp

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# prepare testinspector
+GITHUB_REPO='testinsp'
+SUBDIR='testinsp'
+COMMON=$(dirname $0)
+V="${V-0}" # default to friendly messages
+
+set -eu
+cd $COMMON
+. ./git-utils.sh
+
+if [ ! -e "$COMMON/$SUBDIR" ]; then
+    git clone https://github.com/jscotka/testinsp.git $SUBDIR
+else
+    echo "$SUBDIR/ already exists, skipping"
+fi

test/common/pywrap

@@ -19,9 +19,11 @@ top_srcdir="${realpath%/*}/../.."
 
 # Check out the bots if required
 test -d "${top_srcdir}/bots" || "${top_srcdir}/test/common/make-bots"
+# Check out the test inspector if required
+test -d "$(dirname $realpath)/testinsp" || "$(dirname $realpath)/make-testinsp"
 
 # Prepend the path
-PYTHONPATH="${top_srcdir}/test/common:${top_srcdir}/bots:${top_srcdir}/bots/machine${PYTHONPATH:+:${PYTHONPATH}}"
+PYTHONPATH="${top_srcdir}/test/common/testinsp:${top_srcdir}/test/common:${top_srcdir}/bots:${top_srcdir}/bots/machine${PYTHONPATH:+:${PYTHONPATH}}"
 export PYTHONPATH
 
 # Run the script

test/common/run-tests

@@ -183,7 +183,18 @@ class Test:
             for line in lines[:-1]:
                 if line.startswith(b"WARNING:"):
                     write_line(line)
+            # print test inspector log
+            ti_start = False
+            for line in lines:
+                if b"RESULTS OF TEST INSPECTOR" in line:
+                    ti_start = True
+                    continue
+                if b"END OF TEST INSPECTOR" in line:
+                    break
+                if ti_start:
+                    write_line(b"WARNING: TI -> " + line)
             write_line(lines[-1])
+
         else:
             for line in lines:
                 write_line(line)

test/common/testlib.py

@@ -38,11 +38,13 @@
 import unittest
 from time import sleep
 from typing import Any, Callable, Dict, List, Optional, Union
+from pprint import pprint
 
 import cdp
 import testvm
 from lcov import write_lcov
 from lib.constants import OSTREE_IMAGES
+from testinsp import RunChecks
 
 try:
     from PIL import Image
@@ -1516,6 +1518,29 @@ def nonDestructiveSetup(self):
 
         m = self.machine
 
+        # run TestInspector as last cleanup action
+        def test_inspector_check():
+            print(" ---------- RESULTS OF TEST INSPECTOR ---------- ")
+            inspected_data = self.test_inspecor.check()
+            for k, v in inspected_data.items():
+                if v:
+                    pprint(f">> Test Inspector FAIL - ({k}):")
+                    pprint(v)
+            print(" ---------- END OF TEST INSPECTOR ---------- ")
+        exclude_dict=dict()
+        exclude_dict["ListEtcDir"] = ["/etc/systemd/system/cockpit.service.d/notls.conf"]
+        exclude_dict["ServiceInfo"] = ["cockpit",
+                                       "user@",
+                                       "packagekit",
+                                       "rhsm",
+                                       "realmd",
+                                       "systemd-timedated",
+                                       "systemd-hostnamed",
+                                       "systemd-logind.service"]
+        self.test_inspecor = RunChecks(external_executor=m.execute, exclude_dict=exclude_dict)
+        self.test_inspecor.init()
+        self.addCleanup(test_inspector_check)
+
         # helps with mapping journal output to particular tests
         name = "%s.%s" % (self.__class__.__name__, self._testMethodName)
         m.execute("logger -p user.info 'COCKPITTEST: start %s'" % name)