ci: Run multiple fpgas per job

This consolidates the different FPGA runs into a single job.

Signed-off-by: Steven Koo <steven.koo@ni.com>

4 files changed, 80 insertions, 102 deletions

diff --git a/.ci/templates/job-uhd-devtest-rhombus.yml b/.ci/templates/job-uhd-devtest-rhombus.yml
index a05f47644..8e4c3ba9e 100644
--- a/.ci/templates/job-uhd-devtest-rhombus.yml
+++ b/.ci/templates/job-uhd-devtest-rhombus.yml
@@ -20,135 +20,69 @@ jobs:
     uhdArtifactSource: '${{ parameters.uhdArtifactSource }}'
     redisHost: 'sdr-rhombus'
     dutMatrix:
-      rhombus-x300-UBX-0 XG:
+      rhombus-x300-UBX-0:
         devType: 'x300'
         devModel: 'x300'
         devName: 'rhombus-x300-UBX-0'
         devSerial: '30A6019'
         devBus: 'ip'
         devAddr: '192.168.40.2'
-        devFpga: 'XG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '2516350A6019'
         jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-UBX-0 XG:
+      rhombus-x310-UBX-0:
         devType: 'x300'
         devModel: 'x310'
         devName: 'rhombus-x310-UBX-0'
         devSerial: '3138EF5'
         devBus: 'ip'
         devAddr: '192.168.40.3'
-        devFpga: 'XG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '251635138E98'
         jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-CBX-0 XG:
+      rhombus-x310-CBX-0:
         devType: 'x300'
         devModel: 'x310'
         devName: 'rhombus-x310-CBX-0'
         devSerial: '30796C2'
         devBus: 'ip'
         devAddr: '192.168.40.4'
-        devFpga: 'XG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '2516350796C2'
         jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-WBX-0 XG:
+      rhombus-x310-WBX-0:
         devType: 'x300'
         devModel: 'x310'
         devName: 'rhombus-x310-WBX-0'
         devSerial: '30C5BFF'
         devBus: 'ip'
         devAddr: '192.168.40.5'
-        devFpga: 'XG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '2516350C5BFF'
         jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-TWINRX-0 XG:
+      rhombus-x310-TWINRX-0:
         devType: 'x300'
         devModel: 'x310'
         devName: rhombus-x310-TWINRX-0
         devSerial: 'F43D13'
         devBus: 'ip'
         devAddr: '192.168.40.6'
-        devFpga: 'XG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '251635F43D13'
         jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x300-SBX-0 XG:
+      rhombus-x300-SBX-0:
         devType: 'x300'
         devModel: 'x300'
         devName: rhombus-x300-SBX-0
         devSerial: '32244AD'
         devBus: 'ip'
         devAddr: '192.168.40.7'
-        devFpga: 'XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516352244AD'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x300-UBX-0 HG:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: 'rhombus-x300-UBX-0'
-        devSerial: '30A6019'
-        devBus: 'ip'
-        devAddr: '192.168.40.2'
-        devFpga: 'HG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350A6019'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-UBX-0 HG:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-UBX-0'
-        devSerial: '3138EF5'
-        devBus: 'ip'
-        devAddr: '192.168.40.3'
-        devFpga: 'HG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635138E98'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-CBX-0 HG:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-CBX-0'
-        devSerial: '30796C2'
-        devBus: 'ip'
-        devAddr: '192.168.40.4'
-        devFpga: 'HG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350796C2'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-WBX-0 HG:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-WBX-0'
-        devSerial: '30C5BFF'
-        devBus: 'ip'
-        devAddr: '192.168.40.5'
-        devFpga: 'HG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350C5BFF'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-TWINRX-0 HG:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: rhombus-x310-TWINRX-0
-        devSerial: 'F43D13'
-        devBus: 'ip'
-        devAddr: '192.168.40.6'
-        devFpga: 'HG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635F43D13'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x300-SBX-0 HG:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: rhombus-x300-SBX-0
-        devSerial: '32244AD'
-        devBus: 'ip'
-        devAddr: '192.168.40.7'
-        devFpga: 'HG'
+        devFpga: 'HG,XG'
         devtestPattern: 'x3x0'
         jtagSerial: '2516352244AD'
         jtagServer: 'nitest@sdr-rhombus'
diff --git a/.ci/templates/job-uhd-devtest.yml b/.ci/templates/job-uhd-devtest.yml
index 2febdbbcf..033ab6a51 100644
--- a/.ci/templates/job-uhd-devtest.yml
+++ b/.ci/templates/job-uhd-devtest.yml
@@ -101,7 +101,8 @@ jobs:
       export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
       export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
       python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
-        --jtag_x3xx $(jtagServer),$(jtagSerial),$(Build.BinariesDirectory)/uhddev/build/fpga_images/usrp_$(devModel)_fpga_$(devFpga).bit \
+        --jtag_x3xx $(jtagServer),$(jtagSerial),$(devModel),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
+        --fpgas $(devFpga) \
         ${{ parameters.redisHost }} $(devName) \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
         "python3 ${{ parameters.uhdSrcDir }}/host/tests/devtest/run_testsuite.py \
@@ -126,7 +127,17 @@ jobs:
         $(Common.TestResultsDirectory)/devtest/devtestresults.xml
     continueOnError: true
     displayName: Format devtest xml
-    condition: always()
+    condition: and(always(), eq(variables.devType, 'b200'))
+
+  - script: |
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 ${{ parameters.uhdSrcDir }}/.ci/utils/format_devtest_junitxml.py \
+        --fpgas $(devFpga) \
+        $(Common.TestResultsDirectory)/devtest \
+        $(Common.TestResultsDirectory)/devtest/devtestresults.xml
+    continueOnError: true
+    displayName: Format devtest xml
+    condition: and(always(), ne(variables.devType, 'b200'))
 
   - publish: $(Common.TestResultsDirectory)
     artifact: test-logs-devtest-$(devName)-$(devFpga)-run$(System.JobAttempt)
@@ -137,7 +148,7 @@ jobs:
     inputs:
       testResultsFormat: 'JUnit'
       testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml'
-      testRunTitle: $(devName) $(devFpga) devtest
+      testRunTitle: $(devName) devtest
       buildConfiguration: 'Release'
       mergeTestResults: true
       failTaskOnFailedTests: true
diff --git a/.ci/utils/format_devtest_junitxml.py b/.ci/utils/format_devtest_junitxml.py
index b941a64e6..39b4d13ed 100644
--- a/.ci/utils/format_devtest_junitxml.py
+++ b/.ci/utils/format_devtest_junitxml.py
@@ -8,15 +8,32 @@ class ClassNameTestCase(TestCase):
 parser = argparse.ArgumentParser()
 parser.add_argument("search_path")
 parser.add_argument("output_name")
+parser.add_argument("--fpgas")
 args = parser.parse_args()
 
-xml = JUnitXml()
-for file in glob.glob(args.search_path + "/**/*.xml", recursive=True):
-    xml += JUnitXml.fromfile(file)
+if args.fpgas:
+    return_xml = JUnitXml()
+    for fpga in args.fpgas.split(','):
+        xml = JUnitXml()
+        for file in glob.glob(args.search_path + "/" + fpga + "/**/*.xml", recursive=True):
+            xml += JUnitXml.fromfile(file)
+        for suite in xml:
+            for case in suite:
+                classname_case = ClassNameTestCase.fromelem(case)
+                if classname_case.name == 'test_all':
+                        classname_case.name = fpga + " " + classname_case.classname
+        return_xml += xml
+    return_xml.write(args.output_name)
 
-for suite in xml:
-    for case in suite:
-        classname_case = ClassNameTestCase.fromelem(case)
-        if classname_case.name == 'test_all':
-            classname_case.name = classname_case.classname
-xml.write(args.output_name)
+else:
+    return_xml = JUnitXml()
+    for file in glob.glob(args.search_path + "/**/*.xml", recursive=True):
+        return_xml += JUnitXml.fromfile(file)
+
+    for suite in return_xml:
+        for case in suite:
+            classname_case = ClassNameTestCase.fromelem(case)
+            if classname_case.name == 'test_all':
+                    classname_case.name = classname_case.classname
+
+    return_xml.write(args.output_name)
diff --git a/.ci/utils/mutex_hardware.py b/.ci/utils/mutex_hardware.py
index 5e16c86fb..4c05bf50b 100644
--- a/.ci/utils/mutex_hardware.py
+++ b/.ci/utils/mutex_hardware.py
@@ -15,12 +15,12 @@ from fabric import Connection
 from pottery import Redlock
 from redis import Redis
 
+bitfile_name = "usrp_{}_fpga_{}.bit"
 
-def jtag_x3xx(jtag_args, redis_server):
+def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server):
     remote_working_dir = "pipeline_fpga"
     vivado_program_jtag = "/opt/Xilinx/Vivado_Lab/2020.1/bin/vivado_lab -mode batch -source {}/viv_hardware_utils.tcl -nolog -nojournal -tclargs program".format(
         remote_working_dir)
-    jtag_server, jtag_serial, fpga_path = jtag_args.split(",")
     print("Waiting on jtag mutex for {}".format(jtag_server), flush=True)
     with Redlock(key="hw_jtag_{}".format(jtag_server),
                  masters=redis_server, auto_release_time=1000 * 60 * 5):
@@ -32,6 +32,7 @@ def jtag_x3xx(jtag_args, redis_server):
                 os.path.join(pathlib.Path(
                     __file__).parent.absolute(), "jtag/viv_hardware_utils.tcl"),
                 remote=remote_working_dir)
+            fpga_path = os.path.join(fpga_folder, bitfile_name.format(dev_model, fpga))
             jtag_host.put(fpga_path, remote=remote_working_dir)
             jtag_host.run(vivado_program_jtag + " " +
                           os.path.join(remote_working_dir, os.path.basename(fpga_path)) +
@@ -39,21 +40,34 @@ def jtag_x3xx(jtag_args, redis_server):
         print("Waiting 15 seconds for device to come back up and for Vivado to close", flush=True)
         time.sleep(15)
 
-
 def main(args):
     redis_server = {Redis.from_url(
         "redis://{}:6379/0".format(args.redis_server))}
     print("Waiting to acquire mutex for {}".format(args.dut_name), flush=True)
     with Redlock(key=args.dut_name, masters=redis_server, auto_release_time=1000 * 60 * args.dut_timeout):
         print("Got mutex for {}".format(args.dut_name), flush=True)
-        if(args.jtag_x3xx != None):
-            jtag_x3xx(args.jtag_x3xx, redis_server)
-        for command in args.test_commands:
-            result = subprocess.run(shlex.split(command))
-            if(result.returncode != 0):
-                sys.exit(result.returncode)
-        sys.exit(0)
+        if args.fpgas:
+            working_dir = os.getcwd()
+            return_code = 0
+            for fpga in args.fpgas.split(','):
+                os.mkdir(os.path.join(working_dir, fpga))
+                os.chdir(os.path.join(working_dir, fpga))
+
+                if args.jtag_x3xx:
+                    jtag_server, jtag_serial, dev_model, fpga_folder = args.jtag_x3xx.split(',')
+                    jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server)
 
+                for command in args.test_commands:
+                    result = subprocess.run(shlex.split(command))
+                    if(return_code == 0):
+                        return_code = result.returncode
+            sys.exit(return_code)
+        else:
+            for command in args.test_commands:
+                result = subprocess.run(shlex.split(command))
+                if(result.returncode != 0):
+                    sys.exit(result.returncode)
+            sys.exit(0)
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -62,8 +76,10 @@ if __name__ == "__main__":
     # Provide fpga_path as a local path and it will be copied
     # to jtag_server.
     parser.add_argument("--jtag_x3xx", type=str,
-                        help="user@jtag_server,jtag_serial,fpga_path")
-    parser.add_argument("--dut_timeout", type=int, default=30,
+                        help="user@jtag_server,jtag_serial,dev_model,fpga_folder")
+    parser.add_argument("--fpgas", type=str,
+                        help="Comma delimited list of FPGAs to test")
+    parser.add_argument("--dut_timeout", type=int, default=60,
                         help="Dut mutex timeout in minutes")
     parser.add_argument("redis_server", type=str,
                         help="Redis server for mutex")