Run web API test suite in GitHub Actions (#1009)

- Fixed ignore patterns in .dockerignore
- Added healthchecks to backend and web containers
- Reduced Docker image sizes
- Removed RaSCSI references in various areas (e.g. rascsi -> backend)
- Added compilation-only step to easyinstall.sh
- Moved apt package lists to variables
- Revert to triggering GitHub Actions runs on push
- Updated web/frontend_checks workflow to run black and flake8 against all Python sources
- Capture log files from backend/web containers
- Fix None to float conversion bug when user agent is absent or unrecognised

python/common/src/rascsi/file_cmds.py

@@ -47,6 +47,13 @@ class FileCmds:
         self.token = token
         self.locale = locale
 
+    def send_pb_command(self, command):
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            # TODO: Uncouple/move to common dependency
+            logging.debug(self.ractl.format_pb_command(command))
+
+        return self.sock_cmd.send_pb_command(command.SerializeToString())
+
     # noinspection PyMethodMayBeStatic
     # pylint: disable=no-self-use
     def list_files(self, file_types, dir_path):
@@ -89,7 +96,7 @@ class FileCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
 
@@ -168,7 +175,7 @@ class FileCmds:
         command.params["size"] = str(size)
         command.params["read_only"] = "false"
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -186,7 +193,7 @@ class FileCmds:
 
         command.params["file"] = file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -205,7 +212,7 @@ class FileCmds:
         command.params["from"] = file_name
         command.params["to"] = new_file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -224,7 +231,7 @@ class FileCmds:
         command.params["from"] = file_name
         command.params["to"] = new_file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}

python/common/src/rascsi/ractl_cmds.py

@@ -5,6 +5,7 @@ Module for commands sent to the RaSCSI backend service.
 import rascsi_interface_pb2 as proto
 from rascsi.return_codes import ReturnCodes
 from rascsi.socket_cmds import SocketCmds
+import logging
 
 
 class RaCtlCmds:
@@ -17,6 +18,12 @@ class RaCtlCmds:
         self.token = token
         self.locale = locale
 
+    def send_pb_command(self, command):
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            logging.debug(self.format_pb_command(command))
+
+        return self.sock_cmd.send_pb_command(command.SerializeToString())
+
     def get_server_info(self):
         """
         Sends a SERVER_INFO command to the server.
@@ -35,7 +42,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         version = (
@@ -93,7 +100,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         scsi_ids = []
@@ -114,7 +121,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         ifs = result.network_interfaces_info.name
@@ -133,7 +140,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         device_types = {}
@@ -199,7 +206,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.token
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         images_dir = result.image_files_info.default_image_folder
@@ -273,7 +280,7 @@ class RaCtlCmds:
 
         command.devices.append(devices)
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -295,7 +302,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -310,7 +317,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -332,7 +339,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -360,7 +367,7 @@ class RaCtlCmds:
                 device.unit = int(unit)
             command.devices.append(device)
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
 
@@ -430,7 +437,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -447,7 +454,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -466,7 +473,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -480,7 +487,37 @@ class RaCtlCmds:
         command = proto.PbCommand()
         command.operation = proto.PbOperation.CHECK_AUTHENTICATION
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
+
+    def format_pb_command(self, command):
+        """
+        Formats the Protobuf command for output
+        """
+        message = f"Sending: {proto.PbOperation.Name(command.operation)}"
+
+        params = {
+            name: "***" if name == "token" else value
+            for (name, value) in sorted(command.params.items())
+        }
+        message += f", params: {params}"
+
+        for device in command.devices:
+            formatted_device = {
+                key: value
+                for (key, value) in {
+                    "id": device.id,
+                    "unit": device.unit,
+                    "type": proto.PbDeviceType.Name(device.type) if device.type else None,
+                    "params": device.params,
+                    "vendor": device.vendor,
+                    "product": device.product,
+                    "revision": device.revision,
+                }.items()
+                if key == "id" or value
+            }
+            message += f", device: {formatted_device}"
+
+        return message