Create SysCmds common class, and refactor Python codebase (#697)

* Move the oled script's PiCmds module to common, and rename it SysCmds.

* Use sys_cmds.get_ip_and_host() in web UI code.

* Move the auth_active() method to device_utils

* Rename device_utils to web_utils. Make auth_active() method take the group as argument.

* Migrate all pi_cmds methods to the SysCmds common class.

* Display hostname and ip in Web UI.

* Resolve or suppress pylint warnings.

* Resolve a pylint warning.

* Resolve or suppress pylint warnings.

* Import libraries at the top level for readability. In my testing on a Pi3B+, this leads to ~1.5k more memory being used by the python3 process.

* Change page title as requested by akuker.

* Reenable the import-outside-toplevel pylint rule.

* Resolve pylint warnings.

* Fix error following refactoring.

* Minor UI tweaks.

* Cleanup.

* Break out bridge config validation into a utility method.

* Move the dropzonejs method into the web_utils package

* Move get_logs method into SysCmds class.

* Improve get logs UI.

* Resolve pylint warning.

* Standardize class instance name.

python/common/src/rascsi/file_cmds.py

@@ -4,8 +4,16 @@ Module for methods reading from and writing to the file system
 
 import os
 import logging
-from pathlib import PurePath
 import asyncio
+from pathlib import PurePath
+from zipfile import ZipFile, is_zipfile
+from re import escape, findall
+from time import time
+from subprocess import run, CalledProcessError
+from json import dump, load
+
+import requests
+
 import rascsi_interface_pb2 as proto
 from rascsi.common_settings import CFG_DIR, CONFIG_FILE_SUFFIX, PROPERTIES_SUFFIX, RESERVATIONS
 from rascsi.ractl_cmds import RaCtlCmds
@@ -79,7 +87,6 @@ class FileCmds:
         prop_data = self.list_files(PROPERTIES_SUFFIX, CFG_DIR)
         prop_files = [PurePath(x[0]).stem for x in prop_data]
 
-        from zipfile import ZipFile, is_zipfile
         server_info = self.ractl.get_server_info()
         files = []
         for file in result.image_files_info.image_files:
@@ -225,12 +232,11 @@ class FileCmds:
         members contains all of the full paths to each of the zip archive members
         Returns (dict) with (boolean) status and (list of str) msg
         """
-        from asyncio import run
         server_info = self.ractl.get_server_info()
         prop_flag = False
 
         if not member:
-            unzip_proc = run(self.run_async(
+            unzip_proc = asyncio.run(self.run_async(
                 f"unzip -d {server_info['image_dir']} -n -j "
                 f"{server_info['image_dir']}/{file_name}"
                 ))
@@ -241,14 +247,13 @@ class FileCmds:
                         self.rename_file(f"{server_info['image_dir']}/{name}", f"{CFG_DIR}/{name}")
                         prop_flag = True
         else:
-            from re import escape
             member = escape(member)
-            unzip_proc = run(self.run_async(
+            unzip_proc = asyncio.run(self.run_async(
                 f"unzip -d {server_info['image_dir']} -n -j "
                 f"{server_info['image_dir']}/{file_name} {member}"
                 ))
             # Attempt to unzip a properties file in the same archive dir
-            unzip_prop = run(self.run_async(
+            unzip_prop = asyncio.run(self.run_async(
                 f"unzip -d {CFG_DIR} -n -j "
                 f"{server_info['image_dir']}/{file_name} {member}.{PROPERTIES_SUFFIX}"
                 ))
@@ -258,7 +263,6 @@ class FileCmds:
             logging.warning("Unzipping failed: %s", unzip_proc["stderr"])
             return {"status": False, "msg": unzip_proc["stderr"]}
 
-        from re import findall
         unzipped = findall(
             "(?:inflating|extracting):(.+)\n",
             unzip_proc["stdout"]
@@ -270,8 +274,6 @@ class FileCmds:
         Takes (str) url and one or more (str) *iso_args
         Returns (dict) with (bool) status and (str) msg
         """
-        from time import time
-        from subprocess import run, CalledProcessError
 
         server_info = self.ractl.get_server_info()
 
@@ -287,7 +289,6 @@ class FileCmds:
         if not req_proc["status"]:
             return {"status": False, "msg": req_proc["msg"]}
 
-        from zipfile import is_zipfile, ZipFile
         if is_zipfile(tmp_full_path):
             if "XtraStuf.mac" in str(ZipFile(tmp_full_path).namelist()):
                 logging.info("MacZip file format detected. Will not unzip to retain resource fork.")
@@ -339,7 +340,6 @@ class FileCmds:
         Takes (str) url, (str) save_dir, (str) file_name
         Returns (dict) with (bool) status and (str) msg
         """
-        import requests
         logging.info("Making a request to download %s", url)
 
         try:
@@ -371,7 +371,6 @@ class FileCmds:
         Takes (str) file_name
         Returns (dict) with (bool) status and (str) msg
         """
-        from json import dump
         file_name = f"{CFG_DIR}/{file_name}"
         try:
             with open(file_name, "w", encoding="ISO-8859-1") as json_file:
@@ -433,7 +432,6 @@ class FileCmds:
         Takes (str) file_name
         Returns (dict) with (bool) status and (str) msg
         """
-        from json import load
         file_name = f"{CFG_DIR}/{file_name}"
         try:
             with open(file_name, encoding="ISO-8859-1") as json_file:
@@ -512,7 +510,6 @@ class FileCmds:
         Takes file name base (str) and (list of dicts) conf as arguments
         Returns (dict) with (bool) status and (str) msg
         """
-        from json import dump
         file_path = f"{CFG_DIR}/{file_name}"
         try:
             with open(file_path, "w") as json_file:
@@ -548,7 +545,6 @@ class FileCmds:
         Takes (str) file_path as argument.
         Returns (dict) with (bool) status, (str) msg, (dict) conf
         """
-        from json import load
         try:
             with open(file_path) as json_file:
                 conf = load(json_file)

python/common/src/rascsi/ractl_cmds.py

@@ -162,7 +162,7 @@ class RaCtlCmds:
 
     def get_disk_device_types(self):
         """
-        Returns a (list) of (str) of four letter device acronyms 
+        Returns a (list) of (str) of four letter device acronyms
         that take image files as arguments.
         """
         device_types = self.get_device_types()

python/common/src/rascsi/socket_cmds.py

@@ -3,7 +3,10 @@ Module for sending and receiving data over a socket connection with the RaSCSI b
 """
 
 import logging
+import socket
 from time import sleep
+from struct import pack, unpack
+
 from rascsi.exceptions import (EmptySocketChunkException,
                                InvalidProtobufResponse,
                                FailedSocketConnectionException)
@@ -27,7 +30,6 @@ class SocketCmds:
         tries = 20
         error_msg = ""
 
-        import socket
         while counter < tries:
             try:
                 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
@@ -56,7 +58,6 @@ class SocketCmds:
         Tries to extract and interpret the protobuf header to get response size.
         Reads data from socket in 2048 bytes chunks until all data is received.
         """
-        from struct import pack, unpack
 
         # Sending the magic word "RASCSI" to authenticate with the server
         sock.send(b"RASCSI")

python/common/src/rascsi/sys_cmds.py

@@ -0,0 +1,170 @@
+"""
+Module with methods that interact with the Pi system
+"""
+import subprocess
+import logging
+from subprocess import run
+from shutil import disk_usage
+from re import findall, match
+from socket import socket, gethostname, AF_INET, SOCK_DGRAM
+
+
+class SysCmds:
+    """
+    Class for commands sent to the Pi's Linux system.
+    """
+
+    @staticmethod
+    def running_env():
+        """
+        Returns (str) git and (str) env
+        git contains the git hash of the checked out code
+        env is the various system information where this app is running
+        """
+        try:
+            ra_git_version = (
+                subprocess.run(
+                    ["git", "rev-parse", "HEAD"],
+                    capture_output=True,
+                    check=True,
+                    )
+                .stdout.decode("utf-8")
+                .strip()
+            )
+        except subprocess.CalledProcessError as error:
+            logging.warning("Executed shell command: %s", " ".join(error.cmd))
+            logging.warning("Got error: %s", error.stderr.decode("utf-8"))
+            ra_git_version = ""
+
+        try:
+            pi_version = (
+                subprocess.run(
+                    ["uname", "-a"],
+                    capture_output=True,
+                    check=True,
+                    )
+                .stdout.decode("utf-8")
+                .strip()
+            )
+        except subprocess.CalledProcessError as error:
+            logging.warning("Executed shell command: %s", " ".join(error.cmd))
+            logging.warning("Got error: %s", error.stderr.decode("utf-8"))
+            pi_version = "Unknown"
+
+        return {"git": ra_git_version, "env": pi_version}
+
+    @staticmethod
+    def running_proc(daemon):
+        """
+        Takes (str) daemon
+        Returns (int) proc, which is the number of processes currently running
+        """
+        try:
+            processes = (
+                subprocess.run(
+                    ["ps", "aux"],
+                    capture_output=True,
+                    check=True,
+                    )
+                .stdout.decode("utf-8")
+                .strip()
+            )
+        except subprocess.CalledProcessError as error:
+            logging.warning("Executed shell command: %s", " ".join(error.cmd))
+            logging.warning("Got error: %s", error.stderr.decode("utf-8"))
+            processes = ""
+
+        matching_processes = findall(daemon, processes)
+        return len(matching_processes)
+
+    @staticmethod
+    def is_bridge_setup():
+        """
+        Returns (bool) True if the rascsi_bridge network interface exists
+        """
+        try:
+            bridges = (
+                subprocess.run(
+                    ["brctl", "show"],
+                    capture_output=True,
+                    check=True,
+                    )
+                .stdout.decode("utf-8")
+                .strip()
+            )
+        except subprocess.CalledProcessError as error:
+            logging.warning("Executed shell command: %s", " ".join(error.cmd))
+            logging.warning("Got error: %s", error.stderr.decode("utf-8"))
+            bridges = ""
+
+        if "rascsi_bridge" in bridges:
+            return True
+        return False
+
+    @staticmethod
+    def disk_space():
+        """
+        Returns a (dict) with (int) total (int) used (int) free
+        This is the disk space information of the volume where this app is running
+        """
+        total, used, free = disk_usage(__file__)
+        return {"total": total, "used": used, "free": free}
+
+    @staticmethod
+    def introspect_file(file_path, re_term):
+        """
+        Takes a (str) file_path and (str) re_term in regex format
+        Will introspect file_path for the existance of re_term
+        and return True if found, False if not found
+        """
+        try:
+            ifile = open(file_path, "r", encoding="ISO-8859-1")
+        except (IOError, ValueError, EOFError, TypeError) as error:
+            logging.error(str(error))
+            return False
+        for line in ifile:
+            if match(re_term, line):
+                return True
+        return False
+
+    # pylint: disable=broad-except
+    @staticmethod
+    def get_ip_and_host():
+        """
+        Use a mock socket connection to identify the Pi's hostname and IP address
+        """
+        host = gethostname()
+        sock = socket(AF_INET, SOCK_DGRAM)
+        try:
+            # mock ip address; doesn't have to be reachable
+            sock.connect(('10.255.255.255', 1))
+            ip_addr = sock.getsockname()[0]
+        except Exception:
+            ip_addr = False
+        finally:
+            sock.close()
+        return ip_addr, host
+
+    @staticmethod
+    def get_logs(lines, scope):
+        """
+        Takes (int) lines and (str) scope.
+        Returns either the decoded log output, or the stderr output of journalctl.
+        """
+        if scope != "all":
+            process = run(
+                    ["journalctl", "-n", lines, "-u", scope],
+                    capture_output=True,
+                    check=True,
+                    )
+        else:
+            process = run(
+                    ["journalctl", "-n", lines],
+                    capture_output=True,
+                    check=True,
+                    )
+
+        if process.returncode == 0:
+            return process.returncode, process.stdout.decode("utf-8")
+
+        return process.returncode, process.stderr.decode("utf-8")