Merge tag 'v23.04.01'

PiSCSI version 23.04.01

.github/workflows/cpp.yml

@@ -93,28 +93,16 @@ jobs:
         working-directory: cpp
         run: gcov --preserve-paths $(find -name '*.gcno')
 
-      - uses: actions/cache@v3
-        name: Cache SonarCloud scan cache
-        id: sonar-scan-cache
-        with:
-          path: ~/.sonar_cache/
-          key: sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-          restore-keys: |
-            sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }}
-
       - name: Run sonar-scanner
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: >-
-          (mkdir -p $HOME/.sonar_cache || true) &&
           $HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin/sonar-scanner
           --define sonar.host.url="${{ env.SONAR_SERVER_URL }}"
           --define sonar.projectKey=${{ env.SONAR_PROJECT_KEY }}
           --define sonar.organization=${{ env.SONAR_ORGANIZATION }}
           --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}"
           --define sonar.cfamily.gcov.reportsPath=.
-          --define sonar.cfamily.cache.enabled=true
-          --define sonar.cfamily.cache.path="$HOME/.sonar_cache/"
           --define sonar.coverage.exclusions="cpp/**/test/**"
           --define sonar.cpd.exclusions="cpp/**/test/**"
           --define sonar.inclusions="cpp/**,python/**"

cpp/Makefile

@@ -199,22 +199,22 @@ docs: $(DOC_DIR)/piscsi_man_page.txt $(DOC_DIR)/scsictl_man_page.txt $(DOC_DIR)/
 $(SRC_PISCSI_CORE) $(SRC_SCSICTL_CORE) : $(OBJ_GENERATED)
 
 $(BINDIR)/$(PISCSI): $(SRC_GENERATED) $(OBJ_PISCSI_CORE) $(OBJ_PISCSI) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) -o $@ $(OBJ_PISCSI_CORE) $(OBJ_PISCSI) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lpcap -lprotobuf -lstdc++fs
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(OBJ_PISCSI_CORE) $(OBJ_PISCSI) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lpcap -lprotobuf -lstdc++fs
 
 $(BINDIR)/$(SCSICTL): $(SRC_GENERATED) $(OBJ_SCSICTL_CORE) $(OBJ_SCSICTL) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) -o $@ $(OBJ_SCSICTL_CORE) $(OBJ_SCSICTL) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lprotobuf
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(OBJ_SCSICTL_CORE) $(OBJ_SCSICTL) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lprotobuf
 
 $(BINDIR)/$(SCSIDUMP): $(OBJ_SCSIDUMP) $(OBJ_SHARED) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) -o $@ $(OBJ_SCSIDUMP) $(OBJ_SHARED)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(OBJ_SCSIDUMP) $(OBJ_SHARED)
 
 $(BINDIR)/$(SCSIMON): $(OBJ_SCSIMON) $(OBJ_SHARED) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) -o $@ $(OBJ_SCSIMON) $(OBJ_SHARED)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(OBJ_SCSIMON) $(OBJ_SHARED)
-
-$(BINDIR)/$(PISCSI_TEST): $(SRC_GENERATED) $(OBJ_PISCSI_CORE) $(OBJ_SCSICTL_CORE) $(OBJ_PISCSI_TEST) $(OBJ_SCSICTL_TEST) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) $(TEST_WRAPS) -o $@ $(OBJ_PISCSI_CORE) $(OBJ_SCSICTL_CORE) $(OBJ_PISCSI_TEST) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lpcap -lprotobuf -lstdc++fs -lgmock -lgtest
 
 $(BINDIR)/$(SCSILOOP): $(OBJ_SHARED) $(OBJ_SCSILOOP) | $(BINDIR)
-	$(CXX) $(CXXFLAGS) -o $@  $(OBJ_SHARED) $(OBJ_SCSILOOP)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@  $(OBJ_SHARED) $(OBJ_SCSILOOP)
+
+$(BINDIR)/$(PISCSI_TEST): $(SRC_GENERATED) $(OBJ_PISCSI_CORE) $(OBJ_SCSICTL_CORE) $(OBJ_PISCSI_TEST) $(OBJ_SCSICTL_TEST) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) | $(BINDIR)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) $(TEST_WRAPS) -o $@ $(OBJ_PISCSI_CORE) $(OBJ_SCSICTL_CORE) $(OBJ_PISCSI_TEST) $(OBJ_SHARED) $(OBJ_PROTOBUF) $(OBJ_GENERATED) -lpthread -lpcap -lprotobuf -lstdc++fs -lgmock -lgtest
 
 # Phony rules for building individual utilities
 .PHONY: $(PISCSI) $(SCSICTL) $(SCSIDUMP) $(SCSIMON) $(PISCSI_TEST) $(SCSILOOP)

cpp/devices/device_factory.cpp

@@ -59,6 +59,7 @@ DeviceFactory::DeviceFactory()
 	extension_mapping["hdr"] = SCRM;
 	extension_mapping["mos"] = SCMO;
 	extension_mapping["iso"] = SCCD;
+	extension_mapping["is1"] = SCCD;
 
 	device_mapping["bridge"] = SCBR;
 	device_mapping["daynaport"] = SCDP;
@@ -118,7 +119,8 @@ shared_ptr<PrimaryDevice> DeviceFactory::CreateDevice(PbDeviceType type, int lun
 		break;
 
 	case SCCD:
-		device = make_shared<SCSICD>(lun, sector_sizes.find(SCCD)->second);
+		device = make_shared<SCSICD>(lun, sector_sizes.find(SCCD)->second,
+            GetExtensionLowerCase(filename) == "is1" ? scsi_level::SCSI_1_CCS : scsi_level::SCSI_2);
 		device->SetProduct("SCSI CD-ROM");
 		break;
 

cpp/devices/disk.cpp

@@ -396,7 +396,12 @@ void Disk::SetUpModePages(map<int, vector<byte>>& pages, int page, bool changeab
 void Disk::AddErrorPage(map<int, vector<byte>>& pages, bool) const
 {
 	// Retry count is 0, limit time uses internal default value
-	pages[1] = vector<byte>(12);
+	vector<byte> buf(12);
+
+	// TB, PER, DTE (required for OpenVMS/VAX compatibility, see issue #1117)
+	buf[2] = (byte)0x26;
+
+	pages[1] = buf;
 }
 
 void Disk::AddFormatPage(map<int, vector<byte>>& pages, bool changeable) const

cpp/devices/scsicd.cpp

@@ -21,7 +21,8 @@
 using namespace scsi_defs;
 using namespace scsi_command_util;
 
-SCSICD::SCSICD(int lun, const unordered_set<uint32_t>& sector_sizes) : Disk(SCCD, lun)
+SCSICD::SCSICD(int lun, const unordered_set<uint32_t>& sector_sizes, scsi_defs::scsi_level level) 
+	: Disk(SCCD, lun), scsi_level(level)
 {
 	SetSectorSizes(sector_sizes);
 
@@ -164,7 +165,7 @@ void SCSICD::ReadToc()
 
 vector<uint8_t> SCSICD::InquiryInternal() const
 {
-	return HandleInquiry(device_type::CD_ROM, scsi_level::SCSI_2, true);
+	return HandleInquiry(device_type::CD_ROM, scsi_level, true);
 }
 
 void SCSICD::SetUpModePages(map<int, vector<byte>>& pages, int page, bool changeable) const

cpp/devices/scsicd.h

@@ -22,7 +22,7 @@ class SCSICD : public Disk, private ScsiMmcCommands
 {
 public:
 
-	SCSICD(int, const unordered_set<uint32_t>&);
+	SCSICD(int, const unordered_set<uint32_t>&, scsi_defs::scsi_level = scsi_level::SCSI_2);
 	~SCSICD() override = default;
 
 	bool Init(const unordered_map<string, string>&) override;
@@ -43,6 +43,7 @@ private:
 
 	void AddCDROMPage(map<int, vector<byte>>&, bool) const;
 	void AddCDDAPage(map<int, vector<byte>>&, bool) const;
+	scsi_defs::scsi_level scsi_level;
 
 	void OpenIso();
 	void OpenPhysical();

cpp/devices/scsihd.cpp

@@ -36,9 +36,9 @@ string SCSIHD::GetProductData() const
 	uint64_t capacity = GetBlockCount() * GetSectorSizeInBytes();
 	string unit;
 
-	// 10 GiB and more
+	// 10,000 MiB and more
-	if (capacity >= 1'099'511'627'776) {
+	if (capacity >= 10'485'760'000) {
-		capacity /= 1'099'511'627'776;
+		capacity /= 1'073'741'824;
 		unit = "GiB";
 	}
 	// 1 MiB and more

cpp/hal/gpiobus_raspberry.cpp

@@ -87,7 +87,7 @@ bool GPIOBUS_Raspberry::Init(mode_e mode)
     // Map peripheral region memory
     void *map = mmap(NULL, 0x1000100, PROT_READ | PROT_WRITE, MAP_SHARED, fd, baseaddr);
     if (map == MAP_FAILED) {
-        LOGERROR("Error: Unable to map memory")
+        LOGERROR("Error: Unable to map memory: %s", strerror(errno))
         close(fd);
         return false;
     }
@@ -985,4 +985,4 @@ uint32_t GPIOBUS_Raspberry::Acquire()
 #endif // SIGNAL_CONTROL_MODE
 
     return signals;
-}
+}

cpp/piscsi/piscsi_core.cpp

@@ -62,7 +62,8 @@ void Piscsi::Banner(const vector<char *>& args) const
 		cout << "  hdi : SCSI HD image (Anex86 HD image)\n";
 		cout << "  nhd : SCSI HD image (T98Next HD image)\n";
 		cout << "  mos : SCSI MO image (MO image)\n";
-		cout << "  iso : SCSI CD image (ISO 9660 image)\n" << flush;
+		cout << "  iso : SCSI CD image (ISO 9660 image)\n";
+		cout << "  is1 : SCSI CD image (ISO 9660 image, SCSI-1)\n" << flush;
 
 		exit(EXIT_SUCCESS);
 	}

cpp/shared/piscsi_version.cpp

@@ -14,7 +14,7 @@
 
 // The following should be updated for each release
 const int piscsi_major_version = 23; // Last two digits of year
-const int piscsi_minor_version =  2; // Month
+const int piscsi_minor_version =  4; // Month
 const int piscsi_patch_version =  1;  // Patch number - increment for each update
 
 using namespace std;

cpp/test/device_factory_test.cpp

@@ -29,6 +29,7 @@ TEST(DeviceFactoryTest, GetTypeForFile)
 	EXPECT_EQ(device_factory.GetTypeForFile("test.hdr"), SCRM);
 	EXPECT_EQ(device_factory.GetTypeForFile("test.mos"), SCMO);
 	EXPECT_EQ(device_factory.GetTypeForFile("test.iso"), SCCD);
+	EXPECT_EQ(device_factory.GetTypeForFile("test.is1"), SCCD);
 	EXPECT_EQ(device_factory.GetTypeForFile("test.suffix.iso"), SCCD);
 	EXPECT_EQ(device_factory.GetTypeForFile("bridge"), SCBR);
 	EXPECT_EQ(device_factory.GetTypeForFile("daynaport"), SCDP);
@@ -79,7 +80,7 @@ TEST(DeviceFactoryTest, GetExtensionMapping)
 	DeviceFactory device_factory;
 
 	unordered_map<string, PbDeviceType> mapping = device_factory.GetExtensionMapping();
-	EXPECT_EQ(9, mapping.size());
+	EXPECT_EQ(10, mapping.size());
 	EXPECT_EQ(SCHD, mapping["hd1"]);
 	EXPECT_EQ(SCHD, mapping["hds"]);
 	EXPECT_EQ(SCHD, mapping["hda"]);
@@ -89,6 +90,7 @@ TEST(DeviceFactoryTest, GetExtensionMapping)
 	EXPECT_EQ(SCRM, mapping["hdr"]);
 	EXPECT_EQ(SCMO, mapping["mos"]);
 	EXPECT_EQ(SCCD, mapping["iso"]);
+	EXPECT_EQ(SCCD, mapping["is1"]);
 }
 
 TEST(DeviceFactoryTest, GetDefaultParams)

cpp/test/mocks.h

@@ -3,7 +3,7 @@
 // SCSI Target Emulator PiSCSI
 // for Raspberry Pi
 //
-// Copyright (C) 2022 Uwe Seimet
+// Copyright (C) 2022-2023 Uwe Seimet
 //
 //---------------------------------------------------------------------------
 
@@ -358,6 +358,7 @@ class MockSCSIHD : public SCSIHD //NOSONAR Ignore inheritance hierarchy depth in
 class MockSCSIHD_NEC : public SCSIHD_NEC //NOSONAR Ignore inheritance hierarchy depth in unit tests
 {
 	FRIEND_TEST(ScsiHdNecTest, SetUpModePages);
+	FRIEND_TEST(ScsiHdNecTest, TestAddErrorPage);
 	FRIEND_TEST(ScsiHdNecTest, TestAddFormatPage);
 	FRIEND_TEST(ScsiHdNecTest, TestAddDrivePage);
 	FRIEND_TEST(PiscsiExecutorTest, ProcessDeviceCmd);

cpp/test/piscsi_image_test.cpp

@@ -39,6 +39,8 @@ TEST(PiscsiImageTest, CreateImage)
 	PbCommand command;
 	PiscsiImage image;
 
+	StorageDevice::UnreserveAll();
+
 	EXPECT_FALSE(image.CreateImage(context, command)) << "Filename must be reported as missing";
 
 	SetParam(command, "file", "/a/b/c/filename");
@@ -63,6 +65,8 @@ TEST(PiscsiImageTest, DeleteImage)
 	PbCommand command;
 	PiscsiImage image;
 
+	StorageDevice::UnreserveAll();
+
 	EXPECT_FALSE(image.DeleteImage(context, command)) << "Filename must be reported as missing";
 
 	SetParam(command, "file", "/a/b/c/filename");
@@ -82,6 +86,8 @@ TEST(PiscsiImageTest, RenameImage)
 	PbCommand command;
 	PiscsiImage image;
 
+	StorageDevice::UnreserveAll();
+
 	EXPECT_FALSE(image.RenameImage(context, command)) << "Source filename must be reported as missing";
 
 	SetParam(command, "from", "/a/b/c/filename_from");
@@ -99,6 +105,8 @@ TEST(PiscsiImageTest, CopyImage)
 	PbCommand command;
 	PiscsiImage image;
 
+	StorageDevice::UnreserveAll();
+
 	EXPECT_FALSE(image.CopyImage(context, command)) << "Source filename must be reported as missing";
 
 	SetParam(command, "from", "/a/b/c/filename_from");
@@ -116,6 +124,8 @@ TEST(PiscsiImageTest, SetImagePermissions)
 	PbCommand command;
 	PiscsiImage image;
 
+	StorageDevice::UnreserveAll();
+
 	EXPECT_FALSE(image.SetImagePermissions(context, command)) << "Filename must be reported as missing";
 
 	SetParam(command, "file", "/a/b/c/filename");

cpp/test/piscsi_response_test.cpp

@@ -221,5 +221,5 @@ TEST(PiscsiResponseTest, GetMappingInfo)
 
 	const auto& info = response.GetMappingInfo(result);
 	EXPECT_TRUE(result.status());
-	EXPECT_EQ(9, info->mapping().size());
+	EXPECT_EQ(10, info->mapping().size());
 }

cpp/test/scsicd_test.cpp

@@ -30,6 +30,8 @@ void ScsiCdTest_SetUpModePages(map<int, vector<byte>>& pages)
 TEST(ScsiCdTest, Inquiry)
 {
 	TestInquiry(SCCD, device_type::CD_ROM, scsi_level::SCSI_2, "PiSCSI  SCSI CD-ROM     ", 0x1f, true);
+
+	TestInquiry(SCCD, device_type::CD_ROM, scsi_level::SCSI_1_CCS, "PiSCSI  SCSI CD-ROM     ", 0x1f, true, ".is1");
 }
 
 TEST(ScsiCdTest, SetUpModePages)

cpp/test/scsihd_nec_test.cpp

@@ -3,7 +3,7 @@
 // SCSI Target Emulator PiSCSI
 // for Raspberry Pi
 //
-// Copyright (C) 2022 Uwe Seimet
+// Copyright (C) 2022-2023 Uwe Seimet
 //
 //---------------------------------------------------------------------------
 
@@ -47,6 +47,20 @@ TEST(ScsiHdNecTest, SetUpModePages)
 	ScsiHdNecTest_SetUpModePages(pages);
 }
 
+TEST(ScsiHdNecTest, TestAddErrorPage)
+{
+	map<int, vector<byte>> pages;
+	MockSCSIHD_NEC hd(0);
+
+	hd.SetBlockCount(0x1234);
+	hd.SetReady(true);
+	// Non changeable
+	hd.SetUpModePages(pages, 0x01, false);
+	EXPECT_EQ(1, pages.size()) << "Unexpected number of mode pages";
+	const vector<byte>& page_1 = pages[1];
+	EXPECT_EQ(0x26, to_integer<int>(page_1[2]));
+}
+
 TEST(ScsiHdNecTest, TestAddFormatPage)
 {
 	map<int, vector<byte>> pages;

cpp/test/scsihd_test.cpp

@@ -69,10 +69,10 @@ TEST(ScsiHdTest, GetProductData)
 
 	hd_gb.SetFilename(string(filename));
 	hd_gb.SetSectorSizeInBytes(1024);
-	hd_gb.SetBlockCount(1'099'511'627'776 / 1024);
+	hd_gb.SetBlockCount(10'737'418'240 / 1024);
 	hd_gb.FinalizeSetup(0);
 	s = hd_gb.GetProduct();
-	EXPECT_NE(string::npos, s.find("1 GiB"));
+	EXPECT_NE(string::npos, s.find("10 GiB"));
 	remove(filename);
 }
 

doc/piscsi.1

@@ -34,6 +34,7 @@ PiSCSI will determine the type of device based upon the file extension of the FI
     hda: SCSI Hard Disk image (Apple compatible - typically used with Macintosh computers)
     mos: SCSI Magneto-Optical image (generic - typically used with NeXT, X68000, etc.)
     iso: SCSI CD-ROM or DVD-ROM image (ISO 9660 image)
+    is1: SCSI CD-ROM or DVD-ROM image (ISO 9660 image, SCSI-1)
   
 For example, if you want to specify an Apple-compatible HD image on ID 0, you can use the following command:
     sudo piscsi -ID0 /path/to/drive/hdimage.hda

doc/piscsi_man_page.txt

@@ -29,6 +29,7 @@ DESCRIPTION
            hda: SCSI Hard Disk image (Apple compatible - typically used with Macintosh computers)
            mos: SCSI Magneto-Optical image (generic - typically used with NeXT, X68000, etc.)
            iso: SCSI CD-ROM or DVD-ROM image (ISO 9660 image)
+           is1: SCSI CD-ROM or DVD-ROM image (ISO 9660 image, SCSI-1)
 
        For example, if you want to specify an Apple-compatible HD image on ID 0, you can use the following command:
            sudo piscsi -ID0 /path/to/drive/hdimage.hda

easyinstall.sh

@@ -49,8 +49,7 @@ echo -e $logo
 CONNECT_TYPE="FULLSPEC"
 # clang v11 is the latest distributed by Buster
 COMPILER="clang++-11"
-# Takes half of the CPU cores available, to avoid running out of memory on low spec devices
+CORES=1
-CORES=$(awk 'BEGIN { x = '$(nproc)'; y = 2; print (x / y) }' | numfmt --round=up --format=%.0f)
 USER=$(whoami)
 BASE=$(dirname "$(readlink -f "${0}")")
 CPP_PATH="$BASE/cpp"
@@ -87,6 +86,12 @@ function initialChecks() {
     fi
 }
 
+# Only to be used for pi-gen automated install
+function cacheSudo() {
+    echo "Caching sudo password"
+    echo raspberry | sudo -v -S
+}
+
 # checks that the current user has sudoers privileges
 function sudoCheck() {
     if [[ $HEADLESS ]]; then
@@ -705,10 +710,34 @@ function setupWirelessNetworking() {
     sudo reboot
 }
 
+# Detects or creates the file sharing directory
+function createFileSharingDir() {
+    if [ ! -d "$FILE_SHARE_PATH" ] && [ -d "$HOME/afpshare" ]; then
+        echo
+        echo "File server dir $HOME/afpshare detected. This script will rename it to $FILE_SHARE_PATH."
+        echo
+        echo "Do you want to proceed with the installation? [y/N]"
+        read -r REPLY
+        if [ "$REPLY" == "y" ] || [ "$REPLY" == "Y" ]; then
+            sudo mv "$HOME/afpshare" "$FILE_SHARE_PATH" || exit 1
+        else
+            exit 0
+        fi
+    elif [ -d "$FILE_SHARE_PATH" ]; then
+        echo "Found a $FILE_SHARE_PATH directory; will use it for file sharing."
+    else
+        echo "Creating the $FILE_SHARE_PATH directory and granting read/write permissions to all users..."
+        sudo mkdir -p "$FILE_SHARE_PATH"
+        sudo chown -R "$USER:$USER" "$FILE_SHARE_PATH"
+        chmod -Rv 775 "$FILE_SHARE_PATH"
+    fi
+}
+
 # Downloads, compiles, and installs Netatalk (AppleShare server)
 function installNetatalk() {
-    NETATALK_VERSION="2-230201"
+    NETATALK_VERSION="230302"
     NETATALK_CONFIG_PATH="/etc/netatalk"
+    NETATALK_OPTIONS="--cores=$CORES --share-name='$FILE_SHARE_NAME' --share-path='$FILE_SHARE_PATH'"
 
     if [ -d "$NETATALK_CONFIG_PATH" ]; then
         echo
@@ -723,27 +752,26 @@ function installNetatalk() {
         fi
     fi
 
-    if [ ! -d "$FILE_SHARE_PATH" ] && [ -d "$HOME/afpshare" ]; then
+    echo
-        echo
+    echo "Downloading tarball to $HOME..."
-        echo "File server dir $HOME/afpshare detected. This script will rename it to $FILE_SHARE_PATH."
+    cd $HOME || exit 1
-        echo
+    wget -O "netatalk-2.$NETATALK_VERSION.tar.gz" "https://github.com/rdmark/netatalk-2.x/releases/download/netatalk-2-$NETATALK_VERSION/netatalk-2.$NETATALK_VERSION.tar.gz" </dev/null
-        echo "Do you want to proceed with the installation? [y/N]"
+
-        read -r REPLY
+    echo "Unpacking tarball..."
-        if [ "$REPLY" == "y" ] || [ "$REPLY" == "Y" ]; then
+    tar -xzf "netatalk-2.$NETATALK_VERSION.tar.gz"
-            sudo mv "$HOME/afpshare" "$FILE_SHARE_PATH" || exit 1
+    rm "netatalk-2.$NETATALK_VERSION.tar.gz"
-        else
+
-            exit 0
+    if [ -f "/etc/network/interfaces.d/piscsi_bridge" ]; then
-        fi
+        echo "PiSCSI network bridge detected. Using 'piscsi_bridge' interface for AppleTalk."
+        NETATALK_OPTIONS="$NETATALK_OPTIONS --appletalk-interface=piscsi_bridge"
     fi
 
-    echo "Downloading netatalk-$NETATALK_VERSION to $HOME"
+    [[ $HEADLESS ]] && NETATALK_OPTIONS="$NETATALK_OPTIONS --headless"
-    cd $HOME || exit 1
+    [[ $SKIP_PACKAGES ]] && NETATALK_OPTIONS="$NETATALK_OPTIONS --no-packages"
-    wget -O "netatalk-$NETATALK_VERSION.tar.gz" "https://github.com/rdmark/Netatalk-2.x/archive/refs/tags/netatalk-$NETATALK_VERSION.tar.gz" </dev/null
+    [[ $SKIP_MAKE_CLEAN ]] && NETATALK_OPTIONS="$NETATALK_OPTIONS --no-make-clean"
-    tar -xzvf "netatalk-$NETATALK_VERSION.tar.gz"
-    rm "netatalk-$NETATALK_VERSION.tar.gz"
 
-    cd "$HOME/Netatalk-2.x-netatalk-$NETATALK_VERSION/contrib/shell_utils" || exit 1
+    cd "$HOME/netatalk-2.$NETATALK_VERSION/contrib/shell_utils" || exit 1
-    ./debian_install.sh -j="$CORES" -n="$FILE_SHARE_NAME" -p="$FILE_SHARE_PATH" || exit 1
+    bash -c "./debian_install.sh $NETATALK_OPTIONS" || exit 1
 }
 
 # Appends the images dir as a shared Netatalk volume
@@ -856,26 +884,6 @@ function installSamba() {
         fi
     fi
 
-    if [ ! -d "$FILE_SHARE_PATH" ] && [ -d "$HOME/afpshare" ]; then
-        echo
-        echo "File server dir $HOME/afpshare detected. This script will rename it to $FILE_SHARE_PATH."
-        echo
-        echo "Do you want to proceed with the installation? [y/N]"
-        read -r REPLY
-        if [ "$REPLY" == "y" ] || [ "$REPLY" == "Y" ]; then
-            sudo mv "$HOME/afpshare" "$FILE_SHARE_PATH" || exit 1
-        else
-            exit 0
-        fi
-    elif [ -d "$FILE_SHARE_PATH" ]; then
-        echo "Found a $FILE_SHARE_PATH directory; will use it for file sharing."
-    else
-        echo "Creating the $FILE_SHARE_PATH directory and granting read/write permissions to all users..."
-        sudo mkdir -p "$FILE_SHARE_PATH"
-        sudo chown -R "$USER:$USER" "$FILE_SHARE_PATH"
-        chmod -Rv 775 "$FILE_SHARE_PATH"
-    fi
-
     echo ""
     echo "Installing dependencies..."
     sudo apt-get update || true
@@ -1261,6 +1269,7 @@ function runChoice() {
           ;;
           7)
               echo "Installing AppleShare File Server"
+              createFileSharingDir
               installNetatalk
               echo "Installing AppleShare File Server - Complete!"
           ;;
@@ -1272,6 +1281,7 @@ function runChoice() {
               echo "WARNING: The FTP server may transfer unencrypted data over the network."
               echo "Proceed with this installation only if you are on a private, secure network."
               sudoCheck
+              createFileSharingDir
               installFtp
               echo "Installing FTP File Server - Complete!"
           ;;
@@ -1283,6 +1293,7 @@ function runChoice() {
               echo " - Create a directory in the current user's home directory where shared files will be stored"
               echo " - Create a Samba user for the current user"
               sudoCheck
+              createFileSharingDir
               installSamba
               echo "Installing SMB File Server - Complete!"
           ;;
@@ -1359,6 +1370,25 @@ function runChoice() {
               installPackagesStandalone
               compilePiscsi
           ;;
+          99)
+              echo "Hidden setup mode for running the pi-gen utility"
+              echo "This shouldn't be used by normal users"
+              sudoCache
+              createImagesDir
+              createCfgDir
+              updatePiscsiGit
+              installPackages
+              installHfdisk
+              fetchHardDiskDrivers
+              compilePiscsi
+              installPiscsi
+              enablePiscsiService
+              preparePythonCommon
+              cachePipPackages
+              installPiscsiWebInterface
+              installWebInterfaceService
+              echo "Automated install of the PiSCSI Service $(CONNECT_TYPE) complete!"
+          ;;
           -h|--help|h|help)
               showMenu
           ;;
@@ -1372,7 +1402,7 @@ function runChoice() {
 function readChoice() {
    choice=-1
 
-   until [ $choice -ge "0" ] && [ $choice -le "16" ]; do
+   until [ $choice -ge "0" ] && ([ $choice -eq "99" ] || [ $choice -le "16" ]) ; do
        echo -n "Enter your choice (0-16) or CTRL-C to exit: "
        read -r choice
    done

python/common/src/piscsi/file_cmds.py

@@ -4,7 +4,7 @@ Module for methods reading from and writing to the file system
 
 import logging
 import asyncio
-from os import walk
+from os import walk, path
 from functools import lru_cache
 from pathlib import PurePath, Path
 from zipfile import ZipFile, is_zipfile
@@ -57,7 +57,7 @@ class FileCmds:
     # noinspection PyMethodMayBeStatic
     def list_config_files(self):
         """
-        Finds fils with file ending CONFIG_FILE_SUFFIX in CFG_DIR.
+        Finds files with file ending CONFIG_FILE_SUFFIX in CFG_DIR.
         Returns a (list) of (str) files_list
         """
         files_list = []
@@ -67,6 +67,26 @@ class FileCmds:
                     files_list.append(file)
         return files_list
 
+    # noinspection PyMethodMayBeStatic
+    def list_subdirs(self, directory):
+        """
+        Finds subdirs within the (str) directory dir.
+        Returns a (list) of (str) subdir_list.
+        """
+        subdir_list = []
+        # Filter out file sharing meta data dirs
+        excluded_dirs = ("Network Trash Folder", "Temporary Items", "TheVolumeSettingsFolder")
+        for root, dirs, _files in walk(directory, topdown=True):
+            # Strip out dirs that begin with .
+            dirs[:] = [d for d in dirs if not d[0] == "."]
+            for dir in dirs:
+                if dir not in excluded_dirs:
+                    dirpath = path.join(root, dir)
+                    subdir_list.append(dirpath.replace(directory, "", 1))
+
+        subdir_list.sort()
+        return subdir_list
+
     def list_images(self):
         """
         Sends a IMAGE_FILES_INFO command to the server
@@ -146,7 +166,15 @@ class FileCmds:
         parameters = {"file_path": file_path}
 
         if file_path.exists():
-            file_path.unlink()
+            try:
+                file_path.unlink()
+            except OSError as error:
+                logging.error(error)
+                return {
+                    "status": False,
+                    "return_code": ReturnCodes.DELETEFILE_UNABLE_TO_DELETE,
+                    "parameters": parameters,
+                }
             return {
                 "status": True,
                 "return_code": ReturnCodes.DELETEFILE_SUCCESS,
@@ -159,18 +187,28 @@ class FileCmds:
         }
 
     # noinspection PyMethodMayBeStatic
-    def rename_file(self, file_path, target_path):
+    def rename_file(self, file_path, target_path, overwrite_target=False):
         """
         Takes:
          - (Path) file_path for the file to rename
          - (Path) target_path for the name to rename
+         - optional (bool) overwrite_target
         Returns (dict) with (bool) status, (str) msg, (dict) parameters
         """
         parameters = {"target_path": target_path}
         if not target_path.parent.exists():
             target_path.parent.mkdir(parents=True)
-        if target_path.parent.exists() and not target_path.exists():
+
-            file_path.rename(target_path)
+        if overwrite_target or not target_path.exists():
+            try:
+                file_path.rename(target_path)
+            except OSError as error:
+                logging.error(error)
+                return {
+                    "status": False,
+                    "return_code": ReturnCodes.RENAMEFILE_UNABLE_TO_MOVE,
+                    "parameters": parameters,
+                }
             return {
                 "status": True,
                 "return_code": ReturnCodes.RENAMEFILE_SUCCESS,
@@ -178,23 +216,33 @@ class FileCmds:
             }
         return {
             "status": False,
-            "return_code": ReturnCodes.RENAMEFILE_UNABLE_TO_MOVE,
+            "return_code": ReturnCodes.WRITEFILE_COULD_NOT_OVERWRITE,
             "parameters": parameters,
         }
 
     # noinspection PyMethodMayBeStatic
-    def copy_file(self, file_path, target_path):
+    def copy_file(self, file_path, target_path, overwrite_target=False):
         """
         Takes:
          - (Path) file_path for the file to copy from
          - (Path) target_path for the name to copy to
+         - optional (bool) overwrite_target
         Returns (dict) with (bool) status, (str) msg, (dict) parameters
         """
         parameters = {"target_path": target_path}
         if not target_path.parent.exists():
             target_path.parent.mkdir(parents=True)
-        if target_path.parent.exists() and not target_path.exists():
+
-            copyfile(str(file_path), str(target_path))
+        if overwrite_target or not target_path.exists():
+            try:
+                copyfile(str(file_path), str(target_path))
+            except OSError as error:
+                logging.error(error)
+                return {
+                    "status": False,
+                    "return_code": ReturnCodes.WRITEFILE_COULD_NOT_WRITE,
+                    "parameters": parameters,
+                }
             return {
                 "status": True,
                 "return_code": ReturnCodes.WRITEFILE_SUCCESS,
@@ -202,32 +250,41 @@ class FileCmds:
             }
         return {
             "status": False,
-            "return_code": ReturnCodes.WRITEFILE_COULD_NOT_WRITE,
+            "return_code": ReturnCodes.WRITEFILE_COULD_NOT_OVERWRITE,
             "parameters": parameters,
         }
 
-    def create_empty_image(self, target_path, size):
+    def create_empty_image(self, target_path, size, overwrite_target=False):
         """
-        Takes (Path) target_path and (int) size in bytes
+        Creates a new empty binary file to use as image.
-        Creates a new empty binary file to use as image
+        Takes:
+         - (Path) target_path
+         - (int) size in bytes
+         - optional (bool) overwrite_target
         Returns (dict) with (bool) status, (str) msg, (dict) parameters
         """
         parameters = {"target_path": target_path}
         if not target_path.parent.exists():
             target_path.parent.mkdir(parents=True)
-        if target_path.parent.exists() and not target_path.exists():
+
+        if overwrite_target or not target_path.exists():
             try:
                 with open(f"{target_path}", "wb") as out:
                     out.seek(size - 1)
                     out.write(b"\0")
             except OSError as error:
-                return {"status": False, "msg": str(error)}
+                logging.error(error)
+                return {
+                    "status": False,
+                    "return_code": ReturnCodes.WRITEFILE_COULD_NOT_WRITE,
+                    "parameters": parameters,
+                }
 
             return {"status": True, "msg": ""}
 
         return {
             "status": False,
-            "return_code": ReturnCodes.WRITEFILE_COULD_NOT_WRITE,
+            "return_code": ReturnCodes.WRITEFILE_COULD_NOT_OVERWRITE,
             "parameters": parameters,
         }
 
@@ -262,6 +319,7 @@ class FileCmds:
                         if self.rename_file(
                             Path(file["absolute_path"]),
                             prop_path,
+                            overwrite_target=True,
                         ):
                             properties_files_moved.append(
                                 {
@@ -317,56 +375,39 @@ class FileCmds:
         server_info = self.piscsi.get_server_info()
         full_file_path = Path(server_info["image_dir"]) / file_name
 
-        # Inject hfdisk commands to create Drive with correct partitions
+        # Inject hfdisk commands to create Mac partition table with HFS partitions
-        # https://www.codesrc.com/mediawiki/index.php/HFSFromScratch
-        # i                         initialize partition map
-        # continue with default first block
-        # C                         Create 1st partition with type specified next)
-        # continue with default
-        # 32                        32 blocks (required for HFS+)
-        # Driver_Partition          Partition Name
-        # Apple_Driver              Partition Type  (available types: Apple_Driver,
-        #                           Apple_Driver43, Apple_Free, Apple_HFS...)
-        # C                         Create 2nd partition with type specified next
-        # continue with default first block
-        # continue with default block size (rest of the disk)
-        # ${volumeName}             Partition name provided by user
-        # Apple_HFS                 Partition Type
-        # w                         Write partition map to disk
-        # y                         Confirm partition table
-        # p                         Print partition map
         if disk_format == "HFS":
             partitioning_tool = "hfdisk"
             commands = [
-                "i",
+                "i",  # Initialize partition map
-                "",
+                "",  # Continue with default first block
-                "C",
+                "C",  # Create 1st partition with type specified next)
-                "",
+                "",  # Continue with default
-                "32",
+                "32",  # 32 block (required for HFS+)
-                "Driver_Partition",
+                "Driver_Partition",  # Partition Name
-                "Apple_Driver",
+                "Apple_Driver",  # Partition Type
-                "C",
+                "C",  # Create 2nd partition with type specified next
-                "",
+                "",  # Continue with default first block
-                "",
+                "",  # Continue with default block size (rest of the disk)
-                volume_name,
+                volume_name,  # Partition name
-                "Apple_HFS",
+                "Apple_HFS",  # Partition Type
-                "w",
+                "w",  # Write partition map to disk
-                "y",
+                "y",  # Confirm partition table
-                "p",
+                "p",  # Print partition map (for the log)
             ]
-        # Create a DOS label, primary partition, W95 FAT type
+        # Inject fdisk commands to create primary FAT partition with MS-DOS label
         elif disk_format == "FAT":
             partitioning_tool = "fdisk"
             commands = [
-                "o",
+                "o",  # create a new empty DOS partition table
-                "n",
+                "n",  # add a new partition
-                "p",
+                "p",  # primary partition
-                "",
+                "",  # default partition number
-                "",
+                "",  # default first sector
-                "",
+                "",  # default last sector
-                "t",
+                "t",  # change partition type
-                "b",
+                "b",  # choose W95 FAT32 type
-                "w",
+                "w",  # write table to disk and exit
             ]
         try:
             process = Popen(

python/common/src/piscsi/return_codes.py

@@ -9,12 +9,14 @@ class ReturnCodes:
 
     DELETEFILE_SUCCESS = 0
     DELETEFILE_FILE_NOT_FOUND = 1
+    DELETEFILE_UNABLE_TO_DELETE = 2
     RENAMEFILE_SUCCESS = 10
     RENAMEFILE_UNABLE_TO_MOVE = 11
     DOWNLOADFILETOISO_SUCCESS = 20
     DOWNLOADTODIR_SUCCESS = 30
     WRITEFILE_SUCCESS = 40
     WRITEFILE_COULD_NOT_WRITE = 41
+    WRITEFILE_COULD_NOT_OVERWRITE = 42
     READCONFIG_SUCCESS = 50
     READCONFIG_COULD_NOT_READ = 51
     READCONFIG_INVALID_CONFIG_FILE_FORMAT = 52

python/common/src/piscsi/sys_cmds.py

@@ -21,23 +21,8 @@ class SysCmds:
     @staticmethod
     def running_env():
         """
-        Returns (str) git and (str) env
+        Returns (str) env, with details on the system hardware and software
-        git contains the git hash of the checked out code
-        env is the various system information where this app is running
         """
-        try:
-            ra_git_version = (
-                subprocess.run(
-                    ["git", "rev-parse", "HEAD"],
-                    capture_output=True,
-                    check=True,
-                )
-                .stdout.decode("utf-8")
-                .strip()
-            )
-        except subprocess.CalledProcessError as error:
-            logging.warning(SHELL_ERROR, error.cmd, error.stderr.decode("utf-8"))
-            ra_git_version = ""
 
         PROC_MODEL_PATH = "/proc/device-tree/model"
         SYS_VENDOR_PATH = "/sys/devices/virtual/dmi/id/sys_vendor"
@@ -67,7 +52,6 @@ class SysCmds:
 
         env = uname()
         return {
-            "git": ra_git_version,
             "env": f"{hardware}, {env.system} {env.release} {env.machine}",
         }
 

python/loopback_test/test.py

@@ -1,380 +0,0 @@
-#!/usr/bin/env python3
-# BSD 3-Clause License
-#
-# Copyright (c) 2021, akuker
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its
-#    contributors may be used to endorse or promote products derived from
-#    this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import RPi.GPIO as gpio
-import time
-
-pin_settle_delay = 0.01
-
-err_count = 0
-
-# Define constants for each of the SCSI signals, based upon their
-# raspberry pi pin number (since we're using BOARD mode of RPi.GPIO)
-scsi_d0_gpio = 19
-scsi_d1_gpio = 23
-scsi_d2_gpio = 32
-scsi_d3_gpio = 33
-scsi_d4_gpio = 8
-scsi_d5_gpio = 10
-scsi_d6_gpio = 36
-scsi_d7_gpio = 11
-scsi_dp_gpio = 12
-scsi_atn_gpio = 35
-scsi_rst_gpio = 38
-scsi_ack_gpio = 40
-scsi_req_gpio = 15
-scsi_msg_gpio = 16
-scsi_cd_gpio = 18
-scsi_io_gpio = 22
-scsi_bsy_gpio = 37
-scsi_sel_gpio = 13
-
-# Pin numbers of the direction controllers of the PiSCSI board
-piscsi_ind_gpio = 31
-piscsi_tad_gpio = 26
-piscsi_dtd_gpio = 24
-piscsi_none = -1
-
-# Matrix showing all of the SCSI signals, along what signal they're looped back to.
-# dir_ctrl indicates which direction control pin is associated with that output
-gpio_map = [
-    {
-        "gpio_num": scsi_d0_gpio,
-        "attached_to": scsi_ack_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d1_gpio,
-        "attached_to": scsi_sel_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d2_gpio,
-        "attached_to": scsi_atn_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d3_gpio,
-        "attached_to": scsi_rst_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d4_gpio,
-        "attached_to": scsi_cd_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d5_gpio,
-        "attached_to": scsi_io_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d6_gpio,
-        "attached_to": scsi_msg_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_d7_gpio,
-        "attached_to": scsi_req_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_dp_gpio,
-        "attached_to": scsi_bsy_gpio,
-        "dir_ctrl": piscsi_dtd_gpio,
-    },
-    {
-        "gpio_num": scsi_atn_gpio,
-        "attached_to": scsi_d2_gpio,
-        "dir_ctrl": piscsi_ind_gpio,
-    },
-    {
-        "gpio_num": scsi_rst_gpio,
-        "attached_to": scsi_d3_gpio,
-        "dir_ctrl": piscsi_ind_gpio,
-    },
-    {
-        "gpio_num": scsi_ack_gpio,
-        "attached_to": scsi_d0_gpio,
-        "dir_ctrl": piscsi_ind_gpio,
-    },
-    {
-        "gpio_num": scsi_req_gpio,
-        "attached_to": scsi_d7_gpio,
-        "dir_ctrl": piscsi_tad_gpio,
-    },
-    {
-        "gpio_num": scsi_msg_gpio,
-        "attached_to": scsi_d6_gpio,
-        "dir_ctrl": piscsi_tad_gpio,
-    },
-    {
-        "gpio_num": scsi_cd_gpio,
-        "attached_to": scsi_d4_gpio,
-        "dir_ctrl": piscsi_tad_gpio,
-    },
-    {
-        "gpio_num": scsi_io_gpio,
-        "attached_to": scsi_d5_gpio,
-        "dir_ctrl": piscsi_tad_gpio,
-    },
-    {
-        "gpio_num": scsi_bsy_gpio,
-        "attached_to": scsi_dp_gpio,
-        "dir_ctrl": piscsi_tad_gpio,
-    },
-    {
-        "gpio_num": scsi_sel_gpio,
-        "attached_to": scsi_d1_gpio,
-        "dir_ctrl": piscsi_ind_gpio,
-    },
-]
-
-# List of all of the SCSI signals that is also a dictionary to their human readable name
-scsi_signals = {
-    scsi_d0_gpio: "D0",
-    scsi_d1_gpio: "D1",
-    scsi_d2_gpio: "D2",
-    scsi_d3_gpio: "D3",
-    scsi_d4_gpio: "D4",
-    scsi_d5_gpio: "D5",
-    scsi_d6_gpio: "D6",
-    scsi_d7_gpio: "D7",
-    scsi_dp_gpio: "DP",
-    scsi_atn_gpio: "ATN",
-    scsi_rst_gpio: "RST",
-    scsi_ack_gpio: "ACK",
-    scsi_req_gpio: "REQ",
-    scsi_msg_gpio: "MSG",
-    scsi_cd_gpio: "CD",
-    scsi_io_gpio: "IO",
-    scsi_bsy_gpio: "BSY",
-    scsi_sel_gpio: "SEL",
-}
-
-
-# Debug function that just dumps the status of all of the scsi signals to the console
-def print_all():
-    for cur_gpio in gpio_map:
-        print(
-            cur_gpio["name"] + "=" + str(gpio.input(cur_gpio["gpio_num"])) + "  ",
-            end="",
-            flush=True,
-        )
-    print("")
-
-
-# Set transceivers IC1 and IC2 to OUTPUT
-def set_dtd_out():
-    gpio.output(piscsi_dtd_gpio, gpio.LOW)
-
-
-# Set transceivers IC1 and IC2 to INPUT
-def set_dtd_in():
-    gpio.output(piscsi_dtd_gpio, gpio.HIGH)
-
-
-# Set transceiver IC4 to OUTPUT
-def set_ind_out():
-    gpio.output(piscsi_ind_gpio, gpio.HIGH)
-
-
-# Set transceiver IC4 to INPUT
-def set_ind_in():
-    gpio.output(piscsi_ind_gpio, gpio.LOW)
-
-
-# Set transceiver IC3 to OUTPUT
-def set_tad_out():
-    gpio.output(piscsi_tad_gpio, gpio.HIGH)
-
-
-# Set transceiver IC3 to INPUT
-def set_tad_in():
-    gpio.output(piscsi_tad_gpio, gpio.LOW)
-
-
-# Set the specified transciever to an OUTPUT. All of the other transceivers
-# will be set to inputs. If a non-existent direction gpio is specified, this
-# will set all of the transceivers to inputs.
-def set_output_channel(out_gpio):
-    if out_gpio == piscsi_tad_gpio:
-        set_tad_out()
-    else:
-        set_tad_in()
-    if out_gpio == piscsi_dtd_gpio:
-        set_dtd_out()
-    else:
-        set_dtd_in()
-    if out_gpio == piscsi_ind_gpio:
-        set_ind_out()
-    else:
-        set_ind_in()
-
-
-# Main test procedure. This will execute for each of the SCSI pins to make sure its connected
-# properly.
-def test_gpio_pin(gpio_rec):
-    global err_count
-
-    set_output_channel(gpio_rec["dir_ctrl"])
-
-    ############################################
-    # set the test gpio low
-    gpio.output(gpio_rec["gpio_num"], gpio.LOW)
-
-    time.sleep(pin_settle_delay)
-
-    # loop through all of the gpios
-    for cur_gpio in scsi_signals:
-        # all of the gpios should be high except for the test gpio and the connected gpio
-        cur_val = gpio.input(cur_gpio)
-        if cur_gpio == gpio_rec["gpio_num"]:
-            if cur_val != gpio.LOW:
-                print(
-                    "Error: Test commanded GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " to be low, but it did not respond"
-                )
-                err_count = err_count + 1
-        elif cur_gpio == gpio_rec["attached_to"]:
-            if cur_val != gpio.LOW:
-                print(
-                    "Error: GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " should drive "
-                    + scsi_signals[gpio_rec["attached_to"]]
-                    + " low, but did not"
-                )
-                err_count = err_count + 1
-        else:
-            if cur_val != gpio.HIGH:
-                print(
-                    "Error: GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " incorrectly pulled "
-                    + scsi_signals[cur_gpio]
-                    + " LOW, when it shouldn't have"
-                )
-                err_count = err_count + 1
-
-    ############################################
-    # set the transceivers to input
-    set_output_channel(piscsi_none)
-
-    time.sleep(pin_settle_delay)
-
-    # loop through all of the gpios
-    for cur_gpio in scsi_signals:
-        # all of the gpios should be high except for the test gpio
-        cur_val = gpio.input(cur_gpio)
-        if cur_gpio == gpio_rec["gpio_num"]:
-            if cur_val != gpio.LOW:
-                print(
-                    "Error: Test commanded GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " to be low, but it did not respond"
-                )
-                err_count = err_count + 1
-        else:
-            if cur_val != gpio.HIGH:
-                print(
-                    "Error: GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " incorrectly pulled "
-                    + scsi_signals[cur_gpio]
-                    + " LOW, when it shouldn't have"
-                )
-                err_count = err_count + 1
-
-    # Set the transceiver back to output
-    set_output_channel(gpio_rec["dir_ctrl"])
-
-    #############################################
-    # set the test gpio high
-    gpio.output(gpio_rec["gpio_num"], gpio.HIGH)
-
-    time.sleep(pin_settle_delay)
-
-    # loop through all of the gpios
-    for cur_gpio in scsi_signals:
-        # all of the gpios should be high
-        cur_val = gpio.input(cur_gpio)
-        if cur_gpio == gpio_rec["gpio_num"]:
-            if cur_val != gpio.HIGH:
-                print(
-                    "Error: Test commanded GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " to be high, but it did not respond"
-                )
-                err_count = err_count + 1
-        else:
-            if cur_val != gpio.HIGH:
-                print(
-                    "Error: GPIO "
-                    + scsi_signals[gpio_rec["gpio_num"]]
-                    + " incorrectly pulled "
-                    + scsi_signals[cur_gpio]
-                    + " LOW, when it shouldn't have"
-                )
-                err_count = err_count + 1
-
-
-# Initialize the GPIO library, set all of the gpios associated with SCSI signals to outputs and set
-# all of the direction control gpios to outputs
-def setup():
-    gpio.setmode(gpio.BOARD)
-    gpio.setwarnings(False)
-    for cur_gpio in gpio_map:
-        gpio.setup(cur_gpio["gpio_num"], gpio.OUT, initial=gpio.HIGH)
-
-    # Setup direction control
-    gpio.setup(piscsi_ind_gpio, gpio.OUT)
-    gpio.setup(piscsi_tad_gpio, gpio.OUT)
-    gpio.setup(piscsi_dtd_gpio, gpio.OUT)
-
-
-# Main functions for running the actual test.
-if __name__ == "__main__":
-    # setup the GPIOs
-    setup()
-    # Test each SCSI signal in the gpio_map
-    for cur_gpio in gpio_map:
-        test_gpio_pin(cur_gpio)
-
-    # Print the test results
-    if err_count == 0:
-        print("-------- Test PASSED --------")
-    else:
-        print("!!!!!!!! Test FAILED !!!!!!!!")
-    print("Total errors: " + str(err_count))
-
-    gpio.cleanup()

python/web/src/return_code_mapper.py

@@ -14,6 +14,8 @@ class ReturnCodeMapper:
             _("File deleted: %(file_path)s"),
         ReturnCodes.DELETEFILE_FILE_NOT_FOUND:
             _("File to delete not found: %(file_path)s"),
+        ReturnCodes.DELETEFILE_UNABLE_TO_DELETE:
+            _("Could not delete file: %(file_path)s"),
         ReturnCodes.RENAMEFILE_SUCCESS:
             _("File moved to: %(target_path)s"),
         ReturnCodes.RENAMEFILE_UNABLE_TO_MOVE:
@@ -26,6 +28,8 @@ class ReturnCodeMapper:
             _("File created: %(target_path)s"),
         ReturnCodes.WRITEFILE_COULD_NOT_WRITE:
             _("Could not create file: %(target_path)s"),
+        ReturnCodes.WRITEFILE_COULD_NOT_OVERWRITE:
+            _("A file with name %(target_path)s already exists"),
         ReturnCodes.READCONFIG_SUCCESS:
             _("Loaded configurations from: %(file_name)s"),
         ReturnCodes.READCONFIG_COULD_NOT_READ:

python/web/src/static/themes/classic/style.css

@@ -104,6 +104,15 @@ ul.inline_list {
   list-style: none;
 }
 
+summary.dirname {
+  text-decoration: underline;
+  font-family: monospace;
+}
+
+summary.filename {
+  text-decoration: underline;
+}
+
 .dropzone, .dropzone * {
   box-sizing: border-box;
 }

python/web/src/static/themes/modern/style.css

@@ -53,6 +53,15 @@ input[type="radio"] {
   margin: 0 0.1rem 0 0.75rem;
 }
 
+div.notice {
+  background: var(--danger);
+  border-radius: var(--border-radius);
+  padding: 0.5rem;
+  font-size: 0.75rem;
+  display: inline-block;
+  color: #fff;
+}
+
 /*
  ------------------------------------------------------------------------------
  Tables
@@ -708,6 +717,16 @@ section#files p {
   margin-top: 1rem;
 }
 
+section#files details.subdir summary.dirname {
+  text-decoration: underline;
+  font-family: monospace;
+  margin: 0.5rem 0;
+}
+
+section#files details.contents summary.filename {
+  text-decoration: underline;
+}
+
 @media (max-width: 900px) {
   section#files table#images tr th:nth-child(2),
   section#files table#images tr td:nth-child(2) {

python/web/src/templates/base.html

@@ -23,20 +23,6 @@
     <meta name="theme-color" content="#ffffff">
 
     <link rel="stylesheet" href="{{ url_for('static', filename=current_theme_stylesheet) }}">
-
-    <script type="application/javascript">
-        var processNotify = function(Notification) {
-            document.getElementById("flash").innerHTML = "<div class=\"info\"><div>" + Notification + "{{ _(" This process may take a while, and will continue in the background if you navigate away from this page.") }}</div></div>";
-            window.scrollTo(0,0);
-        }
-
-        var shutdownNotify = function(Notification) {
-            document.getElementById("flash").innerHTML = "<div class=\"warning\"><div>" + Notification + "{{ _(" The Web Interface will become unresponsive momentarily. Reload this page after the Pi has started up again.") }}</div></div>";
-            window.scrollTo(0,0);
-        }
-    </script>
-
-    <script type="application/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.9.3/min/dropzone.min.js"></script>
 </head>
 
 <body class="{{ body_classes|join(' ') }}">
@@ -131,7 +117,7 @@
             {% endif %}
         </div>
         <div>
-            {{ _("PiSCSI version:") }} <b>{{ env["version"] }} <a href="https://github.com/PiSCSI/piscsi/commit/{{ env["running_env"]["git"] }}" target="_blank">{{ env["running_env"]["git"][:7] }}</a></b>
+	    {{ _("PiSCSI software version:") }} <b>{{ env["version"] }}</b>
         </div>
         <div>
             {{ _("Hardware and OS:") }} {{ env["running_env"]["env"] }}

python/web/src/templates/index.html

@@ -1,6 +1,18 @@
 {% extends "base.html" %}
 {% block content %}
 
+<script type="application/javascript">
+    var processNotify = function(Notification) {
+        document.getElementById("flash").innerHTML = "<div class=\"info\"><div>" + Notification + "{{ _(" This process may take a while, and will continue in the background if you navigate away from this page.") }}</div></div>";
+        window.scrollTo(0,0);
+    }
+
+    var shutdownNotify = function(Notification) {
+        document.getElementById("flash").innerHTML = "<div class=\"warning\"><div>" + Notification + "{{ _(" The Web Interface will become unresponsive momentarily. Reload this page after the Pi has started up again.") }}</div></div>";
+        window.scrollTo(0,0);
+    }
+</script>
+
 <section id="current-config">
 <details>
     <summary class="heading">
@@ -197,6 +209,19 @@
     </ul>
 </details>
 
+{% if not files|length: %}
+<div class="notice">
+    {{ _("The images directory is currently empty.") }}
+</div>
+{% else %}
+
+<div>
+{% for subdir, group in formatted_image_files.items() %}
+
+<details class="subdir"{% if subdir == "images/" %} open{% endif %}>
+    <summary class="dirname">
+        {{ subdir }}
+    </summary>
 <table id="images" border="black" cellpadding="3" summary="List of files in the image directory">
     <tbody>
         <tr>
@@ -204,19 +229,12 @@
             <th scope="col">{{ _("Size") }}</th>
             <th scope="col">{{ _("Actions") }}</th>
         </tr>
-        {% if not files|length: %}
+        {% for file in group|sort(attribute='name') %}
-            <tr class="directory-empty">
-                <td colspan="3">
-                    {{ _("The images directory is currently empty.") }}
-                </td>
-            </tr>
-        {% endif %}
-        {% for file in files|sort(attribute='name') %}
         <tr>
             {% if file["prop"] %}
             <td>
-                <details>
+                <details class="contents">
-                    <summary>
+                    <summary class="filename">
                         {{ file["name"] }}
                     </summary>
                     <ul class="inline_list">
@@ -232,8 +250,8 @@
             </td>
             {% elif file["archive_contents"] %}
             <td>
-                <details>
+                <details class="contents">
-                    <summary>
+                    <summary class="filename">
                         {{ file["name"] }}
                     </summary>
                     <ul class="inline_list">
@@ -241,8 +259,8 @@
                             {% if not member["is_properties_file"] %}
                                 <li>
                                     {% if member["related_properties_file"] %}
-                                        <details>
+                                        <details id="contents">
-                                            <summary>
+                                            <summary class="filename">
                                                 <label>{{ member["path"] }}</label>
                                                 <form action="/files/extract_image" method="post" class="file-extract">
                                                     <input name="archive_file" type="hidden" value="{{ file['name'] }}">
@@ -323,14 +341,14 @@
                     <input type="submit" value="{{ _("Attach") }}" title="{{ _("Attach") }}">
                 {% endif %}
                 </form>
-                <form action="/files/rename" method="post" class="file-rename" onsubmit="var new_file_name = prompt('{{ _("Enter new file name for: %(file_name)s", file_name=file["name"]) }}', '{{ file['name'] }}'); if (new_file_name === null) event.preventDefault(); document.getElementById('new_file_name_{{ loop.index }}').value = new_file_name;">
+                <form action="/files/rename" method="post" class="file-rename" onsubmit="var new_file_name = prompt('{{ _("Enter new file name for: %(file_name)s", file_name=file["name"]) }}', '{{ file['name'] }}'); if (new_file_name === null) event.preventDefault(); document.getElementById('new_file_name_{{ subdir }}_{{ loop.index }}').value = new_file_name;">
                     <input name="file_name" type="hidden" value="{{ file['name'] }}">
-                    <input name="new_file_name" id="new_file_name_{{ loop.index }}" type="hidden" value="">
+                    <input name="new_file_name" id="new_file_name_{{ subdir }}_{{ loop.index }}" type="hidden" value="">
                     <input type="submit" value="{{ _("Rename") }}" title="{{ _("Rename") }}">
                 </form>
-                <form action="/files/copy" method="post" class="file-copy" onsubmit="var copy_file_name = prompt('{{ _("Save copy of %(file_name)s as:", file_name=file["name"]) }}', '{{ file['name'] }}'); if (copy_file_name === null) event.preventDefault(); document.getElementById('copy_file_name_{{ loop.index }}').value = copy_file_name;">
+                <form action="/files/copy" method="post" class="file-copy" onsubmit="var copy_file_name = prompt('{{ _("Save copy of %(file_name)s as:", file_name=file["name"]) }}', '{{ file['name'] }}'); if (copy_file_name === null) event.preventDefault(); document.getElementById('copy_file_name_{{ subdir }}_{{ loop.index }}').value = copy_file_name;">
                     <input name="file_name" type="hidden" value="{{ file['name'] }}">
-                    <input name="copy_file_name" id="copy_file_name_{{ loop.index }}" type="hidden" value="">
+                    <input name="copy_file_name" id="copy_file_name_{{ subdir }}_{{ loop.index }}" type="hidden" value="">
                     <input type="submit" value="{{ _("Copy") }}" title="{{ _("Copy") }}">
                 </form>
                 <form action="/files/delete" method="post" class="file-delete" onsubmit="return confirm('{{ _("Delete file: %(file_name)s?", file_name=file["name"]) }}')">
@@ -349,6 +367,10 @@
         {% endfor %}
     </tbody>
 </table>
+</details>
+{% endfor %}
+</div>
+{% endif %}
 <p><small>{{ _("%(disk_space)s MiB disk space remaining on the system", disk_space=env["free_disk_space"]) }}</small></p>
 </section>
 
@@ -361,8 +383,11 @@
     </summary>
     <ul>
         <li>{{ _("Disk Images") }} = {{ env["image_dir"] }}</li>
+	{% if file_server_dir_exists %}
         <li>{{ _("Shared Files") }} = {{ FILE_SERVER_DIR }}</li>
-        <li>{{ _("To access shared files remotely, you may have to install one of the file servers first.") }}</li>
+	{% else %}
+        <li>{{ _("Install a file server and create the shared files directory in order to share files between the Pi and your vintage computers.") }}</li>
+	{% endif %}
     </ul>
 </details>
 
@@ -371,8 +396,22 @@
     <input name="url" id="download_url" required="" type="url">
     <input type="radio" name="destination" id="disk_images" value="disk_images" checked="checked">
     <label for="disk_images">{{ _("Disk Images") }}</label>
+    <select name="images_subdir" id="images_subdir">
+    {% for dir in images_subdirs %}
+    <option value="{{dir}}">{{dir}}</option>
+    {% endfor %}
+    <option value="/" selected>/</option>
+    </select>
+    {% if file_server_dir_exists %}
     <input type="radio" name="destination" id="shared_files" value="shared_files">
     <label for="shared_files">{{ _("Shared Files") }}</label>
+    <select name="shared_subdir" id="shared_subdir">
+    {% for dir in shared_subdirs %}
+    <option value="{{dir}}">{{dir}}</option>
+    {% endfor %}
+    <option value="/" selected>/</option>
+    </select>
+    {% endif %}
     <input type="submit" value="{{ _("Download") }}" onclick="processNotify('{{ _("Downloading File...") }}')">
 </form>
 </section>
@@ -380,6 +419,14 @@
 <section id="upload">
     <a href="/upload" target="_blank"><p>{{ _("Upload Files (new tab)") }}</p></a>
 </section>
+<noscript>
+    <style type="text/css">
+        section#upload { display: none; }
+    </style>
+    <div class="notice">
+        {{ _("The file uploading functionality requires JavaScript.") }}
+    </div>
+</noscript>
 
 <hr/>
 

python/web/src/templates/upload.html

@@ -6,7 +6,9 @@
     <li>{{ _("The largest file size accepted in this form is %(max_file_size)s MiB. Use other file transfer means for larger files.", max_file_size=max_file_size) }}</li>
     <li>{{ _("You have to manually clean up partially uploaded files, as a result of cancelling the upload or closing this page.") }}</li>
     <li>{{ _("Disk Images") }} = {{ env["image_dir"] }}</li>
+    {% if file_server_dir_exists %}
     <li>{{ _("Shared Files") }} = {{ FILE_SERVER_DIR }}</li>
+    {% endif %}
     <li>{{ _("PiSCSI Config") }} = {{ CFG_DIR }}</li>
 </ul>
 
@@ -14,12 +16,28 @@
 <form name="dropper" action="/files/upload" method="post" class="dropzone dz-clickable" enctype="multipart/form-data" id="dropper">
     <input type="radio" name="destination" id="disk_images" value="disk_images" checked="checked">
     <label for="disk_images">{{ _("Disk Images") }}</label>
+    <select name="images_subdir" id="images_subdir">
+    {% for dir in images_subdirs %}
+    <option value="{{dir}}">{{dir}}</option>
+    {% endfor %}
+    <option value="/" selected>/</option>
+    </select>
+    {% if file_server_dir_exists %}
     <input type="radio" name="destination" id="shared_files" value="shared_files">
     <label for="shared_files">{{ _("Shared Files") }}</label>
+    <select name="shared_subdir" id="shared_subdir">
+    {% for dir in shared_subdirs %}
+    <option value="{{dir}}">{{dir}}</option>
+    {% endfor %}
+    <option value="/" selected>/</option>
+    </select>
+    {% endif %}
     <input type="radio" name="destination" id="piscsi_config" value="piscsi_config">
     <label for="piscsi_config">{{ _("PiSCSI Config") }}</label>
 </form>
 
+<script type="application/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.9.3/min/dropzone.min.js"></script>
+
 <script type="application/javascript">
     Dropzone.options.dropper = {
         paramName: 'file',
@@ -54,4 +72,10 @@
     }
 </script>
 
+<noscript>
+    <div class="noscriptmsg">
+        {{ _("The file uploading functionality requires JavaScript.") }}
+    </div>
+</noscript>
+
 {% endblock content %}

python/web/src/web.py

@@ -24,7 +24,6 @@ from flask import (
     send_from_directory,
     make_response,
     session,
-    abort,
     jsonify,
 )
 
@@ -44,11 +43,13 @@ from return_code_mapper import ReturnCodeMapper
 from socket_cmds_flask import SocketCmdsFlask
 
 from web_utils import (
+    working_dirs_exist,
     sort_and_format_devices,
     get_valid_scsi_ids,
     map_device_types_and_names,
     get_device_name,
     map_image_file_descriptions,
+    format_image_list,
     format_drive_properties,
     get_properties_by_drive_name,
     auth_active,
@@ -208,27 +209,15 @@ def index():
     """
     Sets up data structures for and renders the index page
     """
-    if not piscsi_cmd.is_token_auth()["status"] and not APP.config["PISCSI_TOKEN"]:
-        abort(
-            403,
-            _(
-                "PiSCSI is password protected. "
-                "Start the Web Interface with the --password parameter."
-            ),
-        )
-
     server_info = piscsi_cmd.get_server_info()
+    working_dirs_exist((server_info["image_dir"], CFG_DIR))
+
     devices = piscsi_cmd.list_devices()
     device_types = map_device_types_and_names(piscsi_cmd.get_device_types()["device_types"])
     image_files = file_cmd.list_images()
     config_files = file_cmd.list_config_files()
     ip_addr, host = sys_cmd.get_ip_and_host()
-
+    formatted_image_files = format_image_list(image_files["files"], device_types)
-    extended_image_files = []
-    for image in image_files["files"]:
-        if image["detected_type"] != "UNDEFINED":
-            image["detected_type_name"] = device_types[image["detected_type"]]["name"]
-        extended_image_files.append(image)
 
     attached_images = []
     units = 0
@@ -266,7 +255,8 @@ def index():
         bridge_configured=sys_cmd.is_bridge_setup(),
         devices=formatted_devices,
         attached_images=attached_images,
-        files=extended_image_files,
+        formatted_image_files=formatted_image_files,
+        files=image_files["files"],
         config_files=config_files,
         device_types=device_types,
         scan_depth=server_info["scan_depth"],
@@ -278,6 +268,9 @@ def index():
         image_suffixes_to_create=image_suffixes_to_create,
         valid_image_suffixes=valid_image_suffixes,
         drive_properties=format_drive_properties(APP.config["PISCSI_DRIVE_PROPERTIES"]),
+        images_subdirs=file_cmd.list_subdirs(server_info["image_dir"]),
+        shared_subdirs=file_cmd.list_subdirs(FILE_SERVER_DIR),
+        file_server_dir_exists=Path(FILE_SERVER_DIR).exists(),
         RESERVATIONS=RESERVATIONS,
         CFG_DIR=CFG_DIR,
         FILE_SERVER_DIR=FILE_SERVER_DIR,
@@ -303,6 +296,8 @@ def drive_list():
     """
     Sets up the data structures and kicks off the rendering of the drive list page
     """
+    server_info = piscsi_cmd.get_server_info()
+    working_dirs_exist((server_info["image_dir"], CFG_DIR))
 
     return response(
         template="drives.html",
@@ -317,10 +312,15 @@ def upload_page():
     """
     Sets up the data structures and kicks off the rendering of the file uploading page
     """
+    server_info = piscsi_cmd.get_server_info()
+    working_dirs_exist((server_info["image_dir"], CFG_DIR))
 
     return response(
         template="upload.html",
         page_title=_("PiSCSI File Upload"),
+        images_subdirs=file_cmd.list_subdirs(server_info["image_dir"]),
+        shared_subdirs=file_cmd.list_subdirs(FILE_SERVER_DIR),
+        file_server_dir_exists=Path(FILE_SERVER_DIR).exists(),
         max_file_size=int(int(MAX_FILE_SIZE) / 1024 / 1024),
         CFG_DIR=CFG_DIR,
         FILE_SERVER_DIR=FILE_SERVER_DIR,
@@ -514,6 +514,7 @@ def show_diskinfo():
     if not safe_path["status"]:
         return response(error=True, message=safe_path["msg"])
     server_info = piscsi_cmd.get_server_info()
+    working_dirs_exist((server_info["image_dir"], CFG_DIR))
     returncode, diskinfo = sys_cmd.get_diskinfo(Path(server_info["image_dir"]) / file_name)
     if returncode == 0:
         return response(
@@ -959,11 +960,22 @@ def download_file():
     """
     destination = request.form.get("destination")
     url = request.form.get("url")
-    if destination == "shared_files":
+    images_subdir = request.form.get("images_subdir")
-        destination_dir = FILE_SERVER_DIR
+    shared_subdir = request.form.get("shared_subdir")
-    else:
+    if destination == "disk_images":
+        safe_path = is_safe_path(Path("." + images_subdir))
+        if not safe_path["status"]:
+            return make_response(safe_path["msg"], 403)
         server_info = piscsi_cmd.get_server_info()
-        destination_dir = server_info["image_dir"]
+        destination_dir = server_info["image_dir"] + images_subdir
+    elif destination == "shared_files":
+        safe_path = is_safe_path(Path("." + shared_subdir))
+        if not safe_path["status"]:
+            return make_response(safe_path["msg"], 403)
+        destination_dir = FILE_SERVER_DIR + shared_subdir
+    else:
+        return response(error=True, message=_("Unknown destination"))
+
     process = file_cmd.download_to_dir(url, destination_dir, Path(url).name)
     process = ReturnCodeMapper.add_msg(process)
     if process["status"]:
@@ -990,15 +1002,23 @@ def upload_file():
         return make_response(auth["msg"], 403)
 
     destination = request.form.get("destination")
+    images_subdir = request.form.get("images_subdir")
+    shared_subdir = request.form.get("shared_subdir")
     if destination == "disk_images":
+        safe_path = is_safe_path(Path("." + images_subdir))
+        if not safe_path["status"]:
+            return make_response(safe_path["msg"], 403)
         server_info = piscsi_cmd.get_server_info()
-        destination_dir = server_info["image_dir"]
+        destination_dir = server_info["image_dir"] + images_subdir
     elif destination == "shared_files":
-        destination_dir = FILE_SERVER_DIR
+        safe_path = is_safe_path(Path("." + shared_subdir))
+        if not safe_path["status"]:
+            return make_response(safe_path["msg"], 403)
+        destination_dir = FILE_SERVER_DIR + shared_subdir
     elif destination == "piscsi_config":
         destination_dir = CFG_DIR
     else:
-        return make_response("Invalid destination", 403)
+        return make_response(_("Unknown destination"), 403)
 
     return upload_with_dropzonejs(destination_dir)
 
@@ -1453,6 +1473,12 @@ if __name__ == "__main__":
     file_cmd = FileCmds(sock_cmd=sock_cmd, piscsi=piscsi_cmd, token=APP.config["PISCSI_TOKEN"])
     sys_cmd = SysCmds()
 
+    if not piscsi_cmd.is_token_auth()["status"] and not APP.config["PISCSI_TOKEN"]:
+        raise Exception(
+            "PiSCSI is password protected. "
+            "Start the Web Interface with the --password parameter."
+        )
+
     if Path(f"{CFG_DIR}/{DEFAULT_CONFIG}").is_file():
         file_cmd.read_config(DEFAULT_CONFIG)
     if Path(f"{DRIVE_PROPERTIES_FILE}").is_file():

python/web/src/web_utils.py

@@ -7,14 +7,28 @@ from grp import getgrall
 from os import path
 from pathlib import Path
 from ua_parser import user_agent_parser
+from re import findall
 
-from flask import request, make_response
+from flask import request, make_response, abort
 from flask_babel import _
 from werkzeug.utils import secure_filename
 
 from piscsi.sys_cmds import SysCmds
 
 
+def working_dirs_exist(working_dirs):
+    """
+    Method for validating that working dirs exist.
+    Takes (tuple) of (str) working_dirs with paths to required dirs.
+    """
+    for dir_path in working_dirs:
+        if not Path(dir_path).exists():
+            abort(
+                503,
+                _(f"Please create directory: {dir_path}"),
+            )
+
+
 def get_valid_scsi_ids(devices, reserved_ids):
     """
     Takes a list of (dict)s devices, and list of (int)s reserved_ids.
@@ -146,6 +160,33 @@ def get_image_description(file_suffix):
     return file_suffix
 
 
+def format_image_list(image_files, device_types=None):
+    """
+    Takes a (list) of (dict) image_files and optional (list) device_types
+    Returns a formatted (dict) with groups of image_files per subdir key
+    """
+
+    root_image_files = []
+    subdir_image_files = {}
+    for image in image_files:
+        if (image["detected_type"] != "UNDEFINED") and device_types:
+            image["detected_type_name"] = device_types[image["detected_type"]]["name"]
+        subdir_path = findall("^.*/", image["name"])
+        if subdir_path:
+            subdir = subdir_path[0]
+            if f"images/{subdir}" in subdir_image_files.keys():
+                subdir_image_files[f"images/{subdir}"].append(image)
+            else:
+                subdir_image_files[f"images/{subdir}"] = [image]
+        else:
+            root_image_files.append(image)
+
+    formatted_image_files = dict(sorted(subdir_image_files.items()))
+    if root_image_files:
+        formatted_image_files["images/"] = root_image_files
+    return formatted_image_files
+
+
 def format_drive_properties(drive_properties):
     """
     Takes a (dict) with structured drive properties data
@@ -256,10 +297,10 @@ def is_safe_path(file_name):
     Returns True if the path is safe
     Returns False if the path is either absolute, or tries to traverse the file system
     """
-    if file_name.is_absolute() or ".." in str(file_name):
+    if file_name.is_absolute() or ".." in str(file_name) or str(file_name)[0] == "~":
         return {
             "status": False,
-            "msg": _("%(file_name)s is not a valid path", file_name=file_name),
+            "msg": _("No permission to use path '%(file_name)s'", file_name=file_name),
         }
 
     return {"status": True, "msg": ""}

python/web/tests/api/test_files.py

@@ -207,7 +207,8 @@ def test_extract_file(
     http_client.post(
         "/files/download_url",
         data={
-            "destination": "images",
+            "destination": "disk_images",
+            "images_subdir": "/",
             "url": url,
         },
     )
@@ -254,6 +255,7 @@ def test_upload_file(http_client, delete_file):
 
             form_data = {
                 "destination": "disk_images",
+                "images_subdir": "/",
                 "dzuuid": str(uuid.uuid4()),
                 "dzchunkindex": chunk_number,
                 "dzchunksize": chunk_size,
@@ -333,6 +335,7 @@ def test_download_properties(http_client, list_files, delete_file):
 def test_download_url_to_dir(env, httpserver, http_client, list_files, delete_file):
     file_name = str(uuid.uuid4())
     http_path = f"/images/{file_name}"
+    subdir = "/"
     url = httpserver.url_for(http_path)
 
     with open("tests/assets/test_image.hds", mode="rb") as file:
@@ -346,7 +349,8 @@ def test_download_url_to_dir(env, httpserver, http_client, list_files, delete_fi
     response = http_client.post(
         "/files/download_url",
         data={
-            "destination": "images",
+            "destination": "disk_images",
+            "images_subdir": subdir,
             "url": url,
         },
     )
@@ -357,7 +361,8 @@ def test_download_url_to_dir(env, httpserver, http_client, list_files, delete_fi
     assert response_data["status"] == STATUS_SUCCESS
     assert file_name in list_files()
     assert (
-        response_data["messages"][0]["message"] == f"{file_name} downloaded to {env['images_dir']}"
+        response_data["messages"][0]["message"]
+        == f"{file_name} downloaded to {env['images_dir']}{subdir}"
     )
 
     # Cleanup