Add tree file read support (#16)

* Add tree file read support

* Update copyright year

* Add tree file write support

README.md

@@ -8,11 +8,10 @@ Being a work in progress, be warned that this code is likely to corrupt drive im
 There are binaries [here](https://github.com/tjboldt/ProDOS-Utilities/releases/latest)
 
 ## Current TODO list
-1. Allow > 128 KB file support
-2. Create/Delete directories
-3. Add file/directory tests
-4. Add rename
-5. Add in-place file/directory moves
+1. Create/Delete directories
+2. Add file/directory tests
+3. Add rename
+4. Add in-place file/directory moves
 
 ## Example commands and output
 

main.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -17,7 +17,7 @@ import (
 	"github.com/tjboldt/ProDOS-Utilities/prodos"
 )
 
-const version = "0.4.1"
+const version = "0.4.2"
 
 func main() {
 	var fileName string

prodos/basic.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/basic_test.go

@@ -1,3 +1,9 @@
+// Copyright Terence J. Boldt (c)2022-2023
+// Use of this source code is governed by an MIT
+// license that can be found in the LICENSE file.
+
+// This file provides tests for conversion between BASIC and text
+
 package prodos
 
 import (

prodos/bitmap.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/bitmap_test.go

@@ -1,3 +1,10 @@
+// Copyright Terence J. Boldt (c)2021-2023
+// Use of this source code is governed by an MIT
+// license that can be found in the LICENSE file.
+
+// This file provides tests for access to volum bitmap on
+// a ProDOS drive image
+
 package prodos
 
 import (

prodos/block.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/directory.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -281,7 +281,7 @@ func parseVolumeHeader(buffer []byte) VolumeHeader {
 	bitmapBlock := int(buffer[39]) + int(buffer[40])*256
 	totalBlocks := int(buffer[41]) + int(buffer[42])*256
 
-	if version > 0 || minVersion > 0 {
+	if minVersion > 0 {
 		panic("Unsupported ProDOS version")
 	}
 

prodos/doc.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/file.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -8,12 +8,9 @@
 package prodos
 
 import (
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -45,93 +42,6 @@ func LoadFile(reader io.ReaderAt, path string) ([]byte, error) {
 	return buffer, nil
 }
 
-// WriteFileFromFile writes a file to a ProDOS volume from a host file
-func WriteFileFromFile(readerWriter ReaderWriterAt, pathName string, fileType int, auxType int, inFileName string) error {
-	inFile, err := os.ReadFile(inFileName)
-	if err != nil {
-		return err
-	}
-
-	if auxType == 0 && fileType == 0 {
-		auxType, fileType, inFile, err = convertFileByType(inFileName, inFile)
-		if err != nil {
-			return err
-		}
-	}
-
-	if len(pathName) == 0 {
-		_, pathName = filepath.Split(inFileName)
-		pathName = strings.ToUpper(pathName)
-		ext := filepath.Ext(pathName)
-		if len(ext) > 0 {
-			switch ext {
-			case ".SYS", ".TXT", ".BAS", ".BIN":
-				pathName = strings.TrimSuffix(pathName, ext)
-			}
-		}
-	}
-
-	return WriteFile(readerWriter, pathName, fileType, auxType, inFile)
-}
-
-func convertFileByType(inFileName string, inFile []byte) (int, int, []byte, error) {
-	fileType := 0x06  // default to BIN
-	auxType := 0x2000 // default to $2000
-
-	var err error
-
-	// Check for an AppleSingle file as produced by cc65
-	if // Magic number
-	binary.BigEndian.Uint32(inFile[0x00:]) == 0x00051600 &&
-		// Version number
-		binary.BigEndian.Uint32(inFile[0x04:]) == 0x00020000 &&
-		// Number of entries
-		binary.BigEndian.Uint16(inFile[0x18:]) == 0x0002 &&
-		// Data Fork ID
-		binary.BigEndian.Uint32(inFile[0x1A:]) == 0x00000001 &&
-		// Offset
-		binary.BigEndian.Uint32(inFile[0x1E:]) == 0x0000003A &&
-		// Length
-		binary.BigEndian.Uint32(inFile[0x22:]) == uint32(len(inFile))-0x3A &&
-		// ProDOS File Info ID
-		binary.BigEndian.Uint32(inFile[0x26:]) == 0x0000000B &&
-		// Offset
-		binary.BigEndian.Uint32(inFile[0x2A:]) == 0x00000032 &&
-		// Length
-		binary.BigEndian.Uint32(inFile[0x2E:]) == 0x00000008 {
-
-		fileType = int(binary.BigEndian.Uint16(inFile[0x34:]))
-		auxType = int(binary.BigEndian.Uint32(inFile[0x36:]))
-		inFile = inFile[0x3A:]
-	} else {
-		// use extension to determine file type
-		ext := strings.ToUpper(filepath.Ext(inFileName))
-
-		switch ext {
-		case ".BAS":
-			inFile, err = ConvertTextToBasic(string(inFile))
-			fileType = 0xFC
-			auxType = 0x0801
-
-			if err != nil {
-				return 0, 0, nil, err
-			}
-		case ".SYS":
-			fileType = 0xFF
-			auxType = 0x2000
-		case ".BIN":
-			fileType = 0x06
-			auxType = 0x2000
-		case ".TXT":
-			inFile = []byte(strings.ReplaceAll(strings.ReplaceAll(string(inFile), "\r\n", "r"), "\n", "\r"))
-			fileType = 0x04
-			auxType = 0x0000
-		}
-	}
-
-	return auxType, fileType, inFile, err
-}
-
 // WriteFile writes a file to a ProDOS volume from a byte array
 func WriteFile(readerWriter ReaderWriterAt, path string, fileType int, auxType int, buffer []byte) error {
 	directory, fileName := GetDirectoryAndFileNameFromPath(path)
@@ -149,7 +59,7 @@ func WriteFile(readerWriter ReaderWriterAt, path string, fileType int, auxType i
 
 	// seedling file
 	if len(buffer) <= 0x200 {
-		WriteBlock(readerWriter, blockList[0], buffer)
+		writeSeedlingFile(readerWriter, buffer, blockList)
 	}
 
 	// sapling file needs index block
@@ -218,7 +128,7 @@ func DeleteFile(readerWriter ReaderWriterAt, path string) error {
 	}
 
 	// free the blocks
-	blocks, err := getBlocklist(readerWriter, fileEntry)
+	blocks, err := getAllBlockList(readerWriter, fileEntry)
 	if err != nil {
 		return err
 	}
@@ -280,6 +190,10 @@ func updateVolumeBitmap(readerWriter ReaderWriterAt, blockList []int) error {
 	return writeVolumeBitmap(readerWriter, volumeBitmap)
 }
 
+func writeSeedlingFile(writer io.WriterAt, buffer []byte, blockList []int) {
+	WriteBlock(writer, blockList[0], buffer)
+}
+
 func writeSaplingFile(writer io.WriterAt, buffer []byte, blockList []int) {
 	// write index block with pointers to data blocks
 	indexBuffer := make([]byte, 512)
@@ -287,6 +201,9 @@ func writeSaplingFile(writer io.WriterAt, buffer []byte, blockList []int) {
 		if i < len(blockList)-1 {
 			indexBuffer[i] = byte(blockList[i+1] & 0x00FF)
 			indexBuffer[i+256] = byte(blockList[i+1] >> 8)
+		} else {
+			indexBuffer[i] = 0
+			indexBuffer[i+256] = 0
 		}
 	}
 	WriteBlock(writer, blockList[0], indexBuffer)
@@ -313,11 +230,69 @@ func writeSaplingFile(writer io.WriterAt, buffer []byte, blockList []int) {
 }
 
 func writeTreeFile(writer io.WriterAt, buffer []byte, blockList []int) {
+	// write master index block with pointers to index blocks
+	indexBuffer := make([]byte, 512)
+	numberOfIndexBlocks := len(blockList)/256 + 1
+	if len(blockList)%256 == 0 {
+		numberOfIndexBlocks--
+	}
+	for i := 0; i < 256; i++ {
+		if i < numberOfIndexBlocks {
+			indexBuffer[i] = byte(blockList[i+1] & 0x00FF)
+			indexBuffer[i+256] = byte(blockList[i+1] >> 8)
+		} else {
+			indexBuffer[i] = 0
+			indexBuffer[i+256] = 0
+		}
+	}
+	WriteBlock(writer, blockList[0], indexBuffer)
+	numberOfIndexBlocks++
 
+	// write index blocks
+	for i := 0; i < len(blockList)/256+1; i++ {
+		for j := 0; j < 256; j++ {
+			if i*256+j < len(blockList)-numberOfIndexBlocks {
+				indexBuffer[j] = byte(blockList[i*256+numberOfIndexBlocks+j] & 0x00FF)
+				indexBuffer[j+256] = byte(blockList[i*256+j+2] >> 8)
+			} else {
+				indexBuffer[j] = 0
+				indexBuffer[j+256] = 0
+			}
+		}
+		WriteBlock(writer, blockList[i+1], indexBuffer)
+	}
+
+	// write all data blocks
+	blockBuffer := make([]byte, 512)
+	blockPointer := 0
+	blockIndexNumber := numberOfIndexBlocks
+	for i := 0; i < len(buffer); i++ {
+		blockBuffer[blockPointer] = buffer[i]
+		if blockPointer == 511 {
+			WriteBlock(writer, blockList[blockIndexNumber], blockBuffer)
+			blockPointer = 0
+			blockIndexNumber++
+		} else if i == len(buffer)-1 {
+			for j := blockPointer; j < 512; j++ {
+				blockBuffer[j] = 0
+			}
+			WriteBlock(writer, blockList[blockIndexNumber], blockBuffer)
+		} else {
+			blockPointer++
+		}
+	}
+}
+
+func getDataBlocklist(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
+	return getBlocklist(reader, fileEntry, true)
+}
+
+func getAllBlockList(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
+	return getBlocklist(reader, fileEntry, false)
 }
 
 // Returns all blocks, including index blocks
-func getBlocklist(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
+func getBlocklist(reader io.ReaderAt, fileEntry FileEntry, dataOnly bool) ([]int, error) {
 	blocks := make([]int, fileEntry.BlocksUsed)
 
 	switch fileEntry.StorageType {
@@ -329,58 +304,54 @@ func getBlocklist(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
 		if err != nil {
 			return nil, err
 		}
+		blockOffset := 0
+		if !dataOnly {
 			blocks[0] = fileEntry.KeyPointer
+			blockOffset = 1
+		}
 		for i := 0; i < fileEntry.BlocksUsed-1; i++ {
-			blocks[i+1] = int(index[i]) + int(index[i+256])*256
+			blocks[i+blockOffset] = int(index[i]) + int(index[i+256])*256
 		}
 		return blocks, nil
 	case StorageTree:
+		dataBlocks := make([]int, fileEntry.BlocksUsed)
+		numberOfIndexBlocks := fileEntry.BlocksUsed/256 + 1
+		if fileEntry.BlocksUsed%256 != 0 {
+			numberOfIndexBlocks++
+		}
+		indexBlocks := make([]int, numberOfIndexBlocks)
 		masterIndex, err := ReadBlock(reader, fileEntry.KeyPointer)
 		if err != nil {
 			return nil, err
 		}
-		blocks[0] = fileEntry.KeyPointer
+		indexBlocks[0] = fileEntry.KeyPointer
+		indexBlockCount := 1
+
 		for i := 0; i < 128; i++ {
-			index, err := ReadBlock(reader, int(masterIndex[i])+int(masterIndex[i+256])*256)
+			indexBlock := int(masterIndex[i]) + int(masterIndex[i+256])*256
+			if indexBlock == 0 {
+				break
+			}
+			indexBlocks[indexBlockCount] = indexBlock
+			indexBlockCount++
+			index, err := ReadBlock(reader, indexBlock)
 			if err != nil {
 				return nil, err
 			}
 			for j := 0; j < 256 && i*256+j < fileEntry.BlocksUsed; j++ {
 				if (int(index[j]) + int(index[j+256])*256) == 0 {
-					return blocks, nil
-				}
-				blocks[i*256+j] = int(index[j]) + int(index[j+256])*256
+					break
 				}
+				dataBlocks[i*256+j] = int(index[j]) + int(index[j+256])*256
 			}
 		}
 
-	return nil, errors.New("unsupported file storage type")
-}
-
-func getDataBlocklist(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
-	switch fileEntry.StorageType {
-	case StorageSeedling:
-		blocks := make([]int, 1)
-		blocks[0] = fileEntry.KeyPointer
-		return blocks, nil
-	case StorageSapling:
-		blocks := make([]int, fileEntry.BlocksUsed-1)
-		index, err := ReadBlock(reader, fileEntry.KeyPointer)
-		if err != nil {
-			return nil, err
+		if dataOnly {
+			return dataBlocks, nil
 		}
-		for i := 0; i < fileEntry.BlocksUsed-1; i++ {
-			blocks[i] = int(index[i]) + int(index[i+256])*256
-		}
-		return blocks, nil
-		// case StorageTree:
-		// 	blocks := make([]int, fileEntry.BlocksUsed-fileEntry.BlocksUsed/256-1)
-		// 	masterIndex := ReadBlock(reader, fileEntry.KeyPointer)
-		// 	for i := 0; i < 128; i++ {
-		// 		blockNumber := 0
-		// 		blocks[j] = int(index[i]) + int(index[i+256])*256
 
-		// 	}
+		blocks = append(indexBlocks, dataBlocks...)
+		return blocks, nil
 	}
 
 	return nil, errors.New("unsupported file storage type")
@@ -388,20 +359,16 @@ func getDataBlocklist(reader io.ReaderAt, fileEntry FileEntry) ([]int, error) {
 
 func createBlockList(reader io.ReaderAt, fileSize int) ([]int, error) {
 	numberOfBlocks := fileSize / 512
-	//fmt.Printf("Number of blocks %d\n", numberOfBlocks)
 
 	if fileSize%512 > 0 {
-		//fmt.Printf("Adding block for partial usage\n")
 		numberOfBlocks++
 	}
 
 	if fileSize > 0x200 && fileSize <= 0x20000 {
-		//fmt.Printf("Adding index block for sapling file\n")
 		numberOfBlocks++ // add index block
 	}
 
 	if fileSize > 0x20000 && fileSize <= 0x1000000 {
-		//fmt.Printf("Tree file\n")
 		// add index blocks for each 256 blocks
 		numberOfBlocks += numberOfBlocks / 256
 		// add index block for any remaining blocks
@@ -420,7 +387,6 @@ func createBlockList(reader io.ReaderAt, fileSize int) ([]int, error) {
 		return nil, err
 	}
 
-	//fmt.Printf("findFreeBlocks %d\n", numberOfBlocks)
 	blockList := findFreeBlocks(volumeBitmap, numberOfBlocks)
 
 	return blockList, nil

prodos/file_test.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/format.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/format_test.go

@@ -1,3 +1,9 @@
+// Copyright Terence J. Boldt (c)2021-2023
+// Use of this source code is governed by an MIT
+// license that can be found in the LICENSE file.
+
+// This file provides tests for access to format a ProDOS drive image
+
 package prodos
 
 import (

prodos/host.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2022
+// Copyright Terence J. Boldt (c)2022-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -7,8 +7,10 @@
 package prodos
 
 import (
+	"encoding/binary"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // AddFilesFromHostDirectory fills the root volume with files
@@ -28,7 +30,7 @@ func AddFilesFromHostDirectory(
 			return err
 		}
 
-		if !file.IsDir() && info.Size() > 0 && info.Size() <= 0x20000 {
+		if !file.IsDir() && info.Size() > 0 && info.Size() <= 0x1000000 {
 			err = WriteFileFromFile(readerWriter, "", 0, 0, filepath.Join(directory, file.Name()))
 			if err != nil {
 				return err
@@ -38,3 +40,90 @@ func AddFilesFromHostDirectory(
 
 	return nil
 }
+
+// WriteFileFromFile writes a file to a ProDOS volume from a host file
+func WriteFileFromFile(readerWriter ReaderWriterAt, pathName string, fileType int, auxType int, inFileName string) error {
+	inFile, err := os.ReadFile(inFileName)
+	if err != nil {
+		return err
+	}
+
+	if auxType == 0 && fileType == 0 {
+		auxType, fileType, inFile, err = convertFileByType(inFileName, inFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(pathName) == 0 {
+		_, pathName = filepath.Split(inFileName)
+		pathName = strings.ToUpper(pathName)
+		ext := filepath.Ext(pathName)
+		if len(ext) > 0 {
+			switch ext {
+			case ".SYS", ".TXT", ".BAS", ".BIN":
+				pathName = strings.TrimSuffix(pathName, ext)
+			}
+		}
+	}
+
+	return WriteFile(readerWriter, pathName, fileType, auxType, inFile)
+}
+
+func convertFileByType(inFileName string, inFile []byte) (int, int, []byte, error) {
+	fileType := 0x06  // default to BIN
+	auxType := 0x2000 // default to $2000
+
+	var err error
+
+	// Check for an AppleSingle file as produced by cc65
+	if // Magic number
+	binary.BigEndian.Uint32(inFile[0x00:]) == 0x00051600 &&
+		// Version number
+		binary.BigEndian.Uint32(inFile[0x04:]) == 0x00020000 &&
+		// Number of entries
+		binary.BigEndian.Uint16(inFile[0x18:]) == 0x0002 &&
+		// Data Fork ID
+		binary.BigEndian.Uint32(inFile[0x1A:]) == 0x00000001 &&
+		// Offset
+		binary.BigEndian.Uint32(inFile[0x1E:]) == 0x0000003A &&
+		// Length
+		binary.BigEndian.Uint32(inFile[0x22:]) == uint32(len(inFile))-0x3A &&
+		// ProDOS File Info ID
+		binary.BigEndian.Uint32(inFile[0x26:]) == 0x0000000B &&
+		// Offset
+		binary.BigEndian.Uint32(inFile[0x2A:]) == 0x00000032 &&
+		// Length
+		binary.BigEndian.Uint32(inFile[0x2E:]) == 0x00000008 {
+
+		fileType = int(binary.BigEndian.Uint16(inFile[0x34:]))
+		auxType = int(binary.BigEndian.Uint32(inFile[0x36:]))
+		inFile = inFile[0x3A:]
+	} else {
+		// use extension to determine file type
+		ext := strings.ToUpper(filepath.Ext(inFileName))
+
+		switch ext {
+		case ".BAS":
+			inFile, err = ConvertTextToBasic(string(inFile))
+			fileType = 0xFC
+			auxType = 0x0801
+
+			if err != nil {
+				return 0, 0, nil, err
+			}
+		case ".SYS":
+			fileType = 0xFF
+			auxType = 0x2000
+		case ".BIN":
+			fileType = 0x06
+			auxType = 0x2000
+		case ".TXT":
+			inFile = []byte(strings.ReplaceAll(strings.ReplaceAll(string(inFile), "\r\n", "r"), "\n", "\r"))
+			fileType = 0x04
+			auxType = 0x0000
+		}
+	}
+
+	return auxType, fileType, inFile, err
+}

prodos/memfile.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/text.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 

prodos/time.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -11,6 +11,7 @@ import (
 )
 
 // DateTimeToProDOS converts Time to ProDOS date time
+//
 //	       49041 ($BF91)     49040 ($BF90)
 //
 //	        7 6 5 4 3 2 1 0   7 6 5 4 3 2 1 0

prodos/time_test.go

@@ -1,3 +1,9 @@
+// Copyright Terence J. Boldt (c)2021-2023
+// Use of this source code is governed by an MIT
+// license that can be found in the LICENSE file.
+
+// This file provides tests for conversion to and from ProDOS time format
+
 package prodos
 
 import (

readme.go

@@ -1,4 +1,4 @@
-// Copyright Terence J. Boldt (c)2021-2022
+// Copyright Terence J. Boldt (c)2021-2023
 // Use of this source code is governed by an MIT
 // license that can be found in the LICENSE file.
 
@@ -14,7 +14,7 @@ func printReadme() {
 	fmt.Println(`
 MIT License
 
-Copyright (c)2021-2022 Terence Boldt
+Copyright (c)2021-2023 Terence Boldt
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal