diskm8/data.go

package main

import (
	"crypto/md5"
	"encoding/gob"
	"encoding/hex"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/paleotronic/diskm8/disk"
	"github.com/paleotronic/diskm8/loggy"
)

type Disk struct {
	FullPath                string
	Filename                string
	SHA256                  string // Sha of whole disk
	SHA256Active            string // Sha of active sectors/blocks only
	Format                  string
	FormatID                disk.DiskFormat
	Bitmap                  []bool
	Tracks, Sectors, Blocks int
	Files                   DiskCatalog
	ActiveSectors           DiskSectors
	//ActiveBlocks             DiskBlocks
	InactiveSectors DiskSectors
	//InactiveBlocks           DiskBlocks
	MatchFactor              float64
	MatchFiles               map[*DiskFile]*DiskFile
	MissingFiles, ExtraFiles []*DiskFile
	IngestMode               int
	source                   string
}

type ByMatchFactor []*Disk

func (s ByMatchFactor) Len() int {
	return len(s)
}

func (s ByMatchFactor) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s ByMatchFactor) Less(i, j int) bool {
	return s[i].MatchFactor < s[j].MatchFactor
}

type TypeCode int

const (
	TypeMask_AppleDOS TypeCode = 0x0000
	TypeMask_ProDOS   TypeCode = 0x0100
	TypeMask_Pascal   TypeCode = 0x0200
	TypeMask_RDOS     TypeCode = 0x0300
)

type DiskFile struct {
	Filename    string
	Type        string
	Ext         string
	TypeCode    TypeCode
	SHA256      string
	Size        int
	LoadAddress int
	Text        []byte
	Data        []byte
	Locked      bool
	Created     time.Time
	Modified    time.Time
}

func (d *DiskFile) GetNameAdorned() string {

	var ext string
	switch d.TypeCode & 0xff00 {
	case TypeMask_AppleDOS:
		ext = disk.FileType(d.TypeCode & 0xff).Ext()
	case TypeMask_ProDOS:
		ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext()
	case TypeMask_RDOS:
		ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext()
	case TypeMask_Pascal:
		ext = disk.PascalFileType(d.TypeCode & 0xff).Ext()
	}

	return fmt.Sprintf("%s#0x%.4x.%s", d.Filename, d.LoadAddress, ext)

}

func (d *DiskFile) GetName() string {

	var ext string
	switch d.TypeCode & 0xff00 {
	case TypeMask_AppleDOS:
		ext = disk.FileType(d.TypeCode & 0xff).Ext()
	case TypeMask_ProDOS:
		ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext()
	case TypeMask_RDOS:
		ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext()
	case TypeMask_Pascal:
		ext = disk.PascalFileType(d.TypeCode & 0xff).Ext()
	}

	return fmt.Sprintf("%s.%s", d.Filename, ext)

}

type DiskCatalog []*DiskFile
type DiskSectors []*DiskSector
type DiskBlocks []*DiskBlock

type DiskSector struct {
	Track  int
	Sector int

	SHA256 string

	Data []byte
}

type DiskBlock struct {
	Block int

	SHA256 string
}

func (i Disk) LogBitmap(id int) {

	l := loggy.Get(id)

	if i.Tracks > 0 {

		for t := 0; t < i.Tracks; t++ {

			line := fmt.Sprintf("Track %.2d: ", t)

			for s := 0; s < i.Sectors; s++ {
				if i.Bitmap[t*i.Sectors+s] {
					line += fmt.Sprintf("%.2x ", s)
				} else {
					line += ":: "
				}
			}

			l.Logf("%s", line)
		}

	} else if i.Blocks > 0 {

		tr := i.Blocks / 16
		sc := 16

		for t := 0; t < tr; t++ {

			line := fmt.Sprintf("Block %.2d: ", t)

			for s := 0; s < sc; s++ {
				if i.Bitmap[t*i.Sectors+s] {
					line += fmt.Sprintf("%.2x ", s)
				} else {
					line += ":: "
				}
			}

			l.Logf("%s", line)
		}

	}

}

func (d Disk) GetFilename() string {

	sum := md5.Sum([]byte(d.Filename))

	//	fmt.Printf("checksum: [%s] -> [%s]\n", d.Filename, hex.EncodeToString(sum[:]))

	ff := fmt.Sprintf("%s/%d", strings.Trim(filepath.Dir(d.FullPath), "/"), d.FormatID.ID) + "_" + d.SHA256 + "_" + d.SHA256Active + "_" + hex.EncodeToString(sum[:]) + ".fgp"

	if runtime.GOOS == "windows" {
		ff = strings.Replace(ff, ":", "", -1)
		ff = strings.Replace(ff, "\\", "/", -1)
	}

	return ff

}

func (d Disk) WriteToFile(filename string) error {

	// b, err := yaml.Marshal(d)

	// if err != nil {
	// 	return err
	// }
	l := loggy.Get(0)

	_ = os.MkdirAll(filepath.Dir(filename), 0755)

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	enc := gob.NewEncoder(f)
	enc.Encode(d)

	l.Logf("Created %s", filename)

	return nil
}

func (d *Disk) ReadFromFile(filename string) error {
	// b, err := ioutil.ReadFile(filename)
	// if err != nil {
	// 	return err
	// }
	// err = yaml.Unmarshal(b, d)

	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(d)

	d.source = filename

	return err
}

// GetExactBinaryMatches returns disks with the same Global SHA256
func (d *Disk) GetExactBinaryMatches(filter []string) []*Disk {

	l := loggy.Get(0)

	var out []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_"+d.SHA256+"_*_*.fgp")
	if !exists {
		return out
	}

	for _, m := range matches {
		l.Logf(":: Checking %s", m)
		if item, err := cache.Get(m); err == nil {
			if item.FullPath != d.FullPath {
				out = append(out, item)
			}
		}
	}

	return out
}

// GetActiveSectorBinaryMatches returns disks with the same Active SHA256
func (d *Disk) GetActiveSectorBinaryMatches(filter []string) []*Disk {

	l := loggy.Get(0)

	var out []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_"+d.SHA256Active+"_*.fgp")
	if !exists {
		return out
	}

	for _, m := range matches {
		l.Logf(":: Checking %s", m)

		if item, err := cache.Get(m); err == nil {
			if item.FullPath != d.FullPath {
				out = append(out, item)
			}
		}
	}

	return out
}

func (d *Disk) GetFileMap() map[string]*DiskFile {

	out := make(map[string]*DiskFile)

	for _, file := range d.Files {
		f := file
		out[file.SHA256] = f
	}

	return out

}

func (d *Disk) GetUtilizationMap() map[string]string {

	out := make(map[string]string)

	if len(d.ActiveSectors) > 0 {

		for _, block := range d.ActiveSectors {

			key := fmt.Sprintf("T%dS%d", block.Track, block.Sector)
			out[key] = block.SHA256

		}

	}

	return out

}

// CompareChunks returns a value 0-1
func (d *Disk) CompareChunks(b *Disk) (float64, float64, float64, float64) {

	l := loggy.Get(0)

	if d.FormatID != b.FormatID {
		l.Logf("Trying to compare disks of different types")
		return 0, 0, 0, 0
	}

	switch d.FormatID.ID {
	case disk.DF_RDOS_3:
		return d.compareSectorsPositional(b)
	case disk.DF_RDOS_32:
		return d.compareSectorsPositional(b)
	case disk.DF_RDOS_33:
		return d.compareSectorsPositional(b)
	case disk.DF_PASCAL:
		return d.compareBlocksPositional(b)
	case disk.DF_DOS_SECTORS_13:
		return d.compareSectorsPositional(b)
	case disk.DF_DOS_SECTORS_16:
		return d.compareSectorsPositional(b)
	case disk.DF_PRODOS:
		return d.compareBlocksPositional(b)
	case disk.DF_PRODOS_800KB:
		return d.compareBlocksPositional(b)
	}

	return 0, 0, 0, 0

}

func (d *Disk) compareSectorsPositional(b *Disk) (float64, float64, float64, float64) {

	l := loggy.Get(0)

	var sameSectors float64
	var diffSectors float64
	var dNotb float64
	var bNotd float64
	var emptySectors float64
	var dTotal, bTotal float64

	var dmap = d.GetUtilizationMap()
	var bmap = b.GetUtilizationMap()

	for t := 0; t < d.FormatID.TPD(); t++ {

		for s := 0; s < d.FormatID.SPT(); s++ {

			key := fmt.Sprintf("T%dS%d", t, s)

			dCk, dEx := dmap[key]
			bCk, bEx := bmap[key]

			switch {
			case dEx && bEx:
				if dCk == bCk {
					sameSectors += 1
				} else {
					diffSectors += 1
				}
				dTotal += 1
				bTotal += 1
			case dEx && !bEx:
				dNotb += 1
				dTotal += 1
			case !dEx && bEx:
				bNotd += 1
				bTotal += 1
			default:
				emptySectors += 1
			}

		}

	}

	l.Debugf("Same Sectors     : %f", sameSectors)
	l.Debugf("Differing Sectors: %f", diffSectors)
	l.Debugf("Not in other disk: %f", dNotb)
	l.Debugf("Not in this disk : %f", bNotd)

	// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
	return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal

}

func (d *Disk) compareBlocksPositional(b *Disk) (float64, float64, float64, float64) {

	l := loggy.Get(0)

	var sameSectors float64
	var diffSectors float64
	var dNotb float64
	var bNotd float64
	var emptySectors float64
	var dTotal, bTotal float64

	var dmap = d.GetUtilizationMap()
	var bmap = b.GetUtilizationMap()

	for t := 0; t < d.FormatID.BPD(); t++ {

		key := fmt.Sprintf("B%d", t)

		dCk, dEx := dmap[key]
		bCk, bEx := bmap[key]

		switch {
		case dEx && bEx:
			if dCk == bCk {
				sameSectors += 1
			} else {
				diffSectors += 1
			}
			dTotal += 1
			bTotal += 1
		case dEx && !bEx:
			dNotb += 1
			dTotal += 1
		case !dEx && bEx:
			bNotd += 1
			bTotal += 1
		default:
			emptySectors += 1
		}

	}

	l.Debugf("Same Blocks      : %f", sameSectors)
	l.Debugf("Differing Blocks : %f", diffSectors)
	l.Debugf("Not in other disk: %f", dNotb)
	l.Debugf("Not in this disk : %f", bNotd)

	// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
	return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal

}

// GetActiveSectorBinaryMatches returns disks with the same Active SHA256
func (d *Disk) GetPartialMatches(filter []string) ([]*Disk, []*Disk, []*Disk) {

	l := loggy.Get(0)

	var superset []*Disk = make([]*Disk, 0)
	var subset []*Disk = make([]*Disk, 0)
	var identical []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp")
	if !exists {
		return superset, subset, identical
	}

	for _, m := range matches {
		//item := &Disk{}
		if item, err := cache.Get(m); err == nil {
			if item.FullPath != d.FullPath {
				// only here if not looking at same disk
				l.Logf(":: Checking overlapping data blocks %s", item.Filename)
				l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
				dSame, iSame, dDiff, iDiff := d.CompareChunks(item)
				l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename)
				l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename)
				l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100)
				l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100)

				if dSame == 1 && iSame < 1 {
					superset = append(superset, item)
				} else if iSame == 1 && dSame < 1 {
					subset = append(subset, item)
				} else if iSame == 1 && dSame == 1 {
					identical = append(identical, item)
				}
			}
		}
	}

	return superset, subset, identical
}

func (d *Disk) GetPartialMatchesWithThreshold(t float64, filter []string) []*Disk {

	l := loggy.Get(0)

	var matchlist []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp")
	if !exists {
		return matchlist
	}

	var lastPc int = -1
	for i, m := range matches {
		//item := &Disk{}
		if item, err := cache.Get(m); err == nil {

			pc := int(100 * float64(i) / float64(len(matches)))

			if pc != lastPc {
				os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%%   ", pc))
			}

			if item.FullPath != d.FullPath {
				// only here if not looking at same disk
				l.Logf(":: Checking overlapping data blocks %s", item.Filename)
				// l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
				dSame, _, _, _ := d.CompareChunks(item)
				// l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename)
				// l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename)
				// l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100)
				// l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100)

				item.MatchFactor = dSame

				if dSame >= t {
					matchlist = append(matchlist, item)
				}
			}

			fmt.Print("\r")
			lastPc = pc
		}
	}

	return matchlist
}

func Aggregate(f func(d *Disk, collector interface{}), collector interface{}, pathfilter []string) {

	l := loggy.Get(0)

	exists, matches := existsPattern(*baseName, pathfilter, "*_*_*_*.fgp")
	if !exists {
		return
	}

	var lastPc int = -1
	for i, m := range matches {

		pc := int(100 * float64(i) / float64(len(matches)))

		if pc != lastPc {
			os.Stderr.WriteString(fmt.Sprintf("\rAggregating data... %d%%   ", pc))
		}

		l.Logf(":: Checking %s", m)
		//item := &Disk{}
		if item, err := cache.Get(m); err == nil {
			f(item, collector)
		}

	}

	os.Stderr.WriteString("Done.\n")

	return
}

func (d *Disk) CompareFiles(b *Disk) float64 {

	var sameFiles float64
	var missingFiles float64
	var extraFiles float64

	var dmap = d.GetFileMap()
	var bmap = b.GetFileMap()

	for fileCk, info := range dmap {

		if info.Size == 0 {
			continue
		}

		binfo, bEx := bmap[fileCk]

		if bEx {
			sameFiles += 1
			// file match
			if b.MatchFiles == nil {
				b.MatchFiles = make(map[*DiskFile]*DiskFile)
			}
			//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
			b.MatchFiles[binfo] = info
		} else {
			missingFiles += 1
			// file match
			if b.MissingFiles == nil {
				b.MissingFiles = make([]*DiskFile, 0)
			}
			//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
			b.MissingFiles = append(b.MissingFiles, info)
		}

	}

	for fileCk, info := range bmap {

		if info.Size == 0 {
			continue
		}

		_, dEx := dmap[fileCk]

		if !dEx {
			extraFiles += 1
			// file match
			if b.ExtraFiles == nil {
				b.ExtraFiles = make([]*DiskFile, 0)
			}
			//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
			b.ExtraFiles = append(b.ExtraFiles, info)
		}

	}

	// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
	return sameFiles / (sameFiles + extraFiles + missingFiles)

}

func (d *Disk) GetPartialFileMatchesWithThreshold(t float64, filter []string) []*Disk {

	l := loggy.Get(0)

	var matchlist []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
	if !exists {
		return matchlist
	}

	var lastPc int = -1
	for i, m := range matches {
		//item := &Disk{}
		if item, err := cache.Get(m); err == nil {

			pc := int(100 * float64(i) / float64(len(matches)))

			if pc != lastPc {
				os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%%   ", pc))
			}

			if item.FullPath != d.FullPath {
				// only here if not looking at same disk
				l.Logf(":: Checking overlapping files %s", item.Filename)
				dSame := d.CompareFiles(item)

				item.MatchFactor = dSame

				if dSame >= t {
					matchlist = append(matchlist, item)
				}
			}

			fmt.Print("\r")
			lastPc = pc
		}
	}

	return matchlist
}

func (d *Disk) HasFileSHA256(sha string) (bool, *DiskFile) {

	for _, file := range d.Files {
		if sha == file.SHA256 {
			return true, file
		}
	}

	return false, nil

}

func (d *Disk) GetFileChecksum(filename string) (bool, string) {

	for _, f := range d.Files {
		if strings.ToLower(filename) == strings.ToLower(f.Filename) {
			return true, f.SHA256
		}
	}

	return false, ""

}

func (d *Disk) GetFileMatches(filename string, filter []string) []*Disk {

	l := loggy.Get(0)

	var matchlist []*Disk = make([]*Disk, 0)

	exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
	if !exists {
		return matchlist
	}

	fileexists, SHA256 := d.GetFileChecksum(filename)
	if !fileexists {
		os.Stderr.WriteString("File does not exist on this volume: " + filename + "\n")
		return matchlist
	}

	_, srcFile := d.HasFileSHA256(SHA256)

	var lastPc int = -1
	for i, m := range matches {
		//item := &Disk{}
		if item, err := cache.Get(m); err == nil {

			pc := int(100 * float64(i) / float64(len(matches)))

			if pc != lastPc {
				os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%%   ", pc))
			}

			if item.FullPath != d.FullPath {
				// only here if not looking at same disk
				l.Logf(":: Checking overlapping files %s", item.Filename)

				if ex, file := item.HasFileSHA256(SHA256); ex {
					if item.MatchFiles == nil {
						item.MatchFiles = make(map[*DiskFile]*DiskFile)
					}
					item.MatchFiles[srcFile] = file
					matchlist = append(matchlist, item)
				}
			}

			fmt.Print("\r")
			lastPc = pc
		}
	}

	return matchlist
}

// Gets directory with custom format
func (d *Disk) GetDirectory(format string) string {
	out := ""

	for _, file := range d.Files {

		tmp := format
		// size
		tmp = strings.Replace(tmp, "{size:blocks}", fmt.Sprintf("%3d Blocks", file.Size/256+1), -1)
		tmp = strings.Replace(tmp, "{size:kb}", fmt.Sprintf("%4d Kb", file.Size/1024+1), -1)
		tmp = strings.Replace(tmp, "{size:b}", fmt.Sprintf("%6d Bytes", file.Size), -1)
		tmp = strings.Replace(tmp, "{size}", fmt.Sprintf("%6d", file.Size), -1)
		// format
		tmp = strings.Replace(tmp, "{filename}", fmt.Sprintf("%-20s", file.Filename), -1)
		// type
		tmp = strings.Replace(tmp, "{type}", fmt.Sprintf("%-20s", file.Type), -1)
		// sha256
		tmp = strings.Replace(tmp, "{sha256}", file.SHA256, -1)
		// loadaddress
		tmp = strings.Replace(tmp, "{loadaddr}", fmt.Sprintf("0x.%4X", file.LoadAddress), -1)

		out += tmp + "\n"
	}

	return out
}

type CacheContext int

const (
	CC_All CacheContext = iota
	CC_ActiveSectors
	CC_AllSectors
	CC_Files
)

type DiskMetaDataCache struct {
	ctx   CacheContext
	Disks map[string]*Disk
}

var cache = NewCache(CC_All, "")

func (c *DiskMetaDataCache) Get(filename string) (*Disk, error) {
	cached, ok := c.Disks[filename]
	if ok {
		return cached, nil
	}
	item := &Disk{}
	if err := item.ReadFromFile(filename); err == nil {
		c.Disks[filename] = item
		return item, nil
	}
	return nil, errors.New("Not found")
}

func (c *DiskMetaDataCache) Put(filename string, item *Disk) {
	c.Disks[filename] = item
}

func NewCache(ctx CacheContext, pattern string) *DiskMetaDataCache {

	cache := &DiskMetaDataCache{
		ctx:   ctx,
		Disks: make(map[string]*Disk),
	}

	return cache
}

func CreateCache(ctx CacheContext, pattern string, filter []string) *DiskMetaDataCache {

	cache := &DiskMetaDataCache{
		ctx:   ctx,
		Disks: make(map[string]*Disk),
	}

	exists, matches := existsPattern(*baseName, filter, pattern)
	if !exists {
		return cache
	}

	var lastPc int = -1
	for i, m := range matches {
		item := &Disk{}
		if err := item.ReadFromFile(m); err == nil {

			pc := int(100 * float64(i) / float64(len(matches)))

			if pc != lastPc {
				os.Stderr.WriteString(fmt.Sprintf("Caching data... %d%%   ", pc))
			}

			// Load cache
			cache.Put(m, item)

			fmt.Print("\r")
			lastPc = pc
		}
	}

	return cache
}

func SearchPartialFileMatchesWithThreshold(t float64, filter []string) map[string][2]*Disk {

	l := loggy.Get(0)

	matchlist := make(map[string][2]*Disk)

	exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
	if !exists {
		return matchlist
	}

	done := make(map[string]bool)

	var lastPc int = -1
	for i, m := range matches {
		//item := &Disk{}
		if disk, err := cache.Get(m); err == nil {

			d := *disk

			pc := int(100 * float64(i) / float64(len(matches)))

			if pc != lastPc {
				os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%%   ", pc))
			}

			for _, n := range matches {

				if jj, err := cache.Get(n); err == nil {

					item := *jj

					key := d.SHA256 + ":" + item.SHA256
					if item.SHA256 < d.SHA256 {
						key = item.SHA256 + ":" + d.SHA256
					}

					if _, ok := done[key]; ok {
						continue
					}

					if item.FullPath != d.FullPath {
						// only here if not looking at same disk
						l.Logf(":: Checking overlapping files %s", item.Filename)
						dSame := d.CompareFiles(&item)

						item.MatchFactor = dSame

						if dSame >= t {
							matchlist[key] = [2]*Disk{&d, &item}
						}
					}

					done[key] = true

				}

			}

			fmt.Print("\r")
			lastPc = pc
		}
	}

	return matchlist
}

const ingestWorkers = 4
const processWorkers = 6

func exists(path string) bool {

	_, err := os.Stat(path)
	if err != nil {
		return false
	}
	return true

}