943 lines
20 KiB
Go
943 lines
20 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"encoding/gob"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/paleotronic/diskm8/disk"
|
|
"github.com/paleotronic/diskm8/loggy"
|
|
)
|
|
|
|
type Disk struct {
|
|
FullPath string
|
|
Filename string
|
|
SHA256 string // Sha of whole disk
|
|
SHA256Active string // Sha of active sectors/blocks only
|
|
Format string
|
|
FormatID disk.DiskFormat
|
|
Bitmap []bool
|
|
Tracks, Sectors, Blocks int
|
|
Files DiskCatalog
|
|
ActiveSectors DiskSectors
|
|
//ActiveBlocks DiskBlocks
|
|
InactiveSectors DiskSectors
|
|
//InactiveBlocks DiskBlocks
|
|
MatchFactor float64
|
|
MatchFiles map[*DiskFile]*DiskFile
|
|
MissingFiles, ExtraFiles []*DiskFile
|
|
IngestMode int
|
|
source string
|
|
}
|
|
|
|
type ByMatchFactor []*Disk
|
|
|
|
func (s ByMatchFactor) Len() int {
|
|
return len(s)
|
|
}
|
|
|
|
func (s ByMatchFactor) Swap(i, j int) {
|
|
s[i], s[j] = s[j], s[i]
|
|
}
|
|
|
|
func (s ByMatchFactor) Less(i, j int) bool {
|
|
return s[i].MatchFactor < s[j].MatchFactor
|
|
}
|
|
|
|
type TypeCode int
|
|
|
|
const (
|
|
TypeMask_AppleDOS TypeCode = 0x0000
|
|
TypeMask_ProDOS TypeCode = 0x0100
|
|
TypeMask_Pascal TypeCode = 0x0200
|
|
TypeMask_RDOS TypeCode = 0x0300
|
|
)
|
|
|
|
type DiskFile struct {
|
|
Filename string
|
|
Type string
|
|
Ext string
|
|
TypeCode TypeCode
|
|
SHA256 string
|
|
Size int
|
|
LoadAddress int
|
|
Text []byte
|
|
Data []byte
|
|
Locked bool
|
|
Created time.Time
|
|
Modified time.Time
|
|
}
|
|
|
|
func (d *DiskFile) GetNameAdorned() string {
|
|
|
|
var ext string
|
|
switch d.TypeCode & 0xff00 {
|
|
case TypeMask_AppleDOS:
|
|
ext = disk.FileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_ProDOS:
|
|
ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_RDOS:
|
|
ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_Pascal:
|
|
ext = disk.PascalFileType(d.TypeCode & 0xff).Ext()
|
|
}
|
|
|
|
return fmt.Sprintf("%s#0x%.4x.%s", d.Filename, d.LoadAddress, ext)
|
|
|
|
}
|
|
|
|
func (d *DiskFile) GetName() string {
|
|
|
|
var ext string
|
|
switch d.TypeCode & 0xff00 {
|
|
case TypeMask_AppleDOS:
|
|
ext = disk.FileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_ProDOS:
|
|
ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_RDOS:
|
|
ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext()
|
|
case TypeMask_Pascal:
|
|
ext = disk.PascalFileType(d.TypeCode & 0xff).Ext()
|
|
}
|
|
|
|
return fmt.Sprintf("%s.%s", d.Filename, ext)
|
|
|
|
}
|
|
|
|
type DiskCatalog []*DiskFile
|
|
type DiskSectors []*DiskSector
|
|
type DiskBlocks []*DiskBlock
|
|
|
|
type DiskSector struct {
|
|
Track int
|
|
Sector int
|
|
|
|
SHA256 string
|
|
|
|
Data []byte
|
|
}
|
|
|
|
type DiskBlock struct {
|
|
Block int
|
|
|
|
SHA256 string
|
|
}
|
|
|
|
func (i Disk) LogBitmap(id int) {
|
|
|
|
l := loggy.Get(id)
|
|
|
|
if i.Tracks > 0 {
|
|
|
|
for t := 0; t < i.Tracks; t++ {
|
|
|
|
line := fmt.Sprintf("Track %.2d: ", t)
|
|
|
|
for s := 0; s < i.Sectors; s++ {
|
|
if i.Bitmap[t*i.Sectors+s] {
|
|
line += fmt.Sprintf("%.2x ", s)
|
|
} else {
|
|
line += ":: "
|
|
}
|
|
}
|
|
|
|
l.Logf("%s", line)
|
|
}
|
|
|
|
} else if i.Blocks > 0 {
|
|
|
|
tr := i.Blocks / 16
|
|
sc := 16
|
|
|
|
for t := 0; t < tr; t++ {
|
|
|
|
line := fmt.Sprintf("Block %.2d: ", t)
|
|
|
|
for s := 0; s < sc; s++ {
|
|
if i.Bitmap[t*i.Sectors+s] {
|
|
line += fmt.Sprintf("%.2x ", s)
|
|
} else {
|
|
line += ":: "
|
|
}
|
|
}
|
|
|
|
l.Logf("%s", line)
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
func (d Disk) GetFilename() string {
|
|
|
|
sum := md5.Sum([]byte(d.Filename))
|
|
|
|
// fmt.Printf("checksum: [%s] -> [%s]\n", d.Filename, hex.EncodeToString(sum[:]))
|
|
|
|
ff := fmt.Sprintf("%s/%d", strings.Trim(filepath.Dir(d.FullPath), "/"), d.FormatID.ID) + "_" + d.SHA256 + "_" + d.SHA256Active + "_" + hex.EncodeToString(sum[:]) + ".fgp"
|
|
|
|
if runtime.GOOS == "windows" {
|
|
ff = strings.Replace(ff, ":", "", -1)
|
|
ff = strings.Replace(ff, "\\", "/", -1)
|
|
}
|
|
|
|
return ff
|
|
|
|
}
|
|
|
|
func (d Disk) WriteToFile(filename string) error {
|
|
|
|
// b, err := yaml.Marshal(d)
|
|
|
|
// if err != nil {
|
|
// return err
|
|
// }
|
|
l := loggy.Get(0)
|
|
|
|
_ = os.MkdirAll(filepath.Dir(filename), 0755)
|
|
|
|
f, err := os.Create(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
enc := gob.NewEncoder(f)
|
|
enc.Encode(d)
|
|
|
|
l.Logf("Created %s", filename)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (d *Disk) ReadFromFile(filename string) error {
|
|
// b, err := ioutil.ReadFile(filename)
|
|
// if err != nil {
|
|
// return err
|
|
// }
|
|
// err = yaml.Unmarshal(b, d)
|
|
|
|
f, err := os.Open(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
dec := gob.NewDecoder(f)
|
|
err = dec.Decode(d)
|
|
|
|
d.source = filename
|
|
|
|
return err
|
|
}
|
|
|
|
// GetExactBinaryMatches returns disks with the same Global SHA256
|
|
func (d *Disk) GetExactBinaryMatches(filter []string) []*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var out []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_"+d.SHA256+"_*_*.fgp")
|
|
if !exists {
|
|
return out
|
|
}
|
|
|
|
for _, m := range matches {
|
|
l.Logf(":: Checking %s", m)
|
|
if item, err := cache.Get(m); err == nil {
|
|
if item.FullPath != d.FullPath {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
// GetActiveSectorBinaryMatches returns disks with the same Active SHA256
|
|
func (d *Disk) GetActiveSectorBinaryMatches(filter []string) []*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var out []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_"+d.SHA256Active+"_*.fgp")
|
|
if !exists {
|
|
return out
|
|
}
|
|
|
|
for _, m := range matches {
|
|
l.Logf(":: Checking %s", m)
|
|
|
|
if item, err := cache.Get(m); err == nil {
|
|
if item.FullPath != d.FullPath {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func (d *Disk) GetFileMap() map[string]*DiskFile {
|
|
|
|
out := make(map[string]*DiskFile)
|
|
|
|
for _, file := range d.Files {
|
|
f := file
|
|
out[file.SHA256] = f
|
|
}
|
|
|
|
return out
|
|
|
|
}
|
|
|
|
func (d *Disk) GetUtilizationMap() map[string]string {
|
|
|
|
out := make(map[string]string)
|
|
|
|
if len(d.ActiveSectors) > 0 {
|
|
|
|
for _, block := range d.ActiveSectors {
|
|
|
|
key := fmt.Sprintf("T%dS%d", block.Track, block.Sector)
|
|
out[key] = block.SHA256
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return out
|
|
|
|
}
|
|
|
|
// CompareChunks returns a value 0-1
|
|
func (d *Disk) CompareChunks(b *Disk) (float64, float64, float64, float64) {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
if d.FormatID != b.FormatID {
|
|
l.Logf("Trying to compare disks of different types")
|
|
return 0, 0, 0, 0
|
|
}
|
|
|
|
switch d.FormatID.ID {
|
|
case disk.DF_RDOS_3:
|
|
return d.compareSectorsPositional(b)
|
|
case disk.DF_RDOS_32:
|
|
return d.compareSectorsPositional(b)
|
|
case disk.DF_RDOS_33:
|
|
return d.compareSectorsPositional(b)
|
|
case disk.DF_PASCAL:
|
|
return d.compareBlocksPositional(b)
|
|
case disk.DF_DOS_SECTORS_13:
|
|
return d.compareSectorsPositional(b)
|
|
case disk.DF_DOS_SECTORS_16:
|
|
return d.compareSectorsPositional(b)
|
|
case disk.DF_PRODOS:
|
|
return d.compareBlocksPositional(b)
|
|
case disk.DF_PRODOS_800KB:
|
|
return d.compareBlocksPositional(b)
|
|
}
|
|
|
|
return 0, 0, 0, 0
|
|
|
|
}
|
|
|
|
func (d *Disk) compareSectorsPositional(b *Disk) (float64, float64, float64, float64) {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var sameSectors float64
|
|
var diffSectors float64
|
|
var dNotb float64
|
|
var bNotd float64
|
|
var emptySectors float64
|
|
var dTotal, bTotal float64
|
|
|
|
var dmap = d.GetUtilizationMap()
|
|
var bmap = b.GetUtilizationMap()
|
|
|
|
for t := 0; t < d.FormatID.TPD(); t++ {
|
|
|
|
for s := 0; s < d.FormatID.SPT(); s++ {
|
|
|
|
key := fmt.Sprintf("T%dS%d", t, s)
|
|
|
|
dCk, dEx := dmap[key]
|
|
bCk, bEx := bmap[key]
|
|
|
|
switch {
|
|
case dEx && bEx:
|
|
if dCk == bCk {
|
|
sameSectors += 1
|
|
} else {
|
|
diffSectors += 1
|
|
}
|
|
dTotal += 1
|
|
bTotal += 1
|
|
case dEx && !bEx:
|
|
dNotb += 1
|
|
dTotal += 1
|
|
case !dEx && bEx:
|
|
bNotd += 1
|
|
bTotal += 1
|
|
default:
|
|
emptySectors += 1
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
l.Debugf("Same Sectors : %f", sameSectors)
|
|
l.Debugf("Differing Sectors: %f", diffSectors)
|
|
l.Debugf("Not in other disk: %f", dNotb)
|
|
l.Debugf("Not in this disk : %f", bNotd)
|
|
|
|
// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
|
|
return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal
|
|
|
|
}
|
|
|
|
func (d *Disk) compareBlocksPositional(b *Disk) (float64, float64, float64, float64) {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var sameSectors float64
|
|
var diffSectors float64
|
|
var dNotb float64
|
|
var bNotd float64
|
|
var emptySectors float64
|
|
var dTotal, bTotal float64
|
|
|
|
var dmap = d.GetUtilizationMap()
|
|
var bmap = b.GetUtilizationMap()
|
|
|
|
for t := 0; t < d.FormatID.BPD(); t++ {
|
|
|
|
key := fmt.Sprintf("B%d", t)
|
|
|
|
dCk, dEx := dmap[key]
|
|
bCk, bEx := bmap[key]
|
|
|
|
switch {
|
|
case dEx && bEx:
|
|
if dCk == bCk {
|
|
sameSectors += 1
|
|
} else {
|
|
diffSectors += 1
|
|
}
|
|
dTotal += 1
|
|
bTotal += 1
|
|
case dEx && !bEx:
|
|
dNotb += 1
|
|
dTotal += 1
|
|
case !dEx && bEx:
|
|
bNotd += 1
|
|
bTotal += 1
|
|
default:
|
|
emptySectors += 1
|
|
}
|
|
|
|
}
|
|
|
|
l.Debugf("Same Blocks : %f", sameSectors)
|
|
l.Debugf("Differing Blocks : %f", diffSectors)
|
|
l.Debugf("Not in other disk: %f", dNotb)
|
|
l.Debugf("Not in this disk : %f", bNotd)
|
|
|
|
// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
|
|
return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal
|
|
|
|
}
|
|
|
|
// GetActiveSectorBinaryMatches returns disks with the same Active SHA256
|
|
func (d *Disk) GetPartialMatches(filter []string) ([]*Disk, []*Disk, []*Disk) {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var superset []*Disk = make([]*Disk, 0)
|
|
var subset []*Disk = make([]*Disk, 0)
|
|
var identical []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp")
|
|
if !exists {
|
|
return superset, subset, identical
|
|
}
|
|
|
|
for _, m := range matches {
|
|
//item := &Disk{}
|
|
if item, err := cache.Get(m); err == nil {
|
|
if item.FullPath != d.FullPath {
|
|
// only here if not looking at same disk
|
|
l.Logf(":: Checking overlapping data blocks %s", item.Filename)
|
|
l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
|
|
dSame, iSame, dDiff, iDiff := d.CompareChunks(item)
|
|
l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename)
|
|
l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename)
|
|
l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100)
|
|
l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100)
|
|
|
|
if dSame == 1 && iSame < 1 {
|
|
superset = append(superset, item)
|
|
} else if iSame == 1 && dSame < 1 {
|
|
subset = append(subset, item)
|
|
} else if iSame == 1 && dSame == 1 {
|
|
identical = append(identical, item)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return superset, subset, identical
|
|
}
|
|
|
|
func (d *Disk) GetPartialMatchesWithThreshold(t float64, filter []string) []*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var matchlist []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp")
|
|
if !exists {
|
|
return matchlist
|
|
}
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
//item := &Disk{}
|
|
if item, err := cache.Get(m); err == nil {
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc))
|
|
}
|
|
|
|
if item.FullPath != d.FullPath {
|
|
// only here if not looking at same disk
|
|
l.Logf(":: Checking overlapping data blocks %s", item.Filename)
|
|
// l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
|
|
dSame, _, _, _ := d.CompareChunks(item)
|
|
// l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename)
|
|
// l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename)
|
|
// l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100)
|
|
// l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100)
|
|
|
|
item.MatchFactor = dSame
|
|
|
|
if dSame >= t {
|
|
matchlist = append(matchlist, item)
|
|
}
|
|
}
|
|
|
|
fmt.Print("\r")
|
|
lastPc = pc
|
|
}
|
|
}
|
|
|
|
return matchlist
|
|
}
|
|
|
|
func Aggregate(f func(d *Disk, collector interface{}), collector interface{}, pathfilter []string) {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
exists, matches := existsPattern(*baseName, pathfilter, "*_*_*_*.fgp")
|
|
if !exists {
|
|
return
|
|
}
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("\rAggregating data... %d%% ", pc))
|
|
}
|
|
|
|
l.Logf(":: Checking %s", m)
|
|
//item := &Disk{}
|
|
if item, err := cache.Get(m); err == nil {
|
|
f(item, collector)
|
|
}
|
|
|
|
}
|
|
|
|
os.Stderr.WriteString("Done.\n")
|
|
|
|
return
|
|
}
|
|
|
|
func (d *Disk) CompareFiles(b *Disk) float64 {
|
|
|
|
var sameFiles float64
|
|
var missingFiles float64
|
|
var extraFiles float64
|
|
|
|
var dmap = d.GetFileMap()
|
|
var bmap = b.GetFileMap()
|
|
|
|
for fileCk, info := range dmap {
|
|
|
|
if info.Size == 0 {
|
|
continue
|
|
}
|
|
|
|
binfo, bEx := bmap[fileCk]
|
|
|
|
if bEx {
|
|
sameFiles += 1
|
|
// file match
|
|
if b.MatchFiles == nil {
|
|
b.MatchFiles = make(map[*DiskFile]*DiskFile)
|
|
}
|
|
//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
|
|
b.MatchFiles[binfo] = info
|
|
} else {
|
|
missingFiles += 1
|
|
// file match
|
|
if b.MissingFiles == nil {
|
|
b.MissingFiles = make([]*DiskFile, 0)
|
|
}
|
|
//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
|
|
b.MissingFiles = append(b.MissingFiles, info)
|
|
}
|
|
|
|
}
|
|
|
|
for fileCk, info := range bmap {
|
|
|
|
if info.Size == 0 {
|
|
continue
|
|
}
|
|
|
|
_, dEx := dmap[fileCk]
|
|
|
|
if !dEx {
|
|
extraFiles += 1
|
|
// file match
|
|
if b.ExtraFiles == nil {
|
|
b.ExtraFiles = make([]*DiskFile, 0)
|
|
}
|
|
//fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename)
|
|
b.ExtraFiles = append(b.ExtraFiles, info)
|
|
}
|
|
|
|
}
|
|
|
|
// return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal
|
|
return sameFiles / (sameFiles + extraFiles + missingFiles)
|
|
|
|
}
|
|
|
|
func (d *Disk) GetPartialFileMatchesWithThreshold(t float64, filter []string) []*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var matchlist []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
|
|
if !exists {
|
|
return matchlist
|
|
}
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
//item := &Disk{}
|
|
if item, err := cache.Get(m); err == nil {
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc))
|
|
}
|
|
|
|
if item.FullPath != d.FullPath {
|
|
// only here if not looking at same disk
|
|
l.Logf(":: Checking overlapping files %s", item.Filename)
|
|
dSame := d.CompareFiles(item)
|
|
|
|
item.MatchFactor = dSame
|
|
|
|
if dSame >= t {
|
|
matchlist = append(matchlist, item)
|
|
}
|
|
}
|
|
|
|
fmt.Print("\r")
|
|
lastPc = pc
|
|
}
|
|
}
|
|
|
|
return matchlist
|
|
}
|
|
|
|
func (d *Disk) HasFileSHA256(sha string) (bool, *DiskFile) {
|
|
|
|
for _, file := range d.Files {
|
|
if sha == file.SHA256 {
|
|
return true, file
|
|
}
|
|
}
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
func (d *Disk) GetFileChecksum(filename string) (bool, string) {
|
|
|
|
for _, f := range d.Files {
|
|
if strings.ToLower(filename) == strings.ToLower(f.Filename) {
|
|
return true, f.SHA256
|
|
}
|
|
}
|
|
|
|
return false, ""
|
|
|
|
}
|
|
|
|
func (d *Disk) GetFileMatches(filename string, filter []string) []*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
var matchlist []*Disk = make([]*Disk, 0)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
|
|
if !exists {
|
|
return matchlist
|
|
}
|
|
|
|
fileexists, SHA256 := d.GetFileChecksum(filename)
|
|
if !fileexists {
|
|
os.Stderr.WriteString("File does not exist on this volume: " + filename + "\n")
|
|
return matchlist
|
|
}
|
|
|
|
_, srcFile := d.HasFileSHA256(SHA256)
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
//item := &Disk{}
|
|
if item, err := cache.Get(m); err == nil {
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc))
|
|
}
|
|
|
|
if item.FullPath != d.FullPath {
|
|
// only here if not looking at same disk
|
|
l.Logf(":: Checking overlapping files %s", item.Filename)
|
|
|
|
if ex, file := item.HasFileSHA256(SHA256); ex {
|
|
if item.MatchFiles == nil {
|
|
item.MatchFiles = make(map[*DiskFile]*DiskFile)
|
|
}
|
|
item.MatchFiles[srcFile] = file
|
|
matchlist = append(matchlist, item)
|
|
}
|
|
}
|
|
|
|
fmt.Print("\r")
|
|
lastPc = pc
|
|
}
|
|
}
|
|
|
|
return matchlist
|
|
}
|
|
|
|
// Gets directory with custom format
|
|
func (d *Disk) GetDirectory(format string) string {
|
|
out := ""
|
|
|
|
for _, file := range d.Files {
|
|
|
|
tmp := format
|
|
// size
|
|
tmp = strings.Replace(tmp, "{size:blocks}", fmt.Sprintf("%3d Blocks", file.Size/256+1), -1)
|
|
tmp = strings.Replace(tmp, "{size:kb}", fmt.Sprintf("%4d Kb", file.Size/1024+1), -1)
|
|
tmp = strings.Replace(tmp, "{size:b}", fmt.Sprintf("%6d Bytes", file.Size), -1)
|
|
tmp = strings.Replace(tmp, "{size}", fmt.Sprintf("%6d", file.Size), -1)
|
|
// format
|
|
tmp = strings.Replace(tmp, "{filename}", fmt.Sprintf("%-20s", file.Filename), -1)
|
|
// type
|
|
tmp = strings.Replace(tmp, "{type}", fmt.Sprintf("%-20s", file.Type), -1)
|
|
// sha256
|
|
tmp = strings.Replace(tmp, "{sha256}", file.SHA256, -1)
|
|
// loadaddress
|
|
tmp = strings.Replace(tmp, "{loadaddr}", fmt.Sprintf("0x.%4X", file.LoadAddress), -1)
|
|
|
|
out += tmp + "\n"
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
type CacheContext int
|
|
|
|
const (
|
|
CC_All CacheContext = iota
|
|
CC_ActiveSectors
|
|
CC_AllSectors
|
|
CC_Files
|
|
)
|
|
|
|
type DiskMetaDataCache struct {
|
|
ctx CacheContext
|
|
Disks map[string]*Disk
|
|
}
|
|
|
|
var cache = NewCache(CC_All, "")
|
|
|
|
func (c *DiskMetaDataCache) Get(filename string) (*Disk, error) {
|
|
cached, ok := c.Disks[filename]
|
|
if ok {
|
|
return cached, nil
|
|
}
|
|
item := &Disk{}
|
|
if err := item.ReadFromFile(filename); err == nil {
|
|
c.Disks[filename] = item
|
|
return item, nil
|
|
}
|
|
return nil, errors.New("Not found")
|
|
}
|
|
|
|
func (c *DiskMetaDataCache) Put(filename string, item *Disk) {
|
|
c.Disks[filename] = item
|
|
}
|
|
|
|
func NewCache(ctx CacheContext, pattern string) *DiskMetaDataCache {
|
|
|
|
cache := &DiskMetaDataCache{
|
|
ctx: ctx,
|
|
Disks: make(map[string]*Disk),
|
|
}
|
|
|
|
return cache
|
|
}
|
|
|
|
func CreateCache(ctx CacheContext, pattern string, filter []string) *DiskMetaDataCache {
|
|
|
|
cache := &DiskMetaDataCache{
|
|
ctx: ctx,
|
|
Disks: make(map[string]*Disk),
|
|
}
|
|
|
|
exists, matches := existsPattern(*baseName, filter, pattern)
|
|
if !exists {
|
|
return cache
|
|
}
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
item := &Disk{}
|
|
if err := item.ReadFromFile(m); err == nil {
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("Caching data... %d%% ", pc))
|
|
}
|
|
|
|
// Load cache
|
|
cache.Put(m, item)
|
|
|
|
fmt.Print("\r")
|
|
lastPc = pc
|
|
}
|
|
}
|
|
|
|
return cache
|
|
}
|
|
|
|
func SearchPartialFileMatchesWithThreshold(t float64, filter []string) map[string][2]*Disk {
|
|
|
|
l := loggy.Get(0)
|
|
|
|
matchlist := make(map[string][2]*Disk)
|
|
|
|
exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp")
|
|
if !exists {
|
|
return matchlist
|
|
}
|
|
|
|
done := make(map[string]bool)
|
|
|
|
var lastPc int = -1
|
|
for i, m := range matches {
|
|
//item := &Disk{}
|
|
if disk, err := cache.Get(m); err == nil {
|
|
|
|
d := *disk
|
|
|
|
pc := int(100 * float64(i) / float64(len(matches)))
|
|
|
|
if pc != lastPc {
|
|
os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc))
|
|
}
|
|
|
|
for _, n := range matches {
|
|
|
|
if jj, err := cache.Get(n); err == nil {
|
|
|
|
item := *jj
|
|
|
|
key := d.SHA256 + ":" + item.SHA256
|
|
if item.SHA256 < d.SHA256 {
|
|
key = item.SHA256 + ":" + d.SHA256
|
|
}
|
|
|
|
if _, ok := done[key]; ok {
|
|
continue
|
|
}
|
|
|
|
if item.FullPath != d.FullPath {
|
|
// only here if not looking at same disk
|
|
l.Logf(":: Checking overlapping files %s", item.Filename)
|
|
dSame := d.CompareFiles(&item)
|
|
|
|
item.MatchFactor = dSame
|
|
|
|
if dSame >= t {
|
|
matchlist[key] = [2]*Disk{&d, &item}
|
|
}
|
|
}
|
|
|
|
done[key] = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fmt.Print("\r")
|
|
lastPc = pc
|
|
}
|
|
}
|
|
|
|
return matchlist
|
|
}
|
|
|
|
const ingestWorkers = 4
|
|
const processWorkers = 6
|
|
|
|
func exists(path string) bool {
|
|
|
|
_, err := os.Stat(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return true
|
|
|
|
}
|