duplicates

This commit is contained in:
Denis Molony 2016-12-09 22:31:03 +11:00
parent e62dce5065
commit cfdb5fcd5c
8 changed files with 324 additions and 258 deletions

View File

@ -1,40 +1,87 @@
package com.bytezone.diskbrowser.duplicates;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import com.bytezone.common.ComputeCRC32;
public class DiskDetails
{
private final File file;
private long checksum = -1;
private boolean duplicate;
private final long checksum;
private final String rootName; // full path without the root folder
private final String shortName; // file name in lower case
public DiskDetails (File file)
private final List<DiskDetails> duplicateChecksums = new ArrayList<DiskDetails> ();
private final List<DiskDetails> duplicateNames = new ArrayList<DiskDetails> ();
private boolean isDuplicateName;
private boolean isDuplicateChecksum;
public DiskDetails (File file, String rootName, String shortName)
{
this.file = file;
duplicate = false;
this.rootName = rootName;
this.shortName = shortName;
checksum = ComputeCRC32.getChecksumValue (file);
}
public boolean isDuplicate ()
// public boolean isDuplicate ()
// {
// return duplicate;
// }
public void addDuplicateChecksum (DiskDetails diskDetails)
{
return duplicate;
if (this.checksum == diskDetails.checksum)
{
this.duplicateChecksums.add (diskDetails);
diskDetails.isDuplicateChecksum = true;
}
}
public void setDuplicate (boolean value)
public void addDuplicateName (DiskDetails diskDetails)
{
duplicate = value;
if (this.shortName.equals (diskDetails.shortName))
{
this.duplicateNames.add (diskDetails);
diskDetails.isDuplicateName = true;
}
}
public String getAbsolutePath ()
public List<DiskDetails> getDuplicateChecksums ()
{
return file.getAbsolutePath ();
return duplicateChecksums;
}
public List<DiskDetails> getDuplicateNames ()
{
return duplicateNames;
}
public boolean isDuplicateChecksum ()
{
return isDuplicateChecksum;
}
public boolean isDuplicateName ()
{
return isDuplicateName;
}
public String getRootName ()
{
return rootName;
}
public String getShortName ()
{
return shortName;
}
public long getChecksum ()
{
if (checksum < 0)
checksum = ComputeCRC32.getChecksumValue (file);
return checksum;
}
@ -47,7 +94,7 @@ public class DiskDetails
@Override
public String toString ()
{
return String.format ("%s (%s)", file.getAbsolutePath (),
duplicate ? "duplicate" : "OK");
return String.format ("%-40s %3d %s %3d %s", rootName, duplicateChecksums.size (),
isDuplicateChecksum, duplicateNames.size (), isDuplicateName);
}
}

View File

@ -0,0 +1,118 @@
package com.bytezone.diskbrowser.duplicates;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.swing.table.AbstractTableModel;
public class DiskTableModel extends AbstractTableModel
{
static final String[] headers = { "Disk name", "Actual disk", "Checksum" };
Map<String, DiskDetails> fileNameMap;
Map<Long, DiskDetails> checkSumMap;
List<TableLine> lines = new ArrayList<DiskTableModel.TableLine> ();
public DiskTableModel (DuplicateHandler duplicateHandler)
{
fileNameMap = duplicateHandler.getFileNameMap ();
checkSumMap = duplicateHandler.getChecksumMap ();
for (String key : fileNameMap.keySet ())
{
DiskDetails original = fileNameMap.get (key);
if (false)
{
if (original.getDuplicateNames ().size () > 0)
{
lines.add (new TableLine (original));
for (DiskDetails duplicate : original.getDuplicateNames ())
lines.add (new TableLine (duplicate));
}
}
else
{
if (original.getDuplicateChecksums ().size () > 0)
{
lines.add (new TableLine (original));
for (DiskDetails duplicate : original.getDuplicateChecksums ())
lines.add (new TableLine (duplicate));
}
// else if (original.isDuplicateChecksum ())
// {
// lines.add (new TableLine (key, original));
// DiskDetails dd = checkSumMap.get (original.getChecksum ());
// for (DiskDetails duplicate : dd.getDuplicateChecksums ())
// lines.add (new TableLine (key, duplicate));
// }
}
}
}
@Override
public String getColumnName (int column)
{
return headers[column];
}
@Override
public int getRowCount ()
{
return lines.size ();
}
@Override
public int getColumnCount ()
{
return headers.length;
}
@Override
public Class<?> getColumnClass (int columnIndex)
{
switch (columnIndex)
{
case 0:
return String.class;
case 1:
return String.class;
case 2:
return Long.class;
default:
return Object.class;
}
}
@Override
public Object getValueAt (int rowIndex, int columnIndex)
{
TableLine line = lines.get (rowIndex);
switch (columnIndex)
{
case 0:
return line.shortName;
case 1:
return line.diskDetails.getRootName ();
case 2:
return line.checksum;
default:
return "???";
}
}
class TableLine
{
String shortName;
DiskDetails diskDetails;
long checksum;
public TableLine (DiskDetails diskDetails)
{
this.shortName = diskDetails.getShortName ();
this.diskDetails = diskDetails;
this.checksum = diskDetails.getChecksum ();
}
}
}

View File

@ -1,12 +1,10 @@
package com.bytezone.diskbrowser.duplicates;
import java.io.File;
import java.util.*;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
import com.bytezone.diskbrowser.disk.AppleDisk;
import com.bytezone.diskbrowser.disk.Disk;
import com.bytezone.diskbrowser.gui.FileComparator;
import com.bytezone.diskbrowser.utilities.Utility;
@ -19,30 +17,34 @@ public class DuplicateHandler
private final File rootFolder;
private int totalDisks;
private int totalFolders;
private final int rootFolderNameLength;
private final boolean debug = false;
// total files for each suffix
private final Map<String, Integer> typeList = new TreeMap<String, Integer> ();
// list of unique disk names -> List of File duplicates
final Map<String, List<DiskDetails>> duplicateDisks =
new TreeMap<String, List<DiskDetails>> ();
// list of checksum -> DiskDetails
final Map<Long, DiskDetails> checksumMap = new HashMap<Long, DiskDetails> ();
// list of unique disk names -> File
private final Map<String, File> diskNames = new HashMap<String, File> ();
// list of checksum -> File
final Map<Long, List<File>> dosMap = new TreeMap<Long, List<File>> ();
private final Map<String, DiskDetails> fileNameMap =
new TreeMap<String, DiskDetails> ();
public DuplicateHandler (File rootFolder)
{
this.rootFolder = rootFolder;
countDisks ();
rootFolderNameLength = rootFolder.getAbsolutePath ().length ();
}
public Map<String, List<DiskDetails>> getDuplicateDisks ()
public Map<String, DiskDetails> getFileNameMap ()
{
return duplicateDisks;
return fileNameMap;
}
public Map<Long, DiskDetails> getChecksumMap ()
{
return checksumMap;
}
void countDisks ()
@ -62,114 +64,76 @@ public class DuplicateHandler
System.out.printf ("%nTotal ....... %,7d%n%n", grandTotal);
}
File getRootFolder ()
{
return rootFolder;
}
private void traverse (File directory)
{
File[] files = directory.listFiles ();
if (files == null || files.length == 0)
{
System.out.println ("Empty folder : " + directory.getAbsolutePath ());
return;
}
Arrays.sort (files, fileComparator);
if (false)
System.out.printf ("%nFolder: %s%n%n", directory.getAbsolutePath ()
.substring (rootFolder.getAbsolutePath ().length ()));
// Arrays.sort (files, fileComparator);
for (File file : files)
{
String fileName = file.getName ().toLowerCase ();
if (file.isDirectory ())
{
++totalFolders;
traverse (file);
}
else if (Utility.validFileType (file.getName ()))
else if (Utility.validFileType (fileName))
{
if (file.getName ().endsWith (".gz") && !file.getName ().endsWith ("dsk.gz"))
continue;
++totalDisks;
int pos = file.getName ().lastIndexOf ('.');
if (pos > 0)
{
String type = file.getName ().substring (pos + 1).toLowerCase ();
if (typeList.containsKey (type))
{
int t = typeList.get (type);
typeList.put (type, ++t);
}
else
typeList.put (type, 1);
}
if (false)
{
String name = file.getName ();
int nameLength = name.length ();
if (nameLength > MAX_NAME_WIDTH)
name = name.substring (0, 15) + "..."
+ name.substring (nameLength - MAX_NAME_WIDTH + 18);
System.out.printf (FORMAT, name, file.length ());
}
checkDuplicates (file);
// checksumDos (file);
incrementType (file, fileName);
checkDuplicates (file, fileName);
}
}
if (false)
for (String key : duplicateDisks.keySet ())
{
List<DiskDetails> diskDetailsList = duplicateDisks.get (key);
System.out.println (key);
for (DiskDetails diskDetails : diskDetailsList)
System.out.println (diskDetails);
}
}
private void checksumDos (File file)
private void checkDuplicates (File file, String fileName)
{
if (file.length () != 143360 || file.getAbsolutePath ().contains ("/ZDisks/"))
return;
String rootName = file.getAbsolutePath ().substring (rootFolderNameLength);
DiskDetails diskDetails = new DiskDetails (file, rootName, fileName);
Disk disk = new AppleDisk (file, 35, 16);
byte[] buffer = disk.readSector (0, 0);
Checksum checksum = new CRC32 ();
checksum.update (buffer, 0, buffer.length);
long cs = checksum.getValue ();
List<File> files = dosMap.get (cs);
if (files == null)
if (fileNameMap.containsKey (fileName))
{
files = new ArrayList<File> ();
dosMap.put (cs, files);
}
files.add (file);
}
private void checkDuplicates (File file)
{
if (diskNames.containsKey (file.getName ()))
{
List<DiskDetails> diskList = duplicateDisks.get (file.getName ());
if (diskList == null)
{
diskList = new ArrayList<DiskDetails> ();
duplicateDisks.put (file.getName (), diskList);
diskList.add (new DiskDetails (diskNames.get (file.getName ())));// add original
}
diskList.add (new DiskDetails (file)); // add the duplicate
DiskDetails otherDisk = fileNameMap.get (fileName);
otherDisk.addDuplicateName (diskDetails);
}
else
diskNames.put (file.getName (), file);
fileNameMap.put (fileName, diskDetails);
if (checksumMap.containsKey (diskDetails.getChecksum ()))
{
DiskDetails otherDisk = checksumMap.get (diskDetails.getChecksum ());
otherDisk.addDuplicateChecksum (diskDetails);
}
else
checksumMap.put (diskDetails.getChecksum (), diskDetails);
}
// public static void main (String[] args)
// {
// DuplicateHandler dh = new DuplicateHandler (
// new File ("/Users/denismolony/Apple II stuff/AppleDisk Images II/apple disks"));
// }
private void incrementType (File file, String fileName)
{
int pos = file.getName ().lastIndexOf ('.');
if (pos > 0)
{
String type = fileName.substring (pos + 1);
if (typeList.containsKey (type))
{
int t = typeList.get (type);
typeList.put (type, ++t);
}
else
typeList.put (type, 1);
}
}
}

View File

@ -1,53 +0,0 @@
package com.bytezone.diskbrowser.duplicates;
import java.util.ArrayList;
import java.util.List;
import javax.swing.JButton;
import javax.swing.JCheckBox;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.SpringLayout;
import com.bytezone.input.SpringUtilities;
public class DuplicatePanel extends JPanel
{
List<JCheckBox> checkBoxes = new ArrayList<JCheckBox> ();
List<DiskDetails> duplicateDisks;
public DuplicatePanel (List<DiskDetails> duplicateDisks, int folderNameLength,
List<DiskDetails> disksSelected, JButton deleteButton, JButton clearButton)
{
this.duplicateDisks = duplicateDisks;
setLayout (new SpringLayout ());
setAlignmentX (LEFT_ALIGNMENT);
int count = 0;
for (DiskDetails diskDetails : duplicateDisks)
{
JCheckBox checkbox = new JCheckBox ();
checkBoxes.add (checkbox);
checkbox.addActionListener (new CheckBoxActionListener (diskDetails, disksSelected,
deleteButton, clearButton));
add (checkbox);
if (++count == 1)
add (new JLabel ("Original disk"));
else
{
String text = diskDetails.isDuplicate () ? "Duplicate" : "OK";
add (new JLabel (text));
}
String checksum = diskDetails.isDuplicate () || count == 1 ? ""
: " (checksum = " + diskDetails.getChecksum () + ")";
add (new JLabel (
diskDetails.getAbsolutePath ().substring (folderNameLength) + checksum));
}
SpringUtilities.makeCompactGrid (this, //
duplicateDisks.size (), 3, // rows, cols
10, 0, // initX, initY
10, 0); // xPad, yPad
}
}

View File

@ -1,7 +1,6 @@
package com.bytezone.diskbrowser.duplicates;
import java.awt.BorderLayout;
import java.awt.Dimension;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.File;
@ -13,7 +12,7 @@ import javax.swing.*;
public class DuplicateWindow extends JFrame
{
int unfinishedWorkers;
private final JTable table;
int folderNameLength;
Map<String, List<DiskDetails>> duplicateDisks;
File rootFolder;
@ -22,10 +21,9 @@ public class DuplicateWindow extends JFrame
JButton buttonCancel = new JButton ("Cancel");
JButton buttonAll = new JButton ("Select all duplicates");
JButton buttonClear = new JButton ("Clear all");
JPanel mainPanel = new JPanel ();
// JPanel mainPanel = new JPanel ();
List<DiskDetails> disksSelected = new ArrayList<DiskDetails> ();
List<DuplicatePanel> duplicatePanels = new ArrayList<DuplicatePanel> ();
DuplicateHandler duplicateHandler;
@ -33,21 +31,18 @@ public class DuplicateWindow extends JFrame
{
super ("Duplicate Disk Detection - " + rootFolder.getAbsolutePath ());
duplicateHandler = new DuplicateHandler (rootFolder);
Map<String, List<DiskDetails>> duplicateDisks = duplicateHandler.getDuplicateDisks ();
for (List<DiskDetails> diskList : duplicateDisks.values ())
new DuplicateWorker (diskList, this).execute ();
unfinishedWorkers = duplicateDisks.size ();
folderNameLength = rootFolder.getAbsolutePath ().length ();
mainPanel.setLayout (new BoxLayout (mainPanel, BoxLayout.PAGE_AXIS));
JScrollPane sp =
new JScrollPane (mainPanel, ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS,
table = new JTable ();
JScrollPane scrollPane =
new JScrollPane (table, ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS,
ScrollPaneConstants.HORIZONTAL_SCROLLBAR_NEVER);
sp.getVerticalScrollBar ().setUnitIncrement (100);
add (sp, BorderLayout.CENTER);
table.setFillsViewportHeight (true);
// table.setShowGrid (true);
// table.setGridColor (Color.BLACK);
add (scrollPane, BorderLayout.CENTER);
JPanel panel = new JPanel ();
panel.add (buttonClear);
@ -66,20 +61,20 @@ public class DuplicateWindow extends JFrame
@Override
public void actionPerformed (ActionEvent e)
{
for (DuplicatePanel dp : duplicatePanels)
{
int count = 0;
for (JCheckBox cb : dp.checkBoxes)
{
if (count > 0 && dp.duplicateDisks.get (count).isDuplicate ())
if (!cb.isSelected ())
{
cb.setSelected (true); // doesn't fire the actionListener!
disksSelected.add (dp.duplicateDisks.get (count));
}
++count;
}
}
// for (DuplicatePanel dp : duplicatePanels)
// {
// int count = 0;
// for (JCheckBox cb : dp.checkBoxes)
// {
// if (count > 0 && dp.duplicateDisks.get (count).isDuplicate ())
// if (!cb.isSelected ())
// {
// cb.setSelected (true); // doesn't fire the actionListener!
// disksSelected.add (dp.duplicateDisks.get (count));
// }
// ++count;
// }
// }
buttonDelete.setEnabled (disksSelected.size () > 0);
buttonClear.setEnabled (disksSelected.size () > 0);
}
@ -90,9 +85,9 @@ public class DuplicateWindow extends JFrame
@Override
public void actionPerformed (ActionEvent e)
{
for (DuplicatePanel dp : duplicatePanels)
for (JCheckBox cb : dp.checkBoxes)
cb.setSelected (false); // doesn't fire the actionListener!
// for (DuplicatePanel dp : duplicatePanels)
// for (JCheckBox cb : dp.checkBoxes)
// cb.setSelected (false); // doesn't fire the actionListener!
disksSelected.clear ();
buttonDelete.setEnabled (false);
@ -116,55 +111,45 @@ public class DuplicateWindow extends JFrame
{
int totalDeleted = 0;
int totalFailed = 0;
for (DuplicatePanel dp : duplicatePanels)
{
int count = 0;
for (JCheckBox cb : dp.checkBoxes)
{
if (cb.isSelected () && false)
{
DiskDetails dd = dp.duplicateDisks.get (count);
if (dd.delete ())
{
++totalDeleted;
System.out.println ("Deleted : " + dd);
}
else
{
++totalFailed;
System.out.println ("Failed : " + dd);
}
}
++count;
}
}
// for (DuplicatePanel dp : duplicatePanels)
// {
// int count = 0;
// for (JCheckBox cb : dp.checkBoxes)
// {
// if (cb.isSelected ())
// {
// DiskDetails dd = dp.duplicateDisks.get (count);
// if (dd.delete ())
// {
// ++totalDeleted;
// System.out.println ("Deleted : " + dd);
// }
// else
// {
// ++totalFailed;
// System.out.println ("Failed : " + dd);
// }
// }
// ++count;
// }
// }
System.out.printf ("Deleted : %d, Failed : %d%n", totalDeleted, totalFailed);
}
});
setSize (600, 700);
setSize (900, 700);
setLocationRelativeTo (null);
setDefaultCloseOperation (HIDE_ON_CLOSE);
}
public void setDuplicateHandler (DuplicateHandler duplicateHandler)
{
this.duplicateHandler = duplicateHandler;
table.setModel (new DiskTableModel (duplicateHandler));
table.getColumnModel ().getColumn (0).setPreferredWidth (250);
table.getColumnModel ().getColumn (1).setPreferredWidth (500);
table.getColumnModel ().getColumn (2).setPreferredWidth (100);
setVisible (true);
}
// create a DuplicatePanel based on the updated DiskDetails
public synchronized void addResult (List<DiskDetails> duplicateDisks)
{
// create panel and add it to the window
DuplicatePanel dp = new DuplicatePanel (duplicateDisks, folderNameLength,
disksSelected, buttonDelete, buttonClear);
mainPanel.add (dp);
duplicatePanels.add (dp);
validate ();
if (--unfinishedWorkers == 0)
{
buttonAll.setEnabled (true);
buttonCancel.setEnabled (true);
}
else
mainPanel.add (Box.createRigidArea (new Dimension (0, 20)));
}
}
}

View File

@ -1,18 +1,18 @@
package com.bytezone.diskbrowser.duplicates;
import java.util.List;
import java.io.File;
import javax.swing.SwingWorker;
public class DuplicateWorker extends SwingWorker<List<DiskDetails>, Void>
public class DuplicateWorker extends SwingWorker<DuplicateHandler, String>
{
List<DiskDetails> duplicateDisks;
DuplicateHandler duplicateHandler;
DuplicateWindow owner;
public DuplicateWorker (List<DiskDetails> duplicateDisks, DuplicateWindow owner)
public DuplicateWorker (File rootFolder, DuplicateWindow owner)
{
this.duplicateDisks = duplicateDisks;
this.owner = owner;
duplicateHandler = new DuplicateHandler (rootFolder);
}
@Override
@ -20,7 +20,7 @@ public class DuplicateWorker extends SwingWorker<List<DiskDetails>, Void>
{
try
{
owner.addResult (get ());
owner.setDuplicateHandler (get ());
}
catch (Exception e)
{
@ -29,16 +29,10 @@ public class DuplicateWorker extends SwingWorker<List<DiskDetails>, Void>
}
@Override
protected List<DiskDetails> doInBackground () throws Exception
protected DuplicateHandler doInBackground () throws Exception
{
long firstChecksum = -1;
for (DiskDetails dd : duplicateDisks)
{
if (firstChecksum < 0)
firstChecksum = dd.getChecksum ();
else
dd.setDuplicate (dd.getChecksum () == firstChecksum);
}
return duplicateDisks;
duplicateHandler.countDisks ();
return duplicateHandler;
}
}

View File

@ -1,12 +1,16 @@
package com.bytezone.diskbrowser.gui;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.KeyEvent;
import java.io.File;
import javax.swing.Action;
import javax.swing.KeyStroke;
import com.bytezone.common.DefaultAction;
import com.bytezone.diskbrowser.duplicates.DuplicateWindow;
import com.bytezone.diskbrowser.duplicates.DuplicateWorker;
import com.bytezone.diskbrowser.gui.RootDirectoryAction.RootDirectoryChangeListener;
public class DuplicateAction extends DefaultAction implements RootDirectoryChangeListener
@ -22,6 +26,8 @@ public class DuplicateAction extends DefaultAction implements RootDirectoryChang
setIcon (Action.SMALL_ICON, "save_delete_16.png");
setIcon (Action.LARGE_ICON_KEY, "save_delete_32.png");
int mask = Toolkit.getDefaultToolkit ().getMenuShortcutKeyMask ();
putValue (Action.ACCELERATOR_KEY, KeyStroke.getKeyStroke (KeyEvent.VK_D, mask));
setEnabled (false);
}
@ -37,7 +43,10 @@ public class DuplicateAction extends DefaultAction implements RootDirectoryChang
public void actionPerformed (ActionEvent arg0)
{
if (window == null)
{
window = new DuplicateWindow (rootFolder);
new DuplicateWorker (rootFolder, window).execute ();
}
else
window.setVisible (true);
}

View File

@ -69,8 +69,10 @@ public class Utility
if (dotPos2 > 0)
{
String suffix2 = filename.substring (dotPos2 + 1, dotPos).toLowerCase ();
if (suffix.equals ("gz") && (suffix2.equals ("bxy") || suffix2.equals ("bny")))
return false;
// if (suffix.equals ("gz") && (suffix2.equals ("bxy") || suffix2.equals ("bny")))
// return false;
if (suffix.equals ("gz"))
return suffixes.contains (suffix2) && !"gz".equals (suffix2);
}
return suffixes.contains (suffix);